Mercurial > hg > graal-jvmci-8
annotate src/share/vm/runtime/objectMonitor.cpp @ 3979:4dfb2df418f2
6484982: G1: process references during evacuation pauses
Summary: G1 now uses two reference processors - one is used by concurrent marking and the other is used by STW GCs (both full and incremental evacuation pauses). In an evacuation pause, the reference processor is embedded into the closures used to scan objects. Doing so causes causes reference objects to be 'discovered' by the reference processor. At the end of the evacuation pause, these discovered reference objects are processed - preserving (and copying) referent objects (and their reachable graphs) as appropriate.
Reviewed-by: ysr, jwilhelm, brutisso, stefank, tonyp
| author | johnc |
|---|---|
| date | Thu, 22 Sep 2011 10:57:37 -0700 |
| parents | 1d1603768966 |
| children | f08d439fab8c |
| rev | line source |
|---|---|
| 1878 | 1 /* |
|
2426
1d1603768966
7010070: Update all 2010 Oracle-changed OpenJDK files to have the proper copyright dates - second pass
trims
parents:
2177
diff
changeset
|
2 * Copyright (c) 1998, 2011, Oracle and/or its affiliates. All rights reserved. |
| 1878 | 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 * | |
| 5 * This code is free software; you can redistribute it and/or modify it | |
| 6 * under the terms of the GNU General Public License version 2 only, as | |
| 7 * published by the Free Software Foundation. | |
| 8 * | |
| 9 * This code is distributed in the hope that it will be useful, but WITHOUT | |
| 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| 12 * version 2 for more details (a copy is included in the LICENSE file that | |
| 13 * accompanied this code). | |
| 14 * | |
| 15 * You should have received a copy of the GNU General Public License version | |
| 16 * 2 along with this work; if not, write to the Free Software Foundation, | |
| 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| 18 * | |
| 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA | |
| 20 * or visit www.oracle.com if you need additional information or have any | |
| 21 * questions. | |
| 22 * | |
| 23 */ | |
| 24 | |
| 1972 | 25 #include "precompiled.hpp" |
| 26 #include "classfile/vmSymbols.hpp" | |
| 27 #include "memory/resourceArea.hpp" | |
| 28 #include "oops/markOop.hpp" | |
| 29 #include "oops/oop.inline.hpp" | |
| 30 #include "runtime/handles.inline.hpp" | |
| 31 #include "runtime/interfaceSupport.hpp" | |
| 32 #include "runtime/mutexLocker.hpp" | |
| 33 #include "runtime/objectMonitor.hpp" | |
| 34 #include "runtime/objectMonitor.inline.hpp" | |
| 35 #include "runtime/osThread.hpp" | |
| 36 #include "runtime/stubRoutines.hpp" | |
| 37 #include "runtime/thread.hpp" | |
| 38 #include "services/threadService.hpp" | |
| 39 #include "utilities/dtrace.hpp" | |
| 40 #include "utilities/preserveException.hpp" | |
| 41 #ifdef TARGET_OS_FAMILY_linux | |
| 42 # include "os_linux.inline.hpp" | |
| 43 # include "thread_linux.inline.hpp" | |
| 44 #endif | |
| 45 #ifdef TARGET_OS_FAMILY_solaris | |
| 46 # include "os_solaris.inline.hpp" | |
| 47 # include "thread_solaris.inline.hpp" | |
| 48 #endif | |
| 49 #ifdef TARGET_OS_FAMILY_windows | |
| 50 # include "os_windows.inline.hpp" | |
| 51 # include "thread_windows.inline.hpp" | |
| 52 #endif | |
| 1878 | 53 |
| 54 #if defined(__GNUC__) && !defined(IA64) | |
| 55 // Need to inhibit inlining for older versions of GCC to avoid build-time failures | |
| 56 #define ATTR __attribute__((noinline)) | |
| 57 #else | |
| 58 #define ATTR | |
| 59 #endif | |
| 60 | |
| 61 | |
| 62 #ifdef DTRACE_ENABLED | |
| 63 | |
| 64 // Only bother with this argument setup if dtrace is available | |
| 65 // TODO-FIXME: probes should not fire when caller is _blocked. assert() accordingly. | |
| 66 | |
| 67 HS_DTRACE_PROBE_DECL4(hotspot, monitor__notify, | |
| 68 jlong, uintptr_t, char*, int); | |
| 69 HS_DTRACE_PROBE_DECL4(hotspot, monitor__notifyAll, | |
| 70 jlong, uintptr_t, char*, int); | |
| 71 HS_DTRACE_PROBE_DECL4(hotspot, monitor__contended__enter, | |
| 72 jlong, uintptr_t, char*, int); | |
| 73 HS_DTRACE_PROBE_DECL4(hotspot, monitor__contended__entered, | |
| 74 jlong, uintptr_t, char*, int); | |
| 75 HS_DTRACE_PROBE_DECL4(hotspot, monitor__contended__exit, | |
| 76 jlong, uintptr_t, char*, int); | |
| 77 | |
| 78 #define DTRACE_MONITOR_PROBE_COMMON(klassOop, thread) \ | |
| 79 char* bytes = NULL; \ | |
| 80 int len = 0; \ | |
| 81 jlong jtid = SharedRuntime::get_java_tid(thread); \ | |
|
2177
3582bf76420e
6990754: Use native memory and reference counting to implement SymbolTable
coleenp
parents:
1972
diff
changeset
|
82 Symbol* klassname = ((oop)(klassOop))->klass()->klass_part()->name(); \ |
| 1878 | 83 if (klassname != NULL) { \ |
| 84 bytes = (char*)klassname->bytes(); \ | |
| 85 len = klassname->utf8_length(); \ | |
| 86 } | |
| 87 | |
| 88 #define DTRACE_MONITOR_WAIT_PROBE(monitor, klassOop, thread, millis) \ | |
| 89 { \ | |
| 90 if (DTraceMonitorProbes) { \ | |
| 91 DTRACE_MONITOR_PROBE_COMMON(klassOop, thread); \ | |
| 92 HS_DTRACE_PROBE5(hotspot, monitor__wait, jtid, \ | |
| 93 (monitor), bytes, len, (millis)); \ | |
| 94 } \ | |
| 95 } | |
| 96 | |
| 97 #define DTRACE_MONITOR_PROBE(probe, monitor, klassOop, thread) \ | |
| 98 { \ | |
| 99 if (DTraceMonitorProbes) { \ | |
| 100 DTRACE_MONITOR_PROBE_COMMON(klassOop, thread); \ | |
| 101 HS_DTRACE_PROBE4(hotspot, monitor__##probe, jtid, \ | |
| 102 (uintptr_t)(monitor), bytes, len); \ | |
| 103 } \ | |
| 104 } | |
| 105 | |
| 106 #else // ndef DTRACE_ENABLED | |
| 107 | |
| 108 #define DTRACE_MONITOR_WAIT_PROBE(klassOop, thread, millis, mon) {;} | |
| 109 #define DTRACE_MONITOR_PROBE(probe, klassOop, thread, mon) {;} | |
| 110 | |
| 111 #endif // ndef DTRACE_ENABLED | |
| 112 | |
| 113 // Tunables ... | |
| 114 // The knob* variables are effectively final. Once set they should | |
| 115 // never be modified hence. Consider using __read_mostly with GCC. | |
| 116 | |
| 117 int ObjectMonitor::Knob_Verbose = 0 ; | |
| 118 int ObjectMonitor::Knob_SpinLimit = 5000 ; // derived by an external tool - | |
| 119 static int Knob_LogSpins = 0 ; // enable jvmstat tally for spins | |
| 120 static int Knob_HandOff = 0 ; | |
| 121 static int Knob_ReportSettings = 0 ; | |
| 122 | |
| 123 static int Knob_SpinBase = 0 ; // Floor AKA SpinMin | |
| 124 static int Knob_SpinBackOff = 0 ; // spin-loop backoff | |
| 125 static int Knob_CASPenalty = -1 ; // Penalty for failed CAS | |
| 126 static int Knob_OXPenalty = -1 ; // Penalty for observed _owner change | |
| 127 static int Knob_SpinSetSucc = 1 ; // spinners set the _succ field | |
| 128 static int Knob_SpinEarly = 1 ; | |
| 129 static int Knob_SuccEnabled = 1 ; // futile wake throttling | |
| 130 static int Knob_SuccRestrict = 0 ; // Limit successors + spinners to at-most-one | |
| 131 static int Knob_MaxSpinners = -1 ; // Should be a function of # CPUs | |
| 132 static int Knob_Bonus = 100 ; // spin success bonus | |
| 133 static int Knob_BonusB = 100 ; // spin success bonus | |
| 134 static int Knob_Penalty = 200 ; // spin failure penalty | |
| 135 static int Knob_Poverty = 1000 ; | |
| 136 static int Knob_SpinAfterFutile = 1 ; // Spin after returning from park() | |
| 137 static int Knob_FixedSpin = 0 ; | |
| 138 static int Knob_OState = 3 ; // Spinner checks thread state of _owner | |
| 139 static int Knob_UsePause = 1 ; | |
| 140 static int Knob_ExitPolicy = 0 ; | |
| 141 static int Knob_PreSpin = 10 ; // 20-100 likely better | |
| 142 static int Knob_ResetEvent = 0 ; | |
| 143 static int BackOffMask = 0 ; | |
| 144 | |
| 145 static int Knob_FastHSSEC = 0 ; | |
| 146 static int Knob_MoveNotifyee = 2 ; // notify() - disposition of notifyee | |
| 147 static int Knob_QMode = 0 ; // EntryList-cxq policy - queue discipline | |
| 148 static volatile int InitDone = 0 ; | |
| 149 | |
| 150 #define TrySpin TrySpin_VaryDuration | |
| 151 | |
| 152 // ----------------------------------------------------------------------------- | |
| 153 // Theory of operations -- Monitors lists, thread residency, etc: | |
| 154 // | |
| 155 // * A thread acquires ownership of a monitor by successfully | |
| 156 // CAS()ing the _owner field from null to non-null. | |
| 157 // | |
| 158 // * Invariant: A thread appears on at most one monitor list -- | |
| 159 // cxq, EntryList or WaitSet -- at any one time. | |
| 160 // | |
| 161 // * Contending threads "push" themselves onto the cxq with CAS | |
| 162 // and then spin/park. | |
| 163 // | |
| 164 // * After a contending thread eventually acquires the lock it must | |
| 165 // dequeue itself from either the EntryList or the cxq. | |
| 166 // | |
| 167 // * The exiting thread identifies and unparks an "heir presumptive" | |
| 168 // tentative successor thread on the EntryList. Critically, the | |
| 169 // exiting thread doesn't unlink the successor thread from the EntryList. | |
| 170 // After having been unparked, the wakee will recontend for ownership of | |
| 171 // the monitor. The successor (wakee) will either acquire the lock or | |
| 172 // re-park itself. | |
| 173 // | |
| 174 // Succession is provided for by a policy of competitive handoff. | |
| 175 // The exiting thread does _not_ grant or pass ownership to the | |
| 176 // successor thread. (This is also referred to as "handoff" succession"). | |
| 177 // Instead the exiting thread releases ownership and possibly wakes | |
| 178 // a successor, so the successor can (re)compete for ownership of the lock. | |
| 179 // If the EntryList is empty but the cxq is populated the exiting | |
| 180 // thread will drain the cxq into the EntryList. It does so by | |
| 181 // by detaching the cxq (installing null with CAS) and folding | |
| 182 // the threads from the cxq into the EntryList. The EntryList is | |
| 183 // doubly linked, while the cxq is singly linked because of the | |
| 184 // CAS-based "push" used to enqueue recently arrived threads (RATs). | |
| 185 // | |
| 186 // * Concurrency invariants: | |
| 187 // | |
| 188 // -- only the monitor owner may access or mutate the EntryList. | |
| 189 // The mutex property of the monitor itself protects the EntryList | |
| 190 // from concurrent interference. | |
| 191 // -- Only the monitor owner may detach the cxq. | |
| 192 // | |
| 193 // * The monitor entry list operations avoid locks, but strictly speaking | |
| 194 // they're not lock-free. Enter is lock-free, exit is not. | |
| 195 // See http://j2se.east/~dice/PERSIST/040825-LockFreeQueues.html | |
| 196 // | |
| 197 // * The cxq can have multiple concurrent "pushers" but only one concurrent | |
| 198 // detaching thread. This mechanism is immune from the ABA corruption. | |
| 199 // More precisely, the CAS-based "push" onto cxq is ABA-oblivious. | |
| 200 // | |
| 201 // * Taken together, the cxq and the EntryList constitute or form a | |
| 202 // single logical queue of threads stalled trying to acquire the lock. | |
| 203 // We use two distinct lists to improve the odds of a constant-time | |
| 204 // dequeue operation after acquisition (in the ::enter() epilog) and | |
| 205 // to reduce heat on the list ends. (c.f. Michael Scott's "2Q" algorithm). | |
| 206 // A key desideratum is to minimize queue & monitor metadata manipulation | |
| 207 // that occurs while holding the monitor lock -- that is, we want to | |
| 208 // minimize monitor lock holds times. Note that even a small amount of | |
| 209 // fixed spinning will greatly reduce the # of enqueue-dequeue operations | |
| 210 // on EntryList|cxq. That is, spinning relieves contention on the "inner" | |
| 211 // locks and monitor metadata. | |
| 212 // | |
| 213 // Cxq points to the the set of Recently Arrived Threads attempting entry. | |
| 214 // Because we push threads onto _cxq with CAS, the RATs must take the form of | |
| 215 // a singly-linked LIFO. We drain _cxq into EntryList at unlock-time when | |
| 216 // the unlocking thread notices that EntryList is null but _cxq is != null. | |
| 217 // | |
| 218 // The EntryList is ordered by the prevailing queue discipline and | |
| 219 // can be organized in any convenient fashion, such as a doubly-linked list or | |
| 220 // a circular doubly-linked list. Critically, we want insert and delete operations | |
| 221 // to operate in constant-time. If we need a priority queue then something akin | |
| 222 // to Solaris' sleepq would work nicely. Viz., | |
| 223 // http://agg.eng/ws/on10_nightly/source/usr/src/uts/common/os/sleepq.c. | |
| 224 // Queue discipline is enforced at ::exit() time, when the unlocking thread | |
| 225 // drains the cxq into the EntryList, and orders or reorders the threads on the | |
| 226 // EntryList accordingly. | |
| 227 // | |
| 228 // Barring "lock barging", this mechanism provides fair cyclic ordering, | |
| 229 // somewhat similar to an elevator-scan. | |
| 230 // | |
| 231 // * The monitor synchronization subsystem avoids the use of native | |
| 232 // synchronization primitives except for the narrow platform-specific | |
| 233 // park-unpark abstraction. See the comments in os_solaris.cpp regarding | |
| 234 // the semantics of park-unpark. Put another way, this monitor implementation | |
| 235 // depends only on atomic operations and park-unpark. The monitor subsystem | |
| 236 // manages all RUNNING->BLOCKED and BLOCKED->READY transitions while the | |
| 237 // underlying OS manages the READY<->RUN transitions. | |
| 238 // | |
| 239 // * Waiting threads reside on the WaitSet list -- wait() puts | |
| 240 // the caller onto the WaitSet. | |
| 241 // | |
| 242 // * notify() or notifyAll() simply transfers threads from the WaitSet to | |
| 243 // either the EntryList or cxq. Subsequent exit() operations will | |
| 244 // unpark the notifyee. Unparking a notifee in notify() is inefficient - | |
| 245 // it's likely the notifyee would simply impale itself on the lock held | |
| 246 // by the notifier. | |
| 247 // | |
| 248 // * An interesting alternative is to encode cxq as (List,LockByte) where | |
| 249 // the LockByte is 0 iff the monitor is owned. _owner is simply an auxiliary | |
| 250 // variable, like _recursions, in the scheme. The threads or Events that form | |
| 251 // the list would have to be aligned in 256-byte addresses. A thread would | |
| 252 // try to acquire the lock or enqueue itself with CAS, but exiting threads | |
| 253 // could use a 1-0 protocol and simply STB to set the LockByte to 0. | |
| 254 // Note that is is *not* word-tearing, but it does presume that full-word | |
| 255 // CAS operations are coherent with intermix with STB operations. That's true | |
| 256 // on most common processors. | |
| 257 // | |
| 258 // * See also http://blogs.sun.com/dave | |
| 259 | |
| 260 | |
| 261 // ----------------------------------------------------------------------------- | |
| 262 // Enter support | |
| 263 | |
| 264 bool ObjectMonitor::try_enter(Thread* THREAD) { | |
| 265 if (THREAD != _owner) { | |
| 266 if (THREAD->is_lock_owned ((address)_owner)) { | |
| 267 assert(_recursions == 0, "internal state error"); | |
| 268 _owner = THREAD ; | |
| 269 _recursions = 1 ; | |
| 270 OwnerIsThread = 1 ; | |
| 271 return true; | |
| 272 } | |
| 273 if (Atomic::cmpxchg_ptr (THREAD, &_owner, NULL) != NULL) { | |
| 274 return false; | |
| 275 } | |
| 276 return true; | |
| 277 } else { | |
| 278 _recursions++; | |
| 279 return true; | |
| 280 } | |
| 281 } | |
| 282 | |
| 283 void ATTR ObjectMonitor::enter(TRAPS) { | |
| 284 // The following code is ordered to check the most common cases first | |
| 285 // and to reduce RTS->RTO cache line upgrades on SPARC and IA32 processors. | |
| 286 Thread * const Self = THREAD ; | |
| 287 void * cur ; | |
| 288 | |
| 289 cur = Atomic::cmpxchg_ptr (Self, &_owner, NULL) ; | |
| 290 if (cur == NULL) { | |
| 291 // Either ASSERT _recursions == 0 or explicitly set _recursions = 0. | |
| 292 assert (_recursions == 0 , "invariant") ; | |
| 293 assert (_owner == Self, "invariant") ; | |
| 294 // CONSIDER: set or assert OwnerIsThread == 1 | |
| 295 return ; | |
| 296 } | |
| 297 | |
| 298 if (cur == Self) { | |
| 299 // TODO-FIXME: check for integer overflow! BUGID 6557169. | |
| 300 _recursions ++ ; | |
| 301 return ; | |
| 302 } | |
| 303 | |
| 304 if (Self->is_lock_owned ((address)cur)) { | |
| 305 assert (_recursions == 0, "internal state error"); | |
| 306 _recursions = 1 ; | |
| 307 // Commute owner from a thread-specific on-stack BasicLockObject address to | |
| 308 // a full-fledged "Thread *". | |
| 309 _owner = Self ; | |
| 310 OwnerIsThread = 1 ; | |
| 311 return ; | |
| 312 } | |
| 313 | |
| 314 // We've encountered genuine contention. | |
| 315 assert (Self->_Stalled == 0, "invariant") ; | |
| 316 Self->_Stalled = intptr_t(this) ; | |
| 317 | |
| 318 // Try one round of spinning *before* enqueueing Self | |
| 319 // and before going through the awkward and expensive state | |
| 320 // transitions. The following spin is strictly optional ... | |
| 321 // Note that if we acquire the monitor from an initial spin | |
| 322 // we forgo posting JVMTI events and firing DTRACE probes. | |
| 323 if (Knob_SpinEarly && TrySpin (Self) > 0) { | |
| 324 assert (_owner == Self , "invariant") ; | |
| 325 assert (_recursions == 0 , "invariant") ; | |
| 326 assert (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ; | |
| 327 Self->_Stalled = 0 ; | |
| 328 return ; | |
| 329 } | |
| 330 | |
| 331 assert (_owner != Self , "invariant") ; | |
| 332 assert (_succ != Self , "invariant") ; | |
| 333 assert (Self->is_Java_thread() , "invariant") ; | |
| 334 JavaThread * jt = (JavaThread *) Self ; | |
| 335 assert (!SafepointSynchronize::is_at_safepoint(), "invariant") ; | |
| 336 assert (jt->thread_state() != _thread_blocked , "invariant") ; | |
| 337 assert (this->object() != NULL , "invariant") ; | |
| 338 assert (_count >= 0, "invariant") ; | |
| 339 | |
| 340 // Prevent deflation at STW-time. See deflate_idle_monitors() and is_busy(). | |
| 341 // Ensure the object-monitor relationship remains stable while there's contention. | |
| 342 Atomic::inc_ptr(&_count); | |
| 343 | |
| 344 { // Change java thread status to indicate blocked on monitor enter. | |
| 345 JavaThreadBlockedOnMonitorEnterState jtbmes(jt, this); | |
| 346 | |
| 347 DTRACE_MONITOR_PROBE(contended__enter, this, object(), jt); | |
| 348 if (JvmtiExport::should_post_monitor_contended_enter()) { | |
| 349 JvmtiExport::post_monitor_contended_enter(jt, this); | |
| 350 } | |
| 351 | |
| 352 OSThreadContendState osts(Self->osthread()); | |
| 353 ThreadBlockInVM tbivm(jt); | |
| 354 | |
| 355 Self->set_current_pending_monitor(this); | |
| 356 | |
| 357 // TODO-FIXME: change the following for(;;) loop to straight-line code. | |
| 358 for (;;) { | |
| 359 jt->set_suspend_equivalent(); | |
| 360 // cleared by handle_special_suspend_equivalent_condition() | |
| 361 // or java_suspend_self() | |
| 362 | |
| 363 EnterI (THREAD) ; | |
| 364 | |
| 365 if (!ExitSuspendEquivalent(jt)) break ; | |
| 366 | |
| 367 // | |
| 368 // We have acquired the contended monitor, but while we were | |
| 369 // waiting another thread suspended us. We don't want to enter | |
| 370 // the monitor while suspended because that would surprise the | |
| 371 // thread that suspended us. | |
| 372 // | |
| 373 _recursions = 0 ; | |
| 374 _succ = NULL ; | |
| 375 exit (Self) ; | |
| 376 | |
| 377 jt->java_suspend_self(); | |
| 378 } | |
| 379 Self->set_current_pending_monitor(NULL); | |
| 380 } | |
| 381 | |
| 382 Atomic::dec_ptr(&_count); | |
| 383 assert (_count >= 0, "invariant") ; | |
| 384 Self->_Stalled = 0 ; | |
| 385 | |
| 386 // Must either set _recursions = 0 or ASSERT _recursions == 0. | |
| 387 assert (_recursions == 0 , "invariant") ; | |
| 388 assert (_owner == Self , "invariant") ; | |
| 389 assert (_succ != Self , "invariant") ; | |
| 390 assert (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ; | |
| 391 | |
| 392 // The thread -- now the owner -- is back in vm mode. | |
| 393 // Report the glorious news via TI,DTrace and jvmstat. | |
| 394 // The probe effect is non-trivial. All the reportage occurs | |
| 395 // while we hold the monitor, increasing the length of the critical | |
| 396 // section. Amdahl's parallel speedup law comes vividly into play. | |
| 397 // | |
| 398 // Another option might be to aggregate the events (thread local or | |
| 399 // per-monitor aggregation) and defer reporting until a more opportune | |
| 400 // time -- such as next time some thread encounters contention but has | |
| 401 // yet to acquire the lock. While spinning that thread could | |
| 402 // spinning we could increment JVMStat counters, etc. | |
| 403 | |
| 404 DTRACE_MONITOR_PROBE(contended__entered, this, object(), jt); | |
| 405 if (JvmtiExport::should_post_monitor_contended_entered()) { | |
| 406 JvmtiExport::post_monitor_contended_entered(jt, this); | |
| 407 } | |
| 408 if (ObjectMonitor::_sync_ContendedLockAttempts != NULL) { | |
| 409 ObjectMonitor::_sync_ContendedLockAttempts->inc() ; | |
| 410 } | |
| 411 } | |
| 412 | |
| 413 | |
| 414 // Caveat: TryLock() is not necessarily serializing if it returns failure. | |
| 415 // Callers must compensate as needed. | |
| 416 | |
| 417 int ObjectMonitor::TryLock (Thread * Self) { | |
| 418 for (;;) { | |
| 419 void * own = _owner ; | |
| 420 if (own != NULL) return 0 ; | |
| 421 if (Atomic::cmpxchg_ptr (Self, &_owner, NULL) == NULL) { | |
| 422 // Either guarantee _recursions == 0 or set _recursions = 0. | |
| 423 assert (_recursions == 0, "invariant") ; | |
| 424 assert (_owner == Self, "invariant") ; | |
| 425 // CONSIDER: set or assert that OwnerIsThread == 1 | |
| 426 return 1 ; | |
| 427 } | |
| 428 // The lock had been free momentarily, but we lost the race to the lock. | |
| 429 // Interference -- the CAS failed. | |
| 430 // We can either return -1 or retry. | |
| 431 // Retry doesn't make as much sense because the lock was just acquired. | |
| 432 if (true) return -1 ; | |
| 433 } | |
| 434 } | |
| 435 | |
| 436 void ATTR ObjectMonitor::EnterI (TRAPS) { | |
| 437 Thread * Self = THREAD ; | |
| 438 assert (Self->is_Java_thread(), "invariant") ; | |
| 439 assert (((JavaThread *) Self)->thread_state() == _thread_blocked , "invariant") ; | |
| 440 | |
| 441 // Try the lock - TATAS | |
| 442 if (TryLock (Self) > 0) { | |
| 443 assert (_succ != Self , "invariant") ; | |
| 444 assert (_owner == Self , "invariant") ; | |
| 445 assert (_Responsible != Self , "invariant") ; | |
| 446 return ; | |
| 447 } | |
| 448 | |
| 449 DeferredInitialize () ; | |
| 450 | |
| 451 // We try one round of spinning *before* enqueueing Self. | |
| 452 // | |
| 453 // If the _owner is ready but OFFPROC we could use a YieldTo() | |
| 454 // operation to donate the remainder of this thread's quantum | |
| 455 // to the owner. This has subtle but beneficial affinity | |
| 456 // effects. | |
| 457 | |
| 458 if (TrySpin (Self) > 0) { | |
| 459 assert (_owner == Self , "invariant") ; | |
| 460 assert (_succ != Self , "invariant") ; | |
| 461 assert (_Responsible != Self , "invariant") ; | |
| 462 return ; | |
| 463 } | |
| 464 | |
| 465 // The Spin failed -- Enqueue and park the thread ... | |
| 466 assert (_succ != Self , "invariant") ; | |
| 467 assert (_owner != Self , "invariant") ; | |
| 468 assert (_Responsible != Self , "invariant") ; | |
| 469 | |
| 470 // Enqueue "Self" on ObjectMonitor's _cxq. | |
| 471 // | |
| 472 // Node acts as a proxy for Self. | |
| 473 // As an aside, if were to ever rewrite the synchronization code mostly | |
| 474 // in Java, WaitNodes, ObjectMonitors, and Events would become 1st-class | |
| 475 // Java objects. This would avoid awkward lifecycle and liveness issues, | |
| 476 // as well as eliminate a subset of ABA issues. | |
| 477 // TODO: eliminate ObjectWaiter and enqueue either Threads or Events. | |
| 478 // | |
| 479 | |
| 480 ObjectWaiter node(Self) ; | |
| 481 Self->_ParkEvent->reset() ; | |
| 482 node._prev = (ObjectWaiter *) 0xBAD ; | |
| 483 node.TState = ObjectWaiter::TS_CXQ ; | |
| 484 | |
| 485 // Push "Self" onto the front of the _cxq. | |
| 486 // Once on cxq/EntryList, Self stays on-queue until it acquires the lock. | |
| 487 // Note that spinning tends to reduce the rate at which threads | |
| 488 // enqueue and dequeue on EntryList|cxq. | |
| 489 ObjectWaiter * nxt ; | |
| 490 for (;;) { | |
| 491 node._next = nxt = _cxq ; | |
| 492 if (Atomic::cmpxchg_ptr (&node, &_cxq, nxt) == nxt) break ; | |
| 493 | |
| 494 // Interference - the CAS failed because _cxq changed. Just retry. | |
| 495 // As an optional optimization we retry the lock. | |
| 496 if (TryLock (Self) > 0) { | |
| 497 assert (_succ != Self , "invariant") ; | |
| 498 assert (_owner == Self , "invariant") ; | |
| 499 assert (_Responsible != Self , "invariant") ; | |
| 500 return ; | |
| 501 } | |
| 502 } | |
| 503 | |
| 504 // Check for cxq|EntryList edge transition to non-null. This indicates | |
| 505 // the onset of contention. While contention persists exiting threads | |
| 506 // will use a ST:MEMBAR:LD 1-1 exit protocol. When contention abates exit | |
| 507 // operations revert to the faster 1-0 mode. This enter operation may interleave | |
| 508 // (race) a concurrent 1-0 exit operation, resulting in stranding, so we | |
| 509 // arrange for one of the contending thread to use a timed park() operations | |
| 510 // to detect and recover from the race. (Stranding is form of progress failure | |
| 511 // where the monitor is unlocked but all the contending threads remain parked). | |
| 512 // That is, at least one of the contended threads will periodically poll _owner. | |
| 513 // One of the contending threads will become the designated "Responsible" thread. | |
| 514 // The Responsible thread uses a timed park instead of a normal indefinite park | |
| 515 // operation -- it periodically wakes and checks for and recovers from potential | |
| 516 // strandings admitted by 1-0 exit operations. We need at most one Responsible | |
| 517 // thread per-monitor at any given moment. Only threads on cxq|EntryList may | |
| 518 // be responsible for a monitor. | |
| 519 // | |
| 520 // Currently, one of the contended threads takes on the added role of "Responsible". | |
| 521 // A viable alternative would be to use a dedicated "stranding checker" thread | |
| 522 // that periodically iterated over all the threads (or active monitors) and unparked | |
| 523 // successors where there was risk of stranding. This would help eliminate the | |
| 524 // timer scalability issues we see on some platforms as we'd only have one thread | |
| 525 // -- the checker -- parked on a timer. | |
| 526 | |
| 527 if ((SyncFlags & 16) == 0 && nxt == NULL && _EntryList == NULL) { | |
| 528 // Try to assume the role of responsible thread for the monitor. | |
| 529 // CONSIDER: ST vs CAS vs { if (Responsible==null) Responsible=Self } | |
| 530 Atomic::cmpxchg_ptr (Self, &_Responsible, NULL) ; | |
| 531 } | |
| 532 | |
| 533 // The lock have been released while this thread was occupied queueing | |
| 534 // itself onto _cxq. To close the race and avoid "stranding" and | |
| 535 // progress-liveness failure we must resample-retry _owner before parking. | |
| 536 // Note the Dekker/Lamport duality: ST cxq; MEMBAR; LD Owner. | |
| 537 // In this case the ST-MEMBAR is accomplished with CAS(). | |
| 538 // | |
| 539 // TODO: Defer all thread state transitions until park-time. | |
| 540 // Since state transitions are heavy and inefficient we'd like | |
| 541 // to defer the state transitions until absolutely necessary, | |
| 542 // and in doing so avoid some transitions ... | |
| 543 | |
| 544 TEVENT (Inflated enter - Contention) ; | |
| 545 int nWakeups = 0 ; | |
| 546 int RecheckInterval = 1 ; | |
| 547 | |
| 548 for (;;) { | |
| 549 | |
| 550 if (TryLock (Self) > 0) break ; | |
| 551 assert (_owner != Self, "invariant") ; | |
| 552 | |
| 553 if ((SyncFlags & 2) && _Responsible == NULL) { | |
| 554 Atomic::cmpxchg_ptr (Self, &_Responsible, NULL) ; | |
| 555 } | |
| 556 | |
| 557 // park self | |
| 558 if (_Responsible == Self || (SyncFlags & 1)) { | |
| 559 TEVENT (Inflated enter - park TIMED) ; | |
| 560 Self->_ParkEvent->park ((jlong) RecheckInterval) ; | |
| 561 // Increase the RecheckInterval, but clamp the value. | |
| 562 RecheckInterval *= 8 ; | |
| 563 if (RecheckInterval > 1000) RecheckInterval = 1000 ; | |
| 564 } else { | |
| 565 TEVENT (Inflated enter - park UNTIMED) ; | |
| 566 Self->_ParkEvent->park() ; | |
| 567 } | |
| 568 | |
| 569 if (TryLock(Self) > 0) break ; | |
| 570 | |
| 571 // The lock is still contested. | |
| 572 // Keep a tally of the # of futile wakeups. | |
| 573 // Note that the counter is not protected by a lock or updated by atomics. | |
| 574 // That is by design - we trade "lossy" counters which are exposed to | |
| 575 // races during updates for a lower probe effect. | |
| 576 TEVENT (Inflated enter - Futile wakeup) ; | |
| 577 if (ObjectMonitor::_sync_FutileWakeups != NULL) { | |
| 578 ObjectMonitor::_sync_FutileWakeups->inc() ; | |
| 579 } | |
| 580 ++ nWakeups ; | |
| 581 | |
| 582 // Assuming this is not a spurious wakeup we'll normally find _succ == Self. | |
| 583 // We can defer clearing _succ until after the spin completes | |
| 584 // TrySpin() must tolerate being called with _succ == Self. | |
| 585 // Try yet another round of adaptive spinning. | |
| 586 if ((Knob_SpinAfterFutile & 1) && TrySpin (Self) > 0) break ; | |
| 587 | |
| 588 // We can find that we were unpark()ed and redesignated _succ while | |
| 589 // we were spinning. That's harmless. If we iterate and call park(), | |
| 590 // park() will consume the event and return immediately and we'll | |
| 591 // just spin again. This pattern can repeat, leaving _succ to simply | |
| 592 // spin on a CPU. Enable Knob_ResetEvent to clear pending unparks(). | |
| 593 // Alternately, we can sample fired() here, and if set, forgo spinning | |
| 594 // in the next iteration. | |
| 595 | |
| 596 if ((Knob_ResetEvent & 1) && Self->_ParkEvent->fired()) { | |
| 597 Self->_ParkEvent->reset() ; | |
| 598 OrderAccess::fence() ; | |
| 599 } | |
| 600 if (_succ == Self) _succ = NULL ; | |
| 601 | |
| 602 // Invariant: after clearing _succ a thread *must* retry _owner before parking. | |
| 603 OrderAccess::fence() ; | |
| 604 } | |
| 605 | |
| 606 // Egress : | |
| 607 // Self has acquired the lock -- Unlink Self from the cxq or EntryList. | |
| 608 // Normally we'll find Self on the EntryList . | |
| 609 // From the perspective of the lock owner (this thread), the | |
| 610 // EntryList is stable and cxq is prepend-only. | |
| 611 // The head of cxq is volatile but the interior is stable. | |
| 612 // In addition, Self.TState is stable. | |
| 613 | |
| 614 assert (_owner == Self , "invariant") ; | |
| 615 assert (object() != NULL , "invariant") ; | |
| 616 // I'd like to write: | |
| 617 // guarantee (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ; | |
| 618 // but as we're at a safepoint that's not safe. | |
| 619 | |
| 620 UnlinkAfterAcquire (Self, &node) ; | |
| 621 if (_succ == Self) _succ = NULL ; | |
| 622 | |
| 623 assert (_succ != Self, "invariant") ; | |
| 624 if (_Responsible == Self) { | |
| 625 _Responsible = NULL ; | |
| 626 // Dekker pivot-point. | |
| 627 // Consider OrderAccess::storeload() here | |
| 628 | |
| 629 // We may leave threads on cxq|EntryList without a designated | |
| 630 // "Responsible" thread. This is benign. When this thread subsequently | |
| 631 // exits the monitor it can "see" such preexisting "old" threads -- | |
| 632 // threads that arrived on the cxq|EntryList before the fence, above -- | |
| 633 // by LDing cxq|EntryList. Newly arrived threads -- that is, threads | |
| 634 // that arrive on cxq after the ST:MEMBAR, above -- will set Responsible | |
| 635 // non-null and elect a new "Responsible" timer thread. | |
| 636 // | |
| 637 // This thread executes: | |
| 638 // ST Responsible=null; MEMBAR (in enter epilog - here) | |
| 639 // LD cxq|EntryList (in subsequent exit) | |
| 640 // | |
| 641 // Entering threads in the slow/contended path execute: | |
| 642 // ST cxq=nonnull; MEMBAR; LD Responsible (in enter prolog) | |
| 643 // The (ST cxq; MEMBAR) is accomplished with CAS(). | |
| 644 // | |
| 645 // The MEMBAR, above, prevents the LD of cxq|EntryList in the subsequent | |
| 646 // exit operation from floating above the ST Responsible=null. | |
| 647 // | |
| 648 // In *practice* however, EnterI() is always followed by some atomic | |
| 649 // operation such as the decrement of _count in ::enter(). Those atomics | |
| 650 // obviate the need for the explicit MEMBAR, above. | |
| 651 } | |
| 652 | |
| 653 // We've acquired ownership with CAS(). | |
| 654 // CAS is serializing -- it has MEMBAR/FENCE-equivalent semantics. | |
| 655 // But since the CAS() this thread may have also stored into _succ, | |
| 656 // EntryList, cxq or Responsible. These meta-data updates must be | |
| 657 // visible __before this thread subsequently drops the lock. | |
| 658 // Consider what could occur if we didn't enforce this constraint -- | |
| 659 // STs to monitor meta-data and user-data could reorder with (become | |
| 660 // visible after) the ST in exit that drops ownership of the lock. | |
| 661 // Some other thread could then acquire the lock, but observe inconsistent | |
| 662 // or old monitor meta-data and heap data. That violates the JMM. | |
| 663 // To that end, the 1-0 exit() operation must have at least STST|LDST | |
| 664 // "release" barrier semantics. Specifically, there must be at least a | |
| 665 // STST|LDST barrier in exit() before the ST of null into _owner that drops | |
| 666 // the lock. The barrier ensures that changes to monitor meta-data and data | |
| 667 // protected by the lock will be visible before we release the lock, and | |
| 668 // therefore before some other thread (CPU) has a chance to acquire the lock. | |
| 669 // See also: http://gee.cs.oswego.edu/dl/jmm/cookbook.html. | |
| 670 // | |
| 671 // Critically, any prior STs to _succ or EntryList must be visible before | |
| 672 // the ST of null into _owner in the *subsequent* (following) corresponding | |
| 673 // monitorexit. Recall too, that in 1-0 mode monitorexit does not necessarily | |
| 674 // execute a serializing instruction. | |
| 675 | |
| 676 if (SyncFlags & 8) { | |
| 677 OrderAccess::fence() ; | |
| 678 } | |
| 679 return ; | |
| 680 } | |
| 681 | |
| 682 // ReenterI() is a specialized inline form of the latter half of the | |
| 683 // contended slow-path from EnterI(). We use ReenterI() only for | |
| 684 // monitor reentry in wait(). | |
| 685 // | |
| 686 // In the future we should reconcile EnterI() and ReenterI(), adding | |
| 687 // Knob_Reset and Knob_SpinAfterFutile support and restructuring the | |
| 688 // loop accordingly. | |
| 689 | |
| 690 void ATTR ObjectMonitor::ReenterI (Thread * Self, ObjectWaiter * SelfNode) { | |
| 691 assert (Self != NULL , "invariant") ; | |
| 692 assert (SelfNode != NULL , "invariant") ; | |
| 693 assert (SelfNode->_thread == Self , "invariant") ; | |
| 694 assert (_waiters > 0 , "invariant") ; | |
| 695 assert (((oop)(object()))->mark() == markOopDesc::encode(this) , "invariant") ; | |
| 696 assert (((JavaThread *)Self)->thread_state() != _thread_blocked, "invariant") ; | |
| 697 JavaThread * jt = (JavaThread *) Self ; | |
| 698 | |
| 699 int nWakeups = 0 ; | |
| 700 for (;;) { | |
| 701 ObjectWaiter::TStates v = SelfNode->TState ; | |
| 702 guarantee (v == ObjectWaiter::TS_ENTER || v == ObjectWaiter::TS_CXQ, "invariant") ; | |
| 703 assert (_owner != Self, "invariant") ; | |
| 704 | |
| 705 if (TryLock (Self) > 0) break ; | |
| 706 if (TrySpin (Self) > 0) break ; | |
| 707 | |
| 708 TEVENT (Wait Reentry - parking) ; | |
| 709 | |
| 710 // State transition wrappers around park() ... | |
| 711 // ReenterI() wisely defers state transitions until | |
| 712 // it's clear we must park the thread. | |
| 713 { | |
| 714 OSThreadContendState osts(Self->osthread()); | |
| 715 ThreadBlockInVM tbivm(jt); | |
| 716 | |
| 717 // cleared by handle_special_suspend_equivalent_condition() | |
| 718 // or java_suspend_self() | |
| 719 jt->set_suspend_equivalent(); | |
| 720 if (SyncFlags & 1) { | |
| 721 Self->_ParkEvent->park ((jlong)1000) ; | |
| 722 } else { | |
| 723 Self->_ParkEvent->park () ; | |
| 724 } | |
| 725 | |
| 726 // were we externally suspended while we were waiting? | |
| 727 for (;;) { | |
| 728 if (!ExitSuspendEquivalent (jt)) break ; | |
| 729 if (_succ == Self) { _succ = NULL; OrderAccess::fence(); } | |
| 730 jt->java_suspend_self(); | |
| 731 jt->set_suspend_equivalent(); | |
| 732 } | |
| 733 } | |
| 734 | |
| 735 // Try again, but just so we distinguish between futile wakeups and | |
| 736 // successful wakeups. The following test isn't algorithmically | |
| 737 // necessary, but it helps us maintain sensible statistics. | |
| 738 if (TryLock(Self) > 0) break ; | |
| 739 | |
| 740 // The lock is still contested. | |
| 741 // Keep a tally of the # of futile wakeups. | |
| 742 // Note that the counter is not protected by a lock or updated by atomics. | |
| 743 // That is by design - we trade "lossy" counters which are exposed to | |
| 744 // races during updates for a lower probe effect. | |
| 745 TEVENT (Wait Reentry - futile wakeup) ; | |
| 746 ++ nWakeups ; | |
| 747 | |
| 748 // Assuming this is not a spurious wakeup we'll normally | |
| 749 // find that _succ == Self. | |
| 750 if (_succ == Self) _succ = NULL ; | |
| 751 | |
| 752 // Invariant: after clearing _succ a contending thread | |
| 753 // *must* retry _owner before parking. | |
| 754 OrderAccess::fence() ; | |
| 755 | |
| 756 if (ObjectMonitor::_sync_FutileWakeups != NULL) { | |
| 757 ObjectMonitor::_sync_FutileWakeups->inc() ; | |
| 758 } | |
| 759 } | |
| 760 | |
| 761 // Self has acquired the lock -- Unlink Self from the cxq or EntryList . | |
| 762 // Normally we'll find Self on the EntryList. | |
| 763 // Unlinking from the EntryList is constant-time and atomic-free. | |
| 764 // From the perspective of the lock owner (this thread), the | |
| 765 // EntryList is stable and cxq is prepend-only. | |
| 766 // The head of cxq is volatile but the interior is stable. | |
| 767 // In addition, Self.TState is stable. | |
| 768 | |
| 769 assert (_owner == Self, "invariant") ; | |
| 770 assert (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ; | |
| 771 UnlinkAfterAcquire (Self, SelfNode) ; | |
| 772 if (_succ == Self) _succ = NULL ; | |
| 773 assert (_succ != Self, "invariant") ; | |
| 774 SelfNode->TState = ObjectWaiter::TS_RUN ; | |
| 775 OrderAccess::fence() ; // see comments at the end of EnterI() | |
| 776 } | |
| 777 | |
| 778 // after the thread acquires the lock in ::enter(). Equally, we could defer | |
| 779 // unlinking the thread until ::exit()-time. | |
| 780 | |
| 781 void ObjectMonitor::UnlinkAfterAcquire (Thread * Self, ObjectWaiter * SelfNode) | |
| 782 { | |
| 783 assert (_owner == Self, "invariant") ; | |
| 784 assert (SelfNode->_thread == Self, "invariant") ; | |
| 785 | |
| 786 if (SelfNode->TState == ObjectWaiter::TS_ENTER) { | |
| 787 // Normal case: remove Self from the DLL EntryList . | |
| 788 // This is a constant-time operation. | |
| 789 ObjectWaiter * nxt = SelfNode->_next ; | |
| 790 ObjectWaiter * prv = SelfNode->_prev ; | |
| 791 if (nxt != NULL) nxt->_prev = prv ; | |
| 792 if (prv != NULL) prv->_next = nxt ; | |
| 793 if (SelfNode == _EntryList ) _EntryList = nxt ; | |
| 794 assert (nxt == NULL || nxt->TState == ObjectWaiter::TS_ENTER, "invariant") ; | |
| 795 assert (prv == NULL || prv->TState == ObjectWaiter::TS_ENTER, "invariant") ; | |
| 796 TEVENT (Unlink from EntryList) ; | |
| 797 } else { | |
| 798 guarantee (SelfNode->TState == ObjectWaiter::TS_CXQ, "invariant") ; | |
| 799 // Inopportune interleaving -- Self is still on the cxq. | |
| 800 // This usually means the enqueue of self raced an exiting thread. | |
| 801 // Normally we'll find Self near the front of the cxq, so | |
| 802 // dequeueing is typically fast. If needbe we can accelerate | |
| 803 // this with some MCS/CHL-like bidirectional list hints and advisory | |
| 804 // back-links so dequeueing from the interior will normally operate | |
| 805 // in constant-time. | |
| 806 // Dequeue Self from either the head (with CAS) or from the interior | |
| 807 // with a linear-time scan and normal non-atomic memory operations. | |
| 808 // CONSIDER: if Self is on the cxq then simply drain cxq into EntryList | |
| 809 // and then unlink Self from EntryList. We have to drain eventually, | |
| 810 // so it might as well be now. | |
| 811 | |
| 812 ObjectWaiter * v = _cxq ; | |
| 813 assert (v != NULL, "invariant") ; | |
| 814 if (v != SelfNode || Atomic::cmpxchg_ptr (SelfNode->_next, &_cxq, v) != v) { | |
| 815 // The CAS above can fail from interference IFF a "RAT" arrived. | |
| 816 // In that case Self must be in the interior and can no longer be | |
| 817 // at the head of cxq. | |
| 818 if (v == SelfNode) { | |
| 819 assert (_cxq != v, "invariant") ; | |
| 820 v = _cxq ; // CAS above failed - start scan at head of list | |
| 821 } | |
| 822 ObjectWaiter * p ; | |
| 823 ObjectWaiter * q = NULL ; | |
| 824 for (p = v ; p != NULL && p != SelfNode; p = p->_next) { | |
| 825 q = p ; | |
| 826 assert (p->TState == ObjectWaiter::TS_CXQ, "invariant") ; | |
| 827 } | |
| 828 assert (v != SelfNode, "invariant") ; | |
| 829 assert (p == SelfNode, "Node not found on cxq") ; | |
| 830 assert (p != _cxq, "invariant") ; | |
| 831 assert (q != NULL, "invariant") ; | |
| 832 assert (q->_next == p, "invariant") ; | |
| 833 q->_next = p->_next ; | |
| 834 } | |
| 835 TEVENT (Unlink from cxq) ; | |
| 836 } | |
| 837 | |
| 838 // Diagnostic hygiene ... | |
| 839 SelfNode->_prev = (ObjectWaiter *) 0xBAD ; | |
| 840 SelfNode->_next = (ObjectWaiter *) 0xBAD ; | |
| 841 SelfNode->TState = ObjectWaiter::TS_RUN ; | |
| 842 } | |
| 843 | |
| 844 // ----------------------------------------------------------------------------- | |
| 845 // Exit support | |
| 846 // | |
| 847 // exit() | |
| 848 // ~~~~~~ | |
| 849 // Note that the collector can't reclaim the objectMonitor or deflate | |
| 850 // the object out from underneath the thread calling ::exit() as the | |
| 851 // thread calling ::exit() never transitions to a stable state. | |
| 852 // This inhibits GC, which in turn inhibits asynchronous (and | |
| 853 // inopportune) reclamation of "this". | |
| 854 // | |
| 855 // We'd like to assert that: (THREAD->thread_state() != _thread_blocked) ; | |
| 856 // There's one exception to the claim above, however. EnterI() can call | |
| 857 // exit() to drop a lock if the acquirer has been externally suspended. | |
| 858 // In that case exit() is called with _thread_state as _thread_blocked, | |
| 859 // but the monitor's _count field is > 0, which inhibits reclamation. | |
| 860 // | |
| 861 // 1-0 exit | |
| 862 // ~~~~~~~~ | |
| 863 // ::exit() uses a canonical 1-1 idiom with a MEMBAR although some of | |
| 864 // the fast-path operators have been optimized so the common ::exit() | |
| 865 // operation is 1-0. See i486.ad fast_unlock(), for instance. | |
| 866 // The code emitted by fast_unlock() elides the usual MEMBAR. This | |
| 867 // greatly improves latency -- MEMBAR and CAS having considerable local | |
| 868 // latency on modern processors -- but at the cost of "stranding". Absent the | |
| 869 // MEMBAR, a thread in fast_unlock() can race a thread in the slow | |
| 870 // ::enter() path, resulting in the entering thread being stranding | |
| 871 // and a progress-liveness failure. Stranding is extremely rare. | |
| 872 // We use timers (timed park operations) & periodic polling to detect | |
| 873 // and recover from stranding. Potentially stranded threads periodically | |
| 874 // wake up and poll the lock. See the usage of the _Responsible variable. | |
| 875 // | |
| 876 // The CAS() in enter provides for safety and exclusion, while the CAS or | |
| 877 // MEMBAR in exit provides for progress and avoids stranding. 1-0 locking | |
| 878 // eliminates the CAS/MEMBAR from the exist path, but it admits stranding. | |
| 879 // We detect and recover from stranding with timers. | |
| 880 // | |
| 881 // If a thread transiently strands it'll park until (a) another | |
| 882 // thread acquires the lock and then drops the lock, at which time the | |
| 883 // exiting thread will notice and unpark the stranded thread, or, (b) | |
| 884 // the timer expires. If the lock is high traffic then the stranding latency | |
| 885 // will be low due to (a). If the lock is low traffic then the odds of | |
| 886 // stranding are lower, although the worst-case stranding latency | |
| 887 // is longer. Critically, we don't want to put excessive load in the | |
| 888 // platform's timer subsystem. We want to minimize both the timer injection | |
| 889 // rate (timers created/sec) as well as the number of timers active at | |
| 890 // any one time. (more precisely, we want to minimize timer-seconds, which is | |
| 891 // the integral of the # of active timers at any instant over time). | |
| 892 // Both impinge on OS scalability. Given that, at most one thread parked on | |
| 893 // a monitor will use a timer. | |
| 894 | |
| 895 void ATTR ObjectMonitor::exit(TRAPS) { | |
| 896 Thread * Self = THREAD ; | |
| 897 if (THREAD != _owner) { | |
| 898 if (THREAD->is_lock_owned((address) _owner)) { | |
| 899 // Transmute _owner from a BasicLock pointer to a Thread address. | |
| 900 // We don't need to hold _mutex for this transition. | |
| 901 // Non-null to Non-null is safe as long as all readers can | |
| 902 // tolerate either flavor. | |
| 903 assert (_recursions == 0, "invariant") ; | |
| 904 _owner = THREAD ; | |
| 905 _recursions = 0 ; | |
| 906 OwnerIsThread = 1 ; | |
| 907 } else { | |
| 908 // NOTE: we need to handle unbalanced monitor enter/exit | |
| 909 // in native code by throwing an exception. | |
| 910 // TODO: Throw an IllegalMonitorStateException ? | |
| 911 TEVENT (Exit - Throw IMSX) ; | |
| 912 assert(false, "Non-balanced monitor enter/exit!"); | |
| 913 if (false) { | |
| 914 THROW(vmSymbols::java_lang_IllegalMonitorStateException()); | |
| 915 } | |
| 916 return; | |
| 917 } | |
| 918 } | |
| 919 | |
| 920 if (_recursions != 0) { | |
| 921 _recursions--; // this is simple recursive enter | |
| 922 TEVENT (Inflated exit - recursive) ; | |
| 923 return ; | |
| 924 } | |
| 925 | |
| 926 // Invariant: after setting Responsible=null an thread must execute | |
| 927 // a MEMBAR or other serializing instruction before fetching EntryList|cxq. | |
| 928 if ((SyncFlags & 4) == 0) { | |
| 929 _Responsible = NULL ; | |
| 930 } | |
| 931 | |
| 932 for (;;) { | |
| 933 assert (THREAD == _owner, "invariant") ; | |
| 934 | |
| 935 | |
| 936 if (Knob_ExitPolicy == 0) { | |
| 937 // release semantics: prior loads and stores from within the critical section | |
| 938 // must not float (reorder) past the following store that drops the lock. | |
| 939 // On SPARC that requires MEMBAR #loadstore|#storestore. | |
| 940 // But of course in TSO #loadstore|#storestore is not required. | |
| 941 // I'd like to write one of the following: | |
| 942 // A. OrderAccess::release() ; _owner = NULL | |
| 943 // B. OrderAccess::loadstore(); OrderAccess::storestore(); _owner = NULL; | |
| 944 // Unfortunately OrderAccess::release() and OrderAccess::loadstore() both | |
| 945 // store into a _dummy variable. That store is not needed, but can result | |
| 946 // in massive wasteful coherency traffic on classic SMP systems. | |
| 947 // Instead, I use release_store(), which is implemented as just a simple | |
| 948 // ST on x64, x86 and SPARC. | |
| 949 OrderAccess::release_store_ptr (&_owner, NULL) ; // drop the lock | |
| 950 OrderAccess::storeload() ; // See if we need to wake a successor | |
| 951 if ((intptr_t(_EntryList)|intptr_t(_cxq)) == 0 || _succ != NULL) { | |
| 952 TEVENT (Inflated exit - simple egress) ; | |
| 953 return ; | |
| 954 } | |
| 955 TEVENT (Inflated exit - complex egress) ; | |
| 956 | |
| 957 // Normally the exiting thread is responsible for ensuring succession, | |
| 958 // but if other successors are ready or other entering threads are spinning | |
| 959 // then this thread can simply store NULL into _owner and exit without | |
| 960 // waking a successor. The existence of spinners or ready successors | |
| 961 // guarantees proper succession (liveness). Responsibility passes to the | |
| 962 // ready or running successors. The exiting thread delegates the duty. | |
| 963 // More precisely, if a successor already exists this thread is absolved | |
| 964 // of the responsibility of waking (unparking) one. | |
| 965 // | |
| 966 // The _succ variable is critical to reducing futile wakeup frequency. | |
| 967 // _succ identifies the "heir presumptive" thread that has been made | |
| 968 // ready (unparked) but that has not yet run. We need only one such | |
| 969 // successor thread to guarantee progress. | |
| 970 // See http://www.usenix.org/events/jvm01/full_papers/dice/dice.pdf | |
| 971 // section 3.3 "Futile Wakeup Throttling" for details. | |
| 972 // | |
| 973 // Note that spinners in Enter() also set _succ non-null. | |
| 974 // In the current implementation spinners opportunistically set | |
| 975 // _succ so that exiting threads might avoid waking a successor. | |
| 976 // Another less appealing alternative would be for the exiting thread | |
| 977 // to drop the lock and then spin briefly to see if a spinner managed | |
| 978 // to acquire the lock. If so, the exiting thread could exit | |
| 979 // immediately without waking a successor, otherwise the exiting | |
| 980 // thread would need to dequeue and wake a successor. | |
| 981 // (Note that we'd need to make the post-drop spin short, but no | |
| 982 // shorter than the worst-case round-trip cache-line migration time. | |
| 983 // The dropped lock needs to become visible to the spinner, and then | |
| 984 // the acquisition of the lock by the spinner must become visible to | |
| 985 // the exiting thread). | |
| 986 // | |
| 987 | |
| 988 // It appears that an heir-presumptive (successor) must be made ready. | |
| 989 // Only the current lock owner can manipulate the EntryList or | |
| 990 // drain _cxq, so we need to reacquire the lock. If we fail | |
| 991 // to reacquire the lock the responsibility for ensuring succession | |
| 992 // falls to the new owner. | |
| 993 // | |
| 994 if (Atomic::cmpxchg_ptr (THREAD, &_owner, NULL) != NULL) { | |
| 995 return ; | |
| 996 } | |
| 997 TEVENT (Exit - Reacquired) ; | |
| 998 } else { | |
| 999 if ((intptr_t(_EntryList)|intptr_t(_cxq)) == 0 || _succ != NULL) { | |
| 1000 OrderAccess::release_store_ptr (&_owner, NULL) ; // drop the lock | |
| 1001 OrderAccess::storeload() ; | |
| 1002 // Ratify the previously observed values. | |
| 1003 if (_cxq == NULL || _succ != NULL) { | |
| 1004 TEVENT (Inflated exit - simple egress) ; | |
| 1005 return ; | |
| 1006 } | |
| 1007 | |
| 1008 // inopportune interleaving -- the exiting thread (this thread) | |
| 1009 // in the fast-exit path raced an entering thread in the slow-enter | |
| 1010 // path. | |
| 1011 // We have two choices: | |
| 1012 // A. Try to reacquire the lock. | |
| 1013 // If the CAS() fails return immediately, otherwise | |
| 1014 // we either restart/rerun the exit operation, or simply | |
| 1015 // fall-through into the code below which wakes a successor. | |
| 1016 // B. If the elements forming the EntryList|cxq are TSM | |
| 1017 // we could simply unpark() the lead thread and return | |
| 1018 // without having set _succ. | |
| 1019 if (Atomic::cmpxchg_ptr (THREAD, &_owner, NULL) != NULL) { | |
| 1020 TEVENT (Inflated exit - reacquired succeeded) ; | |
| 1021 return ; | |
| 1022 } | |
| 1023 TEVENT (Inflated exit - reacquired failed) ; | |
| 1024 } else { | |
| 1025 TEVENT (Inflated exit - complex egress) ; | |
| 1026 } | |
| 1027 } | |
| 1028 | |
| 1029 guarantee (_owner == THREAD, "invariant") ; | |
| 1030 | |
| 1031 ObjectWaiter * w = NULL ; | |
| 1032 int QMode = Knob_QMode ; | |
| 1033 | |
| 1034 if (QMode == 2 && _cxq != NULL) { | |
| 1035 // QMode == 2 : cxq has precedence over EntryList. | |
| 1036 // Try to directly wake a successor from the cxq. | |
| 1037 // If successful, the successor will need to unlink itself from cxq. | |
| 1038 w = _cxq ; | |
| 1039 assert (w != NULL, "invariant") ; | |
| 1040 assert (w->TState == ObjectWaiter::TS_CXQ, "Invariant") ; | |
| 1041 ExitEpilog (Self, w) ; | |
| 1042 return ; | |
| 1043 } | |
| 1044 | |
| 1045 if (QMode == 3 && _cxq != NULL) { | |
| 1046 // Aggressively drain cxq into EntryList at the first opportunity. | |
| 1047 // This policy ensure that recently-run threads live at the head of EntryList. | |
| 1048 // Drain _cxq into EntryList - bulk transfer. | |
| 1049 // First, detach _cxq. | |
| 1050 // The following loop is tantamount to: w = swap (&cxq, NULL) | |
| 1051 w = _cxq ; | |
| 1052 for (;;) { | |
| 1053 assert (w != NULL, "Invariant") ; | |
| 1054 ObjectWaiter * u = (ObjectWaiter *) Atomic::cmpxchg_ptr (NULL, &_cxq, w) ; | |
| 1055 if (u == w) break ; | |
| 1056 w = u ; | |
| 1057 } | |
| 1058 assert (w != NULL , "invariant") ; | |
| 1059 | |
| 1060 ObjectWaiter * q = NULL ; | |
| 1061 ObjectWaiter * p ; | |
| 1062 for (p = w ; p != NULL ; p = p->_next) { | |
| 1063 guarantee (p->TState == ObjectWaiter::TS_CXQ, "Invariant") ; | |
| 1064 p->TState = ObjectWaiter::TS_ENTER ; | |
| 1065 p->_prev = q ; | |
| 1066 q = p ; | |
| 1067 } | |
| 1068 | |
| 1069 // Append the RATs to the EntryList | |
| 1070 // TODO: organize EntryList as a CDLL so we can locate the tail in constant-time. | |
| 1071 ObjectWaiter * Tail ; | |
| 1072 for (Tail = _EntryList ; Tail != NULL && Tail->_next != NULL ; Tail = Tail->_next) ; | |
| 1073 if (Tail == NULL) { | |
| 1074 _EntryList = w ; | |
| 1075 } else { | |
| 1076 Tail->_next = w ; | |
| 1077 w->_prev = Tail ; | |
| 1078 } | |
| 1079 | |
| 1080 // Fall thru into code that tries to wake a successor from EntryList | |
| 1081 } | |
| 1082 | |
| 1083 if (QMode == 4 && _cxq != NULL) { | |
| 1084 // Aggressively drain cxq into EntryList at the first opportunity. | |
| 1085 // This policy ensure that recently-run threads live at the head of EntryList. | |
| 1086 | |
| 1087 // Drain _cxq into EntryList - bulk transfer. | |
| 1088 // First, detach _cxq. | |
| 1089 // The following loop is tantamount to: w = swap (&cxq, NULL) | |
| 1090 w = _cxq ; | |
| 1091 for (;;) { | |
| 1092 assert (w != NULL, "Invariant") ; | |
| 1093 ObjectWaiter * u = (ObjectWaiter *) Atomic::cmpxchg_ptr (NULL, &_cxq, w) ; | |
| 1094 if (u == w) break ; | |
| 1095 w = u ; | |
| 1096 } | |
| 1097 assert (w != NULL , "invariant") ; | |
| 1098 | |
| 1099 ObjectWaiter * q = NULL ; | |
| 1100 ObjectWaiter * p ; | |
| 1101 for (p = w ; p != NULL ; p = p->_next) { | |
| 1102 guarantee (p->TState == ObjectWaiter::TS_CXQ, "Invariant") ; | |
| 1103 p->TState = ObjectWaiter::TS_ENTER ; | |
| 1104 p->_prev = q ; | |
| 1105 q = p ; | |
| 1106 } | |
| 1107 | |
| 1108 // Prepend the RATs to the EntryList | |
| 1109 if (_EntryList != NULL) { | |
| 1110 q->_next = _EntryList ; | |
| 1111 _EntryList->_prev = q ; | |
| 1112 } | |
| 1113 _EntryList = w ; | |
| 1114 | |
| 1115 // Fall thru into code that tries to wake a successor from EntryList | |
| 1116 } | |
| 1117 | |
| 1118 w = _EntryList ; | |
| 1119 if (w != NULL) { | |
| 1120 // I'd like to write: guarantee (w->_thread != Self). | |
| 1121 // But in practice an exiting thread may find itself on the EntryList. | |
| 1122 // Lets say thread T1 calls O.wait(). Wait() enqueues T1 on O's waitset and | |
| 1123 // then calls exit(). Exit release the lock by setting O._owner to NULL. | |
| 1124 // Lets say T1 then stalls. T2 acquires O and calls O.notify(). The | |
| 1125 // notify() operation moves T1 from O's waitset to O's EntryList. T2 then | |
| 1126 // release the lock "O". T2 resumes immediately after the ST of null into | |
| 1127 // _owner, above. T2 notices that the EntryList is populated, so it | |
| 1128 // reacquires the lock and then finds itself on the EntryList. | |
| 1129 // Given all that, we have to tolerate the circumstance where "w" is | |
| 1130 // associated with Self. | |
| 1131 assert (w->TState == ObjectWaiter::TS_ENTER, "invariant") ; | |
| 1132 ExitEpilog (Self, w) ; | |
| 1133 return ; | |
| 1134 } | |
| 1135 | |
| 1136 // If we find that both _cxq and EntryList are null then just | |
| 1137 // re-run the exit protocol from the top. | |
| 1138 w = _cxq ; | |
| 1139 if (w == NULL) continue ; | |
| 1140 | |
| 1141 // Drain _cxq into EntryList - bulk transfer. | |
| 1142 // First, detach _cxq. | |
| 1143 // The following loop is tantamount to: w = swap (&cxq, NULL) | |
| 1144 for (;;) { | |
| 1145 assert (w != NULL, "Invariant") ; | |
| 1146 ObjectWaiter * u = (ObjectWaiter *) Atomic::cmpxchg_ptr (NULL, &_cxq, w) ; | |
| 1147 if (u == w) break ; | |
| 1148 w = u ; | |
| 1149 } | |
| 1150 TEVENT (Inflated exit - drain cxq into EntryList) ; | |
| 1151 | |
| 1152 assert (w != NULL , "invariant") ; | |
| 1153 assert (_EntryList == NULL , "invariant") ; | |
| 1154 | |
| 1155 // Convert the LIFO SLL anchored by _cxq into a DLL. | |
| 1156 // The list reorganization step operates in O(LENGTH(w)) time. | |
| 1157 // It's critical that this step operate quickly as | |
| 1158 // "Self" still holds the outer-lock, restricting parallelism | |
| 1159 // and effectively lengthening the critical section. | |
| 1160 // Invariant: s chases t chases u. | |
| 1161 // TODO-FIXME: consider changing EntryList from a DLL to a CDLL so | |
| 1162 // we have faster access to the tail. | |
| 1163 | |
| 1164 if (QMode == 1) { | |
| 1165 // QMode == 1 : drain cxq to EntryList, reversing order | |
| 1166 // We also reverse the order of the list. | |
| 1167 ObjectWaiter * s = NULL ; | |
| 1168 ObjectWaiter * t = w ; | |
| 1169 ObjectWaiter * u = NULL ; | |
| 1170 while (t != NULL) { | |
| 1171 guarantee (t->TState == ObjectWaiter::TS_CXQ, "invariant") ; | |
| 1172 t->TState = ObjectWaiter::TS_ENTER ; | |
| 1173 u = t->_next ; | |
| 1174 t->_prev = u ; | |
| 1175 t->_next = s ; | |
| 1176 s = t; | |
| 1177 t = u ; | |
| 1178 } | |
| 1179 _EntryList = s ; | |
| 1180 assert (s != NULL, "invariant") ; | |
| 1181 } else { | |
| 1182 // QMode == 0 or QMode == 2 | |
| 1183 _EntryList = w ; | |
| 1184 ObjectWaiter * q = NULL ; | |
| 1185 ObjectWaiter * p ; | |
| 1186 for (p = w ; p != NULL ; p = p->_next) { | |
| 1187 guarantee (p->TState == ObjectWaiter::TS_CXQ, "Invariant") ; | |
| 1188 p->TState = ObjectWaiter::TS_ENTER ; | |
| 1189 p->_prev = q ; | |
| 1190 q = p ; | |
| 1191 } | |
| 1192 } | |
| 1193 | |
| 1194 // In 1-0 mode we need: ST EntryList; MEMBAR #storestore; ST _owner = NULL | |
| 1195 // The MEMBAR is satisfied by the release_store() operation in ExitEpilog(). | |
| 1196 | |
| 1197 // See if we can abdicate to a spinner instead of waking a thread. | |
| 1198 // A primary goal of the implementation is to reduce the | |
| 1199 // context-switch rate. | |
| 1200 if (_succ != NULL) continue; | |
| 1201 | |
| 1202 w = _EntryList ; | |
| 1203 if (w != NULL) { | |
| 1204 guarantee (w->TState == ObjectWaiter::TS_ENTER, "invariant") ; | |
| 1205 ExitEpilog (Self, w) ; | |
| 1206 return ; | |
| 1207 } | |
| 1208 } | |
| 1209 } | |
| 1210 | |
| 1211 // ExitSuspendEquivalent: | |
| 1212 // A faster alternate to handle_special_suspend_equivalent_condition() | |
| 1213 // | |
| 1214 // handle_special_suspend_equivalent_condition() unconditionally | |
| 1215 // acquires the SR_lock. On some platforms uncontended MutexLocker() | |
| 1216 // operations have high latency. Note that in ::enter() we call HSSEC | |
| 1217 // while holding the monitor, so we effectively lengthen the critical sections. | |
| 1218 // | |
| 1219 // There are a number of possible solutions: | |
| 1220 // | |
| 1221 // A. To ameliorate the problem we might also defer state transitions | |
| 1222 // to as late as possible -- just prior to parking. | |
| 1223 // Given that, we'd call HSSEC after having returned from park(), | |
| 1224 // but before attempting to acquire the monitor. This is only a | |
| 1225 // partial solution. It avoids calling HSSEC while holding the | |
| 1226 // monitor (good), but it still increases successor reacquisition latency -- | |
| 1227 // the interval between unparking a successor and the time the successor | |
| 1228 // resumes and retries the lock. See ReenterI(), which defers state transitions. | |
| 1229 // If we use this technique we can also avoid EnterI()-exit() loop | |
| 1230 // in ::enter() where we iteratively drop the lock and then attempt | |
| 1231 // to reacquire it after suspending. | |
| 1232 // | |
| 1233 // B. In the future we might fold all the suspend bits into a | |
| 1234 // composite per-thread suspend flag and then update it with CAS(). | |
| 1235 // Alternately, a Dekker-like mechanism with multiple variables | |
| 1236 // would suffice: | |
| 1237 // ST Self->_suspend_equivalent = false | |
| 1238 // MEMBAR | |
| 1239 // LD Self_>_suspend_flags | |
| 1240 // | |
| 1241 | |
| 1242 | |
| 1243 bool ObjectMonitor::ExitSuspendEquivalent (JavaThread * jSelf) { | |
| 1244 int Mode = Knob_FastHSSEC ; | |
| 1245 if (Mode && !jSelf->is_external_suspend()) { | |
| 1246 assert (jSelf->is_suspend_equivalent(), "invariant") ; | |
| 1247 jSelf->clear_suspend_equivalent() ; | |
| 1248 if (2 == Mode) OrderAccess::storeload() ; | |
| 1249 if (!jSelf->is_external_suspend()) return false ; | |
| 1250 // We raced a suspension -- fall thru into the slow path | |
| 1251 TEVENT (ExitSuspendEquivalent - raced) ; | |
| 1252 jSelf->set_suspend_equivalent() ; | |
| 1253 } | |
| 1254 return jSelf->handle_special_suspend_equivalent_condition() ; | |
| 1255 } | |
| 1256 | |
| 1257 | |
| 1258 void ObjectMonitor::ExitEpilog (Thread * Self, ObjectWaiter * Wakee) { | |
| 1259 assert (_owner == Self, "invariant") ; | |
| 1260 | |
| 1261 // Exit protocol: | |
| 1262 // 1. ST _succ = wakee | |
| 1263 // 2. membar #loadstore|#storestore; | |
| 1264 // 2. ST _owner = NULL | |
| 1265 // 3. unpark(wakee) | |
| 1266 | |
| 1267 _succ = Knob_SuccEnabled ? Wakee->_thread : NULL ; | |
| 1268 ParkEvent * Trigger = Wakee->_event ; | |
| 1269 | |
| 1270 // Hygiene -- once we've set _owner = NULL we can't safely dereference Wakee again. | |
| 1271 // The thread associated with Wakee may have grabbed the lock and "Wakee" may be | |
| 1272 // out-of-scope (non-extant). | |
| 1273 Wakee = NULL ; | |
| 1274 | |
| 1275 // Drop the lock | |
| 1276 OrderAccess::release_store_ptr (&_owner, NULL) ; | |
| 1277 OrderAccess::fence() ; // ST _owner vs LD in unpark() | |
| 1278 | |
| 1279 if (SafepointSynchronize::do_call_back()) { | |
| 1280 TEVENT (unpark before SAFEPOINT) ; | |
| 1281 } | |
| 1282 | |
| 1283 DTRACE_MONITOR_PROBE(contended__exit, this, object(), Self); | |
| 1284 Trigger->unpark() ; | |
| 1285 | |
| 1286 // Maintain stats and report events to JVMTI | |
| 1287 if (ObjectMonitor::_sync_Parks != NULL) { | |
| 1288 ObjectMonitor::_sync_Parks->inc() ; | |
| 1289 } | |
| 1290 } | |
| 1291 | |
| 1292 | |
| 1293 // ----------------------------------------------------------------------------- | |
| 1294 // Class Loader deadlock handling. | |
| 1295 // | |
| 1296 // complete_exit exits a lock returning recursion count | |
| 1297 // complete_exit/reenter operate as a wait without waiting | |
| 1298 // complete_exit requires an inflated monitor | |
| 1299 // The _owner field is not always the Thread addr even with an | |
| 1300 // inflated monitor, e.g. the monitor can be inflated by a non-owning | |
| 1301 // thread due to contention. | |
| 1302 intptr_t ObjectMonitor::complete_exit(TRAPS) { | |
| 1303 Thread * const Self = THREAD; | |
| 1304 assert(Self->is_Java_thread(), "Must be Java thread!"); | |
| 1305 JavaThread *jt = (JavaThread *)THREAD; | |
| 1306 | |
| 1307 DeferredInitialize(); | |
| 1308 | |
| 1309 if (THREAD != _owner) { | |
| 1310 if (THREAD->is_lock_owned ((address)_owner)) { | |
| 1311 assert(_recursions == 0, "internal state error"); | |
| 1312 _owner = THREAD ; /* Convert from basiclock addr to Thread addr */ | |
| 1313 _recursions = 0 ; | |
| 1314 OwnerIsThread = 1 ; | |
| 1315 } | |
| 1316 } | |
| 1317 | |
| 1318 guarantee(Self == _owner, "complete_exit not owner"); | |
| 1319 intptr_t save = _recursions; // record the old recursion count | |
| 1320 _recursions = 0; // set the recursion level to be 0 | |
| 1321 exit (Self) ; // exit the monitor | |
| 1322 guarantee (_owner != Self, "invariant"); | |
| 1323 return save; | |
| 1324 } | |
| 1325 | |
| 1326 // reenter() enters a lock and sets recursion count | |
| 1327 // complete_exit/reenter operate as a wait without waiting | |
| 1328 void ObjectMonitor::reenter(intptr_t recursions, TRAPS) { | |
| 1329 Thread * const Self = THREAD; | |
| 1330 assert(Self->is_Java_thread(), "Must be Java thread!"); | |
| 1331 JavaThread *jt = (JavaThread *)THREAD; | |
| 1332 | |
| 1333 guarantee(_owner != Self, "reenter already owner"); | |
| 1334 enter (THREAD); // enter the monitor | |
| 1335 guarantee (_recursions == 0, "reenter recursion"); | |
| 1336 _recursions = recursions; | |
| 1337 return; | |
| 1338 } | |
| 1339 | |
| 1340 | |
| 1341 // ----------------------------------------------------------------------------- | |
| 1342 // A macro is used below because there may already be a pending | |
| 1343 // exception which should not abort the execution of the routines | |
| 1344 // which use this (which is why we don't put this into check_slow and | |
| 1345 // call it with a CHECK argument). | |
| 1346 | |
| 1347 #define CHECK_OWNER() \ | |
| 1348 do { \ | |
| 1349 if (THREAD != _owner) { \ | |
| 1350 if (THREAD->is_lock_owned((address) _owner)) { \ | |
| 1351 _owner = THREAD ; /* Convert from basiclock addr to Thread addr */ \ | |
| 1352 _recursions = 0; \ | |
| 1353 OwnerIsThread = 1 ; \ | |
| 1354 } else { \ | |
| 1355 TEVENT (Throw IMSX) ; \ | |
| 1356 THROW(vmSymbols::java_lang_IllegalMonitorStateException()); \ | |
| 1357 } \ | |
| 1358 } \ | |
| 1359 } while (false) | |
| 1360 | |
| 1361 // check_slow() is a misnomer. It's called to simply to throw an IMSX exception. | |
| 1362 // TODO-FIXME: remove check_slow() -- it's likely dead. | |
| 1363 | |
| 1364 void ObjectMonitor::check_slow(TRAPS) { | |
| 1365 TEVENT (check_slow - throw IMSX) ; | |
| 1366 assert(THREAD != _owner && !THREAD->is_lock_owned((address) _owner), "must not be owner"); | |
| 1367 THROW_MSG(vmSymbols::java_lang_IllegalMonitorStateException(), "current thread not owner"); | |
| 1368 } | |
| 1369 | |
| 1370 static int Adjust (volatile int * adr, int dx) { | |
| 1371 int v ; | |
| 1372 for (v = *adr ; Atomic::cmpxchg (v + dx, adr, v) != v; v = *adr) ; | |
| 1373 return v ; | |
| 1374 } | |
| 1375 // ----------------------------------------------------------------------------- | |
| 1376 // Wait/Notify/NotifyAll | |
| 1377 // | |
| 1378 // Note: a subset of changes to ObjectMonitor::wait() | |
| 1379 // will need to be replicated in complete_exit above | |
| 1380 void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) { | |
| 1381 Thread * const Self = THREAD ; | |
| 1382 assert(Self->is_Java_thread(), "Must be Java thread!"); | |
| 1383 JavaThread *jt = (JavaThread *)THREAD; | |
| 1384 | |
| 1385 DeferredInitialize () ; | |
| 1386 | |
| 1387 // Throw IMSX or IEX. | |
| 1388 CHECK_OWNER(); | |
| 1389 | |
| 1390 // check for a pending interrupt | |
| 1391 if (interruptible && Thread::is_interrupted(Self, true) && !HAS_PENDING_EXCEPTION) { | |
| 1392 // post monitor waited event. Note that this is past-tense, we are done waiting. | |
| 1393 if (JvmtiExport::should_post_monitor_waited()) { | |
| 1394 // Note: 'false' parameter is passed here because the | |
| 1395 // wait was not timed out due to thread interrupt. | |
| 1396 JvmtiExport::post_monitor_waited(jt, this, false); | |
| 1397 } | |
| 1398 TEVENT (Wait - Throw IEX) ; | |
| 1399 THROW(vmSymbols::java_lang_InterruptedException()); | |
| 1400 return ; | |
| 1401 } | |
| 1402 TEVENT (Wait) ; | |
| 1403 | |
| 1404 assert (Self->_Stalled == 0, "invariant") ; | |
| 1405 Self->_Stalled = intptr_t(this) ; | |
| 1406 jt->set_current_waiting_monitor(this); | |
| 1407 | |
| 1408 // create a node to be put into the queue | |
| 1409 // Critically, after we reset() the event but prior to park(), we must check | |
| 1410 // for a pending interrupt. | |
| 1411 ObjectWaiter node(Self); | |
| 1412 node.TState = ObjectWaiter::TS_WAIT ; | |
| 1413 Self->_ParkEvent->reset() ; | |
| 1414 OrderAccess::fence(); // ST into Event; membar ; LD interrupted-flag | |
| 1415 | |
| 1416 // Enter the waiting queue, which is a circular doubly linked list in this case | |
| 1417 // but it could be a priority queue or any data structure. | |
| 1418 // _WaitSetLock protects the wait queue. Normally the wait queue is accessed only | |
| 1419 // by the the owner of the monitor *except* in the case where park() | |
| 1420 // returns because of a timeout of interrupt. Contention is exceptionally rare | |
| 1421 // so we use a simple spin-lock instead of a heavier-weight blocking lock. | |
| 1422 | |
| 1423 Thread::SpinAcquire (&_WaitSetLock, "WaitSet - add") ; | |
| 1424 AddWaiter (&node) ; | |
| 1425 Thread::SpinRelease (&_WaitSetLock) ; | |
| 1426 | |
| 1427 if ((SyncFlags & 4) == 0) { | |
| 1428 _Responsible = NULL ; | |
| 1429 } | |
| 1430 intptr_t save = _recursions; // record the old recursion count | |
| 1431 _waiters++; // increment the number of waiters | |
| 1432 _recursions = 0; // set the recursion level to be 1 | |
| 1433 exit (Self) ; // exit the monitor | |
| 1434 guarantee (_owner != Self, "invariant") ; | |
| 1435 | |
| 1436 // As soon as the ObjectMonitor's ownership is dropped in the exit() | |
| 1437 // call above, another thread can enter() the ObjectMonitor, do the | |
| 1438 // notify(), and exit() the ObjectMonitor. If the other thread's | |
| 1439 // exit() call chooses this thread as the successor and the unpark() | |
| 1440 // call happens to occur while this thread is posting a | |
| 1441 // MONITOR_CONTENDED_EXIT event, then we run the risk of the event | |
| 1442 // handler using RawMonitors and consuming the unpark(). | |
| 1443 // | |
| 1444 // To avoid the problem, we re-post the event. This does no harm | |
| 1445 // even if the original unpark() was not consumed because we are the | |
| 1446 // chosen successor for this monitor. | |
| 1447 if (node._notified != 0 && _succ == Self) { | |
| 1448 node._event->unpark(); | |
| 1449 } | |
| 1450 | |
| 1451 // The thread is on the WaitSet list - now park() it. | |
| 1452 // On MP systems it's conceivable that a brief spin before we park | |
| 1453 // could be profitable. | |
| 1454 // | |
| 1455 // TODO-FIXME: change the following logic to a loop of the form | |
| 1456 // while (!timeout && !interrupted && _notified == 0) park() | |
| 1457 | |
| 1458 int ret = OS_OK ; | |
| 1459 int WasNotified = 0 ; | |
| 1460 { // State transition wrappers | |
| 1461 OSThread* osthread = Self->osthread(); | |
| 1462 OSThreadWaitState osts(osthread, true); | |
| 1463 { | |
| 1464 ThreadBlockInVM tbivm(jt); | |
| 1465 // Thread is in thread_blocked state and oop access is unsafe. | |
| 1466 jt->set_suspend_equivalent(); | |
| 1467 | |
| 1468 if (interruptible && (Thread::is_interrupted(THREAD, false) || HAS_PENDING_EXCEPTION)) { | |
| 1469 // Intentionally empty | |
| 1470 } else | |
| 1471 if (node._notified == 0) { | |
| 1472 if (millis <= 0) { | |
| 1473 Self->_ParkEvent->park () ; | |
| 1474 } else { | |
| 1475 ret = Self->_ParkEvent->park (millis) ; | |
| 1476 } | |
| 1477 } | |
| 1478 | |
| 1479 // were we externally suspended while we were waiting? | |
| 1480 if (ExitSuspendEquivalent (jt)) { | |
| 1481 // TODO-FIXME: add -- if succ == Self then succ = null. | |
| 1482 jt->java_suspend_self(); | |
| 1483 } | |
| 1484 | |
| 1485 } // Exit thread safepoint: transition _thread_blocked -> _thread_in_vm | |
| 1486 | |
| 1487 | |
| 1488 // Node may be on the WaitSet, the EntryList (or cxq), or in transition | |
| 1489 // from the WaitSet to the EntryList. | |
| 1490 // See if we need to remove Node from the WaitSet. | |
| 1491 // We use double-checked locking to avoid grabbing _WaitSetLock | |
| 1492 // if the thread is not on the wait queue. | |
| 1493 // | |
| 1494 // Note that we don't need a fence before the fetch of TState. | |
| 1495 // In the worst case we'll fetch a old-stale value of TS_WAIT previously | |
| 1496 // written by the is thread. (perhaps the fetch might even be satisfied | |
| 1497 // by a look-aside into the processor's own store buffer, although given | |
| 1498 // the length of the code path between the prior ST and this load that's | |
| 1499 // highly unlikely). If the following LD fetches a stale TS_WAIT value | |
| 1500 // then we'll acquire the lock and then re-fetch a fresh TState value. | |
| 1501 // That is, we fail toward safety. | |
| 1502 | |
| 1503 if (node.TState == ObjectWaiter::TS_WAIT) { | |
| 1504 Thread::SpinAcquire (&_WaitSetLock, "WaitSet - unlink") ; | |
| 1505 if (node.TState == ObjectWaiter::TS_WAIT) { | |
| 1506 DequeueSpecificWaiter (&node) ; // unlink from WaitSet | |
| 1507 assert(node._notified == 0, "invariant"); | |
| 1508 node.TState = ObjectWaiter::TS_RUN ; | |
| 1509 } | |
| 1510 Thread::SpinRelease (&_WaitSetLock) ; | |
| 1511 } | |
| 1512 | |
| 1513 // The thread is now either on off-list (TS_RUN), | |
| 1514 // on the EntryList (TS_ENTER), or on the cxq (TS_CXQ). | |
| 1515 // The Node's TState variable is stable from the perspective of this thread. | |
| 1516 // No other threads will asynchronously modify TState. | |
| 1517 guarantee (node.TState != ObjectWaiter::TS_WAIT, "invariant") ; | |
| 1518 OrderAccess::loadload() ; | |
| 1519 if (_succ == Self) _succ = NULL ; | |
| 1520 WasNotified = node._notified ; | |
| 1521 | |
| 1522 // Reentry phase -- reacquire the monitor. | |
| 1523 // re-enter contended monitor after object.wait(). | |
| 1524 // retain OBJECT_WAIT state until re-enter successfully completes | |
| 1525 // Thread state is thread_in_vm and oop access is again safe, | |
| 1526 // although the raw address of the object may have changed. | |
| 1527 // (Don't cache naked oops over safepoints, of course). | |
| 1528 | |
| 1529 // post monitor waited event. Note that this is past-tense, we are done waiting. | |
| 1530 if (JvmtiExport::should_post_monitor_waited()) { | |
| 1531 JvmtiExport::post_monitor_waited(jt, this, ret == OS_TIMEOUT); | |
| 1532 } | |
| 1533 OrderAccess::fence() ; | |
| 1534 | |
| 1535 assert (Self->_Stalled != 0, "invariant") ; | |
| 1536 Self->_Stalled = 0 ; | |
| 1537 | |
| 1538 assert (_owner != Self, "invariant") ; | |
| 1539 ObjectWaiter::TStates v = node.TState ; | |
| 1540 if (v == ObjectWaiter::TS_RUN) { | |
| 1541 enter (Self) ; | |
| 1542 } else { | |
| 1543 guarantee (v == ObjectWaiter::TS_ENTER || v == ObjectWaiter::TS_CXQ, "invariant") ; | |
| 1544 ReenterI (Self, &node) ; | |
| 1545 node.wait_reenter_end(this); | |
| 1546 } | |
| 1547 | |
| 1548 // Self has reacquired the lock. | |
| 1549 // Lifecycle - the node representing Self must not appear on any queues. | |
| 1550 // Node is about to go out-of-scope, but even if it were immortal we wouldn't | |
| 1551 // want residual elements associated with this thread left on any lists. | |
| 1552 guarantee (node.TState == ObjectWaiter::TS_RUN, "invariant") ; | |
| 1553 assert (_owner == Self, "invariant") ; | |
| 1554 assert (_succ != Self , "invariant") ; | |
| 1555 } // OSThreadWaitState() | |
| 1556 | |
| 1557 jt->set_current_waiting_monitor(NULL); | |
| 1558 | |
| 1559 guarantee (_recursions == 0, "invariant") ; | |
| 1560 _recursions = save; // restore the old recursion count | |
| 1561 _waiters--; // decrement the number of waiters | |
| 1562 | |
| 1563 // Verify a few postconditions | |
| 1564 assert (_owner == Self , "invariant") ; | |
| 1565 assert (_succ != Self , "invariant") ; | |
| 1566 assert (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ; | |
| 1567 | |
| 1568 if (SyncFlags & 32) { | |
| 1569 OrderAccess::fence() ; | |
| 1570 } | |
| 1571 | |
| 1572 // check if the notification happened | |
| 1573 if (!WasNotified) { | |
| 1574 // no, it could be timeout or Thread.interrupt() or both | |
| 1575 // check for interrupt event, otherwise it is timeout | |
| 1576 if (interruptible && Thread::is_interrupted(Self, true) && !HAS_PENDING_EXCEPTION) { | |
| 1577 TEVENT (Wait - throw IEX from epilog) ; | |
| 1578 THROW(vmSymbols::java_lang_InterruptedException()); | |
| 1579 } | |
| 1580 } | |
| 1581 | |
| 1582 // NOTE: Spurious wake up will be consider as timeout. | |
| 1583 // Monitor notify has precedence over thread interrupt. | |
| 1584 } | |
| 1585 | |
| 1586 | |
| 1587 // Consider: | |
| 1588 // If the lock is cool (cxq == null && succ == null) and we're on an MP system | |
| 1589 // then instead of transferring a thread from the WaitSet to the EntryList | |
| 1590 // we might just dequeue a thread from the WaitSet and directly unpark() it. | |
| 1591 | |
| 1592 void ObjectMonitor::notify(TRAPS) { | |
| 1593 CHECK_OWNER(); | |
| 1594 if (_WaitSet == NULL) { | |
| 1595 TEVENT (Empty-Notify) ; | |
| 1596 return ; | |
| 1597 } | |
| 1598 DTRACE_MONITOR_PROBE(notify, this, object(), THREAD); | |
| 1599 | |
| 1600 int Policy = Knob_MoveNotifyee ; | |
| 1601 | |
| 1602 Thread::SpinAcquire (&_WaitSetLock, "WaitSet - notify") ; | |
| 1603 ObjectWaiter * iterator = DequeueWaiter() ; | |
| 1604 if (iterator != NULL) { | |
| 1605 TEVENT (Notify1 - Transfer) ; | |
| 1606 guarantee (iterator->TState == ObjectWaiter::TS_WAIT, "invariant") ; | |
| 1607 guarantee (iterator->_notified == 0, "invariant") ; | |
| 1608 if (Policy != 4) { | |
| 1609 iterator->TState = ObjectWaiter::TS_ENTER ; | |
| 1610 } | |
| 1611 iterator->_notified = 1 ; | |
| 1612 | |
| 1613 ObjectWaiter * List = _EntryList ; | |
| 1614 if (List != NULL) { | |
| 1615 assert (List->_prev == NULL, "invariant") ; | |
| 1616 assert (List->TState == ObjectWaiter::TS_ENTER, "invariant") ; | |
| 1617 assert (List != iterator, "invariant") ; | |
| 1618 } | |
| 1619 | |
| 1620 if (Policy == 0) { // prepend to EntryList | |
| 1621 if (List == NULL) { | |
| 1622 iterator->_next = iterator->_prev = NULL ; | |
| 1623 _EntryList = iterator ; | |
| 1624 } else { | |
| 1625 List->_prev = iterator ; | |
| 1626 iterator->_next = List ; | |
| 1627 iterator->_prev = NULL ; | |
| 1628 _EntryList = iterator ; | |
| 1629 } | |
| 1630 } else | |
| 1631 if (Policy == 1) { // append to EntryList | |
| 1632 if (List == NULL) { | |
| 1633 iterator->_next = iterator->_prev = NULL ; | |
| 1634 _EntryList = iterator ; | |
| 1635 } else { | |
| 1636 // CONSIDER: finding the tail currently requires a linear-time walk of | |
| 1637 // the EntryList. We can make tail access constant-time by converting to | |
| 1638 // a CDLL instead of using our current DLL. | |
| 1639 ObjectWaiter * Tail ; | |
| 1640 for (Tail = List ; Tail->_next != NULL ; Tail = Tail->_next) ; | |
| 1641 assert (Tail != NULL && Tail->_next == NULL, "invariant") ; | |
| 1642 Tail->_next = iterator ; | |
| 1643 iterator->_prev = Tail ; | |
| 1644 iterator->_next = NULL ; | |
| 1645 } | |
| 1646 } else | |
| 1647 if (Policy == 2) { // prepend to cxq | |
| 1648 // prepend to cxq | |
| 1649 if (List == NULL) { | |
| 1650 iterator->_next = iterator->_prev = NULL ; | |
| 1651 _EntryList = iterator ; | |
| 1652 } else { | |
| 1653 iterator->TState = ObjectWaiter::TS_CXQ ; | |
| 1654 for (;;) { | |
| 1655 ObjectWaiter * Front = _cxq ; | |
| 1656 iterator->_next = Front ; | |
| 1657 if (Atomic::cmpxchg_ptr (iterator, &_cxq, Front) == Front) { | |
| 1658 break ; | |
| 1659 } | |
| 1660 } | |
| 1661 } | |
| 1662 } else | |
| 1663 if (Policy == 3) { // append to cxq | |
| 1664 iterator->TState = ObjectWaiter::TS_CXQ ; | |
| 1665 for (;;) { | |
| 1666 ObjectWaiter * Tail ; | |
| 1667 Tail = _cxq ; | |
| 1668 if (Tail == NULL) { | |
| 1669 iterator->_next = NULL ; | |
| 1670 if (Atomic::cmpxchg_ptr (iterator, &_cxq, NULL) == NULL) { | |
| 1671 break ; | |
| 1672 } | |
| 1673 } else { | |
| 1674 while (Tail->_next != NULL) Tail = Tail->_next ; | |
| 1675 Tail->_next = iterator ; | |
| 1676 iterator->_prev = Tail ; | |
| 1677 iterator->_next = NULL ; | |
| 1678 break ; | |
| 1679 } | |
| 1680 } | |
| 1681 } else { | |
| 1682 ParkEvent * ev = iterator->_event ; | |
| 1683 iterator->TState = ObjectWaiter::TS_RUN ; | |
| 1684 OrderAccess::fence() ; | |
| 1685 ev->unpark() ; | |
| 1686 } | |
| 1687 | |
| 1688 if (Policy < 4) { | |
| 1689 iterator->wait_reenter_begin(this); | |
| 1690 } | |
| 1691 | |
| 1692 // _WaitSetLock protects the wait queue, not the EntryList. We could | |
| 1693 // move the add-to-EntryList operation, above, outside the critical section | |
| 1694 // protected by _WaitSetLock. In practice that's not useful. With the | |
| 1695 // exception of wait() timeouts and interrupts the monitor owner | |
| 1696 // is the only thread that grabs _WaitSetLock. There's almost no contention | |
| 1697 // on _WaitSetLock so it's not profitable to reduce the length of the | |
| 1698 // critical section. | |
| 1699 } | |
| 1700 | |
| 1701 Thread::SpinRelease (&_WaitSetLock) ; | |
| 1702 | |
| 1703 if (iterator != NULL && ObjectMonitor::_sync_Notifications != NULL) { | |
| 1704 ObjectMonitor::_sync_Notifications->inc() ; | |
| 1705 } | |
| 1706 } | |
| 1707 | |
| 1708 | |
| 1709 void ObjectMonitor::notifyAll(TRAPS) { | |
| 1710 CHECK_OWNER(); | |
| 1711 ObjectWaiter* iterator; | |
| 1712 if (_WaitSet == NULL) { | |
| 1713 TEVENT (Empty-NotifyAll) ; | |
| 1714 return ; | |
| 1715 } | |
| 1716 DTRACE_MONITOR_PROBE(notifyAll, this, object(), THREAD); | |
| 1717 | |
| 1718 int Policy = Knob_MoveNotifyee ; | |
| 1719 int Tally = 0 ; | |
| 1720 Thread::SpinAcquire (&_WaitSetLock, "WaitSet - notifyall") ; | |
| 1721 | |
| 1722 for (;;) { | |
| 1723 iterator = DequeueWaiter () ; | |
| 1724 if (iterator == NULL) break ; | |
| 1725 TEVENT (NotifyAll - Transfer1) ; | |
| 1726 ++Tally ; | |
| 1727 | |
| 1728 // Disposition - what might we do with iterator ? | |
| 1729 // a. add it directly to the EntryList - either tail or head. | |
| 1730 // b. push it onto the front of the _cxq. | |
| 1731 // For now we use (a). | |
| 1732 | |
| 1733 guarantee (iterator->TState == ObjectWaiter::TS_WAIT, "invariant") ; | |
| 1734 guarantee (iterator->_notified == 0, "invariant") ; | |
| 1735 iterator->_notified = 1 ; | |
| 1736 if (Policy != 4) { | |
| 1737 iterator->TState = ObjectWaiter::TS_ENTER ; | |
| 1738 } | |
| 1739 | |
| 1740 ObjectWaiter * List = _EntryList ; | |
| 1741 if (List != NULL) { | |
| 1742 assert (List->_prev == NULL, "invariant") ; | |
| 1743 assert (List->TState == ObjectWaiter::TS_ENTER, "invariant") ; | |
| 1744 assert (List != iterator, "invariant") ; | |
| 1745 } | |
| 1746 | |
| 1747 if (Policy == 0) { // prepend to EntryList | |
| 1748 if (List == NULL) { | |
| 1749 iterator->_next = iterator->_prev = NULL ; | |
| 1750 _EntryList = iterator ; | |
| 1751 } else { | |
| 1752 List->_prev = iterator ; | |
| 1753 iterator->_next = List ; | |
| 1754 iterator->_prev = NULL ; | |
| 1755 _EntryList = iterator ; | |
| 1756 } | |
| 1757 } else | |
| 1758 if (Policy == 1) { // append to EntryList | |
| 1759 if (List == NULL) { | |
| 1760 iterator->_next = iterator->_prev = NULL ; | |
| 1761 _EntryList = iterator ; | |
| 1762 } else { | |
| 1763 // CONSIDER: finding the tail currently requires a linear-time walk of | |
| 1764 // the EntryList. We can make tail access constant-time by converting to | |
| 1765 // a CDLL instead of using our current DLL. | |
| 1766 ObjectWaiter * Tail ; | |
| 1767 for (Tail = List ; Tail->_next != NULL ; Tail = Tail->_next) ; | |
| 1768 assert (Tail != NULL && Tail->_next == NULL, "invariant") ; | |
| 1769 Tail->_next = iterator ; | |
| 1770 iterator->_prev = Tail ; | |
| 1771 iterator->_next = NULL ; | |
| 1772 } | |
| 1773 } else | |
| 1774 if (Policy == 2) { // prepend to cxq | |
| 1775 // prepend to cxq | |
| 1776 iterator->TState = ObjectWaiter::TS_CXQ ; | |
| 1777 for (;;) { | |
| 1778 ObjectWaiter * Front = _cxq ; | |
| 1779 iterator->_next = Front ; | |
| 1780 if (Atomic::cmpxchg_ptr (iterator, &_cxq, Front) == Front) { | |
| 1781 break ; | |
| 1782 } | |
| 1783 } | |
| 1784 } else | |
| 1785 if (Policy == 3) { // append to cxq | |
| 1786 iterator->TState = ObjectWaiter::TS_CXQ ; | |
| 1787 for (;;) { | |
| 1788 ObjectWaiter * Tail ; | |
| 1789 Tail = _cxq ; | |
| 1790 if (Tail == NULL) { | |
| 1791 iterator->_next = NULL ; | |
| 1792 if (Atomic::cmpxchg_ptr (iterator, &_cxq, NULL) == NULL) { | |
| 1793 break ; | |
| 1794 } | |
| 1795 } else { | |
| 1796 while (Tail->_next != NULL) Tail = Tail->_next ; | |
| 1797 Tail->_next = iterator ; | |
| 1798 iterator->_prev = Tail ; | |
| 1799 iterator->_next = NULL ; | |
| 1800 break ; | |
| 1801 } | |
| 1802 } | |
| 1803 } else { | |
| 1804 ParkEvent * ev = iterator->_event ; | |
| 1805 iterator->TState = ObjectWaiter::TS_RUN ; | |
| 1806 OrderAccess::fence() ; | |
| 1807 ev->unpark() ; | |
| 1808 } | |
| 1809 | |
| 1810 if (Policy < 4) { | |
| 1811 iterator->wait_reenter_begin(this); | |
| 1812 } | |
| 1813 | |
| 1814 // _WaitSetLock protects the wait queue, not the EntryList. We could | |
| 1815 // move the add-to-EntryList operation, above, outside the critical section | |
| 1816 // protected by _WaitSetLock. In practice that's not useful. With the | |
| 1817 // exception of wait() timeouts and interrupts the monitor owner | |
| 1818 // is the only thread that grabs _WaitSetLock. There's almost no contention | |
| 1819 // on _WaitSetLock so it's not profitable to reduce the length of the | |
| 1820 // critical section. | |
| 1821 } | |
| 1822 | |
| 1823 Thread::SpinRelease (&_WaitSetLock) ; | |
| 1824 | |
| 1825 if (Tally != 0 && ObjectMonitor::_sync_Notifications != NULL) { | |
| 1826 ObjectMonitor::_sync_Notifications->inc(Tally) ; | |
| 1827 } | |
| 1828 } | |
| 1829 | |
| 1830 // ----------------------------------------------------------------------------- | |
| 1831 // Adaptive Spinning Support | |
| 1832 // | |
| 1833 // Adaptive spin-then-block - rational spinning | |
| 1834 // | |
| 1835 // Note that we spin "globally" on _owner with a classic SMP-polite TATAS | |
| 1836 // algorithm. On high order SMP systems it would be better to start with | |
| 1837 // a brief global spin and then revert to spinning locally. In the spirit of MCS/CLH, | |
| 1838 // a contending thread could enqueue itself on the cxq and then spin locally | |
| 1839 // on a thread-specific variable such as its ParkEvent._Event flag. | |
| 1840 // That's left as an exercise for the reader. Note that global spinning is | |
| 1841 // not problematic on Niagara, as the L2$ serves the interconnect and has both | |
| 1842 // low latency and massive bandwidth. | |
| 1843 // | |
| 1844 // Broadly, we can fix the spin frequency -- that is, the % of contended lock | |
| 1845 // acquisition attempts where we opt to spin -- at 100% and vary the spin count | |
| 1846 // (duration) or we can fix the count at approximately the duration of | |
| 1847 // a context switch and vary the frequency. Of course we could also | |
| 1848 // vary both satisfying K == Frequency * Duration, where K is adaptive by monitor. | |
| 1849 // See http://j2se.east/~dice/PERSIST/040824-AdaptiveSpinning.html. | |
| 1850 // | |
| 1851 // This implementation varies the duration "D", where D varies with | |
| 1852 // the success rate of recent spin attempts. (D is capped at approximately | |
| 1853 // length of a round-trip context switch). The success rate for recent | |
| 1854 // spin attempts is a good predictor of the success rate of future spin | |
| 1855 // attempts. The mechanism adapts automatically to varying critical | |
| 1856 // section length (lock modality), system load and degree of parallelism. | |
| 1857 // D is maintained per-monitor in _SpinDuration and is initialized | |
| 1858 // optimistically. Spin frequency is fixed at 100%. | |
| 1859 // | |
| 1860 // Note that _SpinDuration is volatile, but we update it without locks | |
| 1861 // or atomics. The code is designed so that _SpinDuration stays within | |
| 1862 // a reasonable range even in the presence of races. The arithmetic | |
| 1863 // operations on _SpinDuration are closed over the domain of legal values, | |
| 1864 // so at worst a race will install and older but still legal value. | |
| 1865 // At the very worst this introduces some apparent non-determinism. | |
| 1866 // We might spin when we shouldn't or vice-versa, but since the spin | |
| 1867 // count are relatively short, even in the worst case, the effect is harmless. | |
| 1868 // | |
| 1869 // Care must be taken that a low "D" value does not become an | |
| 1870 // an absorbing state. Transient spinning failures -- when spinning | |
| 1871 // is overall profitable -- should not cause the system to converge | |
| 1872 // on low "D" values. We want spinning to be stable and predictable | |
| 1873 // and fairly responsive to change and at the same time we don't want | |
| 1874 // it to oscillate, become metastable, be "too" non-deterministic, | |
| 1875 // or converge on or enter undesirable stable absorbing states. | |
| 1876 // | |
| 1877 // We implement a feedback-based control system -- using past behavior | |
| 1878 // to predict future behavior. We face two issues: (a) if the | |
| 1879 // input signal is random then the spin predictor won't provide optimal | |
| 1880 // results, and (b) if the signal frequency is too high then the control | |
| 1881 // system, which has some natural response lag, will "chase" the signal. | |
| 1882 // (b) can arise from multimodal lock hold times. Transient preemption | |
| 1883 // can also result in apparent bimodal lock hold times. | |
| 1884 // Although sub-optimal, neither condition is particularly harmful, as | |
| 1885 // in the worst-case we'll spin when we shouldn't or vice-versa. | |
| 1886 // The maximum spin duration is rather short so the failure modes aren't bad. | |
| 1887 // To be conservative, I've tuned the gain in system to bias toward | |
| 1888 // _not spinning. Relatedly, the system can sometimes enter a mode where it | |
| 1889 // "rings" or oscillates between spinning and not spinning. This happens | |
| 1890 // when spinning is just on the cusp of profitability, however, so the | |
| 1891 // situation is not dire. The state is benign -- there's no need to add | |
| 1892 // hysteresis control to damp the transition rate between spinning and | |
| 1893 // not spinning. | |
| 1894 // | |
| 1895 | |
| 1896 intptr_t ObjectMonitor::SpinCallbackArgument = 0 ; | |
| 1897 int (*ObjectMonitor::SpinCallbackFunction)(intptr_t, int) = NULL ; | |
| 1898 | |
| 1899 // Spinning: Fixed frequency (100%), vary duration | |
| 1900 | |
| 1901 | |
| 1902 int ObjectMonitor::TrySpin_VaryDuration (Thread * Self) { | |
| 1903 | |
| 1904 // Dumb, brutal spin. Good for comparative measurements against adaptive spinning. | |
| 1905 int ctr = Knob_FixedSpin ; | |
| 1906 if (ctr != 0) { | |
| 1907 while (--ctr >= 0) { | |
| 1908 if (TryLock (Self) > 0) return 1 ; | |
| 1909 SpinPause () ; | |
| 1910 } | |
| 1911 return 0 ; | |
| 1912 } | |
| 1913 | |
| 1914 for (ctr = Knob_PreSpin + 1; --ctr >= 0 ; ) { | |
| 1915 if (TryLock(Self) > 0) { | |
| 1916 // Increase _SpinDuration ... | |
| 1917 // Note that we don't clamp SpinDuration precisely at SpinLimit. | |
| 1918 // Raising _SpurDuration to the poverty line is key. | |
| 1919 int x = _SpinDuration ; | |
| 1920 if (x < Knob_SpinLimit) { | |
| 1921 if (x < Knob_Poverty) x = Knob_Poverty ; | |
| 1922 _SpinDuration = x + Knob_BonusB ; | |
| 1923 } | |
| 1924 return 1 ; | |
| 1925 } | |
| 1926 SpinPause () ; | |
| 1927 } | |
| 1928 | |
| 1929 // Admission control - verify preconditions for spinning | |
| 1930 // | |
| 1931 // We always spin a little bit, just to prevent _SpinDuration == 0 from | |
| 1932 // becoming an absorbing state. Put another way, we spin briefly to | |
| 1933 // sample, just in case the system load, parallelism, contention, or lock | |
| 1934 // modality changed. | |
| 1935 // | |
| 1936 // Consider the following alternative: | |
| 1937 // Periodically set _SpinDuration = _SpinLimit and try a long/full | |
| 1938 // spin attempt. "Periodically" might mean after a tally of | |
| 1939 // the # of failed spin attempts (or iterations) reaches some threshold. | |
| 1940 // This takes us into the realm of 1-out-of-N spinning, where we | |
| 1941 // hold the duration constant but vary the frequency. | |
| 1942 | |
| 1943 ctr = _SpinDuration ; | |
| 1944 if (ctr < Knob_SpinBase) ctr = Knob_SpinBase ; | |
| 1945 if (ctr <= 0) return 0 ; | |
| 1946 | |
| 1947 if (Knob_SuccRestrict && _succ != NULL) return 0 ; | |
| 1948 if (Knob_OState && NotRunnable (Self, (Thread *) _owner)) { | |
| 1949 TEVENT (Spin abort - notrunnable [TOP]); | |
| 1950 return 0 ; | |
| 1951 } | |
| 1952 | |
| 1953 int MaxSpin = Knob_MaxSpinners ; | |
| 1954 if (MaxSpin >= 0) { | |
| 1955 if (_Spinner > MaxSpin) { | |
| 1956 TEVENT (Spin abort -- too many spinners) ; | |
| 1957 return 0 ; | |
| 1958 } | |
| 1959 // Slighty racy, but benign ... | |
| 1960 Adjust (&_Spinner, 1) ; | |
| 1961 } | |
| 1962 | |
| 1963 // We're good to spin ... spin ingress. | |
| 1964 // CONSIDER: use Prefetch::write() to avoid RTS->RTO upgrades | |
| 1965 // when preparing to LD...CAS _owner, etc and the CAS is likely | |
| 1966 // to succeed. | |
| 1967 int hits = 0 ; | |
| 1968 int msk = 0 ; | |
| 1969 int caspty = Knob_CASPenalty ; | |
| 1970 int oxpty = Knob_OXPenalty ; | |
| 1971 int sss = Knob_SpinSetSucc ; | |
| 1972 if (sss && _succ == NULL ) _succ = Self ; | |
| 1973 Thread * prv = NULL ; | |
| 1974 | |
| 1975 // There are three ways to exit the following loop: | |
| 1976 // 1. A successful spin where this thread has acquired the lock. | |
| 1977 // 2. Spin failure with prejudice | |
| 1978 // 3. Spin failure without prejudice | |
| 1979 | |
| 1980 while (--ctr >= 0) { | |
| 1981 | |
| 1982 // Periodic polling -- Check for pending GC | |
| 1983 // Threads may spin while they're unsafe. | |
| 1984 // We don't want spinning threads to delay the JVM from reaching | |
| 1985 // a stop-the-world safepoint or to steal cycles from GC. | |
| 1986 // If we detect a pending safepoint we abort in order that | |
| 1987 // (a) this thread, if unsafe, doesn't delay the safepoint, and (b) | |
| 1988 // this thread, if safe, doesn't steal cycles from GC. | |
| 1989 // This is in keeping with the "no loitering in runtime" rule. | |
| 1990 // We periodically check to see if there's a safepoint pending. | |
| 1991 if ((ctr & 0xFF) == 0) { | |
| 1992 if (SafepointSynchronize::do_call_back()) { | |
| 1993 TEVENT (Spin: safepoint) ; | |
| 1994 goto Abort ; // abrupt spin egress | |
| 1995 } | |
| 1996 if (Knob_UsePause & 1) SpinPause () ; | |
| 1997 | |
| 1998 int (*scb)(intptr_t,int) = SpinCallbackFunction ; | |
| 1999 if (hits > 50 && scb != NULL) { | |
| 2000 int abend = (*scb)(SpinCallbackArgument, 0) ; | |
| 2001 } | |
| 2002 } | |
| 2003 | |
| 2004 if (Knob_UsePause & 2) SpinPause() ; | |
| 2005 | |
| 2006 // Exponential back-off ... Stay off the bus to reduce coherency traffic. | |
| 2007 // This is useful on classic SMP systems, but is of less utility on | |
| 2008 // N1-style CMT platforms. | |
| 2009 // | |
| 2010 // Trade-off: lock acquisition latency vs coherency bandwidth. | |
| 2011 // Lock hold times are typically short. A histogram | |
| 2012 // of successful spin attempts shows that we usually acquire | |
| 2013 // the lock early in the spin. That suggests we want to | |
| 2014 // sample _owner frequently in the early phase of the spin, | |
| 2015 // but then back-off and sample less frequently as the spin | |
| 2016 // progresses. The back-off makes a good citizen on SMP big | |
| 2017 // SMP systems. Oversampling _owner can consume excessive | |
| 2018 // coherency bandwidth. Relatedly, if we _oversample _owner we | |
| 2019 // can inadvertently interfere with the the ST m->owner=null. | |
| 2020 // executed by the lock owner. | |
| 2021 if (ctr & msk) continue ; | |
| 2022 ++hits ; | |
| 2023 if ((hits & 0xF) == 0) { | |
| 2024 // The 0xF, above, corresponds to the exponent. | |
| 2025 // Consider: (msk+1)|msk | |
| 2026 msk = ((msk << 2)|3) & BackOffMask ; | |
| 2027 } | |
| 2028 | |
| 2029 // Probe _owner with TATAS | |
| 2030 // If this thread observes the monitor transition or flicker | |
| 2031 // from locked to unlocked to locked, then the odds that this | |
| 2032 // thread will acquire the lock in this spin attempt go down | |
| 2033 // considerably. The same argument applies if the CAS fails | |
| 2034 // or if we observe _owner change from one non-null value to | |
| 2035 // another non-null value. In such cases we might abort | |
| 2036 // the spin without prejudice or apply a "penalty" to the | |
| 2037 // spin count-down variable "ctr", reducing it by 100, say. | |
| 2038 | |
| 2039 Thread * ox = (Thread *) _owner ; | |
| 2040 if (ox == NULL) { | |
| 2041 ox = (Thread *) Atomic::cmpxchg_ptr (Self, &_owner, NULL) ; | |
| 2042 if (ox == NULL) { | |
| 2043 // The CAS succeeded -- this thread acquired ownership | |
| 2044 // Take care of some bookkeeping to exit spin state. | |
| 2045 if (sss && _succ == Self) { | |
| 2046 _succ = NULL ; | |
| 2047 } | |
| 2048 if (MaxSpin > 0) Adjust (&_Spinner, -1) ; | |
| 2049 | |
| 2050 // Increase _SpinDuration : | |
| 2051 // The spin was successful (profitable) so we tend toward | |
| 2052 // longer spin attempts in the future. | |
| 2053 // CONSIDER: factor "ctr" into the _SpinDuration adjustment. | |
| 2054 // If we acquired the lock early in the spin cycle it | |
| 2055 // makes sense to increase _SpinDuration proportionally. | |
| 2056 // Note that we don't clamp SpinDuration precisely at SpinLimit. | |
| 2057 int x = _SpinDuration ; | |
| 2058 if (x < Knob_SpinLimit) { | |
| 2059 if (x < Knob_Poverty) x = Knob_Poverty ; | |
| 2060 _SpinDuration = x + Knob_Bonus ; | |
| 2061 } | |
| 2062 return 1 ; | |
| 2063 } | |
| 2064 | |
| 2065 // The CAS failed ... we can take any of the following actions: | |
| 2066 // * penalize: ctr -= Knob_CASPenalty | |
| 2067 // * exit spin with prejudice -- goto Abort; | |
| 2068 // * exit spin without prejudice. | |
| 2069 // * Since CAS is high-latency, retry again immediately. | |
| 2070 prv = ox ; | |
| 2071 TEVENT (Spin: cas failed) ; | |
| 2072 if (caspty == -2) break ; | |
| 2073 if (caspty == -1) goto Abort ; | |
| 2074 ctr -= caspty ; | |
| 2075 continue ; | |
| 2076 } | |
| 2077 | |
| 2078 // Did lock ownership change hands ? | |
| 2079 if (ox != prv && prv != NULL ) { | |
| 2080 TEVENT (spin: Owner changed) | |
| 2081 if (oxpty == -2) break ; | |
| 2082 if (oxpty == -1) goto Abort ; | |
| 2083 ctr -= oxpty ; | |
| 2084 } | |
| 2085 prv = ox ; | |
| 2086 | |
| 2087 // Abort the spin if the owner is not executing. | |
| 2088 // The owner must be executing in order to drop the lock. | |
| 2089 // Spinning while the owner is OFFPROC is idiocy. | |
| 2090 // Consider: ctr -= RunnablePenalty ; | |
| 2091 if (Knob_OState && NotRunnable (Self, ox)) { | |
| 2092 TEVENT (Spin abort - notrunnable); | |
| 2093 goto Abort ; | |
| 2094 } | |
| 2095 if (sss && _succ == NULL ) _succ = Self ; | |
| 2096 } | |
| 2097 | |
| 2098 // Spin failed with prejudice -- reduce _SpinDuration. | |
| 2099 // TODO: Use an AIMD-like policy to adjust _SpinDuration. | |
| 2100 // AIMD is globally stable. | |
| 2101 TEVENT (Spin failure) ; | |
| 2102 { | |
| 2103 int x = _SpinDuration ; | |
| 2104 if (x > 0) { | |
| 2105 // Consider an AIMD scheme like: x -= (x >> 3) + 100 | |
| 2106 // This is globally sample and tends to damp the response. | |
| 2107 x -= Knob_Penalty ; | |
| 2108 if (x < 0) x = 0 ; | |
| 2109 _SpinDuration = x ; | |
| 2110 } | |
| 2111 } | |
| 2112 | |
| 2113 Abort: | |
| 2114 if (MaxSpin >= 0) Adjust (&_Spinner, -1) ; | |
| 2115 if (sss && _succ == Self) { | |
| 2116 _succ = NULL ; | |
| 2117 // Invariant: after setting succ=null a contending thread | |
| 2118 // must recheck-retry _owner before parking. This usually happens | |
| 2119 // in the normal usage of TrySpin(), but it's safest | |
| 2120 // to make TrySpin() as foolproof as possible. | |
| 2121 OrderAccess::fence() ; | |
| 2122 if (TryLock(Self) > 0) return 1 ; | |
| 2123 } | |
| 2124 return 0 ; | |
| 2125 } | |
| 2126 | |
| 2127 // NotRunnable() -- informed spinning | |
| 2128 // | |
| 2129 // Don't bother spinning if the owner is not eligible to drop the lock. | |
| 2130 // Peek at the owner's schedctl.sc_state and Thread._thread_values and | |
| 2131 // spin only if the owner thread is _thread_in_Java or _thread_in_vm. | |
| 2132 // The thread must be runnable in order to drop the lock in timely fashion. | |
| 2133 // If the _owner is not runnable then spinning will not likely be | |
| 2134 // successful (profitable). | |
| 2135 // | |
| 2136 // Beware -- the thread referenced by _owner could have died | |
| 2137 // so a simply fetch from _owner->_thread_state might trap. | |
| 2138 // Instead, we use SafeFetchXX() to safely LD _owner->_thread_state. | |
| 2139 // Because of the lifecycle issues the schedctl and _thread_state values | |
| 2140 // observed by NotRunnable() might be garbage. NotRunnable must | |
| 2141 // tolerate this and consider the observed _thread_state value | |
| 2142 // as advisory. | |
| 2143 // | |
| 2144 // Beware too, that _owner is sometimes a BasicLock address and sometimes | |
| 2145 // a thread pointer. We differentiate the two cases with OwnerIsThread. | |
| 2146 // Alternately, we might tag the type (thread pointer vs basiclock pointer) | |
| 2147 // with the LSB of _owner. Another option would be to probablistically probe | |
| 2148 // the putative _owner->TypeTag value. | |
| 2149 // | |
| 2150 // Checking _thread_state isn't perfect. Even if the thread is | |
| 2151 // in_java it might be blocked on a page-fault or have been preempted | |
| 2152 // and sitting on a ready/dispatch queue. _thread state in conjunction | |
| 2153 // with schedctl.sc_state gives us a good picture of what the | |
| 2154 // thread is doing, however. | |
| 2155 // | |
| 2156 // TODO: check schedctl.sc_state. | |
| 2157 // We'll need to use SafeFetch32() to read from the schedctl block. | |
| 2158 // See RFE #5004247 and http://sac.sfbay.sun.com/Archives/CaseLog/arc/PSARC/2005/351/ | |
| 2159 // | |
| 2160 // The return value from NotRunnable() is *advisory* -- the | |
| 2161 // result is based on sampling and is not necessarily coherent. | |
| 2162 // The caller must tolerate false-negative and false-positive errors. | |
| 2163 // Spinning, in general, is probabilistic anyway. | |
| 2164 | |
| 2165 | |
| 2166 int ObjectMonitor::NotRunnable (Thread * Self, Thread * ox) { | |
| 2167 // Check either OwnerIsThread or ox->TypeTag == 2BAD. | |
| 2168 if (!OwnerIsThread) return 0 ; | |
| 2169 | |
| 2170 if (ox == NULL) return 0 ; | |
| 2171 | |
| 2172 // Avoid transitive spinning ... | |
| 2173 // Say T1 spins or blocks trying to acquire L. T1._Stalled is set to L. | |
| 2174 // Immediately after T1 acquires L it's possible that T2, also | |
| 2175 // spinning on L, will see L.Owner=T1 and T1._Stalled=L. | |
| 2176 // This occurs transiently after T1 acquired L but before | |
| 2177 // T1 managed to clear T1.Stalled. T2 does not need to abort | |
| 2178 // its spin in this circumstance. | |
| 2179 intptr_t BlockedOn = SafeFetchN ((intptr_t *) &ox->_Stalled, intptr_t(1)) ; | |
| 2180 | |
| 2181 if (BlockedOn == 1) return 1 ; | |
| 2182 if (BlockedOn != 0) { | |
| 2183 return BlockedOn != intptr_t(this) && _owner == ox ; | |
| 2184 } | |
| 2185 | |
| 2186 assert (sizeof(((JavaThread *)ox)->_thread_state == sizeof(int)), "invariant") ; | |
| 2187 int jst = SafeFetch32 ((int *) &((JavaThread *) ox)->_thread_state, -1) ; ; | |
| 2188 // consider also: jst != _thread_in_Java -- but that's overspecific. | |
| 2189 return jst == _thread_blocked || jst == _thread_in_native ; | |
| 2190 } | |
| 2191 | |
| 2192 | |
| 2193 // ----------------------------------------------------------------------------- | |
| 2194 // WaitSet management ... | |
| 2195 | |
| 2196 ObjectWaiter::ObjectWaiter(Thread* thread) { | |
| 2197 _next = NULL; | |
| 2198 _prev = NULL; | |
| 2199 _notified = 0; | |
| 2200 TState = TS_RUN ; | |
| 2201 _thread = thread; | |
| 2202 _event = thread->_ParkEvent ; | |
| 2203 _active = false; | |
| 2204 assert (_event != NULL, "invariant") ; | |
| 2205 } | |
| 2206 | |
| 2207 void ObjectWaiter::wait_reenter_begin(ObjectMonitor *mon) { | |
| 2208 JavaThread *jt = (JavaThread *)this->_thread; | |
| 2209 _active = JavaThreadBlockedOnMonitorEnterState::wait_reenter_begin(jt, mon); | |
| 2210 } | |
| 2211 | |
| 2212 void ObjectWaiter::wait_reenter_end(ObjectMonitor *mon) { | |
| 2213 JavaThread *jt = (JavaThread *)this->_thread; | |
| 2214 JavaThreadBlockedOnMonitorEnterState::wait_reenter_end(jt, _active); | |
| 2215 } | |
| 2216 | |
| 2217 inline void ObjectMonitor::AddWaiter(ObjectWaiter* node) { | |
| 2218 assert(node != NULL, "should not dequeue NULL node"); | |
| 2219 assert(node->_prev == NULL, "node already in list"); | |
| 2220 assert(node->_next == NULL, "node already in list"); | |
| 2221 // put node at end of queue (circular doubly linked list) | |
| 2222 if (_WaitSet == NULL) { | |
| 2223 _WaitSet = node; | |
| 2224 node->_prev = node; | |
| 2225 node->_next = node; | |
| 2226 } else { | |
| 2227 ObjectWaiter* head = _WaitSet ; | |
| 2228 ObjectWaiter* tail = head->_prev; | |
| 2229 assert(tail->_next == head, "invariant check"); | |
| 2230 tail->_next = node; | |
| 2231 head->_prev = node; | |
| 2232 node->_next = head; | |
| 2233 node->_prev = tail; | |
| 2234 } | |
| 2235 } | |
| 2236 | |
| 2237 inline ObjectWaiter* ObjectMonitor::DequeueWaiter() { | |
| 2238 // dequeue the very first waiter | |
| 2239 ObjectWaiter* waiter = _WaitSet; | |
| 2240 if (waiter) { | |
| 2241 DequeueSpecificWaiter(waiter); | |
| 2242 } | |
| 2243 return waiter; | |
| 2244 } | |
| 2245 | |
| 2246 inline void ObjectMonitor::DequeueSpecificWaiter(ObjectWaiter* node) { | |
| 2247 assert(node != NULL, "should not dequeue NULL node"); | |
| 2248 assert(node->_prev != NULL, "node already removed from list"); | |
| 2249 assert(node->_next != NULL, "node already removed from list"); | |
| 2250 // when the waiter has woken up because of interrupt, | |
| 2251 // timeout or other spurious wake-up, dequeue the | |
| 2252 // waiter from waiting list | |
| 2253 ObjectWaiter* next = node->_next; | |
| 2254 if (next == node) { | |
| 2255 assert(node->_prev == node, "invariant check"); | |
| 2256 _WaitSet = NULL; | |
| 2257 } else { | |
| 2258 ObjectWaiter* prev = node->_prev; | |
| 2259 assert(prev->_next == node, "invariant check"); | |
| 2260 assert(next->_prev == node, "invariant check"); | |
| 2261 next->_prev = prev; | |
| 2262 prev->_next = next; | |
| 2263 if (_WaitSet == node) { | |
| 2264 _WaitSet = next; | |
| 2265 } | |
| 2266 } | |
| 2267 node->_next = NULL; | |
| 2268 node->_prev = NULL; | |
| 2269 } | |
| 2270 | |
| 2271 // ----------------------------------------------------------------------------- | |
| 2272 // PerfData support | |
| 2273 PerfCounter * ObjectMonitor::_sync_ContendedLockAttempts = NULL ; | |
| 2274 PerfCounter * ObjectMonitor::_sync_FutileWakeups = NULL ; | |
| 2275 PerfCounter * ObjectMonitor::_sync_Parks = NULL ; | |
| 2276 PerfCounter * ObjectMonitor::_sync_EmptyNotifications = NULL ; | |
| 2277 PerfCounter * ObjectMonitor::_sync_Notifications = NULL ; | |
| 2278 PerfCounter * ObjectMonitor::_sync_PrivateA = NULL ; | |
| 2279 PerfCounter * ObjectMonitor::_sync_PrivateB = NULL ; | |
| 2280 PerfCounter * ObjectMonitor::_sync_SlowExit = NULL ; | |
| 2281 PerfCounter * ObjectMonitor::_sync_SlowEnter = NULL ; | |
| 2282 PerfCounter * ObjectMonitor::_sync_SlowNotify = NULL ; | |
| 2283 PerfCounter * ObjectMonitor::_sync_SlowNotifyAll = NULL ; | |
| 2284 PerfCounter * ObjectMonitor::_sync_FailedSpins = NULL ; | |
| 2285 PerfCounter * ObjectMonitor::_sync_SuccessfulSpins = NULL ; | |
| 2286 PerfCounter * ObjectMonitor::_sync_MonInCirculation = NULL ; | |
| 2287 PerfCounter * ObjectMonitor::_sync_MonScavenged = NULL ; | |
| 2288 PerfCounter * ObjectMonitor::_sync_Inflations = NULL ; | |
| 2289 PerfCounter * ObjectMonitor::_sync_Deflations = NULL ; | |
| 2290 PerfLongVariable * ObjectMonitor::_sync_MonExtant = NULL ; | |
| 2291 | |
| 2292 // One-shot global initialization for the sync subsystem. | |
| 2293 // We could also defer initialization and initialize on-demand | |
| 2294 // the first time we call inflate(). Initialization would | |
| 2295 // be protected - like so many things - by the MonitorCache_lock. | |
| 2296 | |
| 2297 void ObjectMonitor::Initialize () { | |
| 2298 static int InitializationCompleted = 0 ; | |
| 2299 assert (InitializationCompleted == 0, "invariant") ; | |
| 2300 InitializationCompleted = 1 ; | |
| 2301 if (UsePerfData) { | |
| 2302 EXCEPTION_MARK ; | |
| 2303 #define NEWPERFCOUNTER(n) {n = PerfDataManager::create_counter(SUN_RT, #n, PerfData::U_Events,CHECK); } | |
| 2304 #define NEWPERFVARIABLE(n) {n = PerfDataManager::create_variable(SUN_RT, #n, PerfData::U_Events,CHECK); } | |
| 2305 NEWPERFCOUNTER(_sync_Inflations) ; | |
| 2306 NEWPERFCOUNTER(_sync_Deflations) ; | |
| 2307 NEWPERFCOUNTER(_sync_ContendedLockAttempts) ; | |
| 2308 NEWPERFCOUNTER(_sync_FutileWakeups) ; | |
| 2309 NEWPERFCOUNTER(_sync_Parks) ; | |
| 2310 NEWPERFCOUNTER(_sync_EmptyNotifications) ; | |
| 2311 NEWPERFCOUNTER(_sync_Notifications) ; | |
| 2312 NEWPERFCOUNTER(_sync_SlowEnter) ; | |
| 2313 NEWPERFCOUNTER(_sync_SlowExit) ; | |
| 2314 NEWPERFCOUNTER(_sync_SlowNotify) ; | |
| 2315 NEWPERFCOUNTER(_sync_SlowNotifyAll) ; | |
| 2316 NEWPERFCOUNTER(_sync_FailedSpins) ; | |
| 2317 NEWPERFCOUNTER(_sync_SuccessfulSpins) ; | |
| 2318 NEWPERFCOUNTER(_sync_PrivateA) ; | |
| 2319 NEWPERFCOUNTER(_sync_PrivateB) ; | |
| 2320 NEWPERFCOUNTER(_sync_MonInCirculation) ; | |
| 2321 NEWPERFCOUNTER(_sync_MonScavenged) ; | |
| 2322 NEWPERFVARIABLE(_sync_MonExtant) ; | |
| 2323 #undef NEWPERFCOUNTER | |
| 2324 } | |
| 2325 } | |
| 2326 | |
| 2327 | |
| 2328 // Compile-time asserts | |
| 2329 // When possible, it's better to catch errors deterministically at | |
| 2330 // compile-time than at runtime. The down-side to using compile-time | |
| 2331 // asserts is that error message -- often something about negative array | |
| 2332 // indices -- is opaque. | |
| 2333 | |
| 2334 #define CTASSERT(x) { int tag[1-(2*!(x))]; printf ("Tag @" INTPTR_FORMAT "\n", (intptr_t)tag); } | |
| 2335 | |
| 2336 void ObjectMonitor::ctAsserts() { | |
| 2337 CTASSERT(offset_of (ObjectMonitor, _header) == 0); | |
| 2338 } | |
| 2339 | |
| 2340 | |
| 2341 static char * kvGet (char * kvList, const char * Key) { | |
| 2342 if (kvList == NULL) return NULL ; | |
| 2343 size_t n = strlen (Key) ; | |
| 2344 char * Search ; | |
| 2345 for (Search = kvList ; *Search ; Search += strlen(Search) + 1) { | |
| 2346 if (strncmp (Search, Key, n) == 0) { | |
| 2347 if (Search[n] == '=') return Search + n + 1 ; | |
| 2348 if (Search[n] == 0) return (char *) "1" ; | |
| 2349 } | |
| 2350 } | |
| 2351 return NULL ; | |
| 2352 } | |
| 2353 | |
| 2354 static int kvGetInt (char * kvList, const char * Key, int Default) { | |
| 2355 char * v = kvGet (kvList, Key) ; | |
| 2356 int rslt = v ? ::strtol (v, NULL, 0) : Default ; | |
| 2357 if (Knob_ReportSettings && v != NULL) { | |
| 2358 ::printf (" SyncKnob: %s %d(%d)\n", Key, rslt, Default) ; | |
| 2359 ::fflush (stdout) ; | |
| 2360 } | |
| 2361 return rslt ; | |
| 2362 } | |
| 2363 | |
| 2364 void ObjectMonitor::DeferredInitialize () { | |
| 2365 if (InitDone > 0) return ; | |
| 2366 if (Atomic::cmpxchg (-1, &InitDone, 0) != 0) { | |
| 2367 while (InitDone != 1) ; | |
| 2368 return ; | |
| 2369 } | |
| 2370 | |
| 2371 // One-shot global initialization ... | |
| 2372 // The initialization is idempotent, so we don't need locks. | |
| 2373 // In the future consider doing this via os::init_2(). | |
| 2374 // SyncKnobs consist of <Key>=<Value> pairs in the style | |
| 2375 // of environment variables. Start by converting ':' to NUL. | |
| 2376 | |
| 2377 if (SyncKnobs == NULL) SyncKnobs = "" ; | |
| 2378 | |
| 2379 size_t sz = strlen (SyncKnobs) ; | |
| 2380 char * knobs = (char *) malloc (sz + 2) ; | |
| 2381 if (knobs == NULL) { | |
| 2382 vm_exit_out_of_memory (sz + 2, "Parse SyncKnobs") ; | |
| 2383 guarantee (0, "invariant") ; | |
| 2384 } | |
| 2385 strcpy (knobs, SyncKnobs) ; | |
| 2386 knobs[sz+1] = 0 ; | |
| 2387 for (char * p = knobs ; *p ; p++) { | |
| 2388 if (*p == ':') *p = 0 ; | |
| 2389 } | |
| 2390 | |
| 2391 #define SETKNOB(x) { Knob_##x = kvGetInt (knobs, #x, Knob_##x); } | |
| 2392 SETKNOB(ReportSettings) ; | |
| 2393 SETKNOB(Verbose) ; | |
| 2394 SETKNOB(FixedSpin) ; | |
| 2395 SETKNOB(SpinLimit) ; | |
| 2396 SETKNOB(SpinBase) ; | |
| 2397 SETKNOB(SpinBackOff); | |
| 2398 SETKNOB(CASPenalty) ; | |
| 2399 SETKNOB(OXPenalty) ; | |
| 2400 SETKNOB(LogSpins) ; | |
| 2401 SETKNOB(SpinSetSucc) ; | |
| 2402 SETKNOB(SuccEnabled) ; | |
| 2403 SETKNOB(SuccRestrict) ; | |
| 2404 SETKNOB(Penalty) ; | |
| 2405 SETKNOB(Bonus) ; | |
| 2406 SETKNOB(BonusB) ; | |
| 2407 SETKNOB(Poverty) ; | |
| 2408 SETKNOB(SpinAfterFutile) ; | |
| 2409 SETKNOB(UsePause) ; | |
| 2410 SETKNOB(SpinEarly) ; | |
| 2411 SETKNOB(OState) ; | |
| 2412 SETKNOB(MaxSpinners) ; | |
| 2413 SETKNOB(PreSpin) ; | |
| 2414 SETKNOB(ExitPolicy) ; | |
| 2415 SETKNOB(QMode); | |
| 2416 SETKNOB(ResetEvent) ; | |
| 2417 SETKNOB(MoveNotifyee) ; | |
| 2418 SETKNOB(FastHSSEC) ; | |
| 2419 #undef SETKNOB | |
| 2420 | |
| 2421 if (os::is_MP()) { | |
| 2422 BackOffMask = (1 << Knob_SpinBackOff) - 1 ; | |
| 2423 if (Knob_ReportSettings) ::printf ("BackOffMask=%X\n", BackOffMask) ; | |
| 2424 // CONSIDER: BackOffMask = ROUNDUP_NEXT_POWER2 (ncpus-1) | |
| 2425 } else { | |
| 2426 Knob_SpinLimit = 0 ; | |
| 2427 Knob_SpinBase = 0 ; | |
| 2428 Knob_PreSpin = 0 ; | |
| 2429 Knob_FixedSpin = -1 ; | |
| 2430 } | |
| 2431 | |
| 2432 if (Knob_LogSpins == 0) { | |
| 2433 ObjectMonitor::_sync_FailedSpins = NULL ; | |
| 2434 } | |
| 2435 | |
| 2436 free (knobs) ; | |
| 2437 OrderAccess::fence() ; | |
| 2438 InitDone = 1 ; | |
| 2439 } | |
| 2440 | |
| 2441 #ifndef PRODUCT | |
| 2442 void ObjectMonitor::verify() { | |
| 2443 } | |
| 2444 | |
| 2445 void ObjectMonitor::print() { | |
| 2446 } | |
| 2447 #endif |