annotate src/share/vm/utilities/taskqueue.hpp @ 17716:cdb71841f4bc

6498581: ThreadInterruptTest3 produces wrong output on Windows Summary: There is race condition between os::interrupt and os::is_interrupted on Windows. In JVM_Sleep(Thread.sleep), check if thread gets interrupted, it may see interrupted but not really interrupted so cause spurious waking up (early return from sleep). Fix by checking if interrupt event really gets set thus prevent false return. For intrinsic of _isInterrupted, on Windows, go fastpath only on bit not set. Reviewed-by: acorn, kvn Contributed-by: david.holmes@oracle.com, yumin.qi@oracle.com
author minqi
date Wed, 26 Feb 2014 15:20:41 -0800
parents 190899198332
children 2b8e28fdf503
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
a61af66fc99e Initial load
duke
parents:
diff changeset
1 /*
12087
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
2 * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
0
a61af66fc99e Initial load
duke
parents:
diff changeset
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
a61af66fc99e Initial load
duke
parents:
diff changeset
4 *
a61af66fc99e Initial load
duke
parents:
diff changeset
5 * This code is free software; you can redistribute it and/or modify it
a61af66fc99e Initial load
duke
parents:
diff changeset
6 * under the terms of the GNU General Public License version 2 only, as
a61af66fc99e Initial load
duke
parents:
diff changeset
7 * published by the Free Software Foundation.
a61af66fc99e Initial load
duke
parents:
diff changeset
8 *
a61af66fc99e Initial load
duke
parents:
diff changeset
9 * This code is distributed in the hope that it will be useful, but WITHOUT
a61af66fc99e Initial load
duke
parents:
diff changeset
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
a61af66fc99e Initial load
duke
parents:
diff changeset
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
a61af66fc99e Initial load
duke
parents:
diff changeset
12 * version 2 for more details (a copy is included in the LICENSE file that
a61af66fc99e Initial load
duke
parents:
diff changeset
13 * accompanied this code).
a61af66fc99e Initial load
duke
parents:
diff changeset
14 *
a61af66fc99e Initial load
duke
parents:
diff changeset
15 * You should have received a copy of the GNU General Public License version
a61af66fc99e Initial load
duke
parents:
diff changeset
16 * 2 along with this work; if not, write to the Free Software Foundation,
a61af66fc99e Initial load
duke
parents:
diff changeset
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
a61af66fc99e Initial load
duke
parents:
diff changeset
18 *
1552
c18cbe5936b8 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 1311
diff changeset
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
c18cbe5936b8 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 1311
diff changeset
20 * or visit www.oracle.com if you need additional information or have any
c18cbe5936b8 6941466: Oracle rebranding changes for Hotspot repositories
trims
parents: 1311
diff changeset
21 * questions.
0
a61af66fc99e Initial load
duke
parents:
diff changeset
22 *
a61af66fc99e Initial load
duke
parents:
diff changeset
23 */
a61af66fc99e Initial load
duke
parents:
diff changeset
24
1972
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
25 #ifndef SHARE_VM_UTILITIES_TASKQUEUE_HPP
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
26 #define SHARE_VM_UTILITIES_TASKQUEUE_HPP
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
27
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
28 #include "memory/allocation.hpp"
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
29 #include "memory/allocation.inline.hpp"
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
30 #include "runtime/mutex.hpp"
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
31 #include "utilities/stack.hpp"
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
32 #ifdef TARGET_OS_ARCH_linux_x86
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
33 # include "orderAccess_linux_x86.inline.hpp"
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
34 #endif
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
35 #ifdef TARGET_OS_ARCH_linux_sparc
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
36 # include "orderAccess_linux_sparc.inline.hpp"
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
37 #endif
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
38 #ifdef TARGET_OS_ARCH_linux_zero
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
39 # include "orderAccess_linux_zero.inline.hpp"
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
40 #endif
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
41 #ifdef TARGET_OS_ARCH_solaris_x86
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
42 # include "orderAccess_solaris_x86.inline.hpp"
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
43 #endif
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
44 #ifdef TARGET_OS_ARCH_solaris_sparc
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
45 # include "orderAccess_solaris_sparc.inline.hpp"
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
46 #endif
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
47 #ifdef TARGET_OS_ARCH_windows_x86
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
48 # include "orderAccess_windows_x86.inline.hpp"
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
49 #endif
2192
b92c45f2bc75 7016023: Enable building ARM and PPC from src/closed repository
bobv
parents: 1972
diff changeset
50 #ifdef TARGET_OS_ARCH_linux_arm
b92c45f2bc75 7016023: Enable building ARM and PPC from src/closed repository
bobv
parents: 1972
diff changeset
51 # include "orderAccess_linux_arm.inline.hpp"
b92c45f2bc75 7016023: Enable building ARM and PPC from src/closed repository
bobv
parents: 1972
diff changeset
52 #endif
b92c45f2bc75 7016023: Enable building ARM and PPC from src/closed repository
bobv
parents: 1972
diff changeset
53 #ifdef TARGET_OS_ARCH_linux_ppc
b92c45f2bc75 7016023: Enable building ARM and PPC from src/closed repository
bobv
parents: 1972
diff changeset
54 # include "orderAccess_linux_ppc.inline.hpp"
b92c45f2bc75 7016023: Enable building ARM and PPC from src/closed repository
bobv
parents: 1972
diff changeset
55 #endif
3960
f08d439fab8c 7089790: integrate bsd-port changes
never
parents: 2426
diff changeset
56 #ifdef TARGET_OS_ARCH_bsd_x86
f08d439fab8c 7089790: integrate bsd-port changes
never
parents: 2426
diff changeset
57 # include "orderAccess_bsd_x86.inline.hpp"
f08d439fab8c 7089790: integrate bsd-port changes
never
parents: 2426
diff changeset
58 #endif
f08d439fab8c 7089790: integrate bsd-port changes
never
parents: 2426
diff changeset
59 #ifdef TARGET_OS_ARCH_bsd_zero
f08d439fab8c 7089790: integrate bsd-port changes
never
parents: 2426
diff changeset
60 # include "orderAccess_bsd_zero.inline.hpp"
f08d439fab8c 7089790: integrate bsd-port changes
never
parents: 2426
diff changeset
61 #endif
1972
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
62
1665
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
63 // Simple TaskQueue stats that are collected by default in debug builds.
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
64
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
65 #if !defined(TASKQUEUE_STATS) && defined(ASSERT)
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
66 #define TASKQUEUE_STATS 1
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
67 #elif !defined(TASKQUEUE_STATS)
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
68 #define TASKQUEUE_STATS 0
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
69 #endif
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
70
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
71 #if TASKQUEUE_STATS
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
72 #define TASKQUEUE_STATS_ONLY(code) code
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
73 #else
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
74 #define TASKQUEUE_STATS_ONLY(code)
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
75 #endif // TASKQUEUE_STATS
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
76
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
77 #if TASKQUEUE_STATS
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
78 class TaskQueueStats {
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
79 public:
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
80 enum StatId {
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
81 push, // number of taskqueue pushes
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
82 pop, // number of taskqueue pops
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
83 pop_slow, // subset of taskqueue pops that were done slow-path
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
84 steal_attempt, // number of taskqueue steal attempts
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
85 steal, // number of taskqueue steals
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
86 overflow, // number of overflow pushes
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
87 overflow_max_len, // max length of overflow stack
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
88 last_stat_id
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
89 };
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
90
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
91 public:
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
92 inline TaskQueueStats() { reset(); }
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
93
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
94 inline void record_push() { ++_stats[push]; }
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
95 inline void record_pop() { ++_stats[pop]; }
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
96 inline void record_pop_slow() { record_pop(); ++_stats[pop_slow]; }
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
97 inline void record_steal(bool success);
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
98 inline void record_overflow(size_t new_length);
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
99
1709
5f429ee79634 6966222: G1: simplify TaskQueue overflow handling
jcoomes
parents: 1665
diff changeset
100 TaskQueueStats & operator +=(const TaskQueueStats & addend);
5f429ee79634 6966222: G1: simplify TaskQueue overflow handling
jcoomes
parents: 1665
diff changeset
101
1665
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
102 inline size_t get(StatId id) const { return _stats[id]; }
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
103 inline const size_t* get() const { return _stats; }
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
104
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
105 inline void reset();
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
106
1709
5f429ee79634 6966222: G1: simplify TaskQueue overflow handling
jcoomes
parents: 1665
diff changeset
107 // Print the specified line of the header (does not include a line separator).
1665
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
108 static void print_header(unsigned int line, outputStream* const stream = tty,
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
109 unsigned int width = 10);
1709
5f429ee79634 6966222: G1: simplify TaskQueue overflow handling
jcoomes
parents: 1665
diff changeset
110 // Print the statistics (does not include a line separator).
1665
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
111 void print(outputStream* const stream = tty, unsigned int width = 10) const;
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
112
1709
5f429ee79634 6966222: G1: simplify TaskQueue overflow handling
jcoomes
parents: 1665
diff changeset
113 DEBUG_ONLY(void verify() const;)
5f429ee79634 6966222: G1: simplify TaskQueue overflow handling
jcoomes
parents: 1665
diff changeset
114
1665
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
115 private:
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
116 size_t _stats[last_stat_id];
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
117 static const char * const _names[last_stat_id];
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
118 };
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
119
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
120 void TaskQueueStats::record_steal(bool success) {
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
121 ++_stats[steal_attempt];
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
122 if (success) ++_stats[steal];
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
123 }
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
124
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
125 void TaskQueueStats::record_overflow(size_t new_len) {
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
126 ++_stats[overflow];
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
127 if (new_len > _stats[overflow_max_len]) _stats[overflow_max_len] = new_len;
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
128 }
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
129
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
130 void TaskQueueStats::reset() {
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
131 memset(_stats, 0, sizeof(_stats));
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
132 }
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
133 #endif // TASKQUEUE_STATS
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
134
12087
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
135 // TaskQueueSuper collects functionality common to all GenericTaskQueue instances.
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
136
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
137 template <unsigned int N, MEMFLAGS F>
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
138 class TaskQueueSuper: public CHeapObj<F> {
0
a61af66fc99e Initial load
duke
parents:
diff changeset
139 protected:
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
140 // Internal type for indexing the queue; also used for the tag.
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
141 typedef NOT_LP64(uint16_t) LP64_ONLY(uint32_t) idx_t;
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
142
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
143 // The first free element after the last one pushed (mod N).
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
144 volatile uint _bottom;
0
a61af66fc99e Initial load
duke
parents:
diff changeset
145
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
146 enum { MOD_N_MASK = N - 1 };
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
147
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
148 class Age {
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
149 public:
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
150 Age(size_t data = 0) { _data = data; }
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
151 Age(const Age& age) { _data = age._data; }
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
152 Age(idx_t top, idx_t tag) { _fields._top = top; _fields._tag = tag; }
0
a61af66fc99e Initial load
duke
parents:
diff changeset
153
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
154 Age get() const volatile { return _data; }
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
155 void set(Age age) volatile { _data = age._data; }
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
156
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
157 idx_t top() const volatile { return _fields._top; }
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
158 idx_t tag() const volatile { return _fields._tag; }
0
a61af66fc99e Initial load
duke
parents:
diff changeset
159
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
160 // Increment top; if it wraps, increment tag also.
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
161 void increment() {
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
162 _fields._top = increment_index(_fields._top);
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
163 if (_fields._top == 0) ++_fields._tag;
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
164 }
0
a61af66fc99e Initial load
duke
parents:
diff changeset
165
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
166 Age cmpxchg(const Age new_age, const Age old_age) volatile {
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
167 return (size_t) Atomic::cmpxchg_ptr((intptr_t)new_age._data,
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
168 (volatile intptr_t *)&_data,
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
169 (intptr_t)old_age._data);
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
170 }
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
171
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
172 bool operator ==(const Age& other) const { return _data == other._data; }
0
a61af66fc99e Initial load
duke
parents:
diff changeset
173
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
174 private:
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
175 struct fields {
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
176 idx_t _top;
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
177 idx_t _tag;
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
178 };
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
179 union {
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
180 size_t _data;
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
181 fields _fields;
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
182 };
0
a61af66fc99e Initial load
duke
parents:
diff changeset
183 };
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
184
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
185 volatile Age _age;
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
186
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
187 // These both operate mod N.
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
188 static uint increment_index(uint ind) {
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
189 return (ind + 1) & MOD_N_MASK;
0
a61af66fc99e Initial load
duke
parents:
diff changeset
190 }
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
191 static uint decrement_index(uint ind) {
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
192 return (ind - 1) & MOD_N_MASK;
0
a61af66fc99e Initial load
duke
parents:
diff changeset
193 }
a61af66fc99e Initial load
duke
parents:
diff changeset
194
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
195 // Returns a number in the range [0..N). If the result is "N-1", it should be
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
196 // interpreted as 0.
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
197 uint dirty_size(uint bot, uint top) const {
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
198 return (bot - top) & MOD_N_MASK;
0
a61af66fc99e Initial load
duke
parents:
diff changeset
199 }
a61af66fc99e Initial load
duke
parents:
diff changeset
200
a61af66fc99e Initial load
duke
parents:
diff changeset
201 // Returns the size corresponding to the given "bot" and "top".
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
202 uint size(uint bot, uint top) const {
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
203 uint sz = dirty_size(bot, top);
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
204 // Has the queue "wrapped", so that bottom is less than top? There's a
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
205 // complicated special case here. A pair of threads could perform pop_local
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
206 // and pop_global operations concurrently, starting from a state in which
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
207 // _bottom == _top+1. The pop_local could succeed in decrementing _bottom,
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
208 // and the pop_global in incrementing _top (in which case the pop_global
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
209 // will be awarded the contested queue element.) The resulting state must
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
210 // be interpreted as an empty queue. (We only need to worry about one such
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
211 // event: only the queue owner performs pop_local's, and several concurrent
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
212 // threads attempting to perform the pop_global will all perform the same
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
213 // CAS, and only one can succeed.) Any stealing thread that reads after
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
214 // either the increment or decrement will see an empty queue, and will not
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
215 // join the competitors. The "sz == -1 || sz == N-1" state will not be
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
216 // modified by concurrent queues, so the owner thread can reset the state to
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
217 // _bottom == top so subsequent pushes will be performed normally.
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
218 return (sz == N - 1) ? 0 : sz;
0
a61af66fc99e Initial load
duke
parents:
diff changeset
219 }
a61af66fc99e Initial load
duke
parents:
diff changeset
220
a61af66fc99e Initial load
duke
parents:
diff changeset
221 public:
a61af66fc99e Initial load
duke
parents:
diff changeset
222 TaskQueueSuper() : _bottom(0), _age() {}
a61af66fc99e Initial load
duke
parents:
diff changeset
223
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
224 // Return true if the TaskQueue contains/does not contain any tasks.
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
225 bool peek() const { return _bottom != _age.top(); }
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
226 bool is_empty() const { return size() == 0; }
0
a61af66fc99e Initial load
duke
parents:
diff changeset
227
a61af66fc99e Initial load
duke
parents:
diff changeset
228 // Return an estimate of the number of elements in the queue.
a61af66fc99e Initial load
duke
parents:
diff changeset
229 // The "careful" version admits the possibility of pop_local/pop_global
a61af66fc99e Initial load
duke
parents:
diff changeset
230 // races.
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
231 uint size() const {
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
232 return size(_bottom, _age.top());
0
a61af66fc99e Initial load
duke
parents:
diff changeset
233 }
a61af66fc99e Initial load
duke
parents:
diff changeset
234
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
235 uint dirty_size() const {
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
236 return dirty_size(_bottom, _age.top());
0
a61af66fc99e Initial load
duke
parents:
diff changeset
237 }
a61af66fc99e Initial load
duke
parents:
diff changeset
238
342
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
239 void set_empty() {
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
240 _bottom = 0;
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
241 _age.set(0);
342
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
242 }
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
243
0
a61af66fc99e Initial load
duke
parents:
diff changeset
244 // Maximum number of elements allowed in the queue. This is two less
a61af66fc99e Initial load
duke
parents:
diff changeset
245 // than the actual queue size, for somewhat complicated reasons.
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
246 uint max_elems() const { return N - 2; }
1284
5f1f51edaff6 6928081: G1: rename parameters common with CMS
jmasa
parents: 1025
diff changeset
247
5f1f51edaff6 6928081: G1: rename parameters common with CMS
jmasa
parents: 1025
diff changeset
248 // Total size of queue.
5f1f51edaff6 6928081: G1: rename parameters common with CMS
jmasa
parents: 1025
diff changeset
249 static const uint total_size() { return N; }
1665
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
250
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
251 TASKQUEUE_STATS_ONLY(TaskQueueStats stats;)
0
a61af66fc99e Initial load
duke
parents:
diff changeset
252 };
a61af66fc99e Initial load
duke
parents:
diff changeset
253
12087
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
254 //
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
255 // GenericTaskQueue implements an ABP, Aurora-Blumofe-Plaxton, double-
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
256 // ended-queue (deque), intended for use in work stealing. Queue operations
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
257 // are non-blocking.
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
258 //
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
259 // A queue owner thread performs push() and pop_local() operations on one end
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
260 // of the queue, while other threads may steal work using the pop_global()
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
261 // method.
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
262 //
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
263 // The main difference to the original algorithm is that this
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
264 // implementation allows wrap-around at the end of its allocated
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
265 // storage, which is an array.
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
266 //
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
267 // The original paper is:
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
268 //
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
269 // Arora, N. S., Blumofe, R. D., and Plaxton, C. G.
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
270 // Thread scheduling for multiprogrammed multiprocessors.
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
271 // Theory of Computing Systems 34, 2 (2001), 115-144.
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
272 //
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
273 // The following paper provides an correctness proof and an
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
274 // implementation for weakly ordered memory models including (pseudo-)
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
275 // code containing memory barriers for a Chase-Lev deque. Chase-Lev is
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
276 // similar to ABP, with the main difference that it allows resizing of the
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
277 // underlying storage:
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
278 //
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
279 // Le, N. M., Pop, A., Cohen A., and Nardell, F. Z.
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
280 // Correct and efficient work-stealing for weak memory models
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
281 // Proceedings of the 18th ACM SIGPLAN symposium on Principles and
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
282 // practice of parallel programming (PPoPP 2013), 69-80
61521bd65100 8022784: TaskQueue misses minimal documentation and references for analysis
tschatzl
parents: 11997
diff changeset
283 //
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
284
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
285 template <class E, MEMFLAGS F, unsigned int N = TASKQUEUE_SIZE>
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
286 class GenericTaskQueue: public TaskQueueSuper<N, F> {
9073
83f27710f5f7 7197666: java -d64 -version core dumps in a box with lots of memory
brutisso
parents: 6924
diff changeset
287 ArrayAllocator<E, F> _array_allocator;
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
288 protected:
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
289 typedef typename TaskQueueSuper<N, F>::Age Age;
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
290 typedef typename TaskQueueSuper<N, F>::idx_t idx_t;
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
291
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
292 using TaskQueueSuper<N, F>::_bottom;
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
293 using TaskQueueSuper<N, F>::_age;
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
294 using TaskQueueSuper<N, F>::increment_index;
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
295 using TaskQueueSuper<N, F>::decrement_index;
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
296 using TaskQueueSuper<N, F>::dirty_size;
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
297
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
298 public:
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
299 using TaskQueueSuper<N, F>::max_elems;
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
300 using TaskQueueSuper<N, F>::size;
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
301
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
302 #if TASKQUEUE_STATS
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
303 using TaskQueueSuper<N, F>::stats;
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
304 #endif
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
305
0
a61af66fc99e Initial load
duke
parents:
diff changeset
306 private:
a61af66fc99e Initial load
duke
parents:
diff changeset
307 // Slow paths for push, pop_local. (pop_global has no fast path.)
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
308 bool push_slow(E t, uint dirty_n_elems);
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
309 bool pop_local_slow(uint localBot, Age oldAge);
0
a61af66fc99e Initial load
duke
parents:
diff changeset
310
a61af66fc99e Initial load
duke
parents:
diff changeset
311 public:
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
312 typedef E element_type;
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
313
0
a61af66fc99e Initial load
duke
parents:
diff changeset
314 // Initializes the queue to empty.
a61af66fc99e Initial load
duke
parents:
diff changeset
315 GenericTaskQueue();
a61af66fc99e Initial load
duke
parents:
diff changeset
316
a61af66fc99e Initial load
duke
parents:
diff changeset
317 void initialize();
a61af66fc99e Initial load
duke
parents:
diff changeset
318
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
319 // Push the task "t" on the queue. Returns "false" iff the queue is full.
0
a61af66fc99e Initial load
duke
parents:
diff changeset
320 inline bool push(E t);
a61af66fc99e Initial load
duke
parents:
diff changeset
321
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
322 // Attempts to claim a task from the "local" end of the queue (the most
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
323 // recently pushed). If successful, returns true and sets t to the task;
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
324 // otherwise, returns false (the queue is empty).
12316
190899198332 7195622: CheckUnhandledOops has limited usefulness now
hseigel
parents: 12087
diff changeset
325 inline bool pop_local(volatile E& t);
0
a61af66fc99e Initial load
duke
parents:
diff changeset
326
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
327 // Like pop_local(), but uses the "global" end of the queue (the least
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
328 // recently pushed).
12316
190899198332 7195622: CheckUnhandledOops has limited usefulness now
hseigel
parents: 12087
diff changeset
329 bool pop_global(volatile E& t);
0
a61af66fc99e Initial load
duke
parents:
diff changeset
330
a61af66fc99e Initial load
duke
parents:
diff changeset
331 // Delete any resource associated with the queue.
a61af66fc99e Initial load
duke
parents:
diff changeset
332 ~GenericTaskQueue();
a61af66fc99e Initial load
duke
parents:
diff changeset
333
342
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
334 // apply the closure to all elements in the task queue
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
335 void oops_do(OopClosure* f);
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
336
0
a61af66fc99e Initial load
duke
parents:
diff changeset
337 private:
a61af66fc99e Initial load
duke
parents:
diff changeset
338 // Element array.
a61af66fc99e Initial load
duke
parents:
diff changeset
339 volatile E* _elems;
a61af66fc99e Initial load
duke
parents:
diff changeset
340 };
a61af66fc99e Initial load
duke
parents:
diff changeset
341
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
342 template<class E, MEMFLAGS F, unsigned int N>
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
343 GenericTaskQueue<E, F, N>::GenericTaskQueue() {
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
344 assert(sizeof(Age) == sizeof(size_t), "Depends on this.");
0
a61af66fc99e Initial load
duke
parents:
diff changeset
345 }
a61af66fc99e Initial load
duke
parents:
diff changeset
346
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
347 template<class E, MEMFLAGS F, unsigned int N>
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
348 void GenericTaskQueue<E, F, N>::initialize() {
9073
83f27710f5f7 7197666: java -d64 -version core dumps in a box with lots of memory
brutisso
parents: 6924
diff changeset
349 _elems = _array_allocator.allocate(N);
0
a61af66fc99e Initial load
duke
parents:
diff changeset
350 }
a61af66fc99e Initial load
duke
parents:
diff changeset
351
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
352 template<class E, MEMFLAGS F, unsigned int N>
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
353 void GenericTaskQueue<E, F, N>::oops_do(OopClosure* f) {
342
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
354 // tty->print_cr("START OopTaskQueue::oops_do");
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
355 uint iters = size();
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
356 uint index = _bottom;
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
357 for (uint i = 0; i < iters; ++i) {
342
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
358 index = decrement_index(index);
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
359 // tty->print_cr(" doing entry %d," INTPTR_T " -> " INTPTR_T,
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
360 // index, &_elems[index], _elems[index]);
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
361 E* t = (E*)&_elems[index]; // cast away volatility
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
362 oop* p = (oop*)t;
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
363 assert((*t)->is_oop_or_null(), "Not an oop or null");
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
364 f->do_oop(p);
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
365 }
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
366 // tty->print_cr("END OopTaskQueue::oops_do");
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
367 }
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
368
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
369 template<class E, MEMFLAGS F, unsigned int N>
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
370 bool GenericTaskQueue<E, F, N>::push_slow(E t, uint dirty_n_elems) {
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
371 if (dirty_n_elems == N - 1) {
0
a61af66fc99e Initial load
duke
parents:
diff changeset
372 // Actually means 0, so do the push.
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
373 uint localBot = _bottom;
10973
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
374 // g++ complains if the volatile result of the assignment is
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
375 // unused, so we cast the volatile away. We cannot cast directly
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
376 // to void, because gcc treats that as not using the result of the
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
377 // assignment. However, casting to E& means that we trigger an
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
378 // unused-value warning. So, we cast the E& to void.
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
379 (void)const_cast<E&>(_elems[localBot] = t);
1024
2c03ce058f55 6888847: TaskQueue needs release_store() for correctness on RMO machines
bobv
parents: 907
diff changeset
380 OrderAccess::release_store(&_bottom, increment_index(localBot));
1665
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
381 TASKQUEUE_STATS_ONLY(stats.record_push());
0
a61af66fc99e Initial load
duke
parents:
diff changeset
382 return true;
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
383 }
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
384 return false;
0
a61af66fc99e Initial load
duke
parents:
diff changeset
385 }
a61af66fc99e Initial load
duke
parents:
diff changeset
386
1833
8b10f48633dc 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 1709
diff changeset
387 // pop_local_slow() is done by the owning thread and is trying to
8b10f48633dc 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 1709
diff changeset
388 // get the last task in the queue. It will compete with pop_global()
8b10f48633dc 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 1709
diff changeset
389 // that will be used by other threads. The tag age is incremented
8b10f48633dc 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 1709
diff changeset
390 // whenever the queue goes empty which it will do here if this thread
8b10f48633dc 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 1709
diff changeset
391 // gets the last task or in pop_global() if the queue wraps (top == 0
8b10f48633dc 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 1709
diff changeset
392 // and pop_global() succeeds, see pop_global()).
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
393 template<class E, MEMFLAGS F, unsigned int N>
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
394 bool GenericTaskQueue<E, F, N>::pop_local_slow(uint localBot, Age oldAge) {
0
a61af66fc99e Initial load
duke
parents:
diff changeset
395 // This queue was observed to contain exactly one element; either this
a61af66fc99e Initial load
duke
parents:
diff changeset
396 // thread will claim it, or a competing "pop_global". In either case,
a61af66fc99e Initial load
duke
parents:
diff changeset
397 // the queue will be logically empty afterwards. Create a new Age value
a61af66fc99e Initial load
duke
parents:
diff changeset
398 // that represents the empty queue for the given value of "_bottom". (We
a61af66fc99e Initial load
duke
parents:
diff changeset
399 // must also increment "tag" because of the case where "bottom == 1",
a61af66fc99e Initial load
duke
parents:
diff changeset
400 // "top == 0". A pop_global could read the queue element in that case,
a61af66fc99e Initial load
duke
parents:
diff changeset
401 // then have the owner thread do a pop followed by another push. Without
a61af66fc99e Initial load
duke
parents:
diff changeset
402 // the incrementing of "tag", the pop_global's CAS could succeed,
a61af66fc99e Initial load
duke
parents:
diff changeset
403 // allowing it to believe it has claimed the stale element.)
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
404 Age newAge((idx_t)localBot, oldAge.tag() + 1);
0
a61af66fc99e Initial load
duke
parents:
diff changeset
405 // Perhaps a competing pop_global has already incremented "top", in which
a61af66fc99e Initial load
duke
parents:
diff changeset
406 // case it wins the element.
a61af66fc99e Initial load
duke
parents:
diff changeset
407 if (localBot == oldAge.top()) {
a61af66fc99e Initial load
duke
parents:
diff changeset
408 // No competing pop_global has yet incremented "top"; we'll try to
a61af66fc99e Initial load
duke
parents:
diff changeset
409 // install new_age, thus claiming the element.
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
410 Age tempAge = _age.cmpxchg(newAge, oldAge);
0
a61af66fc99e Initial load
duke
parents:
diff changeset
411 if (tempAge == oldAge) {
a61af66fc99e Initial load
duke
parents:
diff changeset
412 // We win.
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
413 assert(dirty_size(localBot, _age.top()) != N - 1, "sanity");
1665
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
414 TASKQUEUE_STATS_ONLY(stats.record_pop_slow());
0
a61af66fc99e Initial load
duke
parents:
diff changeset
415 return true;
a61af66fc99e Initial load
duke
parents:
diff changeset
416 }
a61af66fc99e Initial load
duke
parents:
diff changeset
417 }
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
418 // We lose; a completing pop_global gets the element. But the queue is empty
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
419 // and top is greater than bottom. Fix this representation of the empty queue
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
420 // to become the canonical one.
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
421 _age.set(newAge);
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
422 assert(dirty_size(localBot, _age.top()) != N - 1, "sanity");
0
a61af66fc99e Initial load
duke
parents:
diff changeset
423 return false;
a61af66fc99e Initial load
duke
parents:
diff changeset
424 }
a61af66fc99e Initial load
duke
parents:
diff changeset
425
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
426 template<class E, MEMFLAGS F, unsigned int N>
12316
190899198332 7195622: CheckUnhandledOops has limited usefulness now
hseigel
parents: 12087
diff changeset
427 bool GenericTaskQueue<E, F, N>::pop_global(volatile E& t) {
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
428 Age oldAge = _age.get();
11997
cd25d3be91c5 8012144: multiple SIGSEGVs fails on staxf
vladidan
parents: 10973
diff changeset
429 // Architectures with weak memory model require a barrier here
cd25d3be91c5 8012144: multiple SIGSEGVs fails on staxf
vladidan
parents: 10973
diff changeset
430 // to guarantee that bottom is not older than age,
cd25d3be91c5 8012144: multiple SIGSEGVs fails on staxf
vladidan
parents: 10973
diff changeset
431 // which is crucial for the correctness of the algorithm.
cd25d3be91c5 8012144: multiple SIGSEGVs fails on staxf
vladidan
parents: 10973
diff changeset
432 #if !(defined SPARC || defined IA32 || defined AMD64)
cd25d3be91c5 8012144: multiple SIGSEGVs fails on staxf
vladidan
parents: 10973
diff changeset
433 OrderAccess::fence();
cd25d3be91c5 8012144: multiple SIGSEGVs fails on staxf
vladidan
parents: 10973
diff changeset
434 #endif
cd25d3be91c5 8012144: multiple SIGSEGVs fails on staxf
vladidan
parents: 10973
diff changeset
435 uint localBot = OrderAccess::load_acquire((volatile juint*)&_bottom);
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
436 uint n_elems = size(localBot, oldAge.top());
0
a61af66fc99e Initial load
duke
parents:
diff changeset
437 if (n_elems == 0) {
a61af66fc99e Initial load
duke
parents:
diff changeset
438 return false;
a61af66fc99e Initial load
duke
parents:
diff changeset
439 }
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
440
10973
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
441 // g++ complains if the volatile result of the assignment is
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
442 // unused, so we cast the volatile away. We cannot cast directly
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
443 // to void, because gcc treats that as not using the result of the
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
444 // assignment. However, casting to E& means that we trigger an
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
445 // unused-value warning. So, we cast the E& to void.
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
446 (void) const_cast<E&>(t = _elems[oldAge.top()]);
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
447 Age newAge(oldAge);
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
448 newAge.increment();
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
449 Age resAge = _age.cmpxchg(newAge, oldAge);
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
450
0
a61af66fc99e Initial load
duke
parents:
diff changeset
451 // Note that using "_bottom" here might fail, since a pop_local might
a61af66fc99e Initial load
duke
parents:
diff changeset
452 // have decremented it.
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
453 assert(dirty_size(localBot, newAge.top()) != N - 1, "sanity");
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
454 return resAge == oldAge;
0
a61af66fc99e Initial load
duke
parents:
diff changeset
455 }
a61af66fc99e Initial load
duke
parents:
diff changeset
456
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
457 template<class E, MEMFLAGS F, unsigned int N>
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
458 GenericTaskQueue<E, F, N>::~GenericTaskQueue() {
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
459 FREE_C_HEAP_ARRAY(E, _elems, F);
0
a61af66fc99e Initial load
duke
parents:
diff changeset
460 }
a61af66fc99e Initial load
duke
parents:
diff changeset
461
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
462 // OverflowTaskQueue is a TaskQueue that also includes an overflow stack for
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
463 // elements that do not fit in the TaskQueue.
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
464 //
1836
894b1d7c7e01 6423256: GC stacks should use a better data structure
jcoomes
parents: 1833
diff changeset
465 // This class hides two methods from super classes:
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
466 //
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
467 // push() - push onto the task queue or, if that fails, onto the overflow stack
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
468 // is_empty() - return true if both the TaskQueue and overflow stack are empty
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
469 //
1836
894b1d7c7e01 6423256: GC stacks should use a better data structure
jcoomes
parents: 1833
diff changeset
470 // Note that size() is not hidden--it returns the number of elements in the
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
471 // TaskQueue, and does not include the size of the overflow stack. This
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
472 // simplifies replacement of GenericTaskQueues with OverflowTaskQueues.
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
473 template<class E, MEMFLAGS F, unsigned int N = TASKQUEUE_SIZE>
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
474 class OverflowTaskQueue: public GenericTaskQueue<E, F, N>
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
475 {
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
476 public:
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
477 typedef Stack<E, F> overflow_t;
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
478 typedef GenericTaskQueue<E, F, N> taskqueue_t;
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
479
1665
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
480 TASKQUEUE_STATS_ONLY(using taskqueue_t::stats;)
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
481
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
482 // Push task t onto the queue or onto the overflow stack. Return true.
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
483 inline bool push(E t);
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
484
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
485 // Attempt to pop from the overflow stack; return true if anything was popped.
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
486 inline bool pop_overflow(E& t);
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
487
1836
894b1d7c7e01 6423256: GC stacks should use a better data structure
jcoomes
parents: 1833
diff changeset
488 inline overflow_t* overflow_stack() { return &_overflow_stack; }
894b1d7c7e01 6423256: GC stacks should use a better data structure
jcoomes
parents: 1833
diff changeset
489
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
490 inline bool taskqueue_empty() const { return taskqueue_t::is_empty(); }
1836
894b1d7c7e01 6423256: GC stacks should use a better data structure
jcoomes
parents: 1833
diff changeset
491 inline bool overflow_empty() const { return _overflow_stack.is_empty(); }
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
492 inline bool is_empty() const {
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
493 return taskqueue_empty() && overflow_empty();
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
494 }
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
495
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
496 private:
1836
894b1d7c7e01 6423256: GC stacks should use a better data structure
jcoomes
parents: 1833
diff changeset
497 overflow_t _overflow_stack;
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
498 };
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
499
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
500 template <class E, MEMFLAGS F, unsigned int N>
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
501 bool OverflowTaskQueue<E, F, N>::push(E t)
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
502 {
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
503 if (!taskqueue_t::push(t)) {
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
504 overflow_stack()->push(t);
1836
894b1d7c7e01 6423256: GC stacks should use a better data structure
jcoomes
parents: 1833
diff changeset
505 TASKQUEUE_STATS_ONLY(stats.record_overflow(overflow_stack()->size()));
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
506 }
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
507 return true;
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
508 }
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
509
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
510 template <class E, MEMFLAGS F, unsigned int N>
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
511 bool OverflowTaskQueue<E, F, N>::pop_overflow(E& t)
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
512 {
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
513 if (overflow_empty()) return false;
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
514 t = overflow_stack()->pop();
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
515 return true;
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
516 }
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
517
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
518 class TaskQueueSetSuper {
0
a61af66fc99e Initial load
duke
parents:
diff changeset
519 protected:
a61af66fc99e Initial load
duke
parents:
diff changeset
520 static int randomParkAndMiller(int* seed0);
a61af66fc99e Initial load
duke
parents:
diff changeset
521 public:
a61af66fc99e Initial load
duke
parents:
diff changeset
522 // Returns "true" if some TaskQueue in the set contains a task.
a61af66fc99e Initial load
duke
parents:
diff changeset
523 virtual bool peek() = 0;
a61af66fc99e Initial load
duke
parents:
diff changeset
524 };
a61af66fc99e Initial load
duke
parents:
diff changeset
525
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
526 template <MEMFLAGS F> class TaskQueueSetSuperImpl: public CHeapObj<F>, public TaskQueueSetSuper {
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
527 };
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
528
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
529 template<class T, MEMFLAGS F>
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
530 class GenericTaskQueueSet: public TaskQueueSetSuperImpl<F> {
0
a61af66fc99e Initial load
duke
parents:
diff changeset
531 private:
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
532 uint _n;
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
533 T** _queues;
0
a61af66fc99e Initial load
duke
parents:
diff changeset
534
a61af66fc99e Initial load
duke
parents:
diff changeset
535 public:
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
536 typedef typename T::element_type E;
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
537
0
a61af66fc99e Initial load
duke
parents:
diff changeset
538 GenericTaskQueueSet(int n) : _n(n) {
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
539 typedef T* GenericTaskQueuePtr;
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
540 _queues = NEW_C_HEAP_ARRAY(GenericTaskQueuePtr, n, F);
0
a61af66fc99e Initial load
duke
parents:
diff changeset
541 for (int i = 0; i < n; i++) {
a61af66fc99e Initial load
duke
parents:
diff changeset
542 _queues[i] = NULL;
a61af66fc99e Initial load
duke
parents:
diff changeset
543 }
a61af66fc99e Initial load
duke
parents:
diff changeset
544 }
a61af66fc99e Initial load
duke
parents:
diff changeset
545
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
546 bool steal_best_of_2(uint queue_num, int* seed, E& t);
0
a61af66fc99e Initial load
duke
parents:
diff changeset
547
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
548 void register_queue(uint i, T* q);
0
a61af66fc99e Initial load
duke
parents:
diff changeset
549
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
550 T* queue(uint n);
0
a61af66fc99e Initial load
duke
parents:
diff changeset
551
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
552 // The thread with queue number "queue_num" (and whose random number seed is
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
553 // at "seed") is trying to steal a task from some other queue. (It may try
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
554 // several queues, according to some configuration parameter.) If some steal
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
555 // succeeds, returns "true" and sets "t" to the stolen task, otherwise returns
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
556 // false.
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
557 bool steal(uint queue_num, int* seed, E& t);
0
a61af66fc99e Initial load
duke
parents:
diff changeset
558
a61af66fc99e Initial load
duke
parents:
diff changeset
559 bool peek();
a61af66fc99e Initial load
duke
parents:
diff changeset
560 };
a61af66fc99e Initial load
duke
parents:
diff changeset
561
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
562 template<class T, MEMFLAGS F> void
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
563 GenericTaskQueueSet<T, F>::register_queue(uint i, T* q) {
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
564 assert(i < _n, "index out of range.");
0
a61af66fc99e Initial load
duke
parents:
diff changeset
565 _queues[i] = q;
a61af66fc99e Initial load
duke
parents:
diff changeset
566 }
a61af66fc99e Initial load
duke
parents:
diff changeset
567
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
568 template<class T, MEMFLAGS F> T*
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
569 GenericTaskQueueSet<T, F>::queue(uint i) {
0
a61af66fc99e Initial load
duke
parents:
diff changeset
570 return _queues[i];
a61af66fc99e Initial load
duke
parents:
diff changeset
571 }
a61af66fc99e Initial load
duke
parents:
diff changeset
572
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
573 template<class T, MEMFLAGS F> bool
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
574 GenericTaskQueueSet<T, F>::steal(uint queue_num, int* seed, E& t) {
1665
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
575 for (uint i = 0; i < 2 * _n; i++) {
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
576 if (steal_best_of_2(queue_num, seed, t)) {
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
577 TASKQUEUE_STATS_ONLY(queue(queue_num)->stats.record_steal(true));
0
a61af66fc99e Initial load
duke
parents:
diff changeset
578 return true;
1665
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
579 }
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
580 }
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
581 TASKQUEUE_STATS_ONLY(queue(queue_num)->stats.record_steal(false));
0
a61af66fc99e Initial load
duke
parents:
diff changeset
582 return false;
a61af66fc99e Initial load
duke
parents:
diff changeset
583 }
a61af66fc99e Initial load
duke
parents:
diff changeset
584
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
585 template<class T, MEMFLAGS F> bool
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
586 GenericTaskQueueSet<T, F>::steal_best_of_2(uint queue_num, int* seed, E& t) {
0
a61af66fc99e Initial load
duke
parents:
diff changeset
587 if (_n > 2) {
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
588 uint k1 = queue_num;
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
589 while (k1 == queue_num) k1 = TaskQueueSetSuper::randomParkAndMiller(seed) % _n;
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
590 uint k2 = queue_num;
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
591 while (k2 == queue_num || k2 == k1) k2 = TaskQueueSetSuper::randomParkAndMiller(seed) % _n;
0
a61af66fc99e Initial load
duke
parents:
diff changeset
592 // Sample both and try the larger.
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
593 uint sz1 = _queues[k1]->size();
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
594 uint sz2 = _queues[k2]->size();
0
a61af66fc99e Initial load
duke
parents:
diff changeset
595 if (sz2 > sz1) return _queues[k2]->pop_global(t);
a61af66fc99e Initial load
duke
parents:
diff changeset
596 else return _queues[k1]->pop_global(t);
a61af66fc99e Initial load
duke
parents:
diff changeset
597 } else if (_n == 2) {
a61af66fc99e Initial load
duke
parents:
diff changeset
598 // Just try the other one.
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
599 uint k = (queue_num + 1) % 2;
0
a61af66fc99e Initial load
duke
parents:
diff changeset
600 return _queues[k]->pop_global(t);
a61af66fc99e Initial load
duke
parents:
diff changeset
601 } else {
a61af66fc99e Initial load
duke
parents:
diff changeset
602 assert(_n == 1, "can't be zero.");
a61af66fc99e Initial load
duke
parents:
diff changeset
603 return false;
a61af66fc99e Initial load
duke
parents:
diff changeset
604 }
a61af66fc99e Initial load
duke
parents:
diff changeset
605 }
a61af66fc99e Initial load
duke
parents:
diff changeset
606
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
607 template<class T, MEMFLAGS F>
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
608 bool GenericTaskQueueSet<T, F>::peek() {
0
a61af66fc99e Initial load
duke
parents:
diff changeset
609 // Try all the queues.
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
610 for (uint j = 0; j < _n; j++) {
0
a61af66fc99e Initial load
duke
parents:
diff changeset
611 if (_queues[j]->peek())
a61af66fc99e Initial load
duke
parents:
diff changeset
612 return true;
a61af66fc99e Initial load
duke
parents:
diff changeset
613 }
a61af66fc99e Initial load
duke
parents:
diff changeset
614 return false;
a61af66fc99e Initial load
duke
parents:
diff changeset
615 }
a61af66fc99e Initial load
duke
parents:
diff changeset
616
342
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
617 // When to terminate from the termination protocol.
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
618 class TerminatorTerminator: public CHeapObj<mtInternal> {
342
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
619 public:
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
620 virtual bool should_exit_termination() = 0;
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
621 };
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
622
0
a61af66fc99e Initial load
duke
parents:
diff changeset
623 // A class to aid in the termination of a set of parallel tasks using
a61af66fc99e Initial load
duke
parents:
diff changeset
624 // TaskQueueSet's for work stealing.
a61af66fc99e Initial load
duke
parents:
diff changeset
625
546
05c6d52fa7a9 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 541
diff changeset
626 #undef TRACESPINNING
05c6d52fa7a9 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 541
diff changeset
627
0
a61af66fc99e Initial load
duke
parents:
diff changeset
628 class ParallelTaskTerminator: public StackObj {
a61af66fc99e Initial load
duke
parents:
diff changeset
629 private:
a61af66fc99e Initial load
duke
parents:
diff changeset
630 int _n_threads;
a61af66fc99e Initial load
duke
parents:
diff changeset
631 TaskQueueSetSuper* _queue_set;
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
632 int _offered_termination;
0
a61af66fc99e Initial load
duke
parents:
diff changeset
633
546
05c6d52fa7a9 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 541
diff changeset
634 #ifdef TRACESPINNING
05c6d52fa7a9 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 541
diff changeset
635 static uint _total_yields;
05c6d52fa7a9 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 541
diff changeset
636 static uint _total_spins;
05c6d52fa7a9 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 541
diff changeset
637 static uint _total_peeks;
05c6d52fa7a9 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 541
diff changeset
638 #endif
05c6d52fa7a9 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 541
diff changeset
639
0
a61af66fc99e Initial load
duke
parents:
diff changeset
640 bool peek_in_queue_set();
a61af66fc99e Initial load
duke
parents:
diff changeset
641 protected:
a61af66fc99e Initial load
duke
parents:
diff changeset
642 virtual void yield();
a61af66fc99e Initial load
duke
parents:
diff changeset
643 void sleep(uint millis);
a61af66fc99e Initial load
duke
parents:
diff changeset
644
a61af66fc99e Initial load
duke
parents:
diff changeset
645 public:
a61af66fc99e Initial load
duke
parents:
diff changeset
646
a61af66fc99e Initial load
duke
parents:
diff changeset
647 // "n_threads" is the number of threads to be terminated. "queue_set" is a
a61af66fc99e Initial load
duke
parents:
diff changeset
648 // queue sets of work queues of other threads.
a61af66fc99e Initial load
duke
parents:
diff changeset
649 ParallelTaskTerminator(int n_threads, TaskQueueSetSuper* queue_set);
a61af66fc99e Initial load
duke
parents:
diff changeset
650
a61af66fc99e Initial load
duke
parents:
diff changeset
651 // The current thread has no work, and is ready to terminate if everyone
a61af66fc99e Initial load
duke
parents:
diff changeset
652 // else is. If returns "true", all threads are terminated. If returns
a61af66fc99e Initial load
duke
parents:
diff changeset
653 // "false", available work has been observed in one of the task queues,
a61af66fc99e Initial load
duke
parents:
diff changeset
654 // so the global task is not complete.
342
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
655 bool offer_termination() {
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
656 return offer_termination(NULL);
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
657 }
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
658
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
659 // As above, but it also terminates if the should_exit_termination()
342
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
660 // method of the terminator parameter returns true. If terminator is
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
661 // NULL, then it is ignored.
37f87013dfd8 6711316: Open source the Garbage-First garbage collector
ysr
parents: 113
diff changeset
662 bool offer_termination(TerminatorTerminator* terminator);
0
a61af66fc99e Initial load
duke
parents:
diff changeset
663
a61af66fc99e Initial load
duke
parents:
diff changeset
664 // Reset the terminator, so that it may be reused again.
a61af66fc99e Initial load
duke
parents:
diff changeset
665 // The caller is responsible for ensuring that this is done
a61af66fc99e Initial load
duke
parents:
diff changeset
666 // in an MT-safe manner, once the previous round of use of
a61af66fc99e Initial load
duke
parents:
diff changeset
667 // the terminator is finished.
a61af66fc99e Initial load
duke
parents:
diff changeset
668 void reset_for_reuse();
1833
8b10f48633dc 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 1709
diff changeset
669 // Same as above but the number of parallel threads is set to the
8b10f48633dc 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 1709
diff changeset
670 // given number.
8b10f48633dc 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 1709
diff changeset
671 void reset_for_reuse(int n_threads);
0
a61af66fc99e Initial load
duke
parents:
diff changeset
672
546
05c6d52fa7a9 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 541
diff changeset
673 #ifdef TRACESPINNING
05c6d52fa7a9 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 541
diff changeset
674 static uint total_yields() { return _total_yields; }
05c6d52fa7a9 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 541
diff changeset
675 static uint total_spins() { return _total_spins; }
05c6d52fa7a9 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 541
diff changeset
676 static uint total_peeks() { return _total_peeks; }
05c6d52fa7a9 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 541
diff changeset
677 static void print_termination_counts();
05c6d52fa7a9 6690928: Use spinning in combination with yields for workstealing termination.
jmasa
parents: 541
diff changeset
678 #endif
0
a61af66fc99e Initial load
duke
parents:
diff changeset
679 };
a61af66fc99e Initial load
duke
parents:
diff changeset
680
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
681 template<class E, MEMFLAGS F, unsigned int N> inline bool
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
682 GenericTaskQueue<E, F, N>::push(E t) {
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
683 uint localBot = _bottom;
11997
cd25d3be91c5 8012144: multiple SIGSEGVs fails on staxf
vladidan
parents: 10973
diff changeset
684 assert(localBot < N, "_bottom out of range.");
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
685 idx_t top = _age.top();
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
686 uint dirty_n_elems = dirty_size(localBot, top);
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
687 assert(dirty_n_elems < N, "n_elems out of range.");
0
a61af66fc99e Initial load
duke
parents:
diff changeset
688 if (dirty_n_elems < max_elems()) {
10973
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
689 // g++ complains if the volatile result of the assignment is
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
690 // unused, so we cast the volatile away. We cannot cast directly
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
691 // to void, because gcc treats that as not using the result of the
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
692 // assignment. However, casting to E& means that we trigger an
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
693 // unused-value warning. So, we cast the E& to void.
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
694 (void) const_cast<E&>(_elems[localBot] = t);
1024
2c03ce058f55 6888847: TaskQueue needs release_store() for correctness on RMO machines
bobv
parents: 907
diff changeset
695 OrderAccess::release_store(&_bottom, increment_index(localBot));
1665
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
696 TASKQUEUE_STATS_ONLY(stats.record_push());
0
a61af66fc99e Initial load
duke
parents:
diff changeset
697 return true;
a61af66fc99e Initial load
duke
parents:
diff changeset
698 } else {
a61af66fc99e Initial load
duke
parents:
diff changeset
699 return push_slow(t, dirty_n_elems);
a61af66fc99e Initial load
duke
parents:
diff changeset
700 }
a61af66fc99e Initial load
duke
parents:
diff changeset
701 }
a61af66fc99e Initial load
duke
parents:
diff changeset
702
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
703 template<class E, MEMFLAGS F, unsigned int N> inline bool
12316
190899198332 7195622: CheckUnhandledOops has limited usefulness now
hseigel
parents: 12087
diff changeset
704 GenericTaskQueue<E, F, N>::pop_local(volatile E& t) {
541
23673011938d 6787254: Work queue capacity can be increased substantially on some platforms
ysr
parents: 375
diff changeset
705 uint localBot = _bottom;
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
706 // This value cannot be N-1. That can only occur as a result of
0
a61af66fc99e Initial load
duke
parents:
diff changeset
707 // the assignment to bottom in this method. If it does, this method
1638
b2a00dd3117c 6957084: simplify TaskQueue overflow handling
jcoomes
parents: 1552
diff changeset
708 // resets the size to 0 before the next call (which is sequential,
0
a61af66fc99e Initial load
duke
parents:
diff changeset
709 // since this is pop_local.)
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
710 uint dirty_n_elems = dirty_size(localBot, _age.top());
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
711 assert(dirty_n_elems != N - 1, "Shouldn't be possible...");
0
a61af66fc99e Initial load
duke
parents:
diff changeset
712 if (dirty_n_elems == 0) return false;
a61af66fc99e Initial load
duke
parents:
diff changeset
713 localBot = decrement_index(localBot);
a61af66fc99e Initial load
duke
parents:
diff changeset
714 _bottom = localBot;
a61af66fc99e Initial load
duke
parents:
diff changeset
715 // This is necessary to prevent any read below from being reordered
a61af66fc99e Initial load
duke
parents:
diff changeset
716 // before the store just above.
a61af66fc99e Initial load
duke
parents:
diff changeset
717 OrderAccess::fence();
10973
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
718 // g++ complains if the volatile result of the assignment is
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
719 // unused, so we cast the volatile away. We cannot cast directly
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
720 // to void, because gcc treats that as not using the result of the
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
721 // assignment. However, casting to E& means that we trigger an
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
722 // unused-value warning. So, we cast the E& to void.
ef57c43512d6 8014431: cleanup warnings indicated by the -Wunused-value compiler option on linux
ccheung
parents: 9073
diff changeset
723 (void) const_cast<E&>(t = _elems[localBot]);
0
a61af66fc99e Initial load
duke
parents:
diff changeset
724 // This is a second read of "age"; the "size()" above is the first.
a61af66fc99e Initial load
duke
parents:
diff changeset
725 // If there's still at least one element in the queue, based on the
a61af66fc99e Initial load
duke
parents:
diff changeset
726 // "_bottom" and "age" we've read, then there can be no interference with
a61af66fc99e Initial load
duke
parents:
diff changeset
727 // a "pop_global" operation, and we're done.
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
728 idx_t tp = _age.top(); // XXX
0
a61af66fc99e Initial load
duke
parents:
diff changeset
729 if (size(localBot, tp) > 0) {
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
730 assert(dirty_size(localBot, tp) != N - 1, "sanity");
1665
a93a9eda13f7 6962947: shared TaskQueue statistics
jcoomes
parents: 1638
diff changeset
731 TASKQUEUE_STATS_ONLY(stats.record_pop());
0
a61af66fc99e Initial load
duke
parents:
diff changeset
732 return true;
a61af66fc99e Initial load
duke
parents:
diff changeset
733 } else {
a61af66fc99e Initial load
duke
parents:
diff changeset
734 // Otherwise, the queue contained exactly one element; we take the slow
a61af66fc99e Initial load
duke
parents:
diff changeset
735 // path.
907
3ee342e25e57 6821693: 64-bit TaskQueue capacity still too small
jcoomes
parents: 845
diff changeset
736 return pop_local_slow(localBot, _age.get());
0
a61af66fc99e Initial load
duke
parents:
diff changeset
737 }
a61af66fc99e Initial load
duke
parents:
diff changeset
738 }
a61af66fc99e Initial load
duke
parents:
diff changeset
739
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
740 typedef GenericTaskQueue<oop, mtGC> OopTaskQueue;
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
741 typedef GenericTaskQueueSet<OopTaskQueue, mtGC> OopTaskQueueSet;
0
a61af66fc99e Initial load
duke
parents:
diff changeset
742
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
743 #ifdef _MSC_VER
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
744 #pragma warning(push)
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
745 // warning C4522: multiple assignment operators specified
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
746 #pragma warning(disable:4522)
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
747 #endif
113
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
748
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
749 // This is a container class for either an oop* or a narrowOop*.
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
750 // Both are pushed onto a task queue and the consumer will test is_narrow()
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
751 // to determine which should be processed.
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
752 class StarTask {
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
753 void* _holder; // either union oop* or narrowOop*
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
754
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
755 enum { COMPRESSED_OOP_MASK = 1 };
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
756
113
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
757 public:
845
df6caf649ff7 6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents: 579
diff changeset
758 StarTask(narrowOop* p) {
df6caf649ff7 6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents: 579
diff changeset
759 assert(((uintptr_t)p & COMPRESSED_OOP_MASK) == 0, "Information loss!");
df6caf649ff7 6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents: 579
diff changeset
760 _holder = (void *)((uintptr_t)p | COMPRESSED_OOP_MASK);
df6caf649ff7 6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents: 579
diff changeset
761 }
df6caf649ff7 6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents: 579
diff changeset
762 StarTask(oop* p) {
df6caf649ff7 6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents: 579
diff changeset
763 assert(((uintptr_t)p & COMPRESSED_OOP_MASK) == 0, "Information loss!");
df6caf649ff7 6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents: 579
diff changeset
764 _holder = (void*)p;
df6caf649ff7 6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents: 579
diff changeset
765 }
113
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
766 StarTask() { _holder = NULL; }
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
767 operator oop*() { return (oop*)_holder; }
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
768 operator narrowOop*() {
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
769 return (narrowOop*)((uintptr_t)_holder & ~COMPRESSED_OOP_MASK);
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
770 }
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
771
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
772 StarTask& operator=(const StarTask& t) {
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
773 _holder = t._holder;
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
774 return *this;
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
775 }
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
776 volatile StarTask& operator=(const volatile StarTask& t) volatile {
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
777 _holder = t._holder;
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
778 return *this;
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
779 }
113
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
780
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
781 bool is_narrow() const {
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
782 return (((uintptr_t)_holder & COMPRESSED_OOP_MASK) != 0);
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
783 }
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
784 };
ba764ed4b6f2 6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents: 0
diff changeset
785
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
786 class ObjArrayTask
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
787 {
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
788 public:
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
789 ObjArrayTask(oop o = NULL, int idx = 0): _obj(o), _index(idx) { }
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
790 ObjArrayTask(oop o, size_t idx): _obj(o), _index(int(idx)) {
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
791 assert(idx <= size_t(max_jint), "too big");
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
792 }
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
793 ObjArrayTask(const ObjArrayTask& t): _obj(t._obj), _index(t._index) { }
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
794
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
795 ObjArrayTask& operator =(const ObjArrayTask& t) {
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
796 _obj = t._obj;
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
797 _index = t._index;
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
798 return *this;
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
799 }
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
800 volatile ObjArrayTask&
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
801 operator =(const volatile ObjArrayTask& t) volatile {
12316
190899198332 7195622: CheckUnhandledOops has limited usefulness now
hseigel
parents: 12087
diff changeset
802 (void)const_cast<oop&>(_obj = t._obj);
1311
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
803 _index = t._index;
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
804 return *this;
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
805 }
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
806
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
807 inline oop obj() const { return _obj; }
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
808 inline int index() const { return _index; }
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
809
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
810 DEBUG_ONLY(bool is_valid() const); // Tasks to be pushed/popped must be valid.
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
811
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
812 private:
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
813 oop _obj;
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
814 int _index;
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
815 };
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
816
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
817 #ifdef _MSC_VER
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
818 #pragma warning(pop)
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
819 #endif
2a1472c30599 4396719: Mark Sweep stack overflow on deeply nested Object arrays
jcoomes
parents: 1284
diff changeset
820
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
821 typedef OverflowTaskQueue<StarTask, mtClass> OopStarTaskQueue;
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
822 typedef GenericTaskQueueSet<OopStarTaskQueue, mtClass> OopStarTaskQueueSet;
0
a61af66fc99e Initial load
duke
parents:
diff changeset
823
6197
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
824 typedef OverflowTaskQueue<size_t, mtInternal> RegionTaskQueue;
d2a62e0f25eb 6995781: Native Memory Tracking (Phase 1)
zgu
parents: 3960
diff changeset
825 typedef GenericTaskQueueSet<RegionTaskQueue, mtClass> RegionTaskQueueSet;
1833
8b10f48633dc 6984287: Regularize how GC parallel workers are specified.
jmasa
parents: 1709
diff changeset
826
1972
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
827
f95d63e2154a 6989984: Use standard include model for Hospot
stefank
parents: 1836
diff changeset
828 #endif // SHARE_VM_UTILITIES_TASKQUEUE_HPP