Mercurial > hg > truffle
annotate src/cpu/x86/vm/c1_MacroAssembler_x86.cpp @ 2013:ec8c74742417
7005241: C1: SEGV in java.util.concurrent.LinkedTransferQueue.xfer() with compressed oops
Summary: Implementation of the CAS primitive for x64 compressed oops was incorrect. It kills rscratch2 register (r11), which is allocatable in C1. Also, we don't need to restore cmpval as it's never used after that, so we need only one temporary register, which can be scratch1.
Reviewed-by: kvn, never
author | iveresov |
---|---|
date | Wed, 08 Dec 2010 02:36:36 -0800 |
parents | ac637b7220d1 |
children | b1a2afa37ec4 |
rev | line source |
---|---|
0 | 1 /* |
1552
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1295
diff
changeset
|
2 * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. |
0 | 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
4 * | |
5 * This code is free software; you can redistribute it and/or modify it | |
6 * under the terms of the GNU General Public License version 2 only, as | |
7 * published by the Free Software Foundation. | |
8 * | |
9 * This code is distributed in the hope that it will be useful, but WITHOUT | |
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
12 * version 2 for more details (a copy is included in the LICENSE file that | |
13 * accompanied this code). | |
14 * | |
15 * You should have received a copy of the GNU General Public License version | |
16 * 2 along with this work; if not, write to the Free Software Foundation, | |
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
18 * | |
1552
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1295
diff
changeset
|
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1295
diff
changeset
|
20 * or visit www.oracle.com if you need additional information or have any |
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1295
diff
changeset
|
21 * questions. |
0 | 22 * |
23 */ | |
24 | |
1972 | 25 #include "precompiled.hpp" |
26 #include "c1/c1_MacroAssembler.hpp" | |
27 #include "c1/c1_Runtime1.hpp" | |
28 #include "classfile/systemDictionary.hpp" | |
29 #include "gc_interface/collectedHeap.hpp" | |
30 #include "interpreter/interpreter.hpp" | |
31 #include "oops/arrayOop.hpp" | |
32 #include "oops/markOop.hpp" | |
33 #include "runtime/basicLock.hpp" | |
34 #include "runtime/biasedLocking.hpp" | |
35 #include "runtime/os.hpp" | |
36 #include "runtime/stubRoutines.hpp" | |
0 | 37 |
38 int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register scratch, Label& slow_case) { | |
304 | 39 const int aligned_mask = BytesPerWord -1; |
0 | 40 const int hdr_offset = oopDesc::mark_offset_in_bytes(); |
41 assert(hdr == rax, "hdr must be rax, for the cmpxchg instruction"); | |
42 assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different"); | |
43 Label done; | |
44 int null_check_offset = -1; | |
45 | |
46 verify_oop(obj); | |
47 | |
48 // save object being locked into the BasicObjectLock | |
304 | 49 movptr(Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()), obj); |
0 | 50 |
51 if (UseBiasedLocking) { | |
52 assert(scratch != noreg, "should have scratch register at this point"); | |
53 null_check_offset = biased_locking_enter(disp_hdr, obj, hdr, scratch, false, done, &slow_case); | |
54 } else { | |
55 null_check_offset = offset(); | |
56 } | |
57 | |
58 // Load object header | |
304 | 59 movptr(hdr, Address(obj, hdr_offset)); |
0 | 60 // and mark it as unlocked |
304 | 61 orptr(hdr, markOopDesc::unlocked_value); |
0 | 62 // save unlocked object header into the displaced header location on the stack |
304 | 63 movptr(Address(disp_hdr, 0), hdr); |
0 | 64 // test if object header is still the same (i.e. unlocked), and if so, store the |
65 // displaced header address in the object header - if it is not the same, get the | |
66 // object header instead | |
67 if (os::is_MP()) MacroAssembler::lock(); // must be immediately before cmpxchg! | |
304 | 68 cmpxchgptr(disp_hdr, Address(obj, hdr_offset)); |
0 | 69 // if the object header was the same, we're done |
70 if (PrintBiasedLockingStatistics) { | |
71 cond_inc32(Assembler::equal, | |
72 ExternalAddress((address)BiasedLocking::fast_path_entry_count_addr())); | |
73 } | |
74 jcc(Assembler::equal, done); | |
75 // if the object header was not the same, it is now in the hdr register | |
76 // => test if it is a stack pointer into the same stack (recursive locking), i.e.: | |
77 // | |
78 // 1) (hdr & aligned_mask) == 0 | |
79 // 2) rsp <= hdr | |
80 // 3) hdr <= rsp + page_size | |
81 // | |
82 // these 3 tests can be done by evaluating the following expression: | |
83 // | |
84 // (hdr - rsp) & (aligned_mask - page_size) | |
85 // | |
86 // assuming both the stack pointer and page_size have their least | |
87 // significant 2 bits cleared and page_size is a power of 2 | |
304 | 88 subptr(hdr, rsp); |
89 andptr(hdr, aligned_mask - os::vm_page_size()); | |
0 | 90 // for recursive locking, the result is zero => save it in the displaced header |
91 // location (NULL in the displaced hdr location indicates recursive locking) | |
304 | 92 movptr(Address(disp_hdr, 0), hdr); |
0 | 93 // otherwise we don't care about the result and handle locking via runtime call |
94 jcc(Assembler::notZero, slow_case); | |
95 // done | |
96 bind(done); | |
97 return null_check_offset; | |
98 } | |
99 | |
100 | |
101 void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) { | |
304 | 102 const int aligned_mask = BytesPerWord -1; |
0 | 103 const int hdr_offset = oopDesc::mark_offset_in_bytes(); |
104 assert(disp_hdr == rax, "disp_hdr must be rax, for the cmpxchg instruction"); | |
105 assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different"); | |
106 Label done; | |
107 | |
108 if (UseBiasedLocking) { | |
109 // load object | |
304 | 110 movptr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes())); |
0 | 111 biased_locking_exit(obj, hdr, done); |
112 } | |
113 | |
114 // load displaced header | |
304 | 115 movptr(hdr, Address(disp_hdr, 0)); |
0 | 116 // if the loaded hdr is NULL we had recursive locking |
304 | 117 testptr(hdr, hdr); |
0 | 118 // if we had recursive locking, we are done |
119 jcc(Assembler::zero, done); | |
120 if (!UseBiasedLocking) { | |
121 // load object | |
304 | 122 movptr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes())); |
0 | 123 } |
124 verify_oop(obj); | |
125 // test if object header is pointing to the displaced header, and if so, restore | |
126 // the displaced header in the object - if the object header is not pointing to | |
127 // the displaced header, get the object header instead | |
128 if (os::is_MP()) MacroAssembler::lock(); // must be immediately before cmpxchg! | |
304 | 129 cmpxchgptr(hdr, Address(obj, hdr_offset)); |
0 | 130 // if the object header was not pointing to the displaced header, |
131 // we do unlocking via runtime call | |
132 jcc(Assembler::notEqual, slow_case); | |
133 // done | |
134 bind(done); | |
135 } | |
136 | |
137 | |
138 // Defines obj, preserves var_size_in_bytes | |
139 void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, Label& slow_case) { | |
140 if (UseTLAB) { | |
141 tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); | |
142 } else { | |
143 eden_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case); | |
144 } | |
145 } | |
146 | |
147 | |
148 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) { | |
149 assert_different_registers(obj, klass, len); | |
150 if (UseBiasedLocking && !len->is_valid()) { | |
151 assert_different_registers(obj, klass, len, t1, t2); | |
304 | 152 movptr(t1, Address(klass, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes())); |
153 movptr(Address(obj, oopDesc::mark_offset_in_bytes()), t1); | |
0 | 154 } else { |
304 | 155 // This assumes that all prototype bits fit in an int32_t |
156 movptr(Address(obj, oopDesc::mark_offset_in_bytes ()), (int32_t)(intptr_t)markOopDesc::prototype()); | |
0 | 157 } |
2002 | 158 #ifdef _LP64 |
159 if (UseCompressedOops) { // Take care not to kill klass | |
160 movptr(t1, klass); | |
161 encode_heap_oop_not_null(t1); | |
162 movl(Address(obj, oopDesc::klass_offset_in_bytes()), t1); | |
163 } else | |
164 #endif | |
165 { | |
166 movptr(Address(obj, oopDesc::klass_offset_in_bytes()), klass); | |
167 } | |
0 | 168 |
169 if (len->is_valid()) { | |
170 movl(Address(obj, arrayOopDesc::length_offset_in_bytes()), len); | |
171 } | |
2002 | 172 #ifdef _LP64 |
173 else if (UseCompressedOops) { | |
174 xorptr(t1, t1); | |
175 store_klass_gap(obj, t1); | |
176 } | |
177 #endif | |
0 | 178 } |
179 | |
180 | |
181 // preserves obj, destroys len_in_bytes | |
182 void C1_MacroAssembler::initialize_body(Register obj, Register len_in_bytes, int hdr_size_in_bytes, Register t1) { | |
183 Label done; | |
184 assert(obj != len_in_bytes && obj != t1 && t1 != len_in_bytes, "registers must be different"); | |
185 assert((hdr_size_in_bytes & (BytesPerWord - 1)) == 0, "header size is not a multiple of BytesPerWord"); | |
186 Register index = len_in_bytes; | |
304 | 187 // index is positive and ptr sized |
188 subptr(index, hdr_size_in_bytes); | |
0 | 189 jcc(Assembler::zero, done); |
190 // initialize topmost word, divide index by 2, check if odd and test if zero | |
191 // note: for the remaining code to work, index must be a multiple of BytesPerWord | |
192 #ifdef ASSERT | |
193 { Label L; | |
304 | 194 testptr(index, BytesPerWord - 1); |
0 | 195 jcc(Assembler::zero, L); |
196 stop("index is not a multiple of BytesPerWord"); | |
197 bind(L); | |
198 } | |
199 #endif | |
304 | 200 xorptr(t1, t1); // use _zero reg to clear memory (shorter code) |
0 | 201 if (UseIncDec) { |
304 | 202 shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set |
0 | 203 } else { |
304 | 204 shrptr(index, 2); // use 2 instructions to avoid partial flag stall |
205 shrptr(index, 1); | |
0 | 206 } |
304 | 207 #ifndef _LP64 |
0 | 208 // index could have been not a multiple of 8 (i.e., bit 2 was set) |
209 { Label even; | |
210 // note: if index was a multiple of 8, than it cannot | |
211 // be 0 now otherwise it must have been 0 before | |
212 // => if it is even, we don't need to check for 0 again | |
213 jcc(Assembler::carryClear, even); | |
214 // clear topmost word (no jump needed if conditional assignment would work here) | |
304 | 215 movptr(Address(obj, index, Address::times_8, hdr_size_in_bytes - 0*BytesPerWord), t1); |
0 | 216 // index could be 0 now, need to check again |
217 jcc(Assembler::zero, done); | |
218 bind(even); | |
219 } | |
304 | 220 #endif // !_LP64 |
0 | 221 // initialize remaining object fields: rdx is a multiple of 2 now |
222 { Label loop; | |
223 bind(loop); | |
304 | 224 movptr(Address(obj, index, Address::times_8, hdr_size_in_bytes - 1*BytesPerWord), t1); |
225 NOT_LP64(movptr(Address(obj, index, Address::times_8, hdr_size_in_bytes - 2*BytesPerWord), t1);) | |
0 | 226 decrement(index); |
227 jcc(Assembler::notZero, loop); | |
228 } | |
229 | |
230 // done | |
231 bind(done); | |
232 } | |
233 | |
234 | |
235 void C1_MacroAssembler::allocate_object(Register obj, Register t1, Register t2, int header_size, int object_size, Register klass, Label& slow_case) { | |
236 assert(obj == rax, "obj must be in rax, for cmpxchg"); | |
237 assert(obj != t1 && obj != t2 && t1 != t2, "registers must be different"); // XXX really? | |
238 assert(header_size >= 0 && object_size >= header_size, "illegal sizes"); | |
239 | |
240 try_allocate(obj, noreg, object_size * BytesPerWord, t1, t2, slow_case); | |
241 | |
242 initialize_object(obj, klass, noreg, object_size * HeapWordSize, t1, t2); | |
243 } | |
244 | |
245 void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2) { | |
246 assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0, | |
247 "con_size_in_bytes is not multiple of alignment"); | |
2002 | 248 const int hdr_size_in_bytes = instanceOopDesc::header_size() * HeapWordSize; |
0 | 249 |
250 initialize_header(obj, klass, noreg, t1, t2); | |
251 | |
252 // clear rest of allocated space | |
253 const Register t1_zero = t1; | |
254 const Register index = t2; | |
255 const int threshold = 6 * BytesPerWord; // approximate break even point for code size (see comments below) | |
256 if (var_size_in_bytes != noreg) { | |
304 | 257 mov(index, var_size_in_bytes); |
0 | 258 initialize_body(obj, index, hdr_size_in_bytes, t1_zero); |
259 } else if (con_size_in_bytes <= threshold) { | |
260 // use explicit null stores | |
261 // code size = 2 + 3*n bytes (n = number of fields to clear) | |
304 | 262 xorptr(t1_zero, t1_zero); // use t1_zero reg to clear memory (shorter code) |
0 | 263 for (int i = hdr_size_in_bytes; i < con_size_in_bytes; i += BytesPerWord) |
304 | 264 movptr(Address(obj, i), t1_zero); |
0 | 265 } else if (con_size_in_bytes > hdr_size_in_bytes) { |
266 // use loop to null out the fields | |
267 // code size = 16 bytes for even n (n = number of fields to clear) | |
268 // initialize last object field first if odd number of fields | |
304 | 269 xorptr(t1_zero, t1_zero); // use t1_zero reg to clear memory (shorter code) |
270 movptr(index, (con_size_in_bytes - hdr_size_in_bytes) >> 3); | |
0 | 271 // initialize last object field if constant size is odd |
272 if (((con_size_in_bytes - hdr_size_in_bytes) & 4) != 0) | |
304 | 273 movptr(Address(obj, con_size_in_bytes - (1*BytesPerWord)), t1_zero); |
0 | 274 // initialize remaining object fields: rdx is a multiple of 2 |
275 { Label loop; | |
276 bind(loop); | |
304 | 277 movptr(Address(obj, index, Address::times_8, hdr_size_in_bytes - (1*BytesPerWord)), |
278 t1_zero); | |
279 NOT_LP64(movptr(Address(obj, index, Address::times_8, hdr_size_in_bytes - (2*BytesPerWord)), | |
280 t1_zero);) | |
0 | 281 decrement(index); |
282 jcc(Assembler::notZero, loop); | |
283 } | |
284 } | |
285 | |
780
c96bf21b756f
6788527: Server vm intermittently fails with assertion "live value must not be garbage" with fastdebug bits
kvn
parents:
304
diff
changeset
|
286 if (CURRENT_ENV->dtrace_alloc_probes()) { |
0 | 287 assert(obj == rax, "must be"); |
288 call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id))); | |
289 } | |
290 | |
291 verify_oop(obj); | |
292 } | |
293 | |
294 void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, Address::ScaleFactor f, Register klass, Label& slow_case) { | |
295 assert(obj == rax, "obj must be in rax, for cmpxchg"); | |
296 assert_different_registers(obj, len, t1, t2, klass); | |
297 | |
298 // determine alignment mask | |
304 | 299 assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work"); |
0 | 300 |
301 // check for negative or excessive length | |
304 | 302 cmpptr(len, (int32_t)max_array_allocation_length); |
0 | 303 jcc(Assembler::above, slow_case); |
304 | |
305 const Register arr_size = t2; // okay to be the same | |
306 // align object end | |
304 | 307 movptr(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask); |
308 lea(arr_size, Address(arr_size, len, f)); | |
309 andptr(arr_size, ~MinObjAlignmentInBytesMask); | |
0 | 310 |
311 try_allocate(obj, arr_size, 0, t1, t2, slow_case); | |
312 | |
313 initialize_header(obj, klass, len, t1, t2); | |
314 | |
315 // clear rest of allocated space | |
316 const Register len_zero = len; | |
317 initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero); | |
318 | |
780
c96bf21b756f
6788527: Server vm intermittently fails with assertion "live value must not be garbage" with fastdebug bits
kvn
parents:
304
diff
changeset
|
319 if (CURRENT_ENV->dtrace_alloc_probes()) { |
0 | 320 assert(obj == rax, "must be"); |
321 call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id))); | |
322 } | |
323 | |
324 verify_oop(obj); | |
325 } | |
326 | |
327 | |
328 | |
329 void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) { | |
330 verify_oop(receiver); | |
331 // explicit NULL check not needed since load from [klass_offset] causes a trap | |
332 // check against inline cache | |
333 assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check"); | |
334 int start_offset = offset(); | |
2002 | 335 |
336 if (UseCompressedOops) { | |
337 load_klass(rscratch1, receiver); | |
338 cmpptr(rscratch1, iCache); | |
339 } else { | |
340 cmpptr(iCache, Address(receiver, oopDesc::klass_offset_in_bytes())); | |
341 } | |
0 | 342 // if icache check fails, then jump to runtime routine |
343 // Note: RECEIVER must still contain the receiver! | |
344 jump_cc(Assembler::notEqual, | |
345 RuntimeAddress(SharedRuntime::get_ic_miss_stub())); | |
304 | 346 const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9); |
2002 | 347 assert(UseCompressedOops || offset() - start_offset == ic_cmp_size, "check alignment in emit_method_entry"); |
0 | 348 } |
349 | |
350 | |
351 void C1_MacroAssembler::build_frame(int frame_size_in_bytes) { | |
352 // Make sure there is enough stack space for this method's activation. | |
353 // Note that we do this before doing an enter(). This matches the | |
354 // ordering of C2's stack overflow check / rsp decrement and allows | |
355 // the SharedRuntime stack overflow handling to be consistent | |
356 // between the two compilers. | |
357 generate_stack_overflow_check(frame_size_in_bytes); | |
358 | |
1295 | 359 push(rbp); |
0 | 360 #ifdef TIERED |
361 // c2 leaves fpu stack dirty. Clean it on entry | |
362 if (UseSSE < 2 ) { | |
363 empty_FPU_stack(); | |
364 } | |
365 #endif // TIERED | |
366 decrement(rsp, frame_size_in_bytes); // does not emit code for frame_size == 0 | |
367 } | |
368 | |
369 | |
1295 | 370 void C1_MacroAssembler::remove_frame(int frame_size_in_bytes) { |
371 increment(rsp, frame_size_in_bytes); // Does not emit code for frame_size == 0 | |
372 pop(rbp); | |
373 } | |
374 | |
375 | |
0 | 376 void C1_MacroAssembler::unverified_entry(Register receiver, Register ic_klass) { |
377 if (C1Breakpoint) int3(); | |
378 inline_cache_check(receiver, ic_klass); | |
379 } | |
380 | |
381 | |
382 void C1_MacroAssembler::verified_entry() { | |
383 if (C1Breakpoint)int3(); | |
384 // build frame | |
385 verify_FPU(0, "method_entry"); | |
386 } | |
387 | |
388 | |
389 #ifndef PRODUCT | |
390 | |
391 void C1_MacroAssembler::verify_stack_oop(int stack_offset) { | |
392 if (!VerifyOops) return; | |
393 verify_oop_addr(Address(rsp, stack_offset)); | |
394 } | |
395 | |
396 void C1_MacroAssembler::verify_not_null_oop(Register r) { | |
397 if (!VerifyOops) return; | |
398 Label not_null; | |
304 | 399 testptr(r, r); |
0 | 400 jcc(Assembler::notZero, not_null); |
401 stop("non-null oop required"); | |
402 bind(not_null); | |
403 verify_oop(r); | |
404 } | |
405 | |
406 void C1_MacroAssembler::invalidate_registers(bool inv_rax, bool inv_rbx, bool inv_rcx, bool inv_rdx, bool inv_rsi, bool inv_rdi) { | |
407 #ifdef ASSERT | |
304 | 408 if (inv_rax) movptr(rax, 0xDEAD); |
409 if (inv_rbx) movptr(rbx, 0xDEAD); | |
410 if (inv_rcx) movptr(rcx, 0xDEAD); | |
411 if (inv_rdx) movptr(rdx, 0xDEAD); | |
412 if (inv_rsi) movptr(rsi, 0xDEAD); | |
413 if (inv_rdi) movptr(rdi, 0xDEAD); | |
0 | 414 #endif |
415 } | |
416 | |
417 #endif // ifndef PRODUCT |