# HG changeset patch # User phh # Date 1294414952 18000 # Node ID b1a2afa37ec4c21adad5b09dc2862a50b3bfb140 # Parent 039eb4201e064db6125d81853e672619168a890f 7003271: Hotspot should track cumulative Java heap bytes allocated on a per-thread basis Summary: Track allocated bytes in Thread's, update on TLAB retirement and direct allocation in Eden and tenured, add JNI methods for ThreadMXBean. Reviewed-by: coleenp, kvn, dholmes, ysr diff -r 039eb4201e06 -r b1a2afa37ec4 src/cpu/sparc/vm/assembler_sparc.cpp --- a/src/cpu/sparc/vm/assembler_sparc.cpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/cpu/sparc/vm/assembler_sparc.cpp Fri Jan 07 10:42:32 2011 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -4083,11 +4083,15 @@ store_klass(t2, top); verify_oop(top); + ld_ptr(G2_thread, in_bytes(JavaThread::tlab_start_offset()), t1); + sub(top, t1, t1); // size of tlab's allocated portion + incr_allocated_bytes(t1, 0, t2); + // refill the tlab with an eden allocation bind(do_refill); ld_ptr(G2_thread, in_bytes(JavaThread::tlab_size_offset()), t1); sll_ptr(t1, LogHeapWordSize, t1); - // add object_size ?? + // allocate new tlab, address returned in top eden_allocate(top, t1, 0, t2, t3, slow_case); st_ptr(top, G2_thread, in_bytes(JavaThread::tlab_start_offset())); @@ -4115,6 +4119,22 @@ delayed()->nop(); } +void MacroAssembler::incr_allocated_bytes(Register var_size_in_bytes, + int con_size_in_bytes, + Register t1) { + // Bump total bytes allocated by this thread + assert(t1->is_global(), "must be global reg"); // so all 64 bits are saved on a context switch + assert_different_registers(var_size_in_bytes, t1); + // v8 support has gone the way of the dodo + ldx(G2_thread, in_bytes(JavaThread::allocated_bytes_offset()), t1); + if (var_size_in_bytes->is_valid()) { + add(t1, var_size_in_bytes, t1); + } else { + add(t1, con_size_in_bytes, t1); + } + stx(t1, G2_thread, in_bytes(JavaThread::allocated_bytes_offset())); +} + Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) { switch (cond) { // Note some conditions are synonyms for others diff -r 039eb4201e06 -r b1a2afa37ec4 src/cpu/sparc/vm/assembler_sparc.hpp --- a/src/cpu/sparc/vm/assembler_sparc.hpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/cpu/sparc/vm/assembler_sparc.hpp Fri Jan 07 10:42:32 2011 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -2388,6 +2388,7 @@ Label& slow_case // continuation point if fast allocation fails ); void tlab_refill(Label& retry_tlab, Label& try_eden, Label& slow_case); + void incr_allocated_bytes(Register var_size_in_bytes, int con_size_in_bytes, Register t1); // interface method calling void lookup_interface_method(Register recv_klass, diff -r 039eb4201e06 -r b1a2afa37ec4 src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp --- a/src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp Fri Jan 07 10:42:32 2011 -0500 @@ -166,7 +166,7 @@ Register obj, // result: pointer to object after successful allocation Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise int con_size_in_bytes, // object size in bytes if known at compile time - Register t1, // temp register + Register t1, // temp register, must be global register for incr_allocated_bytes Register t2, // temp register Label& slow_case // continuation point if fast allocation fails ) { @@ -174,6 +174,7 @@ tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case); } else { eden_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); + incr_allocated_bytes(var_size_in_bytes, con_size_in_bytes, t1); } } @@ -214,7 +215,7 @@ void C1_MacroAssembler::allocate_object( Register obj, // result: pointer to object after successful allocation Register t1, // temp register - Register t2, // temp register + Register t2, // temp register, must be a global register for try_allocate Register t3, // temp register int hdr_size, // object header size in words int obj_size, // object size in words diff -r 039eb4201e06 -r b1a2afa37ec4 src/cpu/sparc/vm/c1_MacroAssembler_sparc.hpp --- a/src/cpu/sparc/vm/c1_MacroAssembler_sparc.hpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/cpu/sparc/vm/c1_MacroAssembler_sparc.hpp Fri Jan 07 10:42:32 2011 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff -r 039eb4201e06 -r b1a2afa37ec4 src/cpu/sparc/vm/c1_Runtime1_sparc.cpp --- a/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp Fri Jan 07 10:42:32 2011 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -448,7 +448,9 @@ // get the instance size __ ld(G5_klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes(), G1_obj_size); + __ tlab_allocate(O0_obj, G1_obj_size, 0, G3_t1, slow_path); + __ initialize_object(O0_obj, G5_klass, G1_obj_size, 0, G3_t1, G4_t2); __ verify_oop(O0_obj); __ mov(O0, I0); @@ -459,6 +461,8 @@ // get the instance size __ ld(G5_klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes(), G1_obj_size); __ eden_allocate(O0_obj, G1_obj_size, 0, G3_t1, G4_t2, slow_path); + __ incr_allocated_bytes(G1_obj_size, 0, G3_t1); + __ initialize_object(O0_obj, G5_klass, G1_obj_size, 0, G3_t1, G4_t2); __ verify_oop(O0_obj); __ mov(O0, I0); @@ -573,6 +577,7 @@ __ and3(G1_arr_size, ~MinObjAlignmentInBytesMask, G1_arr_size); __ eden_allocate(O0_obj, G1_arr_size, 0, G3_t1, O1_t2, slow_path); // preserves G1_arr_size + __ incr_allocated_bytes(G1_arr_size, 0, G3_t1); __ initialize_header(O0_obj, G5_klass, G4_length, G3_t1, O1_t2); __ ldub(klass_lh, G3_t1, klass_lh_header_size_offset); diff -r 039eb4201e06 -r b1a2afa37ec4 src/cpu/sparc/vm/templateTable_sparc.cpp --- a/src/cpu/sparc/vm/templateTable_sparc.cpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/cpu/sparc/vm/templateTable_sparc.cpp Fri Jan 07 10:42:32 2011 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -3393,21 +3393,21 @@ __ delayed()->st_ptr(RnewTopValue, G2_thread, in_bytes(JavaThread::tlab_top_offset())); if (allow_shared_alloc) { - // Check if tlab should be discarded (refill_waste_limit >= free) - __ ld_ptr(G2_thread, in_bytes(JavaThread::tlab_refill_waste_limit_offset()), RtlabWasteLimitValue); - __ sub(RendValue, RoldTopValue, RfreeValue); + // Check if tlab should be discarded (refill_waste_limit >= free) + __ ld_ptr(G2_thread, in_bytes(JavaThread::tlab_refill_waste_limit_offset()), RtlabWasteLimitValue); + __ sub(RendValue, RoldTopValue, RfreeValue); #ifdef _LP64 - __ srlx(RfreeValue, LogHeapWordSize, RfreeValue); + __ srlx(RfreeValue, LogHeapWordSize, RfreeValue); #else - __ srl(RfreeValue, LogHeapWordSize, RfreeValue); + __ srl(RfreeValue, LogHeapWordSize, RfreeValue); #endif - __ cmp(RtlabWasteLimitValue, RfreeValue); - __ brx(Assembler::greaterEqualUnsigned, false, Assembler::pt, slow_case); // tlab waste is small - __ delayed()->nop(); - - // increment waste limit to prevent getting stuck on this slow path - __ add(RtlabWasteLimitValue, ThreadLocalAllocBuffer::refill_waste_limit_increment(), RtlabWasteLimitValue); - __ st_ptr(RtlabWasteLimitValue, G2_thread, in_bytes(JavaThread::tlab_refill_waste_limit_offset())); + __ cmp(RtlabWasteLimitValue, RfreeValue); + __ brx(Assembler::greaterEqualUnsigned, false, Assembler::pt, slow_case); // tlab waste is small + __ delayed()->nop(); + + // increment waste limit to prevent getting stuck on this slow path + __ add(RtlabWasteLimitValue, ThreadLocalAllocBuffer::refill_waste_limit_increment(), RtlabWasteLimitValue); + __ st_ptr(RtlabWasteLimitValue, G2_thread, in_bytes(JavaThread::tlab_refill_waste_limit_offset())); } else { // No allocation in the shared eden. __ br(Assembler::always, false, Assembler::pt, slow_case); @@ -3445,6 +3445,9 @@ __ cmp(RoldTopValue, RnewTopValue); __ brx(Assembler::notEqual, false, Assembler::pn, retry); __ delayed()->nop(); + + // bump total bytes allocated by this thread + __ incr_allocated_bytes(Roffset, 0, G1_scratch); } if (UseTLAB || Universe::heap()->supports_inline_contig_alloc()) { diff -r 039eb4201e06 -r b1a2afa37ec4 src/cpu/x86/vm/assembler_x86.cpp --- a/src/cpu/x86/vm/assembler_x86.cpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/cpu/x86/vm/assembler_x86.cpp Fri Jan 07 10:42:32 2011 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -820,7 +820,20 @@ } -// Now the Assembler instruction (identical for 32/64 bits) +// Now the Assembler instructions (identical for 32/64 bits) + +void Assembler::adcl(Address dst, int32_t imm32) { + InstructionMark im(this); + prefix(dst); + emit_arith_operand(0x81, rdx, dst, imm32); +} + +void Assembler::adcl(Address dst, Register src) { + InstructionMark im(this); + prefix(dst, src); + emit_byte(0x11); + emit_operand(src, dst); +} void Assembler::adcl(Register dst, int32_t imm32) { prefix(dst); @@ -2195,9 +2208,7 @@ void Assembler::orl(Address dst, int32_t imm32) { InstructionMark im(this); prefix(dst); - emit_byte(0x81); - emit_operand(rcx, dst, 4); - emit_long(imm32); + emit_arith_operand(0x81, rcx, dst, imm32); } void Assembler::orl(Register dst, int32_t imm32) { @@ -2205,7 +2216,6 @@ emit_arith(0x81, 0xC8, dst, imm32); } - void Assembler::orl(Register dst, Address src) { InstructionMark im(this); prefix(src, dst); @@ -2213,7 +2223,6 @@ emit_operand(dst, src); } - void Assembler::orl(Register dst, Register src) { (void) prefix_and_encode(dst->encoding(), src->encoding()); emit_arith(0x0B, 0xC0, dst, src); @@ -2692,20 +2701,7 @@ void Assembler::subl(Address dst, int32_t imm32) { InstructionMark im(this); prefix(dst); - if (is8bit(imm32)) { - emit_byte(0x83); - emit_operand(rbp, dst, 1); - emit_byte(imm32 & 0xFF); - } else { - emit_byte(0x81); - emit_operand(rbp, dst, 4); - emit_long(imm32); - } -} - -void Assembler::subl(Register dst, int32_t imm32) { - prefix(dst); - emit_arith(0x81, 0xE8, dst, imm32); + emit_arith_operand(0x81, rbp, dst, imm32); } void Assembler::subl(Address dst, Register src) { @@ -2715,6 +2711,11 @@ emit_operand(src, dst); } +void Assembler::subl(Register dst, int32_t imm32) { + prefix(dst); + emit_arith(0x81, 0xE8, dst, imm32); +} + void Assembler::subl(Register dst, Address src) { InstructionMark im(this); prefix(src, dst); @@ -4333,6 +4334,7 @@ emit_byte(0xD3); emit_byte(0xF8 | encode); } + void Assembler::sbbq(Address dst, int32_t imm32) { InstructionMark im(this); prefixq(dst); @@ -4392,20 +4394,7 @@ void Assembler::subq(Address dst, int32_t imm32) { InstructionMark im(this); prefixq(dst); - if (is8bit(imm32)) { - emit_byte(0x83); - emit_operand(rbp, dst, 1); - emit_byte(imm32 & 0xFF); - } else { - emit_byte(0x81); - emit_operand(rbp, dst, 4); - emit_long(imm32); - } -} - -void Assembler::subq(Register dst, int32_t imm32) { - (void) prefixq_and_encode(dst->encoding()); - emit_arith(0x81, 0xE8, dst, imm32); + emit_arith_operand(0x81, rbp, dst, imm32); } void Assembler::subq(Address dst, Register src) { @@ -4415,6 +4404,11 @@ emit_operand(src, dst); } +void Assembler::subq(Register dst, int32_t imm32) { + (void) prefixq_and_encode(dst->encoding()); + emit_arith(0x81, 0xE8, dst, imm32); +} + void Assembler::subq(Register dst, Address src) { InstructionMark im(this); prefixq(src, dst); @@ -7136,9 +7130,9 @@ } // Preserves rbx, and rdx. -void MacroAssembler::tlab_refill(Label& retry, - Label& try_eden, - Label& slow_case) { +Register MacroAssembler::tlab_refill(Label& retry, + Label& try_eden, + Label& slow_case) { Register top = rax; Register t1 = rcx; Register t2 = rsi; @@ -7185,7 +7179,7 @@ // if tlab is currently allocated (top or end != null) then // fill [top, end + alignment_reserve) with array object - testptr (top, top); + testptr(top, top); jcc(Assembler::zero, do_refill); // set up the mark word @@ -7197,16 +7191,20 @@ movl(Address(top, arrayOopDesc::length_offset_in_bytes()), t1); // set klass to intArrayKlass // dubious reloc why not an oop reloc? - movptr(t1, ExternalAddress((address) Universe::intArrayKlassObj_addr())); + movptr(t1, ExternalAddress((address)Universe::intArrayKlassObj_addr())); // store klass last. concurrent gcs assumes klass length is valid if // klass field is not null. store_klass(top, t1); + movptr(t1, top); + subptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset()))); + incr_allocated_bytes(thread_reg, t1, 0); + // refill the tlab with an eden allocation bind(do_refill); movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset()))); shlptr(t1, LogHeapWordSize); - // add object_size ?? + // allocate new tlab, address returned in top eden_allocate(top, t1, 0, t2, slow_case); // Check that t1 was preserved in eden_allocate. @@ -7234,6 +7232,34 @@ movptr(Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())), top); verify_tlab(); jmp(retry); + + return thread_reg; // for use by caller +} + +void MacroAssembler::incr_allocated_bytes(Register thread, + Register var_size_in_bytes, + int con_size_in_bytes, + Register t1) { +#ifdef _LP64 + if (var_size_in_bytes->is_valid()) { + addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes); + } else { + addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes); + } +#else + if (!thread->is_valid()) { + assert(t1->is_valid(), "need temp reg"); + thread = t1; + get_thread(thread); + } + + if (var_size_in_bytes->is_valid()) { + addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes); + } else { + addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes); + } + adcl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())+4), 0); +#endif } static const double pi_4 = 0.7853981633974483; diff -r 039eb4201e06 -r b1a2afa37ec4 src/cpu/x86/vm/assembler_x86.hpp --- a/src/cpu/x86/vm/assembler_x86.hpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/cpu/x86/vm/assembler_x86.hpp Fri Jan 07 10:42:32 2011 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -674,12 +674,14 @@ // Utilities #ifdef _LP64 - static bool is_simm(int64_t x, int nbits) { return -( CONST64(1) << (nbits-1) ) <= x && x < ( CONST64(1) << (nbits-1) ); } + static bool is_simm(int64_t x, int nbits) { return -(CONST64(1) << (nbits-1)) <= x && + x < (CONST64(1) << (nbits-1)); } static bool is_simm32(int64_t x) { return x == (int64_t)(int32_t)x; } #else - static bool is_simm(int32_t x, int nbits) { return -( 1 << (nbits-1) ) <= x && x < ( 1 << (nbits-1) ); } + static bool is_simm(int32_t x, int nbits) { return -(1 << (nbits-1)) <= x && + x < (1 << (nbits-1)); } static bool is_simm32(int32_t x) { return true; } -#endif // LP64 +#endif // _LP64 // Generic instructions // Does 32bit or 64bit as needed for the platform. In some sense these @@ -705,7 +707,6 @@ void push(void* v); void pop(void* v); - // These do register sized moves/scans void rep_mov(); void rep_set(); @@ -716,6 +717,8 @@ // Vanilla instructions in lexical order + void adcl(Address dst, int32_t imm32); + void adcl(Address dst, Register src); void adcl(Register dst, int32_t imm32); void adcl(Register dst, Address src); void adcl(Register dst, Register src); @@ -724,7 +727,6 @@ void adcq(Register dst, Address src); void adcq(Register dst, Register src); - void addl(Address dst, int32_t imm32); void addl(Address dst, Register src); void addl(Register dst, int32_t imm32); @@ -737,7 +739,6 @@ void addq(Register dst, Address src); void addq(Register dst, Register src); - void addr_nop_4(); void addr_nop_5(); void addr_nop_7(); @@ -759,7 +760,6 @@ void andq(Register dst, Address src); void andq(Register dst, Register src); - // Bitwise Logical AND of Packed Double-Precision Floating-Point Values void andpd(XMMRegister dst, Address src); void andpd(XMMRegister dst, XMMRegister src); @@ -1151,7 +1151,7 @@ #ifdef _LP64 void movq(Register dst, Register src); void movq(Register dst, Address src); - void movq(Address dst, Register src); + void movq(Address dst, Register src); #endif void movq(Address dst, MMXRegister src ); @@ -1177,7 +1177,7 @@ void movsbq(Register dst, Register src); // Move signed 32bit immediate to 64bit extending sign - void movslq(Address dst, int32_t imm64); + void movslq(Address dst, int32_t imm64); void movslq(Register dst, int32_t imm64); void movslq(Register dst, Address src); @@ -1857,7 +1857,10 @@ Register t2, // temp register Label& slow_case // continuation point if fast allocation fails ); - void tlab_refill(Label& retry_tlab, Label& try_eden, Label& slow_case); + Register tlab_refill(Label& retry_tlab, Label& try_eden, Label& slow_case); // returns TLS address + void incr_allocated_bytes(Register thread, + Register var_size_in_bytes, int con_size_in_bytes, + Register t1 = noreg); // interface method calling void lookup_interface_method(Register recv_klass, @@ -2180,9 +2183,9 @@ void divss(XMMRegister dst, Address src) { Assembler::divss(dst, src); } void divss(XMMRegister dst, AddressLiteral src) { Assembler::divss(dst, as_Address(src)); } - void movsd(XMMRegister dst, XMMRegister src) { Assembler::movsd(dst, src); } - void movsd(Address dst, XMMRegister src) { Assembler::movsd(dst, src); } - void movsd(XMMRegister dst, Address src) { Assembler::movsd(dst, src); } + void movsd(XMMRegister dst, XMMRegister src) { Assembler::movsd(dst, src); } + void movsd(Address dst, XMMRegister src) { Assembler::movsd(dst, src); } + void movsd(XMMRegister dst, Address src) { Assembler::movsd(dst, src); } void movsd(XMMRegister dst, AddressLiteral src) { Assembler::movsd(dst, as_Address(src)); } void mulsd(XMMRegister dst, XMMRegister src) { Assembler::mulsd(dst, src); } diff -r 039eb4201e06 -r b1a2afa37ec4 src/cpu/x86/vm/c1_MacroAssembler_x86.cpp --- a/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp Fri Jan 07 10:42:32 2011 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -141,6 +141,7 @@ tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); } else { eden_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case); + incr_allocated_bytes(noreg, var_size_in_bytes, con_size_in_bytes, t1); } } @@ -234,7 +235,7 @@ void C1_MacroAssembler::allocate_object(Register obj, Register t1, Register t2, int header_size, int object_size, Register klass, Label& slow_case) { assert(obj == rax, "obj must be in rax, for cmpxchg"); - assert(obj != t1 && obj != t2 && t1 != t2, "registers must be different"); // XXX really? + assert_different_registers(obj, t1, t2); // XXX really? assert(header_size >= 0 && object_size >= header_size, "illegal sizes"); try_allocate(obj, noreg, object_size * BytesPerWord, t1, t2, slow_case); diff -r 039eb4201e06 -r b1a2afa37ec4 src/cpu/x86/vm/c1_Runtime1_x86.cpp --- a/src/cpu/x86/vm/c1_Runtime1_x86.cpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/cpu/x86/vm/c1_Runtime1_x86.cpp Fri Jan 07 10:42:32 2011 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -977,7 +977,6 @@ // verify that that there is really a valid exception in rax, __ verify_not_null_oop(exception_oop); - oop_maps = new OopMapSet(); OopMap* oop_map = generate_oop_map(sasm, 1); generate_handle_exception(sasm, oop_maps, oop_map); @@ -1037,13 +1036,16 @@ // if we got here then the TLAB allocation failed, so try // refilling the TLAB or allocating directly from eden. Label retry_tlab, try_eden; - __ tlab_refill(retry_tlab, try_eden, slow_path); // does not destroy rdx (klass) + const Register thread = + __ tlab_refill(retry_tlab, try_eden, slow_path); // does not destroy rdx (klass), returns rdi __ bind(retry_tlab); // get the instance size (size is postive so movl is fine for 64bit) __ movl(obj_size, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes())); + __ tlab_allocate(obj, obj_size, 0, t1, t2, slow_path); + __ initialize_object(obj, klass, obj_size, 0, t1, t2); __ verify_oop(obj); __ pop(rbx); @@ -1053,7 +1055,10 @@ __ bind(try_eden); // get the instance size (size is postive so movl is fine for 64bit) __ movl(obj_size, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes())); + __ eden_allocate(obj, obj_size, 0, t1, slow_path); + __ incr_allocated_bytes(thread, obj_size, 0); + __ initialize_object(obj, klass, obj_size, 0, t1, t2); __ verify_oop(obj); __ pop(rbx); @@ -1143,12 +1148,13 @@ // if we got here then the TLAB allocation failed, so try // refilling the TLAB or allocating directly from eden. Label retry_tlab, try_eden; - __ tlab_refill(retry_tlab, try_eden, slow_path); // preserves rbx, & rdx + const Register thread = + __ tlab_refill(retry_tlab, try_eden, slow_path); // preserves rbx & rdx, returns rdi __ bind(retry_tlab); // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F)) - // since size is postive movl does right thing on 64bit + // since size is positive movl does right thing on 64bit __ movl(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes())); // since size is postive movl does right thing on 64bit __ movl(arr_size, length); @@ -1175,7 +1181,7 @@ __ bind(try_eden); // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F)) - // since size is postive movl does right thing on 64bit + // since size is positive movl does right thing on 64bit __ movl(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes())); // since size is postive movl does right thing on 64bit __ movl(arr_size, length); @@ -1188,6 +1194,7 @@ __ andptr(arr_size, ~MinObjAlignmentInBytesMask); __ eden_allocate(obj, arr_size, 0, t1, slow_path); // preserves arr_size + __ incr_allocated_bytes(thread, arr_size, 0); __ initialize_header(obj, klass, length, t1, t2); __ movb(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes() + (Klass::_lh_header_size_shift / BitsPerByte))); diff -r 039eb4201e06 -r b1a2afa37ec4 src/cpu/x86/vm/templateTable_x86_32.cpp --- a/src/cpu/x86/vm/templateTable_x86_32.cpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/cpu/x86/vm/templateTable_x86_32.cpp Fri Jan 07 10:42:32 2011 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -3203,10 +3203,12 @@ const bool allow_shared_alloc = Universe::heap()->supports_inline_contig_alloc() && !CMSIncrementalMode; + const Register thread = rcx; + if (UseTLAB || allow_shared_alloc) { + __ get_thread(thread); + } + if (UseTLAB) { - const Register thread = rcx; - - __ get_thread(thread); __ movptr(rax, Address(thread, in_bytes(JavaThread::tlab_top_offset()))); __ lea(rbx, Address(rax, rdx, Address::times_1)); __ cmpptr(rbx, Address(thread, in_bytes(JavaThread::tlab_end_offset()))); @@ -3247,6 +3249,8 @@ // if someone beat us on the allocation, try again, otherwise continue __ jcc(Assembler::notEqual, retry); + + __ incr_allocated_bytes(thread, rdx, 0); } if (UseTLAB || Universe::heap()->supports_inline_contig_alloc()) { @@ -3256,12 +3260,12 @@ __ decrement(rdx, sizeof(oopDesc)); __ jcc(Assembler::zero, initialize_header); - // Initialize topmost object field, divide rdx by 8, check if odd and - // test if zero. + // Initialize topmost object field, divide rdx by 8, check if odd and + // test if zero. __ xorl(rcx, rcx); // use zero reg to clear memory (shorter code) __ shrl(rdx, LogBytesPerLong); // divide by 2*oopSize and set carry flag if odd - // rdx must have been multiple of 8 + // rdx must have been multiple of 8 #ifdef ASSERT // make sure rdx was multiple of 8 Label L; diff -r 039eb4201e06 -r b1a2afa37ec4 src/cpu/x86/vm/templateTable_x86_64.cpp --- a/src/cpu/x86/vm/templateTable_x86_64.cpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/cpu/x86/vm/templateTable_x86_64.cpp Fri Jan 07 10:42:32 2011 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -3266,6 +3266,8 @@ // if someone beat us on the allocation, try again, otherwise continue __ jcc(Assembler::notEqual, retry); + + __ incr_allocated_bytes(r15_thread, rdx, 0); } if (UseTLAB || Universe::heap()->supports_inline_contig_alloc()) { diff -r 039eb4201e06 -r b1a2afa37ec4 src/os/solaris/vm/os_solaris.cpp --- a/src/os/solaris/vm/os_solaris.cpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/os/solaris/vm/os_solaris.cpp Fri Jan 07 10:42:32 2011 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -283,7 +283,7 @@ is_error_reported(), "sp must be inside of selected thread stack"); - thread->_self_raw_id = raw_id; // mark for quick retrieval + thread->set_self_raw_id(raw_id); // mark for quick retrieval _get_thread_cache[ index ] = thread; } return thread; diff -r 039eb4201e06 -r b1a2afa37ec4 src/os/solaris/vm/thread_solaris.inline.hpp --- a/src/os/solaris/vm/thread_solaris.inline.hpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/os/solaris/vm/thread_solaris.inline.hpp Fri Jan 07 10:42:32 2011 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -53,10 +53,10 @@ uintptr_t raw = pd_raw_thread_id(); int ix = pd_cache_index(raw); - Thread *Candidate = ThreadLocalStorage::_get_thread_cache[ix]; - if (Candidate->_self_raw_id == raw) { + Thread* candidate = ThreadLocalStorage::_get_thread_cache[ix]; + if (candidate->self_raw_id() == raw) { // hit - return Candidate; + return candidate; } else { return ThreadLocalStorage::get_thread_via_cache_slowly(raw, ix); } diff -r 039eb4201e06 -r b1a2afa37ec4 src/share/vm/gc_interface/collectedHeap.inline.hpp --- a/src/share/vm/gc_interface/collectedHeap.inline.hpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/share/vm/gc_interface/collectedHeap.inline.hpp Fri Jan 07 10:42:32 2011 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -153,6 +153,7 @@ check_for_non_bad_heap_word_value(result, size)); assert(!HAS_PENDING_EXCEPTION, "Unexpected exception, will result in uninitialized storage"); + THREAD->incr_allocated_bytes(size * HeapWordSize); return result; } diff -r 039eb4201e06 -r b1a2afa37ec4 src/share/vm/memory/threadLocalAllocBuffer.cpp --- a/src/share/vm/memory/threadLocalAllocBuffer.cpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/share/vm/memory/threadLocalAllocBuffer.cpp Fri Jan 07 10:42:32 2011 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -114,6 +114,11 @@ void ThreadLocalAllocBuffer::make_parsable(bool retire) { if (end() != NULL) { invariants(); + + if (retire) { + myThread()->incr_allocated_bytes(used_bytes()); + } + CollectedHeap::fill_with_object(top(), hard_end(), retire); if (retire || ZeroTLAB) { // "Reset" the TLAB diff -r 039eb4201e06 -r b1a2afa37ec4 src/share/vm/memory/threadLocalAllocBuffer.hpp --- a/src/share/vm/memory/threadLocalAllocBuffer.hpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/share/vm/memory/threadLocalAllocBuffer.hpp Fri Jan 07 10:42:32 2011 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -112,6 +112,8 @@ HeapWord* top() const { return _top; } HeapWord* pf_top() const { return _pf_top; } size_t desired_size() const { return _desired_size; } + size_t used() const { return pointer_delta(top(), start()); } + size_t used_bytes() const { return pointer_delta(top(), start(), 1); } size_t free() const { return pointer_delta(end(), top()); } // Don't discard tlab if remaining space is larger than this. size_t refill_waste_limit() const { return _refill_waste_limit; } diff -r 039eb4201e06 -r b1a2afa37ec4 src/share/vm/opto/macro.cpp --- a/src/share/vm/opto/macro.cpp Fri Jan 07 03:38:19 2011 -0800 +++ b/src/share/vm/opto/macro.cpp Fri Jan 07 10:42:32 2011 -0500 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1158,7 +1158,7 @@ // Note: We set the control input on "eden_end" and "old_eden_top" when using // a TLAB to work around a bug where these values were being moved across // a safepoint. These are not oops, so they cannot be include in the oop - // map, but the can be changed by a GC. The proper way to fix this would + // map, but they can be changed by a GC. The proper way to fix this would // be to set the raw memory state when generating a SafepointNode. However // this will require extensive changes to the loop optimization in order to // prevent a degradation of the optimization. @@ -1167,24 +1167,24 @@ // allocate the Region and Phi nodes for the result result_region = new (C, 3) RegionNode(3); - result_phi_rawmem = new (C, 3) PhiNode( result_region, Type::MEMORY, TypeRawPtr::BOTTOM ); - result_phi_rawoop = new (C, 3) PhiNode( result_region, TypeRawPtr::BOTTOM ); - result_phi_i_o = new (C, 3) PhiNode( result_region, Type::ABIO ); // I/O is used for Prefetch + result_phi_rawmem = new (C, 3) PhiNode(result_region, Type::MEMORY, TypeRawPtr::BOTTOM); + result_phi_rawoop = new (C, 3) PhiNode(result_region, TypeRawPtr::BOTTOM); + result_phi_i_o = new (C, 3) PhiNode(result_region, Type::ABIO); // I/O is used for Prefetch // We need a Region for the loop-back contended case. enum { fall_in_path = 1, contended_loopback_path = 2 }; Node *contended_region; Node *contended_phi_rawmem; - if( UseTLAB ) { + if (UseTLAB) { contended_region = toobig_false; contended_phi_rawmem = mem; } else { contended_region = new (C, 3) RegionNode(3); - contended_phi_rawmem = new (C, 3) PhiNode( contended_region, Type::MEMORY, TypeRawPtr::BOTTOM); + contended_phi_rawmem = new (C, 3) PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM); // Now handle the passing-too-big test. We fall into the contended // loop-back merge point. - contended_region ->init_req( fall_in_path, toobig_false ); - contended_phi_rawmem->init_req( fall_in_path, mem ); + contended_region ->init_req(fall_in_path, toobig_false); + contended_phi_rawmem->init_req(fall_in_path, mem); transform_later(contended_region); transform_later(contended_phi_rawmem); } @@ -1192,78 +1192,101 @@ // Load(-locked) the heap top. // See note above concerning the control input when using a TLAB Node *old_eden_top = UseTLAB - ? new (C, 3) LoadPNode ( ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM ) - : new (C, 3) LoadPLockedNode( contended_region, contended_phi_rawmem, eden_top_adr ); + ? new (C, 3) LoadPNode (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM) + : new (C, 3) LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr); transform_later(old_eden_top); // Add to heap top to get a new heap top - Node *new_eden_top = new (C, 4) AddPNode( top(), old_eden_top, size_in_bytes ); + Node *new_eden_top = new (C, 4) AddPNode(top(), old_eden_top, size_in_bytes); transform_later(new_eden_top); // Check for needing a GC; compare against heap end - Node *needgc_cmp = new (C, 3) CmpPNode( new_eden_top, eden_end ); + Node *needgc_cmp = new (C, 3) CmpPNode(new_eden_top, eden_end); transform_later(needgc_cmp); - Node *needgc_bol = new (C, 2) BoolNode( needgc_cmp, BoolTest::ge ); + Node *needgc_bol = new (C, 2) BoolNode(needgc_cmp, BoolTest::ge); transform_later(needgc_bol); - IfNode *needgc_iff = new (C, 2) IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN ); + IfNode *needgc_iff = new (C, 2) IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN); transform_later(needgc_iff); // Plug the failing-heap-space-need-gc test into the slow-path region - Node *needgc_true = new (C, 1) IfTrueNode( needgc_iff ); + Node *needgc_true = new (C, 1) IfTrueNode(needgc_iff); transform_later(needgc_true); - if( initial_slow_test ) { - slow_region ->init_req( need_gc_path, needgc_true ); + if (initial_slow_test) { + slow_region->init_req(need_gc_path, needgc_true); // This completes all paths into the slow merge point transform_later(slow_region); } else { // No initial slow path needed! // Just fall from the need-GC path straight into the VM call. - slow_region = needgc_true; + slow_region = needgc_true; } // No need for a GC. Setup for the Store-Conditional - Node *needgc_false = new (C, 1) IfFalseNode( needgc_iff ); + Node *needgc_false = new (C, 1) IfFalseNode(needgc_iff); transform_later(needgc_false); // Grab regular I/O before optional prefetch may change it. // Slow-path does no I/O so just set it to the original I/O. - result_phi_i_o->init_req( slow_result_path, i_o ); + result_phi_i_o->init_req(slow_result_path, i_o); i_o = prefetch_allocation(i_o, needgc_false, contended_phi_rawmem, old_eden_top, new_eden_top, length); + // Name successful fast-path variables + Node* fast_oop = old_eden_top; + Node* fast_oop_ctrl; + Node* fast_oop_rawmem; + // Store (-conditional) the modified eden top back down. // StorePConditional produces flags for a test PLUS a modified raw // memory state. - Node *store_eden_top; - Node *fast_oop_ctrl; - if( UseTLAB ) { - store_eden_top = new (C, 4) StorePNode( needgc_false, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, new_eden_top ); + if (UseTLAB) { + Node* store_eden_top = + new (C, 4) StorePNode(needgc_false, contended_phi_rawmem, eden_top_adr, + TypeRawPtr::BOTTOM, new_eden_top); transform_later(store_eden_top); fast_oop_ctrl = needgc_false; // No contention, so this is the fast path + fast_oop_rawmem = store_eden_top; } else { - store_eden_top = new (C, 5) StorePConditionalNode( needgc_false, contended_phi_rawmem, eden_top_adr, new_eden_top, old_eden_top ); + Node* store_eden_top = + new (C, 5) StorePConditionalNode(needgc_false, contended_phi_rawmem, eden_top_adr, + new_eden_top, fast_oop/*old_eden_top*/); transform_later(store_eden_top); - Node *contention_check = new (C, 2) BoolNode( store_eden_top, BoolTest::ne ); + Node *contention_check = new (C, 2) BoolNode(store_eden_top, BoolTest::ne); transform_later(contention_check); store_eden_top = new (C, 1) SCMemProjNode(store_eden_top); transform_later(store_eden_top); // If not using TLABs, check to see if there was contention. - IfNode *contention_iff = new (C, 2) IfNode ( needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN ); + IfNode *contention_iff = new (C, 2) IfNode (needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN); transform_later(contention_iff); - Node *contention_true = new (C, 1) IfTrueNode( contention_iff ); + Node *contention_true = new (C, 1) IfTrueNode(contention_iff); transform_later(contention_true); // If contention, loopback and try again. - contended_region->init_req( contended_loopback_path, contention_true ); - contended_phi_rawmem->init_req( contended_loopback_path, store_eden_top ); + contended_region->init_req(contended_loopback_path, contention_true); + contended_phi_rawmem->init_req(contended_loopback_path, store_eden_top); // Fast-path succeeded with no contention! - Node *contention_false = new (C, 1) IfFalseNode( contention_iff ); + Node *contention_false = new (C, 1) IfFalseNode(contention_iff); transform_later(contention_false); fast_oop_ctrl = contention_false; + + // Bump total allocated bytes for this thread + Node* thread = new (C, 1) ThreadLocalNode(); + transform_later(thread); + Node* alloc_bytes_adr = basic_plus_adr(top()/*not oop*/, thread, + in_bytes(JavaThread::allocated_bytes_offset())); + Node* alloc_bytes = make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr, + 0, TypeLong::LONG, T_LONG); +#ifdef _LP64 + Node* alloc_size = size_in_bytes; +#else + Node* alloc_size = new (C, 2) ConvI2LNode(size_in_bytes); + transform_later(alloc_size); +#endif + Node* new_alloc_bytes = new (C, 3) AddLNode(alloc_bytes, alloc_size); + transform_later(new_alloc_bytes); + fast_oop_rawmem = make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr, + 0, new_alloc_bytes, T_LONG); } - // Rename successful fast-path variables to make meaning more obvious - Node* fast_oop = old_eden_top; - Node* fast_oop_rawmem = store_eden_top; fast_oop_rawmem = initialize_object(alloc, fast_oop_ctrl, fast_oop_rawmem, fast_oop, klass_node, length, size_in_bytes); @@ -1282,11 +1305,11 @@ call->init_req(TypeFunc::Parms+0, thread); call->init_req(TypeFunc::Parms+1, fast_oop); - call->init_req( TypeFunc::Control, fast_oop_ctrl ); - call->init_req( TypeFunc::I_O , top() ) ; // does no i/o - call->init_req( TypeFunc::Memory , fast_oop_rawmem ); - call->init_req( TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr) ); - call->init_req( TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr) ); + call->init_req(TypeFunc::Control, fast_oop_ctrl); + call->init_req(TypeFunc::I_O , top()); // does no i/o + call->init_req(TypeFunc::Memory , fast_oop_rawmem); + call->init_req(TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr)); + call->init_req(TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr)); transform_later(call); fast_oop_ctrl = new (C, 1) ProjNode(call,TypeFunc::Control); transform_later(fast_oop_ctrl); @@ -1295,10 +1318,10 @@ } // Plug in the successful fast-path into the result merge point - result_region ->init_req( fast_result_path, fast_oop_ctrl ); - result_phi_rawoop->init_req( fast_result_path, fast_oop ); - result_phi_i_o ->init_req( fast_result_path, i_o ); - result_phi_rawmem->init_req( fast_result_path, fast_oop_rawmem ); + result_region ->init_req(fast_result_path, fast_oop_ctrl); + result_phi_rawoop->init_req(fast_result_path, fast_oop); + result_phi_i_o ->init_req(fast_result_path, i_o); + result_phi_rawmem->init_req(fast_result_path, fast_oop_rawmem); } else { slow_region = ctrl; } diff -r 039eb4201e06 -r b1a2afa37ec4 src/share/vm/prims/jvmti.xml --- a/src/share/vm/prims/jvmti.xml Fri Jan 07 03:38:19 2011 -0800 +++ b/src/share/vm/prims/jvmti.xml Fri Jan 07 10:42:32 2011 -0500 @@ -1,7 +1,7 @@