# HG changeset patch # User amurillo # Date 1395769722 25200 # Node ID f0ea4d3df1299b6c958e1a72f892c695fca055ad # Parent 7ce7247df891e56d14f0615595c0aa59f4d570f6# Parent 542475eeb98cf5d83cb7621e5f6a90a1e13dab3d Merge diff -r 7ce7247df891 -r f0ea4d3df129 .hgtags --- a/.hgtags Mon Mar 24 13:14:23 2014 -0700 +++ b/.hgtags Tue Mar 25 10:48:42 2014 -0700 @@ -436,3 +436,4 @@ 57eb3e69397e9d5818c5fdaef65b47d9b03f7f88 jdk8u20-b05 804f89b6ff46728d60a69e9a338e63f362f7ac68 hs25.20-b06 c3d92e04873788275eeebec6bcd2948cdbd143a7 jdk8u20-b06 +39eae002499704438142e78f5e0e24d46d0b266f hs25.20-b07 diff -r 7ce7247df891 -r f0ea4d3df129 agent/src/share/classes/sun/jvm/hotspot/gc_implementation/g1/G1CollectedHeap.java --- a/agent/src/share/classes/sun/jvm/hotspot/gc_implementation/g1/G1CollectedHeap.java Mon Mar 24 13:14:23 2014 -0700 +++ b/agent/src/share/classes/sun/jvm/hotspot/gc_implementation/g1/G1CollectedHeap.java Tue Mar 25 10:48:42 2014 -0700 @@ -51,9 +51,9 @@ static private CIntegerField summaryBytesUsedField; // G1MonitoringSupport* _g1mm; static private AddressField g1mmField; - // MasterOldRegionSet _old_set; + // HeapRegionSet _old_set; static private long oldSetFieldOffset; - // MasterHumongousRegionSet _humongous_set; + // HeapRegionSet _humongous_set; static private long humongousSetFieldOffset; static { diff -r 7ce7247df891 -r f0ea4d3df129 agent/src/share/classes/sun/jvm/hotspot/gc_implementation/g1/HeapRegionSetBase.java --- a/agent/src/share/classes/sun/jvm/hotspot/gc_implementation/g1/HeapRegionSetBase.java Mon Mar 24 13:14:23 2014 -0700 +++ b/agent/src/share/classes/sun/jvm/hotspot/gc_implementation/g1/HeapRegionSetBase.java Tue Mar 25 10:48:42 2014 -0700 @@ -40,12 +40,8 @@ // Mirror class for HeapRegionSetBase. Represents a group of regions. public class HeapRegionSetBase extends VMObject { - // uint _length; - static private CIntegerField lengthField; - // uint _region_num; - static private CIntegerField regionNumField; - // size_t _total_used_bytes; - static private CIntegerField totalUsedBytesField; + + static private long countField; static { VM.registerVMInitializedObserver(new Observer() { @@ -58,21 +54,13 @@ static private synchronized void initialize(TypeDataBase db) { Type type = db.lookupType("HeapRegionSetBase"); - lengthField = type.getCIntegerField("_length"); - regionNumField = type.getCIntegerField("_region_num"); - totalUsedBytesField = type.getCIntegerField("_total_used_bytes"); + countField = type.getField("_count").getOffset(); } - public long length() { - return lengthField.getValue(addr); - } - public long regionNum() { - return regionNumField.getValue(addr); - } - - public long totalUsedBytes() { - return totalUsedBytesField.getValue(addr); + public HeapRegionSetCount count() { + Address countFieldAddr = addr.addOffsetTo(countField); + return (HeapRegionSetCount) VMObjectFactory.newObject(HeapRegionSetCount.class, countFieldAddr); } public HeapRegionSetBase(Address addr) { diff -r 7ce7247df891 -r f0ea4d3df129 agent/src/share/classes/sun/jvm/hotspot/gc_implementation/g1/HeapRegionSetCount.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/agent/src/share/classes/sun/jvm/hotspot/gc_implementation/g1/HeapRegionSetCount.java Tue Mar 25 10:48:42 2014 -0700 @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc_implementation.g1; + +import java.util.Iterator; +import java.util.Observable; +import java.util.Observer; + +import sun.jvm.hotspot.debugger.Address; +import sun.jvm.hotspot.runtime.VM; +import sun.jvm.hotspot.runtime.VMObject; +import sun.jvm.hotspot.runtime.VMObjectFactory; +import sun.jvm.hotspot.types.AddressField; +import sun.jvm.hotspot.types.CIntegerField; +import sun.jvm.hotspot.types.Type; +import sun.jvm.hotspot.types.TypeDataBase; + +// Mirror class for HeapRegionSetCount. Represents a group of regions. + +public class HeapRegionSetCount extends VMObject { + + static private CIntegerField lengthField; + static private CIntegerField capacityField; + + static { + VM.registerVMInitializedObserver(new Observer() { + public void update(Observable o, Object data) { + initialize(VM.getVM().getTypeDataBase()); + } + }); + } + + static private synchronized void initialize(TypeDataBase db) { + Type type = db.lookupType("HeapRegionSetCount"); + + lengthField = type.getCIntegerField("_length"); + capacityField = type.getCIntegerField("_capacity"); + } + + public long length() { + return lengthField.getValue(addr); + } + + public long capacity() { + return capacityField.getValue(addr); + } + + public HeapRegionSetCount(Address addr) { + super(addr); + } +} diff -r 7ce7247df891 -r f0ea4d3df129 agent/src/share/classes/sun/jvm/hotspot/tools/HeapSummary.java --- a/agent/src/share/classes/sun/jvm/hotspot/tools/HeapSummary.java Mon Mar 24 13:14:23 2014 -0700 +++ b/agent/src/share/classes/sun/jvm/hotspot/tools/HeapSummary.java Tue Mar 25 10:48:42 2014 -0700 @@ -114,7 +114,8 @@ long survivorRegionNum = g1mm.survivorRegionNum(); HeapRegionSetBase oldSet = g1h.oldSet(); HeapRegionSetBase humongousSet = g1h.humongousSet(); - long oldRegionNum = oldSet.regionNum() + humongousSet.regionNum(); + long oldRegionNum = oldSet.count().length() + + humongousSet.count().capacity() / HeapRegion.grainBytes(); printG1Space("G1 Heap:", g1h.n_regions(), g1h.used(), g1h.capacity()); System.out.println("G1 Young Generation:"); diff -r 7ce7247df891 -r f0ea4d3df129 make/hotspot_version --- a/make/hotspot_version Mon Mar 24 13:14:23 2014 -0700 +++ b/make/hotspot_version Tue Mar 25 10:48:42 2014 -0700 @@ -35,7 +35,7 @@ HS_MAJOR_VER=25 HS_MINOR_VER=20 -HS_BUILD_NUMBER=06 +HS_BUILD_NUMBER=07 JDK_MAJOR_VER=1 JDK_MINOR_VER=8 diff -r 7ce7247df891 -r f0ea4d3df129 src/cpu/sparc/vm/sparc.ad --- a/src/cpu/sparc/vm/sparc.ad Mon Mar 24 13:14:23 2014 -0700 +++ b/src/cpu/sparc/vm/sparc.ad Tue Mar 25 10:48:42 2014 -0700 @@ -2029,19 +2029,6 @@ return L7_REGP_mask(); } -const RegMask Matcher::mathExactI_result_proj_mask() { - return G1_REGI_mask(); -} - -const RegMask Matcher::mathExactL_result_proj_mask() { - return G1_REGL_mask(); -} - -const RegMask Matcher::mathExactI_flags_proj_mask() { - return INT_FLAGS_mask(); -} - - %} diff -r 7ce7247df891 -r f0ea4d3df129 src/cpu/x86/vm/assembler_x86.cpp --- a/src/cpu/x86/vm/assembler_x86.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/cpu/x86/vm/assembler_x86.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -1089,6 +1089,21 @@ emit_arith(0x23, 0xC0, dst, src); } +void Assembler::andnl(Register dst, Register src1, Register src2) { + assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); + int encode = vex_prefix_0F38_and_encode(dst, src1, src2); + emit_int8((unsigned char)0xF2); + emit_int8((unsigned char)(0xC0 | encode)); +} + +void Assembler::andnl(Register dst, Register src1, Address src2) { + InstructionMark im(this); + assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); + vex_prefix_0F38(dst, src1, src2); + emit_int8((unsigned char)0xF2); + emit_operand(dst, src2); +} + void Assembler::bsfl(Register dst, Register src) { int encode = prefix_and_encode(dst->encoding(), src->encoding()); emit_int8(0x0F); @@ -1110,6 +1125,51 @@ emit_int8((unsigned char)(0xC8 | encode)); } +void Assembler::blsil(Register dst, Register src) { + assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); + int encode = vex_prefix_0F38_and_encode(rbx, dst, src); + emit_int8((unsigned char)0xF3); + emit_int8((unsigned char)(0xC0 | encode)); +} + +void Assembler::blsil(Register dst, Address src) { + InstructionMark im(this); + assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); + vex_prefix_0F38(rbx, dst, src); + emit_int8((unsigned char)0xF3); + emit_operand(rbx, src); +} + +void Assembler::blsmskl(Register dst, Register src) { + assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); + int encode = vex_prefix_0F38_and_encode(rdx, dst, src); + emit_int8((unsigned char)0xF3); + emit_int8((unsigned char)(0xC0 | encode)); +} + +void Assembler::blsmskl(Register dst, Address src) { + InstructionMark im(this); + assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); + vex_prefix_0F38(rdx, dst, src); + emit_int8((unsigned char)0xF3); + emit_operand(rdx, src); +} + +void Assembler::blsrl(Register dst, Register src) { + assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); + int encode = vex_prefix_0F38_and_encode(rcx, dst, src); + emit_int8((unsigned char)0xF3); + emit_int8((unsigned char)(0xC0 | encode)); +} + +void Assembler::blsrl(Register dst, Address src) { + InstructionMark im(this); + assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); + vex_prefix_0F38(rcx, dst, src); + emit_int8((unsigned char)0xF3); + emit_operand(rcx, src); +} + void Assembler::call(Label& L, relocInfo::relocType rtype) { // suspect disp32 is always good int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand); @@ -2878,6 +2938,24 @@ emit_operand(dst, src); } +void Assembler::tzcntl(Register dst, Register src) { + assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported"); + emit_int8((unsigned char)0xF3); + int encode = prefix_and_encode(dst->encoding(), src->encoding()); + emit_int8(0x0F); + emit_int8((unsigned char)0xBC); + emit_int8((unsigned char)0xC0 | encode); +} + +void Assembler::tzcntq(Register dst, Register src) { + assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported"); + emit_int8((unsigned char)0xF3); + int encode = prefixq_and_encode(dst->encoding(), src->encoding()); + emit_int8(0x0F); + emit_int8((unsigned char)0xBC); + emit_int8((unsigned char)(0xC0 | encode)); +} + void Assembler::ucomisd(XMMRegister dst, Address src) { NOT_LP64(assert(VM_Version::supports_sse2(), "")); emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66); @@ -4837,6 +4915,21 @@ emit_arith(0x23, 0xC0, dst, src); } +void Assembler::andnq(Register dst, Register src1, Register src2) { + assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); + int encode = vex_prefix_0F38_and_encode_q(dst, src1, src2); + emit_int8((unsigned char)0xF2); + emit_int8((unsigned char)(0xC0 | encode)); +} + +void Assembler::andnq(Register dst, Register src1, Address src2) { + InstructionMark im(this); + assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); + vex_prefix_0F38_q(dst, src1, src2); + emit_int8((unsigned char)0xF2); + emit_operand(dst, src2); +} + void Assembler::bsfq(Register dst, Register src) { int encode = prefixq_and_encode(dst->encoding(), src->encoding()); emit_int8(0x0F); @@ -4858,6 +4951,51 @@ emit_int8((unsigned char)(0xC8 | encode)); } +void Assembler::blsiq(Register dst, Register src) { + assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); + int encode = vex_prefix_0F38_and_encode_q(rbx, dst, src); + emit_int8((unsigned char)0xF3); + emit_int8((unsigned char)(0xC0 | encode)); +} + +void Assembler::blsiq(Register dst, Address src) { + InstructionMark im(this); + assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); + vex_prefix_0F38_q(rbx, dst, src); + emit_int8((unsigned char)0xF3); + emit_operand(rbx, src); +} + +void Assembler::blsmskq(Register dst, Register src) { + assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); + int encode = vex_prefix_0F38_and_encode_q(rdx, dst, src); + emit_int8((unsigned char)0xF3); + emit_int8((unsigned char)(0xC0 | encode)); +} + +void Assembler::blsmskq(Register dst, Address src) { + InstructionMark im(this); + assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); + vex_prefix_0F38_q(rdx, dst, src); + emit_int8((unsigned char)0xF3); + emit_operand(rdx, src); +} + +void Assembler::blsrq(Register dst, Register src) { + assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); + int encode = vex_prefix_0F38_and_encode_q(rcx, dst, src); + emit_int8((unsigned char)0xF3); + emit_int8((unsigned char)(0xC0 | encode)); +} + +void Assembler::blsrq(Register dst, Address src) { + InstructionMark im(this); + assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported"); + vex_prefix_0F38_q(rcx, dst, src); + emit_int8((unsigned char)0xF3); + emit_operand(rcx, src); +} + void Assembler::cdqq() { prefix(REX_W); emit_int8((unsigned char)0x99); diff -r 7ce7247df891 -r f0ea4d3df129 src/cpu/x86/vm/assembler_x86.hpp --- a/src/cpu/x86/vm/assembler_x86.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/cpu/x86/vm/assembler_x86.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -590,10 +590,35 @@ vex_prefix(src, nds_enc, dst_enc, pre, VEX_OPCODE_0F, false, vector256); } + void vex_prefix_0F38(Register dst, Register nds, Address src) { + bool vex_w = false; + bool vector256 = false; + vex_prefix(src, nds->encoding(), dst->encoding(), + VEX_SIMD_NONE, VEX_OPCODE_0F_38, vex_w, vector256); + } + + void vex_prefix_0F38_q(Register dst, Register nds, Address src) { + bool vex_w = true; + bool vector256 = false; + vex_prefix(src, nds->encoding(), dst->encoding(), + VEX_SIMD_NONE, VEX_OPCODE_0F_38, vex_w, vector256); + } int vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool vex_w, bool vector256); + int vex_prefix_0F38_and_encode(Register dst, Register nds, Register src) { + bool vex_w = false; + bool vector256 = false; + return vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), + VEX_SIMD_NONE, VEX_OPCODE_0F_38, vex_w, vector256); + } + int vex_prefix_0F38_and_encode_q(Register dst, Register nds, Register src) { + bool vex_w = true; + bool vector256 = false; + return vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), + VEX_SIMD_NONE, VEX_OPCODE_0F_38, vex_w, vector256); + } int vex_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre, bool vector256 = false, VexOpcode opc = VEX_OPCODE_0F) { @@ -897,6 +922,27 @@ void andq(Register dst, Address src); void andq(Register dst, Register src); + // BMI instructions + void andnl(Register dst, Register src1, Register src2); + void andnl(Register dst, Register src1, Address src2); + void andnq(Register dst, Register src1, Register src2); + void andnq(Register dst, Register src1, Address src2); + + void blsil(Register dst, Register src); + void blsil(Register dst, Address src); + void blsiq(Register dst, Register src); + void blsiq(Register dst, Address src); + + void blsmskl(Register dst, Register src); + void blsmskl(Register dst, Address src); + void blsmskq(Register dst, Register src); + void blsmskq(Register dst, Address src); + + void blsrl(Register dst, Register src); + void blsrl(Register dst, Address src); + void blsrq(Register dst, Register src); + void blsrq(Register dst, Address src); + void bsfl(Register dst, Register src); void bsrl(Register dst, Register src); @@ -1574,6 +1620,9 @@ void testq(Register dst, int32_t imm32); void testq(Register dst, Register src); + // BMI - count trailing zeros + void tzcntl(Register dst, Register src); + void tzcntq(Register dst, Register src); // Unordered Compare Scalar Double-Precision Floating-Point Values and set EFLAGS void ucomisd(XMMRegister dst, Address src); diff -r 7ce7247df891 -r f0ea4d3df129 src/cpu/x86/vm/globals_x86.hpp --- a/src/cpu/x86/vm/globals_x86.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/cpu/x86/vm/globals_x86.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -134,5 +134,11 @@ \ product(bool, UseCountLeadingZerosInstruction, false, \ "Use count leading zeros instruction") \ + \ + product(bool, UseCountTrailingZerosInstruction, false, \ + "Use count trailing zeros instruction") \ + \ + product(bool, UseBMI1Instructions, false, \ + "Use BMI instructions") #endif // CPU_X86_VM_GLOBALS_X86_HPP diff -r 7ce7247df891 -r f0ea4d3df129 src/cpu/x86/vm/vm_version_x86.cpp --- a/src/cpu/x86/vm/vm_version_x86.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/cpu/x86/vm/vm_version_x86.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -50,8 +50,13 @@ const char* VM_Version::_features_str = ""; VM_Version::CpuidInfo VM_Version::_cpuid_info = { 0, }; +// Address of instruction which causes SEGV +address VM_Version::_cpuinfo_segv_addr = 0; +// Address of instruction after the one which causes SEGV +address VM_Version::_cpuinfo_cont_addr = 0; + static BufferBlob* stub_blob; -static const int stub_size = 550; +static const int stub_size = 600; extern "C" { typedef void (*getPsrInfo_stub_t)(void*); @@ -234,9 +239,9 @@ // Check if OS has enabled XGETBV instruction to access XCR0 // (OSXSAVE feature flag) and CPU supports AVX // - __ andl(rcx, 0x18000000); + __ andl(rcx, 0x18000000); // cpuid1 bits osxsave | avx __ cmpl(rcx, 0x18000000); - __ jccb(Assembler::notEqual, sef_cpuid); + __ jccb(Assembler::notEqual, sef_cpuid); // jump if AVX is not supported // // XCR0, XFEATURE_ENABLED_MASK register @@ -247,6 +252,47 @@ __ movl(Address(rsi, 0), rax); __ movl(Address(rsi, 4), rdx); + __ andl(rax, 0x6); // xcr0 bits sse | ymm + __ cmpl(rax, 0x6); + __ jccb(Assembler::notEqual, sef_cpuid); // jump if AVX is not supported + + // + // Some OSs have a bug when upper 128bits of YMM + // registers are not restored after a signal processing. + // Generate SEGV here (reference through NULL) + // and check upper YMM bits after it. + // + VM_Version::set_avx_cpuFeatures(); // Enable temporary to pass asserts + + // load value into all 32 bytes of ymm7 register + __ movl(rcx, VM_Version::ymm_test_value()); + + __ movdl(xmm0, rcx); + __ pshufd(xmm0, xmm0, 0x00); + __ vinsertf128h(xmm0, xmm0, xmm0); + __ vmovdqu(xmm7, xmm0); +#ifdef _LP64 + __ vmovdqu(xmm8, xmm0); + __ vmovdqu(xmm15, xmm0); +#endif + + __ xorl(rsi, rsi); + VM_Version::set_cpuinfo_segv_addr( __ pc() ); + // Generate SEGV + __ movl(rax, Address(rsi, 0)); + + VM_Version::set_cpuinfo_cont_addr( __ pc() ); + // Returns here after signal. Save xmm0 to check it later. + __ lea(rsi, Address(rbp, in_bytes(VM_Version::ymm_save_offset()))); + __ vmovdqu(Address(rsi, 0), xmm0); + __ vmovdqu(Address(rsi, 32), xmm7); +#ifdef _LP64 + __ vmovdqu(Address(rsi, 64), xmm8); + __ vmovdqu(Address(rsi, 96), xmm15); +#endif + + VM_Version::clean_cpuFeatures(); + // // cpuid(0x7) Structured Extended Features // @@ -429,7 +475,7 @@ } char buf[256]; - jio_snprintf(buf, sizeof(buf), "(%u cores per cpu, %u threads per core) family %d model %d stepping %d%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s", + jio_snprintf(buf, sizeof(buf), "(%u cores per cpu, %u threads per core) family %d model %d stepping %d%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s", cores_per_cpu(), threads_per_core(), cpu_family(), _model, _stepping, (supports_cmov() ? ", cmov" : ""), @@ -455,7 +501,9 @@ (supports_ht() ? ", ht": ""), (supports_tsc() ? ", tsc": ""), (supports_tscinv_bit() ? ", tscinvbit": ""), - (supports_tscinv() ? ", tscinv": "")); + (supports_tscinv() ? ", tscinv": ""), + (supports_bmi1() ? ", bmi1" : ""), + (supports_bmi2() ? ", bmi2" : "")); _features_str = strdup(buf); // UseSSE is set to the smaller of what hardware supports and what @@ -538,14 +586,28 @@ if (MaxVectorSize > 32) { FLAG_SET_DEFAULT(MaxVectorSize, 32); } - if (MaxVectorSize > 16 && UseAVX == 0) { - // Only supported with AVX+ + if (MaxVectorSize > 16 && (UseAVX == 0 || !os_supports_avx_vectors())) { + // 32 bytes vectors (in YMM) are only supported with AVX+ FLAG_SET_DEFAULT(MaxVectorSize, 16); } if (UseSSE < 2) { - // Only supported with SSE2+ + // Vectors (in XMM) are only supported with SSE2+ FLAG_SET_DEFAULT(MaxVectorSize, 0); } +#ifdef ASSERT + if (supports_avx() && PrintMiscellaneous && Verbose && TraceNewVectors) { + tty->print_cr("State of YMM registers after signal handle:"); + int nreg = 2 LP64_ONLY(+2); + const char* ymm_name[4] = {"0", "7", "8", "15"}; + for (int i = 0; i < nreg; i++) { + tty->print("YMM%s:", ymm_name[i]); + for (int j = 7; j >=0; j--) { + tty->print(" %x", _cpuid_info.ymm_save[i*8 + j]); + } + tty->cr(); + } + } +#endif } #endif @@ -600,13 +662,6 @@ } } - // Use count leading zeros count instruction if available. - if (supports_lzcnt()) { - if (FLAG_IS_DEFAULT(UseCountLeadingZerosInstruction)) { - UseCountLeadingZerosInstruction = true; - } - } - // some defaults for AMD family 15h if ( cpu_family() == 0x15 ) { // On family 15h processors default is no sw prefetch @@ -683,14 +738,35 @@ } } } -#if defined(COMPILER2) && defined(_ALLBSD_SOURCE) - if (MaxVectorSize > 16) { - // Limit vectors size to 16 bytes on BSD until it fixes - // restoring upper 128bit of YMM registers on return - // from signal handler. - FLAG_SET_DEFAULT(MaxVectorSize, 16); + + // Use count leading zeros count instruction if available. + if (supports_lzcnt()) { + if (FLAG_IS_DEFAULT(UseCountLeadingZerosInstruction)) { + UseCountLeadingZerosInstruction = true; + } + } else if (UseCountLeadingZerosInstruction) { + warning("lzcnt instruction is not available on this CPU"); + FLAG_SET_DEFAULT(UseCountLeadingZerosInstruction, false); + } + + if (supports_bmi1()) { + if (FLAG_IS_DEFAULT(UseBMI1Instructions)) { + UseBMI1Instructions = true; } -#endif // COMPILER2 + } else if (UseBMI1Instructions) { + warning("BMI1 instructions are not available on this CPU"); + FLAG_SET_DEFAULT(UseBMI1Instructions, false); + } + + // Use count trailing zeros instruction if available + if (supports_bmi1()) { + if (FLAG_IS_DEFAULT(UseCountTrailingZerosInstruction)) { + UseCountTrailingZerosInstruction = UseBMI1Instructions; + } + } else if (UseCountTrailingZerosInstruction) { + warning("tzcnt instruction is not available on this CPU"); + FLAG_SET_DEFAULT(UseCountTrailingZerosInstruction, false); + } // Use population count instruction if available. if (supports_popcnt()) { @@ -790,6 +866,11 @@ if (UseAES) { tty->print(" UseAES=1"); } +#ifdef COMPILER2 + if (MaxVectorSize > 0) { + tty->print(" MaxVectorSize=%d", MaxVectorSize); + } +#endif tty->cr(); tty->print("Allocation"); if (AllocatePrefetchStyle <= 0 || UseSSE == 0 && !supports_3dnow_prefetch()) { diff -r 7ce7247df891 -r f0ea4d3df129 src/cpu/x86/vm/vm_version_x86.hpp --- a/src/cpu/x86/vm/vm_version_x86.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/cpu/x86/vm/vm_version_x86.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -141,7 +141,8 @@ struct { uint32_t LahfSahf : 1, CmpLegacy : 1, - : 4, + : 3, + lzcnt_intel : 1, lzcnt : 1, sse4a : 1, misalignsse : 1, @@ -228,6 +229,9 @@ // 0 if this instruction is not available static const char* _features_str; + static address _cpuinfo_segv_addr; // address of instruction which causes SEGV + static address _cpuinfo_cont_addr; // address of instruction after the one which causes SEGV + enum { CPU_CX8 = (1 << 0), // next bits are from cpuid 1 (EDX) CPU_CMOV = (1 << 1), @@ -251,7 +255,9 @@ CPU_AVX2 = (1 << 18), CPU_AES = (1 << 19), CPU_ERMS = (1 << 20), // enhanced 'rep movsb/stosb' instructions - CPU_CLMUL = (1 << 21) // carryless multiply for CRC + CPU_CLMUL = (1 << 21), // carryless multiply for CRC + CPU_BMI1 = (1 << 22), + CPU_BMI2 = (1 << 23) } cpuFeatureFlags; enum { @@ -358,6 +364,9 @@ // extended control register XCR0 (the XFEATURE_ENABLED_MASK register) XemXcr0Eax xem_xcr0_eax; uint32_t xem_xcr0_edx; // reserved + + // Space to save ymm registers after signal handle + int ymm_save[8*4]; // Save ymm0, ymm7, ymm8, ymm15 }; // The actual cpuid info block @@ -423,6 +432,8 @@ if (_cpuid_info.sef_cpuid7_ebx.bits.avx2 != 0) result |= CPU_AVX2; } + if(_cpuid_info.sef_cpuid7_ebx.bits.bmi1 != 0) + result |= CPU_BMI1; if (_cpuid_info.std_cpuid1_edx.bits.tsc != 0) result |= CPU_TSC; if (_cpuid_info.ext_cpuid7_edx.bits.tsc_invariance != 0) @@ -444,10 +455,32 @@ if (_cpuid_info.ext_cpuid1_ecx.bits.sse4a != 0) result |= CPU_SSE4A; } + // Intel features. + if(is_intel()) { + if(_cpuid_info.sef_cpuid7_ebx.bits.bmi2 != 0) + result |= CPU_BMI2; + if(_cpuid_info.ext_cpuid1_ecx.bits.lzcnt_intel != 0) + result |= CPU_LZCNT; + } return result; } + static bool os_supports_avx_vectors() { + if (!supports_avx()) { + return false; + } + // Verify that OS save/restore all bits of AVX registers + // during signal processing. + int nreg = 2 LP64_ONLY(+2); + for (int i = 0; i < 8 * nreg; i++) { // 32 bytes per ymm register + if (_cpuid_info.ymm_save[i] != ymm_test_value()) { + return false; + } + } + return true; + } + static void get_processor_features(); public: @@ -464,6 +497,19 @@ static ByteSize tpl_cpuidB1_offset() { return byte_offset_of(CpuidInfo, tpl_cpuidB1_eax); } static ByteSize tpl_cpuidB2_offset() { return byte_offset_of(CpuidInfo, tpl_cpuidB2_eax); } static ByteSize xem_xcr0_offset() { return byte_offset_of(CpuidInfo, xem_xcr0_eax); } + static ByteSize ymm_save_offset() { return byte_offset_of(CpuidInfo, ymm_save); } + + // The value used to check ymm register after signal handle + static int ymm_test_value() { return 0xCAFEBABE; } + + static void set_cpuinfo_segv_addr(address pc) { _cpuinfo_segv_addr = pc; } + static bool is_cpuinfo_segv_addr(address pc) { return _cpuinfo_segv_addr == pc; } + static void set_cpuinfo_cont_addr(address pc) { _cpuinfo_cont_addr = pc; } + static address cpuinfo_cont_addr() { return _cpuinfo_cont_addr; } + + static void clean_cpuFeatures() { _cpuFeatures = 0; } + static void set_avx_cpuFeatures() { _cpuFeatures = (CPU_SSE | CPU_SSE2 | CPU_AVX); } + // Initialization static void initialize(); @@ -560,7 +606,8 @@ static bool supports_aes() { return (_cpuFeatures & CPU_AES) != 0; } static bool supports_erms() { return (_cpuFeatures & CPU_ERMS) != 0; } static bool supports_clmul() { return (_cpuFeatures & CPU_CLMUL) != 0; } - + static bool supports_bmi1() { return (_cpuFeatures & CPU_BMI1) != 0; } + static bool supports_bmi2() { return (_cpuFeatures & CPU_BMI2) != 0; } // Intel features static bool is_intel_family_core() { return is_intel() && extended_cpu_family() == CPU_FAMILY_INTEL_CORE; } diff -r 7ce7247df891 -r f0ea4d3df129 src/cpu/x86/vm/x86_32.ad --- a/src/cpu/x86/vm/x86_32.ad Mon Mar 24 13:14:23 2014 -0700 +++ b/src/cpu/x86/vm/x86_32.ad Tue Mar 25 10:48:42 2014 -0700 @@ -1534,19 +1534,6 @@ return EBP_REG_mask(); } -const RegMask Matcher::mathExactI_result_proj_mask() { - return EAX_REG_mask(); -} - -const RegMask Matcher::mathExactL_result_proj_mask() { - ShouldNotReachHere(); - return RegMask(); -} - -const RegMask Matcher::mathExactI_flags_proj_mask() { - return INT_FLAGS_mask(); -} - // Returns true if the high 32 bits of the value is known to be zero. bool is_operand_hi32_zero(Node* n) { int opc = n->Opcode(); @@ -5168,6 +5155,19 @@ %} instruct countTrailingZerosI(rRegI dst, rRegI src, eFlagsReg cr) %{ + predicate(UseCountTrailingZerosInstruction); + match(Set dst (CountTrailingZerosI src)); + effect(KILL cr); + + format %{ "TZCNT $dst, $src\t# count trailing zeros (int)" %} + ins_encode %{ + __ tzcntl($dst$$Register, $src$$Register); + %} + ins_pipe(ialu_reg); +%} + +instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, eFlagsReg cr) %{ + predicate(!UseCountTrailingZerosInstruction); match(Set dst (CountTrailingZerosI src)); effect(KILL cr); @@ -5187,6 +5187,30 @@ %} instruct countTrailingZerosL(rRegI dst, eRegL src, eFlagsReg cr) %{ + predicate(UseCountTrailingZerosInstruction); + match(Set dst (CountTrailingZerosL src)); + effect(TEMP dst, KILL cr); + + format %{ "TZCNT $dst, $src.lo\t# count trailing zeros (long) \n\t" + "JNC done\n\t" + "TZCNT $dst, $src.hi\n\t" + "ADD $dst, 32\n" + "done:" %} + ins_encode %{ + Register Rdst = $dst$$Register; + Register Rsrc = $src$$Register; + Label done; + __ tzcntl(Rdst, Rsrc); + __ jccb(Assembler::carryClear, done); + __ tzcntl(Rdst, HIGH_FROM_LOW(Rsrc)); + __ addl(Rdst, BitsPerInt); + __ bind(done); + %} + ins_pipe(ialu_reg); +%} + +instruct countTrailingZerosL_bsf(rRegI dst, eRegL src, eFlagsReg cr) %{ + predicate(!UseCountTrailingZerosInstruction); match(Set dst (CountTrailingZerosL src)); effect(TEMP dst, KILL cr); @@ -6999,44 +7023,6 @@ //----------Arithmetic Instructions-------------------------------------------- //----------Addition Instructions---------------------------------------------- -instruct addExactI_eReg(eAXRegI dst, rRegI src, eFlagsReg cr) -%{ - match(AddExactI dst src); - effect(DEF cr); - - format %{ "ADD $dst, $src\t# addExact int" %} - ins_encode %{ - __ addl($dst$$Register, $src$$Register); - %} - ins_pipe(ialu_reg_reg); -%} - -instruct addExactI_eReg_imm(eAXRegI dst, immI src, eFlagsReg cr) -%{ - match(AddExactI dst src); - effect(DEF cr); - - format %{ "ADD $dst, $src\t# addExact int" %} - ins_encode %{ - __ addl($dst$$Register, $src$$constant); - %} - ins_pipe(ialu_reg_reg); -%} - -instruct addExactI_eReg_mem(eAXRegI dst, memory src, eFlagsReg cr) -%{ - match(AddExactI dst (LoadI src)); - effect(DEF cr); - - ins_cost(125); - format %{ "ADD $dst,$src\t# addExact int" %} - ins_encode %{ - __ addl($dst$$Register, $src$$Address); - %} - ins_pipe( ialu_reg_mem ); -%} - - // Integer Addition Instructions instruct addI_eReg(rRegI dst, rRegI src, eFlagsReg cr) %{ match(Set dst (AddI dst src)); @@ -7346,43 +7332,6 @@ //----------Subtraction Instructions------------------------------------------- -instruct subExactI_eReg(eAXRegI dst, rRegI src, eFlagsReg cr) -%{ - match(SubExactI dst src); - effect(DEF cr); - - format %{ "SUB $dst, $src\t# subExact int" %} - ins_encode %{ - __ subl($dst$$Register, $src$$Register); - %} - ins_pipe(ialu_reg_reg); -%} - -instruct subExactI_eReg_imm(eAXRegI dst, immI src, eFlagsReg cr) -%{ - match(SubExactI dst src); - effect(DEF cr); - - format %{ "SUB $dst, $src\t# subExact int" %} - ins_encode %{ - __ subl($dst$$Register, $src$$constant); - %} - ins_pipe(ialu_reg_reg); -%} - -instruct subExactI_eReg_mem(eAXRegI dst, memory src, eFlagsReg cr) -%{ - match(SubExactI dst (LoadI src)); - effect(DEF cr); - - ins_cost(125); - format %{ "SUB $dst,$src\t# subExact int" %} - ins_encode %{ - __ subl($dst$$Register, $src$$Address); - %} - ins_pipe( ialu_reg_mem ); -%} - // Integer Subtraction Instructions instruct subI_eReg(rRegI dst, rRegI src, eFlagsReg cr) %{ match(Set dst (SubI dst src)); @@ -7451,17 +7400,6 @@ ins_pipe( ialu_reg ); %} -instruct negExactI_eReg(eAXRegI dst, eFlagsReg cr) %{ - match(NegExactI dst); - effect(DEF cr); - - format %{ "NEG $dst\t# negExact int"%} - ins_encode %{ - __ negl($dst$$Register); - %} - ins_pipe(ialu_reg); -%} - //----------Multiplication/Division Instructions------------------------------- // Integer Multiplication Instructions // Multiply Register @@ -7673,46 +7611,6 @@ ins_pipe( pipe_slow ); %} -instruct mulExactI_eReg(eAXRegI dst, rRegI src, eFlagsReg cr) -%{ - match(MulExactI dst src); - effect(DEF cr); - - ins_cost(300); - format %{ "IMUL $dst, $src\t# mulExact int" %} - ins_encode %{ - __ imull($dst$$Register, $src$$Register); - %} - ins_pipe(ialu_reg_reg_alu0); -%} - -instruct mulExactI_eReg_imm(eAXRegI dst, rRegI src, immI imm, eFlagsReg cr) -%{ - match(MulExactI src imm); - effect(DEF cr); - - ins_cost(300); - format %{ "IMUL $dst, $src, $imm\t# mulExact int" %} - ins_encode %{ - __ imull($dst$$Register, $src$$Register, $imm$$constant); - %} - ins_pipe(ialu_reg_reg_alu0); -%} - -instruct mulExactI_eReg_mem(eAXRegI dst, memory src, eFlagsReg cr) -%{ - match(MulExactI dst (LoadI src)); - effect(DEF cr); - - ins_cost(350); - format %{ "IMUL $dst, $src\t# mulExact int" %} - ins_encode %{ - __ imull($dst$$Register, $src$$Address); - %} - ins_pipe(ialu_reg_mem_alu0); -%} - - // Integer DIV with Register instruct divI_eReg(eAXRegI rax, eDXRegI rdx, eCXRegI div, eFlagsReg cr) %{ match(Set rax (DivI rax div)); @@ -8156,6 +8054,123 @@ ins_pipe( ialu_mem_imm ); %} +// BMI1 instructions +instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, eFlagsReg cr) %{ + match(Set dst (AndI (XorI src1 minus_1) src2)); + predicate(UseBMI1Instructions); + effect(KILL cr); + + format %{ "ANDNL $dst, $src1, $src2" %} + + ins_encode %{ + __ andnl($dst$$Register, $src1$$Register, $src2$$Register); + %} + ins_pipe(ialu_reg); +%} + +instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, eFlagsReg cr) %{ + match(Set dst (AndI (XorI src1 minus_1) (LoadI src2) )); + predicate(UseBMI1Instructions); + effect(KILL cr); + + ins_cost(125); + format %{ "ANDNL $dst, $src1, $src2" %} + + ins_encode %{ + __ andnl($dst$$Register, $src1$$Register, $src2$$Address); + %} + ins_pipe(ialu_reg_mem); +%} + +instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, eFlagsReg cr) %{ + match(Set dst (AndI (SubI imm_zero src) src)); + predicate(UseBMI1Instructions); + effect(KILL cr); + + format %{ "BLSIL $dst, $src" %} + + ins_encode %{ + __ blsil($dst$$Register, $src$$Register); + %} + ins_pipe(ialu_reg); +%} + +instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, eFlagsReg cr) %{ + match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) )); + predicate(UseBMI1Instructions); + effect(KILL cr); + + ins_cost(125); + format %{ "BLSIL $dst, $src" %} + + ins_encode %{ + __ blsil($dst$$Register, $src$$Address); + %} + ins_pipe(ialu_reg_mem); +%} + +instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, eFlagsReg cr) +%{ + match(Set dst (XorI (AddI src minus_1) src)); + predicate(UseBMI1Instructions); + effect(KILL cr); + + format %{ "BLSMSKL $dst, $src" %} + + ins_encode %{ + __ blsmskl($dst$$Register, $src$$Register); + %} + + ins_pipe(ialu_reg); +%} + +instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, eFlagsReg cr) +%{ + match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) )); + predicate(UseBMI1Instructions); + effect(KILL cr); + + ins_cost(125); + format %{ "BLSMSKL $dst, $src" %} + + ins_encode %{ + __ blsmskl($dst$$Register, $src$$Address); + %} + + ins_pipe(ialu_reg_mem); +%} + +instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, eFlagsReg cr) +%{ + match(Set dst (AndI (AddI src minus_1) src) ); + predicate(UseBMI1Instructions); + effect(KILL cr); + + format %{ "BLSRL $dst, $src" %} + + ins_encode %{ + __ blsrl($dst$$Register, $src$$Register); + %} + + ins_pipe(ialu_reg); +%} + +instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, eFlagsReg cr) +%{ + match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) )); + predicate(UseBMI1Instructions); + effect(KILL cr); + + ins_cost(125); + format %{ "BLSRL $dst, $src" %} + + ins_encode %{ + __ blsrl($dst$$Register, $src$$Address); + %} + + ins_pipe(ialu_reg_mem); +%} + // Or Instructions // Or Register with Register instruct orI_eReg(rRegI dst, rRegI src, eFlagsReg cr) %{ @@ -8578,6 +8593,91 @@ instruct cadd_cmpLTMask_mem(ncxRegI p, ncxRegI q, memory y, eCXRegI tmp, eFlagsReg cr) %{ match(Set p (AddI (AndI (CmpLTMask p q) (LoadI y)) (SubI p q))); */ +//----------Overflow Math Instructions----------------------------------------- + +instruct overflowAddI_eReg(eFlagsReg cr, eAXRegI op1, rRegI op2) +%{ + match(Set cr (OverflowAddI op1 op2)); + effect(DEF cr, USE_KILL op1, USE op2); + + format %{ "ADD $op1, $op2\t# overflow check int" %} + + ins_encode %{ + __ addl($op1$$Register, $op2$$Register); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct overflowAddI_rReg_imm(eFlagsReg cr, eAXRegI op1, immI op2) +%{ + match(Set cr (OverflowAddI op1 op2)); + effect(DEF cr, USE_KILL op1, USE op2); + + format %{ "ADD $op1, $op2\t# overflow check int" %} + + ins_encode %{ + __ addl($op1$$Register, $op2$$constant); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct overflowSubI_rReg(eFlagsReg cr, rRegI op1, rRegI op2) +%{ + match(Set cr (OverflowSubI op1 op2)); + + format %{ "CMP $op1, $op2\t# overflow check int" %} + ins_encode %{ + __ cmpl($op1$$Register, $op2$$Register); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct overflowSubI_rReg_imm(eFlagsReg cr, rRegI op1, immI op2) +%{ + match(Set cr (OverflowSubI op1 op2)); + + format %{ "CMP $op1, $op2\t# overflow check int" %} + ins_encode %{ + __ cmpl($op1$$Register, $op2$$constant); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct overflowNegI_rReg(eFlagsReg cr, immI0 zero, eAXRegI op2) +%{ + match(Set cr (OverflowSubI zero op2)); + effect(DEF cr, USE_KILL op2); + + format %{ "NEG $op2\t# overflow check int" %} + ins_encode %{ + __ negl($op2$$Register); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct overflowMulI_rReg(eFlagsReg cr, eAXRegI op1, rRegI op2) +%{ + match(Set cr (OverflowMulI op1 op2)); + effect(DEF cr, USE_KILL op1, USE op2); + + format %{ "IMUL $op1, $op2\t# overflow check int" %} + ins_encode %{ + __ imull($op1$$Register, $op2$$Register); + %} + ins_pipe(ialu_reg_reg_alu0); +%} + +instruct overflowMulI_rReg_imm(eFlagsReg cr, rRegI op1, immI op2, rRegI tmp) +%{ + match(Set cr (OverflowMulI op1 op2)); + effect(DEF cr, TEMP tmp, USE op1, USE op2); + + format %{ "IMUL $tmp, $op1, $op2\t# overflow check int" %} + ins_encode %{ + __ imull($tmp$$Register, $op1$$Register, $op2$$constant); + %} + ins_pipe(ialu_reg_reg_alu0); +%} //----------Long Instructions------------------------------------------------ // Add Long Register with Register @@ -8693,6 +8793,210 @@ ins_pipe( ialu_reg_long_mem ); %} +// BMI1 instructions +instruct andnL_eReg_eReg_eReg(eRegL dst, eRegL src1, eRegL src2, immL_M1 minus_1, eFlagsReg cr) %{ + match(Set dst (AndL (XorL src1 minus_1) src2)); + predicate(UseBMI1Instructions); + effect(KILL cr, TEMP dst); + + format %{ "ANDNL $dst.lo, $src1.lo, $src2.lo\n\t" + "ANDNL $dst.hi, $src1.hi, $src2.hi" + %} + + ins_encode %{ + Register Rdst = $dst$$Register; + Register Rsrc1 = $src1$$Register; + Register Rsrc2 = $src2$$Register; + __ andnl(Rdst, Rsrc1, Rsrc2); + __ andnl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc1), HIGH_FROM_LOW(Rsrc2)); + %} + ins_pipe(ialu_reg_reg_long); +%} + +instruct andnL_eReg_eReg_mem(eRegL dst, eRegL src1, memory src2, immL_M1 minus_1, eFlagsReg cr) %{ + match(Set dst (AndL (XorL src1 minus_1) (LoadL src2) )); + predicate(UseBMI1Instructions); + effect(KILL cr, TEMP dst); + + ins_cost(125); + format %{ "ANDNL $dst.lo, $src1.lo, $src2\n\t" + "ANDNL $dst.hi, $src1.hi, $src2+4" + %} + + ins_encode %{ + Register Rdst = $dst$$Register; + Register Rsrc1 = $src1$$Register; + Address src2_hi = Address::make_raw($src2$$base, $src2$$index, $src2$$scale, $src2$$disp + 4, relocInfo::none); + + __ andnl(Rdst, Rsrc1, $src2$$Address); + __ andnl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc1), src2_hi); + %} + ins_pipe(ialu_reg_mem); +%} + +instruct blsiL_eReg_eReg(eRegL dst, eRegL src, immL0 imm_zero, eFlagsReg cr) %{ + match(Set dst (AndL (SubL imm_zero src) src)); + predicate(UseBMI1Instructions); + effect(KILL cr, TEMP dst); + + format %{ "MOVL $dst.hi, 0\n\t" + "BLSIL $dst.lo, $src.lo\n\t" + "JNZ done\n\t" + "BLSIL $dst.hi, $src.hi\n" + "done:" + %} + + ins_encode %{ + Label done; + Register Rdst = $dst$$Register; + Register Rsrc = $src$$Register; + __ movl(HIGH_FROM_LOW(Rdst), 0); + __ blsil(Rdst, Rsrc); + __ jccb(Assembler::notZero, done); + __ blsil(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc)); + __ bind(done); + %} + ins_pipe(ialu_reg); +%} + +instruct blsiL_eReg_mem(eRegL dst, memory src, immL0 imm_zero, eFlagsReg cr) %{ + match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) )); + predicate(UseBMI1Instructions); + effect(KILL cr, TEMP dst); + + ins_cost(125); + format %{ "MOVL $dst.hi, 0\n\t" + "BLSIL $dst.lo, $src\n\t" + "JNZ done\n\t" + "BLSIL $dst.hi, $src+4\n" + "done:" + %} + + ins_encode %{ + Label done; + Register Rdst = $dst$$Register; + Address src_hi = Address::make_raw($src$$base, $src$$index, $src$$scale, $src$$disp + 4, relocInfo::none); + + __ movl(HIGH_FROM_LOW(Rdst), 0); + __ blsil(Rdst, $src$$Address); + __ jccb(Assembler::notZero, done); + __ blsil(HIGH_FROM_LOW(Rdst), src_hi); + __ bind(done); + %} + ins_pipe(ialu_reg_mem); +%} + +instruct blsmskL_eReg_eReg(eRegL dst, eRegL src, immL_M1 minus_1, eFlagsReg cr) +%{ + match(Set dst (XorL (AddL src minus_1) src)); + predicate(UseBMI1Instructions); + effect(KILL cr, TEMP dst); + + format %{ "MOVL $dst.hi, 0\n\t" + "BLSMSKL $dst.lo, $src.lo\n\t" + "JNC done\n\t" + "BLSMSKL $dst.hi, $src.hi\n" + "done:" + %} + + ins_encode %{ + Label done; + Register Rdst = $dst$$Register; + Register Rsrc = $src$$Register; + __ movl(HIGH_FROM_LOW(Rdst), 0); + __ blsmskl(Rdst, Rsrc); + __ jccb(Assembler::carryClear, done); + __ blsmskl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc)); + __ bind(done); + %} + + ins_pipe(ialu_reg); +%} + +instruct blsmskL_eReg_mem(eRegL dst, memory src, immL_M1 minus_1, eFlagsReg cr) +%{ + match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) )); + predicate(UseBMI1Instructions); + effect(KILL cr, TEMP dst); + + ins_cost(125); + format %{ "MOVL $dst.hi, 0\n\t" + "BLSMSKL $dst.lo, $src\n\t" + "JNC done\n\t" + "BLSMSKL $dst.hi, $src+4\n" + "done:" + %} + + ins_encode %{ + Label done; + Register Rdst = $dst$$Register; + Address src_hi = Address::make_raw($src$$base, $src$$index, $src$$scale, $src$$disp + 4, relocInfo::none); + + __ movl(HIGH_FROM_LOW(Rdst), 0); + __ blsmskl(Rdst, $src$$Address); + __ jccb(Assembler::carryClear, done); + __ blsmskl(HIGH_FROM_LOW(Rdst), src_hi); + __ bind(done); + %} + + ins_pipe(ialu_reg_mem); +%} + +instruct blsrL_eReg_eReg(eRegL dst, eRegL src, immL_M1 minus_1, eFlagsReg cr) +%{ + match(Set dst (AndL (AddL src minus_1) src) ); + predicate(UseBMI1Instructions); + effect(KILL cr, TEMP dst); + + format %{ "MOVL $dst.hi, $src.hi\n\t" + "BLSRL $dst.lo, $src.lo\n\t" + "JNC done\n\t" + "BLSRL $dst.hi, $src.hi\n" + "done:" + %} + + ins_encode %{ + Label done; + Register Rdst = $dst$$Register; + Register Rsrc = $src$$Register; + __ movl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc)); + __ blsrl(Rdst, Rsrc); + __ jccb(Assembler::carryClear, done); + __ blsrl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc)); + __ bind(done); + %} + + ins_pipe(ialu_reg); +%} + +instruct blsrL_eReg_mem(eRegL dst, memory src, immL_M1 minus_1, eFlagsReg cr) +%{ + match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src) )); + predicate(UseBMI1Instructions); + effect(KILL cr, TEMP dst); + + ins_cost(125); + format %{ "MOVL $dst.hi, $src+4\n\t" + "BLSRL $dst.lo, $src\n\t" + "JNC done\n\t" + "BLSRL $dst.hi, $src+4\n" + "done:" + %} + + ins_encode %{ + Label done; + Register Rdst = $dst$$Register; + Address src_hi = Address::make_raw($src$$base, $src$$index, $src$$scale, $src$$disp + 4, relocInfo::none); + __ movl(HIGH_FROM_LOW(Rdst), src_hi); + __ blsrl(Rdst, $src$$Address); + __ jccb(Assembler::carryClear, done); + __ blsrl(HIGH_FROM_LOW(Rdst), src_hi); + __ bind(done); + %} + + ins_pipe(ialu_reg_mem); +%} + // Or Long Register with Register instruct orl_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{ match(Set dst (OrL dst src)); diff -r 7ce7247df891 -r f0ea4d3df129 src/cpu/x86/vm/x86_64.ad --- a/src/cpu/x86/vm/x86_64.ad Mon Mar 24 13:14:23 2014 -0700 +++ b/src/cpu/x86/vm/x86_64.ad Tue Mar 25 10:48:42 2014 -0700 @@ -1649,18 +1649,6 @@ return PTR_RBP_REG_mask(); } -const RegMask Matcher::mathExactI_result_proj_mask() { - return INT_RAX_REG_mask(); -} - -const RegMask Matcher::mathExactL_result_proj_mask() { - return LONG_RAX_REG_mask(); -} - -const RegMask Matcher::mathExactI_flags_proj_mask() { - return INT_FLAGS_mask(); -} - %} //----------ENCODING BLOCK----------------------------------------------------- @@ -6026,6 +6014,19 @@ %} instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{ + predicate(UseCountTrailingZerosInstruction); + match(Set dst (CountTrailingZerosI src)); + effect(KILL cr); + + format %{ "tzcntl $dst, $src\t# count trailing zeros (int)" %} + ins_encode %{ + __ tzcntl($dst$$Register, $src$$Register); + %} + ins_pipe(ialu_reg); +%} + +instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{ + predicate(!UseCountTrailingZerosInstruction); match(Set dst (CountTrailingZerosI src)); effect(KILL cr); @@ -6045,6 +6046,19 @@ %} instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{ + predicate(UseCountTrailingZerosInstruction); + match(Set dst (CountTrailingZerosL src)); + effect(KILL cr); + + format %{ "tzcntq $dst, $src\t# count trailing zeros (long)" %} + ins_encode %{ + __ tzcntq($dst$$Register, $src$$Register); + %} + ins_pipe(ialu_reg); +%} + +instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{ + predicate(!UseCountTrailingZerosInstruction); match(Set dst (CountTrailingZerosL src)); effect(KILL cr); @@ -6728,82 +6742,6 @@ //----------Arithmetic Instructions-------------------------------------------- //----------Addition Instructions---------------------------------------------- -instruct addExactI_rReg(rax_RegI dst, rRegI src, rFlagsReg cr) -%{ - match(AddExactI dst src); - effect(DEF cr); - - format %{ "addl $dst, $src\t# addExact int" %} - ins_encode %{ - __ addl($dst$$Register, $src$$Register); - %} - ins_pipe(ialu_reg_reg); -%} - -instruct addExactI_rReg_imm(rax_RegI dst, immI src, rFlagsReg cr) -%{ - match(AddExactI dst src); - effect(DEF cr); - - format %{ "addl $dst, $src\t# addExact int" %} - ins_encode %{ - __ addl($dst$$Register, $src$$constant); - %} - ins_pipe(ialu_reg_reg); -%} - -instruct addExactI_rReg_mem(rax_RegI dst, memory src, rFlagsReg cr) -%{ - match(AddExactI dst (LoadI src)); - effect(DEF cr); - - ins_cost(125); // XXX - format %{ "addl $dst, $src\t# addExact int" %} - ins_encode %{ - __ addl($dst$$Register, $src$$Address); - %} - - ins_pipe(ialu_reg_mem); -%} - -instruct addExactL_rReg(rax_RegL dst, rRegL src, rFlagsReg cr) -%{ - match(AddExactL dst src); - effect(DEF cr); - - format %{ "addq $dst, $src\t# addExact long" %} - ins_encode %{ - __ addq($dst$$Register, $src$$Register); - %} - ins_pipe(ialu_reg_reg); -%} - -instruct addExactL_rReg_imm(rax_RegL dst, immL32 src, rFlagsReg cr) -%{ - match(AddExactL dst src); - effect(DEF cr); - - format %{ "addq $dst, $src\t# addExact long" %} - ins_encode %{ - __ addq($dst$$Register, $src$$constant); - %} - ins_pipe(ialu_reg_reg); -%} - -instruct addExactL_rReg_mem(rax_RegL dst, memory src, rFlagsReg cr) -%{ - match(AddExactL dst (LoadL src)); - effect(DEF cr); - - ins_cost(125); // XXX - format %{ "addq $dst, $src\t# addExact long" %} - ins_encode %{ - __ addq($dst$$Register, $src$$Address); - %} - - ins_pipe(ialu_reg_mem); -%} - instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr) %{ match(Set dst (AddI dst src)); @@ -7416,80 +7354,6 @@ ins_pipe(ialu_mem_imm); %} -instruct subExactI_rReg(rax_RegI dst, rRegI src, rFlagsReg cr) -%{ - match(SubExactI dst src); - effect(DEF cr); - - format %{ "subl $dst, $src\t# subExact int" %} - ins_encode %{ - __ subl($dst$$Register, $src$$Register); - %} - ins_pipe(ialu_reg_reg); -%} - -instruct subExactI_rReg_imm(rax_RegI dst, immI src, rFlagsReg cr) -%{ - match(SubExactI dst src); - effect(DEF cr); - - format %{ "subl $dst, $src\t# subExact int" %} - ins_encode %{ - __ subl($dst$$Register, $src$$constant); - %} - ins_pipe(ialu_reg_reg); -%} - -instruct subExactI_rReg_mem(rax_RegI dst, memory src, rFlagsReg cr) -%{ - match(SubExactI dst (LoadI src)); - effect(DEF cr); - - ins_cost(125); - format %{ "subl $dst, $src\t# subExact int" %} - ins_encode %{ - __ subl($dst$$Register, $src$$Address); - %} - ins_pipe(ialu_reg_mem); -%} - -instruct subExactL_rReg(rax_RegL dst, rRegL src, rFlagsReg cr) -%{ - match(SubExactL dst src); - effect(DEF cr); - - format %{ "subq $dst, $src\t# subExact long" %} - ins_encode %{ - __ subq($dst$$Register, $src$$Register); - %} - ins_pipe(ialu_reg_reg); -%} - -instruct subExactL_rReg_imm(rax_RegL dst, immL32 src, rFlagsReg cr) -%{ - match(SubExactL dst (LoadL src)); - effect(DEF cr); - - format %{ "subq $dst, $src\t# subExact long" %} - ins_encode %{ - __ subq($dst$$Register, $src$$constant); - %} - ins_pipe(ialu_reg_reg); -%} - -instruct subExactL_rReg_mem(rax_RegI dst, memory src, rFlagsReg cr) -%{ - match(SubExactI dst src); - effect(DEF cr); - - ins_cost(125); - format %{ "subq $dst, $src\t# subExact long" %} - ins_encode %{ - __ subq($dst$$Register, $src$$Address); - %} - ins_pipe(ialu_reg_mem); -%} - instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr) %{ match(Set dst (SubL dst src)); @@ -7606,31 +7470,6 @@ ins_pipe(ialu_reg); %} -instruct negExactI_rReg(rax_RegI dst, rFlagsReg cr) -%{ - match(NegExactI dst); - effect(KILL cr); - - format %{ "negl $dst\t# negExact int" %} - ins_encode %{ - __ negl($dst$$Register); - %} - ins_pipe(ialu_reg); -%} - -instruct negExactL_rReg(rax_RegL dst, rFlagsReg cr) -%{ - match(NegExactL dst); - effect(KILL cr); - - format %{ "negq $dst\t# negExact long" %} - ins_encode %{ - __ negq($dst$$Register); - %} - ins_pipe(ialu_reg); -%} - - //----------Multiplication/Division Instructions------------------------------- // Integer Multiplication Instructions // Multiply Register @@ -7747,86 +7586,6 @@ ins_pipe(ialu_reg_reg_alu0); %} - -instruct mulExactI_rReg(rax_RegI dst, rRegI src, rFlagsReg cr) -%{ - match(MulExactI dst src); - effect(DEF cr); - - ins_cost(300); - format %{ "imull $dst, $src\t# mulExact int" %} - ins_encode %{ - __ imull($dst$$Register, $src$$Register); - %} - ins_pipe(ialu_reg_reg_alu0); -%} - - -instruct mulExactI_rReg_imm(rax_RegI dst, rRegI src, immI imm, rFlagsReg cr) -%{ - match(MulExactI src imm); - effect(DEF cr); - - ins_cost(300); - format %{ "imull $dst, $src, $imm\t# mulExact int" %} - ins_encode %{ - __ imull($dst$$Register, $src$$Register, $imm$$constant); - %} - ins_pipe(ialu_reg_reg_alu0); -%} - -instruct mulExactI_rReg_mem(rax_RegI dst, memory src, rFlagsReg cr) -%{ - match(MulExactI dst (LoadI src)); - effect(DEF cr); - - ins_cost(350); - format %{ "imull $dst, $src\t# mulExact int" %} - ins_encode %{ - __ imull($dst$$Register, $src$$Address); - %} - ins_pipe(ialu_reg_mem_alu0); -%} - -instruct mulExactL_rReg(rax_RegL dst, rRegL src, rFlagsReg cr) -%{ - match(MulExactL dst src); - effect(DEF cr); - - ins_cost(300); - format %{ "imulq $dst, $src\t# mulExact long" %} - ins_encode %{ - __ imulq($dst$$Register, $src$$Register); - %} - ins_pipe(ialu_reg_reg_alu0); -%} - -instruct mulExactL_rReg_imm(rax_RegL dst, rRegL src, immL32 imm, rFlagsReg cr) -%{ - match(MulExactL src imm); - effect(DEF cr); - - ins_cost(300); - format %{ "imulq $dst, $src, $imm\t# mulExact long" %} - ins_encode %{ - __ imulq($dst$$Register, $src$$Register, $imm$$constant); - %} - ins_pipe(ialu_reg_reg_alu0); -%} - -instruct mulExactL_rReg_mem(rax_RegL dst, memory src, rFlagsReg cr) -%{ - match(MulExactL dst (LoadL src)); - effect(DEF cr); - - ins_cost(350); - format %{ "imulq $dst, $src\t# mulExact long" %} - ins_encode %{ - __ imulq($dst$$Register, $src$$Address); - %} - ins_pipe(ialu_reg_mem_alu0); -%} - instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr) %{ @@ -8879,6 +8638,122 @@ ins_pipe(ialu_mem_imm); %} +// BMI1 instructions +instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{ + match(Set dst (AndI (XorI src1 minus_1) (LoadI src2))); + predicate(UseBMI1Instructions); + effect(KILL cr); + + ins_cost(125); + format %{ "andnl $dst, $src1, $src2" %} + + ins_encode %{ + __ andnl($dst$$Register, $src1$$Register, $src2$$Address); + %} + ins_pipe(ialu_reg_mem); +%} + +instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{ + match(Set dst (AndI (XorI src1 minus_1) src2)); + predicate(UseBMI1Instructions); + effect(KILL cr); + + format %{ "andnl $dst, $src1, $src2" %} + + ins_encode %{ + __ andnl($dst$$Register, $src1$$Register, $src2$$Register); + %} + ins_pipe(ialu_reg); +%} + +instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{ + match(Set dst (AndI (SubI imm_zero src) src)); + predicate(UseBMI1Instructions); + effect(KILL cr); + + format %{ "blsil $dst, $src" %} + + ins_encode %{ + __ blsil($dst$$Register, $src$$Register); + %} + ins_pipe(ialu_reg); +%} + +instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{ + match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) )); + predicate(UseBMI1Instructions); + effect(KILL cr); + + ins_cost(125); + format %{ "blsil $dst, $src" %} + + ins_encode %{ + __ blsil($dst$$Register, $src$$Address); + %} + ins_pipe(ialu_reg_mem); +%} + +instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr) +%{ + match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) ); + predicate(UseBMI1Instructions); + effect(KILL cr); + + ins_cost(125); + format %{ "blsmskl $dst, $src" %} + + ins_encode %{ + __ blsmskl($dst$$Register, $src$$Address); + %} + ins_pipe(ialu_reg_mem); +%} + +instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr) +%{ + match(Set dst (XorI (AddI src minus_1) src)); + predicate(UseBMI1Instructions); + effect(KILL cr); + + format %{ "blsmskl $dst, $src" %} + + ins_encode %{ + __ blsmskl($dst$$Register, $src$$Register); + %} + + ins_pipe(ialu_reg); +%} + +instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr) +%{ + match(Set dst (AndI (AddI src minus_1) src) ); + predicate(UseBMI1Instructions); + effect(KILL cr); + + format %{ "blsrl $dst, $src" %} + + ins_encode %{ + __ blsrl($dst$$Register, $src$$Register); + %} + + ins_pipe(ialu_reg_mem); +%} + +instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr) +%{ + match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) ); + predicate(UseBMI1Instructions); + effect(KILL cr); + + ins_cost(125); + format %{ "blsrl $dst, $src" %} + + ins_encode %{ + __ blsrl($dst$$Register, $src$$Address); + %} + + ins_pipe(ialu_reg); +%} + // Or Instructions // Or Register with Register instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr) @@ -9110,6 +8985,122 @@ ins_pipe(ialu_mem_imm); %} +// BMI1 instructions +instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{ + match(Set dst (AndL (XorL src1 minus_1) (LoadL src2))); + predicate(UseBMI1Instructions); + effect(KILL cr); + + ins_cost(125); + format %{ "andnq $dst, $src1, $src2" %} + + ins_encode %{ + __ andnq($dst$$Register, $src1$$Register, $src2$$Address); + %} + ins_pipe(ialu_reg_mem); +%} + +instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{ + match(Set dst (AndL (XorL src1 minus_1) src2)); + predicate(UseBMI1Instructions); + effect(KILL cr); + + format %{ "andnq $dst, $src1, $src2" %} + + ins_encode %{ + __ andnq($dst$$Register, $src1$$Register, $src2$$Register); + %} + ins_pipe(ialu_reg_mem); +%} + +instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{ + match(Set dst (AndL (SubL imm_zero src) src)); + predicate(UseBMI1Instructions); + effect(KILL cr); + + format %{ "blsiq $dst, $src" %} + + ins_encode %{ + __ blsiq($dst$$Register, $src$$Register); + %} + ins_pipe(ialu_reg); +%} + +instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{ + match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) )); + predicate(UseBMI1Instructions); + effect(KILL cr); + + ins_cost(125); + format %{ "blsiq $dst, $src" %} + + ins_encode %{ + __ blsiq($dst$$Register, $src$$Address); + %} + ins_pipe(ialu_reg_mem); +%} + +instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr) +%{ + match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) ); + predicate(UseBMI1Instructions); + effect(KILL cr); + + ins_cost(125); + format %{ "blsmskq $dst, $src" %} + + ins_encode %{ + __ blsmskq($dst$$Register, $src$$Address); + %} + ins_pipe(ialu_reg_mem); +%} + +instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr) +%{ + match(Set dst (XorL (AddL src minus_1) src)); + predicate(UseBMI1Instructions); + effect(KILL cr); + + format %{ "blsmskq $dst, $src" %} + + ins_encode %{ + __ blsmskq($dst$$Register, $src$$Register); + %} + + ins_pipe(ialu_reg); +%} + +instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr) +%{ + match(Set dst (AndL (AddL src minus_1) src) ); + predicate(UseBMI1Instructions); + effect(KILL cr); + + format %{ "blsrq $dst, $src" %} + + ins_encode %{ + __ blsrq($dst$$Register, $src$$Register); + %} + + ins_pipe(ialu_reg); +%} + +instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr) +%{ + match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) ); + predicate(UseBMI1Instructions); + effect(KILL cr); + + ins_cost(125); + format %{ "blsrq $dst, $src" %} + + ins_encode %{ + __ blsrq($dst$$Register, $src$$Address); + %} + + ins_pipe(ialu_reg); +%} + // Or Instructions // Or Register with Register instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr) @@ -10435,6 +10426,174 @@ ins_pipe( pipe_slow ); %} +//----------Overflow Math Instructions----------------------------------------- + +instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2) +%{ + match(Set cr (OverflowAddI op1 op2)); + effect(DEF cr, USE_KILL op1, USE op2); + + format %{ "addl $op1, $op2\t# overflow check int" %} + + ins_encode %{ + __ addl($op1$$Register, $op2$$Register); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2) +%{ + match(Set cr (OverflowAddI op1 op2)); + effect(DEF cr, USE_KILL op1, USE op2); + + format %{ "addl $op1, $op2\t# overflow check int" %} + + ins_encode %{ + __ addl($op1$$Register, $op2$$constant); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2) +%{ + match(Set cr (OverflowAddL op1 op2)); + effect(DEF cr, USE_KILL op1, USE op2); + + format %{ "addq $op1, $op2\t# overflow check long" %} + ins_encode %{ + __ addq($op1$$Register, $op2$$Register); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2) +%{ + match(Set cr (OverflowAddL op1 op2)); + effect(DEF cr, USE_KILL op1, USE op2); + + format %{ "addq $op1, $op2\t# overflow check long" %} + ins_encode %{ + __ addq($op1$$Register, $op2$$constant); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2) +%{ + match(Set cr (OverflowSubI op1 op2)); + + format %{ "cmpl $op1, $op2\t# overflow check int" %} + ins_encode %{ + __ cmpl($op1$$Register, $op2$$Register); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2) +%{ + match(Set cr (OverflowSubI op1 op2)); + + format %{ "cmpl $op1, $op2\t# overflow check int" %} + ins_encode %{ + __ cmpl($op1$$Register, $op2$$constant); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2) +%{ + match(Set cr (OverflowSubL op1 op2)); + + format %{ "cmpq $op1, $op2\t# overflow check long" %} + ins_encode %{ + __ cmpq($op1$$Register, $op2$$Register); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2) +%{ + match(Set cr (OverflowSubL op1 op2)); + + format %{ "cmpq $op1, $op2\t# overflow check long" %} + ins_encode %{ + __ cmpq($op1$$Register, $op2$$constant); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2) +%{ + match(Set cr (OverflowSubI zero op2)); + effect(DEF cr, USE_KILL op2); + + format %{ "negl $op2\t# overflow check int" %} + ins_encode %{ + __ negl($op2$$Register); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2) +%{ + match(Set cr (OverflowSubL zero op2)); + effect(DEF cr, USE_KILL op2); + + format %{ "negq $op2\t# overflow check long" %} + ins_encode %{ + __ negq($op2$$Register); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2) +%{ + match(Set cr (OverflowMulI op1 op2)); + effect(DEF cr, USE_KILL op1, USE op2); + + format %{ "imull $op1, $op2\t# overflow check int" %} + ins_encode %{ + __ imull($op1$$Register, $op2$$Register); + %} + ins_pipe(ialu_reg_reg_alu0); +%} + +instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp) +%{ + match(Set cr (OverflowMulI op1 op2)); + effect(DEF cr, TEMP tmp, USE op1, USE op2); + + format %{ "imull $tmp, $op1, $op2\t# overflow check int" %} + ins_encode %{ + __ imull($tmp$$Register, $op1$$Register, $op2$$constant); + %} + ins_pipe(ialu_reg_reg_alu0); +%} + +instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2) +%{ + match(Set cr (OverflowMulL op1 op2)); + effect(DEF cr, USE_KILL op1, USE op2); + + format %{ "imulq $op1, $op2\t# overflow check long" %} + ins_encode %{ + __ imulq($op1$$Register, $op2$$Register); + %} + ins_pipe(ialu_reg_reg_alu0); +%} + +instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp) +%{ + match(Set cr (OverflowMulL op1 op2)); + effect(DEF cr, TEMP tmp, USE op1, USE op2); + + format %{ "imulq $tmp, $op1, $op2\t# overflow check long" %} + ins_encode %{ + __ imulq($tmp$$Register, $op1$$Register, $op2$$constant); + %} + ins_pipe(ialu_reg_reg_alu0); +%} + //----------Control Flow Instructions------------------------------------------ // Signed compare Instructions diff -r 7ce7247df891 -r f0ea4d3df129 src/os/linux/vm/os_linux.cpp --- a/src/os/linux/vm/os_linux.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/os/linux/vm/os_linux.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -109,6 +109,8 @@ #define MAX_PATH (2 * K) +#define MAX_SECS 100000000 + // for timer info max values which include all bits #define ALL_64_BITS CONST64(0xFFFFFFFFFFFFFFFF) @@ -2471,7 +2473,6 @@ sem_t _semaphore; }; - Semaphore::Semaphore() { sem_init(&_semaphore, 0, 0); } @@ -2493,8 +2494,22 @@ } bool Semaphore::timedwait(unsigned int sec, int nsec) { + struct timespec ts; - unpackTime(&ts, false, (sec * NANOSECS_PER_SEC) + nsec); + // Semaphore's are always associated with CLOCK_REALTIME + os::Linux::clock_gettime(CLOCK_REALTIME, &ts); + // see unpackTime for discussion on overflow checking + if (sec >= MAX_SECS) { + ts.tv_sec += MAX_SECS; + ts.tv_nsec = 0; + } else { + ts.tv_sec += sec; + ts.tv_nsec += nsec; + if (ts.tv_nsec >= NANOSECS_PER_SEC) { + ts.tv_nsec -= NANOSECS_PER_SEC; + ++ts.tv_sec; // note: this must be <= max_secs + } + } while (1) { int result = sem_timedwait(&_semaphore, &ts); @@ -5808,7 +5823,6 @@ * is no need to track notifications. */ -#define MAX_SECS 100000000 /* * This code is common to linux and solaris and will be moved to a * common place in dolphin. diff -r 7ce7247df891 -r f0ea4d3df129 src/os/windows/vm/os_windows.cpp --- a/src/os/windows/vm/os_windows.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/os/windows/vm/os_windows.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -2437,6 +2437,12 @@ } } + if ((exception_code == EXCEPTION_ACCESS_VIOLATION) && + VM_Version::is_cpuinfo_segv_addr(pc)) { + // Verify that OS save/restore AVX registers. + return Handle_Exception(exceptionInfo, VM_Version::cpuinfo_cont_addr()); + } + if (t != NULL && t->is_Java_thread()) { JavaThread* thread = (JavaThread*) t; bool in_java = thread->thread_state() == _thread_in_Java; diff -r 7ce7247df891 -r f0ea4d3df129 src/os_cpu/bsd_x86/vm/os_bsd_x86.cpp --- a/src/os_cpu/bsd_x86/vm/os_bsd_x86.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/os_cpu/bsd_x86/vm/os_bsd_x86.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -492,6 +492,11 @@ } } + if ((sig == SIGSEGV || sig == SIGBUS) && VM_Version::is_cpuinfo_segv_addr(pc)) { + // Verify that OS save/restore AVX registers. + stub = VM_Version::cpuinfo_cont_addr(); + } + // We test if stub is already set (by the stack overflow code // above) so it is not overwritten by the code that follows. This // check is not required on other platforms, because on other diff -r 7ce7247df891 -r f0ea4d3df129 src/os_cpu/linux_x86/vm/os_linux_x86.cpp --- a/src/os_cpu/linux_x86/vm/os_linux_x86.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/os_cpu/linux_x86/vm/os_linux_x86.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -337,6 +337,11 @@ } } + if ((sig == SIGSEGV) && VM_Version::is_cpuinfo_segv_addr(pc)) { + // Verify that OS save/restore AVX registers. + stub = VM_Version::cpuinfo_cont_addr(); + } + if (thread->thread_state() == _thread_in_Java) { // Java thread running in Java code => find exception handler if any // a fault inside compiled code, the interpreter, or a stub diff -r 7ce7247df891 -r f0ea4d3df129 src/os_cpu/solaris_x86/vm/os_solaris_x86.cpp --- a/src/os_cpu/solaris_x86/vm/os_solaris_x86.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/os_cpu/solaris_x86/vm/os_solaris_x86.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -459,6 +459,11 @@ } } + if ((sig == SIGSEGV) && VM_Version::is_cpuinfo_segv_addr(pc)) { + // Verify that OS save/restore AVX registers. + stub = VM_Version::cpuinfo_cont_addr(); + } + if (thread->thread_state() == _thread_in_vm) { if (sig == SIGBUS && info->si_code == BUS_OBJERR && thread->doing_unsafe_access()) { stub = StubRoutines::handler_for_unsafe_access(); diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/adlc/archDesc.cpp --- a/src/share/vm/adlc/archDesc.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/adlc/archDesc.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -1192,15 +1192,12 @@ || strcmp(idealName,"CmpF") == 0 || strcmp(idealName,"FastLock") == 0 || strcmp(idealName,"FastUnlock") == 0 - || strcmp(idealName,"AddExactI") == 0 - || strcmp(idealName,"AddExactL") == 0 - || strcmp(idealName,"SubExactI") == 0 - || strcmp(idealName,"SubExactL") == 0 - || strcmp(idealName,"MulExactI") == 0 - || strcmp(idealName,"MulExactL") == 0 - || strcmp(idealName,"NegExactI") == 0 - || strcmp(idealName,"NegExactL") == 0 - || strcmp(idealName,"FlagsProj") == 0 + || strcmp(idealName,"OverflowAddI") == 0 + || strcmp(idealName,"OverflowAddL") == 0 + || strcmp(idealName,"OverflowSubI") == 0 + || strcmp(idealName,"OverflowSubL") == 0 + || strcmp(idealName,"OverflowMulI") == 0 + || strcmp(idealName,"OverflowMulL") == 0 || strcmp(idealName,"Bool") == 0 || strcmp(idealName,"Binary") == 0 ) { // Removed ConI from the must_clone list. CPUs that cannot use diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/adlc/formssel.cpp --- a/src/share/vm/adlc/formssel.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/adlc/formssel.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -649,6 +649,7 @@ int USE_of_memory = 0; int DEF_of_memory = 0; const char* last_memory_DEF = NULL; // to test DEF/USE pairing in asserts + const char* last_memory_USE = NULL; Component *unique = NULL; Component *comp = NULL; ComponentList &components = (ComponentList &)_components; @@ -670,7 +671,16 @@ assert(0 == strcmp(last_memory_DEF, comp->_name), "every memory DEF is followed by a USE of the same name"); last_memory_DEF = NULL; } - USE_of_memory++; + // Handles same memory being used multiple times in the case of BMI1 instructions. + if (last_memory_USE != NULL) { + if (strcmp(comp->_name, last_memory_USE) != 0) { + USE_of_memory++; + } + } else { + USE_of_memory++; + } + last_memory_USE = comp->_name; + if (DEF_of_memory == 0) // defs take precedence unique = comp; } else { diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/ci/ciClassList.hpp --- a/src/share/vm/ci/ciClassList.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/ci/ciClassList.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -103,6 +103,7 @@ friend class ciMethodType; \ friend class ciReceiverTypeData; \ friend class ciTypeEntries; \ +friend class ciSpeculativeTrapData; \ friend class ciSymbol; \ friend class ciArray; \ friend class ciObjArray; \ diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/ci/ciMethodData.cpp --- a/src/share/vm/ci/ciMethodData.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/ci/ciMethodData.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -78,6 +78,36 @@ _parameters = NULL; } +void ciMethodData::load_extra_data() { + MethodData* mdo = get_MethodData(); + + // speculative trap entries also hold a pointer to a Method so need to be translated + DataLayout* dp_src = mdo->extra_data_base(); + DataLayout* end_src = mdo->extra_data_limit(); + DataLayout* dp_dst = extra_data_base(); + for (;; dp_src = MethodData::next_extra(dp_src), dp_dst = MethodData::next_extra(dp_dst)) { + assert(dp_src < end_src, "moved past end of extra data"); + // New traps in the MDO can be added as we translate the copy so + // look at the entries in the copy. + switch(dp_dst->tag()) { + case DataLayout::speculative_trap_data_tag: { + ciSpeculativeTrapData* data_dst = new ciSpeculativeTrapData(dp_dst); + SpeculativeTrapData* data_src = new SpeculativeTrapData(dp_src); + data_dst->translate_from(data_src); + break; + } + case DataLayout::bit_data_tag: + break; + case DataLayout::no_tag: + case DataLayout::arg_info_data_tag: + // An empty slot or ArgInfoData entry marks the end of the trap data + return; + default: + fatal(err_msg("bad tag = %d", dp_dst->tag())); + } + } +} + void ciMethodData::load_data() { MethodData* mdo = get_MethodData(); if (mdo == NULL) { @@ -116,6 +146,8 @@ parameters->translate_from(mdo->parameters_type_data()); } + load_extra_data(); + // Note: Extra data are all BitData, and do not need translation. _current_mileage = MethodData::mileage_of(mdo->method()); _invocation_counter = mdo->invocation_count(); @@ -156,6 +188,12 @@ set_type(translate_klass(k)); } +void ciSpeculativeTrapData::translate_from(const ProfileData* data) { + Method* m = data->as_SpeculativeTrapData()->method(); + ciMethod* ci_m = CURRENT_ENV->get_method(m); + set_method(ci_m); +} + // Get the data at an arbitrary (sort of) data index. ciProfileData* ciMethodData::data_at(int data_index) { if (out_of_bounds(data_index)) { @@ -203,32 +241,64 @@ return next; } -// Translate a bci to its corresponding data, or NULL. -ciProfileData* ciMethodData::bci_to_data(int bci) { - ciProfileData* data = data_before(bci); - for ( ; is_valid(data); data = next_data(data)) { - if (data->bci() == bci) { - set_hint_di(dp_to_di(data->dp())); - return data; - } else if (data->bci() > bci) { - break; - } - } +ciProfileData* ciMethodData::bci_to_extra_data(int bci, ciMethod* m, bool& two_free_slots) { // bci_to_extra_data(bci) ... DataLayout* dp = data_layout_at(data_size()); DataLayout* end = data_layout_at(data_size() + extra_data_size()); - for (; dp < end; dp = MethodData::next_extra(dp)) { - if (dp->tag() == DataLayout::no_tag) { + two_free_slots = false; + for (;dp < end; dp = MethodData::next_extra(dp)) { + switch(dp->tag()) { + case DataLayout::no_tag: _saw_free_extra_data = true; // observed an empty slot (common case) + two_free_slots = (MethodData::next_extra(dp)->tag() == DataLayout::no_tag); return NULL; + case DataLayout::arg_info_data_tag: + return NULL; // ArgInfoData is at the end of extra data section. + case DataLayout::bit_data_tag: + if (m == NULL && dp->bci() == bci) { + return new ciBitData(dp); + } + break; + case DataLayout::speculative_trap_data_tag: { + ciSpeculativeTrapData* data = new ciSpeculativeTrapData(dp); + // data->method() might be null if the MDO is snapshotted + // concurrently with a trap + if (m != NULL && data->method() == m && dp->bci() == bci) { + return data; + } + break; + } + default: + fatal(err_msg("bad tag = %d", dp->tag())); } - if (dp->tag() == DataLayout::arg_info_data_tag) { - break; // ArgInfoData is at the end of extra data section. + } + return NULL; +} + +// Translate a bci to its corresponding data, or NULL. +ciProfileData* ciMethodData::bci_to_data(int bci, ciMethod* m) { + // If m is not NULL we look for a SpeculativeTrapData entry + if (m == NULL) { + ciProfileData* data = data_before(bci); + for ( ; is_valid(data); data = next_data(data)) { + if (data->bci() == bci) { + set_hint_di(dp_to_di(data->dp())); + return data; + } else if (data->bci() > bci) { + break; + } } - if (dp->bci() == bci) { - assert(dp->tag() == DataLayout::bit_data_tag, "sane"); - return new ciBitData(dp); - } + } + bool two_free_slots = false; + ciProfileData* result = bci_to_extra_data(bci, m, two_free_slots); + if (result != NULL) { + return result; + } + if (m != NULL && !two_free_slots) { + // We were looking for a SpeculativeTrapData entry we didn't + // find. Room is not available for more SpeculativeTrapData + // entries, look in the non SpeculativeTrapData entries. + return bci_to_data(bci, NULL); } return NULL; } @@ -525,18 +595,25 @@ st->print_cr("--- Extra data:"); DataLayout* dp = data_layout_at(data_size()); DataLayout* end = data_layout_at(data_size() + extra_data_size()); - for (; dp < end; dp = MethodData::next_extra(dp)) { - if (dp->tag() == DataLayout::no_tag) continue; - if (dp->tag() == DataLayout::bit_data_tag) { + for (;; dp = MethodData::next_extra(dp)) { + assert(dp < end, "moved past end of extra data"); + switch (dp->tag()) { + case DataLayout::no_tag: + continue; + case DataLayout::bit_data_tag: data = new BitData(dp); - } else { - assert(dp->tag() == DataLayout::arg_info_data_tag, "must be BitData or ArgInfo"); + break; + case DataLayout::arg_info_data_tag: data = new ciArgInfoData(dp); dp = end; // ArgInfoData is at the end of extra data section. + break; + default: + fatal(err_msg("unexpected tag %d", dp->tag())); } st->print("%d", dp_to_di(data->dp())); st->fill_to(6); data->print_data_on(st); + if (dp >= end) return; } } @@ -569,8 +646,8 @@ st->cr(); } -void ciCallTypeData::print_data_on(outputStream* st) const { - print_shared(st, "ciCallTypeData"); +void ciCallTypeData::print_data_on(outputStream* st, const char* extra) const { + print_shared(st, "ciCallTypeData", extra); if (has_arguments()) { tab(st, true); st->print("argument types"); @@ -599,18 +676,18 @@ } } -void ciReceiverTypeData::print_data_on(outputStream* st) const { - print_shared(st, "ciReceiverTypeData"); +void ciReceiverTypeData::print_data_on(outputStream* st, const char* extra) const { + print_shared(st, "ciReceiverTypeData", extra); print_receiver_data_on(st); } -void ciVirtualCallData::print_data_on(outputStream* st) const { - print_shared(st, "ciVirtualCallData"); +void ciVirtualCallData::print_data_on(outputStream* st, const char* extra) const { + print_shared(st, "ciVirtualCallData", extra); rtd_super()->print_receiver_data_on(st); } -void ciVirtualCallTypeData::print_data_on(outputStream* st) const { - print_shared(st, "ciVirtualCallTypeData"); +void ciVirtualCallTypeData::print_data_on(outputStream* st, const char* extra) const { + print_shared(st, "ciVirtualCallTypeData", extra); rtd_super()->print_receiver_data_on(st); if (has_arguments()) { tab(st, true); @@ -624,8 +701,15 @@ } } -void ciParametersTypeData::print_data_on(outputStream* st) const { - st->print_cr("Parametertypes"); +void ciParametersTypeData::print_data_on(outputStream* st, const char* extra) const { + st->print_cr("ciParametersTypeData"); parameters()->print_data_on(st); } + +void ciSpeculativeTrapData::print_data_on(outputStream* st, const char* extra) const { + st->print_cr("ciSpeculativeTrapData"); + tab(st); + method()->print_short_name(st); + st->cr(); +} #endif diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/ci/ciMethodData.hpp --- a/src/share/vm/ci/ciMethodData.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/ci/ciMethodData.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -31,6 +31,7 @@ #include "ci/ciUtilities.hpp" #include "oops/methodData.hpp" #include "oops/oop.inline.hpp" +#include "runtime/deoptimization.hpp" class ciBitData; class ciCounterData; @@ -44,6 +45,7 @@ class ciCallTypeData; class ciVirtualCallTypeData; class ciParametersTypeData; +class ciSpeculativeTrapData;; typedef ProfileData ciProfileData; @@ -173,7 +175,7 @@ } #ifndef PRODUCT - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, const char* extra) const; #endif }; @@ -200,7 +202,7 @@ } void translate_receiver_data_from(const ProfileData* data); #ifndef PRODUCT - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, const char* extra) const; void print_receiver_data_on(outputStream* st) const; #endif }; @@ -225,7 +227,7 @@ rtd_super()->translate_receiver_data_from(data); } #ifndef PRODUCT - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, const char* extra) const; #endif }; @@ -287,7 +289,7 @@ } #ifndef PRODUCT - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, const char* extra) const; #endif }; @@ -336,7 +338,26 @@ } #ifndef PRODUCT - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, const char* extra) const; +#endif +}; + +class ciSpeculativeTrapData : public SpeculativeTrapData { +public: + ciSpeculativeTrapData(DataLayout* layout) : SpeculativeTrapData(layout) {} + + virtual void translate_from(const ProfileData* data); + + ciMethod* method() const { + return (ciMethod*)intptr_at(method_offset); + } + + void set_method(ciMethod* m) { + set_intptr_at(method_offset, (intptr_t)m); + } + +#ifndef PRODUCT + void print_data_on(outputStream* st, const char* extra) const; #endif }; @@ -436,6 +457,16 @@ ciArgInfoData *arg_info() const; + address data_base() const { + return (address) _data; + } + DataLayout* limit_data_position() const { + return (DataLayout*)((address)data_base() + _data_size); + } + + void load_extra_data(); + ciProfileData* bci_to_extra_data(int bci, ciMethod* m, bool& two_free_slots); + public: bool is_method_data() const { return true; } @@ -475,9 +506,11 @@ ciProfileData* next_data(ciProfileData* current); bool is_valid(ciProfileData* current) { return current != NULL; } - // Get the data at an arbitrary bci, or NULL if there is none. - ciProfileData* bci_to_data(int bci); - ciProfileData* bci_to_extra_data(int bci, bool create_if_missing); + DataLayout* extra_data_base() const { return limit_data_position(); } + + // Get the data at an arbitrary bci, or NULL if there is none. If m + // is not NULL look for a SpeculativeTrapData if any first. + ciProfileData* bci_to_data(int bci, ciMethod* m = NULL); uint overflow_trap_count() const { return _orig.overflow_trap_count(); @@ -496,12 +529,13 @@ // Helpful query functions that decode trap_state. int has_trap_at(ciProfileData* data, int reason); - int has_trap_at(int bci, int reason) { - return has_trap_at(bci_to_data(bci), reason); + int has_trap_at(int bci, ciMethod* m, int reason) { + assert((m != NULL) == Deoptimization::reason_is_speculate(reason), "inconsistent method/reason"); + return has_trap_at(bci_to_data(bci, m), reason); } int trap_recompiled_at(ciProfileData* data); - int trap_recompiled_at(int bci) { - return trap_recompiled_at(bci_to_data(bci)); + int trap_recompiled_at(int bci, ciMethod* m) { + return trap_recompiled_at(bci_to_data(bci, m)); } void clear_escape_info(); diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/g1/concurrentMark.cpp --- a/src/share/vm/gc_implementation/g1/concurrentMark.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -1809,8 +1809,8 @@ uint _regions_claimed; size_t _freed_bytes; FreeRegionList* _local_cleanup_list; - OldRegionSet* _old_proxy_set; - HumongousRegionSet* _humongous_proxy_set; + HeapRegionSetCount _old_regions_removed; + HeapRegionSetCount _humongous_regions_removed; HRRSCleanupTask* _hrrs_cleanup_task; double _claimed_region_time; double _max_region_time; @@ -1819,19 +1819,19 @@ G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1, int worker_num, FreeRegionList* local_cleanup_list, - OldRegionSet* old_proxy_set, - HumongousRegionSet* humongous_proxy_set, HRRSCleanupTask* hrrs_cleanup_task) : _g1(g1), _worker_num(worker_num), _max_live_bytes(0), _regions_claimed(0), _freed_bytes(0), _claimed_region_time(0.0), _max_region_time(0.0), _local_cleanup_list(local_cleanup_list), - _old_proxy_set(old_proxy_set), - _humongous_proxy_set(humongous_proxy_set), + _old_regions_removed(), + _humongous_regions_removed(), _hrrs_cleanup_task(hrrs_cleanup_task) { } size_t freed_bytes() { return _freed_bytes; } + const HeapRegionSetCount& old_regions_removed() { return _old_regions_removed; } + const HeapRegionSetCount& humongous_regions_removed() { return _humongous_regions_removed; } bool doHeapRegion(HeapRegion *hr) { if (hr->continuesHumongous()) { @@ -1844,13 +1844,22 @@ _regions_claimed++; hr->note_end_of_marking(); _max_live_bytes += hr->max_live_bytes(); - _g1->free_region_if_empty(hr, - &_freed_bytes, - _local_cleanup_list, - _old_proxy_set, - _humongous_proxy_set, - _hrrs_cleanup_task, - true /* par */); + + if (hr->used() > 0 && hr->max_live_bytes() == 0 && !hr->is_young()) { + _freed_bytes += hr->used(); + hr->set_containing_set(NULL); + if (hr->isHumongous()) { + assert(hr->startsHumongous(), "we should only see starts humongous"); + _humongous_regions_removed.increment(1u, hr->capacity()); + _g1->free_humongous_region(hr, _local_cleanup_list, true); + } else { + _old_regions_removed.increment(1u, hr->capacity()); + _g1->free_region(hr, _local_cleanup_list, true); + } + } else { + hr->rem_set()->do_cleanup_work(_hrrs_cleanup_task); + } + double region_time = (os::elapsedTime() - start); _claimed_region_time += region_time; if (region_time > _max_region_time) { @@ -1883,12 +1892,8 @@ void work(uint worker_id) { double start = os::elapsedTime(); FreeRegionList local_cleanup_list("Local Cleanup List"); - OldRegionSet old_proxy_set("Local Cleanup Old Proxy Set"); - HumongousRegionSet humongous_proxy_set("Local Cleanup Humongous Proxy Set"); HRRSCleanupTask hrrs_cleanup_task; G1NoteEndOfConcMarkClosure g1_note_end(_g1h, worker_id, &local_cleanup_list, - &old_proxy_set, - &humongous_proxy_set, &hrrs_cleanup_task); if (G1CollectedHeap::use_parallel_gc_threads()) { _g1h->heap_region_par_iterate_chunked(&g1_note_end, worker_id, @@ -1900,13 +1905,10 @@ assert(g1_note_end.complete(), "Shouldn't have yielded!"); // Now update the lists - _g1h->update_sets_after_freeing_regions(g1_note_end.freed_bytes(), - NULL /* free_list */, - &old_proxy_set, - &humongous_proxy_set, - true /* par */); + _g1h->remove_from_old_sets(g1_note_end.old_regions_removed(), g1_note_end.humongous_regions_removed()); { MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); + _g1h->decrement_summary_bytes(g1_note_end.freed_bytes()); _max_live_bytes += g1_note_end.max_live_bytes(); _freed_bytes += g1_note_end.freed_bytes(); @@ -1920,7 +1922,7 @@ G1HRPrinter* hr_printer = _g1h->hr_printer(); if (hr_printer->is_active()) { - HeapRegionLinkedListIterator iter(&local_cleanup_list); + FreeRegionListIterator iter(&local_cleanup_list); while (iter.more_available()) { HeapRegion* hr = iter.get_next(); hr_printer->cleanup(hr); @@ -1971,7 +1973,6 @@ return; } - HRSPhaseSetter x(HRSPhaseCleanup); g1h->verify_region_sets_optional(); if (VerifyDuringGC) { @@ -2144,7 +2145,7 @@ G1CollectedHeap* g1h = G1CollectedHeap::heap(); - _cleanup_list.verify_optional(); + _cleanup_list.verify_list(); FreeRegionList tmp_free_list("Tmp Free List"); if (G1ConcRegionFreeingVerbose) { diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/g1/concurrentMark.hpp --- a/src/share/vm/gc_implementation/g1/concurrentMark.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -25,7 +25,7 @@ #ifndef SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_HPP #define SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_HPP -#include "gc_implementation/g1/heapRegionSets.hpp" +#include "gc_implementation/g1/heapRegionSet.hpp" #include "utilities/taskqueue.hpp" class G1CollectedHeap; diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -1298,7 +1298,6 @@ size_t metadata_prev_used = MetaspaceAux::allocated_used_bytes(); - HRSPhaseSetter x(HRSPhaseFullGC); verify_region_sets_optional(); const bool do_clear_all_soft_refs = clear_all_soft_refs || @@ -1928,10 +1927,10 @@ _g1mm(NULL), _refine_cte_cl(NULL), _full_collection(false), - _free_list("Master Free List"), - _secondary_free_list("Secondary Free List"), - _old_set("Old Set"), - _humongous_set("Master Humongous Set"), + _free_list("Master Free List", new MasterFreeRegionListMtSafeChecker()), + _secondary_free_list("Secondary Free List", new SecondaryFreeRegionListMtSafeChecker()), + _old_set("Old Set", false /* humongous */, new OldRegionSetMtSafeChecker()), + _humongous_set("Master Humongous Set", true /* humongous */, new HumongousRegionSetMtSafeChecker()), _free_regions_coming(false), _young_list(new YoungList(this)), _gc_time_stamp(0), @@ -2079,7 +2078,7 @@ guarantee(HeapRegion::CardsPerRegion < max_cards_per_region, "too many cards per region"); - HeapRegionSet::set_unrealistically_long_length(max_regions() + 1); + FreeRegionList::set_unrealistically_long_length(max_regions() + 1); _bot_shared = new G1BlockOffsetSharedArray(_reserved, heap_word_size(init_byte_size)); @@ -3013,7 +3012,17 @@ } size_t G1CollectedHeap::tlab_capacity(Thread* ignored) const { - return HeapRegion::GrainBytes; + return (_g1_policy->young_list_target_length() - young_list()->survivor_length()) * HeapRegion::GrainBytes; +} + +size_t G1CollectedHeap::tlab_used(Thread* ignored) const { + return young_list()->eden_used_bytes(); +} + +// For G1 TLABs should not contain humongous objects, so the maximum TLAB size +// must be smaller than the humongous object limit. +size_t G1CollectedHeap::max_tlab_size() const { + return align_size_down(_humongous_object_threshold_in_words - 1, MinObjAlignment); } size_t G1CollectedHeap::unsafe_max_tlab_alloc(Thread* ignored) const { @@ -3025,11 +3034,11 @@ // humongous objects. HeapRegion* hr = _mutator_alloc_region.get(); - size_t max_tlab_size = _humongous_object_threshold_in_words * wordSize; + size_t max_tlab = max_tlab_size() * wordSize; if (hr == NULL) { - return max_tlab_size; + return max_tlab; } else { - return MIN2(MAX2(hr->free(), (size_t) MinTLABSize), max_tlab_size); + return MIN2(MAX2(hr->free(), (size_t) MinTLABSize), max_tlab); } } @@ -3668,6 +3677,7 @@ // always_do_update_barrier = false; assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer"); // Fill TLAB's and such + accumulate_statistics_all_tlabs(); ensure_parsability(true); if (G1SummarizeRSetStats && (G1SummarizeRSetStatsPeriod > 0) && @@ -3692,6 +3702,8 @@ "derived pointer present")); // always_do_update_barrier = true; + resize_all_tlabs(); + // We have just completed a GC. Update the soft reference // policy with the new heap occupancy Universe::update_heap_info_at_gc(); @@ -3892,7 +3904,6 @@ print_heap_before_gc(); trace_heap_before_gc(_gc_tracer_stw); - HRSPhaseSetter x(HRSPhaseEvacuation); verify_region_sets_optional(); verify_dirty_young_regions(); @@ -5954,28 +5965,7 @@ COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); } -void G1CollectedHeap::free_region_if_empty(HeapRegion* hr, - size_t* pre_used, - FreeRegionList* free_list, - OldRegionSet* old_proxy_set, - HumongousRegionSet* humongous_proxy_set, - HRRSCleanupTask* hrrs_cleanup_task, - bool par) { - if (hr->used() > 0 && hr->max_live_bytes() == 0 && !hr->is_young()) { - if (hr->isHumongous()) { - assert(hr->startsHumongous(), "we should only see starts humongous"); - free_humongous_region(hr, pre_used, free_list, humongous_proxy_set, par); - } else { - _old_set.remove_with_proxy(hr, old_proxy_set); - free_region(hr, pre_used, free_list, par); - } - } else { - hr->rem_set()->do_cleanup_work(hrrs_cleanup_task); - } -} - void G1CollectedHeap::free_region(HeapRegion* hr, - size_t* pre_used, FreeRegionList* free_list, bool par) { assert(!hr->isHumongous(), "this is only for non-humongous regions"); @@ -5988,70 +5978,56 @@ if (!hr->is_young()) { _cg1r->hot_card_cache()->reset_card_counts(hr); } - *pre_used += hr->used(); hr->hr_clear(par, true /* clear_space */); free_list->add_as_head(hr); } void G1CollectedHeap::free_humongous_region(HeapRegion* hr, - size_t* pre_used, FreeRegionList* free_list, - HumongousRegionSet* humongous_proxy_set, bool par) { assert(hr->startsHumongous(), "this is only for starts humongous regions"); assert(free_list != NULL, "pre-condition"); - assert(humongous_proxy_set != NULL, "pre-condition"); - - size_t hr_used = hr->used(); + size_t hr_capacity = hr->capacity(); - size_t hr_pre_used = 0; - _humongous_set.remove_with_proxy(hr, humongous_proxy_set); // We need to read this before we make the region non-humongous, // otherwise the information will be gone. uint last_index = hr->last_hc_index(); hr->set_notHumongous(); - free_region(hr, &hr_pre_used, free_list, par); + free_region(hr, free_list, par); uint i = hr->hrs_index() + 1; while (i < last_index) { HeapRegion* curr_hr = region_at(i); assert(curr_hr->continuesHumongous(), "invariant"); curr_hr->set_notHumongous(); - free_region(curr_hr, &hr_pre_used, free_list, par); + free_region(curr_hr, free_list, par); i += 1; } - assert(hr_pre_used == hr_used, - err_msg("hr_pre_used: "SIZE_FORMAT" and hr_used: "SIZE_FORMAT" " - "should be the same", hr_pre_used, hr_used)); - *pre_used += hr_pre_used; -} - -void G1CollectedHeap::update_sets_after_freeing_regions(size_t pre_used, - FreeRegionList* free_list, - OldRegionSet* old_proxy_set, - HumongousRegionSet* humongous_proxy_set, - bool par) { - if (pre_used > 0) { - Mutex* lock = (par) ? ParGCRareEvent_lock : NULL; - MutexLockerEx x(lock, Mutex::_no_safepoint_check_flag); - assert(_summary_bytes_used >= pre_used, - err_msg("invariant: _summary_bytes_used: "SIZE_FORMAT" " - "should be >= pre_used: "SIZE_FORMAT, - _summary_bytes_used, pre_used)); - _summary_bytes_used -= pre_used; - } - if (free_list != NULL && !free_list->is_empty()) { +} + +void G1CollectedHeap::remove_from_old_sets(const HeapRegionSetCount& old_regions_removed, + const HeapRegionSetCount& humongous_regions_removed) { + if (old_regions_removed.length() > 0 || humongous_regions_removed.length() > 0) { + MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag); + _old_set.bulk_remove(old_regions_removed); + _humongous_set.bulk_remove(humongous_regions_removed); + } + +} + +void G1CollectedHeap::prepend_to_freelist(FreeRegionList* list) { + assert(list != NULL, "list can't be null"); + if (!list->is_empty()) { MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag); - _free_list.add_as_head(free_list); - } - if (old_proxy_set != NULL && !old_proxy_set->is_empty()) { - MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag); - _old_set.update_from_proxy(old_proxy_set); - } - if (humongous_proxy_set != NULL && !humongous_proxy_set->is_empty()) { - MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag); - _humongous_set.update_from_proxy(humongous_proxy_set); - } + _free_list.add_as_head(list); + } +} + +void G1CollectedHeap::decrement_summary_bytes(size_t bytes) { + assert(_summary_bytes_used >= bytes, + err_msg("invariant: _summary_bytes_used: "SIZE_FORMAT" should be >= bytes: "SIZE_FORMAT, + _summary_bytes_used, bytes)); + _summary_bytes_used -= bytes; } class G1ParCleanupCTTask : public AbstractGangTask { @@ -6244,7 +6220,8 @@ // And the region is empty. assert(!used_mr.is_empty(), "Should not have empty regions in a CS."); - free_region(cur, &pre_used, &local_free_list, false /* par */); + pre_used += cur->used(); + free_region(cur, &local_free_list, false /* par */); } else { cur->uninstall_surv_rate_group(); if (cur->is_young()) { @@ -6272,10 +6249,8 @@ young_time_ms += elapsed_ms; } - update_sets_after_freeing_regions(pre_used, &local_free_list, - NULL /* old_proxy_set */, - NULL /* humongous_proxy_set */, - false /* par */); + prepend_to_freelist(&local_free_list); + decrement_summary_bytes(pre_used); policy->phase_times()->record_young_free_cset_time_ms(young_time_ms); policy->phase_times()->record_non_young_free_cset_time_ms(non_young_time_ms); } @@ -6387,10 +6362,10 @@ class TearDownRegionSetsClosure : public HeapRegionClosure { private: - OldRegionSet *_old_set; + HeapRegionSet *_old_set; public: - TearDownRegionSetsClosure(OldRegionSet* old_set) : _old_set(old_set) { } + TearDownRegionSetsClosure(HeapRegionSet* old_set) : _old_set(old_set) { } bool doHeapRegion(HeapRegion* r) { if (r->is_empty()) { @@ -6429,13 +6404,13 @@ class RebuildRegionSetsClosure : public HeapRegionClosure { private: bool _free_list_only; - OldRegionSet* _old_set; + HeapRegionSet* _old_set; FreeRegionList* _free_list; size_t _total_used; public: RebuildRegionSetsClosure(bool free_list_only, - OldRegionSet* old_set, FreeRegionList* free_list) : + HeapRegionSet* old_set, FreeRegionList* free_list) : _free_list_only(free_list_only), _old_set(old_set), _free_list(free_list), _total_used(0) { assert(_free_list->is_empty(), "pre-condition"); @@ -6632,23 +6607,22 @@ class VerifyRegionListsClosure : public HeapRegionClosure { private: - FreeRegionList* _free_list; - OldRegionSet* _old_set; - HumongousRegionSet* _humongous_set; - uint _region_count; + HeapRegionSet* _old_set; + HeapRegionSet* _humongous_set; + FreeRegionList* _free_list; public: - VerifyRegionListsClosure(OldRegionSet* old_set, - HumongousRegionSet* humongous_set, + HeapRegionSetCount _old_count; + HeapRegionSetCount _humongous_count; + HeapRegionSetCount _free_count; + + VerifyRegionListsClosure(HeapRegionSet* old_set, + HeapRegionSet* humongous_set, FreeRegionList* free_list) : - _old_set(old_set), _humongous_set(humongous_set), - _free_list(free_list), _region_count(0) { } - - uint region_count() { return _region_count; } + _old_set(old_set), _humongous_set(humongous_set), _free_list(free_list), + _old_count(), _humongous_count(), _free_count(){ } bool doHeapRegion(HeapRegion* hr) { - _region_count += 1; - if (hr->continuesHumongous()) { return false; } @@ -6656,14 +6630,31 @@ if (hr->is_young()) { // TODO } else if (hr->startsHumongous()) { - _humongous_set->verify_next_region(hr); + assert(hr->containing_set() == _humongous_set, err_msg("Heap region %u is starts humongous but not in humongous set.", hr->region_num())); + _humongous_count.increment(1u, hr->capacity()); } else if (hr->is_empty()) { - _free_list->verify_next_region(hr); + assert(hr->containing_set() == _free_list, err_msg("Heap region %u is empty but not on the free list.", hr->region_num())); + _free_count.increment(1u, hr->capacity()); } else { - _old_set->verify_next_region(hr); + assert(hr->containing_set() == _old_set, err_msg("Heap region %u is old but not in the old set.", hr->region_num())); + _old_count.increment(1u, hr->capacity()); } return false; } + + void verify_counts(HeapRegionSet* old_set, HeapRegionSet* humongous_set, FreeRegionList* free_list) { + guarantee(old_set->length() == _old_count.length(), err_msg("Old set count mismatch. Expected %u, actual %u.", old_set->length(), _old_count.length())); + guarantee(old_set->total_capacity_bytes() == _old_count.capacity(), err_msg("Old set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, + old_set->total_capacity_bytes(), _old_count.capacity())); + + guarantee(humongous_set->length() == _humongous_count.length(), err_msg("Hum set count mismatch. Expected %u, actual %u.", humongous_set->length(), _humongous_count.length())); + guarantee(humongous_set->total_capacity_bytes() == _humongous_count.capacity(), err_msg("Hum set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, + humongous_set->total_capacity_bytes(), _humongous_count.capacity())); + + guarantee(free_list->length() == _free_count.length(), err_msg("Free list count mismatch. Expected %u, actual %u.", free_list->length(), _free_count.length())); + guarantee(free_list->total_capacity_bytes() == _free_count.capacity(), err_msg("Free list capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, + free_list->total_capacity_bytes(), _free_count.capacity())); + } }; HeapRegion* G1CollectedHeap::new_heap_region(uint hrs_index, @@ -6679,16 +6670,14 @@ assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); // First, check the explicit lists. - _free_list.verify(); + _free_list.verify_list(); { // Given that a concurrent operation might be adding regions to // the secondary free list we have to take the lock before // verifying it. MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); - _secondary_free_list.verify(); - } - _old_set.verify(); - _humongous_set.verify(); + _secondary_free_list.verify_list(); + } // If a concurrent region freeing operation is in progress it will // be difficult to correctly attributed any free regions we come @@ -6711,16 +6700,10 @@ // Finally, make sure that the region accounting in the lists is // consistent with what we see in the heap. - _old_set.verify_start(); - _humongous_set.verify_start(); - _free_list.verify_start(); VerifyRegionListsClosure cl(&_old_set, &_humongous_set, &_free_list); heap_region_iterate(&cl); - - _old_set.verify_end(); - _humongous_set.verify_end(); - _free_list.verify_end(); + cl.verify_counts(&_old_set, &_humongous_set, &_free_list); } // Optimized nmethod scanning diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -34,7 +34,7 @@ #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" #include "gc_implementation/g1/g1YCTypes.hpp" #include "gc_implementation/g1/heapRegionSeq.hpp" -#include "gc_implementation/g1/heapRegionSets.hpp" +#include "gc_implementation/g1/heapRegionSet.hpp" #include "gc_implementation/shared/hSpaceCounters.hpp" #include "gc_implementation/shared/parGCAllocBuffer.hpp" #include "memory/barrierSet.hpp" @@ -243,18 +243,18 @@ MemRegion _g1_committed; // The master free list. It will satisfy all new region allocations. - MasterFreeRegionList _free_list; + FreeRegionList _free_list; // The secondary free list which contains regions that have been // freed up during the cleanup process. This will be appended to the // master free list when appropriate. - SecondaryFreeRegionList _secondary_free_list; + FreeRegionList _secondary_free_list; // It keeps track of the old regions. - MasterOldRegionSet _old_set; + HeapRegionSet _old_set; // It keeps track of the humongous regions. - MasterHumongousRegionSet _humongous_set; + HeapRegionSet _humongous_set; // The number of regions we could create by expansion. uint _expansion_regions; @@ -757,6 +757,26 @@ G1HRPrinter* hr_printer() { return &_hr_printer; } + // Frees a non-humongous region by initializing its contents and + // adding it to the free list that's passed as a parameter (this is + // usually a local list which will be appended to the master free + // list later). The used bytes of freed regions are accumulated in + // pre_used. If par is true, the region's RSet will not be freed + // up. The assumption is that this will be done later. + void free_region(HeapRegion* hr, + FreeRegionList* free_list, + bool par); + + // Frees a humongous region by collapsing it into individual regions + // and calling free_region() for each of them. The freed regions + // will be added to the free list that's passed as a parameter (this + // is usually a local list which will be appended to the master free + // list later). The used bytes of freed regions are accumulated in + // pre_used. If par is true, the region's RSet will not be freed + // up. The assumption is that this will be done later. + void free_humongous_region(HeapRegion* hr, + FreeRegionList* free_list, + bool par); protected: // Shrink the garbage-first heap by at most the given size (in bytes!). @@ -840,30 +860,6 @@ // string table, and referents of reachable weak refs. void g1_process_weak_roots(OopClosure* root_closure); - // Frees a non-humongous region by initializing its contents and - // adding it to the free list that's passed as a parameter (this is - // usually a local list which will be appended to the master free - // list later). The used bytes of freed regions are accumulated in - // pre_used. If par is true, the region's RSet will not be freed - // up. The assumption is that this will be done later. - void free_region(HeapRegion* hr, - size_t* pre_used, - FreeRegionList* free_list, - bool par); - - // Frees a humongous region by collapsing it into individual regions - // and calling free_region() for each of them. The freed regions - // will be added to the free list that's passed as a parameter (this - // is usually a local list which will be appended to the master free - // list later). The used bytes of freed regions are accumulated in - // pre_used. If par is true, the region's RSet will not be freed - // up. The assumption is that this will be done later. - void free_humongous_region(HeapRegion* hr, - size_t* pre_used, - FreeRegionList* free_list, - HumongousRegionSet* humongous_proxy_set, - bool par); - // Notifies all the necessary spaces that the committed space has // been updated (either expanded or shrunk). It should be called // after _g1_storage is updated. @@ -1242,10 +1238,6 @@ bool is_on_master_free_list(HeapRegion* hr) { return hr->containing_set() == &_free_list; } - - bool is_in_humongous_set(HeapRegion* hr) { - return hr->containing_set() == &_humongous_set; - } #endif // ASSERT // Wrapper for the region list operations that can be called from @@ -1298,27 +1290,9 @@ // True iff an evacuation has failed in the most-recent collection. bool evacuation_failed() { return _evacuation_failed; } - // It will free a region if it has allocated objects in it that are - // all dead. It calls either free_region() or - // free_humongous_region() depending on the type of the region that - // is passed to it. - void free_region_if_empty(HeapRegion* hr, - size_t* pre_used, - FreeRegionList* free_list, - OldRegionSet* old_proxy_set, - HumongousRegionSet* humongous_proxy_set, - HRRSCleanupTask* hrrs_cleanup_task, - bool par); - - // It appends the free list to the master free list and updates the - // master humongous list according to the contents of the proxy - // list. It also adjusts the total used bytes according to pre_used - // (if par is true, it will do so by taking the ParGCRareEvent_lock). - void update_sets_after_freeing_regions(size_t pre_used, - FreeRegionList* free_list, - OldRegionSet* old_proxy_set, - HumongousRegionSet* humongous_proxy_set, - bool par); + void remove_from_old_sets(const HeapRegionSetCount& old_regions_removed, const HeapRegionSetCount& humongous_regions_removed); + void prepend_to_freelist(FreeRegionList* list); + void decrement_summary_bytes(size_t bytes); // Returns "TRUE" iff "p" points into the committed areas of the heap. virtual bool is_in(const void* p) const; @@ -1481,9 +1455,11 @@ // Section on thread-local allocation buffers (TLABs) // See CollectedHeap for semantics. - virtual bool supports_tlab_allocation() const; - virtual size_t tlab_capacity(Thread* thr) const; - virtual size_t unsafe_max_tlab_alloc(Thread* thr) const; + bool supports_tlab_allocation() const; + size_t tlab_capacity(Thread* ignored) const; + size_t tlab_used(Thread* ignored) const; + size_t max_tlab_size() const; + size_t unsafe_max_tlab_alloc(Thread* ignored) const; // Can a compiler initialize a new object without store barriers? // This permission only extends from the creation of a new object @@ -1568,7 +1544,7 @@ void set_region_short_lived_locked(HeapRegion* hr); // add appropriate methods for any other surv rate groups - YoungList* young_list() { return _young_list; } + YoungList* young_list() const { return _young_list; } // debugging bool check_young_list_well_formed() { diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -30,6 +30,7 @@ #include "gc_implementation/g1/g1AllocRegion.inline.hpp" #include "gc_implementation/g1/g1CollectorPolicy.hpp" #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" +#include "gc_implementation/g1/heapRegionSet.inline.hpp" #include "gc_implementation/g1/heapRegionSeq.inline.hpp" #include "utilities/taskqueue.hpp" diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp --- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -820,6 +820,8 @@ // do that for any other surv rate groups } + size_t young_list_target_length() const { return _young_list_target_length; } + bool is_young_list_full() { uint young_list_length = _g1->young_list()->length(); uint young_list_target_length = _young_list_target_length; diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/g1/g1MarkSweep.cpp --- a/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -196,17 +196,19 @@ G1CollectedHeap* _g1h; ModRefBarrierSet* _mrbs; CompactPoint _cp; - HumongousRegionSet _humongous_proxy_set; + HeapRegionSetCount _humongous_regions_removed; void free_humongous_region(HeapRegion* hr) { HeapWord* end = hr->end(); - size_t dummy_pre_used; FreeRegionList dummy_free_list("Dummy Free List for G1MarkSweep"); assert(hr->startsHumongous(), "Only the start of a humongous region should be freed."); - _g1h->free_humongous_region(hr, &dummy_pre_used, &dummy_free_list, - &_humongous_proxy_set, false /* par */); + + hr->set_containing_set(NULL); + _humongous_regions_removed.increment(1u, hr->capacity()); + + _g1h->free_humongous_region(hr, &dummy_free_list, false /* par */); hr->prepare_for_compaction(&_cp); // Also clear the part of the card table that will be unused after // compaction. @@ -219,16 +221,13 @@ : _g1h(G1CollectedHeap::heap()), _mrbs(_g1h->g1_barrier_set()), _cp(NULL, cs, cs->initialize_threshold()), - _humongous_proxy_set("G1MarkSweep Humongous Proxy Set") { } + _humongous_regions_removed() { } void update_sets() { // We'll recalculate total used bytes and recreate the free list // at the end of the GC, so no point in updating those values here. - _g1h->update_sets_after_freeing_regions(0, /* pre_used */ - NULL, /* free_list */ - NULL, /* old_proxy_set */ - &_humongous_proxy_set, - false /* par */); + HeapRegionSetCount empty_set; + _g1h->remove_from_old_sets(empty_set, _humongous_regions_removed); } bool doHeapRegion(HeapRegion* hr) { diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/g1/heapRegionSeq.cpp --- a/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -25,7 +25,7 @@ #include "precompiled.hpp" #include "gc_implementation/g1/heapRegion.hpp" #include "gc_implementation/g1/heapRegionSeq.inline.hpp" -#include "gc_implementation/g1/heapRegionSets.hpp" +#include "gc_implementation/g1/heapRegionSet.hpp" #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" #include "memory/allocation.hpp" diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/g1/heapRegionSet.cpp --- a/src/share/vm/gc_implementation/g1/heapRegionSet.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegionSet.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -23,171 +23,60 @@ */ #include "precompiled.hpp" +#include "gc_implementation/g1/heapRegionRemSet.hpp" #include "gc_implementation/g1/heapRegionSet.inline.hpp" -uint HeapRegionSetBase::_unrealistically_long_length = 0; -HRSPhase HeapRegionSetBase::_phase = HRSPhaseNone; - -//////////////////// HeapRegionSetBase //////////////////// - -void HeapRegionSetBase::set_unrealistically_long_length(uint len) { - guarantee(_unrealistically_long_length == 0, "should only be set once"); - _unrealistically_long_length = len; -} +uint FreeRegionList::_unrealistically_long_length = 0; void HeapRegionSetBase::fill_in_ext_msg(hrs_ext_msg* msg, const char* message) { - msg->append("[%s] %s ln: %u rn: %u cy: "SIZE_FORMAT" ud: "SIZE_FORMAT, - name(), message, length(), region_num(), - total_capacity_bytes(), total_used_bytes()); + msg->append("[%s] %s ln: %u cy: "SIZE_FORMAT, + name(), message, length(), total_capacity_bytes()); fill_in_ext_msg_extra(msg); } -bool HeapRegionSetBase::verify_region(HeapRegion* hr, - HeapRegionSetBase* expected_containing_set) { - const char* error_message = NULL; - - if (!regions_humongous()) { - if (hr->isHumongous()) { - error_message = "the region should not be humongous"; - } - } else { - if (!hr->isHumongous() || !hr->startsHumongous()) { - error_message = "the region should be 'starts humongous'"; - } - } - - if (!regions_empty()) { - if (hr->is_empty()) { - error_message = "the region should not be empty"; - } - } else { - if (!hr->is_empty()) { - error_message = "the region should be empty"; - } - } - -#ifdef ASSERT - // The _containing_set field is only available when ASSERT is defined. - if (hr->containing_set() != expected_containing_set) { - error_message = "inconsistent containing set found"; - } -#endif // ASSERT - - const char* extra_error_message = verify_region_extra(hr); - if (extra_error_message != NULL) { - error_message = extra_error_message; - } - - if (error_message != NULL) { - outputStream* out = tty; - out->cr(); - out->print_cr("## [%s] %s", name(), error_message); - out->print_cr("## Offending Region: "PTR_FORMAT, hr); - out->print_cr(" "HR_FORMAT, HR_FORMAT_PARAMS(hr)); -#ifdef ASSERT - out->print_cr(" containing set: "PTR_FORMAT, hr->containing_set()); -#endif // ASSERT - out->print_cr("## Offending Region Set: "PTR_FORMAT, this); - print_on(out); - return false; - } else { - return true; - } +#ifndef PRODUCT +void HeapRegionSetBase::verify_region(HeapRegion* hr) { + assert(hr->containing_set() == this, err_msg("Inconsistent containing set for %u", hr->hrs_index())); + assert(!hr->is_young(), err_msg("Adding young region %u", hr->hrs_index())); // currently we don't use these sets for young regions + assert(hr->isHumongous() == regions_humongous(), err_msg("Wrong humongous state for region %u and set %s", hr->hrs_index(), name())); + assert(hr->is_empty() == regions_empty(), err_msg("Wrong empty state for region %u and set %s", hr->hrs_index(), name())); + assert(hr->rem_set()->verify_ready_for_par_iteration(), err_msg("Wrong iteration state %u", hr->hrs_index())); } +#endif void HeapRegionSetBase::verify() { // It's important that we also observe the MT safety protocol even // for the verification calls. If we do verification without the // appropriate locks and the set changes underneath our feet // verification might fail and send us on a wild goose chase. - hrs_assert_mt_safety_ok(this); - - guarantee(( is_empty() && length() == 0 && region_num() == 0 && - total_used_bytes() == 0 && total_capacity_bytes() == 0) || - (!is_empty() && length() >= 0 && region_num() >= 0 && - total_used_bytes() >= 0 && total_capacity_bytes() >= 0), - hrs_ext_msg(this, "invariant")); + check_mt_safety(); - guarantee((!regions_humongous() && region_num() == length()) || - ( regions_humongous() && region_num() >= length()), - hrs_ext_msg(this, "invariant")); - - guarantee(!regions_empty() || total_used_bytes() == 0, - hrs_ext_msg(this, "invariant")); - - guarantee(total_used_bytes() <= total_capacity_bytes(), + guarantee(( is_empty() && length() == 0 && total_capacity_bytes() == 0) || + (!is_empty() && length() >= 0 && total_capacity_bytes() >= 0), hrs_ext_msg(this, "invariant")); } void HeapRegionSetBase::verify_start() { // See comment in verify() about MT safety and verification. - hrs_assert_mt_safety_ok(this); + check_mt_safety(); assert(!_verify_in_progress, hrs_ext_msg(this, "verification should not be in progress")); // Do the basic verification first before we do the checks over the regions. HeapRegionSetBase::verify(); - _calc_length = 0; - _calc_region_num = 0; - _calc_total_capacity_bytes = 0; - _calc_total_used_bytes = 0; _verify_in_progress = true; } -void HeapRegionSetBase::verify_next_region(HeapRegion* hr) { - // See comment in verify() about MT safety and verification. - hrs_assert_mt_safety_ok(this); - assert(_verify_in_progress, - hrs_ext_msg(this, "verification should be in progress")); - - guarantee(verify_region(hr, this), hrs_ext_msg(this, "region verification")); - - _calc_length += 1; - _calc_region_num += hr->region_num(); - _calc_total_capacity_bytes += hr->capacity(); - _calc_total_used_bytes += hr->used(); -} - void HeapRegionSetBase::verify_end() { // See comment in verify() about MT safety and verification. - hrs_assert_mt_safety_ok(this); + check_mt_safety(); assert(_verify_in_progress, hrs_ext_msg(this, "verification should be in progress")); - guarantee(length() == _calc_length, - hrs_err_msg("[%s] length: %u should be == calc length: %u", - name(), length(), _calc_length)); - - guarantee(region_num() == _calc_region_num, - hrs_err_msg("[%s] region num: %u should be == calc region num: %u", - name(), region_num(), _calc_region_num)); - - guarantee(total_capacity_bytes() == _calc_total_capacity_bytes, - hrs_err_msg("[%s] capacity bytes: "SIZE_FORMAT" should be == " - "calc capacity bytes: "SIZE_FORMAT, - name(), - total_capacity_bytes(), _calc_total_capacity_bytes)); - - guarantee(total_used_bytes() == _calc_total_used_bytes, - hrs_err_msg("[%s] used bytes: "SIZE_FORMAT" should be == " - "calc used bytes: "SIZE_FORMAT, - name(), total_used_bytes(), _calc_total_used_bytes)); - _verify_in_progress = false; } -void HeapRegionSetBase::clear_phase() { - assert(_phase != HRSPhaseNone, "pre-condition"); - _phase = HRSPhaseNone; -} - -void HeapRegionSetBase::set_phase(HRSPhase phase) { - assert(_phase == HRSPhaseNone, "pre-condition"); - assert(phase != HRSPhaseNone, "pre-condition"); - _phase = phase; -} - void HeapRegionSetBase::print_on(outputStream* out, bool print_contents) { out->cr(); out->print_cr("Set: %s ("PTR_FORMAT")", name(), this); @@ -196,76 +85,38 @@ out->print_cr(" empty : %s", BOOL_TO_STR(regions_empty())); out->print_cr(" Attributes"); out->print_cr(" length : %14u", length()); - out->print_cr(" region num : %14u", region_num()); out->print_cr(" total capacity : "SIZE_FORMAT_W(14)" bytes", total_capacity_bytes()); - out->print_cr(" total used : "SIZE_FORMAT_W(14)" bytes", - total_used_bytes()); -} - -void HeapRegionSetBase::clear() { - _length = 0; - _region_num = 0; - _total_used_bytes = 0; } -HeapRegionSetBase::HeapRegionSetBase(const char* name) +HeapRegionSetBase::HeapRegionSetBase(const char* name, bool humongous, bool empty, HRSMtSafeChecker* mt_safety_checker) : _name(name), _verify_in_progress(false), - _calc_length(0), _calc_region_num(0), - _calc_total_capacity_bytes(0), _calc_total_used_bytes(0) { } - -//////////////////// HeapRegionSet //////////////////// - -void HeapRegionSet::update_from_proxy(HeapRegionSet* proxy_set) { - hrs_assert_mt_safety_ok(this); - hrs_assert_mt_safety_ok(proxy_set); - hrs_assert_sets_match(this, proxy_set); - - verify_optional(); - proxy_set->verify_optional(); - - if (proxy_set->is_empty()) return; + _is_humongous(humongous), _is_empty(empty), _mt_safety_checker(mt_safety_checker), + _count() +{ } - assert(proxy_set->length() <= _length, - hrs_err_msg("[%s] proxy set length: %u should be <= length: %u", - name(), proxy_set->length(), _length)); - _length -= proxy_set->length(); - - assert(proxy_set->region_num() <= _region_num, - hrs_err_msg("[%s] proxy set region num: %u should be <= region num: %u", - name(), proxy_set->region_num(), _region_num)); - _region_num -= proxy_set->region_num(); - - assert(proxy_set->total_used_bytes() <= _total_used_bytes, - hrs_err_msg("[%s] proxy set used bytes: "SIZE_FORMAT" " - "should be <= used bytes: "SIZE_FORMAT, - name(), proxy_set->total_used_bytes(), - _total_used_bytes)); - _total_used_bytes -= proxy_set->total_used_bytes(); - - proxy_set->clear(); - - verify_optional(); - proxy_set->verify_optional(); +void FreeRegionList::set_unrealistically_long_length(uint len) { + guarantee(_unrealistically_long_length == 0, "should only be set once"); + _unrealistically_long_length = len; } -//////////////////// HeapRegionLinkedList //////////////////// - -void HeapRegionLinkedList::fill_in_ext_msg_extra(hrs_ext_msg* msg) { +void FreeRegionList::fill_in_ext_msg_extra(hrs_ext_msg* msg) { msg->append(" hd: "PTR_FORMAT" tl: "PTR_FORMAT, head(), tail()); } -void HeapRegionLinkedList::add_as_head(HeapRegionLinkedList* from_list) { - hrs_assert_mt_safety_ok(this); - hrs_assert_mt_safety_ok(from_list); +void FreeRegionList::add_as_head_or_tail(FreeRegionList* from_list, bool as_head) { + check_mt_safety(); + from_list->check_mt_safety(); verify_optional(); from_list->verify_optional(); - if (from_list->is_empty()) return; + if (from_list->is_empty()) { + return; + } #ifdef ASSERT - HeapRegionLinkedListIterator iter(from_list); + FreeRegionListIterator iter(from_list); while (iter.more_available()) { HeapRegion* hr = iter.get_next(); // In set_containing_set() we check that we either set the value @@ -276,70 +127,43 @@ } #endif // ASSERT - if (_head != NULL) { - assert(length() > 0 && _tail != NULL, hrs_ext_msg(this, "invariant")); - from_list->_tail->set_next(_head); - } else { + if (_head == NULL) { assert(length() == 0 && _tail == NULL, hrs_ext_msg(this, "invariant")); + _head = from_list->_head; _tail = from_list->_tail; + } else { + assert(length() > 0 && _tail != NULL, hrs_ext_msg(this, "invariant")); + if (as_head) { + from_list->_tail->set_next(_head); + _head = from_list->_head; + } else { + _tail->set_next(from_list->_head); + _tail = from_list->_tail; + } } - _head = from_list->_head; - _length += from_list->length(); - _region_num += from_list->region_num(); - _total_used_bytes += from_list->total_used_bytes(); + _count.increment(from_list->length(), from_list->total_capacity_bytes()); from_list->clear(); verify_optional(); from_list->verify_optional(); } -void HeapRegionLinkedList::add_as_tail(HeapRegionLinkedList* from_list) { - hrs_assert_mt_safety_ok(this); - hrs_assert_mt_safety_ok(from_list); - - verify_optional(); - from_list->verify_optional(); - - if (from_list->is_empty()) return; - -#ifdef ASSERT - HeapRegionLinkedListIterator iter(from_list); - while (iter.more_available()) { - HeapRegion* hr = iter.get_next(); - // In set_containing_set() we check that we either set the value - // from NULL to non-NULL or vice versa to catch bugs. So, we have - // to NULL it first before setting it to the value. - hr->set_containing_set(NULL); - hr->set_containing_set(this); - } -#endif // ASSERT - - if (_tail != NULL) { - assert(length() > 0 && _head != NULL, hrs_ext_msg(this, "invariant")); - _tail->set_next(from_list->_head); - } else { - assert(length() == 0 && _head == NULL, hrs_ext_msg(this, "invariant")); - _head = from_list->_head; - } - _tail = from_list->_tail; - - _length += from_list->length(); - _region_num += from_list->region_num(); - _total_used_bytes += from_list->total_used_bytes(); - from_list->clear(); - - verify_optional(); - from_list->verify_optional(); +void FreeRegionList::add_as_head(FreeRegionList* from_list) { + add_as_head_or_tail(from_list, true /* as_head */); } -void HeapRegionLinkedList::remove_all() { - hrs_assert_mt_safety_ok(this); +void FreeRegionList::add_as_tail(FreeRegionList* from_list) { + add_as_head_or_tail(from_list, false /* as_head */); +} + +void FreeRegionList::remove_all() { + check_mt_safety(); verify_optional(); HeapRegion* curr = _head; while (curr != NULL) { - hrs_assert_region_ok(this, curr, this); + verify_region(curr); HeapRegion* next = curr->next(); curr->set_next(NULL); @@ -351,8 +175,8 @@ verify_optional(); } -void HeapRegionLinkedList::remove_all_pending(uint target_count) { - hrs_assert_mt_safety_ok(this); +void FreeRegionList::remove_all_pending(uint target_count) { + check_mt_safety(); assert(target_count > 1, hrs_ext_msg(this, "pre-condition")); assert(!is_empty(), hrs_ext_msg(this, "pre-condition")); @@ -363,7 +187,7 @@ HeapRegion* prev = NULL; uint count = 0; while (curr != NULL) { - hrs_assert_region_ok(this, curr, this); + verify_region(curr); HeapRegion* next = curr->next(); if (curr->pending_removal()) { @@ -387,7 +211,7 @@ } curr->set_next(NULL); - remove_internal(curr); + remove(curr); curr->set_pending_removal(false); count += 1; @@ -414,46 +238,26 @@ verify_optional(); } -void HeapRegionLinkedList::verify() { +void FreeRegionList::verify() { // See comment in HeapRegionSetBase::verify() about MT safety and // verification. - hrs_assert_mt_safety_ok(this); + check_mt_safety(); // This will also do the basic verification too. verify_start(); - HeapRegion* curr = _head; - HeapRegion* prev1 = NULL; - HeapRegion* prev0 = NULL; - uint count = 0; - while (curr != NULL) { - verify_next_region(curr); - - count += 1; - guarantee(count < _unrealistically_long_length, - hrs_err_msg("[%s] the calculated length: %u " - "seems very long, is there maybe a cycle? " - "curr: "PTR_FORMAT" prev0: "PTR_FORMAT" " - "prev1: "PTR_FORMAT" length: %u", - name(), count, curr, prev0, prev1, length())); - - prev1 = prev0; - prev0 = curr; - curr = curr->next(); - } - - guarantee(_tail == prev0, hrs_ext_msg(this, "post-condition")); + verify_list(); verify_end(); } -void HeapRegionLinkedList::clear() { - HeapRegionSetBase::clear(); +void FreeRegionList::clear() { + _count = HeapRegionSetCount(); _head = NULL; _tail = NULL; } -void HeapRegionLinkedList::print_on(outputStream* out, bool print_contents) { +void FreeRegionList::print_on(outputStream* out, bool print_contents) { HeapRegionSetBase::print_on(out, print_contents); out->print_cr(" Linking"); out->print_cr(" head : "PTR_FORMAT, _head); @@ -461,10 +265,115 @@ if (print_contents) { out->print_cr(" Contents"); - HeapRegionLinkedListIterator iter(this); + FreeRegionListIterator iter(this); while (iter.more_available()) { HeapRegion* hr = iter.get_next(); hr->print_on(out); } } } + +void FreeRegionList::verify_list() { + HeapRegion* curr = head(); + HeapRegion* prev1 = NULL; + HeapRegion* prev0 = NULL; + uint count = 0; + size_t capacity = 0; + while (curr != NULL) { + verify_region(curr); + + count++; + guarantee(count < _unrealistically_long_length, + hrs_err_msg("[%s] the calculated length: %u seems very long, is there maybe a cycle? curr: "PTR_FORMAT" prev0: "PTR_FORMAT" " "prev1: "PTR_FORMAT" length: %u", name(), count, curr, prev0, prev1, length())); + + capacity += curr->capacity(); + + prev1 = prev0; + prev0 = curr; + curr = curr->next(); + } + + guarantee(tail() == prev0, err_msg("Expected %s to end with %u but it ended with %u.", name(), tail()->hrs_index(), prev0->hrs_index())); + + guarantee(length() == count, err_msg("%s count mismatch. Expected %u, actual %u.", name(), length(), count)); + guarantee(total_capacity_bytes() == capacity, err_msg("%s capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, + name(), total_capacity_bytes(), capacity)); +} + +// Note on the check_mt_safety() methods below: +// +// Verification of the "master" heap region sets / lists that are +// maintained by G1CollectedHeap is always done during a STW pause and +// by the VM thread at the start / end of the pause. The standard +// verification methods all assert check_mt_safety(). This is +// important as it ensures that verification is done without +// concurrent updates taking place at the same time. It follows, that, +// for the "master" heap region sets / lists, the check_mt_safety() +// method should include the VM thread / STW case. + +void MasterFreeRegionListMtSafeChecker::check() { + // Master Free List MT safety protocol: + // (a) If we're at a safepoint, operations on the master free list + // should be invoked by either the VM thread (which will serialize + // them) or by the GC workers while holding the + // FreeList_lock. + // (b) If we're not at a safepoint, operations on the master free + // list should be invoked while holding the Heap_lock. + + if (SafepointSynchronize::is_at_safepoint()) { + guarantee(Thread::current()->is_VM_thread() || + FreeList_lock->owned_by_self(), "master free list MT safety protocol at a safepoint"); + } else { + guarantee(Heap_lock->owned_by_self(), "master free list MT safety protocol outside a safepoint"); + } +} + +void SecondaryFreeRegionListMtSafeChecker::check() { + // Secondary Free List MT safety protocol: + // Operations on the secondary free list should always be invoked + // while holding the SecondaryFreeList_lock. + + guarantee(SecondaryFreeList_lock->owned_by_self(), "secondary free list MT safety protocol"); +} + +void OldRegionSetMtSafeChecker::check() { + // Master Old Set MT safety protocol: + // (a) If we're at a safepoint, operations on the master old set + // should be invoked: + // - by the VM thread (which will serialize them), or + // - by the GC workers while holding the FreeList_lock, if we're + // at a safepoint for an evacuation pause (this lock is taken + // anyway when an GC alloc region is retired so that a new one + // is allocated from the free list), or + // - by the GC workers while holding the OldSets_lock, if we're at a + // safepoint for a cleanup pause. + // (b) If we're not at a safepoint, operations on the master old set + // should be invoked while holding the Heap_lock. + + if (SafepointSynchronize::is_at_safepoint()) { + guarantee(Thread::current()->is_VM_thread() + || FreeList_lock->owned_by_self() || OldSets_lock->owned_by_self(), + "master old set MT safety protocol at a safepoint"); + } else { + guarantee(Heap_lock->owned_by_self(), "master old set MT safety protocol outside a safepoint"); + } +} + +void HumongousRegionSetMtSafeChecker::check() { + // Humongous Set MT safety protocol: + // (a) If we're at a safepoint, operations on the master humongous + // set should be invoked by either the VM thread (which will + // serialize them) or by the GC workers while holding the + // OldSets_lock. + // (b) If we're not at a safepoint, operations on the master + // humongous set should be invoked while holding the Heap_lock. + + if (SafepointSynchronize::is_at_safepoint()) { + guarantee(Thread::current()->is_VM_thread() || + OldSets_lock->owned_by_self(), + "master humongous set MT safety protocol at a safepoint"); + } else { + guarantee(Heap_lock->owned_by_self(), + "master humongous set MT safety protocol outside a safepoint"); + } +} diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/g1/heapRegionSet.hpp --- a/src/share/vm/gc_implementation/g1/heapRegionSet.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegionSet.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -38,135 +38,108 @@ #define HEAP_REGION_SET_FORCE_VERIFY defined(ASSERT) #endif // HEAP_REGION_SET_FORCE_VERIFY -//////////////////// HeapRegionSetBase //////////////////// +class hrs_ext_msg; + +class HRSMtSafeChecker : public CHeapObj { +public: + virtual void check() = 0; +}; + +class MasterFreeRegionListMtSafeChecker : public HRSMtSafeChecker { public: void check(); }; +class SecondaryFreeRegionListMtSafeChecker : public HRSMtSafeChecker { public: void check(); }; +class HumongousRegionSetMtSafeChecker : public HRSMtSafeChecker { public: void check(); }; +class OldRegionSetMtSafeChecker : public HRSMtSafeChecker { public: void check(); }; + +class HeapRegionSetCount VALUE_OBJ_CLASS_SPEC { + friend class VMStructs; + uint _length; + size_t _capacity; + +public: + HeapRegionSetCount() : _length(0), _capacity(0) { } + + const uint length() const { return _length; } + const size_t capacity() const { return _capacity; } + + void increment(uint length_to_add, size_t capacity_to_add) { + _length += length_to_add; + _capacity += capacity_to_add; + } + + void decrement(const uint length_to_remove, const size_t capacity_to_remove) { + _length -= length_to_remove; + _capacity -= capacity_to_remove; + } +}; // Base class for all the classes that represent heap region sets. It // contains the basic attributes that each set needs to maintain // (e.g., length, region num, used bytes sum) plus any shared // functionality (e.g., verification). -class hrs_ext_msg; - -typedef enum { - HRSPhaseNone, - HRSPhaseEvacuation, - HRSPhaseCleanup, - HRSPhaseFullGC -} HRSPhase; - -class HRSPhaseSetter; - class HeapRegionSetBase VALUE_OBJ_CLASS_SPEC { - friend class hrs_ext_msg; - friend class HRSPhaseSetter; friend class VMStructs; +private: + bool _is_humongous; + bool _is_empty; + HRSMtSafeChecker* _mt_safety_checker; protected: - static uint _unrealistically_long_length; - // The number of regions added to the set. If the set contains // only humongous regions, this reflects only 'starts humongous' // regions and does not include 'continues humongous' ones. - uint _length; - - // The total number of regions represented by the set. If the set - // does not contain humongous regions, this should be the same as - // _length. If the set contains only humongous regions, this will - // include the 'continues humongous' regions. - uint _region_num; - - // We don't keep track of the total capacity explicitly, we instead - // recalculate it based on _region_num and the heap region size. - - // The sum of used bytes in the all the regions in the set. - size_t _total_used_bytes; + HeapRegionSetCount _count; const char* _name; - bool _verify_in_progress; - uint _calc_length; - uint _calc_region_num; - size_t _calc_total_capacity_bytes; - size_t _calc_total_used_bytes; - - // This is here so that it can be used in the subclasses to assert - // something different depending on which phase the GC is in. This - // can be particularly helpful in the check_mt_safety() methods. - static HRSPhase _phase; - - // Only used by HRSPhaseSetter. - static void clear_phase(); - static void set_phase(HRSPhase phase); + bool _verify_in_progress; // verify_region() is used to ensure that the contents of a region - // added to / removed from a set are consistent. Different sets - // make different assumptions about the regions added to them. So - // each set can override verify_region_extra(), which is called - // from verify_region(), and do any extra verification it needs to - // perform in that. - virtual const char* verify_region_extra(HeapRegion* hr) { return NULL; } - bool verify_region(HeapRegion* hr, - HeapRegionSetBase* expected_containing_set); + // added to / removed from a set are consistent. + void verify_region(HeapRegion* hr) PRODUCT_RETURN; // Indicates whether all regions in the set should be humongous or // not. Only used during verification. - virtual bool regions_humongous() = 0; + bool regions_humongous() { return _is_humongous; } // Indicates whether all regions in the set should be empty or // not. Only used during verification. - virtual bool regions_empty() = 0; + bool regions_empty() { return _is_empty; } + + void check_mt_safety() { + if (_mt_safety_checker != NULL) { + _mt_safety_checker->check(); + } + } + + virtual void fill_in_ext_msg_extra(hrs_ext_msg* msg) { } + + HeapRegionSetBase(const char* name, bool humongous, bool empty, HRSMtSafeChecker* mt_safety_checker); + +public: + const char* name() { return _name; } - // Subclasses can optionally override this to do MT safety protocol - // checks. It is called in an assert from all methods that perform - // updates on the set (and subclasses should also call it too). - virtual bool check_mt_safety() { return true; } + uint length() { return _count.length(); } + + bool is_empty() { return _count.length() == 0; } + + size_t total_capacity_bytes() { + return _count.capacity(); + } + + // It updates the fields of the set to reflect hr being added to + // the set and tags the region appropriately. + inline void add(HeapRegion* hr); + + // It updates the fields of the set to reflect hr being removed + // from the set and tags the region appropriately. + inline void remove(HeapRegion* hr); // fill_in_ext_msg() writes the the values of the set's attributes // in the custom err_msg (hrs_ext_msg). fill_in_ext_msg_extra() // allows subclasses to append further information. - virtual void fill_in_ext_msg_extra(hrs_ext_msg* msg) { } void fill_in_ext_msg(hrs_ext_msg* msg, const char* message); - // It updates the fields of the set to reflect hr being added to - // the set. - inline void update_for_addition(HeapRegion* hr); - - // It updates the fields of the set to reflect hr being added to - // the set and tags the region appropriately. - inline void add_internal(HeapRegion* hr); - - // It updates the fields of the set to reflect hr being removed - // from the set. - inline void update_for_removal(HeapRegion* hr); - - // It updates the fields of the set to reflect hr being removed - // from the set and tags the region appropriately. - inline void remove_internal(HeapRegion* hr); - - // It clears all the fields of the sets. Note: it will not iterate - // over the set and remove regions from it. It assumes that the - // caller has already done so. It will literally just clear the fields. - virtual void clear(); - - HeapRegionSetBase(const char* name); - -public: - static void set_unrealistically_long_length(uint len); - - const char* name() { return _name; } - - uint length() { return _length; } - - bool is_empty() { return _length == 0; } - - uint region_num() { return _region_num; } - - size_t total_capacity_bytes() { - return (size_t) region_num() << HeapRegion::LogOfHRGrainBytes; - } - - size_t total_used_bytes() { return _total_used_bytes; } - virtual void verify(); void verify_start(); void verify_next_region(HeapRegion* hr); @@ -187,7 +160,6 @@ // assert/guarantee-specific message it also prints out the values of // the fields of the associated set. This can be very helpful in // diagnosing failures. - class hrs_ext_msg : public hrs_err_msg { public: hrs_ext_msg(HeapRegionSetBase* set, const char* message) : hrs_err_msg("") { @@ -195,32 +167,6 @@ } }; -class HRSPhaseSetter { -public: - HRSPhaseSetter(HRSPhase phase) { - HeapRegionSetBase::set_phase(phase); - } - ~HRSPhaseSetter() { - HeapRegionSetBase::clear_phase(); - } -}; - -// These two macros are provided for convenience, to keep the uses of -// these two asserts a bit more concise. - -#define hrs_assert_mt_safety_ok(_set_) \ - do { \ - assert((_set_)->check_mt_safety(), hrs_ext_msg((_set_), "MT safety")); \ - } while (0) - -#define hrs_assert_region_ok(_set_, _hr_, _expected_) \ - do { \ - assert((_set_)->verify_region((_hr_), (_expected_)), \ - hrs_ext_msg((_set_), "region verification")); \ - } while (0) - -//////////////////// HeapRegionSet //////////////////// - #define hrs_assert_sets_match(_set1_, _set2_) \ do { \ assert(((_set1_)->regions_humongous() == \ @@ -236,63 +182,33 @@ // the same interface (namely, the HeapRegionSetBase API). class HeapRegionSet : public HeapRegionSetBase { -protected: - virtual const char* verify_region_extra(HeapRegion* hr) { - if (hr->next() != NULL) { - return "next() should always be NULL as we do not link the regions"; - } - - return HeapRegionSetBase::verify_region_extra(hr); - } - - HeapRegionSet(const char* name) : HeapRegionSetBase(name) { - clear(); - } - public: - // It adds hr to the set. The region should not be a member of - // another set. - inline void add(HeapRegion* hr); + HeapRegionSet(const char* name, bool humongous, HRSMtSafeChecker* mt_safety_checker): + HeapRegionSetBase(name, humongous, false /* empty */, mt_safety_checker) { } - // It removes hr from the set. The region should be a member of - // this set. - inline void remove(HeapRegion* hr); - - // It removes a region from the set. Instead of updating the fields - // of the set to reflect this removal, it accumulates the updates - // in proxy_set. The idea is that proxy_set is thread-local to - // avoid multiple threads updating the fields of the set - // concurrently and having to synchronize. The method - // update_from_proxy() will update the fields of the set from the - // proxy_set. - inline void remove_with_proxy(HeapRegion* hr, HeapRegionSet* proxy_set); - - // After multiple calls to remove_with_proxy() the updates to the - // fields of the set are accumulated in proxy_set. This call - // updates the fields of the set from proxy_set. - void update_from_proxy(HeapRegionSet* proxy_set); + void bulk_remove(const HeapRegionSetCount& removed) { + _count.decrement(removed.length(), removed.capacity()); + } }; -//////////////////// HeapRegionLinkedList //////////////////// - // A set that links all the regions added to it in a singly-linked // list. We should try to avoid doing operations that iterate over // such lists in performance critical paths. Typically we should // add / remove one region at a time or concatenate two lists. All // those operations are done in constant time. -class HeapRegionLinkedListIterator; +class FreeRegionListIterator; -class HeapRegionLinkedList : public HeapRegionSetBase { - friend class HeapRegionLinkedListIterator; +class FreeRegionList : public HeapRegionSetBase { + friend class FreeRegionListIterator; private: HeapRegion* _head; HeapRegion* _tail; - // These are provided for use by the friend classes. - HeapRegion* head() { return _head; } - HeapRegion* tail() { return _tail; } + static uint _unrealistically_long_length; + + void add_as_head_or_tail(FreeRegionList* from_list, bool as_head); protected: virtual void fill_in_ext_msg_extra(hrs_ext_msg* msg); @@ -300,11 +216,19 @@ // See the comment for HeapRegionSetBase::clear() virtual void clear(); - HeapRegionLinkedList(const char* name) : HeapRegionSetBase(name) { +public: + FreeRegionList(const char* name, HRSMtSafeChecker* mt_safety_checker = NULL): + HeapRegionSetBase(name, false /* humongous */, true /* empty */, mt_safety_checker) { clear(); } -public: + void verify_list(); + + HeapRegion* head() { return _head; } + HeapRegion* tail() { return _tail; } + + static void set_unrealistically_long_length(uint len); + // It adds hr to the list as the new head. The region should not be // a member of another set. inline void add_as_head(HeapRegion* hr); @@ -323,12 +247,12 @@ // It moves the regions from from_list to this list and empties // from_list. The new regions will appear in the same order as they // were in from_list and be linked in the beginning of this list. - void add_as_head(HeapRegionLinkedList* from_list); + void add_as_head(FreeRegionList* from_list); // It moves the regions from from_list to this list and empties // from_list. The new regions will appear in the same order as they // were in from_list and be linked in the end of this list. - void add_as_tail(HeapRegionLinkedList* from_list); + void add_as_tail(FreeRegionList* from_list); // It empties the list by removing all regions from it. void remove_all(); @@ -346,14 +270,12 @@ virtual void print_on(outputStream* out, bool print_contents = false); }; -//////////////////// HeapRegionLinkedListIterator //////////////////// - // Iterator class that provides a convenient way to iterate over the // regions of a HeapRegionLinkedList instance. -class HeapRegionLinkedListIterator : public StackObj { +class FreeRegionListIterator : public StackObj { private: - HeapRegionLinkedList* _list; + FreeRegionList* _list; HeapRegion* _curr; public: @@ -369,12 +291,12 @@ // do the "cycle" check. HeapRegion* hr = _curr; - assert(_list->verify_region(hr, _list), "region verification"); + _list->verify_region(hr); _curr = hr->next(); return hr; } - HeapRegionLinkedListIterator(HeapRegionLinkedList* list) + FreeRegionListIterator(FreeRegionList* list) : _curr(NULL), _list(list) { _curr = list->head(); } diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/g1/heapRegionSet.inline.hpp --- a/src/share/vm/gc_implementation/g1/heapRegionSet.inline.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegionSet.inline.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -27,87 +27,32 @@ #include "gc_implementation/g1/heapRegionSet.hpp" -//////////////////// HeapRegionSetBase //////////////////// - -inline void HeapRegionSetBase::update_for_addition(HeapRegion* hr) { - // Assumes the caller has already verified the region. - - _length += 1; - _region_num += hr->region_num(); - _total_used_bytes += hr->used(); -} - -inline void HeapRegionSetBase::add_internal(HeapRegion* hr) { - hrs_assert_region_ok(this, hr, NULL); +inline void HeapRegionSetBase::add(HeapRegion* hr) { + check_mt_safety(); + assert(hr->containing_set() == NULL, hrs_ext_msg(this, "should not already have a containing set %u")); assert(hr->next() == NULL, hrs_ext_msg(this, "should not already be linked")); - update_for_addition(hr); + _count.increment(1u, hr->capacity()); hr->set_containing_set(this); + verify_region(hr); } -inline void HeapRegionSetBase::update_for_removal(HeapRegion* hr) { - // Assumes the caller has already verified the region. - assert(_length > 0, hrs_ext_msg(this, "pre-condition")); - _length -= 1; - - uint region_num_diff = hr->region_num(); - assert(region_num_diff <= _region_num, - hrs_err_msg("[%s] region's region num: %u " - "should be <= region num: %u", - name(), region_num_diff, _region_num)); - _region_num -= region_num_diff; - - size_t used_bytes = hr->used(); - assert(used_bytes <= _total_used_bytes, - hrs_err_msg("[%s] region's used bytes: "SIZE_FORMAT" " - "should be <= used bytes: "SIZE_FORMAT, - name(), used_bytes, _total_used_bytes)); - _total_used_bytes -= used_bytes; -} - -inline void HeapRegionSetBase::remove_internal(HeapRegion* hr) { - hrs_assert_region_ok(this, hr, this); +inline void HeapRegionSetBase::remove(HeapRegion* hr) { + check_mt_safety(); + verify_region(hr); assert(hr->next() == NULL, hrs_ext_msg(this, "should already be unlinked")); hr->set_containing_set(NULL); - update_for_removal(hr); -} - -//////////////////// HeapRegionSet //////////////////// - -inline void HeapRegionSet::add(HeapRegion* hr) { - hrs_assert_mt_safety_ok(this); - // add_internal() will verify the region. - add_internal(hr); -} - -inline void HeapRegionSet::remove(HeapRegion* hr) { - hrs_assert_mt_safety_ok(this); - // remove_internal() will verify the region. - remove_internal(hr); + assert(_count.length() > 0, hrs_ext_msg(this, "pre-condition")); + _count.decrement(1u, hr->capacity()); } -inline void HeapRegionSet::remove_with_proxy(HeapRegion* hr, - HeapRegionSet* proxy_set) { - // No need to fo the MT safety check here given that this method - // does not update the contents of the set but instead accumulates - // the changes in proxy_set which is assumed to be thread-local. - hrs_assert_sets_match(this, proxy_set); - hrs_assert_region_ok(this, hr, this); - - hr->set_containing_set(NULL); - proxy_set->update_for_addition(hr); -} - -//////////////////// HeapRegionLinkedList //////////////////// - -inline void HeapRegionLinkedList::add_as_head(HeapRegion* hr) { - hrs_assert_mt_safety_ok(this); +inline void FreeRegionList::add_as_head(HeapRegion* hr) { assert((length() == 0 && _head == NULL && _tail == NULL) || (length() > 0 && _head != NULL && _tail != NULL), hrs_ext_msg(this, "invariant")); - // add_internal() will verify the region. - add_internal(hr); + // add() will verify the region and check mt safety. + add(hr); // Now link the region. if (_head != NULL) { @@ -118,13 +63,13 @@ _head = hr; } -inline void HeapRegionLinkedList::add_as_tail(HeapRegion* hr) { - hrs_assert_mt_safety_ok(this); +inline void FreeRegionList::add_as_tail(HeapRegion* hr) { + check_mt_safety(); assert((length() == 0 && _head == NULL && _tail == NULL) || (length() > 0 && _head != NULL && _tail != NULL), hrs_ext_msg(this, "invariant")); - // add_internal() will verify the region. - add_internal(hr); + // add() will verify the region and check mt safety + add(hr); // Now link the region. if (_tail != NULL) { @@ -135,8 +80,7 @@ _tail = hr; } -inline HeapRegion* HeapRegionLinkedList::remove_head() { - hrs_assert_mt_safety_ok(this); +inline HeapRegion* FreeRegionList::remove_head() { assert(!is_empty(), hrs_ext_msg(this, "the list should not be empty")); assert(length() > 0 && _head != NULL && _tail != NULL, hrs_ext_msg(this, "invariant")); @@ -149,14 +93,13 @@ } hr->set_next(NULL); - // remove_internal() will verify the region. - remove_internal(hr); + // remove() will verify the region and check mt safety + remove(hr); return hr; } -inline HeapRegion* HeapRegionLinkedList::remove_head_or_null() { - hrs_assert_mt_safety_ok(this); - +inline HeapRegion* FreeRegionList::remove_head_or_null() { + check_mt_safety(); if (!is_empty()) { return remove_head(); } else { diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/g1/heapRegionSets.cpp --- a/src/share/vm/gc_implementation/g1/heapRegionSets.cpp Mon Mar 24 13:14:23 2014 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,175 +0,0 @@ -/* - * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/heapRegionRemSet.hpp" -#include "gc_implementation/g1/heapRegionSets.hpp" - -// Note on the check_mt_safety() methods below: -// -// Verification of the "master" heap region sets / lists that are -// maintained by G1CollectedHeap is always done during a STW pause and -// by the VM thread at the start / end of the pause. The standard -// verification methods all assert check_mt_safety(). This is -// important as it ensures that verification is done without -// concurrent updates taking place at the same time. It follows, that, -// for the "master" heap region sets / lists, the check_mt_safety() -// method should include the VM thread / STW case. - -//////////////////// FreeRegionList //////////////////// - -const char* FreeRegionList::verify_region_extra(HeapRegion* hr) { - if (hr->is_young()) { - return "the region should not be young"; - } - // The superclass will check that the region is empty and - // not humongous. - return HeapRegionLinkedList::verify_region_extra(hr); -} - -//////////////////// MasterFreeRegionList //////////////////// - -const char* MasterFreeRegionList::verify_region_extra(HeapRegion* hr) { - // We should reset the RSet for parallel iteration before we add it - // to the master free list so that it is ready when the region is - // re-allocated. - if (!hr->rem_set()->verify_ready_for_par_iteration()) { - return "the region's RSet should be ready for parallel iteration"; - } - return FreeRegionList::verify_region_extra(hr); -} - -bool MasterFreeRegionList::check_mt_safety() { - // Master Free List MT safety protocol: - // (a) If we're at a safepoint, operations on the master free list - // should be invoked by either the VM thread (which will serialize - // them) or by the GC workers while holding the - // FreeList_lock. - // (b) If we're not at a safepoint, operations on the master free - // list should be invoked while holding the Heap_lock. - - if (SafepointSynchronize::is_at_safepoint()) { - guarantee(Thread::current()->is_VM_thread() || - FreeList_lock->owned_by_self(), - hrs_ext_msg(this, "master free list MT safety protocol " - "at a safepoint")); - } else { - guarantee(Heap_lock->owned_by_self(), - hrs_ext_msg(this, "master free list MT safety protocol " - "outside a safepoint")); - } - - return FreeRegionList::check_mt_safety(); -} - -//////////////////// SecondaryFreeRegionList //////////////////// - -bool SecondaryFreeRegionList::check_mt_safety() { - // Secondary Free List MT safety protocol: - // Operations on the secondary free list should always be invoked - // while holding the SecondaryFreeList_lock. - - guarantee(SecondaryFreeList_lock->owned_by_self(), - hrs_ext_msg(this, "secondary free list MT safety protocol")); - - return FreeRegionList::check_mt_safety(); -} - -//////////////////// OldRegionSet //////////////////// - -const char* OldRegionSet::verify_region_extra(HeapRegion* hr) { - if (hr->is_young()) { - return "the region should not be young"; - } - // The superclass will check that the region is not empty and not - // humongous. - return HeapRegionSet::verify_region_extra(hr); -} - -//////////////////// MasterOldRegionSet //////////////////// - -bool MasterOldRegionSet::check_mt_safety() { - // Master Old Set MT safety protocol: - // (a) If we're at a safepoint, operations on the master old set - // should be invoked: - // - by the VM thread (which will serialize them), or - // - by the GC workers while holding the FreeList_lock, if we're - // at a safepoint for an evacuation pause (this lock is taken - // anyway when an GC alloc region is retired so that a new one - // is allocated from the free list), or - // - by the GC workers while holding the OldSets_lock, if we're at a - // safepoint for a cleanup pause. - // (b) If we're not at a safepoint, operations on the master old set - // should be invoked while holding the Heap_lock. - - if (SafepointSynchronize::is_at_safepoint()) { - guarantee(Thread::current()->is_VM_thread() || - _phase == HRSPhaseEvacuation && FreeList_lock->owned_by_self() || - _phase == HRSPhaseCleanup && OldSets_lock->owned_by_self(), - hrs_ext_msg(this, "master old set MT safety protocol " - "at a safepoint")); - } else { - guarantee(Heap_lock->owned_by_self(), - hrs_ext_msg(this, "master old set MT safety protocol " - "outside a safepoint")); - } - - return OldRegionSet::check_mt_safety(); -} - -//////////////////// HumongousRegionSet //////////////////// - -const char* HumongousRegionSet::verify_region_extra(HeapRegion* hr) { - if (hr->is_young()) { - return "the region should not be young"; - } - // The superclass will check that the region is not empty and - // humongous. - return HeapRegionSet::verify_region_extra(hr); -} - -//////////////////// MasterHumongousRegionSet //////////////////// - -bool MasterHumongousRegionSet::check_mt_safety() { - // Master Humongous Set MT safety protocol: - // (a) If we're at a safepoint, operations on the master humongous - // set should be invoked by either the VM thread (which will - // serialize them) or by the GC workers while holding the - // OldSets_lock. - // (b) If we're not at a safepoint, operations on the master - // humongous set should be invoked while holding the Heap_lock. - - if (SafepointSynchronize::is_at_safepoint()) { - guarantee(Thread::current()->is_VM_thread() || - OldSets_lock->owned_by_self(), - hrs_ext_msg(this, "master humongous set MT safety protocol " - "at a safepoint")); - } else { - guarantee(Heap_lock->owned_by_self(), - hrs_ext_msg(this, "master humongous set MT safety protocol " - "outside a safepoint")); - } - - return HumongousRegionSet::check_mt_safety(); -} diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/g1/heapRegionSets.hpp --- a/src/share/vm/gc_implementation/g1/heapRegionSets.hpp Mon Mar 24 13:14:23 2014 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSETS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSETS_HPP - -#include "gc_implementation/g1/heapRegionSet.inline.hpp" - -//////////////////// FreeRegionList //////////////////// - -class FreeRegionList : public HeapRegionLinkedList { -protected: - virtual const char* verify_region_extra(HeapRegion* hr); - - virtual bool regions_humongous() { return false; } - virtual bool regions_empty() { return true; } - -public: - FreeRegionList(const char* name) : HeapRegionLinkedList(name) { } -}; - -//////////////////// MasterFreeRegionList //////////////////// - -class MasterFreeRegionList : public FreeRegionList { -protected: - virtual const char* verify_region_extra(HeapRegion* hr); - virtual bool check_mt_safety(); - -public: - MasterFreeRegionList(const char* name) : FreeRegionList(name) { } -}; - -//////////////////// SecondaryFreeRegionList //////////////////// - -class SecondaryFreeRegionList : public FreeRegionList { -protected: - virtual bool check_mt_safety(); - -public: - SecondaryFreeRegionList(const char* name) : FreeRegionList(name) { } -}; - -//////////////////// OldRegionSet //////////////////// - -class OldRegionSet : public HeapRegionSet { -protected: - virtual const char* verify_region_extra(HeapRegion* hr); - - virtual bool regions_humongous() { return false; } - virtual bool regions_empty() { return false; } - -public: - OldRegionSet(const char* name) : HeapRegionSet(name) { } -}; - -//////////////////// MasterOldRegionSet //////////////////// - -class MasterOldRegionSet : public OldRegionSet { -private: -protected: - virtual bool check_mt_safety(); - -public: - MasterOldRegionSet(const char* name) : OldRegionSet(name) { } -}; - -//////////////////// HumongousRegionSet //////////////////// - -class HumongousRegionSet : public HeapRegionSet { -protected: - virtual const char* verify_region_extra(HeapRegion* hr); - - virtual bool regions_humongous() { return true; } - virtual bool regions_empty() { return false; } - -public: - HumongousRegionSet(const char* name) : HeapRegionSet(name) { } -}; - -//////////////////// MasterHumongousRegionSet //////////////////// - -class MasterHumongousRegionSet : public HumongousRegionSet { -protected: - virtual bool check_mt_safety(); - -public: - MasterHumongousRegionSet(const char* name) : HumongousRegionSet(name) { } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSETS_HPP diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/g1/vmStructs_g1.hpp --- a/src/share/vm/gc_implementation/g1/vmStructs_g1.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/vmStructs_g1.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -57,9 +57,10 @@ nonstatic_field(G1MonitoringSupport, _old_committed, size_t) \ nonstatic_field(G1MonitoringSupport, _old_used, size_t) \ \ - nonstatic_field(HeapRegionSetBase, _length, uint) \ - nonstatic_field(HeapRegionSetBase, _region_num, uint) \ - nonstatic_field(HeapRegionSetBase, _total_used_bytes, size_t) \ + nonstatic_field(HeapRegionSetBase, _count, HeapRegionSetCount) \ + \ + nonstatic_field(HeapRegionSetCount, _length, uint) \ + nonstatic_field(HeapRegionSetCount, _capacity, size_t) \ #define VM_TYPES_G1(declare_type, declare_toplevel_type) \ @@ -71,6 +72,7 @@ declare_type(HeapRegion, ContiguousSpace) \ declare_toplevel_type(HeapRegionSeq) \ declare_toplevel_type(HeapRegionSetBase) \ + declare_toplevel_type(HeapRegionSetCount) \ declare_toplevel_type(G1MonitoringSupport) \ \ declare_toplevel_type(G1CollectedHeap*) \ diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.cpp --- a/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -492,6 +492,10 @@ return young_gen()->eden_space()->tlab_capacity(thr); } +size_t ParallelScavengeHeap::tlab_used(Thread* thr) const { + return young_gen()->eden_space()->tlab_used(thr); +} + size_t ParallelScavengeHeap::unsafe_max_tlab_alloc(Thread* thr) const { return young_gen()->eden_space()->unsafe_max_tlab_alloc(thr); } diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.hpp --- a/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -189,6 +189,7 @@ bool supports_tlab_allocation() const { return true; } size_t tlab_capacity(Thread* thr) const; + size_t tlab_used(Thread* thr) const; size_t unsafe_max_tlab_alloc(Thread* thr) const; // Can a compiler initialize a new object without store barriers? diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp --- a/src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -173,6 +173,26 @@ return lgrp_spaces()->at(i)->space()->capacity_in_bytes(); } +size_t MutableNUMASpace::tlab_used(Thread *thr) const { + // Please see the comments for tlab_capacity(). + guarantee(thr != NULL, "No thread"); + int lgrp_id = thr->lgrp_id(); + if (lgrp_id == -1) { + if (lgrp_spaces()->length() > 0) { + return (used_in_bytes()) / lgrp_spaces()->length(); + } else { + assert(false, "There should be at least one locality group"); + return 0; + } + } + int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); + if (i == -1) { + return 0; + } + return lgrp_spaces()->at(i)->space()->used_in_bytes(); +} + + size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const { // Please see the comments for tlab_capacity(). guarantee(thr != NULL, "No thread"); diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/shared/mutableNUMASpace.hpp --- a/src/share/vm/gc_implementation/shared/mutableNUMASpace.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/shared/mutableNUMASpace.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -217,6 +217,7 @@ using MutableSpace::capacity_in_words; virtual size_t capacity_in_words(Thread* thr) const; virtual size_t tlab_capacity(Thread* thr) const; + virtual size_t tlab_used(Thread* thr) const; virtual size_t unsafe_max_tlab_alloc(Thread* thr) const; // Allocation (return NULL if full) diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/shared/mutableSpace.hpp --- a/src/share/vm/gc_implementation/shared/mutableSpace.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/shared/mutableSpace.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -124,6 +124,7 @@ virtual size_t used_in_words() const { return pointer_delta(top(), bottom()); } virtual size_t free_in_words() const { return pointer_delta(end(), top()); } virtual size_t tlab_capacity(Thread* thr) const { return capacity_in_bytes(); } + virtual size_t tlab_used(Thread* thr) const { return used_in_bytes(); } virtual size_t unsafe_max_tlab_alloc(Thread* thr) const { return free_in_bytes(); } // Allocation (return NULL if full) diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/shared/parGCAllocBuffer.cpp --- a/src/share/vm/gc_implementation/shared/parGCAllocBuffer.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/shared/parGCAllocBuffer.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -89,6 +89,10 @@ // scavenge; it clears the sensor accumulators. void PLABStats::adjust_desired_plab_sz(uint no_of_gc_workers) { assert(ResizePLAB, "Not set"); + + assert(is_object_aligned(max_size()) && min_size() <= max_size(), + "PLAB clipping computation may be incorrect"); + if (_allocated == 0) { assert(_unused == 0, err_msg("Inconsistency in PLAB stats: " diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_implementation/shared/parGCAllocBuffer.hpp --- a/src/share/vm/gc_implementation/shared/parGCAllocBuffer.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_implementation/shared/parGCAllocBuffer.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -181,16 +181,7 @@ _used(0), _desired_plab_sz(desired_plab_sz_), _filter(wt) - { - size_t min_sz = min_size(); - size_t max_sz = max_size(); - size_t aligned_min_sz = align_object_size(min_sz); - size_t aligned_max_sz = align_object_size(max_sz); - assert(min_sz <= aligned_min_sz && max_sz >= aligned_max_sz && - min_sz <= max_sz, - "PLAB clipping computation in adjust_desired_plab_sz()" - " may be incorrect"); - } + { } static const size_t min_size() { return ParGCAllocBuffer::min_size(); diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_interface/collectedHeap.cpp --- a/src/share/vm/gc_interface/collectedHeap.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_interface/collectedHeap.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -320,6 +320,21 @@ assert(thread->deferred_card_mark().is_empty(), "invariant"); } +size_t CollectedHeap::max_tlab_size() const { + // TLABs can't be bigger than we can fill with a int[Integer.MAX_VALUE]. + // This restriction could be removed by enabling filling with multiple arrays. + // If we compute that the reasonable way as + // header_size + ((sizeof(jint) * max_jint) / HeapWordSize) + // we'll overflow on the multiply, so we do the divide first. + // We actually lose a little by dividing first, + // but that just makes the TLAB somewhat smaller than the biggest array, + // which is fine, since we'll be able to fill that. + size_t max_int_size = typeArrayOopDesc::header_size(T_INT) + + sizeof(jint) * + ((juint) max_jint / (size_t) HeapWordSize); + return align_size_down(max_int_size, MinObjAlignment); +} + // Helper for ReduceInitialCardMarks. For performance, // compiled code may elide card-marks for initializing stores // to a newly allocated object along the fast-path. We diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/gc_interface/collectedHeap.hpp --- a/src/share/vm/gc_interface/collectedHeap.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/gc_interface/collectedHeap.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -403,14 +403,16 @@ // the following methods: // Returns "true" iff the heap supports thread-local allocation buffers. // The default is "no". - virtual bool supports_tlab_allocation() const { - return false; - } + virtual bool supports_tlab_allocation() const = 0; + // The amount of space available for thread-local allocation buffers. - virtual size_t tlab_capacity(Thread *thr) const { - guarantee(false, "thread-local allocation buffers not supported"); - return 0; - } + virtual size_t tlab_capacity(Thread *thr) const = 0; + + // The amount of used space for thread-local allocation buffers for the given thread. + virtual size_t tlab_used(Thread *thr) const = 0; + + virtual size_t max_tlab_size() const; + // An estimate of the maximum allocation that could be performed // for thread-local allocation buffers without triggering any // collection or expansion activity. diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/interpreter/bytecodeTracer.cpp --- a/src/share/vm/interpreter/bytecodeTracer.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/interpreter/bytecodeTracer.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -596,7 +596,7 @@ if (data != NULL) { st->print(" %d", mdo->dp_to_di(data->dp())); st->fill_to(6); - data->print_data_on(st); + data->print_data_on(st, mdo); } } } diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/memory/defNewGeneration.cpp --- a/src/share/vm/memory/defNewGeneration.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/memory/defNewGeneration.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -1086,6 +1086,10 @@ return eden()->capacity(); } +size_t DefNewGeneration::tlab_used() const { + return eden()->used(); +} + size_t DefNewGeneration::unsafe_max_tlab_alloc() const { return unsafe_max_alloc_nogc(); } diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/memory/defNewGeneration.hpp --- a/src/share/vm/memory/defNewGeneration.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/memory/defNewGeneration.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -239,6 +239,7 @@ // Thread-local allocation buffers bool supports_tlab_allocation() const { return true; } size_t tlab_capacity() const; + size_t tlab_used() const; size_t unsafe_max_tlab_alloc() const; // Grow the generation by the specified number of bytes. diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/memory/genCollectedHeap.cpp --- a/src/share/vm/memory/genCollectedHeap.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/memory/genCollectedHeap.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -937,6 +937,16 @@ return result; } +size_t GenCollectedHeap::tlab_used(Thread* thr) const { + size_t result = 0; + for (int i = 0; i < _n_gens; i += 1) { + if (_gens[i]->supports_tlab_allocation()) { + result += _gens[i]->tlab_used(); + } + } + return result; +} + size_t GenCollectedHeap::unsafe_max_tlab_alloc(Thread* thr) const { size_t result = 0; for (int i = 0; i < _n_gens; i += 1) { diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/memory/genCollectedHeap.hpp --- a/src/share/vm/memory/genCollectedHeap.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/memory/genCollectedHeap.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -256,6 +256,7 @@ // Section on TLAB's. virtual bool supports_tlab_allocation() const; virtual size_t tlab_capacity(Thread* thr) const; + virtual size_t tlab_used(Thread* thr) const; virtual size_t unsafe_max_tlab_alloc(Thread* thr) const; virtual HeapWord* allocate_new_tlab(size_t size); diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/memory/generation.hpp --- a/src/share/vm/memory/generation.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/memory/generation.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -299,6 +299,10 @@ guarantee(false, "Generation doesn't support thread local allocation buffers"); return 0; } + virtual size_t tlab_used() const { + guarantee(false, "Generation doesn't support thread local allocation buffers"); + return 0; + } virtual size_t unsafe_max_tlab_alloc() const { guarantee(false, "Generation doesn't support thread local allocation buffers"); return 0; diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/memory/threadLocalAllocBuffer.cpp --- a/src/share/vm/memory/threadLocalAllocBuffer.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/memory/threadLocalAllocBuffer.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -34,6 +34,7 @@ // Thread-Local Edens support // static member initialization +size_t ThreadLocalAllocBuffer::_max_size = 0; unsigned ThreadLocalAllocBuffer::_target_refills = 0; GlobalTLABStats* ThreadLocalAllocBuffer::_global_stats = NULL; @@ -45,7 +46,7 @@ void ThreadLocalAllocBuffer::accumulate_statistics_before_gc() { global_stats()->initialize(); - for(JavaThread *thread = Threads::first(); thread; thread = thread->next()) { + for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) { thread->tlab().accumulate_statistics(); thread->tlab().initialize_statistics(); } @@ -60,28 +61,32 @@ } void ThreadLocalAllocBuffer::accumulate_statistics() { - size_t capacity = Universe::heap()->tlab_capacity(myThread()) / HeapWordSize; - size_t unused = Universe::heap()->unsafe_max_tlab_alloc(myThread()) / HeapWordSize; - size_t used = capacity - unused; - - // Update allocation history if a reasonable amount of eden was allocated. - bool update_allocation_history = used > 0.5 * capacity; + Thread* thread = myThread(); + size_t capacity = Universe::heap()->tlab_capacity(thread); + size_t used = Universe::heap()->tlab_used(thread); _gc_waste += (unsigned)remaining(); + size_t total_allocated = thread->allocated_bytes(); + size_t allocated_since_last_gc = total_allocated - _allocated_before_last_gc; + _allocated_before_last_gc = total_allocated; if (PrintTLAB && (_number_of_refills > 0 || Verbose)) { print_stats("gc"); } if (_number_of_refills > 0) { + // Update allocation history if a reasonable amount of eden was allocated. + bool update_allocation_history = used > 0.5 * capacity; if (update_allocation_history) { // Average the fraction of eden allocated in a tlab by this // thread for use in the next resize operation. // _gc_waste is not subtracted because it's included in // "used". - size_t allocation = _number_of_refills * desired_size(); - double alloc_frac = allocation / (double) used; + // The result can be larger than 1.0 due to direct to old allocations. + // These allocations should ideally not be counted but since it is not possible + // to filter them out here we just cap the fraction to be at most 1.0. + double alloc_frac = MIN2(1.0, (double) allocated_since_last_gc / used); _allocation_fraction.sample(alloc_frac); } global_stats()->update_allocating_threads(); @@ -126,33 +131,32 @@ } void ThreadLocalAllocBuffer::resize_all_tlabs() { - for(JavaThread *thread = Threads::first(); thread; thread = thread->next()) { - thread->tlab().resize(); + if (ResizeTLAB) { + for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) { + thread->tlab().resize(); + } } } void ThreadLocalAllocBuffer::resize() { + // Compute the next tlab size using expected allocation amount + assert(ResizeTLAB, "Should not call this otherwise"); + size_t alloc = (size_t)(_allocation_fraction.average() * + (Universe::heap()->tlab_capacity(myThread()) / HeapWordSize)); + size_t new_size = alloc / _target_refills; - if (ResizeTLAB) { - // Compute the next tlab size using expected allocation amount - size_t alloc = (size_t)(_allocation_fraction.average() * - (Universe::heap()->tlab_capacity(myThread()) / HeapWordSize)); - size_t new_size = alloc / _target_refills; - - new_size = MIN2(MAX2(new_size, min_size()), max_size()); - - size_t aligned_new_size = align_object_size(new_size); + new_size = MIN2(MAX2(new_size, min_size()), max_size()); - if (PrintTLAB && Verbose) { - gclog_or_tty->print("TLAB new size: thread: " INTPTR_FORMAT " [id: %2d]" - " refills %d alloc: %8.6f desired_size: " SIZE_FORMAT " -> " SIZE_FORMAT "\n", - myThread(), myThread()->osthread()->thread_id(), - _target_refills, _allocation_fraction.average(), desired_size(), aligned_new_size); - } - set_desired_size(aligned_new_size); + size_t aligned_new_size = align_object_size(new_size); - set_refill_waste_limit(initial_refill_waste_limit()); + if (PrintTLAB && Verbose) { + gclog_or_tty->print("TLAB new size: thread: " INTPTR_FORMAT " [id: %2d]" + " refills %d alloc: %8.6f desired_size: " SIZE_FORMAT " -> " SIZE_FORMAT "\n", + myThread(), myThread()->osthread()->thread_id(), + _target_refills, _allocation_fraction.average(), desired_size(), aligned_new_size); } + set_desired_size(aligned_new_size); + set_refill_waste_limit(initial_refill_waste_limit()); } void ThreadLocalAllocBuffer::initialize_statistics() { @@ -248,31 +252,13 @@ return init_sz; } -const size_t ThreadLocalAllocBuffer::max_size() { - - // TLABs can't be bigger than we can fill with a int[Integer.MAX_VALUE]. - // This restriction could be removed by enabling filling with multiple arrays. - // If we compute that the reasonable way as - // header_size + ((sizeof(jint) * max_jint) / HeapWordSize) - // we'll overflow on the multiply, so we do the divide first. - // We actually lose a little by dividing first, - // but that just makes the TLAB somewhat smaller than the biggest array, - // which is fine, since we'll be able to fill that. - - size_t unaligned_max_size = typeArrayOopDesc::header_size(T_INT) + - sizeof(jint) * - ((juint) max_jint / (size_t) HeapWordSize); - return align_size_down(unaligned_max_size, MinObjAlignment); -} - void ThreadLocalAllocBuffer::print_stats(const char* tag) { Thread* thrd = myThread(); size_t waste = _gc_waste + _slow_refill_waste + _fast_refill_waste; size_t alloc = _number_of_refills * _desired_size; double waste_percent = alloc == 0 ? 0.0 : 100.0 * waste / alloc; - size_t tlab_used = Universe::heap()->tlab_capacity(thrd) - - Universe::heap()->unsafe_max_tlab_alloc(thrd); + size_t tlab_used = Universe::heap()->tlab_used(thrd); gclog_or_tty->print("TLAB: %s thread: " INTPTR_FORMAT " [id: %2d]" " desired_size: " SIZE_FORMAT "KB" " slow allocs: %d refill waste: " SIZE_FORMAT "B" diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/memory/threadLocalAllocBuffer.hpp --- a/src/share/vm/memory/threadLocalAllocBuffer.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/memory/threadLocalAllocBuffer.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -45,7 +45,9 @@ HeapWord* _end; // allocation end (excluding alignment_reserve) size_t _desired_size; // desired size (including alignment_reserve) size_t _refill_waste_limit; // hold onto tlab if free() is larger than this + size_t _allocated_before_last_gc; // total bytes allocated up until the last gc + static size_t _max_size; // maximum size of any TLAB static unsigned _target_refills; // expected number of refills between GCs unsigned _number_of_refills; @@ -99,12 +101,13 @@ static GlobalTLABStats* global_stats() { return _global_stats; } public: - ThreadLocalAllocBuffer() : _allocation_fraction(TLABAllocationWeight) { + ThreadLocalAllocBuffer() : _allocation_fraction(TLABAllocationWeight), _allocated_before_last_gc(0) { // do nothing. tlabs must be inited by initialize() calls } static const size_t min_size() { return align_object_size(MinTLABSize / HeapWordSize); } - static const size_t max_size(); + static const size_t max_size() { assert(_max_size != 0, "max_size not set up"); return _max_size; } + static void set_max_size(size_t max_size) { _max_size = max_size; } HeapWord* start() const { return _start; } HeapWord* end() const { return _end; } diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/memory/universe.cpp --- a/src/share/vm/memory/universe.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/memory/universe.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -816,6 +816,8 @@ Universe::_collectedHeap = new GenCollectedHeap(gc_policy); } + ThreadLocalAllocBuffer::set_max_size(Universe::heap()->max_tlab_size()); + jint status = Universe::heap()->initialize(); if (status != JNI_OK) { return status; diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/oops/instanceKlass.cpp --- a/src/share/vm/oops/instanceKlass.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/oops/instanceKlass.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -2234,15 +2234,7 @@ for (int m = 0; m < methods()->length(); m++) { MethodData* mdo = methods()->at(m)->method_data(); if (mdo != NULL) { - for (ProfileData* data = mdo->first_data(); - mdo->is_valid(data); - data = mdo->next_data(data)) { - data->clean_weak_klass_links(is_alive); - } - ParametersTypeData* parameters = mdo->parameters_type_data(); - if (parameters != NULL) { - parameters->clean_weak_klass_links(is_alive); - } + mdo->clean_method_data(is_alive); } } } diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/oops/instanceKlass.hpp --- a/src/share/vm/oops/instanceKlass.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/oops/instanceKlass.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -306,7 +306,7 @@ // three cases: // NULL: no implementor. // A Klass* that's not itself: one implementor. - // Itsef: more than one implementors. + // Itself: more than one implementors. // embedded host klass follows here // The embedded host klass only exists in an anonymous class for // dynamic language support (JSR 292 enabled). The host class grants diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/oops/methodData.cpp --- a/src/share/vm/oops/methodData.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/oops/methodData.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -80,8 +80,42 @@ _data = NULL; } +char* ProfileData::print_data_on_helper(const MethodData* md) const { + DataLayout* dp = md->extra_data_base(); + DataLayout* end = md->extra_data_limit(); + stringStream ss; + for (;; dp = MethodData::next_extra(dp)) { + assert(dp < end, "moved past end of extra data"); + switch(dp->tag()) { + case DataLayout::speculative_trap_data_tag: + if (dp->bci() == bci()) { + SpeculativeTrapData* data = new SpeculativeTrapData(dp); + int trap = data->trap_state(); + char buf[100]; + ss.print("trap/"); + data->method()->print_short_name(&ss); + ss.print("(%s) ", Deoptimization::format_trap_state(buf, sizeof(buf), trap)); + } + break; + case DataLayout::bit_data_tag: + break; + case DataLayout::no_tag: + case DataLayout::arg_info_data_tag: + return ss.as_string(); + break; + default: + fatal(err_msg("unexpected tag %d", dp->tag())); + } + } + return NULL; +} + +void ProfileData::print_data_on(outputStream* st, const MethodData* md) const { + print_data_on(st, print_data_on_helper(md)); +} + #ifndef PRODUCT -void ProfileData::print_shared(outputStream* st, const char* name) const { +void ProfileData::print_shared(outputStream* st, const char* name, const char* extra) const { st->print("bci: %d", bci()); st->fill_to(tab_width_one); st->print("%s", name); @@ -91,9 +125,13 @@ char buf[100]; st->print("trap(%s) ", Deoptimization::format_trap_state(buf, sizeof(buf), trap)); } + if (extra != NULL) { + st->print(extra); + } int flags = data()->flags(); - if (flags != 0) + if (flags != 0) { st->print("flags(%d) ", flags); + } } void ProfileData::tab(outputStream* st, bool first) const { @@ -109,8 +147,8 @@ #ifndef PRODUCT -void BitData::print_data_on(outputStream* st) const { - print_shared(st, "BitData"); +void BitData::print_data_on(outputStream* st, const char* extra) const { + print_shared(st, "BitData", extra); } #endif // !PRODUCT @@ -120,8 +158,8 @@ // A CounterData corresponds to a simple counter. #ifndef PRODUCT -void CounterData::print_data_on(outputStream* st) const { - print_shared(st, "CounterData"); +void CounterData::print_data_on(outputStream* st, const char* extra) const { + print_shared(st, "CounterData", extra); st->print_cr("count(%u)", count()); } #endif // !PRODUCT @@ -150,8 +188,8 @@ } #ifndef PRODUCT -void JumpData::print_data_on(outputStream* st) const { - print_shared(st, "JumpData"); +void JumpData::print_data_on(outputStream* st, const char* extra) const { + print_shared(st, "JumpData", extra); st->print_cr("taken(%u) displacement(%d)", taken(), displacement()); } #endif // !PRODUCT @@ -332,8 +370,8 @@ st->cr(); } -void CallTypeData::print_data_on(outputStream* st) const { - CounterData::print_data_on(st); +void CallTypeData::print_data_on(outputStream* st, const char* extra) const { + CounterData::print_data_on(st, extra); if (has_arguments()) { tab(st, true); st->print("argument types"); @@ -346,8 +384,8 @@ } } -void VirtualCallTypeData::print_data_on(outputStream* st) const { - VirtualCallData::print_data_on(st); +void VirtualCallTypeData::print_data_on(outputStream* st, const char* extra) const { + VirtualCallData::print_data_on(st, extra); if (has_arguments()) { tab(st, true); st->print("argument types"); @@ -400,12 +438,12 @@ } } } -void ReceiverTypeData::print_data_on(outputStream* st) const { - print_shared(st, "ReceiverTypeData"); +void ReceiverTypeData::print_data_on(outputStream* st, const char* extra) const { + print_shared(st, "ReceiverTypeData", extra); print_receiver_data_on(st); } -void VirtualCallData::print_data_on(outputStream* st) const { - print_shared(st, "VirtualCallData"); +void VirtualCallData::print_data_on(outputStream* st, const char* extra) const { + print_shared(st, "VirtualCallData", extra); print_receiver_data_on(st); } #endif // !PRODUCT @@ -456,8 +494,8 @@ #ifndef PRODUCT -void RetData::print_data_on(outputStream* st) const { - print_shared(st, "RetData"); +void RetData::print_data_on(outputStream* st, const char* extra) const { + print_shared(st, "RetData", extra); uint row; int entries = 0; for (row = 0; row < row_limit(); row++) { @@ -491,8 +529,8 @@ } #ifndef PRODUCT -void BranchData::print_data_on(outputStream* st) const { - print_shared(st, "BranchData"); +void BranchData::print_data_on(outputStream* st, const char* extra) const { + print_shared(st, "BranchData", extra); st->print_cr("taken(%u) displacement(%d)", taken(), displacement()); tab(st); @@ -565,8 +603,8 @@ } #ifndef PRODUCT -void MultiBranchData::print_data_on(outputStream* st) const { - print_shared(st, "MultiBranchData"); +void MultiBranchData::print_data_on(outputStream* st, const char* extra) const { + print_shared(st, "MultiBranchData", extra); st->print_cr("default_count(%u) displacement(%d)", default_count(), default_displacement()); int cases = number_of_cases(); @@ -579,8 +617,8 @@ #endif #ifndef PRODUCT -void ArgInfoData::print_data_on(outputStream* st) const { - print_shared(st, "ArgInfoData"); +void ArgInfoData::print_data_on(outputStream* st, const char* extra) const { + print_shared(st, "ArgInfoData", extra); int nargs = number_of_args(); for (int i = 0; i < nargs; i++) { st->print(" 0x%x", arg_modified(i)); @@ -611,10 +649,17 @@ } #ifndef PRODUCT -void ParametersTypeData::print_data_on(outputStream* st) const { - st->print("parameter types"); +void ParametersTypeData::print_data_on(outputStream* st, const char* extra) const { + st->print("parameter types", extra); _parameters.print_data_on(st); } + +void SpeculativeTrapData::print_data_on(outputStream* st, const char* extra) const { + print_shared(st, "SpeculativeTrapData", extra); + tab(st); + method()->print_short_name(st); + st->cr(); +} #endif // ================================================================== @@ -740,7 +785,27 @@ return DataLayout::compute_size_in_bytes(cell_count); } -int MethodData::compute_extra_data_count(int data_size, int empty_bc_count) { +bool MethodData::is_speculative_trap_bytecode(Bytecodes::Code code) { + // Bytecodes for which we may use speculation + switch (code) { + case Bytecodes::_checkcast: + case Bytecodes::_instanceof: + case Bytecodes::_aastore: + case Bytecodes::_invokevirtual: + case Bytecodes::_invokeinterface: + case Bytecodes::_if_acmpeq: + case Bytecodes::_if_acmpne: + case Bytecodes::_invokestatic: +#ifdef COMPILER2 + return UseTypeSpeculation; +#endif + default: + return false; + } + return false; +} + +int MethodData::compute_extra_data_count(int data_size, int empty_bc_count, bool needs_speculative_traps) { if (ProfileTraps) { // Assume that up to 3% of BCIs with no MDP will need to allocate one. int extra_data_count = (uint)(empty_bc_count * 3) / 128 + 1; @@ -751,7 +816,18 @@ extra_data_count = one_percent_of_data; if (extra_data_count > empty_bc_count) extra_data_count = empty_bc_count; // no need for more - return extra_data_count; + + // Make sure we have a minimum number of extra data slots to + // allocate SpeculativeTrapData entries. We would want to have one + // entry per compilation that inlines this method and for which + // some type speculation assumption fails. So the room we need for + // the SpeculativeTrapData entries doesn't directly depend on the + // size of the method. Because it's hard to estimate, we reserve + // space for an arbitrary number of entries. + int spec_data_count = (needs_speculative_traps ? SpecTrapLimitExtraEntries : 0) * + (SpeculativeTrapData::static_cell_count() + DataLayout::header_size_in_cells()); + + return MAX2(extra_data_count, spec_data_count); } else { return 0; } @@ -764,15 +840,17 @@ BytecodeStream stream(method); Bytecodes::Code c; int empty_bc_count = 0; // number of bytecodes lacking data + bool needs_speculative_traps = false; while ((c = stream.next()) >= 0) { int size_in_bytes = compute_data_size(&stream); data_size += size_in_bytes; if (size_in_bytes == 0) empty_bc_count += 1; + needs_speculative_traps = needs_speculative_traps || is_speculative_trap_bytecode(c); } int object_size = in_bytes(data_offset()) + data_size; // Add some extra DataLayout cells (at least one) to track stray traps. - int extra_data_count = compute_extra_data_count(data_size, empty_bc_count); + int extra_data_count = compute_extra_data_count(data_size, empty_bc_count, needs_speculative_traps); object_size += extra_data_count * DataLayout::compute_size_in_bytes(0); // Add a cell to record information about modified arguments. @@ -988,7 +1066,8 @@ } // Initialize the MethodData* corresponding to a given method. -MethodData::MethodData(methodHandle method, int size, TRAPS) { +MethodData::MethodData(methodHandle method, int size, TRAPS) + : _extra_data_lock(Monitor::leaf, "MDO extra data lock") { No_Safepoint_Verifier no_safepoint; // init function atomic wrt GC ResourceMark rm; // Set the method back-pointer. @@ -1004,18 +1083,23 @@ _data[0] = 0; // apparently not set below. BytecodeStream stream(method); Bytecodes::Code c; + bool needs_speculative_traps = false; while ((c = stream.next()) >= 0) { int size_in_bytes = initialize_data(&stream, data_size); data_size += size_in_bytes; if (size_in_bytes == 0) empty_bc_count += 1; + needs_speculative_traps = needs_speculative_traps || is_speculative_trap_bytecode(c); } _data_size = data_size; int object_size = in_bytes(data_offset()) + data_size; // Add some extra DataLayout cells (at least one) to track stray traps. - int extra_data_count = compute_extra_data_count(data_size, empty_bc_count); + int extra_data_count = compute_extra_data_count(data_size, empty_bc_count, needs_speculative_traps); int extra_size = extra_data_count * DataLayout::compute_size_in_bytes(0); + // Let's zero the space for the extra data + Copy::zero_to_bytes(((address)_data) + data_size, extra_size); + // Add a cell to record information about modified arguments. // Set up _args_modified array after traps cells so that // the code for traps cells works. @@ -1027,17 +1111,17 @@ int arg_data_size = DataLayout::compute_size_in_bytes(arg_size+1); object_size += extra_size + arg_data_size; - int args_cell = ParametersTypeData::compute_cell_count(method()); + int parms_cell = ParametersTypeData::compute_cell_count(method()); // If we are profiling parameters, we reserver an area near the end // of the MDO after the slots for bytecodes (because there's no bci // for method entry so they don't fit with the framework for the // profiling of bytecodes). We store the offset within the MDO of // this area (or -1 if no parameter is profiled) - if (args_cell > 0) { - object_size += DataLayout::compute_size_in_bytes(args_cell); + if (parms_cell > 0) { + object_size += DataLayout::compute_size_in_bytes(parms_cell); _parameters_type_data_di = data_size + extra_size + arg_data_size; DataLayout *dp = data_layout_at(data_size + extra_size + arg_data_size); - dp->initialize(DataLayout::parameters_type_data_tag, 0, args_cell); + dp->initialize(DataLayout::parameters_type_data_tag, 0, parms_cell); } else { _parameters_type_data_di = -1; } @@ -1128,39 +1212,114 @@ break; } } - return bci_to_extra_data(bci, false); + return bci_to_extra_data(bci, NULL, false); } -// Translate a bci to its corresponding extra data, or NULL. -ProfileData* MethodData::bci_to_extra_data(int bci, bool create_if_missing) { - DataLayout* dp = extra_data_base(); - DataLayout* end = extra_data_limit(); - DataLayout* avail = NULL; - for (; dp < end; dp = next_extra(dp)) { +DataLayout* MethodData::next_extra(DataLayout* dp) { + int nb_cells = 0; + switch(dp->tag()) { + case DataLayout::bit_data_tag: + case DataLayout::no_tag: + nb_cells = BitData::static_cell_count(); + break; + case DataLayout::speculative_trap_data_tag: + nb_cells = SpeculativeTrapData::static_cell_count(); + break; + default: + fatal(err_msg("unexpected tag %d", dp->tag())); + } + return (DataLayout*)((address)dp + DataLayout::compute_size_in_bytes(nb_cells)); +} + +ProfileData* MethodData::bci_to_extra_data_helper(int bci, Method* m, DataLayout*& dp, bool concurrent) { + DataLayout* end = extra_data_limit(); + + for (;; dp = next_extra(dp)) { + assert(dp < end, "moved past end of extra data"); // No need for "OrderAccess::load_acquire" ops, // since the data structure is monotonic. - if (dp->tag() == DataLayout::no_tag) break; - if (dp->tag() == DataLayout::arg_info_data_tag) { - dp = end; // ArgInfoData is at the end of extra data section. + switch(dp->tag()) { + case DataLayout::no_tag: + return NULL; + case DataLayout::arg_info_data_tag: + dp = end; + return NULL; // ArgInfoData is at the end of extra data section. + case DataLayout::bit_data_tag: + if (m == NULL && dp->bci() == bci) { + return new BitData(dp); + } break; - } - if (dp->bci() == bci) { - assert(dp->tag() == DataLayout::bit_data_tag, "sane"); - return new BitData(dp); + case DataLayout::speculative_trap_data_tag: + if (m != NULL) { + SpeculativeTrapData* data = new SpeculativeTrapData(dp); + // data->method() may be null in case of a concurrent + // allocation. Maybe it's for the same method. Try to use that + // entry in that case. + if (dp->bci() == bci) { + if (data->method() == NULL) { + assert(concurrent, "impossible because no concurrent allocation"); + return NULL; + } else if (data->method() == m) { + return data; + } + } + } + break; + default: + fatal(err_msg("unexpected tag %d", dp->tag())); } } + return NULL; +} + + +// Translate a bci to its corresponding extra data, or NULL. +ProfileData* MethodData::bci_to_extra_data(int bci, Method* m, bool create_if_missing) { + // This code assumes an entry for a SpeculativeTrapData is 2 cells + assert(2*DataLayout::compute_size_in_bytes(BitData::static_cell_count()) == + DataLayout::compute_size_in_bytes(SpeculativeTrapData::static_cell_count()), + "code needs to be adjusted"); + + DataLayout* dp = extra_data_base(); + DataLayout* end = extra_data_limit(); + + // Allocation in the extra data space has to be atomic because not + // all entries have the same size and non atomic concurrent + // allocation would result in a corrupted extra data space. + ProfileData* result = bci_to_extra_data_helper(bci, m, dp, true); + if (result != NULL) { + return result; + } + if (create_if_missing && dp < end) { - // Allocate this one. There is no mutual exclusion, - // so two threads could allocate different BCIs to the - // same data layout. This means these extra data - // records, like most other MDO contents, must not be - // trusted too much. + MutexLocker ml(&_extra_data_lock); + // Check again now that we have the lock. Another thread may + // have added extra data entries. + ProfileData* result = bci_to_extra_data_helper(bci, m, dp, false); + if (result != NULL || dp >= end) { + return result; + } + + assert(dp->tag() == DataLayout::no_tag || (dp->tag() == DataLayout::speculative_trap_data_tag && m != NULL), "should be free"); + assert(next_extra(dp)->tag() == DataLayout::no_tag || next_extra(dp)->tag() == DataLayout::arg_info_data_tag, "should be free or arg info"); + u1 tag = m == NULL ? DataLayout::bit_data_tag : DataLayout::speculative_trap_data_tag; + // SpeculativeTrapData is 2 slots. Make sure we have room. + if (m != NULL && next_extra(dp)->tag() != DataLayout::no_tag) { + return NULL; + } DataLayout temp; - temp.initialize(DataLayout::bit_data_tag, bci, 0); - dp->release_set_header(temp.header()); - assert(dp->tag() == DataLayout::bit_data_tag, "sane"); - //NO: assert(dp->bci() == bci, "no concurrent allocation"); - return new BitData(dp); + temp.initialize(tag, bci, 0); + + dp->set_header(temp.header()); + assert(dp->tag() == tag, "sane"); + assert(dp->bci() == bci, "no concurrent allocation"); + if (tag == DataLayout::bit_data_tag) { + return new BitData(dp); + } else { + SpeculativeTrapData* data = new SpeculativeTrapData(dp); + data->set_method(m); + return data; + } } return NULL; } @@ -1205,25 +1364,35 @@ for ( ; is_valid(data); data = next_data(data)) { st->print("%d", dp_to_di(data->dp())); st->fill_to(6); - data->print_data_on(st); + data->print_data_on(st, this); } st->print_cr("--- Extra data:"); DataLayout* dp = extra_data_base(); DataLayout* end = extra_data_limit(); - for (; dp < end; dp = next_extra(dp)) { + for (;; dp = next_extra(dp)) { + assert(dp < end, "moved past end of extra data"); // No need for "OrderAccess::load_acquire" ops, // since the data structure is monotonic. - if (dp->tag() == DataLayout::no_tag) continue; - if (dp->tag() == DataLayout::bit_data_tag) { + switch(dp->tag()) { + case DataLayout::no_tag: + continue; + case DataLayout::bit_data_tag: data = new BitData(dp); - } else { - assert(dp->tag() == DataLayout::arg_info_data_tag, "must be BitData or ArgInfo"); + break; + case DataLayout::speculative_trap_data_tag: + data = new SpeculativeTrapData(dp); + break; + case DataLayout::arg_info_data_tag: data = new ArgInfoData(dp); dp = end; // ArgInfoData is at the end of extra data section. + break; + default: + fatal(err_msg("unexpected tag %d", dp->tag())); } st->print("%d", dp_to_di(data->dp())); st->fill_to(6); data->print_data_on(st); + if (dp >= end) return; } } #endif @@ -1346,3 +1515,110 @@ assert(profile_parameters_jsr292_only(), "inconsistent"); return m->is_compiled_lambda_form(); } + +void MethodData::clean_extra_data_helper(DataLayout* dp, int shift, bool reset) { + if (shift == 0) { + return; + } + if (!reset) { + // Move all cells of trap entry at dp left by "shift" cells + intptr_t* start = (intptr_t*)dp; + intptr_t* end = (intptr_t*)next_extra(dp); + for (intptr_t* ptr = start; ptr < end; ptr++) { + *(ptr-shift) = *ptr; + } + } else { + // Reset "shift" cells stopping at dp + intptr_t* start = ((intptr_t*)dp) - shift; + intptr_t* end = (intptr_t*)dp; + for (intptr_t* ptr = start; ptr < end; ptr++) { + *ptr = 0; + } + } +} + +// Remove SpeculativeTrapData entries that reference an unloaded +// method +void MethodData::clean_extra_data(BoolObjectClosure* is_alive) { + DataLayout* dp = extra_data_base(); + DataLayout* end = extra_data_limit(); + + int shift = 0; + for (; dp < end; dp = next_extra(dp)) { + switch(dp->tag()) { + case DataLayout::speculative_trap_data_tag: { + SpeculativeTrapData* data = new SpeculativeTrapData(dp); + Method* m = data->method(); + assert(m != NULL, "should have a method"); + if (!m->method_holder()->is_loader_alive(is_alive)) { + // "shift" accumulates the number of cells for dead + // SpeculativeTrapData entries that have been seen so + // far. Following entries must be shifted left by that many + // cells to remove the dead SpeculativeTrapData entries. + shift += (int)((intptr_t*)next_extra(dp) - (intptr_t*)dp); + } else { + // Shift this entry left if it follows dead + // SpeculativeTrapData entries + clean_extra_data_helper(dp, shift); + } + break; + } + case DataLayout::bit_data_tag: + // Shift this entry left if it follows dead SpeculativeTrapData + // entries + clean_extra_data_helper(dp, shift); + continue; + case DataLayout::no_tag: + case DataLayout::arg_info_data_tag: + // We are at end of the live trap entries. The previous "shift" + // cells contain entries that are either dead or were shifted + // left. They need to be reset to no_tag + clean_extra_data_helper(dp, shift, true); + return; + default: + fatal(err_msg("unexpected tag %d", dp->tag())); + } + } +} + +// Verify there's no unloaded method referenced by a +// SpeculativeTrapData entry +void MethodData::verify_extra_data_clean(BoolObjectClosure* is_alive) { +#ifdef ASSERT + DataLayout* dp = extra_data_base(); + DataLayout* end = extra_data_limit(); + + for (; dp < end; dp = next_extra(dp)) { + switch(dp->tag()) { + case DataLayout::speculative_trap_data_tag: { + SpeculativeTrapData* data = new SpeculativeTrapData(dp); + Method* m = data->method(); + assert(m != NULL && m->method_holder()->is_loader_alive(is_alive), "Method should exist"); + break; + } + case DataLayout::bit_data_tag: + continue; + case DataLayout::no_tag: + case DataLayout::arg_info_data_tag: + return; + default: + fatal(err_msg("unexpected tag %d", dp->tag())); + } + } +#endif +} + +void MethodData::clean_method_data(BoolObjectClosure* is_alive) { + for (ProfileData* data = first_data(); + is_valid(data); + data = next_data(data)) { + data->clean_weak_klass_links(is_alive); + } + ParametersTypeData* parameters = parameters_type_data(); + if (parameters != NULL) { + parameters->clean_weak_klass_links(is_alive); + } + + clean_extra_data(is_alive); + verify_extra_data_clean(is_alive); +} diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/oops/methodData.hpp --- a/src/share/vm/oops/methodData.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/oops/methodData.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -120,7 +120,8 @@ arg_info_data_tag, call_type_data_tag, virtual_call_type_data_tag, - parameters_type_data_tag + parameters_type_data_tag, + speculative_trap_data_tag }; enum { @@ -189,9 +190,6 @@ void set_header(intptr_t value) { _header._bits = value; } - void release_set_header(intptr_t value) { - OrderAccess::release_store_ptr(&_header._bits, value); - } intptr_t header() { return _header._bits; } @@ -266,6 +264,7 @@ class MultiBranchData; class ArgInfoData; class ParametersTypeData; +class SpeculativeTrapData; // ProfileData // @@ -286,6 +285,8 @@ // This is a pointer to a section of profiling data. DataLayout* _data; + char* print_data_on_helper(const MethodData* md) const; + protected: DataLayout* data() { return _data; } const DataLayout* data() const { return _data; } @@ -400,6 +401,7 @@ virtual bool is_CallTypeData() const { return false; } virtual bool is_VirtualCallTypeData()const { return false; } virtual bool is_ParametersTypeData() const { return false; } + virtual bool is_SpeculativeTrapData()const { return false; } BitData* as_BitData() const { @@ -454,6 +456,10 @@ assert(is_ParametersTypeData(), "wrong type"); return is_ParametersTypeData() ? (ParametersTypeData*)this : NULL; } + SpeculativeTrapData* as_SpeculativeTrapData() const { + assert(is_SpeculativeTrapData(), "wrong type"); + return is_SpeculativeTrapData() ? (SpeculativeTrapData*)this : NULL; + } // Subclass specific initialization @@ -469,12 +475,14 @@ // translation here, and the required translators are in the ci subclasses. virtual void translate_from(const ProfileData* data) {} - virtual void print_data_on(outputStream* st) const { + virtual void print_data_on(outputStream* st, const char* extra = NULL) const { ShouldNotReachHere(); } + void print_data_on(outputStream* st, const MethodData* md) const; + #ifndef PRODUCT - void print_shared(outputStream* st, const char* name) const; + void print_shared(outputStream* st, const char* name, const char* extra) const; void tab(outputStream* st, bool first = false) const; #endif }; @@ -522,7 +530,7 @@ } #ifndef PRODUCT - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, const char* extra = NULL) const; #endif }; @@ -566,7 +574,7 @@ } #ifndef PRODUCT - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, const char* extra = NULL) const; #endif }; @@ -639,7 +647,7 @@ void post_initialize(BytecodeStream* stream, MethodData* mdo); #ifndef PRODUCT - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, const char* extra = NULL) const; #endif }; @@ -1050,7 +1058,7 @@ } #ifndef PRODUCT - virtual void print_data_on(outputStream* st) const; + virtual void print_data_on(outputStream* st, const char* extra = NULL) const; #endif }; @@ -1158,7 +1166,7 @@ #ifndef PRODUCT void print_receiver_data_on(outputStream* st) const; - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, const char* extra = NULL) const; #endif }; @@ -1191,7 +1199,7 @@ } #ifndef PRODUCT - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, const char* extra = NULL) const; #endif }; @@ -1317,7 +1325,7 @@ } #ifndef PRODUCT - virtual void print_data_on(outputStream* st) const; + virtual void print_data_on(outputStream* st, const char* extra = NULL) const; #endif }; @@ -1416,7 +1424,7 @@ void post_initialize(BytecodeStream* stream, MethodData* mdo); #ifndef PRODUCT - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, const char* extra = NULL) const; #endif }; @@ -1480,7 +1488,7 @@ void post_initialize(BytecodeStream* stream, MethodData* mdo); #ifndef PRODUCT - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, const char* extra = NULL) const; #endif }; @@ -1637,7 +1645,7 @@ void post_initialize(BytecodeStream* stream, MethodData* mdo); #ifndef PRODUCT - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, const char* extra = NULL) const; #endif }; @@ -1664,7 +1672,7 @@ } #ifndef PRODUCT - void print_data_on(outputStream* st) const; + void print_data_on(outputStream* st, const char* extra = NULL) const; #endif }; @@ -1725,7 +1733,7 @@ } #ifndef PRODUCT - virtual void print_data_on(outputStream* st) const; + virtual void print_data_on(outputStream* st, const char* extra = NULL) const; #endif static ByteSize stack_slot_offset(int i) { @@ -1737,6 +1745,54 @@ } }; +// SpeculativeTrapData +// +// A SpeculativeTrapData is used to record traps due to type +// speculation. It records the root of the compilation: that type +// speculation is wrong in the context of one compilation (for +// method1) doesn't mean it's wrong in the context of another one (for +// method2). Type speculation could have more/different data in the +// context of the compilation of method2 and it's worthwhile to try an +// optimization that failed for compilation of method1 in the context +// of compilation of method2. +// Space for SpeculativeTrapData entries is allocated from the extra +// data space in the MDO. If we run out of space, the trap data for +// the ProfileData at that bci is updated. +class SpeculativeTrapData : public ProfileData { +protected: + enum { + method_offset, + speculative_trap_cell_count + }; +public: + SpeculativeTrapData(DataLayout* layout) : ProfileData(layout) { + assert(layout->tag() == DataLayout::speculative_trap_data_tag, "wrong type"); + } + + virtual bool is_SpeculativeTrapData() const { return true; } + + static int static_cell_count() { + return speculative_trap_cell_count; + } + + virtual int cell_count() const { + return static_cell_count(); + } + + // Direct accessor + Method* method() const { + return (Method*)intptr_at(method_offset); + } + + void set_method(Method* m) { + set_intptr_at(method_offset, (intptr_t)m); + } + +#ifndef PRODUCT + virtual void print_data_on(outputStream* st, const char* extra = NULL) const; +#endif +}; + // MethodData* // // A MethodData* holds information which has been collected about @@ -1794,16 +1850,18 @@ // Cached hint for bci_to_dp and bci_to_data int _hint_di; + Mutex _extra_data_lock; + MethodData(methodHandle method, int size, TRAPS); public: static MethodData* allocate(ClassLoaderData* loader_data, methodHandle method, TRAPS); - MethodData() {}; // For ciMethodData + MethodData() : _extra_data_lock(Monitor::leaf, "MDO extra data lock") {}; // For ciMethodData bool is_methodData() const volatile { return true; } // Whole-method sticky bits and flags enum { - _trap_hist_limit = 17, // decoupled from Deoptimization::Reason_LIMIT + _trap_hist_limit = 18, // decoupled from Deoptimization::Reason_LIMIT _trap_hist_mask = max_jubyte, _extra_data_count = 4 // extra DataLayout headers, for trap history }; // Public flag values @@ -1858,6 +1916,7 @@ // Helper for size computation static int compute_data_size(BytecodeStream* stream); static int bytecode_cell_count(Bytecodes::Code code); + static bool is_speculative_trap_bytecode(Bytecodes::Code code); enum { no_profile_data = -1, variable_cell_count = -2 }; // Helper for initialization @@ -1901,8 +1960,9 @@ // What is the index of the first data entry? int first_di() const { return 0; } + ProfileData* bci_to_extra_data_helper(int bci, Method* m, DataLayout*& dp, bool concurrent); // Find or create an extra ProfileData: - ProfileData* bci_to_extra_data(int bci, bool create_if_missing); + ProfileData* bci_to_extra_data(int bci, Method* m, bool create_if_missing); // return the argument info cell ArgInfoData *arg_info(); @@ -1925,6 +1985,10 @@ static bool profile_parameters_jsr292_only(); static bool profile_all_parameters(); + void clean_extra_data(BoolObjectClosure* is_alive); + void clean_extra_data_helper(DataLayout* dp, int shift, bool reset = false); + void verify_extra_data_clean(BoolObjectClosure* is_alive); + public: static int header_size() { return sizeof(MethodData)/wordSize; @@ -1933,7 +1997,7 @@ // Compute the size of a MethodData* before it is created. static int compute_allocation_size_in_bytes(methodHandle method); static int compute_allocation_size_in_words(methodHandle method); - static int compute_extra_data_count(int data_size, int empty_bc_count); + static int compute_extra_data_count(int data_size, int empty_bc_count, bool needs_speculative_traps); // Determine if a given bytecode can have profile information. static bool bytecode_has_profile(Bytecodes::Code code) { @@ -2074,9 +2138,26 @@ ProfileData* bci_to_data(int bci); // Same, but try to create an extra_data record if one is needed: - ProfileData* allocate_bci_to_data(int bci) { - ProfileData* data = bci_to_data(bci); - return (data != NULL) ? data : bci_to_extra_data(bci, true); + ProfileData* allocate_bci_to_data(int bci, Method* m) { + ProfileData* data = NULL; + // If m not NULL, try to allocate a SpeculativeTrapData entry + if (m == NULL) { + data = bci_to_data(bci); + } + if (data != NULL) { + return data; + } + data = bci_to_extra_data(bci, m, true); + if (data != NULL) { + return data; + } + // If SpeculativeTrapData allocation fails try to allocate a + // regular entry + data = bci_to_data(bci); + if (data != NULL) { + return data; + } + return bci_to_extra_data(bci, NULL, true); } // Add a handful of extra data records, for trap tracking. @@ -2084,7 +2165,7 @@ DataLayout* extra_data_limit() const { return (DataLayout*)((address)this + size_in_bytes()); } int extra_data_size() const { return (address)extra_data_limit() - (address)extra_data_base(); } - static DataLayout* next_extra(DataLayout* dp) { return (DataLayout*)((address)dp + in_bytes(DataLayout::cell_offset(0))); } + static DataLayout* next_extra(DataLayout* dp); // Return (uint)-1 for overflow. uint trap_count(int reason) const { @@ -2184,6 +2265,8 @@ static bool profile_return(); static bool profile_parameters(); static bool profile_return_jsr292_only(); + + void clean_method_data(BoolObjectClosure* is_alive); }; #endif // SHARE_VM_OOPS_METHODDATAOOP_HPP diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/c2_globals.hpp --- a/src/share/vm/opto/c2_globals.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/c2_globals.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -637,14 +637,19 @@ diagnostic(bool, OptimizeExpensiveOps, true, \ "Find best control for expensive operations") \ \ - experimental(bool, UseMathExactIntrinsics, false, \ + product(bool, UseMathExactIntrinsics, true, \ "Enables intrinsification of various java.lang.Math functions") \ \ experimental(bool, ReplaceInParentMaps, false, \ "Propagate type improvements in callers of inlinee if possible") \ \ - experimental(bool, UseTypeSpeculation, false, \ - "Speculatively propagate types from profiles") + product(bool, UseTypeSpeculation, true, \ + "Speculatively propagate types from profiles") \ + \ + diagnostic(bool, UseInlineDepthForSpeculativeTypes, true, \ + "Carry inline depth of profile point with speculative type " \ + "and give priority to profiling from lower inline depth") \ + C2_FLAGS(DECLARE_DEVELOPER_FLAG, DECLARE_PD_DEVELOPER_FLAG, DECLARE_PRODUCT_FLAG, DECLARE_PD_PRODUCT_FLAG, DECLARE_DIAGNOSTIC_FLAG, DECLARE_EXPERIMENTAL_FLAG, DECLARE_NOTPRODUCT_FLAG) diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/classes.hpp --- a/src/share/vm/opto/classes.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/classes.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -29,8 +29,6 @@ macro(AbsF) macro(AbsI) macro(AddD) -macro(AddExactI) -macro(AddExactL) macro(AddF) macro(AddI) macro(AddL) @@ -135,7 +133,6 @@ macro(ExpD) macro(FastLock) macro(FastUnlock) -macro(FlagsProj) macro(Goto) macro(Halt) macro(If) @@ -170,9 +167,6 @@ macro(LoopLimit) macro(Mach) macro(MachProj) -macro(MathExact) -macro(MathExactI) -macro(MathExactL) macro(MaxI) macro(MemBarAcquire) macro(MemBarAcquireLock) @@ -192,22 +186,24 @@ macro(MoveL2D) macro(MoveD2L) macro(MulD) -macro(MulExactI) -macro(MulExactL) macro(MulF) macro(MulHiL) macro(MulI) macro(MulL) macro(Multi) macro(NegD) -macro(NegExactI) -macro(NegExactL) macro(NegF) macro(NeverBranch) macro(Opaque1) macro(Opaque2) macro(OrI) macro(OrL) +macro(OverflowAddI) +macro(OverflowSubI) +macro(OverflowMulI) +macro(OverflowAddL) +macro(OverflowSubL) +macro(OverflowMulL) macro(PCTable) macro(Parm) macro(PartialSubtypeCheck) @@ -251,8 +247,6 @@ macro(StrEquals) macro(StrIndexOf) macro(SubD) -macro(SubExactI) -macro(SubExactL) macro(SubF) macro(SubI) macro(SubL) diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/compile.cpp --- a/src/share/vm/opto/compile.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/compile.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -3012,42 +3012,6 @@ n->set_req(MemBarNode::Precedent, top()); } break; - // Must set a control edge on all nodes that produce a FlagsProj - // so they can't escape the block that consumes the flags. - // Must also set the non throwing branch as the control - // for all nodes that depends on the result. Unless the node - // already have a control that isn't the control of the - // flag producer - case Op_FlagsProj: - { - MathExactNode* math = (MathExactNode*) n->in(0); - Node* ctrl = math->control_node(); - Node* non_throwing = math->non_throwing_branch(); - math->set_req(0, ctrl); - - Node* result = math->result_node(); - if (result != NULL) { - for (DUIterator_Fast jmax, j = result->fast_outs(jmax); j < jmax; j++) { - Node* out = result->fast_out(j); - // Phi nodes shouldn't be moved. They would only match below if they - // had the same control as the MathExactNode. The only time that - // would happen is if the Phi is also an input to the MathExact - // - // Cmp nodes shouldn't have control set at all. - if (out->is_Phi() || - out->is_Cmp()) { - continue; - } - - if (out->in(0) == NULL) { - out->set_req(0, non_throwing); - } else if (out->in(0) == ctrl) { - out->set_req(0, non_throwing); - } - } - } - } - break; default: assert( !n->is_Call(), "" ); assert( !n->is_Mem(), "" ); @@ -3269,7 +3233,8 @@ // because of a transient condition during start-up in the interpreter. return false; } - if (md->has_trap_at(bci, reason) != 0) { + ciMethod* m = Deoptimization::reason_is_speculate(reason) ? this->method() : NULL; + if (md->has_trap_at(bci, m, reason) != 0) { // Assume PerBytecodeTrapLimit==0, for a more conservative heuristic. // Also, if there are multiple reasons, or if there is no per-BCI record, // assume the worst. @@ -3287,7 +3252,7 @@ // Less-accurate variant which does not require a method and bci. bool Compile::too_many_traps(Deoptimization::DeoptReason reason, ciMethodData* logmd) { - if (trap_count(reason) >= (uint)PerMethodTrapLimit) { + if (trap_count(reason) >= Deoptimization::per_method_trap_limit(reason)) { // Too many traps globally. // Note that we use cumulative trap_count, not just md->trap_count. if (log()) { @@ -3322,10 +3287,11 @@ uint m_cutoff = (uint) PerMethodRecompilationCutoff / 2 + 1; // not zero Deoptimization::DeoptReason per_bc_reason = Deoptimization::reason_recorded_per_bytecode_if_any(reason); + ciMethod* m = Deoptimization::reason_is_speculate(reason) ? this->method() : NULL; if ((per_bc_reason == Deoptimization::Reason_none - || md->has_trap_at(bci, reason) != 0) + || md->has_trap_at(bci, m, reason) != 0) // The trap frequency measure we care about is the recompile count: - && md->trap_recompiled_at(bci) + && md->trap_recompiled_at(bci, m) && md->overflow_recompile_count() >= bc_cutoff) { // Do not emit a trap here if it has already caused recompilations. // Also, if there are multiple reasons, or if there is no per-BCI record, diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/doCall.cpp --- a/src/share/vm/opto/doCall.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/doCall.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -250,7 +250,7 @@ CallGenerator* miss_cg; Deoptimization::DeoptReason reason = morphism == 2 ? Deoptimization::Reason_bimorphic : - Deoptimization::Reason_class_check; + (speculative_receiver_type == NULL ? Deoptimization::Reason_class_check : Deoptimization::Reason_speculate_class_check); if ((morphism == 1 || (morphism == 2 && next_hit_cg != NULL)) && !too_many_traps(jvms->method(), jvms->bci(), reason) ) { diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/graphKit.cpp --- a/src/share/vm/opto/graphKit.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/graphKit.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -611,9 +611,10 @@ // Usual case: Bail to interpreter. // Reserve the right to recompile if we haven't seen anything yet. + assert(!Deoptimization::reason_is_speculate(reason), "unsupported"); Deoptimization::DeoptAction action = Deoptimization::Action_maybe_recompile; if (treat_throw_as_hot - && (method()->method_data()->trap_recompiled_at(bci()) + && (method()->method_data()->trap_recompiled_at(bci(), NULL) || C->too_many_traps(reason))) { // We cannot afford to take more traps here. Suffer in the interpreter. if (C->log() != NULL) @@ -2108,30 +2109,33 @@ * @return node with improved type */ Node* GraphKit::record_profile_for_speculation(Node* n, ciKlass* exact_kls) { - const TypeOopPtr* current_type = _gvn.type(n)->isa_oopptr(); + const Type* current_type = _gvn.type(n); assert(UseTypeSpeculation, "type speculation must be on"); - if (exact_kls != NULL && - // nothing to improve if type is already exact - (current_type == NULL || - (!current_type->klass_is_exact() && - (current_type->speculative() == NULL || - !current_type->speculative()->klass_is_exact())))) { + + const TypeOopPtr* speculative = current_type->speculative(); + + if (current_type->would_improve_type(exact_kls, jvms()->depth())) { const TypeKlassPtr* tklass = TypeKlassPtr::make(exact_kls); const TypeOopPtr* xtype = tklass->as_instance_type(); assert(xtype->klass_is_exact(), "Should be exact"); - + // record the new speculative type's depth + speculative = xtype->with_inline_depth(jvms()->depth()); + } + + if (speculative != current_type->speculative()) { // Build a type with a speculative type (what we think we know // about the type but will need a guard when we use it) - const TypeOopPtr* spec_type = TypeOopPtr::make(TypePtr::BotPTR, Type::OffsetBot, TypeOopPtr::InstanceBot, xtype); - // We're changing the type, we need a new cast node to carry the - // new type. The new type depends on the control: what profiling - // tells us is only valid from here as far as we can tell. - Node* cast = new(C) CastPPNode(n, spec_type); - cast->init_req(0, control()); + const TypeOopPtr* spec_type = TypeOopPtr::make(TypePtr::BotPTR, Type::OffsetBot, TypeOopPtr::InstanceBot, speculative); + // We're changing the type, we need a new CheckCast node to carry + // the new type. The new type depends on the control: what + // profiling tells us is only valid from here as far as we can + // tell. + Node* cast = new(C) CheckCastPPNode(control(), n, current_type->remove_speculative()->join_speculative(spec_type)); cast = _gvn.transform(cast); replace_in_map(n, cast); n = cast; } + return n; } @@ -2141,7 +2145,7 @@ * * @param n receiver node * - * @return node with improved type + * @return node with improved type */ Node* GraphKit::record_profiled_receiver_for_speculation(Node* n) { if (!UseTypeSpeculation) { @@ -2734,12 +2738,14 @@ // Subsequent type checks will always fold up. Node* GraphKit::maybe_cast_profiled_receiver(Node* not_null_obj, ciKlass* require_klass, - ciKlass* spec_klass, + ciKlass* spec_klass, bool safe_for_replace) { if (!UseTypeProfile || !TypeProfileCasts) return NULL; + Deoptimization::DeoptReason reason = spec_klass == NULL ? Deoptimization::Reason_class_check : Deoptimization::Reason_speculate_class_check; + // Make sure we haven't already deoptimized from this tactic. - if (too_many_traps(Deoptimization::Reason_class_check)) + if (too_many_traps(reason)) return NULL; // (No, this isn't a call, but it's enough like a virtual call @@ -2761,7 +2767,7 @@ &exact_obj); { PreserveJVMState pjvms(this); set_control(slow_ctl); - uncommon_trap(Deoptimization::Reason_class_check, + uncommon_trap(reason, Deoptimization::Action_maybe_recompile); } if (safe_for_replace) { @@ -2788,8 +2794,10 @@ bool not_null) { // type == NULL if profiling tells us this object is always null if (type != NULL) { - if (!too_many_traps(Deoptimization::Reason_null_check) && - !too_many_traps(Deoptimization::Reason_class_check)) { + Deoptimization::DeoptReason class_reason = Deoptimization::Reason_speculate_class_check; + Deoptimization::DeoptReason null_reason = Deoptimization::Reason_null_check; + if (!too_many_traps(null_reason) && + !too_many_traps(class_reason)) { Node* not_null_obj = NULL; // not_null is true if we know the object is not null and // there's no need for a null check @@ -2808,7 +2816,7 @@ { PreserveJVMState pjvms(this); set_control(slow_ctl); - uncommon_trap(Deoptimization::Reason_class_check, + uncommon_trap(class_reason, Deoptimization::Action_maybe_recompile); } replace_in_map(not_null_obj, exact_obj); @@ -2877,7 +2885,7 @@ } if (known_statically && UseTypeSpeculation) { - // If we know the type check always succeed then we don't use the + // If we know the type check always succeeds then we don't use the // profiling data at this bytecode. Don't lose it, feed it to the // type system as a speculative type. not_null_obj = record_profiled_receiver_for_speculation(not_null_obj); diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/graphKit.hpp --- a/src/share/vm/opto/graphKit.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/graphKit.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -406,7 +406,7 @@ // Use the type profile to narrow an object type. Node* maybe_cast_profiled_receiver(Node* not_null_obj, ciKlass* require_klass, - ciKlass* spec, + ciKlass* spec, bool safe_for_replace); // Cast obj to type and emit guard unless we had too many traps here already diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/ifnode.cpp --- a/src/share/vm/opto/ifnode.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/ifnode.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -76,7 +76,6 @@ if( !i1->is_Bool() ) return NULL; BoolNode *b = i1->as_Bool(); Node *cmp = b->in(1); - if( cmp->is_FlagsProj() ) return NULL; if( !cmp->is_Cmp() ) return NULL; i1 = cmp->in(1); if( i1 == NULL || !i1->is_Phi() ) return NULL; diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/lcm.cpp --- a/src/share/vm/opto/lcm.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/lcm.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -472,13 +472,6 @@ break; } - // For nodes that produce a FlagsProj, make the node adjacent to the - // use of the FlagsProj - if (use->is_FlagsProj() && get_block_for_node(use) == block) { - found_machif = true; - break; - } - // More than this instruction pending for successor to be ready, // don't choose this if other opportunities are ready if (ready_cnt.at(use->_idx) > 1) diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/library_call.cpp --- a/src/share/vm/opto/library_call.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/library_call.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -203,7 +203,9 @@ bool inline_math_native(vmIntrinsics::ID id); bool inline_trig(vmIntrinsics::ID id); bool inline_math(vmIntrinsics::ID id); - void inline_math_mathExact(Node* math); + template + bool inline_math_overflow(Node* arg1, Node* arg2); + void inline_math_mathExact(Node* math, Node* test); bool inline_math_addExactI(bool is_increment); bool inline_math_addExactL(bool is_increment); bool inline_math_multiplyExactI(); @@ -517,31 +519,31 @@ case vmIntrinsics::_incrementExactI: case vmIntrinsics::_addExactI: - if (!Matcher::match_rule_supported(Op_AddExactI) || !UseMathExactIntrinsics) return NULL; + if (!Matcher::match_rule_supported(Op_OverflowAddI) || !UseMathExactIntrinsics) return NULL; break; case vmIntrinsics::_incrementExactL: case vmIntrinsics::_addExactL: - if (!Matcher::match_rule_supported(Op_AddExactL) || !UseMathExactIntrinsics) return NULL; + if (!Matcher::match_rule_supported(Op_OverflowAddL) || !UseMathExactIntrinsics) return NULL; break; case vmIntrinsics::_decrementExactI: case vmIntrinsics::_subtractExactI: - if (!Matcher::match_rule_supported(Op_SubExactI) || !UseMathExactIntrinsics) return NULL; + if (!Matcher::match_rule_supported(Op_OverflowSubI) || !UseMathExactIntrinsics) return NULL; break; case vmIntrinsics::_decrementExactL: case vmIntrinsics::_subtractExactL: - if (!Matcher::match_rule_supported(Op_SubExactL) || !UseMathExactIntrinsics) return NULL; + if (!Matcher::match_rule_supported(Op_OverflowSubL) || !UseMathExactIntrinsics) return NULL; break; case vmIntrinsics::_negateExactI: - if (!Matcher::match_rule_supported(Op_NegExactI) || !UseMathExactIntrinsics) return NULL; + if (!Matcher::match_rule_supported(Op_OverflowSubI) || !UseMathExactIntrinsics) return NULL; break; case vmIntrinsics::_negateExactL: - if (!Matcher::match_rule_supported(Op_NegExactL) || !UseMathExactIntrinsics) return NULL; + if (!Matcher::match_rule_supported(Op_OverflowSubL) || !UseMathExactIntrinsics) return NULL; break; case vmIntrinsics::_multiplyExactI: - if (!Matcher::match_rule_supported(Op_MulExactI) || !UseMathExactIntrinsics) return NULL; + if (!Matcher::match_rule_supported(Op_OverflowMulI) || !UseMathExactIntrinsics) return NULL; break; case vmIntrinsics::_multiplyExactL: - if (!Matcher::match_rule_supported(Op_MulExactL) || !UseMathExactIntrinsics) return NULL; + if (!Matcher::match_rule_supported(Op_OverflowMulL) || !UseMathExactIntrinsics) return NULL; break; default: @@ -1970,18 +1972,8 @@ return true; } -void LibraryCallKit::inline_math_mathExact(Node* math) { - // If we didn't get the expected opcode it means we have optimized - // the node to something else and don't need the exception edge. - if (!math->is_MathExact()) { - set_result(math); - return; - } - - Node* result = _gvn.transform( new(C) ProjNode(math, MathExactNode::result_proj_node)); - Node* flags = _gvn.transform( new(C) FlagsProjNode(math, MathExactNode::flags_proj_node)); - - Node* bol = _gvn.transform( new (C) BoolNode(flags, BoolTest::overflow) ); +void LibraryCallKit::inline_math_mathExact(Node* math, Node *test) { + Node* bol = _gvn.transform( new (C) BoolNode(test, BoolTest::overflow) ); IfNode* check = create_and_map_if(control(), bol, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN); Node* fast_path = _gvn.transform( new (C) IfFalseNode(check)); Node* slow_path = _gvn.transform( new (C) IfTrueNode(check) ); @@ -1999,108 +1991,50 @@ } set_control(fast_path); - set_result(result); + set_result(math); +} + +template +bool LibraryCallKit::inline_math_overflow(Node* arg1, Node* arg2) { + typedef typename OverflowOp::MathOp MathOp; + + MathOp* mathOp = new(C) MathOp(arg1, arg2); + Node* operation = _gvn.transform( mathOp ); + Node* ofcheck = _gvn.transform( new(C) OverflowOp(arg1, arg2) ); + inline_math_mathExact(operation, ofcheck); + return true; } bool LibraryCallKit::inline_math_addExactI(bool is_increment) { - Node* arg1 = argument(0); - Node* arg2 = NULL; - - if (is_increment) { - arg2 = intcon(1); - } else { - arg2 = argument(1); - } - - Node* add = _gvn.transform( new(C) AddExactINode(NULL, arg1, arg2) ); - inline_math_mathExact(add); - return true; + return inline_math_overflow(argument(0), is_increment ? intcon(1) : argument(1)); } bool LibraryCallKit::inline_math_addExactL(bool is_increment) { - Node* arg1 = argument(0); // type long - // argument(1) == TOP - Node* arg2 = NULL; - - if (is_increment) { - arg2 = longcon(1); - } else { - arg2 = argument(2); // type long - // argument(3) == TOP - } - - Node* add = _gvn.transform(new(C) AddExactLNode(NULL, arg1, arg2)); - inline_math_mathExact(add); - return true; + return inline_math_overflow(argument(0), is_increment ? longcon(1) : argument(2)); } bool LibraryCallKit::inline_math_subtractExactI(bool is_decrement) { - Node* arg1 = argument(0); - Node* arg2 = NULL; - - if (is_decrement) { - arg2 = intcon(1); - } else { - arg2 = argument(1); - } - - Node* sub = _gvn.transform(new(C) SubExactINode(NULL, arg1, arg2)); - inline_math_mathExact(sub); - return true; + return inline_math_overflow(argument(0), is_decrement ? intcon(1) : argument(1)); } bool LibraryCallKit::inline_math_subtractExactL(bool is_decrement) { - Node* arg1 = argument(0); // type long - // argument(1) == TOP - Node* arg2 = NULL; - - if (is_decrement) { - arg2 = longcon(1); - } else { - arg2 = argument(2); // type long - // argument(3) == TOP - } - - Node* sub = _gvn.transform(new(C) SubExactLNode(NULL, arg1, arg2)); - inline_math_mathExact(sub); - return true; + return inline_math_overflow(argument(0), is_decrement ? longcon(1) : argument(2)); } bool LibraryCallKit::inline_math_negateExactI() { - Node* arg1 = argument(0); - - Node* neg = _gvn.transform(new(C) NegExactINode(NULL, arg1)); - inline_math_mathExact(neg); - return true; + return inline_math_overflow(intcon(0), argument(0)); } bool LibraryCallKit::inline_math_negateExactL() { - Node* arg1 = argument(0); - // argument(1) == TOP - - Node* neg = _gvn.transform(new(C) NegExactLNode(NULL, arg1)); - inline_math_mathExact(neg); - return true; + return inline_math_overflow(longcon(0), argument(0)); } bool LibraryCallKit::inline_math_multiplyExactI() { - Node* arg1 = argument(0); - Node* arg2 = argument(1); - - Node* mul = _gvn.transform(new(C) MulExactINode(NULL, arg1, arg2)); - inline_math_mathExact(mul); - return true; + return inline_math_overflow(argument(0), argument(1)); } bool LibraryCallKit::inline_math_multiplyExactL() { - Node* arg1 = argument(0); - // argument(1) == TOP - Node* arg2 = argument(2); - // argument(3) == TOP - - Node* mul = _gvn.transform(new(C) MulExactLNode(NULL, arg1, arg2)); - inline_math_mathExact(mul); - return true; + return inline_math_overflow(argument(0), argument(2)); } Node* diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/loopTransform.cpp --- a/src/share/vm/opto/loopTransform.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/loopTransform.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -713,10 +713,6 @@ case Op_ModL: body_size += 30; break; case Op_DivL: body_size += 30; break; case Op_MulL: body_size += 10; break; - case Op_FlagsProj: - // Can't handle unrolling of loops containing - // nodes that generate a FlagsProj at the moment - return false; case Op_StrComp: case Op_StrEquals: case Op_StrIndexOf: @@ -780,10 +776,6 @@ continue; // not RC Node *cmp = bol->in(1); - if (cmp->is_FlagsProj()) { - continue; - } - Node *rc_exp = cmp->in(1); Node *limit = cmp->in(2); diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/loopopts.cpp --- a/src/share/vm/opto/loopopts.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/loopopts.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -43,12 +43,6 @@ return NULL; } - if (n->is_MathExact()) { - // MathExact has projections that are not correctly handled in the code - // below. - return NULL; - } - int wins = 0; assert(!n->is_CFG(), ""); assert(region->is_Region(), ""); @@ -2362,8 +2356,7 @@ opc == Op_Catch || opc == Op_CatchProj || opc == Op_Jump || - opc == Op_JumpProj || - opc == Op_FlagsProj) { + opc == Op_JumpProj) { #if !defined(PRODUCT) if (TracePartialPeeling) { tty->print_cr("\nExit control too complex: lp: %d", head->_idx); diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/matcher.cpp --- a/src/share/vm/opto/matcher.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/matcher.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -1908,6 +1908,105 @@ return OptoReg::as_OptoReg(regs.first()); } +// This function identifies sub-graphs in which a 'load' node is +// input to two different nodes, and such that it can be matched +// with BMI instructions like blsi, blsr, etc. +// Example : for b = -a[i] & a[i] can be matched to blsi r32, m32. +// The graph is (AndL (SubL Con0 LoadL*) LoadL*), where LoadL* +// refers to the same node. +#ifdef X86 +// Match the generic fused operations pattern (op1 (op2 Con{ConType} mop) mop) +// This is a temporary solution until we make DAGs expressible in ADL. +template +class FusedPatternMatcher { + Node* _op1_node; + Node* _mop_node; + int _con_op; + + static int match_next(Node* n, int next_op, int next_op_idx) { + if (n->in(1) == NULL || n->in(2) == NULL) { + return -1; + } + + if (next_op_idx == -1) { // n is commutative, try rotations + if (n->in(1)->Opcode() == next_op) { + return 1; + } else if (n->in(2)->Opcode() == next_op) { + return 2; + } + } else { + assert(next_op_idx > 0 && next_op_idx <= 2, "Bad argument index"); + if (n->in(next_op_idx)->Opcode() == next_op) { + return next_op_idx; + } + } + return -1; + } +public: + FusedPatternMatcher(Node* op1_node, Node *mop_node, int con_op) : + _op1_node(op1_node), _mop_node(mop_node), _con_op(con_op) { } + + bool match(int op1, int op1_op2_idx, // op1 and the index of the op1->op2 edge, -1 if op1 is commutative + int op2, int op2_con_idx, // op2 and the index of the op2->con edge, -1 if op2 is commutative + typename ConType::NativeType con_value) { + if (_op1_node->Opcode() != op1) { + return false; + } + if (_mop_node->outcnt() > 2) { + return false; + } + op1_op2_idx = match_next(_op1_node, op2, op1_op2_idx); + if (op1_op2_idx == -1) { + return false; + } + // Memory operation must be the other edge + int op1_mop_idx = (op1_op2_idx & 1) + 1; + + // Check that the mop node is really what we want + if (_op1_node->in(op1_mop_idx) == _mop_node) { + Node *op2_node = _op1_node->in(op1_op2_idx); + if (op2_node->outcnt() > 1) { + return false; + } + assert(op2_node->Opcode() == op2, "Should be"); + op2_con_idx = match_next(op2_node, _con_op, op2_con_idx); + if (op2_con_idx == -1) { + return false; + } + // Memory operation must be the other edge + int op2_mop_idx = (op2_con_idx & 1) + 1; + // Check that the memory operation is the same node + if (op2_node->in(op2_mop_idx) == _mop_node) { + // Now check the constant + const Type* con_type = op2_node->in(op2_con_idx)->bottom_type(); + if (con_type != Type::TOP && ConType::as_self(con_type)->get_con() == con_value) { + return true; + } + } + } + return false; + } +}; + + +bool Matcher::is_bmi_pattern(Node *n, Node *m) { + if (n != NULL && m != NULL) { + if (m->Opcode() == Op_LoadI) { + FusedPatternMatcher bmii(n, m, Op_ConI); + return bmii.match(Op_AndI, -1, Op_SubI, 1, 0) || + bmii.match(Op_AndI, -1, Op_AddI, -1, -1) || + bmii.match(Op_XorI, -1, Op_AddI, -1, -1); + } else if (m->Opcode() == Op_LoadL) { + FusedPatternMatcher bmil(n, m, Op_ConL); + return bmil.match(Op_AndL, -1, Op_SubL, 1, 0) || + bmil.match(Op_AndL, -1, Op_AddL, -1, -1) || + bmil.match(Op_XorL, -1, Op_AddL, -1, -1); + } + } + return false; +} +#endif // X86 + // A method-klass-holder may be passed in the inline_cache_reg // and then expanded into the inline_cache_reg and a method_oop register // defined in ad_.cpp @@ -1984,7 +2083,6 @@ case Op_Catch: case Op_CatchProj: case Op_CProj: - case Op_FlagsProj: case Op_JumpProj: case Op_JProj: case Op_NeverBranch: @@ -2064,6 +2162,14 @@ set_shared(m->in(AddPNode::Base)->in(1)); } + // if 'n' and 'm' are part of a graph for BMI instruction, clone this node. +#ifdef X86 + if (UseBMI1Instructions && is_bmi_pattern(n, m)) { + mstack.push(m, Visit); + continue; + } +#endif + // Clone addressing expressions as they are "free" in memory access instructions if( mem_op && i == MemNode::Address && mop == Op_AddP ) { // Some inputs for address expression are not put on stack diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/matcher.hpp --- a/src/share/vm/opto/matcher.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/matcher.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -79,6 +79,9 @@ // Find shared Nodes, or Nodes that otherwise are Matcher roots void find_shared( Node *n ); +#ifdef X86 + bool is_bmi_pattern(Node *n, Node *m); +#endif // Debug and profile information for nodes in old space: GrowableArray* _old_node_note_array; @@ -340,10 +343,6 @@ // Register for MODL projection of divmodL static RegMask modL_proj_mask(); - static const RegMask mathExactI_result_proj_mask(); - static const RegMask mathExactL_result_proj_mask(); - static const RegMask mathExactI_flags_proj_mask(); - // Use hardware DIV instruction when it is faster than // a code which use multiply for division by constant. static bool use_asm_for_ldiv_by_con( jlong divisor ); diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/mathexactnode.cpp --- a/src/share/vm/opto/mathexactnode.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/mathexactnode.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -31,358 +31,93 @@ #include "opto/mathexactnode.hpp" #include "opto/subnode.hpp" -MathExactNode::MathExactNode(Node* ctrl, Node* in1) : MultiNode(2) { - init_class_id(Class_MathExact); - init_req(0, ctrl); - init_req(1, in1); -} - -MathExactNode::MathExactNode(Node* ctrl, Node* in1, Node* in2) : MultiNode(3) { - init_class_id(Class_MathExact); - init_req(0, ctrl); - init_req(1, in1); - init_req(2, in2); -} - -BoolNode* MathExactNode::bool_node() const { - Node* flags = flags_node(); - BoolNode* boolnode = flags->unique_out()->as_Bool(); - assert(boolnode != NULL, "must have BoolNode"); - return boolnode; -} - -IfNode* MathExactNode::if_node() const { - BoolNode* boolnode = bool_node(); - IfNode* ifnode = boolnode->unique_out()->as_If(); - assert(ifnode != NULL, "must have IfNode"); - return ifnode; -} - -Node* MathExactNode::control_node() const { - IfNode* ifnode = if_node(); - return ifnode->in(0); -} - -Node* MathExactNode::non_throwing_branch() const { - IfNode* ifnode = if_node(); - if (bool_node()->_test._test == BoolTest::overflow) { - return ifnode->proj_out(0); - } - return ifnode->proj_out(1); -} - -// If the MathExactNode won't overflow we have to replace the -// FlagsProjNode and ProjNode that is generated by the MathExactNode -Node* MathExactNode::no_overflow(PhaseGVN* phase, Node* new_result) { - PhaseIterGVN* igvn = phase->is_IterGVN(); - if (igvn) { - ProjNode* result = result_node(); - ProjNode* flags = flags_node(); - - if (result != NULL) { - igvn->replace_node(result, new_result); - } +template +class AddHelper { +public: + typedef typename OverflowOp::TypeClass TypeClass; + typedef typename TypeClass::NativeType NativeType; - if (flags != NULL) { - BoolNode* boolnode = bool_node(); - switch (boolnode->_test._test) { - case BoolTest::overflow: - // if the check is for overflow - never taken - igvn->replace_node(boolnode, phase->intcon(0)); - break; - case BoolTest::no_overflow: - // if the check is for no overflow - always taken - igvn->replace_node(boolnode, phase->intcon(1)); - break; - default: - fatal("Unexpected value of BoolTest"); - break; - } - flags->del_req(0); + static bool will_overflow(NativeType value1, NativeType value2) { + NativeType result = value1 + value2; + // Hacker's Delight 2-12 Overflow if both arguments have the opposite sign of the result + if (((value1 ^ result) & (value2 ^ result)) >= 0) { + return false; } - } - return new_result; -} - -Node* MathExactINode::match(const ProjNode* proj, const Matcher* m) { - uint ideal_reg = proj->ideal_reg(); - RegMask rm; - if (proj->_con == result_proj_node) { - rm = m->mathExactI_result_proj_mask(); - } else { - assert(proj->_con == flags_proj_node, "must be result or flags"); - assert(ideal_reg == Op_RegFlags, "sanity"); - rm = m->mathExactI_flags_proj_mask(); - } - return new (m->C) MachProjNode(this, proj->_con, rm, ideal_reg); -} - -Node* MathExactLNode::match(const ProjNode* proj, const Matcher* m) { - uint ideal_reg = proj->ideal_reg(); - RegMask rm; - if (proj->_con == result_proj_node) { - rm = m->mathExactL_result_proj_mask(); - } else { - assert(proj->_con == flags_proj_node, "must be result or flags"); - assert(ideal_reg == Op_RegFlags, "sanity"); - rm = m->mathExactI_flags_proj_mask(); - } - return new (m->C) MachProjNode(this, proj->_con, rm, ideal_reg); -} - -Node* AddExactINode::Ideal(PhaseGVN* phase, bool can_reshape) { - Node* arg1 = in(1); - Node* arg2 = in(2); - - const Type* type1 = phase->type(arg1); - const Type* type2 = phase->type(arg2); - - if (type1 != Type::TOP && type1->singleton() && - type2 != Type::TOP && type2->singleton()) { - jint val1 = arg1->get_int(); - jint val2 = arg2->get_int(); - jint result = val1 + val2; - // Hacker's Delight 2-12 Overflow if both arguments have the opposite sign of the result - if ( (((val1 ^ result) & (val2 ^ result)) >= 0)) { - Node* con_result = ConINode::make(phase->C, result); - return no_overflow(phase, con_result); - } - return NULL; + return true; } - if (type1 == TypeInt::ZERO || type2 == TypeInt::ZERO) { // (Add 0 x) == x - Node* add_result = new (phase->C) AddINode(arg1, arg2); - return no_overflow(phase, add_result); - } - - if (type2->singleton()) { - return NULL; // no change - keep constant on the right + static bool can_overflow(const Type* type1, const Type* type2) { + if (type1 == TypeClass::ZERO || type2 == TypeClass::ZERO) { + return false; + } + return true; } - - if (type1->singleton()) { - // Make it x + Constant - move constant to the right - swap_edges(1, 2); - return this; - } - - if (arg2->is_Load()) { - return NULL; // no change - keep load on the right - } - - if (arg1->is_Load()) { - // Make it x + Load - move load to the right - swap_edges(1, 2); - return this; - } +}; - if (arg1->_idx > arg2->_idx) { - // Sort the edges - swap_edges(1, 2); - return this; - } - - return NULL; -} - -Node* AddExactLNode::Ideal(PhaseGVN* phase, bool can_reshape) { - Node* arg1 = in(1); - Node* arg2 = in(2); +template +class SubHelper { +public: + typedef typename OverflowOp::TypeClass TypeClass; + typedef typename TypeClass::NativeType NativeType; - const Type* type1 = phase->type(arg1); - const Type* type2 = phase->type(arg2); - - if (type1 != Type::TOP && type1->singleton() && - type2 != Type::TOP && type2->singleton()) { - jlong val1 = arg1->get_long(); - jlong val2 = arg2->get_long(); - jlong result = val1 + val2; - // Hacker's Delight 2-12 Overflow if both arguments have the opposite sign of the result - if ( (((val1 ^ result) & (val2 ^ result)) >= 0)) { - Node* con_result = ConLNode::make(phase->C, result); - return no_overflow(phase, con_result); + static bool will_overflow(NativeType value1, NativeType value2) { + NativeType result = value1 - value2; + // hacker's delight 2-12 overflow iff the arguments have different signs and + // the sign of the result is different than the sign of arg1 + if (((value1 ^ value2) & (value1 ^ result)) >= 0) { + return false; } - return NULL; + return true; } - if (type1 == TypeLong::ZERO || type2 == TypeLong::ZERO) { // (Add 0 x) == x - Node* add_result = new (phase->C) AddLNode(arg1, arg2); - return no_overflow(phase, add_result); + static bool can_overflow(const Type* type1, const Type* type2) { + if (type2 == TypeClass::ZERO) { + return false; + } + return true; } - - if (type2->singleton()) { - return NULL; // no change - keep constant on the right - } - - if (type1->singleton()) { - // Make it x + Constant - move constant to the right - swap_edges(1, 2); - return this; - } +}; - if (arg2->is_Load()) { - return NULL; // no change - keep load on the right - } - - if (arg1->is_Load()) { - // Make it x + Load - move load to the right - swap_edges(1, 2); - return this; - } - - if (arg1->_idx > arg2->_idx) { - // Sort the edges - swap_edges(1, 2); - return this; - } +template +class MulHelper { +public: + typedef typename OverflowOp::TypeClass TypeClass; - return NULL; -} - -Node* SubExactINode::Ideal(PhaseGVN* phase, bool can_reshape) { - Node* arg1 = in(1); - Node* arg2 = in(2); - - const Type* type1 = phase->type(arg1); - const Type* type2 = phase->type(arg2); - - if (type1 != Type::TOP && type1->singleton() && - type2 != Type::TOP && type2->singleton()) { - jint val1 = arg1->get_int(); - jint val2 = arg2->get_int(); - jint result = val1 - val2; + static bool can_overflow(const Type* type1, const Type* type2) { + if (type1 == TypeClass::ZERO || type2 == TypeClass::ZERO) { + return false; + } else if (type1 == TypeClass::ONE || type2 == TypeClass::ONE) { + return false; + } + return true; + } +}; - // Hacker's Delight 2-12 Overflow iff the arguments have different signs and - // the sign of the result is different than the sign of arg1 - if (((val1 ^ val2) & (val1 ^ result)) >= 0) { - Node* con_result = ConINode::make(phase->C, result); - return no_overflow(phase, con_result); - } - return NULL; - } - - if (type1 == TypeInt::ZERO || type2 == TypeInt::ZERO) { - // Sub with zero is the same as add with zero - Node* add_result = new (phase->C) AddINode(arg1, arg2); - return no_overflow(phase, add_result); - } - - return NULL; +bool OverflowAddINode::will_overflow(jint v1, jint v2) const { + return AddHelper::will_overflow(v1, v2); } -Node* SubExactLNode::Ideal(PhaseGVN* phase, bool can_reshape) { - Node* arg1 = in(1); - Node* arg2 = in(2); - - const Type* type1 = phase->type(arg1); - const Type* type2 = phase->type(arg2); - - if (type1 != Type::TOP && type1->singleton() && - type2 != Type::TOP && type2->singleton()) { - jlong val1 = arg1->get_long(); - jlong val2 = arg2->get_long(); - jlong result = val1 - val2; - - // Hacker's Delight 2-12 Overflow iff the arguments have different signs and - // the sign of the result is different than the sign of arg1 - if (((val1 ^ val2) & (val1 ^ result)) >= 0) { - Node* con_result = ConLNode::make(phase->C, result); - return no_overflow(phase, con_result); - } - return NULL; - } - - if (type1 == TypeLong::ZERO || type2 == TypeLong::ZERO) { - // Sub with zero is the same as add with zero - Node* add_result = new (phase->C) AddLNode(arg1, arg2); - return no_overflow(phase, add_result); - } - - return NULL; -} - -Node* NegExactINode::Ideal(PhaseGVN* phase, bool can_reshape) { - Node *arg = in(1); - - const Type* type = phase->type(arg); - if (type != Type::TOP && type->singleton()) { - jint value = arg->get_int(); - if (value != min_jint) { - Node* neg_result = ConINode::make(phase->C, -value); - return no_overflow(phase, neg_result); - } - } - return NULL; +bool OverflowSubINode::will_overflow(jint v1, jint v2) const { + return SubHelper::will_overflow(v1, v2); } -Node* NegExactLNode::Ideal(PhaseGVN* phase, bool can_reshape) { - Node *arg = in(1); - - const Type* type = phase->type(arg); - if (type != Type::TOP && type->singleton()) { - jlong value = arg->get_long(); - if (value != min_jlong) { - Node* neg_result = ConLNode::make(phase->C, -value); - return no_overflow(phase, neg_result); +bool OverflowMulINode::will_overflow(jint v1, jint v2) const { + jlong result = (jlong) v1 * (jlong) v2; + if ((jint) result == result) { + return false; } - } - return NULL; + return true; } -Node* MulExactINode::Ideal(PhaseGVN* phase, bool can_reshape) { - Node* arg1 = in(1); - Node* arg2 = in(2); - - const Type* type1 = phase->type(arg1); - const Type* type2 = phase->type(arg2); - - if (type1 != Type::TOP && type1->singleton() && - type2 != Type::TOP && type2->singleton()) { - jint val1 = arg1->get_int(); - jint val2 = arg2->get_int(); - jlong result = (jlong) val1 * (jlong) val2; - if ((jint) result == result) { - // no overflow - Node* mul_result = ConINode::make(phase->C, result); - return no_overflow(phase, mul_result); - } - } - - if (type1 == TypeInt::ZERO || type2 == TypeInt::ZERO) { - return no_overflow(phase, ConINode::make(phase->C, 0)); - } - - if (type1 == TypeInt::ONE) { - Node* mul_result = new (phase->C) AddINode(arg2, phase->intcon(0)); - return no_overflow(phase, mul_result); - } - if (type2 == TypeInt::ONE) { - Node* mul_result = new (phase->C) AddINode(arg1, phase->intcon(0)); - return no_overflow(phase, mul_result); - } - - if (type1 == TypeInt::MINUS_1) { - return new (phase->C) NegExactINode(NULL, arg2); - } - - if (type2 == TypeInt::MINUS_1) { - return new (phase->C) NegExactINode(NULL, arg1); - } - - return NULL; +bool OverflowAddLNode::will_overflow(jlong v1, jlong v2) const { + return AddHelper::will_overflow(v1, v2); } -Node* MulExactLNode::Ideal(PhaseGVN* phase, bool can_reshape) { - Node* arg1 = in(1); - Node* arg2 = in(2); +bool OverflowSubLNode::will_overflow(jlong v1, jlong v2) const { + return SubHelper::will_overflow(v1, v2); +} - const Type* type1 = phase->type(arg1); - const Type* type2 = phase->type(arg2); - - if (type1 != Type::TOP && type1->singleton() && - type2 != Type::TOP && type2->singleton()) { - jlong val1 = arg1->get_long(); - jlong val2 = arg2->get_long(); - +bool OverflowMulLNode::will_overflow(jlong val1, jlong val2) const { jlong result = val1 * val2; jlong ax = (val1 < 0 ? -val1 : val1); jlong ay = (val2 < 0 ? -val2 : val2); @@ -398,33 +133,125 @@ } } - if (!overflow) { - Node* mul_result = ConLNode::make(phase->C, result); - return no_overflow(phase, mul_result); + return overflow; +} + +bool OverflowAddINode::can_overflow(const Type* t1, const Type* t2) const { + return AddHelper::can_overflow(t1, t2); +} + +bool OverflowSubINode::can_overflow(const Type* t1, const Type* t2) const { + if (in(1) == in(2)) { + return false; + } + return SubHelper::can_overflow(t1, t2); +} + +bool OverflowMulINode::can_overflow(const Type* t1, const Type* t2) const { + return MulHelper::can_overflow(t1, t2); +} + +bool OverflowAddLNode::can_overflow(const Type* t1, const Type* t2) const { + return AddHelper::can_overflow(t1, t2); +} + +bool OverflowSubLNode::can_overflow(const Type* t1, const Type* t2) const { + if (in(1) == in(2)) { + return false; + } + return SubHelper::can_overflow(t1, t2); +} + +bool OverflowMulLNode::can_overflow(const Type* t1, const Type* t2) const { + return MulHelper::can_overflow(t1, t2); +} + +const Type* OverflowNode::sub(const Type* t1, const Type* t2) const { + fatal(err_msg_res("sub() should not be called for '%s'", NodeClassNames[this->Opcode()])); + return TypeInt::CC; +} + +template +struct IdealHelper { + typedef typename OverflowOp::TypeClass TypeClass; // TypeInt, TypeLong + typedef typename TypeClass::NativeType NativeType; + + static Node* Ideal(const OverflowOp* node, PhaseGVN* phase, bool can_reshape) { + Node* arg1 = node->in(1); + Node* arg2 = node->in(2); + const Type* type1 = phase->type(arg1); + const Type* type2 = phase->type(arg2); + + if (type1 == NULL || type2 == NULL) { + return NULL; } - } - if (type1 == TypeLong::ZERO || type2 == TypeLong::ZERO) { - return no_overflow(phase, ConLNode::make(phase->C, 0)); + if (type1 != Type::TOP && type1->singleton() && + type2 != Type::TOP && type2->singleton()) { + NativeType val1 = TypeClass::as_self(type1)->get_con(); + NativeType val2 = TypeClass::as_self(type2)->get_con(); + if (node->will_overflow(val1, val2) == false) { + Node* con_result = ConINode::make(phase->C, 0); + return con_result; + } + return NULL; + } + return NULL; } - if (type1 == TypeLong::ONE) { - Node* mul_result = new (phase->C) AddLNode(arg2, phase->longcon(0)); - return no_overflow(phase, mul_result); - } - if (type2 == TypeLong::ONE) { - Node* mul_result = new (phase->C) AddLNode(arg1, phase->longcon(0)); - return no_overflow(phase, mul_result); - } + static const Type* Value(const OverflowOp* node, PhaseTransform* phase) { + const Type *t1 = phase->type( node->in(1) ); + const Type *t2 = phase->type( node->in(2) ); + if( t1 == Type::TOP ) return Type::TOP; + if( t2 == Type::TOP ) return Type::TOP; + + const TypeClass* i1 = TypeClass::as_self(t1); + const TypeClass* i2 = TypeClass::as_self(t2); + + if (i1 == NULL || i2 == NULL) { + return TypeInt::CC; + } - if (type1 == TypeLong::MINUS_1) { - return new (phase->C) NegExactLNode(NULL, arg2); - } + if (t1->singleton() && t2->singleton()) { + NativeType val1 = i1->get_con(); + NativeType val2 = i2->get_con(); + if (node->will_overflow(val1, val2)) { + return TypeInt::CC; + } + return TypeInt::ZERO; + } else if (i1 != TypeClass::TYPE_DOMAIN && i2 != TypeClass::TYPE_DOMAIN) { + if (node->will_overflow(i1->_lo, i2->_lo)) { + return TypeInt::CC; + } else if (node->will_overflow(i1->_lo, i2->_hi)) { + return TypeInt::CC; + } else if (node->will_overflow(i1->_hi, i2->_lo)) { + return TypeInt::CC; + } else if (node->will_overflow(i1->_hi, i2->_hi)) { + return TypeInt::CC; + } + return TypeInt::ZERO; + } - if (type2 == TypeLong::MINUS_1) { - return new (phase->C) NegExactLNode(NULL, arg1); + if (!node->can_overflow(t1, t2)) { + return TypeInt::ZERO; + } + return TypeInt::CC; } +}; - return NULL; +Node* OverflowINode::Ideal(PhaseGVN* phase, bool can_reshape) { + return IdealHelper::Ideal(this, phase, can_reshape); } +Node* OverflowLNode::Ideal(PhaseGVN* phase, bool can_reshape) { + return IdealHelper::Ideal(this, phase, can_reshape); +} + +const Type* OverflowINode::Value(PhaseTransform* phase) const { + return IdealHelper::Value(this, phase); +} + +const Type* OverflowLNode::Value(PhaseTransform* phase) const { + return IdealHelper::Value(this, phase); +} + diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/mathexactnode.hpp --- a/src/share/vm/opto/mathexactnode.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/mathexactnode.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -27,128 +27,111 @@ #include "opto/multnode.hpp" #include "opto/node.hpp" +#include "opto/addnode.hpp" #include "opto/subnode.hpp" #include "opto/type.hpp" -class BoolNode; -class IfNode; -class Node; - class PhaseGVN; class PhaseTransform; -class MathExactNode : public MultiNode { +class OverflowNode : public CmpNode { public: - MathExactNode(Node* ctrl, Node* in1); - MathExactNode(Node* ctrl, Node* in1, Node* in2); - enum { - result_proj_node = 0, - flags_proj_node = 1 - }; - virtual int Opcode() const; - virtual Node* Identity(PhaseTransform* phase) { return this; } - virtual Node* Ideal(PhaseGVN* phase, bool can_reshape) { return NULL; } - virtual const Type* Value(PhaseTransform* phase) const { return bottom_type(); } - virtual uint hash() const { return NO_HASH; } - virtual bool is_CFG() const { return false; } - virtual uint ideal_reg() const { return NotAMachineReg; } + OverflowNode(Node* in1, Node* in2) : CmpNode(in1, in2) {} - ProjNode* result_node() const { return proj_out(result_proj_node); } - ProjNode* flags_node() const { return proj_out(flags_proj_node); } - Node* control_node() const; - Node* non_throwing_branch() const; -protected: - IfNode* if_node() const; - BoolNode* bool_node() const; - Node* no_overflow(PhaseGVN *phase, Node* new_result); -}; - -class MathExactINode : public MathExactNode { - public: - MathExactINode(Node* ctrl, Node* in1) : MathExactNode(ctrl, in1) {} - MathExactINode(Node* ctrl, Node* in1, Node* in2) : MathExactNode(ctrl, in1, in2) {} - virtual int Opcode() const; - virtual Node* match(const ProjNode* proj, const Matcher* m); - virtual const Type* bottom_type() const { return TypeTuple::INT_CC_PAIR; } -}; - -class MathExactLNode : public MathExactNode { -public: - MathExactLNode(Node* ctrl, Node* in1) : MathExactNode(ctrl, in1) {} - MathExactLNode(Node* ctrl, Node* in1, Node* in2) : MathExactNode(ctrl, in1, in2) {} - virtual int Opcode() const; - virtual Node* match(const ProjNode* proj, const Matcher* m); - virtual const Type* bottom_type() const { return TypeTuple::LONG_CC_PAIR; } -}; - -class AddExactINode : public MathExactINode { -public: - AddExactINode(Node* ctrl, Node* in1, Node* in2) : MathExactINode(ctrl, in1, in2) {} - virtual int Opcode() const; - virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); + virtual uint ideal_reg() const { return Op_RegFlags; } + virtual const Type* sub(const Type* t1, const Type* t2) const; }; -class AddExactLNode : public MathExactLNode { -public: - AddExactLNode(Node* ctrl, Node* in1, Node* in2) : MathExactLNode(ctrl, in1, in2) {} - virtual int Opcode() const; - virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); -}; - -class SubExactINode : public MathExactINode { -public: - SubExactINode(Node* ctrl, Node* in1, Node* in2) : MathExactINode(ctrl, in1, in2) {} - virtual int Opcode() const; - virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); -}; - -class SubExactLNode : public MathExactLNode { -public: - SubExactLNode(Node* ctrl, Node* in1, Node* in2) : MathExactLNode(ctrl, in1, in2) {} - virtual int Opcode() const; - virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); -}; - -class NegExactINode : public MathExactINode { -public: - NegExactINode(Node* ctrl, Node* in1) : MathExactINode(ctrl, in1) {} - virtual int Opcode() const; - virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); -}; - -class NegExactLNode : public MathExactLNode { +class OverflowINode : public OverflowNode { public: - NegExactLNode(Node* ctrl, Node* in1) : MathExactLNode(ctrl, in1) {} - virtual int Opcode() const; - virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); -}; - -class MulExactINode : public MathExactINode { -public: - MulExactINode(Node* ctrl, Node* in1, Node* in2) : MathExactINode(ctrl, in1, in2) {} - virtual int Opcode() const; - virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); -}; + typedef TypeInt TypeClass; -class MulExactLNode : public MathExactLNode { -public: - MulExactLNode(Node* ctrl, Node* in1, Node* in2) : MathExactLNode(ctrl, in1, in2) {} - virtual int Opcode() const; + OverflowINode(Node* in1, Node* in2) : OverflowNode(in1, in2) {} virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); -}; + virtual const Type* Value(PhaseTransform* phase) const; -class FlagsProjNode : public ProjNode { -public: - FlagsProjNode(Node* src, uint con) : ProjNode(src, con) { - init_class_id(Class_FlagsProj); - } - - virtual int Opcode() const; - virtual bool is_CFG() const { return false; } - virtual const Type* bottom_type() const { return TypeInt::CC; } - virtual uint ideal_reg() const { return Op_RegFlags; } + virtual bool will_overflow(jint v1, jint v2) const = 0; + virtual bool can_overflow(const Type* t1, const Type* t2) const = 0; }; +class OverflowLNode : public OverflowNode { +public: + typedef TypeLong TypeClass; + + OverflowLNode(Node* in1, Node* in2) : OverflowNode(in1, in2) {} + virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); + virtual const Type* Value(PhaseTransform* phase) const; + + virtual bool will_overflow(jlong v1, jlong v2) const = 0; + virtual bool can_overflow(const Type* t1, const Type* t2) const = 0; +}; + +class OverflowAddINode : public OverflowINode { +public: + typedef AddINode MathOp; + + OverflowAddINode(Node* in1, Node* in2) : OverflowINode(in1, in2) {} + virtual int Opcode() const; + + virtual bool will_overflow(jint v1, jint v2) const; + virtual bool can_overflow(const Type* t1, const Type* t2) const; +}; + +class OverflowSubINode : public OverflowINode { +public: + typedef SubINode MathOp; + + OverflowSubINode(Node* in1, Node* in2) : OverflowINode(in1, in2) {} + virtual int Opcode() const; + + virtual bool will_overflow(jint v1, jint v2) const; + virtual bool can_overflow(const Type* t1, const Type* t2) const; +}; + +class OverflowMulINode : public OverflowINode { +public: + typedef MulINode MathOp; + + OverflowMulINode(Node* in1, Node* in2) : OverflowINode(in1, in2) {} + virtual int Opcode() const; + + virtual bool will_overflow(jint v1, jint v2) const; + virtual bool can_overflow(const Type* t1, const Type* t2) const; +}; + +class OverflowAddLNode : public OverflowLNode { +public: + typedef AddLNode MathOp; + + OverflowAddLNode(Node* in1, Node* in2) : OverflowLNode(in1, in2) {} + virtual int Opcode() const; + + virtual bool will_overflow(jlong v1, jlong v2) const; + virtual bool can_overflow(const Type* t1, const Type* t2) const; +}; + +class OverflowSubLNode : public OverflowLNode { +public: + typedef SubLNode MathOp; + + OverflowSubLNode(Node* in1, Node* in2) : OverflowLNode(in1, in2) {} + virtual int Opcode() const; + + virtual bool will_overflow(jlong v1, jlong v2) const; + virtual bool can_overflow(const Type* t1, const Type* t2) const; +}; + +class OverflowMulLNode : public OverflowLNode { +public: + typedef MulLNode MathOp; + + OverflowMulLNode(Node* in1, Node* in2) : OverflowLNode(in1, in2) {} + virtual int Opcode() const; + + virtual bool will_overflow(jlong v1, jlong v2) const; + virtual bool can_overflow(const Type* t1, const Type* t2) const; +}; + #endif diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/multnode.cpp --- a/src/share/vm/opto/multnode.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/multnode.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -54,11 +54,6 @@ assert(Opcode() != Op_If || proj->Opcode() == (which_proj?Op_IfTrue:Op_IfFalse), "bad if #2"); return proj; } - } else if (p->is_FlagsProj()) { - FlagsProjNode *proj = p->as_FlagsProj(); - if (proj->_con == which_proj) { - return proj; - } } else { assert(p == this && this->is_Start(), "else must be proj"); continue; diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/node.hpp --- a/src/share/vm/opto/node.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/node.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -69,7 +69,6 @@ class EncodePKlassNode; class FastLockNode; class FastUnlockNode; -class FlagsProjNode; class IfNode; class IfFalseNode; class IfTrueNode; @@ -100,7 +99,6 @@ class MachSpillCopyNode; class MachTempNode; class Matcher; -class MathExactNode; class MemBarNode; class MemBarStoreStoreNode; class MemNode; @@ -569,7 +567,6 @@ DEFINE_CLASS_ID(MemBar, Multi, 3) DEFINE_CLASS_ID(Initialize, MemBar, 0) DEFINE_CLASS_ID(MemBarStoreStore, MemBar, 1) - DEFINE_CLASS_ID(MathExact, Multi, 4) DEFINE_CLASS_ID(Mach, Node, 1) DEFINE_CLASS_ID(MachReturn, Mach, 0) @@ -626,7 +623,6 @@ DEFINE_CLASS_ID(Cmp, Sub, 0) DEFINE_CLASS_ID(FastLock, Cmp, 0) DEFINE_CLASS_ID(FastUnlock, Cmp, 1) - DEFINE_CLASS_ID(FlagsProj, Cmp, 2) DEFINE_CLASS_ID(MergeMem, Node, 7) DEFINE_CLASS_ID(Bool, Node, 8) @@ -730,7 +726,6 @@ DEFINE_CLASS_QUERY(EncodePKlass) DEFINE_CLASS_QUERY(FastLock) DEFINE_CLASS_QUERY(FastUnlock) - DEFINE_CLASS_QUERY(FlagsProj) DEFINE_CLASS_QUERY(If) DEFINE_CLASS_QUERY(IfFalse) DEFINE_CLASS_QUERY(IfTrue) @@ -759,7 +754,6 @@ DEFINE_CLASS_QUERY(MachSafePoint) DEFINE_CLASS_QUERY(MachSpillCopy) DEFINE_CLASS_QUERY(MachTemp) - DEFINE_CLASS_QUERY(MathExact) DEFINE_CLASS_QUERY(Mem) DEFINE_CLASS_QUERY(MemBar) DEFINE_CLASS_QUERY(MemBarStoreStore) diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/subnode.cpp --- a/src/share/vm/opto/subnode.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/subnode.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -1126,11 +1126,15 @@ Node *cmp = in(1); if( !cmp->is_Sub() ) return NULL; int cop = cmp->Opcode(); - if( cop == Op_FastLock || cop == Op_FastUnlock || cop == Op_FlagsProj) return NULL; + if( cop == Op_FastLock || cop == Op_FastUnlock) return NULL; Node *cmp1 = cmp->in(1); Node *cmp2 = cmp->in(2); if( !cmp1 ) return NULL; + if (_test._test == BoolTest::overflow || _test._test == BoolTest::no_overflow) { + return NULL; + } + // Constant on left? Node *con = cmp1; uint op2 = cmp2->Opcode(); diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/type.cpp --- a/src/share/vm/opto/type.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/type.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -301,6 +301,7 @@ TypeInt::POS1 = TypeInt::make(1,max_jint, WidenMin); // Positive values TypeInt::INT = TypeInt::make(min_jint,max_jint, WidenMax); // 32-bit integers TypeInt::SYMINT = TypeInt::make(-max_jint,max_jint,WidenMin); // symmetric range + TypeInt::TYPE_DOMAIN = TypeInt::INT; // CmpL is overloaded both as the bytecode computation returning // a trinary (-1,0,+1) integer result AND as an efficient long // compare returning optimizer ideal-type flags. @@ -317,6 +318,7 @@ TypeLong::LONG = TypeLong::make(min_jlong,max_jlong,WidenMax); // 64-bit integers TypeLong::INT = TypeLong::make((jlong)min_jint,(jlong)max_jint,WidenMin); TypeLong::UINT = TypeLong::make(0,(jlong)max_juint,WidenMin); + TypeLong::TYPE_DOMAIN = TypeLong::LONG; const Type **fboth =(const Type**)shared_type_arena->Amalloc_4(2*sizeof(Type*)); fboth[0] = Type::CONTROL; @@ -1156,6 +1158,7 @@ const TypeInt *TypeInt::POS1; // Positive 32-bit integers const TypeInt *TypeInt::INT; // 32-bit integers const TypeInt *TypeInt::SYMINT; // symmetric range [-max_jint..max_jint] +const TypeInt *TypeInt::TYPE_DOMAIN; // alias for TypeInt::INT //------------------------------TypeInt---------------------------------------- TypeInt::TypeInt( jint lo, jint hi, int w ) : Type(Int), _lo(lo), _hi(hi), _widen(w) { @@ -1413,6 +1416,7 @@ const TypeLong *TypeLong::LONG; // 64-bit integers const TypeLong *TypeLong::INT; // 32-bit subrange const TypeLong *TypeLong::UINT; // 32-bit unsigned subrange +const TypeLong *TypeLong::TYPE_DOMAIN; // alias for TypeLong::LONG //------------------------------TypeLong--------------------------------------- TypeLong::TypeLong( jlong lo, jlong hi, int w ) : Type(Long), _lo(lo), _hi(hi), _widen(w) { @@ -2453,7 +2457,7 @@ const TypeOopPtr *TypeOopPtr::BOTTOM; //------------------------------TypeOopPtr------------------------------------- -TypeOopPtr::TypeOopPtr(TYPES t, PTR ptr, ciKlass* k, bool xk, ciObject* o, int offset, int instance_id, const TypeOopPtr* speculative) +TypeOopPtr::TypeOopPtr(TYPES t, PTR ptr, ciKlass* k, bool xk, ciObject* o, int offset, int instance_id, const TypeOopPtr* speculative, int inline_depth) : TypePtr(t, ptr, offset), _const_oop(o), _klass(k), _klass_is_exact(xk), @@ -2461,7 +2465,8 @@ _is_ptr_to_narrowklass(false), _is_ptr_to_boxed_value(false), _instance_id(instance_id), - _speculative(speculative) { + _speculative(speculative), + _inline_depth(inline_depth){ if (Compile::current()->eliminate_boxing() && (t == InstPtr) && (offset > 0) && xk && (k != 0) && k->is_instance_klass()) { _is_ptr_to_boxed_value = k->as_instance_klass()->is_boxed_value_offset(offset); @@ -2528,12 +2533,12 @@ //------------------------------make------------------------------------------- const TypeOopPtr *TypeOopPtr::make(PTR ptr, - int offset, int instance_id, const TypeOopPtr* speculative) { + int offset, int instance_id, const TypeOopPtr* speculative, int inline_depth) { assert(ptr != Constant, "no constant generic pointers"); ciKlass* k = Compile::current()->env()->Object_klass(); bool xk = false; ciObject* o = NULL; - return (TypeOopPtr*)(new TypeOopPtr(OopPtr, ptr, k, xk, o, offset, instance_id, speculative))->hashcons(); + return (TypeOopPtr*)(new TypeOopPtr(OopPtr, ptr, k, xk, o, offset, instance_id, speculative, inline_depth))->hashcons(); } @@ -2541,7 +2546,7 @@ const Type *TypeOopPtr::cast_to_ptr_type(PTR ptr) const { assert(_base == OopPtr, "subclass must override cast_to_ptr_type"); if( ptr == _ptr ) return this; - return make(ptr, _offset, _instance_id, _speculative); + return make(ptr, _offset, _instance_id, _speculative, _inline_depth); } //-----------------------------cast_to_instance_id---------------------------- @@ -2638,7 +2643,7 @@ case AnyNull: { int instance_id = meet_instance_id(InstanceTop); const TypeOopPtr* speculative = _speculative; - return make(ptr, offset, instance_id, speculative); + return make(ptr, offset, instance_id, speculative, _inline_depth); } case BotPTR: case NotNull: @@ -2651,7 +2656,8 @@ const TypeOopPtr *tp = t->is_oopptr(); int instance_id = meet_instance_id(tp->instance_id()); const TypeOopPtr* speculative = xmeet_speculative(tp); - return make(meet_ptr(tp->ptr()), meet_offset(tp->offset()), instance_id, speculative); + int depth = meet_inline_depth(tp->inline_depth()); + return make(meet_ptr(tp->ptr()), meet_offset(tp->offset()), instance_id, speculative, depth); } case InstPtr: // For these, flip the call around to cut down @@ -2668,7 +2674,7 @@ const Type *TypeOopPtr::xdual() const { assert(klass() == Compile::current()->env()->Object_klass(), "no klasses here"); assert(const_oop() == NULL, "no constants here"); - return new TypeOopPtr(_base, dual_ptr(), klass(), klass_is_exact(), const_oop(), dual_offset(), dual_instance_id(), dual_speculative()); + return new TypeOopPtr(_base, dual_ptr(), klass(), klass_is_exact(), const_oop(), dual_offset(), dual_instance_id(), dual_speculative(), dual_inline_depth()); } //--------------------------make_from_klass_common----------------------------- @@ -2759,7 +2765,7 @@ } else if (!o->should_be_constant()) { return TypeAryPtr::make(TypePtr::NotNull, arr0, klass, true, 0); } - const TypeAryPtr* arr = TypeAryPtr::make(TypePtr::Constant, o, arr0, klass, true, 0, InstanceBot, NULL, is_autobox_cache); + const TypeAryPtr* arr = TypeAryPtr::make(TypePtr::Constant, o, arr0, klass, true, 0, InstanceBot, NULL, InlineDepthBottom, is_autobox_cache); return arr; } else if (klass->is_type_array_klass()) { // Element is an typeArray @@ -2848,7 +2854,8 @@ const TypeOopPtr *a = (const TypeOopPtr*)t; if (_klass_is_exact != a->_klass_is_exact || _instance_id != a->_instance_id || - !eq_speculative(a)) return false; + !eq_speculative(a) || + _inline_depth != a->_inline_depth) return false; ciObject* one = const_oop(); ciObject* two = a->const_oop(); if (one == NULL || two == NULL) { @@ -2866,6 +2873,7 @@ _klass_is_exact + _instance_id + hash_speculative() + + _inline_depth + TypePtr::hash(); } @@ -2886,6 +2894,7 @@ else if (_instance_id != InstanceBot) st->print(",iid=%d",_instance_id); + dump_inline_depth(st); dump_speculative(st); } @@ -2899,6 +2908,16 @@ st->print(")"); } } + +void TypeOopPtr::dump_inline_depth(outputStream *st) const { + if (_inline_depth != InlineDepthBottom) { + if (_inline_depth == InlineDepthTop) { + st->print(" (inline_depth=InlineDepthTop)"); + } else { + st->print(" (inline_depth=%d)", _inline_depth); + } + } +} #endif //------------------------------singleton-------------------------------------- @@ -2912,7 +2931,7 @@ //------------------------------add_offset------------------------------------- const TypePtr *TypeOopPtr::add_offset(intptr_t offset) const { - return make(_ptr, xadd_offset(offset), _instance_id, add_offset_speculative(offset)); + return make(_ptr, xadd_offset(offset), _instance_id, add_offset_speculative(offset), _inline_depth); } /** @@ -2922,7 +2941,52 @@ if (_speculative == NULL) { return this; } - return make(_ptr, _offset, _instance_id, NULL); + assert(_inline_depth == InlineDepthTop || _inline_depth == InlineDepthBottom, "non speculative type shouldn't have inline depth"); + return make(_ptr, _offset, _instance_id, NULL, _inline_depth); +} + +/** + * Return same type but with a different inline depth (used for speculation) + * + * @param depth depth to meet with + */ +const TypeOopPtr* TypeOopPtr::with_inline_depth(int depth) const { + if (!UseInlineDepthForSpeculativeTypes) { + return this; + } + return make(_ptr, _offset, _instance_id, _speculative, depth); +} + +/** + * Check whether new profiling would improve speculative type + * + * @param exact_kls class from profiling + * @param inline_depth inlining depth of profile point + * + * @return true if type profile is valuable + */ +bool TypeOopPtr::would_improve_type(ciKlass* exact_kls, int inline_depth) const { + // no way to improve an already exact type + if (klass_is_exact()) { + return false; + } + // no profiling? + if (exact_kls == NULL) { + return false; + } + // no speculative type or non exact speculative type? + if (speculative_type() == NULL) { + return true; + } + // If the node already has an exact speculative type keep it, + // unless it was provided by profiling that is at a deeper + // inlining level. Profiling at a higher inlining depth is + // expected to be less accurate. + if (_speculative->inline_depth() == InlineDepthBottom) { + return false; + } + assert(_speculative->inline_depth() != InlineDepthTop, "can't do the comparison"); + return inline_depth < _speculative->inline_depth(); } //------------------------------meet_instance_id-------------------------------- @@ -3025,6 +3089,21 @@ return _speculative->hash(); } +/** + * dual of the inline depth for this type (used for speculation) + */ +int TypeOopPtr::dual_inline_depth() const { + return -inline_depth(); +} + +/** + * meet of 2 inline depth (used for speculation) + * + * @param depth depth to meet with + */ +int TypeOopPtr::meet_inline_depth(int depth) const { + return MAX2(inline_depth(), depth); +} //============================================================================= // Convenience common pre-built types. @@ -3035,8 +3114,8 @@ const TypeInstPtr *TypeInstPtr::KLASS; //------------------------------TypeInstPtr------------------------------------- -TypeInstPtr::TypeInstPtr(PTR ptr, ciKlass* k, bool xk, ciObject* o, int off, int instance_id, const TypeOopPtr* speculative) - : TypeOopPtr(InstPtr, ptr, k, xk, o, off, instance_id, speculative), _name(k->name()) { +TypeInstPtr::TypeInstPtr(PTR ptr, ciKlass* k, bool xk, ciObject* o, int off, int instance_id, const TypeOopPtr* speculative, int inline_depth) + : TypeOopPtr(InstPtr, ptr, k, xk, o, off, instance_id, speculative, inline_depth), _name(k->name()) { assert(k != NULL && (k->is_loaded() || o == NULL), "cannot have constants with non-loaded klass"); @@ -3049,7 +3128,8 @@ ciObject* o, int offset, int instance_id, - const TypeOopPtr* speculative) { + const TypeOopPtr* speculative, + int inline_depth) { assert( !k->is_loaded() || k->is_instance_klass(), "Must be for instance"); // Either const_oop() is NULL or else ptr is Constant assert( (!o && ptr != Constant) || (o && ptr == Constant), @@ -3070,7 +3150,7 @@ // Now hash this baby TypeInstPtr *result = - (TypeInstPtr*)(new TypeInstPtr(ptr, k, xk, o ,offset, instance_id, speculative))->hashcons(); + (TypeInstPtr*)(new TypeInstPtr(ptr, k, xk, o ,offset, instance_id, speculative, inline_depth))->hashcons(); return result; } @@ -3103,7 +3183,7 @@ if( ptr == _ptr ) return this; // Reconstruct _sig info here since not a problem with later lazy // construction, _sig will show up on demand. - return make(ptr, klass(), klass_is_exact(), const_oop(), _offset, _instance_id, _speculative); + return make(ptr, klass(), klass_is_exact(), const_oop(), _offset, _instance_id, _speculative, _inline_depth); } @@ -3115,13 +3195,13 @@ ciInstanceKlass* ik = _klass->as_instance_klass(); if( (ik->is_final() || _const_oop) ) return this; // cannot clear xk if( ik->is_interface() ) return this; // cannot set xk - return make(ptr(), klass(), klass_is_exact, const_oop(), _offset, _instance_id, _speculative); + return make(ptr(), klass(), klass_is_exact, const_oop(), _offset, _instance_id, _speculative, _inline_depth); } //-----------------------------cast_to_instance_id---------------------------- const TypeOopPtr *TypeInstPtr::cast_to_instance_id(int instance_id) const { if( instance_id == _instance_id ) return this; - return make(_ptr, klass(), _klass_is_exact, const_oop(), _offset, instance_id, _speculative); + return make(_ptr, klass(), _klass_is_exact, const_oop(), _offset, instance_id, _speculative, _inline_depth); } //------------------------------xmeet_unloaded--------------------------------- @@ -3132,6 +3212,7 @@ PTR ptr = meet_ptr(tinst->ptr()); int instance_id = meet_instance_id(tinst->instance_id()); const TypeOopPtr* speculative = xmeet_speculative(tinst); + int depth = meet_inline_depth(tinst->inline_depth()); const TypeInstPtr *loaded = is_loaded() ? this : tinst; const TypeInstPtr *unloaded = is_loaded() ? tinst : this; @@ -3152,7 +3233,7 @@ assert(loaded->ptr() != TypePtr::Null, "insanity check"); // if( loaded->ptr() == TypePtr::TopPTR ) { return unloaded; } - else if (loaded->ptr() == TypePtr::AnyNull) { return TypeInstPtr::make(ptr, unloaded->klass(), false, NULL, off, instance_id, speculative); } + else if (loaded->ptr() == TypePtr::AnyNull) { return TypeInstPtr::make(ptr, unloaded->klass(), false, NULL, off, instance_id, speculative, depth); } else if (loaded->ptr() == TypePtr::BotPTR ) { return TypeInstPtr::BOTTOM; } else if (loaded->ptr() == TypePtr::Constant || loaded->ptr() == TypePtr::NotNull) { if (unloaded->ptr() == TypePtr::BotPTR ) { return TypeInstPtr::BOTTOM; } @@ -3209,6 +3290,7 @@ PTR ptr = meet_ptr(tp->ptr()); int instance_id = meet_instance_id(tp->instance_id()); const TypeOopPtr* speculative = xmeet_speculative(tp); + int depth = meet_inline_depth(tp->inline_depth()); switch (ptr) { case TopPTR: case AnyNull: // Fall 'down' to dual of object klass @@ -3216,12 +3298,12 @@ // below the centerline when the superclass is exact. We need to // do the same here. if (klass()->equals(ciEnv::current()->Object_klass()) && !klass_is_exact()) { - return TypeAryPtr::make(ptr, tp->ary(), tp->klass(), tp->klass_is_exact(), offset, instance_id, speculative); + return TypeAryPtr::make(ptr, tp->ary(), tp->klass(), tp->klass_is_exact(), offset, instance_id, speculative, depth); } else { // cannot subclass, so the meet has to fall badly below the centerline ptr = NotNull; instance_id = InstanceBot; - return TypeInstPtr::make( ptr, ciEnv::current()->Object_klass(), false, NULL, offset, instance_id, speculative); + return TypeInstPtr::make( ptr, ciEnv::current()->Object_klass(), false, NULL, offset, instance_id, speculative, depth); } case Constant: case NotNull: @@ -3236,7 +3318,7 @@ if (klass()->equals(ciEnv::current()->Object_klass()) && !klass_is_exact()) { // that is, tp's array type is a subtype of my klass return TypeAryPtr::make(ptr, (ptr == Constant ? tp->const_oop() : NULL), - tp->ary(), tp->klass(), tp->klass_is_exact(), offset, instance_id, speculative); + tp->ary(), tp->klass(), tp->klass_is_exact(), offset, instance_id, speculative, depth); } } // The other case cannot happen, since I cannot be a subtype of an array. @@ -3244,7 +3326,7 @@ if( ptr == Constant ) ptr = NotNull; instance_id = InstanceBot; - return make(ptr, ciEnv::current()->Object_klass(), false, NULL, offset, instance_id, speculative); + return make(ptr, ciEnv::current()->Object_klass(), false, NULL, offset, instance_id, speculative, depth); default: typerr(t); } } @@ -3259,14 +3341,16 @@ case AnyNull: { int instance_id = meet_instance_id(InstanceTop); const TypeOopPtr* speculative = xmeet_speculative(tp); + int depth = meet_inline_depth(tp->inline_depth()); return make(ptr, klass(), klass_is_exact(), - (ptr == Constant ? const_oop() : NULL), offset, instance_id, speculative); + (ptr == Constant ? const_oop() : NULL), offset, instance_id, speculative, depth); } case NotNull: case BotPTR: { int instance_id = meet_instance_id(tp->instance_id()); const TypeOopPtr* speculative = xmeet_speculative(tp); - return TypeOopPtr::make(ptr, offset, instance_id, speculative); + int depth = meet_inline_depth(tp->inline_depth()); + return TypeOopPtr::make(ptr, offset, instance_id, speculative, depth); } default: typerr(t); } @@ -3286,7 +3370,7 @@ int instance_id = meet_instance_id(InstanceTop); const TypeOopPtr* speculative = _speculative; return make(ptr, klass(), klass_is_exact(), - (ptr == Constant ? const_oop() : NULL), offset, instance_id, speculative); + (ptr == Constant ? const_oop() : NULL), offset, instance_id, speculative, _inline_depth); } case NotNull: case BotPTR: @@ -3318,13 +3402,14 @@ PTR ptr = meet_ptr( tinst->ptr() ); int instance_id = meet_instance_id(tinst->instance_id()); const TypeOopPtr* speculative = xmeet_speculative(tinst); + int depth = meet_inline_depth(tinst->inline_depth()); // Check for easy case; klasses are equal (and perhaps not loaded!) // If we have constants, then we created oops so classes are loaded // and we can handle the constants further down. This case handles // both-not-loaded or both-loaded classes if (ptr != Constant && klass()->equals(tinst->klass()) && klass_is_exact() == tinst->klass_is_exact()) { - return make(ptr, klass(), klass_is_exact(), NULL, off, instance_id, speculative); + return make(ptr, klass(), klass_is_exact(), NULL, off, instance_id, speculative, depth); } // Classes require inspection in the Java klass hierarchy. Must be loaded. @@ -3388,7 +3473,7 @@ // Find out which constant. o = (this_klass == klass()) ? const_oop() : tinst->const_oop(); } - return make(ptr, k, xk, o, off, instance_id, speculative); + return make(ptr, k, xk, o, off, instance_id, speculative, depth); } // Either oop vs oop or interface vs interface or interface vs Object @@ -3465,7 +3550,7 @@ else ptr = NotNull; } - return make(ptr, this_klass, this_xk, o, off, instance_id, speculative); + return make(ptr, this_klass, this_xk, o, off, instance_id, speculative, depth); } // Else classes are not equal // Since klasses are different, we require a LCA in the Java @@ -3476,7 +3561,7 @@ // Now we find the LCA of Java classes ciKlass* k = this_klass->least_common_ancestor(tinst_klass); - return make(ptr, k, false, NULL, off, instance_id, speculative); + return make(ptr, k, false, NULL, off, instance_id, speculative, depth); } // End of case InstPtr } // End of switch @@ -3500,7 +3585,7 @@ // Dual: do NOT dual on klasses. This means I do NOT understand the Java // inheritance mechanism. const Type *TypeInstPtr::xdual() const { - return new TypeInstPtr(dual_ptr(), klass(), klass_is_exact(), const_oop(), dual_offset(), dual_instance_id(), dual_speculative()); + return new TypeInstPtr(dual_ptr(), klass(), klass_is_exact(), const_oop(), dual_offset(), dual_instance_id(), dual_speculative(), dual_inline_depth()); } //------------------------------eq--------------------------------------------- @@ -3557,6 +3642,7 @@ else if (_instance_id != InstanceBot) st->print(",iid=%d",_instance_id); + dump_inline_depth(st); dump_speculative(st); } #endif @@ -3570,7 +3656,15 @@ if (_speculative == NULL) { return this; } - return make(_ptr, klass(), klass_is_exact(), const_oop(), _offset, _instance_id, NULL); + assert(_inline_depth == InlineDepthTop || _inline_depth == InlineDepthBottom, "non speculative type shouldn't have inline depth"); + return make(_ptr, klass(), klass_is_exact(), const_oop(), _offset, _instance_id, NULL, _inline_depth); +} + +const TypeOopPtr *TypeInstPtr::with_inline_depth(int depth) const { + if (!UseInlineDepthForSpeculativeTypes) { + return this; + } + return make(_ptr, klass(), klass_is_exact(), const_oop(), _offset, _instance_id, _speculative, depth); } //============================================================================= @@ -3587,30 +3681,30 @@ const TypeAryPtr *TypeAryPtr::DOUBLES; //------------------------------make------------------------------------------- -const TypeAryPtr *TypeAryPtr::make(PTR ptr, const TypeAry *ary, ciKlass* k, bool xk, int offset, int instance_id, const TypeOopPtr* speculative) { +const TypeAryPtr *TypeAryPtr::make(PTR ptr, const TypeAry *ary, ciKlass* k, bool xk, int offset, int instance_id, const TypeOopPtr* speculative, int inline_depth) { assert(!(k == NULL && ary->_elem->isa_int()), "integral arrays must be pre-equipped with a class"); if (!xk) xk = ary->ary_must_be_exact(); assert(instance_id <= 0 || xk || !UseExactTypes, "instances are always exactly typed"); if (!UseExactTypes) xk = (ptr == Constant); - return (TypeAryPtr*)(new TypeAryPtr(ptr, NULL, ary, k, xk, offset, instance_id, false, speculative))->hashcons(); + return (TypeAryPtr*)(new TypeAryPtr(ptr, NULL, ary, k, xk, offset, instance_id, false, speculative, inline_depth))->hashcons(); } //------------------------------make------------------------------------------- -const TypeAryPtr *TypeAryPtr::make(PTR ptr, ciObject* o, const TypeAry *ary, ciKlass* k, bool xk, int offset, int instance_id, const TypeOopPtr* speculative, bool is_autobox_cache) { +const TypeAryPtr *TypeAryPtr::make(PTR ptr, ciObject* o, const TypeAry *ary, ciKlass* k, bool xk, int offset, int instance_id, const TypeOopPtr* speculative, int inline_depth, bool is_autobox_cache) { assert(!(k == NULL && ary->_elem->isa_int()), "integral arrays must be pre-equipped with a class"); assert( (ptr==Constant && o) || (ptr!=Constant && !o), "" ); if (!xk) xk = (o != NULL) || ary->ary_must_be_exact(); assert(instance_id <= 0 || xk || !UseExactTypes, "instances are always exactly typed"); if (!UseExactTypes) xk = (ptr == Constant); - return (TypeAryPtr*)(new TypeAryPtr(ptr, o, ary, k, xk, offset, instance_id, is_autobox_cache, speculative))->hashcons(); + return (TypeAryPtr*)(new TypeAryPtr(ptr, o, ary, k, xk, offset, instance_id, is_autobox_cache, speculative, inline_depth))->hashcons(); } //------------------------------cast_to_ptr_type------------------------------- const Type *TypeAryPtr::cast_to_ptr_type(PTR ptr) const { if( ptr == _ptr ) return this; - return make(ptr, const_oop(), _ary, klass(), klass_is_exact(), _offset, _instance_id, _speculative); + return make(ptr, const_oop(), _ary, klass(), klass_is_exact(), _offset, _instance_id, _speculative, _inline_depth); } @@ -3619,13 +3713,13 @@ if( klass_is_exact == _klass_is_exact ) return this; if (!UseExactTypes) return this; if (_ary->ary_must_be_exact()) return this; // cannot clear xk - return make(ptr(), const_oop(), _ary, klass(), klass_is_exact, _offset, _instance_id, _speculative); + return make(ptr(), const_oop(), _ary, klass(), klass_is_exact, _offset, _instance_id, _speculative, _inline_depth); } //-----------------------------cast_to_instance_id---------------------------- const TypeOopPtr *TypeAryPtr::cast_to_instance_id(int instance_id) const { if( instance_id == _instance_id ) return this; - return make(_ptr, const_oop(), _ary, klass(), _klass_is_exact, _offset, instance_id, _speculative); + return make(_ptr, const_oop(), _ary, klass(), _klass_is_exact, _offset, instance_id, _speculative, _inline_depth); } //-----------------------------narrow_size_type------------------------------- @@ -3688,7 +3782,7 @@ new_size = narrow_size_type(new_size); if (new_size == size()) return this; const TypeAry* new_ary = TypeAry::make(elem(), new_size, is_stable()); - return make(ptr(), const_oop(), new_ary, klass(), klass_is_exact(), _offset, _instance_id, _speculative); + return make(ptr(), const_oop(), new_ary, klass(), klass_is_exact(), _offset, _instance_id, _speculative, _inline_depth); } @@ -3767,19 +3861,20 @@ const TypeOopPtr *tp = t->is_oopptr(); int offset = meet_offset(tp->offset()); PTR ptr = meet_ptr(tp->ptr()); + int depth = meet_inline_depth(tp->inline_depth()); switch (tp->ptr()) { case TopPTR: case AnyNull: { int instance_id = meet_instance_id(InstanceTop); const TypeOopPtr* speculative = xmeet_speculative(tp); return make(ptr, (ptr == Constant ? const_oop() : NULL), - _ary, _klass, _klass_is_exact, offset, instance_id, speculative); + _ary, _klass, _klass_is_exact, offset, instance_id, speculative, depth); } case BotPTR: case NotNull: { int instance_id = meet_instance_id(tp->instance_id()); const TypeOopPtr* speculative = xmeet_speculative(tp); - return TypeOopPtr::make(ptr, offset, instance_id, speculative); + return TypeOopPtr::make(ptr, offset, instance_id, speculative, depth); } default: ShouldNotReachHere(); } @@ -3803,7 +3898,7 @@ int instance_id = meet_instance_id(InstanceTop); const TypeOopPtr* speculative = _speculative; return make(ptr, (ptr == Constant ? const_oop() : NULL), - _ary, _klass, _klass_is_exact, offset, instance_id, speculative); + _ary, _klass, _klass_is_exact, offset, instance_id, speculative, _inline_depth); } default: ShouldNotReachHere(); } @@ -3820,6 +3915,7 @@ PTR ptr = meet_ptr(tap->ptr()); int instance_id = meet_instance_id(tap->instance_id()); const TypeOopPtr* speculative = xmeet_speculative(tap); + int depth = meet_inline_depth(tap->inline_depth()); ciKlass* lazy_klass = NULL; if (tary->_elem->isa_int()) { // Integral array element types have irrelevant lattice relations. @@ -3860,7 +3956,7 @@ } else { xk = (tap->_klass_is_exact | this->_klass_is_exact); } - return make(ptr, const_oop(), tary, lazy_klass, xk, off, instance_id, speculative); + return make(ptr, const_oop(), tary, lazy_klass, xk, off, instance_id, speculative, depth); case Constant: { ciObject* o = const_oop(); if( _ptr == Constant ) { @@ -3879,7 +3975,7 @@ // Only precise for identical arrays xk = this->_klass_is_exact && (klass() == tap->klass()); } - return TypeAryPtr::make(ptr, o, tary, lazy_klass, xk, off, instance_id, speculative); + return TypeAryPtr::make(ptr, o, tary, lazy_klass, xk, off, instance_id, speculative, depth); } case NotNull: case BotPTR: @@ -3888,7 +3984,7 @@ xk = tap->_klass_is_exact; else xk = (tap->_klass_is_exact & this->_klass_is_exact) && (klass() == tap->klass()); // Only precise for identical arrays - return TypeAryPtr::make(ptr, NULL, tary, lazy_klass, xk, off, instance_id, speculative); + return TypeAryPtr::make(ptr, NULL, tary, lazy_klass, xk, off, instance_id, speculative, depth); default: ShouldNotReachHere(); } } @@ -3900,6 +3996,7 @@ PTR ptr = meet_ptr(tp->ptr()); int instance_id = meet_instance_id(tp->instance_id()); const TypeOopPtr* speculative = xmeet_speculative(tp); + int depth = meet_inline_depth(tp->inline_depth()); switch (ptr) { case TopPTR: case AnyNull: // Fall 'down' to dual of object klass @@ -3907,12 +4004,12 @@ // below the centerline when the superclass is exact. We need to // do the same here. if (tp->klass()->equals(ciEnv::current()->Object_klass()) && !tp->klass_is_exact()) { - return TypeAryPtr::make(ptr, _ary, _klass, _klass_is_exact, offset, instance_id, speculative); + return TypeAryPtr::make(ptr, _ary, _klass, _klass_is_exact, offset, instance_id, speculative, depth); } else { // cannot subclass, so the meet has to fall badly below the centerline ptr = NotNull; instance_id = InstanceBot; - return TypeInstPtr::make(ptr, ciEnv::current()->Object_klass(), false, NULL,offset, instance_id, speculative); + return TypeInstPtr::make(ptr, ciEnv::current()->Object_klass(), false, NULL,offset, instance_id, speculative, depth); } case Constant: case NotNull: @@ -3927,7 +4024,7 @@ if (tp->klass()->equals(ciEnv::current()->Object_klass()) && !tp->klass_is_exact()) { // that is, my array type is a subtype of 'tp' klass return make(ptr, (ptr == Constant ? const_oop() : NULL), - _ary, _klass, _klass_is_exact, offset, instance_id, speculative); + _ary, _klass, _klass_is_exact, offset, instance_id, speculative, depth); } } // The other case cannot happen, since t cannot be a subtype of an array. @@ -3935,7 +4032,7 @@ if( ptr == Constant ) ptr = NotNull; instance_id = InstanceBot; - return TypeInstPtr::make(ptr, ciEnv::current()->Object_klass(), false, NULL,offset, instance_id, speculative); + return TypeInstPtr::make(ptr, ciEnv::current()->Object_klass(), false, NULL,offset, instance_id, speculative, depth); default: typerr(t); } } @@ -3946,7 +4043,7 @@ //------------------------------xdual------------------------------------------ // Dual: compute field-by-field dual const Type *TypeAryPtr::xdual() const { - return new TypeAryPtr(dual_ptr(), _const_oop, _ary->dual()->is_ary(),_klass, _klass_is_exact, dual_offset(), dual_instance_id(), is_autobox_cache(), dual_speculative()); + return new TypeAryPtr(dual_ptr(), _const_oop, _ary->dual()->is_ary(),_klass, _klass_is_exact, dual_offset(), dual_instance_id(), is_autobox_cache(), dual_speculative(), dual_inline_depth()); } //----------------------interface_vs_oop--------------------------------------- @@ -3999,6 +4096,7 @@ else if (_instance_id != InstanceBot) st->print(",iid=%d",_instance_id); + dump_inline_depth(st); dump_speculative(st); } #endif @@ -4010,11 +4108,22 @@ //------------------------------add_offset------------------------------------- const TypePtr *TypeAryPtr::add_offset(intptr_t offset) const { - return make(_ptr, _const_oop, _ary, _klass, _klass_is_exact, xadd_offset(offset), _instance_id, add_offset_speculative(offset)); + return make(_ptr, _const_oop, _ary, _klass, _klass_is_exact, xadd_offset(offset), _instance_id, add_offset_speculative(offset), _inline_depth); } const Type *TypeAryPtr::remove_speculative() const { - return make(_ptr, _const_oop, _ary->remove_speculative()->is_ary(), _klass, _klass_is_exact, _offset, _instance_id, NULL); + if (_speculative == NULL) { + return this; + } + assert(_inline_depth == InlineDepthTop || _inline_depth == InlineDepthBottom, "non speculative type shouldn't have inline depth"); + return make(_ptr, _const_oop, _ary->remove_speculative()->is_ary(), _klass, _klass_is_exact, _offset, _instance_id, NULL, _inline_depth); +} + +const TypeOopPtr *TypeAryPtr::with_inline_depth(int depth) const { + if (!UseInlineDepthForSpeculativeTypes) { + return this; + } + return make(_ptr, _const_oop, _ary->remove_speculative()->is_ary(), _klass, _klass_is_exact, _offset, _instance_id, _speculative, depth); } //============================================================================= diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/opto/type.hpp --- a/src/share/vm/opto/type.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/opto/type.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -415,10 +415,15 @@ bool is_autobox_cache = false); // Speculative type. See TypeInstPtr + virtual const TypeOopPtr* speculative() const { return NULL; } virtual ciKlass* speculative_type() const { return NULL; } const Type* maybe_remove_speculative(bool include_speculative) const; virtual const Type* remove_speculative() const { return this; } + virtual bool would_improve_type(ciKlass* exact_kls, int inline_depth) const { + return exact_kls != NULL; + } + private: // support arrays static const BasicType _basic_type[]; @@ -489,6 +494,7 @@ virtual const Type *filter_helper(const Type *kills, bool include_speculative) const; public: + typedef jint NativeType; virtual bool eq( const Type *t ) const; virtual int hash() const; // Type specific hashing virtual bool singleton(void) const; // TRUE if type is a singleton @@ -531,6 +537,9 @@ static const TypeInt *POS1; static const TypeInt *INT; static const TypeInt *SYMINT; // symmetric range [-max_jint..max_jint] + static const TypeInt *TYPE_DOMAIN; // alias for TypeInt::INT + + static const TypeInt *as_self(const Type *t) { return t->is_int(); } #ifndef PRODUCT virtual void dump2( Dict &d, uint depth, outputStream *st ) const; #endif @@ -546,6 +555,7 @@ // Do not kill _widen bits. virtual const Type *filter_helper(const Type *kills, bool include_speculative) const; public: + typedef jlong NativeType; virtual bool eq( const Type *t ) const; virtual int hash() const; // Type specific hashing virtual bool singleton(void) const; // TRUE if type is a singleton @@ -565,6 +575,7 @@ virtual bool is_finite() const; // Has a finite value + virtual const Type *xmeet( const Type *t ) const; virtual const Type *xdual() const; // Compute dual right now. virtual const Type *widen( const Type *t, const Type* limit_type ) const; @@ -577,6 +588,11 @@ static const TypeLong *LONG; static const TypeLong *INT; // 32-bit subrange [min_jint..max_jint] static const TypeLong *UINT; // 32-bit unsigned [0..max_juint] + static const TypeLong *TYPE_DOMAIN; // alias for TypeLong::LONG + + // static convenience methods. + static const TypeLong *as_self(const Type *t) { return t->is_long(); } + #ifndef PRODUCT virtual void dump2( Dict &d, uint, outputStream *st ) const;// Specialized per-Type dumping #endif @@ -831,7 +847,7 @@ // Some kind of oop (Java pointer), either klass or instance or array. class TypeOopPtr : public TypePtr { protected: - TypeOopPtr(TYPES t, PTR ptr, ciKlass* k, bool xk, ciObject* o, int offset, int instance_id, const TypeOopPtr* speculative); + TypeOopPtr(TYPES t, PTR ptr, ciKlass* k, bool xk, ciObject* o, int offset, int instance_id, const TypeOopPtr* speculative, int inline_depth); public: virtual bool eq( const Type *t ) const; virtual int hash() const; // Type specific hashing @@ -842,6 +858,10 @@ }; protected: + enum { + InlineDepthBottom = INT_MAX, + InlineDepthTop = -InlineDepthBottom + }; // Oop is NULL, unless this is a constant oop. ciObject* _const_oop; // Constant oop // If _klass is NULL, then so is _sig. This is an unloaded klass. @@ -862,6 +882,11 @@ // use it, then we have to emit a guard: this part of the type is // not something we know but something we speculate about the type. const TypeOopPtr* _speculative; + // For speculative types, we record at what inlining depth the + // profiling point that provided the data is. We want to favor + // profile data coming from outer scopes which are likely better for + // the current compilation. + int _inline_depth; static const TypeOopPtr* make_from_klass_common(ciKlass* klass, bool klass_change, bool try_for_exact); @@ -877,6 +902,12 @@ #ifndef PRODUCT void dump_speculative(outputStream *st) const; #endif + // utility methods to work on the inline depth of the type + int dual_inline_depth() const; + int meet_inline_depth(int depth) const; +#ifndef PRODUCT + void dump_inline_depth(outputStream *st) const; +#endif // Do not allow interface-vs.-noninterface joins to collapse to top. virtual const Type *filter_helper(const Type *kills, bool include_speculative) const; @@ -907,7 +938,7 @@ bool not_null_elements = false); // Make a generic (unclassed) pointer to an oop. - static const TypeOopPtr* make(PTR ptr, int offset, int instance_id, const TypeOopPtr* speculative); + static const TypeOopPtr* make(PTR ptr, int offset, int instance_id, const TypeOopPtr* speculative = NULL, int inline_depth = InlineDepthBottom); ciObject* const_oop() const { return _const_oop; } virtual ciKlass* klass() const { return _klass; } @@ -921,7 +952,7 @@ bool is_known_instance() const { return _instance_id > 0; } int instance_id() const { return _instance_id; } bool is_known_instance_field() const { return is_known_instance() && _offset >= 0; } - const TypeOopPtr* speculative() const { return _speculative; } + virtual const TypeOopPtr* speculative() const { return _speculative; } virtual intptr_t get_con() const; @@ -954,18 +985,23 @@ if (_speculative != NULL) { const TypeOopPtr* speculative = _speculative->join(this)->is_oopptr(); if (speculative->klass_is_exact()) { - return speculative->klass(); + return speculative->klass(); } } return NULL; } + int inline_depth() const { + return _inline_depth; + } + virtual const TypeOopPtr* with_inline_depth(int depth) const; + virtual bool would_improve_type(ciKlass* exact_kls, int inline_depth) const; }; //------------------------------TypeInstPtr------------------------------------ // Class of Java object pointers, pointing either to non-array Java instances // or to a Klass* (including array klasses). class TypeInstPtr : public TypeOopPtr { - TypeInstPtr(PTR ptr, ciKlass* k, bool xk, ciObject* o, int offset, int instance_id, const TypeOopPtr* speculative); + TypeInstPtr(PTR ptr, ciKlass* k, bool xk, ciObject* o, int offset, int instance_id, const TypeOopPtr* speculative, int inline_depth); virtual bool eq( const Type *t ) const; virtual int hash() const; // Type specific hashing @@ -1001,7 +1037,7 @@ } // Make a pointer to an oop. - static const TypeInstPtr *make(PTR ptr, ciKlass* k, bool xk, ciObject* o, int offset, int instance_id = InstanceBot, const TypeOopPtr* speculative = NULL); + static const TypeInstPtr *make(PTR ptr, ciKlass* k, bool xk, ciObject* o, int offset, int instance_id = InstanceBot, const TypeOopPtr* speculative = NULL, int inline_depth = InlineDepthBottom); /** Create constant type for a constant boxed value */ const Type* get_const_boxed_value() const; @@ -1020,6 +1056,7 @@ virtual const TypePtr *add_offset( intptr_t offset ) const; // Return same type without a speculative part virtual const Type* remove_speculative() const; + virtual const TypeOopPtr* with_inline_depth(int depth) const; // the core of the computation of the meet of 2 types virtual const Type *xmeet_helper(const Type *t) const; @@ -1041,8 +1078,8 @@ // Class of Java array pointers class TypeAryPtr : public TypeOopPtr { TypeAryPtr( PTR ptr, ciObject* o, const TypeAry *ary, ciKlass* k, bool xk, - int offset, int instance_id, bool is_autobox_cache, const TypeOopPtr* speculative) - : TypeOopPtr(AryPtr,ptr,k,xk,o,offset, instance_id, speculative), + int offset, int instance_id, bool is_autobox_cache, const TypeOopPtr* speculative, int inline_depth) + : TypeOopPtr(AryPtr,ptr,k,xk,o,offset, instance_id, speculative, inline_depth), _ary(ary), _is_autobox_cache(is_autobox_cache) { @@ -1080,9 +1117,9 @@ bool is_autobox_cache() const { return _is_autobox_cache; } - static const TypeAryPtr *make( PTR ptr, const TypeAry *ary, ciKlass* k, bool xk, int offset, int instance_id = InstanceBot, const TypeOopPtr* speculative = NULL); + static const TypeAryPtr *make( PTR ptr, const TypeAry *ary, ciKlass* k, bool xk, int offset, int instance_id = InstanceBot, const TypeOopPtr* speculative = NULL, int inline_depth = InlineDepthBottom); // Constant pointer to array - static const TypeAryPtr *make( PTR ptr, ciObject* o, const TypeAry *ary, ciKlass* k, bool xk, int offset, int instance_id = InstanceBot, const TypeOopPtr* speculative = NULL, bool is_autobox_cache = false); + static const TypeAryPtr *make( PTR ptr, ciObject* o, const TypeAry *ary, ciKlass* k, bool xk, int offset, int instance_id = InstanceBot, const TypeOopPtr* speculative = NULL, int inline_depth = InlineDepthBottom, bool is_autobox_cache= false); // Return a 'ptr' version of this type virtual const Type *cast_to_ptr_type(PTR ptr) const; @@ -1098,6 +1135,7 @@ virtual const TypePtr *add_offset( intptr_t offset ) const; // Return same type without a speculative part virtual const Type* remove_speculative() const; + virtual const TypeOopPtr* with_inline_depth(int depth) const; // the core of the computation of the meet of 2 types virtual const Type *xmeet_helper(const Type *t) const; diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/runtime/deoptimization.cpp --- a/src/share/vm/runtime/deoptimization.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/runtime/deoptimization.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -1486,6 +1486,7 @@ bool maybe_prior_trap = false; bool maybe_prior_recompile = false; pdata = query_update_method_data(trap_mdo, trap_bci, reason, + nm->method(), //outputs: this_trap_count, maybe_prior_trap, @@ -1531,7 +1532,7 @@ } // Go back to the compiler if there are too many traps in this method. - if (this_trap_count >= (uint)PerMethodTrapLimit) { + if (this_trap_count >= per_method_trap_limit(reason)) { // If there are too many traps in this method, force a recompile. // This will allow the compiler to see the limit overflow, and // take corrective action, if possible. @@ -1619,6 +1620,7 @@ Deoptimization::query_update_method_data(MethodData* trap_mdo, int trap_bci, Deoptimization::DeoptReason reason, + Method* compiled_method, //outputs: uint& ret_this_trap_count, bool& ret_maybe_prior_trap, @@ -1642,9 +1644,16 @@ // Find the profile data for this BCI. If there isn't one, // try to allocate one from the MDO's set of spares. // This will let us detect a repeated trap at this point. - pdata = trap_mdo->allocate_bci_to_data(trap_bci); + pdata = trap_mdo->allocate_bci_to_data(trap_bci, reason_is_speculate(reason) ? compiled_method : NULL); if (pdata != NULL) { + if (reason_is_speculate(reason) && !pdata->is_SpeculativeTrapData()) { + if (LogCompilation && xtty != NULL) { + ttyLocker ttyl; + // no more room for speculative traps in this MDO + xtty->elem("speculative_traps_oom"); + } + } // Query the trap state of this profile datum. int tstate0 = pdata->trap_state(); if (!trap_state_has_reason(tstate0, per_bc_reason)) @@ -1682,8 +1691,10 @@ uint ignore_this_trap_count; bool ignore_maybe_prior_trap; bool ignore_maybe_prior_recompile; + assert(!reason_is_speculate(reason), "reason speculate only used by compiler"); query_update_method_data(trap_mdo, trap_bci, (DeoptReason)reason, + NULL, ignore_this_trap_count, ignore_maybe_prior_trap, ignore_maybe_prior_recompile); @@ -1811,7 +1822,8 @@ "div0_check", "age", "predicate", - "loop_limit_check" + "loop_limit_check", + "speculate_class_check" }; const char* Deoptimization::_trap_action_name[Action_LIMIT] = { // Note: Keep this in sync. with enum DeoptAction. diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/runtime/deoptimization.hpp --- a/src/share/vm/runtime/deoptimization.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/runtime/deoptimization.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -59,6 +59,7 @@ Reason_age, // nmethod too old; tier threshold reached Reason_predicate, // compiler generated predicate failed Reason_loop_limit_check, // compiler generated loop limits check failed + Reason_speculate_class_check, // saw unexpected object class from type speculation Reason_LIMIT, // Note: Keep this enum in sync. with _trap_reason_name. Reason_RECORDED_LIMIT = Reason_bimorphic // some are not recorded per bc @@ -311,10 +312,23 @@ return reason; else if (reason == Reason_div0_check) // null check due to divide-by-zero? return Reason_null_check; // recorded per BCI as a null check + else if (reason == Reason_speculate_class_check) + return Reason_class_check; else return Reason_none; } + static bool reason_is_speculate(int reason) { + if (reason == Reason_speculate_class_check) { + return true; + } + return false; + } + + static uint per_method_trap_limit(int reason) { + return reason_is_speculate(reason) ? (uint)PerMethodSpecTrapLimit : (uint)PerMethodTrapLimit; + } + static const char* trap_reason_name(int reason); static const char* trap_action_name(int action); // Format like reason='foo' action='bar' index='123'. @@ -337,6 +351,7 @@ static ProfileData* query_update_method_data(MethodData* trap_mdo, int trap_bci, DeoptReason reason, + Method* compiled_method, //outputs: uint& ret_this_trap_count, bool& ret_maybe_prior_trap, diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/runtime/globals.hpp --- a/src/share/vm/runtime/globals.hpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/runtime/globals.hpp Tue Mar 25 10:48:42 2014 -0700 @@ -3055,9 +3055,15 @@ product(intx, PerMethodTrapLimit, 100, \ "Limit on traps (of one kind) in a method (includes inlines)") \ \ + experimental(intx, PerMethodSpecTrapLimit, 5000, \ + "Limit on speculative traps (of one kind) in a method (includes inlines)") \ + \ product(intx, PerBytecodeTrapLimit, 4, \ "Limit on traps (of one kind) at a particular BCI") \ \ + experimental(intx, SpecTrapLimitExtraEntries, 3, \ + "Extra method data trap entries for speculation") \ + \ develop(intx, InlineFrequencyRatio, 20, \ "Ratio of call site execution to caller method invocation") \ \ diff -r 7ce7247df891 -r f0ea4d3df129 src/share/vm/runtime/vmStructs.cpp --- a/src/share/vm/runtime/vmStructs.cpp Mon Mar 24 13:14:23 2014 -0700 +++ b/src/share/vm/runtime/vmStructs.cpp Tue Mar 25 10:48:42 2014 -0700 @@ -1938,15 +1938,6 @@ declare_c2_type(CmpF3Node, CmpFNode) \ declare_c2_type(CmpDNode, CmpNode) \ declare_c2_type(CmpD3Node, CmpDNode) \ - declare_c2_type(MathExactNode, MultiNode) \ - declare_c2_type(MathExactINode, MathExactNode) \ - declare_c2_type(AddExactINode, MathExactINode) \ - declare_c2_type(AddExactLNode, MathExactLNode) \ - declare_c2_type(SubExactINode, MathExactINode) \ - declare_c2_type(SubExactLNode, MathExactLNode) \ - declare_c2_type(NegExactINode, MathExactINode) \ - declare_c2_type(MulExactINode, MathExactINode) \ - declare_c2_type(FlagsProjNode, ProjNode) \ declare_c2_type(BoolNode, Node) \ declare_c2_type(AbsNode, Node) \ declare_c2_type(AbsINode, AbsNode) \ @@ -2027,6 +2018,15 @@ declare_c2_type(ExtractLNode, ExtractNode) \ declare_c2_type(ExtractFNode, ExtractNode) \ declare_c2_type(ExtractDNode, ExtractNode) \ + declare_c2_type(OverflowNode, CmpNode) \ + declare_c2_type(OverflowINode, OverflowNode) \ + declare_c2_type(OverflowAddINode, OverflowINode) \ + declare_c2_type(OverflowSubINode, OverflowINode) \ + declare_c2_type(OverflowMulINode, OverflowINode) \ + declare_c2_type(OverflowLNode, OverflowNode) \ + declare_c2_type(OverflowAddLNode, OverflowLNode) \ + declare_c2_type(OverflowSubLNode, OverflowLNode) \ + declare_c2_type(OverflowMulLNode, OverflowLNode) \ \ /*********************/ \ /* Adapter Blob Entries */ \ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/codegen/BMI1.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/compiler/codegen/BMI1.java Tue Mar 25 10:48:42 2014 -0700 @@ -0,0 +1,301 @@ +/* + * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @bug 8031321 + * @summary Support BMI1 instructions on x86/x64 + * @run main/othervm -Xbatch -XX:-TieredCompilation -XX:CompileCommand=compileonly,BMITests.* BMI1 + * + */ + +class MemI { + public int x; + public MemI(int x) { this.x = x; } +} + +class MemL { + public long x; + public MemL(long x) { this.x = x; } +} + +class BMITests { + static int andnl(int src1, int src2) { + return ~src1 & src2; + } + static long andnq(long src1, long src2) { + return ~src1 & src2; + } + static int andnl(int src1, MemI src2) { + return ~src1 & src2.x; + } + static long andnq(long src1, MemL src2) { + return ~src1 & src2.x; + } + static int blsil(int src1) { + return src1 & -src1; + } + static long blsiq(long src1) { + return src1 & -src1; + } + static int blsil(MemI src1) { + return src1.x & -src1.x; + } + static long blsiq(MemL src1) { + return src1.x & -src1.x; + } + static int blsmskl(int src1) { + return (src1 - 1) ^ src1; + } + static long blsmskq(long src1) { + return (src1 - 1) ^ src1; + } + static int blsmskl(MemI src1) { + return (src1.x - 1) ^ src1.x; + } + static long blsmskq(MemL src1) { + return (src1.x - 1) ^ src1.x; + } + static int blsrl(int src1) { + return (src1 - 1) & src1; + } + static long blsrq(long src1) { + return (src1 - 1) & src1; + } + static int blsrl(MemI src1) { + return (src1.x - 1) & src1.x; + } + static long blsrq(MemL src1) { + return (src1.x - 1) & src1.x; + } + static int lzcntl(int src1) { + return Integer.numberOfLeadingZeros(src1); + } + static int lzcntq(long src1) { + return Long.numberOfLeadingZeros(src1); + } + static int tzcntl(int src1) { + return Integer.numberOfTrailingZeros(src1); + } + static int tzcntq(long src1) { + return Long.numberOfTrailingZeros(src1); + } +} + +public class BMI1 { + private final static int ITERATIONS = 1000000; + + public static void main(String[] args) { + int ix = 0x01234567; + int iy = 0x89abcdef; + MemI imy = new MemI(iy); + long lx = 0x0123456701234567L; + long ly = 0x89abcdef89abcdefL; + MemL lmy = new MemL(ly); + + { // match(Set dst (AndI (XorI src1 minus_1) src2)) + int z = BMITests.andnl(ix, iy); + for (int i = 0; i < ITERATIONS; i++) { + int ii = BMITests.andnl(ix, iy); + if (ii != z) { + throw new Error("andnl with register failed"); + } + } + } + { // match(Set dst (AndL (XorL src1 minus_1) src2)) + long z = BMITests.andnq(lx, ly); + for (int i = 0; i < ITERATIONS; i++) { + long ll = BMITests.andnq(lx, ly); + if (ll != z) { + throw new Error("andnq with register failed"); + } + } + } + { // match(Set dst (AndI (XorI src1 minus_1) (LoadI src2))) + int z = BMITests.andnl(ix, imy); + for (int i = 0; i < ITERATIONS; i++) { + int ii = BMITests.andnl(ix, imy); + if (ii != z) { + throw new Error("andnl with memory failed"); + } + } + } + { // match(Set dst (AndL (XorL src1 minus_1) (LoadL src2))) + long z = BMITests.andnq(lx, lmy); + for (int i = 0; i < ITERATIONS; i++) { + long ll = BMITests.andnq(lx, lmy); + if (ll != z) { + throw new Error("andnq with memory failed"); + } + } + } + { // match(Set dst (AndI (SubI imm_zero src) src)) + int z = BMITests.blsil(ix); + for (int i = 0; i < ITERATIONS; i++) { + int ii = BMITests.blsil(ix); + if (ii != z) { + throw new Error("blsil with register failed"); + } + } + } + { // match(Set dst (AndL (SubL imm_zero src) src)) + long z = BMITests.blsiq(lx); + for (int i = 0; i < ITERATIONS; i++) { + long ll = BMITests.blsiq(lx); + if (ll != z) { + throw new Error("blsiq with register failed"); + } + } + } + { // match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) )) + int z = BMITests.blsil(imy); + for (int i = 0; i < ITERATIONS; i++) { + int ii = BMITests.blsil(imy); + if (ii != z) { + throw new Error("blsil with memory failed"); + } + } + } + { // match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) )) + long z = BMITests.blsiq(lmy); + for (int i = 0; i < ITERATIONS; i++) { + long ll = BMITests.blsiq(lmy); + if (ll != z) { + throw new Error("blsiq with memory failed"); + } + } + } + + { // match(Set dst (XorI (AddI src minus_1) src)) + int z = BMITests.blsmskl(ix); + for (int i = 0; i < ITERATIONS; i++) { + int ii = BMITests.blsmskl(ix); + if (ii != z) { + throw new Error("blsmskl with register failed"); + } + } + } + { // match(Set dst (XorL (AddL src minus_1) src)) + long z = BMITests.blsmskq(lx); + for (int i = 0; i < ITERATIONS; i++) { + long ll = BMITests.blsmskq(lx); + if (ll != z) { + throw new Error("blsmskq with register failed"); + } + } + } + { // match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) ) + int z = BMITests.blsmskl(imy); + for (int i = 0; i < ITERATIONS; i++) { + int ii = BMITests.blsmskl(imy); + if (ii != z) { + throw new Error("blsmskl with memory failed"); + } + } + } + { // match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) ) + long z = BMITests.blsmskq(lmy); + for (int i = 0; i < ITERATIONS; i++) { + long ll = BMITests.blsmskq(lmy); + if (ll != z) { + throw new Error("blsmskq with memory failed"); + } + } + } + + { // match(Set dst (AndI (AddI src minus_1) src) ) + int z = BMITests.blsrl(ix); + for (int i = 0; i < ITERATIONS; i++) { + int ii = BMITests.blsrl(ix); + if (ii != z) { + throw new Error("blsrl with register failed"); + } + } + } + { // match(Set dst (AndL (AddL src minus_1) src) ) + long z = BMITests.blsrq(lx); + for (int i = 0; i < ITERATIONS; i++) { + long ll = BMITests.blsrq(lx); + if (ll != z) { + throw new Error("blsrq with register failed"); + } + } + } + { // match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) ) + int z = BMITests.blsrl(imy); + for (int i = 0; i < ITERATIONS; i++) { + int ii = BMITests.blsrl(imy); + if (ii != z) { + throw new Error("blsrl with memory failed"); + } + } + } + { // match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) ) + long z = BMITests.blsrq(lmy); + for (int i = 0; i < ITERATIONS; i++) { + long ll = BMITests.blsrq(lmy); + if (ll != z) { + throw new Error("blsrq with memory failed"); + } + } + } + + { + int z = BMITests.lzcntl(ix); + for (int i = 0; i < ITERATIONS; i++) { + int ii = BMITests.lzcntl(ix); + if (ii != z) { + throw new Error("lzcntl failed"); + } + } + } + { + int z = BMITests.lzcntq(lx); + for (int i = 0; i < ITERATIONS; i++) { + int ii = BMITests.lzcntq(lx); + if (ii != z) { + throw new Error("lzcntq failed"); + } + } + } + + { + int z = BMITests.tzcntl(ix); + for (int i = 0; i < ITERATIONS; i++) { + int ii = BMITests.tzcntl(ix); + if (ii != z) { + throw new Error("tzcntl failed"); + } + } + } + { + int z = BMITests.tzcntq(lx); + for (int i = 0; i < ITERATIONS; i++) { + int ii = BMITests.tzcntq(lx); + if (ii != z) { + throw new Error("tzcntq failed"); + } + } + } + } +} diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/AddExactICondTest.java --- a/test/compiler/intrinsics/mathexact/AddExactICondTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/AddExactICondTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8024924 * @summary Test non constant addExact * @compile AddExactICondTest.java - * @run main AddExactICondTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main AddExactICondTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/AddExactIConstantTest.java --- a/test/compiler/intrinsics/mathexact/AddExactIConstantTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/AddExactIConstantTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8024924 * @summary Test constant addExact * @compile AddExactIConstantTest.java Verify.java - * @run main AddExactIConstantTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main AddExactIConstantTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/AddExactILoadTest.java --- a/test/compiler/intrinsics/mathexact/AddExactILoadTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/AddExactILoadTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8024924 * @summary Test non constant addExact * @compile AddExactILoadTest.java Verify.java - * @run main AddExactILoadTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main AddExactILoadTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/AddExactILoopDependentTest.java --- a/test/compiler/intrinsics/mathexact/AddExactILoopDependentTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/AddExactILoopDependentTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8024924 * @summary Test non constant addExact * @compile AddExactILoopDependentTest.java Verify.java - * @run main AddExactILoopDependentTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main AddExactILoopDependentTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/AddExactINonConstantTest.java --- a/test/compiler/intrinsics/mathexact/AddExactINonConstantTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/AddExactINonConstantTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8024924 * @summary Test non constant addExact * @compile AddExactINonConstantTest.java Verify.java - * @run main AddExactINonConstantTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main AddExactINonConstantTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/AddExactIRepeatTest.java --- a/test/compiler/intrinsics/mathexact/AddExactIRepeatTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/AddExactIRepeatTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8025657 * @summary Test repeating addExact * @compile AddExactIRepeatTest.java Verify.java - * @run main AddExactIRepeatTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main AddExactIRepeatTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/AddExactLConstantTest.java --- a/test/compiler/intrinsics/mathexact/AddExactLConstantTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/AddExactLConstantTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test constant addExact * @compile AddExactLConstantTest.java Verify.java - * @run main AddExactLConstantTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main AddExactLConstantTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/AddExactLNonConstantTest.java --- a/test/compiler/intrinsics/mathexact/AddExactLNonConstantTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/AddExactLNonConstantTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test non constant addExact * @compile AddExactLNonConstantTest.java Verify.java - * @run main AddExactLNonConstantTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main AddExactLNonConstantTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/CompareTest.java --- a/test/compiler/intrinsics/mathexact/CompareTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/CompareTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026722 * @summary Verify that the compare after addExact is a signed compare * @compile CompareTest.java - * @run main CompareTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main CompareTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/DecExactITest.java --- a/test/compiler/intrinsics/mathexact/DecExactITest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/DecExactITest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test decrementExact * @compile DecExactITest.java Verify.java - * @run main DecExactITest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main DecExactITest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/DecExactLTest.java --- a/test/compiler/intrinsics/mathexact/DecExactLTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/DecExactLTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test decrementExact * @compile DecExactLTest.java Verify.java - * @run main DecExactLTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main DecExactLTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/GVNTest.java --- a/test/compiler/intrinsics/mathexact/GVNTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/GVNTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8028207 * @summary Verify that GVN doesn't mess up the two addExacts * @compile GVNTest.java - * @run main GVNTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main GVNTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/IncExactITest.java --- a/test/compiler/intrinsics/mathexact/IncExactITest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/IncExactITest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test incrementExact * @compile IncExactITest.java Verify.java - * @run main IncExactITest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main IncExactITest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/IncExactLTest.java --- a/test/compiler/intrinsics/mathexact/IncExactLTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/IncExactLTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test incrementExact * @compile IncExactLTest.java Verify.java - * @run main IncExactLTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main IncExactLTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/MulExactICondTest.java --- a/test/compiler/intrinsics/mathexact/MulExactICondTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/MulExactICondTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test multiplyExact as condition * @compile MulExactICondTest.java - * @run main MulExactICondTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main MulExactICondTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/MulExactIConstantTest.java --- a/test/compiler/intrinsics/mathexact/MulExactIConstantTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/MulExactIConstantTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test constant multiplyExact * @compile MulExactIConstantTest.java Verify.java - * @run main MulExactIConstantTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main MulExactIConstantTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/MulExactILoadTest.java --- a/test/compiler/intrinsics/mathexact/MulExactILoadTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/MulExactILoadTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test multiplyExact * @compile MulExactILoadTest.java Verify.java - * @run main MulExactILoadTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main MulExactILoadTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/MulExactILoopDependentTest.java --- a/test/compiler/intrinsics/mathexact/MulExactILoopDependentTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/MulExactILoopDependentTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test loop dependent multiplyExact * @compile MulExactILoopDependentTest.java Verify.java - * @run main MulExactILoopDependentTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main MulExactILoopDependentTest * */ public class MulExactILoopDependentTest { diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/MulExactINonConstantTest.java --- a/test/compiler/intrinsics/mathexact/MulExactINonConstantTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/MulExactINonConstantTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test non constant multiplyExact * @compile MulExactINonConstantTest.java Verify.java - * @run main MulExactINonConstantTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main MulExactINonConstantTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/MulExactIRepeatTest.java --- a/test/compiler/intrinsics/mathexact/MulExactIRepeatTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/MulExactIRepeatTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test repeating multiplyExact * @compile MulExactIRepeatTest.java Verify.java - * @run main MulExactIRepeatTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main MulExactIRepeatTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/MulExactLConstantTest.java --- a/test/compiler/intrinsics/mathexact/MulExactLConstantTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/MulExactLConstantTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test constant mulExact * @compile MulExactLConstantTest.java Verify.java - * @run main MulExactLConstantTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main MulExactLConstantTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/MulExactLNonConstantTest.java --- a/test/compiler/intrinsics/mathexact/MulExactLNonConstantTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/MulExactLNonConstantTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test non constant mulExact * @compile MulExactLNonConstantTest.java Verify.java - * @run main MulExactLNonConstantTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main MulExactLNonConstantTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/NegExactIConstantTest.java --- a/test/compiler/intrinsics/mathexact/NegExactIConstantTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/NegExactIConstantTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test constant negExact * @compile NegExactIConstantTest.java Verify.java - * @run main NegExactIConstantTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main NegExactIConstantTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/NegExactILoadTest.java --- a/test/compiler/intrinsics/mathexact/NegExactILoadTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/NegExactILoadTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,14 +26,14 @@ * @bug 8026844 * @summary Test negExact * @compile NegExactILoadTest.java Verify.java - * @run main NegExactILoadTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main NegExactILoadTest * */ public class NegExactILoadTest { public static void main(String[] args) { - Verify.LoadTest.init(); - Verify.LoadTest.verify(new Verify.UnaryToBinary(new Verify.NegExactI())); + Verify.LoadTest.init(); + Verify.LoadTest.verify(new Verify.UnaryToBinary(new Verify.NegExactI())); } } diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/NegExactILoopDependentTest.java --- a/test/compiler/intrinsics/mathexact/NegExactILoopDependentTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/NegExactILoopDependentTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test negExact loop dependent * @compile NegExactILoopDependentTest.java Verify.java - * @run main NegExactILoopDependentTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main NegExactILoopDependentTest * */ public class NegExactILoopDependentTest { diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/NegExactINonConstantTest.java --- a/test/compiler/intrinsics/mathexact/NegExactINonConstantTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/NegExactINonConstantTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test non constant negExact * @compile NegExactINonConstantTest.java Verify.java - * @run main NegExactINonConstantTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main NegExactINonConstantTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/NegExactLConstantTest.java --- a/test/compiler/intrinsics/mathexact/NegExactLConstantTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/NegExactLConstantTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test constant negExact * @compile NegExactLConstantTest.java Verify.java - * @run main NegExactLConstantTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main NegExactLConstantTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/NegExactLNonConstantTest.java --- a/test/compiler/intrinsics/mathexact/NegExactLNonConstantTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/NegExactLNonConstantTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test constant negExact * @compile NegExactLNonConstantTest.java Verify.java - * @run main NegExactLNonConstantTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main NegExactLNonConstantTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/NestedMathExactTest.java --- a/test/compiler/intrinsics/mathexact/NestedMathExactTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/NestedMathExactTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8027444 * @summary Test nested loops * @compile NestedMathExactTest.java - * @run main NestedMathExactTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main NestedMathExactTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/SplitThruPhiTest.java --- a/test/compiler/intrinsics/mathexact/SplitThruPhiTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/SplitThruPhiTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8028198 * @summary Verify that split through phi does the right thing * @compile SplitThruPhiTest.java - * @run main SplitThruPhiTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main SplitThruPhiTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/SubExactICondTest.java --- a/test/compiler/intrinsics/mathexact/SubExactICondTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/SubExactICondTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test subtractExact as condition * @compile SubExactICondTest.java Verify.java - * @run main SubExactICondTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main SubExactICondTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/SubExactIConstantTest.java --- a/test/compiler/intrinsics/mathexact/SubExactIConstantTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/SubExactIConstantTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test constant subtractExact * @compile SubExactIConstantTest.java Verify.java - * @run main SubExactIConstantTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main SubExactIConstantTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/SubExactILoadTest.java --- a/test/compiler/intrinsics/mathexact/SubExactILoadTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/SubExactILoadTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test non constant subtractExact * @compile SubExactILoadTest.java Verify.java - * @run main SubExactILoadTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main SubExactILoadTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/SubExactILoopDependentTest.java --- a/test/compiler/intrinsics/mathexact/SubExactILoopDependentTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/SubExactILoopDependentTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test non constant subtractExact * @compile SubExactILoopDependentTest.java Verify.java - * @run main SubExactILoopDependentTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main SubExactILoopDependentTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/SubExactINonConstantTest.java --- a/test/compiler/intrinsics/mathexact/SubExactINonConstantTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/SubExactINonConstantTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test non constant subtractExact * @compile SubExactINonConstantTest.java Verify.java - * @run main SubExactINonConstantTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main SubExactINonConstantTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/SubExactIRepeatTest.java --- a/test/compiler/intrinsics/mathexact/SubExactIRepeatTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/SubExactIRepeatTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,7 +26,7 @@ * @bug 8026844 * @summary Test repeating subtractExact * @compile SubExactIRepeatTest.java Verify.java - * @run main SubExactIRepeatTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main SubExactIRepeatTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/SubExactLConstantTest.java --- a/test/compiler/intrinsics/mathexact/SubExactLConstantTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/SubExactLConstantTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -27,7 +27,7 @@ * @bug 8027353 * @summary Test constant subtractExact * @compile SubExactLConstantTest.java Verify.java - * @run main SubExactLConstantTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main SubExactLConstantTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/SubExactLNonConstantTest.java --- a/test/compiler/intrinsics/mathexact/SubExactLNonConstantTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/SubExactLNonConstantTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -27,7 +27,7 @@ * @bug 8027353 * @summary Test non constant subtractExact * @compile SubExactLNonConstantTest.java Verify.java - * @run main SubExactLNonConstantTest -XX:+IgnoreUnrecognizedVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseMathExactIntrinsics + * @run main SubExactLNonConstantTest * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/Verify.java --- a/test/compiler/intrinsics/mathexact/Verify.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/Verify.java Tue Mar 25 10:48:42 2014 -0700 @@ -160,6 +160,7 @@ public static class NonConstantTest { public static java.util.Random rnd = new java.util.Random(); + public static int[] values = new int[] { Integer.MAX_VALUE, Integer.MIN_VALUE }; public static void verify(BinaryMethod method) { for (int i = 0; i < 50000; ++i) { @@ -169,6 +170,10 @@ Verify.verifyBinary(rnd1 + 1, rnd2, method); Verify.verifyBinary(rnd1 - 1, rnd2, method); Verify.verifyBinary(rnd1, rnd2 - 1, method); + Verify.verifyBinary(0, values[0], method); + Verify.verifyBinary(values[0], 0, method); + Verify.verifyBinary(0, values[1], method); + Verify.verifyBinary(values[1], 0, method); } } } diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/sanity/AddExactIntTest.java --- a/test/compiler/intrinsics/mathexact/sanity/AddExactIntTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/sanity/AddExactIntTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,11 +26,11 @@ * @library /testlibrary /testlibrary/whitebox /compiler/whitebox * @build AddExactIntTest * @run main ClassFileInstaller sun.hotspot.WhiteBox - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs_neg.log -XX:-UseMathExactIntrinsics AddExactIntTest - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs.log -XX:+UseMathExactIntrinsics AddExactIntTest diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/sanity/AddExactLongTest.java --- a/test/compiler/intrinsics/mathexact/sanity/AddExactLongTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/sanity/AddExactLongTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,11 +26,11 @@ * @library /testlibrary /testlibrary/whitebox /compiler/whitebox * @build AddExactLongTest * @run main ClassFileInstaller sun.hotspot.WhiteBox - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs_neg.log -XX:-UseMathExactIntrinsics AddExactLongTest - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs.log -XX:+UseMathExactIntrinsics AddExactLongTest diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/sanity/DecrementExactIntTest.java --- a/test/compiler/intrinsics/mathexact/sanity/DecrementExactIntTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/sanity/DecrementExactIntTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,11 +26,11 @@ * @library /testlibrary /testlibrary/whitebox /compiler/whitebox * @build DecrementExactIntTest * @run main ClassFileInstaller sun.hotspot.WhiteBox - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs_neg.log -XX:-UseMathExactIntrinsics DecrementExactIntTest - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs.log -XX:+UseMathExactIntrinsics DecrementExactIntTest diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/sanity/DecrementExactLongTest.java --- a/test/compiler/intrinsics/mathexact/sanity/DecrementExactLongTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/sanity/DecrementExactLongTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,11 +26,11 @@ * @library /testlibrary /testlibrary/whitebox /compiler/whitebox * @build DecrementExactLongTest * @run main ClassFileInstaller sun.hotspot.WhiteBox - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs_neg.log -XX:-UseMathExactIntrinsics DecrementExactLongTest - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs.log -XX:+UseMathExactIntrinsics DecrementExactLongTest @@ -42,4 +42,4 @@ public static void main(String[] args) throws Exception { new IntrinsicBase.LongTest(MathIntrinsic.LongIntrinsic.Decrement).test(); } -} \ No newline at end of file +} diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/sanity/IncrementExactIntTest.java --- a/test/compiler/intrinsics/mathexact/sanity/IncrementExactIntTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/sanity/IncrementExactIntTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,11 +26,11 @@ * @library /testlibrary /testlibrary/whitebox /compiler/whitebox * @build IncrementExactIntTest * @run main ClassFileInstaller sun.hotspot.WhiteBox - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs_neg.log -XX:-UseMathExactIntrinsics IncrementExactIntTest - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs.log -XX:+UseMathExactIntrinsics IncrementExactIntTest diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/sanity/IncrementExactLongTest.java --- a/test/compiler/intrinsics/mathexact/sanity/IncrementExactLongTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/sanity/IncrementExactLongTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,11 +26,11 @@ * @library /testlibrary /testlibrary/whitebox /compiler/whitebox * @build IncrementExactLongTest * @run main ClassFileInstaller sun.hotspot.WhiteBox - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs_neg.log -XX:-UseMathExactIntrinsics IncrementExactLongTest - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs.log -XX:+UseMathExactIntrinsics IncrementExactLongTest diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/sanity/MultiplyExactIntTest.java --- a/test/compiler/intrinsics/mathexact/sanity/MultiplyExactIntTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/sanity/MultiplyExactIntTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,11 +26,11 @@ * @library /testlibrary /testlibrary/whitebox /compiler/whitebox * @build MultiplyExactIntTest * @run main ClassFileInstaller sun.hotspot.WhiteBox - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs_neg.log -XX:-UseMathExactIntrinsics MultiplyExactIntTest - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs.log -XX:+UseMathExactIntrinsics MultiplyExactIntTest diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/sanity/MultiplyExactLongTest.java --- a/test/compiler/intrinsics/mathexact/sanity/MultiplyExactLongTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/sanity/MultiplyExactLongTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,11 +26,11 @@ * @library /testlibrary /testlibrary/whitebox /compiler/whitebox * @build MultiplyExactLongTest * @run main ClassFileInstaller sun.hotspot.WhiteBox - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs_neg.log -XX:-UseMathExactIntrinsics MultiplyExactLongTest - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs.log -XX:+UseMathExactIntrinsics MultiplyExactLongTest diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/sanity/NegateExactIntTest.java --- a/test/compiler/intrinsics/mathexact/sanity/NegateExactIntTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/sanity/NegateExactIntTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,11 +26,11 @@ * @library /testlibrary /testlibrary/whitebox /compiler/whitebox * @build NegateExactIntTest * @run main ClassFileInstaller sun.hotspot.WhiteBox - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs_neg.log -XX:-UseMathExactIntrinsics NegateExactIntTest - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs.log -XX:+UseMathExactIntrinsics NegateExactIntTest diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/sanity/NegateExactLongTest.java --- a/test/compiler/intrinsics/mathexact/sanity/NegateExactLongTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/sanity/NegateExactLongTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,11 +26,11 @@ * @library /testlibrary /testlibrary/whitebox /compiler/whitebox * @build NegateExactLongTest * @run main ClassFileInstaller sun.hotspot.WhiteBox - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs_neg.log -XX:-UseMathExactIntrinsics NegateExactLongTest - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs.log -XX:+UseMathExactIntrinsics NegateExactLongTest diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/sanity/SubtractExactIntTest.java --- a/test/compiler/intrinsics/mathexact/sanity/SubtractExactIntTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/sanity/SubtractExactIntTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,11 +26,11 @@ * @library /testlibrary /testlibrary/whitebox /compiler/whitebox * @build SubtractExactIntTest * @run main ClassFileInstaller sun.hotspot.WhiteBox - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs_neg.log -XX:-UseMathExactIntrinsics SubtractExactIntTest - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs.log -XX:+UseMathExactIntrinsics SubtractExactIntTest diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/intrinsics/mathexact/sanity/SubtractExactLongTest.java --- a/test/compiler/intrinsics/mathexact/sanity/SubtractExactLongTest.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/intrinsics/mathexact/sanity/SubtractExactLongTest.java Tue Mar 25 10:48:42 2014 -0700 @@ -26,11 +26,11 @@ * @library /testlibrary /testlibrary/whitebox /compiler/whitebox * @build SubtractExactLongTest * @run main ClassFileInstaller sun.hotspot.WhiteBox - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs_neg.log -XX:-UseMathExactIntrinsics SubtractExactLongTest - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions * -XX:+IgnoreUnrecognizedVMOptions -XX:+WhiteBoxAPI -XX:+LogCompilation * -XX:CompileCommand=compileonly,MathIntrinsic*::execMathMethod * -XX:LogFile=hs.log -XX:+UseMathExactIntrinsics SubtractExactLongTest diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/types/TestMeetTopArrayExactConstantArray.java --- a/test/compiler/types/TestMeetTopArrayExactConstantArray.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/types/TestMeetTopArrayExactConstantArray.java Tue Mar 25 10:48:42 2014 -0700 @@ -25,7 +25,7 @@ * @test * @bug 8027571 * @summary meet of TopPTR exact array with constant array is not symmetric - * @run main/othervm -XX:+IgnoreUnrecognizedVMOptions -XX:-UseOnStackReplacement -XX:TypeProfileLevel=222 -XX:+UnlockExperimentalVMOptions -XX:+UseTypeSpeculation -XX:-BackgroundCompilation TestMeetTopArrayExactConstantArray + * @run main/othervm -XX:+IgnoreUnrecognizedVMOptions -XX:-UseOnStackReplacement -XX:TypeProfileLevel=222 -XX:+UseTypeSpeculation -XX:-BackgroundCompilation TestMeetTopArrayExactConstantArray * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/types/TestSpeculationFailedHigherEqual.java --- a/test/compiler/types/TestSpeculationFailedHigherEqual.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/types/TestSpeculationFailedHigherEqual.java Tue Mar 25 10:48:42 2014 -0700 @@ -25,7 +25,7 @@ * @test * @bug 8027422 * @summary type methods shouldn't always operate on speculative part - * @run main/othervm -XX:+IgnoreUnrecognizedVMOptions -XX:TypeProfileLevel=222 -XX:+UnlockExperimentalVMOptions -XX:+UseTypeSpeculation -XX:-BackgroundCompilation TestSpeculationFailedHigherEqual + * @run main/othervm -XX:+IgnoreUnrecognizedVMOptions -XX:TypeProfileLevel=222 -XX:+UseTypeSpeculation -XX:-BackgroundCompilation TestSpeculationFailedHigherEqual * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/types/TypeSpeculation.java --- a/test/compiler/types/TypeSpeculation.java Mon Mar 24 13:14:23 2014 -0700 +++ b/test/compiler/types/TypeSpeculation.java Tue Mar 25 10:48:42 2014 -0700 @@ -25,7 +25,7 @@ * @test * @bug 8024070 * @summary Test that type speculation doesn't cause incorrect execution - * @run main/othervm -XX:+IgnoreUnrecognizedVMOptions -XX:-UseOnStackReplacement -XX:-BackgroundCompilation -XX:TypeProfileLevel=222 -XX:+UnlockExperimentalVMOptions -XX:+UseTypeSpeculation TypeSpeculation + * @run main/othervm -XX:+IgnoreUnrecognizedVMOptions -XX:-UseOnStackReplacement -XX:-BackgroundCompilation -XX:TypeProfileLevel=222 -XX:+UseTypeSpeculation TypeSpeculation * */ diff -r 7ce7247df891 -r f0ea4d3df129 test/compiler/uncommontrap/TestSpecTrapClassUnloading.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/compiler/uncommontrap/TestSpecTrapClassUnloading.java Tue Mar 25 10:48:42 2014 -0700 @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @bug 8031752 + * @summary speculative traps need to be cleaned up at GC + * @run main/othervm -XX:+IgnoreUnrecognizedVMOptions -XX:-TieredCompilation -XX:-UseOnStackReplacement -XX:-BackgroundCompilation -XX:+UseTypeSpeculation -XX:TypeProfileLevel=222 -XX:CompileCommand=exclude,java.lang.reflect.Method::invoke -XX:CompileCommand=exclude,sun.reflect.DelegatingMethodAccessorImpl::invoke -Xmx1M TestSpecTrapClassUnloading + * + */ + +import java.lang.reflect.Method; + +public class TestSpecTrapClassUnloading { + static class B { + final public boolean m(Object o) { + if (o.getClass() == B.class) { + return true; + } + return false; + } + } + + static class MemoryChunk { + MemoryChunk other; + long[] array; + MemoryChunk(MemoryChunk other) { + other = other; + array = new long[1024 * 1024 * 1024]; + } + } + + static void m1(B b, Object o) { + b.m(o); + } + + static void m2(B b, Object o) { + b.m(o); + } + + public static void main(String[] args) throws Exception { + Method m = B.class.getMethod("m", Object.class); + Object o = new Object(); + B b = new B(); + + // add speculative trap in B.m() for m1 + for (int i = 0; i < 20000; i++) { + m1(b, b); + } + m1(b, o); + + // add speculative trap in B.m() for code generated by reflection + for (int i = 0; i < 20000; i++) { + m.invoke(b, b); + } + m.invoke(b, o); + + m = null; + + // add speculative trap in B.m() for m2 + for (int i = 0; i < 20000; i++) { + m2(b, b); + } + m2(b, o); + + // Exhaust memory which causes the code generated by + // reflection to be unloaded but B.m() is not. + MemoryChunk root = null; + try { + while (true) { + root = new MemoryChunk(root); + } + } catch(OutOfMemoryError e) { + root = null; + } + } +}