Mercurial > hg > truffle
changeset 20340:fbc31318922c hs25.40-b06
Merge
| author | amurillo |
|---|---|
| date | Thu, 21 Aug 2014 14:42:54 -0700 |
| parents | 4d5c3760ebb4 (current diff) 7b2fc3129653 (diff) |
| children | 38539608359a |
| files | |
| diffstat | 74 files changed, 2954 insertions(+), 1789 deletions(-) [+] |
line wrap: on
line diff
--- a/agent/src/share/classes/sun/jvm/hotspot/gc_implementation/g1/G1CollectedHeap.java Thu Aug 21 11:43:17 2014 -0700 +++ b/agent/src/share/classes/sun/jvm/hotspot/gc_implementation/g1/G1CollectedHeap.java Thu Aug 21 14:42:54 2014 -0700 @@ -45,8 +45,8 @@ public class G1CollectedHeap extends SharedHeap { // HeapRegionSeq _seq; static private long hrsFieldOffset; - // MemRegion _g1_committed; - static private long g1CommittedFieldOffset; + // MemRegion _g1_reserved; + static private long g1ReservedFieldOffset; // size_t _summary_bytes_used; static private CIntegerField summaryBytesUsedField; // G1MonitoringSupport* _g1mm; @@ -68,7 +68,6 @@ Type type = db.lookupType("G1CollectedHeap"); hrsFieldOffset = type.getField("_hrs").getOffset(); - g1CommittedFieldOffset = type.getField("_g1_committed").getOffset(); summaryBytesUsedField = type.getCIntegerField("_summary_bytes_used"); g1mmField = type.getAddressField("_g1mm"); oldSetFieldOffset = type.getField("_old_set").getOffset(); @@ -76,9 +75,7 @@ } public long capacity() { - Address g1CommittedAddr = addr.addOffsetTo(g1CommittedFieldOffset); - MemRegion g1Committed = new MemRegion(g1CommittedAddr); - return g1Committed.byteSize(); + return hrs().capacity(); } public long used() {
--- a/agent/src/share/classes/sun/jvm/hotspot/gc_implementation/g1/G1HeapRegionTable.java Thu Aug 21 11:43:17 2014 -0700 +++ b/agent/src/share/classes/sun/jvm/hotspot/gc_implementation/g1/G1HeapRegionTable.java Thu Aug 21 14:42:54 2014 -0700 @@ -93,19 +93,35 @@ private class HeapRegionIterator implements Iterator<HeapRegion> { private long index; private long length; + private HeapRegion next; - @Override - public boolean hasNext() { return index < length; } + public HeapRegion positionToNext() { + HeapRegion result = next; + while (index < length && at(index) == null) { + index++; + } + if (index < length) { + next = at(index); + index++; // restart search at next element + } else { + next = null; + } + return result; + } @Override - public HeapRegion next() { return at(index++); } + public boolean hasNext() { return next != null; } + + @Override + public HeapRegion next() { return positionToNext(); } @Override - public void remove() { /* not supported */ } + public void remove() { /* not supported */ } - HeapRegionIterator(long committedLength) { + HeapRegionIterator(long totalLength) { index = 0; - length = committedLength; + length = totalLength; + positionToNext(); } }
--- a/agent/src/share/classes/sun/jvm/hotspot/gc_implementation/g1/HeapRegionSeq.java Thu Aug 21 11:43:17 2014 -0700 +++ b/agent/src/share/classes/sun/jvm/hotspot/gc_implementation/g1/HeapRegionSeq.java Thu Aug 21 14:42:54 2014 -0700 @@ -43,7 +43,7 @@ // G1HeapRegionTable _regions static private long regionsFieldOffset; // uint _committed_length - static private CIntegerField committedLengthField; + static private CIntegerField numCommittedField; static { VM.registerVMInitializedObserver(new Observer() { @@ -57,7 +57,7 @@ Type type = db.lookupType("HeapRegionSeq"); regionsFieldOffset = type.getField("_regions").getOffset(); - committedLengthField = type.getCIntegerField("_committed_length"); + numCommittedField = type.getCIntegerField("_num_committed"); } private G1HeapRegionTable regions() { @@ -66,16 +66,20 @@ regionsAddr); } + public long capacity() { + return length() * HeapRegion.grainBytes(); + } + public long length() { return regions().length(); } public long committedLength() { - return committedLengthField.getValue(addr); + return numCommittedField.getValue(addr); } public Iterator<HeapRegion> heapRegionIterator() { - return regions().heapRegionIterator(committedLength()); + return regions().heapRegionIterator(length()); } public HeapRegionSeq(Address addr) {
--- a/make/hotspot_version Thu Aug 21 11:43:17 2014 -0700 +++ b/make/hotspot_version Thu Aug 21 14:42:54 2014 -0700 @@ -35,7 +35,7 @@ HS_MAJOR_VER=25 HS_MINOR_VER=40 -HS_BUILD_NUMBER=05 +HS_BUILD_NUMBER=06 JDK_MAJOR_VER=1 JDK_MINOR_VER=8
--- a/make/jprt.properties Thu Aug 21 11:43:17 2014 -0700 +++ b/make/jprt.properties Thu Aug 21 14:42:54 2014 -0700 @@ -33,7 +33,7 @@ # This tells jprt what default release we want to build -jprt.hotspot.default.release=jdk8u20 +jprt.hotspot.default.release=jdk8u40 jprt.tools.default.release=${jprt.submit.option.release?${jprt.submit.option.release}:${jprt.hotspot.default.release}} @@ -47,65 +47,65 @@ # sparc etc. # Define the Solaris platforms we want for the various releases -jprt.my.solaris.sparcv9.jdk8u20=solaris_sparcv9_5.10 +jprt.my.solaris.sparcv9.jdk8u40=solaris_sparcv9_5.10 jprt.my.solaris.sparcv9.jdk7=solaris_sparcv9_5.10 jprt.my.solaris.sparcv9.jdk7u8=${jprt.my.solaris.sparcv9.jdk7} jprt.my.solaris.sparcv9=${jprt.my.solaris.sparcv9.${jprt.tools.default.release}} -jprt.my.solaris.x64.jdk8u20=solaris_x64_5.10 +jprt.my.solaris.x64.jdk8u40=solaris_x64_5.10 jprt.my.solaris.x64.jdk7=solaris_x64_5.10 jprt.my.solaris.x64.jdk7u8=${jprt.my.solaris.x64.jdk7} jprt.my.solaris.x64=${jprt.my.solaris.x64.${jprt.tools.default.release}} -jprt.my.linux.i586.jdk8u20=linux_i586_2.6 +jprt.my.linux.i586.jdk8u40=linux_i586_2.6 jprt.my.linux.i586.jdk7=linux_i586_2.6 jprt.my.linux.i586.jdk7u8=${jprt.my.linux.i586.jdk7} jprt.my.linux.i586=${jprt.my.linux.i586.${jprt.tools.default.release}} -jprt.my.linux.x64.jdk8u20=linux_x64_2.6 +jprt.my.linux.x64.jdk8u40=linux_x64_2.6 jprt.my.linux.x64.jdk7=linux_x64_2.6 jprt.my.linux.x64.jdk7u8=${jprt.my.linux.x64.jdk7} jprt.my.linux.x64=${jprt.my.linux.x64.${jprt.tools.default.release}} -jprt.my.linux.ppc.jdk8u20=linux_ppc_2.6 +jprt.my.linux.ppc.jdk8u40=linux_ppc_2.6 jprt.my.linux.ppc.jdk7=linux_ppc_2.6 jprt.my.linux.ppc.jdk7u8=${jprt.my.linux.ppc.jdk7} jprt.my.linux.ppc=${jprt.my.linux.ppc.${jprt.tools.default.release}} -jprt.my.linux.ppcv2.jdk8u20=linux_ppcv2_2.6 +jprt.my.linux.ppcv2.jdk8u40=linux_ppcv2_2.6 jprt.my.linux.ppcv2.jdk7=linux_ppcv2_2.6 jprt.my.linux.ppcv2.jdk7u8=${jprt.my.linux.ppcv2.jdk7} jprt.my.linux.ppcv2=${jprt.my.linux.ppcv2.${jprt.tools.default.release}} -jprt.my.linux.armvfpsflt.jdk8u20=linux_armvfpsflt_2.6 +jprt.my.linux.armvfpsflt.jdk8u40=linux_armvfpsflt_2.6 jprt.my.linux.armvfpsflt=${jprt.my.linux.armvfpsflt.${jprt.tools.default.release}} -jprt.my.linux.armvfphflt.jdk8u20=linux_armvfphflt_2.6 +jprt.my.linux.armvfphflt.jdk8u40=linux_armvfphflt_2.6 jprt.my.linux.armvfphflt=${jprt.my.linux.armvfphflt.${jprt.tools.default.release}} # The ARM GP vfp-sflt build is not currently supported -#jprt.my.linux.armvs.jdk8u20=linux_armvs_2.6 +#jprt.my.linux.armvs.jdk8u40=linux_armvs_2.6 #jprt.my.linux.armvs=${jprt.my.linux.armvs.${jprt.tools.default.release}} -jprt.my.linux.armvh.jdk8u20=linux_armvh_2.6 +jprt.my.linux.armvh.jdk8u40=linux_armvh_2.6 jprt.my.linux.armvh=${jprt.my.linux.armvh.${jprt.tools.default.release}} -jprt.my.linux.armsflt.jdk8u20=linux_armsflt_2.6 +jprt.my.linux.armsflt.jdk8u40=linux_armsflt_2.6 jprt.my.linux.armsflt.jdk7=linux_armsflt_2.6 jprt.my.linux.armsflt.jdk7u8=${jprt.my.linux.armsflt.jdk7} jprt.my.linux.armsflt=${jprt.my.linux.armsflt.${jprt.tools.default.release}} -jprt.my.macosx.x64.jdk8u20=macosx_x64_10.7 +jprt.my.macosx.x64.jdk8u40=macosx_x64_10.7 jprt.my.macosx.x64.jdk7=macosx_x64_10.7 jprt.my.macosx.x64.jdk7u8=${jprt.my.macosx.x64.jdk7} jprt.my.macosx.x64=${jprt.my.macosx.x64.${jprt.tools.default.release}} -jprt.my.windows.i586.jdk8u20=windows_i586_6.1 +jprt.my.windows.i586.jdk8u40=windows_i586_6.1 jprt.my.windows.i586.jdk7=windows_i586_6.1 jprt.my.windows.i586.jdk7u8=${jprt.my.windows.i586.jdk7} jprt.my.windows.i586=${jprt.my.windows.i586.${jprt.tools.default.release}} -jprt.my.windows.x64.jdk8u20=windows_x64_6.1 +jprt.my.windows.x64.jdk8u40=windows_x64_6.1 jprt.my.windows.x64.jdk7=windows_x64_6.1 jprt.my.windows.x64.jdk7u8=${jprt.my.windows.x64.jdk7} jprt.my.windows.x64=${jprt.my.windows.x64.${jprt.tools.default.release}} @@ -137,7 +137,7 @@ jprt.build.targets.all=${jprt.build.targets.standard}, \ ${jprt.build.targets.embedded}, ${jprt.build.targets.open} -jprt.build.targets.jdk8u20=${jprt.build.targets.all} +jprt.build.targets.jdk8u40=${jprt.build.targets.all} jprt.build.targets.jdk7=${jprt.build.targets.all} jprt.build.targets.jdk7u8=${jprt.build.targets.all} jprt.build.targets=${jprt.build.targets.${jprt.tools.default.release}} @@ -343,7 +343,7 @@ ${jprt.my.windows.i586.test.targets}, \ ${jprt.my.windows.x64.test.targets} -jprt.test.targets.jdk8u20=${jprt.test.targets.standard} +jprt.test.targets.jdk8u40=${jprt.test.targets.standard} jprt.test.targets.jdk7=${jprt.test.targets.standard} jprt.test.targets.jdk7u8=${jprt.test.targets.jdk7} jprt.test.targets=${jprt.test.targets.${jprt.tools.default.release}} @@ -393,7 +393,7 @@ jprt.make.rule.test.targets.embedded = \ ${jprt.make.rule.test.targets.standard.client} -jprt.make.rule.test.targets.jdk8u20=${jprt.make.rule.test.targets.standard} +jprt.make.rule.test.targets.jdk8u40=${jprt.make.rule.test.targets.standard} jprt.make.rule.test.targets.jdk7=${jprt.make.rule.test.targets.standard} jprt.make.rule.test.targets.jdk7u8=${jprt.make.rule.test.targets.jdk7} jprt.make.rule.test.targets=${jprt.make.rule.test.targets.${jprt.tools.default.release}}
--- a/src/share/vm/classfile/verifier.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/classfile/verifier.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -2231,6 +2231,181 @@ } } +// Look at the method's handlers. If the bci is in the handler's try block +// then check if the handler_pc is already on the stack. If not, push it. +void ClassVerifier::push_handlers(ExceptionTable* exhandlers, + GrowableArray<u4>* handler_stack, + u4 bci) { + int exlength = exhandlers->length(); + for(int x = 0; x < exlength; x++) { + if (bci >= exhandlers->start_pc(x) && bci < exhandlers->end_pc(x)) { + handler_stack->append_if_missing(exhandlers->handler_pc(x)); + } + } +} + +// Return TRUE if all code paths starting with start_bc_offset end in +// bytecode athrow or loop. +bool ClassVerifier::ends_in_athrow(u4 start_bc_offset) { + ResourceMark rm; + // Create bytecode stream. + RawBytecodeStream bcs(method()); + u4 code_length = method()->code_size(); + bcs.set_start(start_bc_offset); + u4 target; + // Create stack for storing bytecode start offsets for if* and *switch. + GrowableArray<u4>* bci_stack = new GrowableArray<u4>(30); + // Create stack for handlers for try blocks containing this handler. + GrowableArray<u4>* handler_stack = new GrowableArray<u4>(30); + // Create list of visited branch opcodes (goto* and if*). + GrowableArray<u4>* visited_branches = new GrowableArray<u4>(30); + ExceptionTable exhandlers(_method()); + + while (true) { + if (bcs.is_last_bytecode()) { + // if no more starting offsets to parse or if at the end of the + // method then return false. + if ((bci_stack->is_empty()) || ((u4)bcs.end_bci() == code_length)) + return false; + // Pop a bytecode starting offset and scan from there. + bcs.set_start(bci_stack->pop()); + } + Bytecodes::Code opcode = bcs.raw_next(); + u4 bci = bcs.bci(); + + // If the bytecode is in a TRY block, push its handlers so they + // will get parsed. + push_handlers(&exhandlers, handler_stack, bci); + + switch (opcode) { + case Bytecodes::_if_icmpeq: + case Bytecodes::_if_icmpne: + case Bytecodes::_if_icmplt: + case Bytecodes::_if_icmpge: + case Bytecodes::_if_icmpgt: + case Bytecodes::_if_icmple: + case Bytecodes::_ifeq: + case Bytecodes::_ifne: + case Bytecodes::_iflt: + case Bytecodes::_ifge: + case Bytecodes::_ifgt: + case Bytecodes::_ifle: + case Bytecodes::_if_acmpeq: + case Bytecodes::_if_acmpne: + case Bytecodes::_ifnull: + case Bytecodes::_ifnonnull: + target = bcs.dest(); + if (visited_branches->contains(bci)) { + if (bci_stack->is_empty()) return true; + // Pop a bytecode starting offset and scan from there. + bcs.set_start(bci_stack->pop()); + } else { + if (target > bci) { // forward branch + if (target >= code_length) return false; + // Push the branch target onto the stack. + bci_stack->push(target); + // then, scan bytecodes starting with next. + bcs.set_start(bcs.next_bci()); + } else { // backward branch + // Push bytecode offset following backward branch onto the stack. + bci_stack->push(bcs.next_bci()); + // Check bytecodes starting with branch target. + bcs.set_start(target); + } + // Record target so we don't branch here again. + visited_branches->append(bci); + } + break; + + case Bytecodes::_goto: + case Bytecodes::_goto_w: + target = (opcode == Bytecodes::_goto ? bcs.dest() : bcs.dest_w()); + if (visited_branches->contains(bci)) { + if (bci_stack->is_empty()) return true; + // Been here before, pop new starting offset from stack. + bcs.set_start(bci_stack->pop()); + } else { + if (target >= code_length) return false; + // Continue scanning from the target onward. + bcs.set_start(target); + // Record target so we don't branch here again. + visited_branches->append(bci); + } + break; + + // Check that all switch alternatives end in 'athrow' bytecodes. Since it + // is difficult to determine where each switch alternative ends, parse + // each switch alternative until either hit a 'return', 'athrow', or reach + // the end of the method's bytecodes. This is gross but should be okay + // because: + // 1. tableswitch and lookupswitch byte codes in handlers for ctor explicit + // constructor invocations should be rare. + // 2. if each switch alternative ends in an athrow then the parsing should be + // short. If there is no athrow then it is bogus code, anyway. + case Bytecodes::_lookupswitch: + case Bytecodes::_tableswitch: + { + address aligned_bcp = (address) round_to((intptr_t)(bcs.bcp() + 1), jintSize); + u4 default_offset = Bytes::get_Java_u4(aligned_bcp) + bci; + int keys, delta; + if (opcode == Bytecodes::_tableswitch) { + jint low = (jint)Bytes::get_Java_u4(aligned_bcp + jintSize); + jint high = (jint)Bytes::get_Java_u4(aligned_bcp + 2*jintSize); + // This is invalid, but let the regular bytecode verifier + // report this because the user will get a better error message. + if (low > high) return true; + keys = high - low + 1; + delta = 1; + } else { + keys = (int)Bytes::get_Java_u4(aligned_bcp + jintSize); + delta = 2; + } + // Invalid, let the regular bytecode verifier deal with it. + if (keys < 0) return true; + + // Push the offset of the next bytecode onto the stack. + bci_stack->push(bcs.next_bci()); + + // Push the switch alternatives onto the stack. + for (int i = 0; i < keys; i++) { + u4 target = bci + (jint)Bytes::get_Java_u4(aligned_bcp+(3+i*delta)*jintSize); + if (target > code_length) return false; + bci_stack->push(target); + } + + // Start bytecode parsing for the switch at the default alternative. + if (default_offset > code_length) return false; + bcs.set_start(default_offset); + break; + } + + case Bytecodes::_return: + return false; + + case Bytecodes::_athrow: + { + if (bci_stack->is_empty()) { + if (handler_stack->is_empty()) { + return true; + } else { + // Parse the catch handlers for try blocks containing athrow. + bcs.set_start(handler_stack->pop()); + } + } else { + // Pop a bytecode offset and starting scanning from there. + bcs.set_start(bci_stack->pop()); + } + } + break; + + default: + ; + } // end switch + } // end while loop + + return false; +} + void ClassVerifier::verify_invoke_init( RawBytecodeStream* bcs, u2 ref_class_index, VerificationType ref_class_type, StackMapFrame* current_frame, u4 code_length, bool *this_uninit, @@ -2250,18 +2425,26 @@ return; } - // Make sure that this call is not done from within a TRY block because - // that can result in returning an incomplete object. Simply checking - // (bci >= start_pc) also ensures that this call is not done after a TRY - // block. That is also illegal because this call must be the first Java - // statement in the constructor. + // Check if this call is done from inside of a TRY block. If so, make + // sure that all catch clause paths end in a throw. Otherwise, this + // can result in returning an incomplete object. ExceptionTable exhandlers(_method()); int exlength = exhandlers.length(); for(int i = 0; i < exlength; i++) { - if (bci >= exhandlers.start_pc(i)) { - verify_error(ErrorContext::bad_code(bci), - "Bad <init> method call from after the start of a try block"); - return; + u2 start_pc = exhandlers.start_pc(i); + u2 end_pc = exhandlers.end_pc(i); + + if (bci >= start_pc && bci < end_pc) { + if (!ends_in_athrow(exhandlers.handler_pc(i))) { + verify_error(ErrorContext::bad_code(bci), + "Bad <init> method call from after the start of a try block"); + return; + } else if (VerboseVerification) { + ResourceMark rm; + tty->print_cr( + "Survived call to ends_in_athrow(): %s", + current_class()->name()->as_C_string()); + } } }
--- a/src/share/vm/classfile/verifier.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/classfile/verifier.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -30,6 +30,7 @@ #include "oops/klass.hpp" #include "oops/method.hpp" #include "runtime/handles.hpp" +#include "utilities/growableArray.hpp" #include "utilities/exceptions.hpp" // The verifier class @@ -303,6 +304,16 @@ StackMapFrame* current_frame, u4 code_length, bool* this_uninit, constantPoolHandle cp, TRAPS); + // Used by ends_in_athrow() to push all handlers that contain bci onto + // the handler_stack, if the handler is not already on the stack. + void push_handlers(ExceptionTable* exhandlers, + GrowableArray<u4>* handler_stack, + u4 bci); + + // Returns true if all paths starting with start_bc_offset end in athrow + // bytecode or loop. + bool ends_in_athrow(u4 start_bc_offset); + void verify_invoke_instructions( RawBytecodeStream* bcs, u4 code_length, StackMapFrame* current_frame, bool* this_uninit, VerificationType return_type,
--- a/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -1514,6 +1514,8 @@ gclog_or_tty->print_cr("cms_allocation_rate=%g", stats().cms_allocation_rate()); gclog_or_tty->print_cr("occupancy=%3.7f", _cmsGen->occupancy()); gclog_or_tty->print_cr("initiatingOccupancy=%3.7f", _cmsGen->initiating_occupancy()); + gclog_or_tty->print_cr("cms_time_since_begin=%3.7f", stats().cms_time_since_begin()); + gclog_or_tty->print_cr("cms_time_since_end=%3.7f", stats().cms_time_since_end()); gclog_or_tty->print_cr("metadata initialized %d", MetaspaceGC::should_concurrent_collect()); } @@ -1576,6 +1578,28 @@ return true; } + // CMSTriggerInterval starts a CMS cycle if enough time has passed. + if (CMSTriggerInterval >= 0) { + if (CMSTriggerInterval == 0) { + // Trigger always + return true; + } + + // Check the CMS time since begin (we do not check the stats validity + // as we want to be able to trigger the first CMS cycle as well) + if (stats().cms_time_since_begin() >= (CMSTriggerInterval / ((double) MILLIUNITS))) { + if (Verbose && PrintGCDetails) { + if (stats().valid()) { + gclog_or_tty->print_cr("CMSCollector: collect because of trigger interval (time since last begin %3.7f secs)", + stats().cms_time_since_begin()); + } else { + gclog_or_tty->print_cr("CMSCollector: collect because of trigger interval (first collection)"); + } + } + return true; + } + } + return false; }
--- a/src/share/vm/gc_implementation/g1/concurrentG1Refine.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/concurrentG1Refine.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -81,8 +81,8 @@ } } -void ConcurrentG1Refine::init() { - _hot_card_cache.initialize(); +void ConcurrentG1Refine::init(G1RegionToSpaceMapper* card_counts_storage) { + _hot_card_cache.initialize(card_counts_storage); } void ConcurrentG1Refine::stop() {
--- a/src/share/vm/gc_implementation/g1/concurrentG1Refine.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/concurrentG1Refine.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -34,6 +34,7 @@ class ConcurrentG1RefineThread; class G1CollectedHeap; class G1HotCardCache; +class G1RegionToSpaceMapper; class G1RemSet; class DirtyCardQueue; @@ -74,7 +75,7 @@ ConcurrentG1Refine(G1CollectedHeap* g1h, CardTableEntryClosure* refine_closure); ~ConcurrentG1Refine(); - void init(); // Accomplish some initialization that has to wait. + void init(G1RegionToSpaceMapper* card_counts_storage); void stop(); void reinitialize_threads();
--- a/src/share/vm/gc_implementation/g1/concurrentMark.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -36,6 +36,7 @@ #include "gc_implementation/g1/heapRegion.inline.hpp" #include "gc_implementation/g1/heapRegionRemSet.hpp" #include "gc_implementation/g1/heapRegionSeq.inline.hpp" +#include "gc_implementation/g1/heapRegionSet.inline.hpp" #include "gc_implementation/shared/vmGCOperations.hpp" #include "gc_implementation/shared/gcTimer.hpp" #include "gc_implementation/shared/gcTrace.hpp" @@ -98,12 +99,12 @@ } #ifndef PRODUCT -bool CMBitMapRO::covers(ReservedSpace heap_rs) const { +bool CMBitMapRO::covers(MemRegion heap_rs) const { // assert(_bm.map() == _virtual_space.low(), "map inconsistency"); assert(((size_t)_bm.size() * ((size_t)1 << _shifter)) == _bmWordSize, "size inconsistency"); - return _bmStartWord == (HeapWord*)(heap_rs.base()) && - _bmWordSize == heap_rs.size()>>LogHeapWordSize; + return _bmStartWord == (HeapWord*)(heap_rs.start()) && + _bmWordSize == heap_rs.word_size(); } #endif @@ -111,33 +112,73 @@ _bm.print_on_error(st, prefix); } -bool CMBitMap::allocate(ReservedSpace heap_rs) { - _bmStartWord = (HeapWord*)(heap_rs.base()); - _bmWordSize = heap_rs.size()/HeapWordSize; // heap_rs.size() is in bytes - ReservedSpace brs(ReservedSpace::allocation_align_size_up( - (_bmWordSize >> (_shifter + LogBitsPerByte)) + 1)); - if (!brs.is_reserved()) { - warning("ConcurrentMark marking bit map allocation failure"); +size_t CMBitMap::compute_size(size_t heap_size) { + return heap_size / mark_distance(); +} + +size_t CMBitMap::mark_distance() { + return MinObjAlignmentInBytes * BitsPerByte; +} + +void CMBitMap::initialize(MemRegion heap, G1RegionToSpaceMapper* storage) { + _bmStartWord = heap.start(); + _bmWordSize = heap.word_size(); + + _bm.set_map((BitMap::bm_word_t*) storage->reserved().start()); + _bm.set_size(_bmWordSize >> _shifter); + + storage->set_mapping_changed_listener(&_listener); +} + +void CMBitMapMappingChangedListener::on_commit(uint start_region, size_t num_regions) { + // We need to clear the bitmap on commit, removing any existing information. + MemRegion mr(G1CollectedHeap::heap()->bottom_addr_for_region(start_region), num_regions * HeapRegion::GrainWords); + _bm->clearRange(mr); +} + +// Closure used for clearing the given mark bitmap. +class ClearBitmapHRClosure : public HeapRegionClosure { + private: + ConcurrentMark* _cm; + CMBitMap* _bitmap; + bool _may_yield; // The closure may yield during iteration. If yielded, abort the iteration. + public: + ClearBitmapHRClosure(ConcurrentMark* cm, CMBitMap* bitmap, bool may_yield) : HeapRegionClosure(), _cm(cm), _bitmap(bitmap), _may_yield(may_yield) { + assert(!may_yield || cm != NULL, "CM must be non-NULL if this closure is expected to yield."); + } + + virtual bool doHeapRegion(HeapRegion* r) { + size_t const chunk_size_in_words = M / HeapWordSize; + + HeapWord* cur = r->bottom(); + HeapWord* const end = r->end(); + + while (cur < end) { + MemRegion mr(cur, MIN2(cur + chunk_size_in_words, end)); + _bitmap->clearRange(mr); + + cur += chunk_size_in_words; + + // Abort iteration if after yielding the marking has been aborted. + if (_may_yield && _cm->do_yield_check() && _cm->has_aborted()) { + return true; + } + // Repeat the asserts from before the start of the closure. We will do them + // as asserts here to minimize their overhead on the product. However, we + // will have them as guarantees at the beginning / end of the bitmap + // clearing to get some checking in the product. + assert(!_may_yield || _cm->cmThread()->during_cycle(), "invariant"); + assert(!_may_yield || !G1CollectedHeap::heap()->mark_in_progress(), "invariant"); + } + return false; } - MemTracker::record_virtual_memory_type((address)brs.base(), mtGC); - // For now we'll just commit all of the bit map up front. - // Later on we'll try to be more parsimonious with swap. - if (!_virtual_space.initialize(brs, brs.size())) { - warning("ConcurrentMark marking bit map backing store failure"); - return false; - } - assert(_virtual_space.committed_size() == brs.size(), - "didn't reserve backing store for all of concurrent marking bit map?"); - _bm.set_map((BitMap::bm_word_t*)_virtual_space.low()); - assert(_virtual_space.committed_size() << (_shifter + LogBitsPerByte) >= - _bmWordSize, "inconsistency in bit map sizing"); - _bm.set_size(_bmWordSize >> _shifter); - return true; -} +}; void CMBitMap::clearAll() { - _bm.clear(); + ClearBitmapHRClosure cl(NULL, this, false /* may_yield */); + G1CollectedHeap::heap()->heap_region_iterate(&cl); + guarantee(cl.complete(), "Must have completed iteration."); return; } @@ -482,10 +523,10 @@ return MAX2((n_par_threads + 2) / 4, 1U); } -ConcurrentMark::ConcurrentMark(G1CollectedHeap* g1h, ReservedSpace heap_rs) : +ConcurrentMark::ConcurrentMark(G1CollectedHeap* g1h, G1RegionToSpaceMapper* prev_bitmap_storage, G1RegionToSpaceMapper* next_bitmap_storage) : _g1h(g1h), - _markBitMap1(log2_intptr(MinObjAlignment)), - _markBitMap2(log2_intptr(MinObjAlignment)), + _markBitMap1(), + _markBitMap2(), _parallel_marking_threads(0), _max_parallel_marking_threads(0), _sleep_factor(0.0), @@ -494,7 +535,7 @@ _cleanup_task_overhead(1.0), _cleanup_list("Cleanup List"), _region_bm((BitMap::idx_t)(g1h->max_regions()), false /* in_resource_area*/), - _card_bm((heap_rs.size() + CardTableModRefBS::card_size - 1) >> + _card_bm((g1h->reserved_region().byte_size() + CardTableModRefBS::card_size - 1) >> CardTableModRefBS::card_shift, false /* in_resource_area*/), @@ -544,14 +585,8 @@ "heap end = " INTPTR_FORMAT, p2i(_heap_start), p2i(_heap_end)); } - if (!_markBitMap1.allocate(heap_rs)) { - warning("Failed to allocate first CM bit map"); - return; - } - if (!_markBitMap2.allocate(heap_rs)) { - warning("Failed to allocate second CM bit map"); - return; - } + _markBitMap1.initialize(g1h->reserved_region(), prev_bitmap_storage); + _markBitMap2.initialize(g1h->reserved_region(), next_bitmap_storage); // Create & start a ConcurrentMark thread. _cmThread = new ConcurrentMarkThread(this); @@ -562,8 +597,8 @@ } assert(CGC_lock != NULL, "Where's the CGC_lock?"); - assert(_markBitMap1.covers(heap_rs), "_markBitMap1 inconsistency"); - assert(_markBitMap2.covers(heap_rs), "_markBitMap2 inconsistency"); + assert(_markBitMap1.covers(g1h->reserved_region()), "_markBitMap1 inconsistency"); + assert(_markBitMap2.covers(g1h->reserved_region()), "_markBitMap2 inconsistency"); SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set(); satb_qs.set_buffer_size(G1SATBBufferSize); @@ -723,38 +758,17 @@ clear_all_count_data(); // so that the call below can read a sensible value - _heap_start = (HeapWord*) heap_rs.base(); + _heap_start = g1h->reserved_region().start(); set_non_marking_state(); _completed_initialization = true; } -void ConcurrentMark::update_g1_committed(bool force) { - // If concurrent marking is not in progress, then we do not need to - // update _heap_end. - if (!concurrent_marking_in_progress() && !force) return; - - MemRegion committed = _g1h->g1_committed(); - assert(committed.start() == _heap_start, "start shouldn't change"); - HeapWord* new_end = committed.end(); - if (new_end > _heap_end) { - // The heap has been expanded. - - _heap_end = new_end; - } - // Notice that the heap can also shrink. However, this only happens - // during a Full GC (at least currently) and the entire marking - // phase will bail out and the task will not be restarted. So, let's - // do nothing. -} - void ConcurrentMark::reset() { // Starting values for these two. This should be called in a STW - // phase. CM will be notified of any future g1_committed expansions - // will be at the end of evacuation pauses, when tasks are - // inactive. - MemRegion committed = _g1h->g1_committed(); - _heap_start = committed.start(); - _heap_end = committed.end(); + // phase. + MemRegion reserved = _g1h->g1_reserved(); + _heap_start = reserved.start(); + _heap_end = reserved.end(); // Separated the asserts so that we know which one fires. assert(_heap_start != NULL, "heap bounds should look ok"); @@ -826,7 +840,6 @@ assert(out_of_regions(), err_msg("only way to get here: _finger: "PTR_FORMAT", _heap_end: "PTR_FORMAT, p2i(_finger), p2i(_heap_end))); - update_g1_committed(true); } } @@ -845,7 +858,6 @@ void ConcurrentMark::clearNextBitmap() { G1CollectedHeap* g1h = G1CollectedHeap::heap(); - G1CollectorPolicy* g1p = g1h->g1_policy(); // Make sure that the concurrent mark thread looks to still be in // the current cycle. @@ -857,41 +869,36 @@ // is the case. guarantee(!g1h->mark_in_progress(), "invariant"); - // clear the mark bitmap (no grey objects to start with). - // We need to do this in chunks and offer to yield in between - // each chunk. - HeapWord* start = _nextMarkBitMap->startWord(); - HeapWord* end = _nextMarkBitMap->endWord(); - HeapWord* cur = start; - size_t chunkSize = M; - while (cur < end) { - HeapWord* next = cur + chunkSize; - if (next > end) { - next = end; - } - MemRegion mr(cur,next); - _nextMarkBitMap->clearRange(mr); - cur = next; - do_yield_check(); - - // Repeat the asserts from above. We'll do them as asserts here to - // minimize their overhead on the product. However, we'll have - // them as guarantees at the beginning / end of the bitmap - // clearing to get some checking in the product. - assert(cmThread()->during_cycle(), "invariant"); - assert(!g1h->mark_in_progress(), "invariant"); + ClearBitmapHRClosure cl(this, _nextMarkBitMap, true /* may_yield */); + g1h->heap_region_iterate(&cl); + + // Clear the liveness counting data. If the marking has been aborted, the abort() + // call already did that. + if (cl.complete()) { + clear_all_count_data(); } - // Clear the liveness counting data - clear_all_count_data(); - // Repeat the asserts from above. guarantee(cmThread()->during_cycle(), "invariant"); guarantee(!g1h->mark_in_progress(), "invariant"); } +class CheckBitmapClearHRClosure : public HeapRegionClosure { + CMBitMap* _bitmap; + bool _error; + public: + CheckBitmapClearHRClosure(CMBitMap* bitmap) : _bitmap(bitmap) { + } + + virtual bool doHeapRegion(HeapRegion* r) { + return _bitmap->getNextMarkedWordAddress(r->bottom(), r->end()) != r->end(); + } +}; + bool ConcurrentMark::nextMarkBitmapIsClear() { - return _nextMarkBitMap->getNextMarkedWordAddress(_heap_start, _heap_end) == _heap_end; + CheckBitmapClearHRClosure cl(_nextMarkBitMap); + _g1h->heap_region_iterate(&cl); + return cl.complete(); } class NoteStartOfMarkHRClosure: public HeapRegionClosure { @@ -2191,10 +2198,10 @@ _cleanup_list.length()); } - // Noone else should be accessing the _cleanup_list at this point, - // so it's not necessary to take any locks + // No one else should be accessing the _cleanup_list at this point, + // so it is not necessary to take any locks while (!_cleanup_list.is_empty()) { - HeapRegion* hr = _cleanup_list.remove_head(); + HeapRegion* hr = _cleanup_list.remove_region(true /* from_head */); assert(hr != NULL, "Got NULL from a non-empty list"); hr->par_clear(); tmp_free_list.add_ordered(hr); @@ -2800,7 +2807,6 @@ str = " O"; } else { HeapRegion* hr = _g1h->heap_region_containing(obj); - guarantee(hr != NULL, "invariant"); bool over_tams = _g1h->allocated_since_marking(obj, hr, _vo); bool marked = _g1h->is_marked(obj, _vo); @@ -2979,22 +2985,25 @@ // claim_region() and a humongous object allocation might force us // to do a bit of unnecessary work (due to some unnecessary bitmap // iterations) but it should not introduce and correctness issues. - HeapRegion* curr_region = _g1h->heap_region_containing_raw(finger); - HeapWord* bottom = curr_region->bottom(); - HeapWord* end = curr_region->end(); - HeapWord* limit = curr_region->next_top_at_mark_start(); - - if (verbose_low()) { - gclog_or_tty->print_cr("[%u] curr_region = "PTR_FORMAT" " - "["PTR_FORMAT", "PTR_FORMAT"), " - "limit = "PTR_FORMAT, - worker_id, p2i(curr_region), p2i(bottom), p2i(end), p2i(limit)); - } + HeapRegion* curr_region = _g1h->heap_region_containing_raw(finger); + + // Above heap_region_containing_raw may return NULL as we always scan claim + // until the end of the heap. In this case, just jump to the next region. + HeapWord* end = curr_region != NULL ? curr_region->end() : finger + HeapRegion::GrainWords; // Is the gap between reading the finger and doing the CAS too long? HeapWord* res = (HeapWord*) Atomic::cmpxchg_ptr(end, &_finger, finger); - if (res == finger) { + if (res == finger && curr_region != NULL) { // we succeeded + HeapWord* bottom = curr_region->bottom(); + HeapWord* limit = curr_region->next_top_at_mark_start(); + + if (verbose_low()) { + gclog_or_tty->print_cr("[%u] curr_region = "PTR_FORMAT" " + "["PTR_FORMAT", "PTR_FORMAT"), " + "limit = "PTR_FORMAT, + worker_id, p2i(curr_region), p2i(bottom), p2i(end), p2i(limit)); + } // notice that _finger == end cannot be guaranteed here since, // someone else might have moved the finger even further @@ -3025,10 +3034,17 @@ } else { assert(_finger > finger, "the finger should have moved forward"); if (verbose_low()) { - gclog_or_tty->print_cr("[%u] somebody else moved the finger, " - "global finger = "PTR_FORMAT", " - "our finger = "PTR_FORMAT, - worker_id, p2i(_finger), p2i(finger)); + if (curr_region == NULL) { + gclog_or_tty->print_cr("[%u] found uncommitted region, moving finger, " + "global finger = "PTR_FORMAT", " + "our finger = "PTR_FORMAT, + worker_id, p2i(_finger), p2i(finger)); + } else { + gclog_or_tty->print_cr("[%u] somebody else moved the finger, " + "global finger = "PTR_FORMAT", " + "our finger = "PTR_FORMAT, + worker_id, p2i(_finger), p2i(finger)); + } } // read it again @@ -3143,8 +3159,10 @@ // happens, heap_region_containing() will return the bottom of the // corresponding starts humongous region and the check below will // not hold any more. + // Since we always iterate over all regions, we might get a NULL HeapRegion + // here. HeapRegion* global_hr = _g1h->heap_region_containing_raw(global_finger); - guarantee(global_finger == global_hr->bottom(), + guarantee(global_hr == NULL || global_finger == global_hr->bottom(), err_msg("global finger: "PTR_FORMAT" region: "HR_FORMAT, p2i(global_finger), HR_FORMAT_PARAMS(global_hr))); } @@ -3157,7 +3175,7 @@ if (task_finger != NULL && task_finger < _heap_end) { // See above note on the global finger verification. HeapRegion* task_hr = _g1h->heap_region_containing_raw(task_finger); - guarantee(task_finger == task_hr->bottom() || + guarantee(task_hr == NULL || task_finger == task_hr->bottom() || !task_hr->in_collection_set(), err_msg("task finger: "PTR_FORMAT" region: "HR_FORMAT, p2i(task_finger), HR_FORMAT_PARAMS(task_hr))); @@ -3565,9 +3583,8 @@ } void CMTask::setup_for_region(HeapRegion* hr) { - // Separated the asserts so that we know which one fires. assert(hr != NULL, - "claim_region() should have filtered out continues humongous regions"); + "claim_region() should have filtered out NULL regions"); assert(!hr->continuesHumongous(), "claim_region() should have filtered out continues humongous regions"); @@ -4674,7 +4691,6 @@ _hum_prev_live_bytes(0), _hum_next_live_bytes(0), _total_remset_bytes(0), _total_strong_code_roots_bytes(0) { G1CollectedHeap* g1h = G1CollectedHeap::heap(); - MemRegion g1_committed = g1h->g1_committed(); MemRegion g1_reserved = g1h->g1_reserved(); double now = os::elapsedTime(); @@ -4682,10 +4698,8 @@ _out->cr(); _out->print_cr(G1PPRL_LINE_PREFIX" PHASE %s @ %1.3f", phase_name, now); _out->print_cr(G1PPRL_LINE_PREFIX" HEAP" - G1PPRL_SUM_ADDR_FORMAT("committed") G1PPRL_SUM_ADDR_FORMAT("reserved") G1PPRL_SUM_BYTE_FORMAT("region-size"), - p2i(g1_committed.start()), p2i(g1_committed.end()), p2i(g1_reserved.start()), p2i(g1_reserved.end()), HeapRegion::GrainBytes); _out->print_cr(G1PPRL_LINE_PREFIX);
--- a/src/share/vm/gc_implementation/g1/concurrentMark.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -27,10 +27,12 @@ #include "classfile/javaClasses.hpp" #include "gc_implementation/g1/heapRegionSet.hpp" +#include "gc_implementation/g1/g1RegionToSpaceMapper.hpp" #include "gc_implementation/shared/gcId.hpp" #include "utilities/taskqueue.hpp" class G1CollectedHeap; +class CMBitMap; class CMTask; typedef GenericTaskQueue<oop, mtGC> CMTaskQueue; typedef GenericTaskQueueSet<CMTaskQueue, mtGC> CMTaskQueueSet; @@ -57,7 +59,6 @@ HeapWord* _bmStartWord; // base address of range covered by map size_t _bmWordSize; // map size (in #HeapWords covered) const int _shifter; // map to char or bit - VirtualSpace _virtual_space; // underlying the bit map BitMap _bm; // the bit map itself public: @@ -115,42 +116,41 @@ void print_on_error(outputStream* st, const char* prefix) const; // debugging - NOT_PRODUCT(bool covers(ReservedSpace rs) const;) + NOT_PRODUCT(bool covers(MemRegion rs) const;) +}; + +class CMBitMapMappingChangedListener : public G1MappingChangedListener { + private: + CMBitMap* _bm; + public: + CMBitMapMappingChangedListener() : _bm(NULL) {} + + void set_bitmap(CMBitMap* bm) { _bm = bm; } + + virtual void on_commit(uint start_idx, size_t num_regions); }; class CMBitMap : public CMBitMapRO { + private: + CMBitMapMappingChangedListener _listener; public: - // constructor - CMBitMap(int shifter) : - CMBitMapRO(shifter) {} + static size_t compute_size(size_t heap_size); + // Returns the amount of bytes on the heap between two marks in the bitmap. + static size_t mark_distance(); - // Allocates the back store for the marking bitmap - bool allocate(ReservedSpace heap_rs); + CMBitMap() : CMBitMapRO(LogMinObjAlignment), _listener() { _listener.set_bitmap(this); } - // write marks - void mark(HeapWord* addr) { - assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), - "outside underlying space?"); - _bm.set_bit(heapWordToOffset(addr)); - } - void clear(HeapWord* addr) { - assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), - "outside underlying space?"); - _bm.clear_bit(heapWordToOffset(addr)); - } - bool parMark(HeapWord* addr) { - assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), - "outside underlying space?"); - return _bm.par_set_bit(heapWordToOffset(addr)); - } - bool parClear(HeapWord* addr) { - assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), - "outside underlying space?"); - return _bm.par_clear_bit(heapWordToOffset(addr)); - } + // Initializes the underlying BitMap to cover the given area. + void initialize(MemRegion heap, G1RegionToSpaceMapper* storage); + + // Write marks. + inline void mark(HeapWord* addr); + inline void clear(HeapWord* addr); + inline bool parMark(HeapWord* addr); + inline bool parClear(HeapWord* addr); + void markRange(MemRegion mr); - void clearAll(); void clearRange(MemRegion mr); // Starting at the bit corresponding to "addr" (inclusive), find the next @@ -161,6 +161,9 @@ // the run. If there is no "1" bit at or after "addr", return an empty // MemRegion. MemRegion getAndClearMarkedRegion(HeapWord* addr, HeapWord* end_addr); + + // Clear the whole mark bitmap. + void clearAll(); }; // Represents a marking stack used by ConcurrentMarking in the G1 collector. @@ -680,7 +683,7 @@ return _task_queues->steal(worker_id, hash_seed, obj); } - ConcurrentMark(G1CollectedHeap* g1h, ReservedSpace heap_rs); + ConcurrentMark(G1CollectedHeap* g1h, G1RegionToSpaceMapper* prev_bitmap_storage, G1RegionToSpaceMapper* next_bitmap_storage); ~ConcurrentMark(); ConcurrentMarkThread* cmThread() { return _cmThread; } @@ -736,7 +739,8 @@ // Clear the next marking bitmap (will be called concurrently). void clearNextBitmap(); - // Return whether the next mark bitmap has no marks set. + // Return whether the next mark bitmap has no marks set. To be used for assertions + // only. Will not yield to pause requests. bool nextMarkBitmapIsClear(); // These two do the work that needs to be done before and after the @@ -794,12 +798,6 @@ bool verify_thread_buffers, bool verify_fingers) PRODUCT_RETURN; - // It is called at the end of an evacuation pause during marking so - // that CM is notified of where the new end of the heap is. It - // doesn't do anything if concurrent_marking_in_progress() is false, - // unless the force parameter is true. - void update_g1_committed(bool force = false); - bool isMarked(oop p) const { assert(p != NULL && p->is_oop(), "expected an oop"); HeapWord* addr = (HeapWord*)p;
--- a/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -268,6 +268,36 @@ return iterate(cl, mr); } +#define check_mark(addr) \ + assert(_bmStartWord <= (addr) && (addr) < (_bmStartWord + _bmWordSize), \ + "outside underlying space?"); \ + assert(G1CollectedHeap::heap()->is_in_exact(addr), \ + err_msg("Trying to access not available bitmap "PTR_FORMAT \ + " corresponding to "PTR_FORMAT" (%u)", \ + p2i(this), p2i(addr), G1CollectedHeap::heap()->addr_to_region(addr))); + +inline void CMBitMap::mark(HeapWord* addr) { + check_mark(addr); + _bm.set_bit(heapWordToOffset(addr)); +} + +inline void CMBitMap::clear(HeapWord* addr) { + check_mark(addr); + _bm.clear_bit(heapWordToOffset(addr)); +} + +inline bool CMBitMap::parMark(HeapWord* addr) { + check_mark(addr); + return _bm.par_set_bit(heapWordToOffset(addr)); +} + +inline bool CMBitMap::parClear(HeapWord* addr) { + check_mark(addr); + return _bm.par_clear_bit(heapWordToOffset(addr)); +} + +#undef check_mark + inline void CMTask::push(oop obj) { HeapWord* objAddr = (HeapWord*) obj; assert(_g1h->is_in_g1_reserved(objAddr), "invariant");
--- a/src/share/vm/gc_implementation/g1/g1AllocRegion.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1AllocRegion.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -173,7 +173,7 @@ // Should be called when we want to release the active region which // is returned after it's been retired. - HeapRegion* release(); + virtual HeapRegion* release(); #if G1_ALLOC_REGION_TRACING void trace(const char* str, size_t word_size = 0, HeapWord* result = NULL);
--- a/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -32,64 +32,37 @@ PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC +void G1BlockOffsetSharedArrayMappingChangedListener::on_commit(uint start_idx, size_t num_regions) { + // Nothing to do. The BOT is hard-wired to be part of the HeapRegion, and we cannot + // retrieve it here since this would cause firing of several asserts. The code + // executed after commit of a region already needs to do some re-initialization of + // the HeapRegion, so we combine that. +} + ////////////////////////////////////////////////////////////////////// // G1BlockOffsetSharedArray ////////////////////////////////////////////////////////////////////// -G1BlockOffsetSharedArray::G1BlockOffsetSharedArray(MemRegion reserved, - size_t init_word_size) : - _reserved(reserved), _end(NULL) -{ - size_t size = compute_size(reserved.word_size()); - ReservedSpace rs(ReservedSpace::allocation_align_size_up(size)); - if (!rs.is_reserved()) { - vm_exit_during_initialization("Could not reserve enough space for heap offset array"); - } - if (!_vs.initialize(rs, 0)) { - vm_exit_during_initialization("Could not reserve enough space for heap offset array"); - } +G1BlockOffsetSharedArray::G1BlockOffsetSharedArray(MemRegion heap, G1RegionToSpaceMapper* storage) : + _reserved(), _end(NULL), _listener(), _offset_array(NULL) { + + _reserved = heap; + _end = NULL; - MemTracker::record_virtual_memory_type((address)rs.base(), mtGC); + MemRegion bot_reserved = storage->reserved(); - _offset_array = (u_char*)_vs.low_boundary(); - resize(init_word_size); + _offset_array = (u_char*)bot_reserved.start(); + _end = _reserved.end(); + + storage->set_mapping_changed_listener(&_listener); + if (TraceBlockOffsetTable) { gclog_or_tty->print_cr("G1BlockOffsetSharedArray::G1BlockOffsetSharedArray: "); gclog_or_tty->print_cr(" " " rs.base(): " INTPTR_FORMAT " rs.size(): " INTPTR_FORMAT " rs end(): " INTPTR_FORMAT, - rs.base(), rs.size(), rs.base() + rs.size()); - gclog_or_tty->print_cr(" " - " _vs.low_boundary(): " INTPTR_FORMAT - " _vs.high_boundary(): " INTPTR_FORMAT, - _vs.low_boundary(), - _vs.high_boundary()); - } -} - -void G1BlockOffsetSharedArray::resize(size_t new_word_size) { - assert(new_word_size <= _reserved.word_size(), "Resize larger than reserved"); - size_t new_size = compute_size(new_word_size); - size_t old_size = _vs.committed_size(); - size_t delta; - char* high = _vs.high(); - _end = _reserved.start() + new_word_size; - if (new_size > old_size) { - delta = ReservedSpace::page_align_size_up(new_size - old_size); - assert(delta > 0, "just checking"); - if (!_vs.expand_by(delta)) { - // Do better than this for Merlin - vm_exit_out_of_memory(delta, OOM_MMAP_ERROR, "offset table expansion"); - } - assert(_vs.high() == high + delta, "invalid expansion"); - // Initialization of the contents is left to the - // G1BlockOffsetArray that uses it. - } else { - delta = ReservedSpace::page_align_size_down(old_size - new_size); - if (delta == 0) return; - _vs.shrink_by(delta); - assert(_vs.high() == high - delta, "invalid expansion"); + bot_reserved.start(), bot_reserved.byte_size(), bot_reserved.end()); } } @@ -100,18 +73,7 @@ } void G1BlockOffsetSharedArray::set_offset_array(HeapWord* left, HeapWord* right, u_char offset) { - check_index(index_for(right - 1), "right address out of range"); - assert(left < right, "Heap addresses out of order"); - size_t num_cards = pointer_delta(right, left) >> LogN_words; - if (UseMemSetInBOT) { - memset(&_offset_array[index_for(left)], offset, num_cards); - } else { - size_t i = index_for(left); - const size_t end = i + num_cards; - for (; i < end; i++) { - _offset_array[i] = offset; - } - } + set_offset_array(index_for(left), index_for(right -1), offset); } ////////////////////////////////////////////////////////////////////// @@ -651,6 +613,25 @@ _next_offset_index = 0; } +HeapWord* G1BlockOffsetArrayContigSpace::initialize_threshold_raw() { + assert(!Universe::heap()->is_in_reserved(_array->_offset_array), + "just checking"); + _next_offset_index = _array->index_for_raw(_bottom); + _next_offset_index++; + _next_offset_threshold = + _array->address_for_index_raw(_next_offset_index); + return _next_offset_threshold; +} + +void G1BlockOffsetArrayContigSpace::zero_bottom_entry_raw() { + assert(!Universe::heap()->is_in_reserved(_array->_offset_array), + "just checking"); + size_t bottom_index = _array->index_for_raw(_bottom); + assert(_array->address_for_index_raw(bottom_index) == _bottom, + "Precondition of call"); + _array->set_offset_array_raw(bottom_index, 0); +} + HeapWord* G1BlockOffsetArrayContigSpace::initialize_threshold() { assert(!Universe::heap()->is_in_reserved(_array->_offset_array), "just checking"); @@ -675,8 +656,7 @@ assert(new_top <= _end, "_end should have already been updated"); // The first BOT entry should have offset 0. - zero_bottom_entry(); - initialize_threshold(); + reset_bot(); alloc_block(_bottom, new_top); }
--- a/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -25,6 +25,7 @@ #ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1BLOCKOFFSETTABLE_HPP #define SHARE_VM_GC_IMPLEMENTATION_G1_G1BLOCKOFFSETTABLE_HPP +#include "gc_implementation/g1/g1RegionToSpaceMapper.hpp" #include "memory/memRegion.hpp" #include "runtime/virtualspace.hpp" #include "utilities/globalDefinitions.hpp" @@ -106,6 +107,11 @@ inline HeapWord* block_start_const(const void* addr) const; }; +class G1BlockOffsetSharedArrayMappingChangedListener : public G1MappingChangedListener { + public: + virtual void on_commit(uint start_idx, size_t num_regions); +}; + // This implementation of "G1BlockOffsetTable" divides the covered region // into "N"-word subregions (where "N" = 2^"LogN". An array with an entry // for each such subregion indicates how far back one must go to find the @@ -125,6 +131,7 @@ friend class VMStructs; private: + G1BlockOffsetSharedArrayMappingChangedListener _listener; // The reserved region covered by the shared array. MemRegion _reserved; @@ -133,16 +140,8 @@ // Array for keeping offsets for retrieving object start fast given an // address. - VirtualSpace _vs; u_char* _offset_array; // byte array keeping backwards offsets - void check_index(size_t index, const char* msg) const { - assert(index < _vs.committed_size(), - err_msg("%s - " - "index: " SIZE_FORMAT ", _vs.committed_size: " SIZE_FORMAT, - msg, index, _vs.committed_size())); - } - void check_offset(size_t offset, const char* msg) const { assert(offset <= N_words, err_msg("%s - " @@ -152,63 +151,33 @@ // Bounds checking accessors: // For performance these have to devolve to array accesses in product builds. - u_char offset_array(size_t index) const { - check_index(index, "index out of range"); - return _offset_array[index]; - } + inline u_char offset_array(size_t index) const; void set_offset_array(HeapWord* left, HeapWord* right, u_char offset); - void set_offset_array(size_t index, u_char offset) { - check_index(index, "index out of range"); - check_offset(offset, "offset too large"); + void set_offset_array_raw(size_t index, u_char offset) { _offset_array[index] = offset; } - void set_offset_array(size_t index, HeapWord* high, HeapWord* low) { - check_index(index, "index out of range"); - assert(high >= low, "addresses out of order"); - check_offset(pointer_delta(high, low), "offset too large"); - _offset_array[index] = (u_char) pointer_delta(high, low); - } + inline void set_offset_array(size_t index, u_char offset); + + inline void set_offset_array(size_t index, HeapWord* high, HeapWord* low); - void set_offset_array(size_t left, size_t right, u_char offset) { - check_index(right, "right index out of range"); - assert(left <= right, "indexes out of order"); - size_t num_cards = right - left + 1; - if (UseMemSetInBOT) { - memset(&_offset_array[left], offset, num_cards); - } else { - size_t i = left; - const size_t end = i + num_cards; - for (; i < end; i++) { - _offset_array[i] = offset; - } - } - } + inline void set_offset_array(size_t left, size_t right, u_char offset); - void check_offset_array(size_t index, HeapWord* high, HeapWord* low) const { - check_index(index, "index out of range"); - assert(high >= low, "addresses out of order"); - check_offset(pointer_delta(high, low), "offset too large"); - assert(_offset_array[index] == pointer_delta(high, low), "Wrong offset"); - } + inline void check_offset_array(size_t index, HeapWord* high, HeapWord* low) const; bool is_card_boundary(HeapWord* p) const; +public: + // Return the number of slots needed for an offset array // that covers mem_region_words words. - // We always add an extra slot because if an object - // ends on a card boundary we put a 0 in the next - // offset array slot, so we want that slot always - // to be reserved. - - size_t compute_size(size_t mem_region_words) { - size_t number_of_slots = (mem_region_words / N_words) + 1; - return ReservedSpace::page_align_size_up(number_of_slots); + static size_t compute_size(size_t mem_region_words) { + size_t number_of_slots = (mem_region_words / N_words); + return ReservedSpace::allocation_align_size_up(number_of_slots); } -public: enum SomePublicConstants { LogN = 9, LogN_words = LogN - LogHeapWordSize, @@ -222,25 +191,21 @@ // least "init_word_size".) The contents of the initial table are // undefined; it is the responsibility of the constituent // G1BlockOffsetTable(s) to initialize cards. - G1BlockOffsetSharedArray(MemRegion reserved, size_t init_word_size); - - // Notes a change in the committed size of the region covered by the - // table. The "new_word_size" may not be larger than the size of the - // reserved region this table covers. - void resize(size_t new_word_size); + G1BlockOffsetSharedArray(MemRegion heap, G1RegionToSpaceMapper* storage); void set_bottom(HeapWord* new_bottom); - // Updates all the BlockOffsetArray's sharing this shared array to - // reflect the current "top"'s of their spaces. - void update_offset_arrays(); - // Return the appropriate index into "_offset_array" for "p". inline size_t index_for(const void* p) const; + inline size_t index_for_raw(const void* p) const; // Return the address indicating the start of the region corresponding to // "index" in "_offset_array". inline HeapWord* address_for_index(size_t index) const; + // Variant of address_for_index that does not check the index for validity. + inline HeapWord* address_for_index_raw(size_t index) const { + return _reserved.start() + (index << LogN_words); + } }; // And here is the G1BlockOffsetTable subtype that uses the array. @@ -480,6 +445,14 @@ blk_start, blk_end); } + // Variant of zero_bottom_entry that does not check for availability of the + // memory first. + void zero_bottom_entry_raw(); + // Variant of initialize_threshold that does not check for availability of the + // memory first. + HeapWord* initialize_threshold_raw(); + // Zero out the entry for _bottom (offset will be zero). + void zero_bottom_entry(); public: G1BlockOffsetArrayContigSpace(G1BlockOffsetSharedArray* array, MemRegion mr); @@ -487,8 +460,10 @@ // bottom of the covered region. HeapWord* initialize_threshold(); - // Zero out the entry for _bottom (offset will be zero). - void zero_bottom_entry(); + void reset_bot() { + zero_bottom_entry_raw(); + initialize_threshold_raw(); + } // Return the next threshold, the point at which the table should be // updated.
--- a/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.inline.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.inline.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -47,14 +47,69 @@ } } +#define check_index(index, msg) \ + assert((index) < (_reserved.word_size() >> LogN_words), \ + err_msg("%s - index: "SIZE_FORMAT", _vs.committed_size: "SIZE_FORMAT, \ + msg, (index), (_reserved.word_size() >> LogN_words))); \ + assert(G1CollectedHeap::heap()->is_in_exact(address_for_index_raw(index)), \ + err_msg("Index "SIZE_FORMAT" corresponding to "PTR_FORMAT \ + " (%u) is not in committed area.", \ + (index), \ + p2i(address_for_index_raw(index)), \ + G1CollectedHeap::heap()->addr_to_region(address_for_index_raw(index)))); + +u_char G1BlockOffsetSharedArray::offset_array(size_t index) const { + check_index(index, "index out of range"); + return _offset_array[index]; +} + +void G1BlockOffsetSharedArray::set_offset_array(size_t index, u_char offset) { + check_index(index, "index out of range"); + set_offset_array_raw(index, offset); +} + +void G1BlockOffsetSharedArray::set_offset_array(size_t index, HeapWord* high, HeapWord* low) { + check_index(index, "index out of range"); + assert(high >= low, "addresses out of order"); + size_t offset = pointer_delta(high, low); + check_offset(offset, "offset too large"); + set_offset_array(index, (u_char)offset); +} + +void G1BlockOffsetSharedArray::set_offset_array(size_t left, size_t right, u_char offset) { + check_index(right, "right index out of range"); + assert(left <= right, "indexes out of order"); + size_t num_cards = right - left + 1; + if (UseMemSetInBOT) { + memset(&_offset_array[left], offset, num_cards); + } else { + size_t i = left; + const size_t end = i + num_cards; + for (; i < end; i++) { + _offset_array[i] = offset; + } + } +} + +void G1BlockOffsetSharedArray::check_offset_array(size_t index, HeapWord* high, HeapWord* low) const { + check_index(index, "index out of range"); + assert(high >= low, "addresses out of order"); + check_offset(pointer_delta(high, low), "offset too large"); + assert(_offset_array[index] == pointer_delta(high, low), "Wrong offset"); +} + +// Variant of index_for that does not check the index for validity. +inline size_t G1BlockOffsetSharedArray::index_for_raw(const void* p) const { + return pointer_delta((char*)p, _reserved.start(), sizeof(char)) >> LogN; +} + inline size_t G1BlockOffsetSharedArray::index_for(const void* p) const { char* pc = (char*)p; assert(pc >= (char*)_reserved.start() && pc < (char*)_reserved.end(), err_msg("p (" PTR_FORMAT ") not in reserved [" PTR_FORMAT ", " PTR_FORMAT ")", p2i(p), p2i(_reserved.start()), p2i(_reserved.end()))); - size_t delta = pointer_delta(pc, _reserved.start(), sizeof(char)); - size_t result = delta >> LogN; + size_t result = index_for_raw(p); check_index(result, "bad index from address"); return result; } @@ -62,7 +117,7 @@ inline HeapWord* G1BlockOffsetSharedArray::address_for_index(size_t index) const { check_index(index, "index out of range"); - HeapWord* result = _reserved.start() + (index << LogN_words); + HeapWord* result = address_for_index_raw(index); assert(result >= _reserved.start() && result < _reserved.end(), err_msg("bad address from index result " PTR_FORMAT " _reserved.start() " PTR_FORMAT " _reserved.end() " @@ -71,6 +126,8 @@ return result; } +#undef check_index + inline size_t G1BlockOffsetArray::block_size(const HeapWord* p) const { return gsp()->block_size(p);
--- a/src/share/vm/gc_implementation/g1/g1CardCounts.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CardCounts.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -33,31 +33,26 @@ PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC +void G1CardCountsMappingChangedListener::on_commit(uint start_idx, size_t num_regions) { + MemRegion mr(G1CollectedHeap::heap()->bottom_addr_for_region(start_idx), num_regions * HeapRegion::GrainWords); + _counts->clear_range(mr); +} + void G1CardCounts::clear_range(size_t from_card_num, size_t to_card_num) { if (has_count_table()) { - assert(from_card_num >= 0 && from_card_num < _committed_max_card_num, - err_msg("from card num out of range: "SIZE_FORMAT, from_card_num)); assert(from_card_num < to_card_num, err_msg("Wrong order? from: " SIZE_FORMAT ", to: "SIZE_FORMAT, from_card_num, to_card_num)); - assert(to_card_num <= _committed_max_card_num, - err_msg("to card num out of range: " - "to: "SIZE_FORMAT ", " - "max: "SIZE_FORMAT, - to_card_num, _committed_max_card_num)); - - to_card_num = MIN2(_committed_max_card_num, to_card_num); - Copy::fill_to_bytes(&_card_counts[from_card_num], (to_card_num - from_card_num)); } } G1CardCounts::G1CardCounts(G1CollectedHeap *g1h): - _g1h(g1h), _card_counts(NULL), - _reserved_max_card_num(0), _committed_max_card_num(0), - _committed_size(0) {} + _listener(), _g1h(g1h), _card_counts(NULL), _reserved_max_card_num(0) { + _listener.set_cardcounts(this); +} -void G1CardCounts::initialize() { +void G1CardCounts::initialize(G1RegionToSpaceMapper* mapper) { assert(_g1h->max_capacity() > 0, "initialization order"); assert(_g1h->capacity() == 0, "initialization order"); @@ -70,70 +65,9 @@ _ct_bs = _g1h->g1_barrier_set(); _ct_bot = _ct_bs->byte_for_const(_g1h->reserved_region().start()); - // Allocate/Reserve the counts table - size_t reserved_bytes = _g1h->max_capacity(); - _reserved_max_card_num = reserved_bytes >> CardTableModRefBS::card_shift; - - size_t reserved_size = _reserved_max_card_num * sizeof(jbyte); - ReservedSpace rs(ReservedSpace::allocation_align_size_up(reserved_size)); - if (!rs.is_reserved()) { - warning("Could not reserve enough space for the card counts table"); - guarantee(!has_reserved_count_table(), "should be NULL"); - return; - } - - MemTracker::record_virtual_memory_type((address)rs.base(), mtGC); - - _card_counts_storage.initialize(rs, 0); - _card_counts = (jubyte*) _card_counts_storage.low(); - } -} - -void G1CardCounts::resize(size_t heap_capacity) { - // Expand the card counts table to handle a heap with the given capacity. - - if (!has_reserved_count_table()) { - // Don't expand if we failed to reserve the card counts table. - return; - } - - assert(_committed_size == - ReservedSpace::allocation_align_size_up(_committed_size), - err_msg("Unaligned? committed_size: " SIZE_FORMAT, _committed_size)); - - // Verify that the committed space for the card counts matches our - // committed max card num. Note for some allocation alignments, the - // amount of space actually committed for the counts table will be able - // to span more cards than the number spanned by the maximum heap. - size_t prev_committed_size = _committed_size; - size_t prev_committed_card_num = committed_to_card_num(prev_committed_size); - - assert(prev_committed_card_num == _committed_max_card_num, - err_msg("Card mismatch: " - "prev: " SIZE_FORMAT ", " - "committed: "SIZE_FORMAT", " - "reserved: "SIZE_FORMAT, - prev_committed_card_num, _committed_max_card_num, _reserved_max_card_num)); - - size_t new_size = (heap_capacity >> CardTableModRefBS::card_shift) * sizeof(jbyte); - size_t new_committed_size = ReservedSpace::allocation_align_size_up(new_size); - size_t new_committed_card_num = committed_to_card_num(new_committed_size); - - if (_committed_max_card_num < new_committed_card_num) { - // we need to expand the backing store for the card counts - size_t expand_size = new_committed_size - prev_committed_size; - - if (!_card_counts_storage.expand_by(expand_size)) { - warning("Card counts table backing store commit failure"); - return; - } - assert(_card_counts_storage.committed_size() == new_committed_size, - "expansion commit failure"); - - _committed_size = new_committed_size; - _committed_max_card_num = new_committed_card_num; - - clear_range(prev_committed_card_num, _committed_max_card_num); + _card_counts = (jubyte*) mapper->reserved().start(); + _reserved_max_card_num = mapper->reserved().byte_size(); + mapper->set_mapping_changed_listener(&_listener); } } @@ -149,12 +83,13 @@ uint count = 0; if (has_count_table()) { size_t card_num = ptr_2_card_num(card_ptr); - if (card_num < _committed_max_card_num) { - count = (uint) _card_counts[card_num]; - if (count < G1ConcRSHotCardLimit) { - _card_counts[card_num] = - (jubyte)(MIN2((uintx)(_card_counts[card_num] + 1), G1ConcRSHotCardLimit)); - } + assert(card_num < _reserved_max_card_num, + err_msg("Card "SIZE_FORMAT" outside of card counts table (max size "SIZE_FORMAT")", + card_num, _reserved_max_card_num)); + count = (uint) _card_counts[card_num]; + if (count < G1ConcRSHotCardLimit) { + _card_counts[card_num] = + (jubyte)(MIN2((uintx)(_card_counts[card_num] + 1), G1ConcRSHotCardLimit)); } } return count; @@ -165,31 +100,23 @@ } void G1CardCounts::clear_region(HeapRegion* hr) { - assert(!hr->isHumongous(), "Should have been cleared"); - if (has_count_table()) { - HeapWord* bottom = hr->bottom(); + MemRegion mr(hr->bottom(), hr->end()); + clear_range(mr); +} - // We use the last address in hr as hr could be the - // last region in the heap. In which case trying to find - // the card for hr->end() will be an OOB accesss to the - // card table. - HeapWord* last = hr->end() - 1; - assert(_g1h->g1_committed().contains(last), - err_msg("last not in committed: " - "last: " PTR_FORMAT ", " - "committed: [" PTR_FORMAT ", " PTR_FORMAT ")", - last, - _g1h->g1_committed().start(), - _g1h->g1_committed().end())); - - const jbyte* from_card_ptr = _ct_bs->byte_for_const(bottom); - const jbyte* last_card_ptr = _ct_bs->byte_for_const(last); +void G1CardCounts::clear_range(MemRegion mr) { + if (has_count_table()) { + const jbyte* from_card_ptr = _ct_bs->byte_for_const(mr.start()); + // We use the last address in the range as the range could represent the + // last region in the heap. In which case trying to find the card will be an + // OOB access to the card table. + const jbyte* last_card_ptr = _ct_bs->byte_for_const(mr.last()); #ifdef ASSERT HeapWord* start_addr = _ct_bs->addr_for(from_card_ptr); - assert(start_addr == hr->bottom(), "alignment"); + assert(start_addr == mr.start(), "MemRegion start must be aligned to a card."); HeapWord* last_addr = _ct_bs->addr_for(last_card_ptr); - assert((last_addr + CardTableModRefBS::card_size_in_words) == hr->end(), "alignment"); + assert((last_addr + CardTableModRefBS::card_size_in_words) == mr.end(), "MemRegion end must be aligned to a card."); #endif // ASSERT // Clear the counts for the (exclusive) card range. @@ -199,14 +126,22 @@ } } +class G1CardCountsClearClosure : public HeapRegionClosure { + private: + G1CardCounts* _card_counts; + public: + G1CardCountsClearClosure(G1CardCounts* card_counts) : + HeapRegionClosure(), _card_counts(card_counts) { } + + + virtual bool doHeapRegion(HeapRegion* r) { + _card_counts->clear_region(r); + return false; + } +}; + void G1CardCounts::clear_all() { assert(SafepointSynchronize::is_at_safepoint(), "don't call this otherwise"); - clear_range((size_t)0, _committed_max_card_num); + G1CardCountsClearClosure cl(this); + _g1h->heap_region_iterate(&cl); } - -G1CardCounts::~G1CardCounts() { - if (has_reserved_count_table()) { - _card_counts_storage.release(); - } -} -
--- a/src/share/vm/gc_implementation/g1/g1CardCounts.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CardCounts.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -25,14 +25,26 @@ #ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1CARDCOUNTS_HPP #define SHARE_VM_GC_IMPLEMENTATION_G1_G1CARDCOUNTS_HPP +#include "gc_implementation/g1/g1RegionToSpaceMapper.hpp" #include "memory/allocation.hpp" #include "runtime/virtualspace.hpp" #include "utilities/globalDefinitions.hpp" class CardTableModRefBS; +class G1CardCounts; class G1CollectedHeap; +class G1RegionToSpaceMapper; class HeapRegion; +class G1CardCountsMappingChangedListener : public G1MappingChangedListener { + private: + G1CardCounts* _counts; + public: + void set_cardcounts(G1CardCounts* counts) { _counts = counts; } + + virtual void on_commit(uint start_idx, size_t num_regions); +}; + // Table to track the number of times a card has been refined. Once // a card has been refined a certain number of times, it is // considered 'hot' and its refinement is delayed by inserting the @@ -41,6 +53,8 @@ // is 'drained' during the next evacuation pause. class G1CardCounts: public CHeapObj<mtGC> { + G1CardCountsMappingChangedListener _listener; + G1CollectedHeap* _g1h; // The table of counts @@ -49,27 +63,18 @@ // Max capacity of the reserved space for the counts table size_t _reserved_max_card_num; - // Max capacity of the committed space for the counts table - size_t _committed_max_card_num; - - // Size of committed space for the counts table - size_t _committed_size; - // CardTable bottom. const jbyte* _ct_bot; // Barrier set CardTableModRefBS* _ct_bs; - // The virtual memory backing the counts table - VirtualSpace _card_counts_storage; - // Returns true if the card counts table has been reserved. bool has_reserved_count_table() { return _card_counts != NULL; } // Returns true if the card counts table has been reserved and committed. bool has_count_table() { - return has_reserved_count_table() && _committed_max_card_num > 0; + return has_reserved_count_table(); } size_t ptr_2_card_num(const jbyte* card_ptr) { @@ -79,37 +84,24 @@ "_ct_bot: " PTR_FORMAT, p2i(card_ptr), p2i(_ct_bot))); size_t card_num = pointer_delta(card_ptr, _ct_bot, sizeof(jbyte)); - assert(card_num >= 0 && card_num < _committed_max_card_num, + assert(card_num >= 0 && card_num < _reserved_max_card_num, err_msg("card pointer out of range: " PTR_FORMAT, p2i(card_ptr))); return card_num; } jbyte* card_num_2_ptr(size_t card_num) { - assert(card_num >= 0 && card_num < _committed_max_card_num, + assert(card_num >= 0 && card_num < _reserved_max_card_num, err_msg("card num out of range: "SIZE_FORMAT, card_num)); return (jbyte*) (_ct_bot + card_num); } - // Helper routine. - // Returns the number of cards that can be counted by the given committed - // table size, with a maximum of the number of cards spanned by the max - // capacity of the heap. - size_t committed_to_card_num(size_t committed_size) { - return MIN2(_reserved_max_card_num, committed_size / sizeof(jbyte)); - } - // Clear the counts table for the given (exclusive) index range. void clear_range(size_t from_card_num, size_t to_card_num); public: G1CardCounts(G1CollectedHeap* g1h); - ~G1CardCounts(); - void initialize(); - - // Resize the committed space for the card counts table in - // response to a resize of the committed space for the heap. - void resize(size_t heap_capacity); + void initialize(G1RegionToSpaceMapper* mapper); // Increments the refinement count for the given card. // Returns the pre-increment count value. @@ -122,8 +114,10 @@ // Clears the card counts for the cards spanned by the region void clear_region(HeapRegion* hr); + // Clears the card counts for the cards spanned by the MemRegion + void clear_range(MemRegion mr); + // Clear the entire card counts table during GC. - // Updates the policy stats with the duration. void clear_all(); };
--- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -43,12 +43,13 @@ #include "gc_implementation/g1/g1MarkSweep.hpp" #include "gc_implementation/g1/g1OopClosures.inline.hpp" #include "gc_implementation/g1/g1ParScanThreadState.inline.hpp" +#include "gc_implementation/g1/g1RegionToSpaceMapper.hpp" #include "gc_implementation/g1/g1RemSet.inline.hpp" #include "gc_implementation/g1/g1StringDedup.hpp" #include "gc_implementation/g1/g1YCTypes.hpp" #include "gc_implementation/g1/heapRegion.inline.hpp" #include "gc_implementation/g1/heapRegionRemSet.hpp" -#include "gc_implementation/g1/heapRegionSeq.inline.hpp" +#include "gc_implementation/g1/heapRegionSet.inline.hpp" #include "gc_implementation/g1/vm_operations_g1.hpp" #include "gc_implementation/shared/gcHeapSummary.hpp" #include "gc_implementation/shared/gcTimer.hpp" @@ -377,6 +378,14 @@ gclog_or_tty->cr(); } +void G1RegionMappingChangedListener::reset_from_card_cache(uint start_idx, size_t num_regions) { + OtherRegionsTable::invalidate(start_idx, num_regions); +} + +void G1RegionMappingChangedListener::on_commit(uint start_idx, size_t num_regions) { + reset_from_card_cache(start_idx, num_regions); +} + void G1CollectedHeap::push_dirty_cards_region(HeapRegion* hr) { // Claim the right to put the region on the dirty cards region list @@ -442,24 +451,18 @@ // implementation of is_scavengable() for G1 will indicate that // all nmethods must be scanned during a partial collection. bool G1CollectedHeap::is_in_partial_collection(const void* p) { - HeapRegion* hr = heap_region_containing(p); - return hr != NULL && hr->in_collection_set(); + if (p == NULL) { + return false; + } + return heap_region_containing(p)->in_collection_set(); } #endif // Returns true if the reference points to an object that // can move in an incremental collection. bool G1CollectedHeap::is_scavengable(const void* p) { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - G1CollectorPolicy* g1p = g1h->g1_policy(); HeapRegion* hr = heap_region_containing(p); - if (hr == NULL) { - // null - assert(p == NULL, err_msg("Not NULL " PTR_FORMAT ,p)); - return false; - } else { - return !hr->isHumongous(); - } + return !hr->isHumongous(); } void G1CollectedHeap::check_ct_logs_at_safepoint() { @@ -525,9 +528,9 @@ // again to allocate from it. append_secondary_free_list(); - assert(!_free_list.is_empty(), "if the secondary_free_list was not " + assert(_hrs.num_free_regions() > 0, "if the secondary_free_list was not " "empty we should have moved at least one entry to the free_list"); - HeapRegion* res = _free_list.remove_region(is_old); + HeapRegion* res = _hrs.allocate_free_region(is_old); if (G1ConcRegionFreeingVerbose) { gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " "allocated "HR_FORMAT" from secondary_free_list", @@ -568,7 +571,7 @@ } } - res = _free_list.remove_region(is_old); + res = _hrs.allocate_free_region(is_old); if (res == NULL) { if (G1ConcRegionFreeingVerbose) { @@ -593,8 +596,8 @@ // Given that expand() succeeded in expanding the heap, and we // always expand the heap by an amount aligned to the heap // region size, the free list should in theory not be empty. - // In either case remove_region() will check for NULL. - res = _free_list.remove_region(is_old); + // In either case allocate_free_region() will check for NULL. + res = _hrs.allocate_free_region(is_old); } else { _expand_heap_after_alloc_failure = false; } @@ -602,55 +605,11 @@ return res; } -uint G1CollectedHeap::humongous_obj_allocate_find_first(uint num_regions, - size_t word_size) { - assert(isHumongous(word_size), "word_size should be humongous"); - assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition"); - - uint first = G1_NULL_HRS_INDEX; - if (num_regions == 1) { - // Only one region to allocate, no need to go through the slower - // path. The caller will attempt the expansion if this fails, so - // let's not try to expand here too. - HeapRegion* hr = new_region(word_size, true /* is_old */, false /* do_expand */); - if (hr != NULL) { - first = hr->hrs_index(); - } else { - first = G1_NULL_HRS_INDEX; - } - } else { - // We can't allocate humongous regions while cleanupComplete() is - // running, since some of the regions we find to be empty might not - // yet be added to the free list and it is not straightforward to - // know which list they are on so that we can remove them. Note - // that we only need to do this if we need to allocate more than - // one region to satisfy the current humongous allocation - // request. If we are only allocating one region we use the common - // region allocation code (see above). - wait_while_free_regions_coming(); - append_secondary_free_list_if_not_empty_with_lock(); - - if (free_regions() >= num_regions) { - first = _hrs.find_contiguous(num_regions); - if (first != G1_NULL_HRS_INDEX) { - for (uint i = first; i < first + num_regions; ++i) { - HeapRegion* hr = region_at(i); - assert(hr->is_empty(), "sanity"); - assert(is_on_master_free_list(hr), "sanity"); - hr->set_pending_removal(true); - } - _free_list.remove_all_pending(num_regions); - } - } - } - return first; -} - HeapWord* G1CollectedHeap::humongous_obj_allocate_initialize_regions(uint first, uint num_regions, size_t word_size) { - assert(first != G1_NULL_HRS_INDEX, "pre-condition"); + assert(first != G1_NO_HRS_INDEX, "pre-condition"); assert(isHumongous(word_size), "word_size should be humongous"); assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition"); @@ -788,42 +747,70 @@ verify_region_sets_optional(); - size_t word_size_rounded = round_to(word_size, HeapRegion::GrainWords); - uint num_regions = (uint) (word_size_rounded / HeapRegion::GrainWords); - uint x_num = expansion_regions(); - uint fs = _hrs.free_suffix(); - uint first = humongous_obj_allocate_find_first(num_regions, word_size); - if (first == G1_NULL_HRS_INDEX) { - // The only thing we can do now is attempt expansion. - if (fs + x_num >= num_regions) { - // If the number of regions we're trying to allocate for this - // object is at most the number of regions in the free suffix, - // then the call to humongous_obj_allocate_find_first() above - // should have succeeded and we wouldn't be here. - // - // We should only be trying to expand when the free suffix is - // not sufficient for the object _and_ we have some expansion - // room available. - assert(num_regions > fs, "earlier allocation should have succeeded"); - + uint first = G1_NO_HRS_INDEX; + uint obj_regions = (uint)(align_size_up_(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords); + + if (obj_regions == 1) { + // Only one region to allocate, try to use a fast path by directly allocating + // from the free lists. Do not try to expand here, we will potentially do that + // later. + HeapRegion* hr = new_region(word_size, true /* is_old */, false /* do_expand */); + if (hr != NULL) { + first = hr->hrs_index(); + } + } else { + // We can't allocate humongous regions spanning more than one region while + // cleanupComplete() is running, since some of the regions we find to be + // empty might not yet be added to the free list. It is not straightforward + // to know in which list they are on so that we can remove them. We only + // need to do this if we need to allocate more than one region to satisfy the + // current humongous allocation request. If we are only allocating one region + // we use the one-region region allocation code (see above), that already + // potentially waits for regions from the secondary free list. + wait_while_free_regions_coming(); + append_secondary_free_list_if_not_empty_with_lock(); + + // Policy: Try only empty regions (i.e. already committed first). Maybe we + // are lucky enough to find some. + first = _hrs.find_contiguous_only_empty(obj_regions); + if (first != G1_NO_HRS_INDEX) { + _hrs.allocate_free_regions_starting_at(first, obj_regions); + } + } + + if (first == G1_NO_HRS_INDEX) { + // Policy: We could not find enough regions for the humongous object in the + // free list. Look through the heap to find a mix of free and uncommitted regions. + // If so, try expansion. + first = _hrs.find_contiguous_empty_or_unavailable(obj_regions); + if (first != G1_NO_HRS_INDEX) { + // We found something. Make sure these regions are committed, i.e. expand + // the heap. Alternatively we could do a defragmentation GC. ergo_verbose1(ErgoHeapSizing, "attempt heap expansion", ergo_format_reason("humongous allocation request failed") ergo_format_byte("allocation request"), word_size * HeapWordSize); - if (expand((num_regions - fs) * HeapRegion::GrainBytes)) { - // Even though the heap was expanded, it might not have - // reached the desired size. So, we cannot assume that the - // allocation will succeed. - first = humongous_obj_allocate_find_first(num_regions, word_size); + + _hrs.expand_at(first, obj_regions); + g1_policy()->record_new_heap_size(num_regions()); + +#ifdef ASSERT + for (uint i = first; i < first + obj_regions; ++i) { + HeapRegion* hr = region_at(i); + assert(hr->is_empty(), "sanity"); + assert(is_on_master_free_list(hr), "sanity"); } +#endif + _hrs.allocate_free_regions_starting_at(first, obj_regions); + } else { + // Policy: Potentially trigger a defragmentation GC. } } HeapWord* result = NULL; - if (first != G1_NULL_HRS_INDEX) { - result = - humongous_obj_allocate_initialize_regions(first, num_regions, word_size); + if (first != G1_NO_HRS_INDEX) { + result = humongous_obj_allocate_initialize_regions(first, obj_regions, word_size); assert(result != NULL, "it should always return a valid result"); // A successful humongous object allocation changes the used space @@ -1386,7 +1373,7 @@ G1MarkSweep::invoke_at_safepoint(ref_processor_stw(), do_clear_all_soft_refs); } - assert(free_regions() == 0, "we should not have added any free regions"); + assert(num_free_regions() == 0, "we should not have added any free regions"); rebuild_region_sets(false /* free_list_only */); // Enqueue any discovered reference objects that have @@ -1751,21 +1738,6 @@ return NULL; } -void G1CollectedHeap::update_committed_space(HeapWord* old_end, - HeapWord* new_end) { - assert(old_end != new_end, "don't call this otherwise"); - assert((HeapWord*) _g1_storage.high() == new_end, "invariant"); - - // Update the committed mem region. - _g1_committed.set_end(new_end); - // Tell the card table about the update. - Universe::heap()->barrier_set()->resize_covered_region(_g1_committed); - // Tell the BOT about the update. - _bot_shared->resize(_g1_committed.word_size()); - // Tell the hot card cache about the update - _cg1r->hot_card_cache()->resize_card_counts(capacity()); -} - bool G1CollectedHeap::expand(size_t expand_bytes) { size_t aligned_expand_bytes = ReservedSpace::page_align_size_up(expand_bytes); aligned_expand_bytes = align_size_up(aligned_expand_bytes, @@ -1776,55 +1748,22 @@ ergo_format_byte("attempted expansion amount"), expand_bytes, aligned_expand_bytes); - if (_g1_storage.uncommitted_size() == 0) { + if (is_maximal_no_gc()) { ergo_verbose0(ErgoHeapSizing, "did not expand the heap", ergo_format_reason("heap already fully expanded")); return false; } - // First commit the memory. - HeapWord* old_end = (HeapWord*) _g1_storage.high(); - bool successful = _g1_storage.expand_by(aligned_expand_bytes); - if (successful) { - // Then propagate this update to the necessary data structures. - HeapWord* new_end = (HeapWord*) _g1_storage.high(); - update_committed_space(old_end, new_end); - - FreeRegionList expansion_list("Local Expansion List"); - MemRegion mr = _hrs.expand_by(old_end, new_end, &expansion_list); - assert(mr.start() == old_end, "post-condition"); - // mr might be a smaller region than what was requested if - // expand_by() was unable to allocate the HeapRegion instances - assert(mr.end() <= new_end, "post-condition"); - - size_t actual_expand_bytes = mr.byte_size(); + uint regions_to_expand = (uint)(aligned_expand_bytes / HeapRegion::GrainBytes); + assert(regions_to_expand > 0, "Must expand by at least one region"); + + uint expanded_by = _hrs.expand_by(regions_to_expand); + + if (expanded_by > 0) { + size_t actual_expand_bytes = expanded_by * HeapRegion::GrainBytes; assert(actual_expand_bytes <= aligned_expand_bytes, "post-condition"); - assert(actual_expand_bytes == expansion_list.total_capacity_bytes(), - "post-condition"); - if (actual_expand_bytes < aligned_expand_bytes) { - // We could not expand _hrs to the desired size. In this case we - // need to shrink the committed space accordingly. - assert(mr.end() < new_end, "invariant"); - - size_t diff_bytes = aligned_expand_bytes - actual_expand_bytes; - // First uncommit the memory. - _g1_storage.shrink_by(diff_bytes); - // Then propagate this update to the necessary data structures. - update_committed_space(new_end, mr.end()); - } - _free_list.add_as_tail(&expansion_list); - - if (_hr_printer.is_active()) { - HeapWord* curr = mr.start(); - while (curr < mr.end()) { - HeapWord* curr_end = curr + HeapRegion::GrainWords; - _hr_printer.commit(curr, curr_end); - curr = curr_end; - } - assert(curr == mr.end(), "post-condition"); - } - g1_policy()->record_new_heap_size(n_regions()); + g1_policy()->record_new_heap_size(num_regions()); } else { ergo_verbose0(ErgoHeapSizing, "did not expand the heap", @@ -1832,12 +1771,12 @@ // The expansion of the virtual storage space was unsuccessful. // Let's see if it was because we ran out of swap. if (G1ExitOnExpansionFailure && - _g1_storage.uncommitted_size() >= aligned_expand_bytes) { + _hrs.available() >= regions_to_expand) { // We had head room... vm_exit_out_of_memory(aligned_expand_bytes, OOM_MMAP_ERROR, "G1 heap expansion"); } } - return successful; + return regions_to_expand > 0; } void G1CollectedHeap::shrink_helper(size_t shrink_bytes) { @@ -1848,7 +1787,6 @@ uint num_regions_to_remove = (uint)(shrink_bytes / HeapRegion::GrainBytes); uint num_regions_removed = _hrs.shrink_by(num_regions_to_remove); - HeapWord* old_end = (HeapWord*) _g1_storage.high(); size_t shrunk_bytes = num_regions_removed * HeapRegion::GrainBytes; ergo_verbose3(ErgoHeapSizing, @@ -1858,22 +1796,7 @@ ergo_format_byte("attempted shrinking amount"), shrink_bytes, aligned_shrink_bytes, shrunk_bytes); if (num_regions_removed > 0) { - _g1_storage.shrink_by(shrunk_bytes); - HeapWord* new_end = (HeapWord*) _g1_storage.high(); - - if (_hr_printer.is_active()) { - HeapWord* curr = old_end; - while (curr > new_end) { - HeapWord* curr_end = curr; - curr -= HeapRegion::GrainWords; - _hr_printer.uncommit(curr, curr_end); - } - } - - _expansion_regions += num_regions_removed; - update_committed_space(old_end, new_end); - HeapRegionRemSet::shrink_heap(n_regions()); - g1_policy()->record_new_heap_size(n_regions()); + g1_policy()->record_new_heap_size(num_regions()); } else { ergo_verbose0(ErgoHeapSizing, "did not shrink the heap", @@ -1924,7 +1847,6 @@ _g1mm(NULL), _refine_cte_cl(NULL), _full_collection(false), - _free_list("Master Free List", new MasterFreeRegionListMtSafeChecker()), _secondary_free_list("Secondary Free List", new SecondaryFreeRegionListMtSafeChecker()), _old_set("Old Set", false /* humongous */, new OldRegionSetMtSafeChecker()), _humongous_set("Master Humongous Set", true /* humongous */, new HumongousRegionSetMtSafeChecker()), @@ -2038,8 +1960,6 @@ _reserved.set_start((HeapWord*)heap_rs.base()); _reserved.set_end((HeapWord*)(heap_rs.base() + heap_rs.size())); - _expansion_regions = (uint) (max_byte_size / HeapRegion::GrainBytes); - // Create the gen rem set (and barrier set) for the entire reserved region. _rem_set = collector_policy()->create_rem_set(_reserved, 2); set_barrier_set(rem_set()->bs()); @@ -2053,20 +1973,65 @@ // Carve out the G1 part of the heap. - ReservedSpace g1_rs = heap_rs.first_part(max_byte_size); - _g1_reserved = MemRegion((HeapWord*)g1_rs.base(), - g1_rs.size()/HeapWordSize); - - _g1_storage.initialize(g1_rs, 0); - _g1_committed = MemRegion((HeapWord*)_g1_storage.low(), (size_t) 0); - _hrs.initialize((HeapWord*) _g1_reserved.start(), - (HeapWord*) _g1_reserved.end()); - assert(_hrs.max_length() == _expansion_regions, - err_msg("max length: %u expansion regions: %u", - _hrs.max_length(), _expansion_regions)); - - // Do later initialization work for concurrent refinement. - _cg1r->init(); + ReservedSpace g1_rs = heap_rs.first_part(max_byte_size); + G1RegionToSpaceMapper* heap_storage = + G1RegionToSpaceMapper::create_mapper(g1_rs, + UseLargePages ? os::large_page_size() : os::vm_page_size(), + HeapRegion::GrainBytes, + 1, + mtJavaHeap); + heap_storage->set_mapping_changed_listener(&_listener); + + // Reserve space for the block offset table. We do not support automatic uncommit + // for the card table at this time. BOT only. + ReservedSpace bot_rs(G1BlockOffsetSharedArray::compute_size(g1_rs.size() / HeapWordSize)); + G1RegionToSpaceMapper* bot_storage = + G1RegionToSpaceMapper::create_mapper(bot_rs, + os::vm_page_size(), + HeapRegion::GrainBytes, + G1BlockOffsetSharedArray::N_bytes, + mtGC); + + ReservedSpace cardtable_rs(G1SATBCardTableLoggingModRefBS::compute_size(g1_rs.size() / HeapWordSize)); + G1RegionToSpaceMapper* cardtable_storage = + G1RegionToSpaceMapper::create_mapper(cardtable_rs, + os::vm_page_size(), + HeapRegion::GrainBytes, + G1BlockOffsetSharedArray::N_bytes, + mtGC); + + // Reserve space for the card counts table. + ReservedSpace card_counts_rs(G1BlockOffsetSharedArray::compute_size(g1_rs.size() / HeapWordSize)); + G1RegionToSpaceMapper* card_counts_storage = + G1RegionToSpaceMapper::create_mapper(card_counts_rs, + os::vm_page_size(), + HeapRegion::GrainBytes, + G1BlockOffsetSharedArray::N_bytes, + mtGC); + + // Reserve space for prev and next bitmap. + size_t bitmap_size = CMBitMap::compute_size(g1_rs.size()); + + ReservedSpace prev_bitmap_rs(ReservedSpace::allocation_align_size_up(bitmap_size)); + G1RegionToSpaceMapper* prev_bitmap_storage = + G1RegionToSpaceMapper::create_mapper(prev_bitmap_rs, + os::vm_page_size(), + HeapRegion::GrainBytes, + CMBitMap::mark_distance(), + mtGC); + + ReservedSpace next_bitmap_rs(ReservedSpace::allocation_align_size_up(bitmap_size)); + G1RegionToSpaceMapper* next_bitmap_storage = + G1RegionToSpaceMapper::create_mapper(next_bitmap_rs, + os::vm_page_size(), + HeapRegion::GrainBytes, + CMBitMap::mark_distance(), + mtGC); + + _hrs.initialize(heap_storage, prev_bitmap_storage, next_bitmap_storage, bot_storage, cardtable_storage, card_counts_storage); + g1_barrier_set()->initialize(cardtable_storage); + // Do later initialization work for concurrent refinement. + _cg1r->init(card_counts_storage); // 6843694 - ensure that the maximum region index can fit // in the remembered set structures. @@ -2080,17 +2045,16 @@ FreeRegionList::set_unrealistically_long_length(max_regions() + 1); - _bot_shared = new G1BlockOffsetSharedArray(_reserved, - heap_word_size(init_byte_size)); + _bot_shared = new G1BlockOffsetSharedArray(_reserved, bot_storage); _g1h = this; - _in_cset_fast_test.initialize(_g1_reserved.start(), _g1_reserved.end(), HeapRegion::GrainBytes); - _humongous_is_live.initialize(_g1_reserved.start(), _g1_reserved.end(), HeapRegion::GrainBytes); + _in_cset_fast_test.initialize(_hrs.reserved().start(), _hrs.reserved().end(), HeapRegion::GrainBytes); + _humongous_is_live.initialize(_hrs.reserved().start(), _hrs.reserved().end(), HeapRegion::GrainBytes); // Create the ConcurrentMark data structure and thread. // (Must do this late, so that "max_regions" is defined.) - _cm = new ConcurrentMark(this, heap_rs); + _cm = new ConcurrentMark(this, prev_bitmap_storage, next_bitmap_storage); if (_cm == NULL || !_cm->completed_initialization()) { vm_shutdown_during_initialization("Could not create/initialize ConcurrentMark"); return JNI_ENOMEM; @@ -2145,12 +2109,10 @@ // counts and that mechanism. SpecializationStats::clear(); - // Here we allocate the dummy full region that is required by the - // G1AllocRegion class. If we don't pass an address in the reserved - // space here, lots of asserts fire. - - HeapRegion* dummy_region = new_heap_region(0 /* index of bottom region */, - _g1_reserved.start()); + // Here we allocate the dummy HeapRegion that is required by the + // G1AllocRegion class. + HeapRegion* dummy_region = _hrs.get_dummy_region(); + // We'll re-use the same region whether the alloc region will // require BOT updates or not and, if it doesn't, then a non-young // region will complain that it cannot support allocations without @@ -2266,7 +2228,7 @@ } size_t G1CollectedHeap::capacity() const { - return _g1_committed.byte_size(); + return _hrs.length() * HeapRegion::GrainBytes; } void G1CollectedHeap::reset_gc_time_stamps(HeapRegion* hr) { @@ -2375,7 +2337,7 @@ } size_t G1CollectedHeap::unsafe_max_alloc() { - if (free_regions() > 0) return HeapRegion::GrainBytes; + if (num_free_regions() > 0) return HeapRegion::GrainBytes; // otherwise, is there space in the current allocation region? // We need to store the current allocation region in a local variable @@ -2590,8 +2552,8 @@ } bool G1CollectedHeap::is_in(const void* p) const { - if (_g1_committed.contains(p)) { - // Given that we know that p is in the committed space, + if (_hrs.reserved().contains(p)) { + // Given that we know that p is in the reserved space, // heap_region_containing_raw() should successfully // return the containing region. HeapRegion* hr = heap_region_containing_raw(p); @@ -2601,6 +2563,18 @@ } } +#ifdef ASSERT +bool G1CollectedHeap::is_in_exact(const void* p) const { + bool contains = reserved_region().contains(p); + bool available = _hrs.is_available(addr_to_region((HeapWord*)p)); + if (contains && available) { + return true; + } else { + return false; + } +} +#endif + // Iteration functions. // Applies an ExtendedOopClosure onto all references of objects within a HeapRegion. @@ -2665,83 +2639,9 @@ void G1CollectedHeap::heap_region_par_iterate_chunked(HeapRegionClosure* cl, uint worker_id, - uint no_of_par_workers, - jint claim_value) { - const uint regions = n_regions(); - const uint max_workers = (G1CollectedHeap::use_parallel_gc_threads() ? - no_of_par_workers : - 1); - assert(UseDynamicNumberOfGCThreads || - no_of_par_workers == workers()->total_workers(), - "Non dynamic should use fixed number of workers"); - // try to spread out the starting points of the workers - const HeapRegion* start_hr = - start_region_for_worker(worker_id, no_of_par_workers); - const uint start_index = start_hr->hrs_index(); - - // each worker will actually look at all regions - for (uint count = 0; count < regions; ++count) { - const uint index = (start_index + count) % regions; - assert(0 <= index && index < regions, "sanity"); - HeapRegion* r = region_at(index); - // we'll ignore "continues humongous" regions (we'll process them - // when we come across their corresponding "start humongous" - // region) and regions already claimed - if (r->claim_value() == claim_value || r->continuesHumongous()) { - continue; - } - // OK, try to claim it - if (r->claimHeapRegion(claim_value)) { - // success! - assert(!r->continuesHumongous(), "sanity"); - if (r->startsHumongous()) { - // If the region is "starts humongous" we'll iterate over its - // "continues humongous" first; in fact we'll do them - // first. The order is important. In on case, calling the - // closure on the "starts humongous" region might de-allocate - // and clear all its "continues humongous" regions and, as a - // result, we might end up processing them twice. So, we'll do - // them first (notice: most closures will ignore them anyway) and - // then we'll do the "starts humongous" region. - for (uint ch_index = index + 1; ch_index < regions; ++ch_index) { - HeapRegion* chr = region_at(ch_index); - - // if the region has already been claimed or it's not - // "continues humongous" we're done - if (chr->claim_value() == claim_value || - !chr->continuesHumongous()) { - break; - } - - // No one should have claimed it directly. We can given - // that we claimed its "starts humongous" region. - assert(chr->claim_value() != claim_value, "sanity"); - assert(chr->humongous_start_region() == r, "sanity"); - - if (chr->claimHeapRegion(claim_value)) { - // we should always be able to claim it; no one else should - // be trying to claim this region - - bool res2 = cl->doHeapRegion(chr); - assert(!res2, "Should not abort"); - - // Right now, this holds (i.e., no closure that actually - // does something with "continues humongous" regions - // clears them). We might have to weaken it in the future, - // but let's leave these two asserts here for extra safety. - assert(chr->continuesHumongous(), "should still be the case"); - assert(chr->humongous_start_region() == r, "sanity"); - } else { - guarantee(false, "we should not reach here"); - } - } - } - - assert(!r->continuesHumongous(), "sanity"); - bool res = cl->doHeapRegion(r); - assert(!res, "Should not abort"); - } - } + uint num_workers, + jint claim_value) const { + _hrs.par_iterate(cl, worker_id, num_workers, claim_value); } class ResetClaimValuesClosure: public HeapRegionClosure { @@ -2919,17 +2819,6 @@ return result; } -HeapRegion* G1CollectedHeap::start_region_for_worker(uint worker_i, - uint no_of_par_workers) { - uint worker_num = - G1CollectedHeap::use_parallel_gc_threads() ? no_of_par_workers : 1U; - assert(UseDynamicNumberOfGCThreads || - no_of_par_workers == workers()->total_workers(), - "Non dynamic should use fixed number of workers"); - const uint start_index = n_regions() * worker_i / worker_num; - return region_at(start_index); -} - void G1CollectedHeap::collection_set_iterate(HeapRegionClosure* cl) { HeapRegion* r = g1_policy()->collection_set(); while (r != NULL) { @@ -2972,33 +2861,24 @@ } HeapRegion* G1CollectedHeap::next_compaction_region(const HeapRegion* from) const { - // We're not using an iterator given that it will wrap around when - // it reaches the last region and this is not what we want here. - for (uint index = from->hrs_index() + 1; index < n_regions(); index++) { - HeapRegion* hr = region_at(index); - if (!hr->isHumongous()) { - return hr; - } - } - return NULL; + HeapRegion* result = _hrs.next_region_in_heap(from); + while (result != NULL && result->isHumongous()) { + result = _hrs.next_region_in_heap(result); + } + return result; } Space* G1CollectedHeap::space_containing(const void* addr) const { - Space* res = heap_region_containing(addr); - return res; + return heap_region_containing(addr); } HeapWord* G1CollectedHeap::block_start(const void* addr) const { Space* sp = space_containing(addr); - if (sp != NULL) { - return sp->block_start(addr); - } - return NULL; + return sp->block_start(addr); } size_t G1CollectedHeap::block_size(const HeapWord* addr) const { Space* sp = space_containing(addr); - assert(sp != NULL, "block_size of address outside of heap"); return sp->block_size(addr); } @@ -3043,7 +2923,7 @@ } size_t G1CollectedHeap::max_capacity() const { - return _g1_reserved.byte_size(); + return _hrs.reserved().byte_size(); } jlong G1CollectedHeap::millis_since_last_gc() { @@ -3572,9 +3452,9 @@ st->print(" total " SIZE_FORMAT "K, used " SIZE_FORMAT "K", capacity()/K, used_unlocked()/K); st->print(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " INTPTR_FORMAT ")", - _g1_storage.low_boundary(), - _g1_storage.high(), - _g1_storage.high_boundary()); + _hrs.reserved().start(), + _hrs.reserved().start() + _hrs.length() + HeapRegion::GrainWords, + _hrs.reserved().end()); st->cr(); st->print(" region size " SIZE_FORMAT "K, ", HeapRegion::GrainBytes / K); uint young_regions = _young_list->length(); @@ -4264,10 +4144,7 @@ // No need for an ergo verbose message here, // expansion_amount() does this when it returns a value > 0. if (!expand(expand_bytes)) { - // We failed to expand the heap so let's verify that - // committed/uncommitted amount match the backing store - assert(capacity() == _g1_storage.committed_size(), "committed size mismatch"); - assert(max_capacity() == _g1_storage.reserved_size(), "reserved size mismatch"); + // We failed to expand the heap. Cannot do anything about it. } } } @@ -4327,10 +4204,6 @@ // RETIRE events are generated before the end GC event. _hr_printer.end_gc(false /* full */, (size_t) total_collections()); - if (mark_in_progress()) { - concurrent_mark()->update_g1_committed(); - } - #ifdef TRACESPINNING ParallelTaskTerminator::print_termination_counts(); #endif @@ -4652,30 +4525,19 @@ ParGCAllocBuffer(gclab_word_size), _retired(true) { } void G1ParCopyHelper::mark_object(oop obj) { -#ifdef ASSERT - HeapRegion* hr = _g1->heap_region_containing(obj); - assert(hr != NULL, "sanity"); - assert(!hr->in_collection_set(), "should not mark objects in the CSet"); -#endif // ASSERT + assert(!_g1->heap_region_containing(obj)->in_collection_set(), "should not mark objects in the CSet"); // We know that the object is not moving so it's safe to read its size. _cm->grayRoot(obj, (size_t) obj->size(), _worker_id); } void G1ParCopyHelper::mark_forwarded_object(oop from_obj, oop to_obj) { -#ifdef ASSERT assert(from_obj->is_forwarded(), "from obj should be forwarded"); assert(from_obj->forwardee() == to_obj, "to obj should be the forwardee"); assert(from_obj != to_obj, "should not be self-forwarded"); - HeapRegion* from_hr = _g1->heap_region_containing(from_obj); - assert(from_hr != NULL, "sanity"); - assert(from_hr->in_collection_set(), "from obj should be in the CSet"); - - HeapRegion* to_hr = _g1->heap_region_containing(to_obj); - assert(to_hr != NULL, "sanity"); - assert(!to_hr->in_collection_set(), "should not mark objects in the CSet"); -#endif // ASSERT + assert(_g1->heap_region_containing(from_obj)->in_collection_set(), "from obj should be in the CSet"); + assert(!_g1->heap_region_containing(to_obj)->in_collection_set(), "should not mark objects in the CSet"); // The object might be in the process of being copied by another // worker so we cannot trust that its to-space image is @@ -6176,6 +6038,7 @@ bool locked) { assert(!hr->isHumongous(), "this is only for non-humongous regions"); assert(!hr->is_empty(), "the region should not be empty"); + assert(_hrs.is_available(hr->hrs_index()), "region should be committed"); assert(free_list != NULL, "pre-condition"); if (G1VerifyBitmaps) { @@ -6230,7 +6093,7 @@ assert(list != NULL, "list can't be null"); if (!list->is_empty()) { MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag); - _free_list.add_ordered(list); + _hrs.insert_list_into_free_list(list); } } @@ -6838,22 +6701,22 @@ // this is that during a full GC string deduplication needs to know if // a collected region was young or old when the full GC was initiated. } - _free_list.remove_all(); + _hrs.remove_all_free_regions(); } class RebuildRegionSetsClosure : public HeapRegionClosure { private: bool _free_list_only; HeapRegionSet* _old_set; - FreeRegionList* _free_list; + HeapRegionSeq* _hrs; size_t _total_used; public: RebuildRegionSetsClosure(bool free_list_only, - HeapRegionSet* old_set, FreeRegionList* free_list) : + HeapRegionSet* old_set, HeapRegionSeq* hrs) : _free_list_only(free_list_only), - _old_set(old_set), _free_list(free_list), _total_used(0) { - assert(_free_list->is_empty(), "pre-condition"); + _old_set(old_set), _hrs(hrs), _total_used(0) { + assert(_hrs->num_free_regions() == 0, "pre-condition"); if (!free_list_only) { assert(_old_set->is_empty(), "pre-condition"); } @@ -6866,7 +6729,7 @@ if (r->is_empty()) { // Add free regions to the free list - _free_list->add_as_tail(r); + _hrs->insert_into_free_list(r); } else if (!_free_list_only) { assert(!r->is_young(), "we should not come across young regions"); @@ -6894,7 +6757,7 @@ _young_list->empty_list(); } - RebuildRegionSetsClosure cl(free_list_only, &_old_set, &_free_list); + RebuildRegionSetsClosure cl(free_list_only, &_old_set, &_hrs); heap_region_iterate(&cl); if (!free_list_only) { @@ -6912,11 +6775,7 @@ bool G1CollectedHeap::is_in_closed_subset(const void* p) const { HeapRegion* hr = heap_region_containing(p); - if (hr == NULL) { - return false; - } else { - return hr->is_in(p); - } + return hr->is_in(p); } // Methods for the mutator alloc region @@ -7053,13 +6912,42 @@ _g1h->retire_gc_alloc_region(alloc_region, allocated_bytes, GCAllocForTenured); } + +HeapRegion* OldGCAllocRegion::release() { + HeapRegion* cur = get(); + if (cur != NULL) { + // Determine how far we are from the next card boundary. If it is smaller than + // the minimum object size we can allocate into, expand into the next card. + HeapWord* top = cur->top(); + HeapWord* aligned_top = (HeapWord*)align_ptr_up(top, G1BlockOffsetSharedArray::N_bytes); + + size_t to_allocate_words = pointer_delta(aligned_top, top, HeapWordSize); + + if (to_allocate_words != 0) { + // We are not at a card boundary. Fill up, possibly into the next, taking the + // end of the region and the minimum object size into account. + to_allocate_words = MIN2(pointer_delta(cur->end(), cur->top(), HeapWordSize), + MAX2(to_allocate_words, G1CollectedHeap::min_fill_size())); + + // Skip allocation if there is not enough space to allocate even the smallest + // possible object. In this case this region will not be retained, so the + // original problem cannot occur. + if (to_allocate_words >= G1CollectedHeap::min_fill_size()) { + HeapWord* dummy = attempt_allocation(to_allocate_words, true /* bot_updates */); + CollectedHeap::fill_with_object(dummy, to_allocate_words); + } + } + } + return G1AllocRegion::release(); +} + // Heap region set verification class VerifyRegionListsClosure : public HeapRegionClosure { private: HeapRegionSet* _old_set; HeapRegionSet* _humongous_set; - FreeRegionList* _free_list; + HeapRegionSeq* _hrs; public: HeapRegionSetCount _old_count; @@ -7068,8 +6956,8 @@ VerifyRegionListsClosure(HeapRegionSet* old_set, HeapRegionSet* humongous_set, - FreeRegionList* free_list) : - _old_set(old_set), _humongous_set(humongous_set), _free_list(free_list), + HeapRegionSeq* hrs) : + _old_set(old_set), _humongous_set(humongous_set), _hrs(hrs), _old_count(), _humongous_count(), _free_count(){ } bool doHeapRegion(HeapRegion* hr) { @@ -7080,19 +6968,19 @@ if (hr->is_young()) { // TODO } else if (hr->startsHumongous()) { - assert(hr->containing_set() == _humongous_set, err_msg("Heap region %u is starts humongous but not in humongous set.", hr->region_num())); + assert(hr->containing_set() == _humongous_set, err_msg("Heap region %u is starts humongous but not in humongous set.", hr->hrs_index())); _humongous_count.increment(1u, hr->capacity()); } else if (hr->is_empty()) { - assert(hr->containing_set() == _free_list, err_msg("Heap region %u is empty but not on the free list.", hr->region_num())); + assert(_hrs->is_free(hr), err_msg("Heap region %u is empty but not on the free list.", hr->hrs_index())); _free_count.increment(1u, hr->capacity()); } else { - assert(hr->containing_set() == _old_set, err_msg("Heap region %u is old but not in the old set.", hr->region_num())); + assert(hr->containing_set() == _old_set, err_msg("Heap region %u is old but not in the old set.", hr->hrs_index())); _old_count.increment(1u, hr->capacity()); } return false; } - void verify_counts(HeapRegionSet* old_set, HeapRegionSet* humongous_set, FreeRegionList* free_list) { + void verify_counts(HeapRegionSet* old_set, HeapRegionSet* humongous_set, HeapRegionSeq* free_list) { guarantee(old_set->length() == _old_count.length(), err_msg("Old set count mismatch. Expected %u, actual %u.", old_set->length(), _old_count.length())); guarantee(old_set->total_capacity_bytes() == _old_count.capacity(), err_msg("Old set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, old_set->total_capacity_bytes(), _old_count.capacity())); @@ -7101,26 +6989,17 @@ guarantee(humongous_set->total_capacity_bytes() == _humongous_count.capacity(), err_msg("Hum set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, humongous_set->total_capacity_bytes(), _humongous_count.capacity())); - guarantee(free_list->length() == _free_count.length(), err_msg("Free list count mismatch. Expected %u, actual %u.", free_list->length(), _free_count.length())); + guarantee(free_list->num_free_regions() == _free_count.length(), err_msg("Free list count mismatch. Expected %u, actual %u.", free_list->num_free_regions(), _free_count.length())); guarantee(free_list->total_capacity_bytes() == _free_count.capacity(), err_msg("Free list capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, free_list->total_capacity_bytes(), _free_count.capacity())); } }; -HeapRegion* G1CollectedHeap::new_heap_region(uint hrs_index, - HeapWord* bottom) { - HeapWord* end = bottom + HeapRegion::GrainWords; - MemRegion mr(bottom, end); - assert(_g1_reserved.contains(mr), "invariant"); - // This might return NULL if the allocation fails - return new HeapRegion(hrs_index, _bot_shared, mr); -} - void G1CollectedHeap::verify_region_sets() { assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); // First, check the explicit lists. - _free_list.verify_list(); + _hrs.verify(); { // Given that a concurrent operation might be adding regions to // the secondary free list we have to take the lock before @@ -7151,9 +7030,9 @@ // Finally, make sure that the region accounting in the lists is // consistent with what we see in the heap. - VerifyRegionListsClosure cl(&_old_set, &_humongous_set, &_free_list); + VerifyRegionListsClosure cl(&_old_set, &_humongous_set, &_hrs); heap_region_iterate(&cl); - cl.verify_counts(&_old_set, &_humongous_set, &_free_list); + cl.verify_counts(&_old_set, &_humongous_set, &_hrs); } // Optimized nmethod scanning
--- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -183,6 +183,13 @@ public: OldGCAllocRegion() : G1AllocRegion("Old GC Alloc Region", true /* bot_updates */) { } + + // This specialization of release() makes sure that the last card that has been + // allocated into has been completely filled by a dummy object. + // This avoids races when remembered set scanning wants to update the BOT of the + // last card in the retained old gc alloc region, and allocation threads + // allocating into that card at the same time. + virtual HeapRegion* release(); }; // The G1 STW is alive closure. @@ -199,6 +206,13 @@ class RefineCardTableEntryClosure; +class G1RegionMappingChangedListener : public G1MappingChangedListener { + private: + void reset_from_card_cache(uint start_idx, size_t num_regions); + public: + virtual void on_commit(uint start_idx, size_t num_regions); +}; + class G1CollectedHeap : public SharedHeap { friend class VM_CollectForMetadataAllocation; friend class VM_G1CollectForAllocation; @@ -237,19 +251,9 @@ static size_t _humongous_object_threshold_in_words; - // Storage for the G1 heap. - VirtualSpace _g1_storage; - MemRegion _g1_reserved; - - // The part of _g1_storage that is currently committed. - MemRegion _g1_committed; - - // The master free list. It will satisfy all new region allocations. - FreeRegionList _free_list; - // The secondary free list which contains regions that have been - // freed up during the cleanup process. This will be appended to the - // master free list when appropriate. + // freed up during the cleanup process. This will be appended to + // the master free list when appropriate. FreeRegionList _secondary_free_list; // It keeps track of the old regions. @@ -283,6 +287,9 @@ // after heap shrinking (free_list_only == true). void rebuild_region_sets(bool free_list_only); + // Callback for region mapping changed events. + G1RegionMappingChangedListener _listener; + // The sequence of all heap regions in the heap. HeapRegionSeq _hrs; @@ -513,14 +520,6 @@ // humongous object, set is_old to true. If not, to false. HeapRegion* new_region(size_t word_size, bool is_old, bool do_expand); - // Attempt to satisfy a humongous allocation request of the given - // size by finding a contiguous set of free regions of num_regions - // length and remove them from the master free list. Return the - // index of the first region or G1_NULL_HRS_INDEX if the search - // was unsuccessful. - uint humongous_obj_allocate_find_first(uint num_regions, - size_t word_size); - // Initialize a contiguous set of free regions of length num_regions // and starting at index first so that they appear as a single // humongous region. @@ -862,11 +861,6 @@ CodeBlobClosure* scan_strong_code, uint worker_i); - // Notifies all the necessary spaces that the committed space has - // been updated (either expanded or shrunk). It should be called - // after _g1_storage is updated. - void update_committed_space(HeapWord* old_end, HeapWord* new_end); - // The concurrent marker (and the thread it runs in.) ConcurrentMark* _cm; ConcurrentMarkThread* _cmThread; @@ -1186,27 +1180,20 @@ virtual size_t unsafe_max_alloc(); virtual bool is_maximal_no_gc() const { - return _g1_storage.uncommitted_size() == 0; + return _hrs.available() == 0; } - // The total number of regions in the heap. - uint n_regions() const { return _hrs.length(); } + // The current number of regions in the heap. + uint num_regions() const { return _hrs.length(); } // The max number of regions in the heap. uint max_regions() const { return _hrs.max_length(); } // The number of regions that are completely free. - uint free_regions() const { return _free_list.length(); } + uint num_free_regions() const { return _hrs.num_free_regions(); } // The number of regions that are not completely free. - uint used_regions() const { return n_regions() - free_regions(); } - - // The number of regions available for "regular" expansion. - uint expansion_regions() const { return _expansion_regions; } - - // Factory method for HeapRegion instances. It will return NULL if - // the allocation fails. - HeapRegion* new_heap_region(uint hrs_index, HeapWord* bottom); + uint num_used_regions() const { return num_regions() - num_free_regions(); } void verify_not_dirty_region(HeapRegion* hr) PRODUCT_RETURN; void verify_dirty_region(HeapRegion* hr) PRODUCT_RETURN; @@ -1255,7 +1242,7 @@ #ifdef ASSERT bool is_on_master_free_list(HeapRegion* hr) { - return hr->containing_set() == &_free_list; + return _hrs.is_free(hr); } #endif // ASSERT @@ -1267,7 +1254,7 @@ } void append_secondary_free_list() { - _free_list.add_ordered(&_secondary_free_list); + _hrs.insert_list_into_free_list(&_secondary_free_list); } void append_secondary_free_list_if_not_empty_with_lock() { @@ -1313,6 +1300,11 @@ // Returns "TRUE" iff "p" points into the committed areas of the heap. virtual bool is_in(const void* p) const; +#ifdef ASSERT + // Returns whether p is in one of the available areas of the heap. Slow but + // extensive version. + bool is_in_exact(const void* p) const; +#endif // Return "TRUE" iff the given object address is within the collection // set. Slow implementation. @@ -1373,25 +1365,19 @@ // Return "TRUE" iff the given object address is in the reserved // region of g1. bool is_in_g1_reserved(const void* p) const { - return _g1_reserved.contains(p); + return _hrs.reserved().contains(p); } // Returns a MemRegion that corresponds to the space that has been // reserved for the heap - MemRegion g1_reserved() { - return _g1_reserved; - } - - // Returns a MemRegion that corresponds to the space that has been - // committed in the heap - MemRegion g1_committed() { - return _g1_committed; + MemRegion g1_reserved() const { + return _hrs.reserved(); } virtual bool is_in_closed_subset(const void* p) const; - G1SATBCardTableModRefBS* g1_barrier_set() { - return (G1SATBCardTableModRefBS*) barrier_set(); + G1SATBCardTableLoggingModRefBS* g1_barrier_set() { + return (G1SATBCardTableLoggingModRefBS*) barrier_set(); } // This resets the card table to all zeros. It is used after @@ -1425,6 +1411,8 @@ // within the heap. inline uint addr_to_region(HeapWord* addr) const; + inline HeapWord* bottom_addr_for_region(uint index) const; + // Divide the heap region sequence into "chunks" of some size (the number // of regions divided by the number of parallel threads times some // overpartition factor, currently 4). Assumes that this will be called @@ -1438,10 +1426,10 @@ // setting the claim value of the second and subsequent regions of the // chunk.) For now requires that "doHeapRegion" always returns "false", // i.e., that a closure never attempt to abort a traversal. - void heap_region_par_iterate_chunked(HeapRegionClosure* blk, - uint worker, - uint no_of_par_workers, - jint claim_value); + void heap_region_par_iterate_chunked(HeapRegionClosure* cl, + uint worker_id, + uint num_workers, + jint claim_value) const; // It resets all the region claim values to the default. void reset_heap_region_claim_values(); @@ -1466,11 +1454,6 @@ // starting region for iterating over the current collection set. HeapRegion* start_cset_region_for_worker(uint worker_i); - // This is a convenience method that is used by the - // HeapRegionIterator classes to calculate the starting region for - // each worker so that they do not all start from the same region. - HeapRegion* start_region_for_worker(uint worker_i, uint no_of_par_workers); - // Iterate over the regions (if any) in the current collection set. void collection_set_iterate(HeapRegionClosure* blk); @@ -1483,17 +1466,15 @@ // space containing a given address, or else returns NULL. virtual Space* space_containing(const void* addr) const; - // A G1CollectedHeap will contain some number of heap regions. This - // finds the region containing a given address, or else returns NULL. + // Returns the HeapRegion that contains addr. addr must not be NULL. + template <class T> + inline HeapRegion* heap_region_containing_raw(const T addr) const; + + // Returns the HeapRegion that contains addr. addr must not be NULL. + // If addr is within a humongous continues region, it returns its humongous start region. template <class T> inline HeapRegion* heap_region_containing(const T addr) const; - // Like the above, but requires "addr" to be in the heap (to avoid a - // null-check), and unlike the above, may return an continuing humongous - // region. - template <class T> - inline HeapRegion* heap_region_containing_raw(const T addr) const; - // A CollectedHeap is divided into a dense sequence of "blocks"; that is, // each address in the (reserved) heap is a member of exactly // one block. The defining characteristic of a block is that it is @@ -1635,7 +1616,6 @@ // the region to which the object belongs. An object is dead // iff a) it was not allocated since the last mark and b) it // is not marked. - bool is_obj_dead(const oop obj, const HeapRegion* hr) const { return !hr->obj_allocated_since_prev_marking(obj) && @@ -1645,7 +1625,6 @@ // This function returns true when an object has been // around since the previous marking and hasn't yet // been marked during this marking. - bool is_obj_ill(const oop obj, const HeapRegion* hr) const { return !hr->obj_allocated_since_next_marking(obj) &&
--- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -47,23 +47,26 @@ return (uint)(pointer_delta(addr, _reserved.start(), sizeof(uint8_t)) >> HeapRegion::LogOfHRGrainBytes); } -template <class T> -inline HeapRegion* -G1CollectedHeap::heap_region_containing(const T addr) const { - HeapRegion* hr = _hrs.addr_to_region((HeapWord*) addr); - // hr can be null if addr in perm_gen - if (hr != NULL && hr->continuesHumongous()) { - hr = hr->humongous_start_region(); - } - return hr; +inline HeapWord* G1CollectedHeap::bottom_addr_for_region(uint index) const { + return _hrs.reserved().start() + index * HeapRegion::GrainWords; } template <class T> -inline HeapRegion* -G1CollectedHeap::heap_region_containing_raw(const T addr) const { - assert(_g1_reserved.contains((const void*) addr), "invariant"); - HeapRegion* res = _hrs.addr_to_region_unsafe((HeapWord*) addr); - return res; +inline HeapRegion* G1CollectedHeap::heap_region_containing_raw(const T addr) const { + assert(addr != NULL, "invariant"); + assert(is_in_g1_reserved((const void*) addr), + err_msg("Address "PTR_FORMAT" is outside of the heap ranging from ["PTR_FORMAT" to "PTR_FORMAT")", + p2i((void*)addr), p2i(g1_reserved().start()), p2i(g1_reserved().end()))); + return _hrs.addr_to_region((HeapWord*) addr); +} + +template <class T> +inline HeapRegion* G1CollectedHeap::heap_region_containing(const T addr) const { + HeapRegion* hr = heap_region_containing_raw(addr); + if (hr->continuesHumongous()) { + return hr->humongous_start_region(); + } + return hr; } inline void G1CollectedHeap::reset_gc_time_stamp() { @@ -88,10 +91,9 @@ return r != NULL && r->in_collection_set(); } -inline HeapWord* -G1CollectedHeap::attempt_allocation(size_t word_size, - unsigned int* gc_count_before_ret, - int* gclocker_retry_count_ret) { +inline HeapWord* G1CollectedHeap::attempt_allocation(size_t word_size, + unsigned int* gc_count_before_ret, + int* gclocker_retry_count_ret) { assert_heap_not_locked_and_not_at_safepoint(); assert(!isHumongous(word_size), "attempt_allocation() should not " "be called for humongous allocation requests"); @@ -154,8 +156,7 @@ // have to keep calling heap_region_containing_raw() in the // asserts below. DEBUG_ONLY(HeapRegion* containing_hr = heap_region_containing_raw(start);) - assert(containing_hr != NULL && start != NULL && word_size > 0, - "pre-condition"); + assert(word_size > 0, "pre-condition"); assert(containing_hr->is_in(start), "it should contain start"); assert(containing_hr->is_young(), "it should be young"); assert(!containing_hr->isHumongous(), "it should not be humongous"); @@ -252,8 +253,7 @@ } } -inline bool -G1CollectedHeap::evacuation_should_fail() { +inline bool G1CollectedHeap::evacuation_should_fail() { if (!G1EvacuationFailureALot || !_evacuation_failure_alot_for_current_gc) { return false; } @@ -277,8 +277,10 @@ #endif // #ifndef PRODUCT inline bool G1CollectedHeap::is_in_young(const oop obj) { - HeapRegion* hr = heap_region_containing(obj); - return hr != NULL && hr->is_young(); + if (obj == NULL) { + return false; + } + return heap_region_containing(obj)->is_young(); } // We don't need barriers for initializing stores to objects @@ -291,21 +293,17 @@ } inline bool G1CollectedHeap::is_obj_dead(const oop obj) const { - const HeapRegion* hr = heap_region_containing(obj); - if (hr == NULL) { - if (obj == NULL) return false; - else return true; + if (obj == NULL) { + return false; } - else return is_obj_dead(obj, hr); + return is_obj_dead(obj, heap_region_containing(obj)); } inline bool G1CollectedHeap::is_obj_ill(const oop obj) const { - const HeapRegion* hr = heap_region_containing(obj); - if (hr == NULL) { - if (obj == NULL) return false; - else return true; + if (obj == NULL) { + return false; } - else return is_obj_ill(obj, hr); + return is_obj_ill(obj, heap_region_containing(obj)); } inline void G1CollectedHeap::set_humongous_is_live(oop obj) {
--- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -455,7 +455,7 @@ } else { _young_list_fixed_length = _young_gen_sizer->min_desired_young_length(); } - _free_regions_at_end_of_collection = _g1->free_regions(); + _free_regions_at_end_of_collection = _g1->num_free_regions(); update_young_list_target_length(); // We may immediately start allocating regions and placing them on the @@ -828,7 +828,7 @@ record_survivor_regions(0, NULL, NULL); - _free_regions_at_end_of_collection = _g1->free_regions(); + _free_regions_at_end_of_collection = _g1->num_free_regions(); // Reset survivors SurvRateGroup. _survivor_surv_rate_group->reset(); update_young_list_target_length(); @@ -1180,7 +1180,7 @@ _in_marking_window = new_in_marking_window; _in_marking_window_im = new_in_marking_window_im; - _free_regions_at_end_of_collection = _g1->free_regions(); + _free_regions_at_end_of_collection = _g1->num_free_regions(); update_young_list_target_length(); // Note that _mmu_tracker->max_gc_time() returns the time in seconds. @@ -1202,7 +1202,7 @@ _survivor_used_bytes_before_gc = young_list->survivor_used_bytes(); _heap_capacity_bytes_before_gc = _g1->capacity(); _heap_used_bytes_before_gc = _g1->used(); - _cur_collection_pause_used_regions_at_start = _g1->used_regions(); + _cur_collection_pause_used_regions_at_start = _g1->num_used_regions(); _eden_capacity_bytes_before_gc = (_young_list_target_length * HeapRegion::GrainBytes) - _survivor_used_bytes_before_gc; @@ -1617,7 +1617,7 @@ G1CollectorPolicy::record_concurrent_mark_cleanup_end(int no_of_gc_threads) { _collectionSetChooser->clear(); - uint region_num = _g1->n_regions(); + uint region_num = _g1->num_regions(); if (G1CollectedHeap::use_parallel_gc_threads()) { const uint OverpartitionFactor = 4; uint WorkUnit; @@ -1638,7 +1638,7 @@ MAX2(region_num / (uint) (ParallelGCThreads * OverpartitionFactor), MinWorkUnit); } - _collectionSetChooser->prepare_for_par_region_addition(_g1->n_regions(), + _collectionSetChooser->prepare_for_par_region_addition(_g1->num_regions(), WorkUnit); ParKnownGarbageTask parKnownGarbageTask(_collectionSetChooser, (int) WorkUnit); @@ -1935,7 +1935,7 @@ // of them are available. G1CollectedHeap* g1h = G1CollectedHeap::heap(); - const size_t region_num = g1h->n_regions(); + const size_t region_num = g1h->num_regions(); const size_t perc = (size_t) G1OldCSetRegionThresholdPercent; size_t result = region_num * perc / 100; // emulate ceiling
--- a/src/share/vm/gc_implementation/g1/g1HotCardCache.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1HotCardCache.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -33,7 +33,7 @@ G1HotCardCache::G1HotCardCache(G1CollectedHeap *g1h): _g1h(g1h), _hot_cache(NULL), _use_cache(false), _card_counts(g1h) {} -void G1HotCardCache::initialize() { +void G1HotCardCache::initialize(G1RegionToSpaceMapper* card_counts_storage) { if (default_use_cache()) { _use_cache = true; @@ -49,7 +49,7 @@ _hot_cache_par_chunk_size = MAX2(1, _hot_cache_size / (int)n_workers); _hot_cache_par_claimed_idx = 0; - _card_counts.initialize(); + _card_counts.initialize(card_counts_storage); } } @@ -135,11 +135,8 @@ // above, are discarded prior to re-enabling the cache near the end of the GC. } -void G1HotCardCache::resize_card_counts(size_t heap_capacity) { - _card_counts.resize(heap_capacity); -} - void G1HotCardCache::reset_card_counts(HeapRegion* hr) { + assert(!hr->isHumongous(), "Should have been cleared"); _card_counts.clear_region(hr); }
--- a/src/share/vm/gc_implementation/g1/g1HotCardCache.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1HotCardCache.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -78,7 +78,7 @@ G1HotCardCache(G1CollectedHeap* g1h); ~G1HotCardCache(); - void initialize(); + void initialize(G1RegionToSpaceMapper* card_counts_storage); bool use_cache() { return _use_cache; } @@ -115,9 +115,6 @@ bool hot_cache_is_empty() { return _n_hot == 0; } - // Resizes the card counts table to match the given capacity - void resize_card_counts(size_t heap_capacity); - // Zeros the values in the card counts table for entire committed heap void reset_card_counts();
--- a/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -130,9 +130,7 @@ if (!oopDesc::is_null(heap_oop)) { oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); HeapRegion* hr = _g1h->heap_region_containing((HeapWord*) obj); - if (hr != NULL) { - _cm->grayRoot(obj, obj->size(), _worker_id, hr); - } + _cm->grayRoot(obj, obj->size(), _worker_id, hr); } } @@ -159,57 +157,61 @@ template <class T> inline void G1UpdateRSOrPushRefOopClosure::do_oop_nv(T* p) { oop obj = oopDesc::load_decode_heap_oop(p); + if (obj == NULL) { + return; + } #ifdef ASSERT // can't do because of races // assert(obj == NULL || obj->is_oop(), "expected an oop"); // Do the safe subset of is_oop - if (obj != NULL) { #ifdef CHECK_UNHANDLED_OOPS - oopDesc* o = obj.obj(); + oopDesc* o = obj.obj(); #else - oopDesc* o = obj; + oopDesc* o = obj; #endif // CHECK_UNHANDLED_OOPS - assert((intptr_t)o % MinObjAlignmentInBytes == 0, "not oop aligned"); - assert(Universe::heap()->is_in_reserved(obj), "must be in heap"); - } + assert((intptr_t)o % MinObjAlignmentInBytes == 0, "not oop aligned"); + assert(Universe::heap()->is_in_reserved(obj), "must be in heap"); #endif // ASSERT assert(_from != NULL, "from region must be non-NULL"); assert(_from->is_in_reserved(p), "p is not in from"); HeapRegion* to = _g1->heap_region_containing(obj); - if (to != NULL && _from != to) { - // The _record_refs_into_cset flag is true during the RSet - // updating part of an evacuation pause. It is false at all - // other times: - // * rebuilding the rembered sets after a full GC - // * during concurrent refinement. - // * updating the remembered sets of regions in the collection - // set in the event of an evacuation failure (when deferred - // updates are enabled). + if (_from == to) { + // Normally this closure should only be called with cross-region references. + // But since Java threads are manipulating the references concurrently and we + // reload the values things may have changed. + return; + } + // The _record_refs_into_cset flag is true during the RSet + // updating part of an evacuation pause. It is false at all + // other times: + // * rebuilding the remembered sets after a full GC + // * during concurrent refinement. + // * updating the remembered sets of regions in the collection + // set in the event of an evacuation failure (when deferred + // updates are enabled). - if (_record_refs_into_cset && to->in_collection_set()) { - // We are recording references that point into the collection - // set and this particular reference does exactly that... - // If the referenced object has already been forwarded - // to itself, we are handling an evacuation failure and - // we have already visited/tried to copy this object - // there is no need to retry. - if (!self_forwarded(obj)) { - assert(_push_ref_cl != NULL, "should not be null"); - // Push the reference in the refs queue of the G1ParScanThreadState - // instance for this worker thread. - _push_ref_cl->do_oop(p); - } + if (_record_refs_into_cset && to->in_collection_set()) { + // We are recording references that point into the collection + // set and this particular reference does exactly that... + // If the referenced object has already been forwarded + // to itself, we are handling an evacuation failure and + // we have already visited/tried to copy this object + // there is no need to retry. + if (!self_forwarded(obj)) { + assert(_push_ref_cl != NULL, "should not be null"); + // Push the reference in the refs queue of the G1ParScanThreadState + // instance for this worker thread. + _push_ref_cl->do_oop(p); + } - // Deferred updates to the CSet are either discarded (in the normal case), - // or processed (if an evacuation failure occurs) at the end - // of the collection. - // See G1RemSet::cleanup_after_oops_into_collection_set_do(). - return; - } - + // Deferred updates to the CSet are either discarded (in the normal case), + // or processed (if an evacuation failure occurs) at the end + // of the collection. + // See G1RemSet::cleanup_after_oops_into_collection_set_do(). + } else { // We either don't care about pushing references that point into the // collection set (i.e. we're not during an evacuation pause) _or_ // the reference doesn't point into the collection set. Either way
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc_implementation/g1/g1PageBasedVirtualSpace.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc_implementation/g1/g1PageBasedVirtualSpace.hpp" +#include "oops/markOop.hpp" +#include "oops/oop.inline.hpp" +#include "services/memTracker.hpp" +#ifdef TARGET_OS_FAMILY_linux +# include "os_linux.inline.hpp" +#endif +#ifdef TARGET_OS_FAMILY_solaris +# include "os_solaris.inline.hpp" +#endif +#ifdef TARGET_OS_FAMILY_windows +# include "os_windows.inline.hpp" +#endif +#ifdef TARGET_OS_FAMILY_aix +# include "os_aix.inline.hpp" +#endif +#ifdef TARGET_OS_FAMILY_bsd +# include "os_bsd.inline.hpp" +#endif +#include "utilities/bitMap.inline.hpp" + +G1PageBasedVirtualSpace::G1PageBasedVirtualSpace() : _low_boundary(NULL), + _high_boundary(NULL), _committed(), _page_size(0), _special(false), _executable(false) { +} + +bool G1PageBasedVirtualSpace::initialize_with_granularity(ReservedSpace rs, size_t page_size) { + if (!rs.is_reserved()) { + return false; // Allocation failed. + } + assert(_low_boundary == NULL, "VirtualSpace already initialized"); + assert(page_size > 0, "Granularity must be non-zero."); + + _low_boundary = rs.base(); + _high_boundary = _low_boundary + rs.size(); + + _special = rs.special(); + _executable = rs.executable(); + + _page_size = page_size; + + assert(_committed.size() == 0, "virtual space initialized more than once"); + uintx size_in_bits = rs.size() / page_size; + _committed.resize(size_in_bits, /* in_resource_area */ false); + + return true; +} + + +G1PageBasedVirtualSpace::~G1PageBasedVirtualSpace() { + release(); +} + +void G1PageBasedVirtualSpace::release() { + // This does not release memory it never reserved. + // Caller must release via rs.release(); + _low_boundary = NULL; + _high_boundary = NULL; + _special = false; + _executable = false; + _page_size = 0; + _committed.resize(0, false); +} + +size_t G1PageBasedVirtualSpace::committed_size() const { + return _committed.count_one_bits() * _page_size; +} + +size_t G1PageBasedVirtualSpace::reserved_size() const { + return pointer_delta(_high_boundary, _low_boundary, sizeof(char)); +} + +size_t G1PageBasedVirtualSpace::uncommitted_size() const { + return reserved_size() - committed_size(); +} + +uintptr_t G1PageBasedVirtualSpace::addr_to_page_index(char* addr) const { + return (addr - _low_boundary) / _page_size; +} + +bool G1PageBasedVirtualSpace::is_area_committed(uintptr_t start, size_t size_in_pages) const { + uintptr_t end = start + size_in_pages; + return _committed.get_next_zero_offset(start, end) >= end; +} + +bool G1PageBasedVirtualSpace::is_area_uncommitted(uintptr_t start, size_t size_in_pages) const { + uintptr_t end = start + size_in_pages; + return _committed.get_next_one_offset(start, end) >= end; +} + +char* G1PageBasedVirtualSpace::page_start(uintptr_t index) { + return _low_boundary + index * _page_size; +} + +size_t G1PageBasedVirtualSpace::byte_size_for_pages(size_t num) { + return num * _page_size; +} + +MemRegion G1PageBasedVirtualSpace::commit(uintptr_t start, size_t size_in_pages) { + // We need to make sure to commit all pages covered by the given area. + guarantee(is_area_uncommitted(start, size_in_pages), "Specified area is not uncommitted"); + + if (!_special) { + os::commit_memory_or_exit(page_start(start), byte_size_for_pages(size_in_pages), _executable, + err_msg("Failed to commit pages from "SIZE_FORMAT" of length "SIZE_FORMAT, start, size_in_pages)); + } + _committed.set_range(start, start + size_in_pages); + + MemRegion result((HeapWord*)page_start(start), byte_size_for_pages(size_in_pages) / HeapWordSize); + return result; +} + +MemRegion G1PageBasedVirtualSpace::uncommit(uintptr_t start, size_t size_in_pages) { + guarantee(is_area_committed(start, size_in_pages), "checking"); + + if (!_special) { + os::uncommit_memory(page_start(start), byte_size_for_pages(size_in_pages)); + } + + _committed.clear_range(start, start + size_in_pages); + + MemRegion result((HeapWord*)page_start(start), byte_size_for_pages(size_in_pages) / HeapWordSize); + return result; +} + +bool G1PageBasedVirtualSpace::contains(const void* p) const { + return _low_boundary <= (const char*) p && (const char*) p < _high_boundary; +} + +#ifndef PRODUCT +void G1PageBasedVirtualSpace::print_on(outputStream* out) { + out->print ("Virtual space:"); + if (special()) out->print(" (pinned in memory)"); + out->cr(); + out->print_cr(" - committed: " SIZE_FORMAT, committed_size()); + out->print_cr(" - reserved: " SIZE_FORMAT, reserved_size()); + out->print_cr(" - [low_b, high_b]: [" INTPTR_FORMAT ", " INTPTR_FORMAT "]", p2i(_low_boundary), p2i(_high_boundary)); +} + +void G1PageBasedVirtualSpace::print() { + print_on(tty); +} +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc_implementation/g1/g1PageBasedVirtualSpace.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1PAGEBASEDVIRTUALSPACE_HPP +#define SHARE_VM_GC_IMPLEMENTATION_G1_G1PAGEBASEDVIRTUALSPACE_HPP + +#include "memory/allocation.hpp" +#include "memory/memRegion.hpp" +#include "runtime/virtualspace.hpp" +#include "utilities/bitMap.hpp" + +// Virtual space management helper for a virtual space with an OS page allocation +// granularity. +// (De-)Allocation requests are always OS page aligned by passing a page index +// and multiples of pages. +// The implementation gives an error when trying to commit or uncommit pages that +// have already been committed or uncommitted. +class G1PageBasedVirtualSpace VALUE_OBJ_CLASS_SPEC { + friend class VMStructs; + private: + // Reserved area addresses. + char* _low_boundary; + char* _high_boundary; + + // The commit/uncommit granularity in bytes. + size_t _page_size; + + // Bitmap used for verification of commit/uncommit operations. + BitMap _committed; + + // Indicates that the entire space has been committed and pinned in memory, + // os::commit_memory() or os::uncommit_memory() have no function. + bool _special; + + // Indicates whether the committed space should be executable. + bool _executable; + + // Returns the index of the page which contains the given address. + uintptr_t addr_to_page_index(char* addr) const; + // Returns the address of the given page index. + char* page_start(uintptr_t index); + // Returns the byte size of the given number of pages. + size_t byte_size_for_pages(size_t num); + + // Returns true if the entire area is backed by committed memory. + bool is_area_committed(uintptr_t start, size_t size_in_pages) const; + // Returns true if the entire area is not backed by committed memory. + bool is_area_uncommitted(uintptr_t start, size_t size_in_pages) const; + + public: + + // Commit the given area of pages starting at start being size_in_pages large. + MemRegion commit(uintptr_t start, size_t size_in_pages); + + // Uncommit the given area of pages starting at start being size_in_pages large. + MemRegion uncommit(uintptr_t start, size_t size_in_pages); + + bool special() const { return _special; } + + // Initialization + G1PageBasedVirtualSpace(); + bool initialize_with_granularity(ReservedSpace rs, size_t page_size); + + // Destruction + ~G1PageBasedVirtualSpace(); + + // Amount of reserved memory. + size_t reserved_size() const; + // Memory used in this virtual space. + size_t committed_size() const; + // Memory left to use/expand in this virtual space. + size_t uncommitted_size() const; + + bool contains(const void* p) const; + + MemRegion reserved() { + MemRegion x((HeapWord*)_low_boundary, reserved_size() / HeapWordSize); + return x; + } + + void release(); + + void check_for_contiguity() PRODUCT_RETURN; + + // Debugging + void print_on(outputStream* out) PRODUCT_RETURN; + void print(); +}; + +#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1PAGEBASEDVIRTUALSPACE_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc_implementation/g1/g1RegionToSpaceMapper.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc_implementation/g1/g1BiasedArray.hpp" +#include "gc_implementation/g1/g1RegionToSpaceMapper.hpp" +#include "memory/allocation.inline.hpp" +#include "runtime/virtualspace.hpp" +#include "services/memTracker.hpp" +#include "utilities/bitMap.inline.hpp" + +G1RegionToSpaceMapper::G1RegionToSpaceMapper(ReservedSpace rs, + size_t commit_granularity, + size_t region_granularity, + MemoryType type) : + _storage(), + _commit_granularity(commit_granularity), + _region_granularity(region_granularity), + _listener(NULL), + _commit_map() { + guarantee(is_power_of_2(commit_granularity), "must be"); + guarantee(is_power_of_2(region_granularity), "must be"); + _storage.initialize_with_granularity(rs, commit_granularity); + + MemTracker::record_virtual_memory_type((address)rs.base(), type); +} + +// G1RegionToSpaceMapper implementation where the region granularity is larger than +// or the same as the commit granularity. +// Basically, the space corresponding to one region region spans several OS pages. +class G1RegionsLargerThanCommitSizeMapper : public G1RegionToSpaceMapper { + private: + size_t _pages_per_region; + + public: + G1RegionsLargerThanCommitSizeMapper(ReservedSpace rs, + size_t os_commit_granularity, + size_t alloc_granularity, + size_t commit_factor, + MemoryType type) : + G1RegionToSpaceMapper(rs, os_commit_granularity, alloc_granularity, type), + _pages_per_region(alloc_granularity / (os_commit_granularity * commit_factor)) { + + guarantee(alloc_granularity >= os_commit_granularity, "allocation granularity smaller than commit granularity"); + _commit_map.resize(rs.size() * commit_factor / alloc_granularity, /* in_resource_area */ false); + } + + virtual void commit_regions(uintptr_t start_idx, size_t num_regions) { + _storage.commit(start_idx * _pages_per_region, num_regions * _pages_per_region); + _commit_map.set_range(start_idx, start_idx + num_regions); + fire_on_commit(start_idx, num_regions); + } + + virtual void uncommit_regions(uintptr_t start_idx, size_t num_regions) { + _storage.uncommit(start_idx * _pages_per_region, num_regions * _pages_per_region); + _commit_map.clear_range(start_idx, start_idx + num_regions); + } +}; + +// G1RegionToSpaceMapper implementation where the region granularity is smaller +// than the commit granularity. +// Basically, the contents of one OS page span several regions. +class G1RegionsSmallerThanCommitSizeMapper : public G1RegionToSpaceMapper { + private: + class CommitRefcountArray : public G1BiasedMappedArray<uint> { + protected: + virtual uint default_value() const { return 0; } + }; + + size_t _regions_per_page; + + CommitRefcountArray _refcounts; + + uintptr_t region_idx_to_page_idx(uint region) const { + return region / _regions_per_page; + } + + public: + G1RegionsSmallerThanCommitSizeMapper(ReservedSpace rs, + size_t os_commit_granularity, + size_t alloc_granularity, + size_t commit_factor, + MemoryType type) : + G1RegionToSpaceMapper(rs, os_commit_granularity, alloc_granularity, type), + _regions_per_page((os_commit_granularity * commit_factor) / alloc_granularity), _refcounts() { + + guarantee((os_commit_granularity * commit_factor) >= alloc_granularity, "allocation granularity smaller than commit granularity"); + _refcounts.initialize((HeapWord*)rs.base(), (HeapWord*)(rs.base() + rs.size()), os_commit_granularity); + _commit_map.resize(rs.size() * commit_factor / alloc_granularity, /* in_resource_area */ false); + } + + virtual void commit_regions(uintptr_t start_idx, size_t num_regions) { + for (uintptr_t i = start_idx; i < start_idx + num_regions; i++) { + assert(!_commit_map.at(i), err_msg("Trying to commit storage at region "INTPTR_FORMAT" that is already committed", i)); + uintptr_t idx = region_idx_to_page_idx(i); + uint old_refcount = _refcounts.get_by_index(idx); + if (old_refcount == 0) { + _storage.commit(idx, 1); + } + _refcounts.set_by_index(idx, old_refcount + 1); + _commit_map.set_bit(i); + fire_on_commit(i, 1); + } + } + + virtual void uncommit_regions(uintptr_t start_idx, size_t num_regions) { + for (uintptr_t i = start_idx; i < start_idx + num_regions; i++) { + assert(_commit_map.at(i), err_msg("Trying to uncommit storage at region "INTPTR_FORMAT" that is not committed", i)); + uintptr_t idx = region_idx_to_page_idx(i); + uint old_refcount = _refcounts.get_by_index(idx); + assert(old_refcount > 0, "must be"); + if (old_refcount == 1) { + _storage.uncommit(idx, 1); + } + _refcounts.set_by_index(idx, old_refcount - 1); + _commit_map.clear_bit(i); + } + } +}; + +void G1RegionToSpaceMapper::fire_on_commit(uint start_idx, size_t num_regions) { + if (_listener != NULL) { + _listener->on_commit(start_idx, num_regions); + } +} + +G1RegionToSpaceMapper* G1RegionToSpaceMapper::create_mapper(ReservedSpace rs, + size_t os_commit_granularity, + size_t region_granularity, + size_t commit_factor, + MemoryType type) { + + if (region_granularity >= (os_commit_granularity * commit_factor)) { + return new G1RegionsLargerThanCommitSizeMapper(rs, os_commit_granularity, region_granularity, commit_factor, type); + } else { + return new G1RegionsSmallerThanCommitSizeMapper(rs, os_commit_granularity, region_granularity, commit_factor, type); + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc_implementation/g1/g1RegionToSpaceMapper.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1REGIONTOSPACEMAPPER_HPP +#define SHARE_VM_GC_IMPLEMENTATION_G1_G1REGIONTOSPACEMAPPER_HPP + +#include "gc_implementation/g1/g1PageBasedVirtualSpace.hpp" +#include "memory/allocation.hpp" +#include "utilities/debug.hpp" + +class G1MappingChangedListener VALUE_OBJ_CLASS_SPEC { + public: + // Fired after commit of the memory, i.e. the memory this listener is registered + // for can be accessed. + virtual void on_commit(uint start_idx, size_t num_regions) = 0; +}; + +// Maps region based commit/uncommit requests to the underlying page sized virtual +// space. +class G1RegionToSpaceMapper : public CHeapObj<mtGC> { + private: + G1MappingChangedListener* _listener; + protected: + // Backing storage. + G1PageBasedVirtualSpace _storage; + size_t _commit_granularity; + size_t _region_granularity; + // Mapping management + BitMap _commit_map; + + G1RegionToSpaceMapper(ReservedSpace rs, size_t commit_granularity, size_t region_granularity, MemoryType type); + + void fire_on_commit(uint start_idx, size_t num_regions); + public: + MemRegion reserved() { return _storage.reserved(); } + + void set_mapping_changed_listener(G1MappingChangedListener* listener) { _listener = listener; } + + virtual ~G1RegionToSpaceMapper() { + _commit_map.resize(0, /* in_resource_area */ false); + } + + bool is_committed(uintptr_t idx) const { + return _commit_map.at(idx); + } + + virtual void commit_regions(uintptr_t start_idx, size_t num_regions = 1) = 0; + virtual void uncommit_regions(uintptr_t start_idx, size_t num_regions = 1) = 0; + + // Creates an appropriate G1RegionToSpaceMapper for the given parameters. + // The byte_translation_factor defines how many bytes in a region correspond to + // a single byte in the data structure this mapper is for. + // Eg. in the card table, this value corresponds to the size a single card + // table entry corresponds to. + static G1RegionToSpaceMapper* create_mapper(ReservedSpace rs, + size_t os_commit_granularity, + size_t region_granularity, + size_t byte_translation_factor, + MemoryType type); +}; + +#endif /* SHARE_VM_GC_IMPLEMENTATION_G1_G1REGIONTOSPACEMAPPER_HPP */
--- a/src/share/vm/gc_implementation/g1/g1RemSet.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1RemSet.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -211,7 +211,6 @@ #endif HeapRegion* card_region = _g1h->heap_region_containing(card_start); - assert(card_region != NULL, "Yielding cards not in the heap?"); _cards++; if (!card_region->is_on_dirty_cards_region_list()) { @@ -406,7 +405,6 @@ HeapWord* start = _ct_bs->addr_for(card_ptr); // And find the region containing it. HeapRegion* r = _g1->heap_region_containing(start); - assert(r != NULL, "unexpected null"); // Scan oops in the card looking for references into the collection set // Don't use addr_for(card_ptr + 1) which can ask for @@ -556,6 +554,12 @@ bool G1RemSet::refine_card(jbyte* card_ptr, uint worker_i, bool check_for_refs_into_cset) { + assert(_g1->is_in_exact(_ct_bs->addr_for(card_ptr)), + err_msg("Card at "PTR_FORMAT" index "SIZE_FORMAT" representing heap at "PTR_FORMAT" (%u) must be in committed heap", + p2i(card_ptr), + _ct_bs->index_for(_ct_bs->addr_for(card_ptr)), + _ct_bs->addr_for(card_ptr), + _g1->addr_to_region(_ct_bs->addr_for(card_ptr)))); // If the card is no longer dirty, nothing to do. if (*card_ptr != CardTableModRefBS::dirty_card_val()) { @@ -568,11 +572,6 @@ HeapWord* start = _ct_bs->addr_for(card_ptr); // And find the region containing it. HeapRegion* r = _g1->heap_region_containing(start); - if (r == NULL) { - // Again no need to return that this card contains refs that - // point into the collection set. - return false; // Not in the G1 heap (might be in perm, for example.) - } // Why do we have to check here whether a card is on a young region, // given that we dirty young regions and, as a result, the @@ -625,10 +624,6 @@ start = _ct_bs->addr_for(card_ptr); r = _g1->heap_region_containing(start); - if (r == NULL) { - // Not in the G1 heap - return false; - } // Checking whether the region we got back from the cache // is young here is inappropriate. The region could have been
--- a/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -46,26 +46,28 @@ template <class T> inline void G1RemSet::par_write_ref(HeapRegion* from, T* p, int tid) { oop obj = oopDesc::load_decode_heap_oop(p); + if (obj == NULL) { + return; + } + #ifdef ASSERT // can't do because of races // assert(obj == NULL || obj->is_oop(), "expected an oop"); // Do the safe subset of is_oop - if (obj != NULL) { #ifdef CHECK_UNHANDLED_OOPS - oopDesc* o = obj.obj(); + oopDesc* o = obj.obj(); #else - oopDesc* o = obj; + oopDesc* o = obj; #endif // CHECK_UNHANDLED_OOPS - assert((intptr_t)o % MinObjAlignmentInBytes == 0, "not oop aligned"); - assert(Universe::heap()->is_in_reserved(obj), "must be in heap"); - } + assert((intptr_t)o % MinObjAlignmentInBytes == 0, "not oop aligned"); + assert(Universe::heap()->is_in_reserved(obj), "must be in heap"); #endif // ASSERT assert(from == NULL || from->is_in_reserved(p), "p is not in from"); HeapRegion* to = _g1->heap_region_containing(obj); - if (to != NULL && from != to) { + if (from != to) { assert(to->rem_set() != NULL, "Need per-region 'into' remsets."); to->rem_set()->add_reference(p, tid); }
--- a/src/share/vm/gc_implementation/g1/g1SATBCardTableModRefBS.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1SATBCardTableModRefBS.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -23,6 +23,7 @@ */ #include "precompiled.hpp" +#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" #include "gc_implementation/g1/heapRegion.hpp" #include "gc_implementation/g1/satbQueue.hpp" @@ -37,7 +38,6 @@ _kind = G1SATBCT; } - void G1SATBCardTableModRefBS::enqueue(oop pre_val) { // Nulls should have been already filtered. assert(pre_val->is_oop(true), "Error"); @@ -124,13 +124,52 @@ } #endif +void G1SATBCardTableLoggingModRefBSChangedListener::on_commit(uint start_idx, size_t num_regions) { + MemRegion mr(G1CollectedHeap::heap()->bottom_addr_for_region(start_idx), num_regions * HeapRegion::GrainWords); + _card_table->clear(mr); +} + G1SATBCardTableLoggingModRefBS:: G1SATBCardTableLoggingModRefBS(MemRegion whole_heap, int max_covered_regions) : G1SATBCardTableModRefBS(whole_heap, max_covered_regions), - _dcqs(JavaThread::dirty_card_queue_set()) + _dcqs(JavaThread::dirty_card_queue_set()), + _listener() { _kind = G1SATBCTLogging; + _listener.set_card_table(this); +} + +void G1SATBCardTableLoggingModRefBS::initialize(G1RegionToSpaceMapper* mapper) { + mapper->set_mapping_changed_listener(&_listener); + + _byte_map_size = mapper->reserved().byte_size(); + + _guard_index = cards_required(_whole_heap.word_size()) - 1; + _last_valid_index = _guard_index - 1; + + HeapWord* low_bound = _whole_heap.start(); + HeapWord* high_bound = _whole_heap.end(); + + _cur_covered_regions = 1; + _covered[0] = _whole_heap; + + _byte_map = (jbyte*) mapper->reserved().start(); + byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift); + assert(byte_for(low_bound) == &_byte_map[0], "Checking start of map"); + assert(byte_for(high_bound-1) <= &_byte_map[_last_valid_index], "Checking end of map"); + + if (TraceCardTableModRefBS) { + gclog_or_tty->print_cr("G1SATBCardTableModRefBS::G1SATBCardTableModRefBS: "); + gclog_or_tty->print_cr(" " + " &_byte_map[0]: " INTPTR_FORMAT + " &_byte_map[_last_valid_index]: " INTPTR_FORMAT, + p2i(&_byte_map[0]), + p2i(&_byte_map[_last_valid_index])); + gclog_or_tty->print_cr(" " + " byte_map_base: " INTPTR_FORMAT, + p2i(byte_map_base)); + } } void
--- a/src/share/vm/gc_implementation/g1/g1SATBCardTableModRefBS.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/g1SATBCardTableModRefBS.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -25,6 +25,7 @@ #ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1SATBCARDTABLEMODREFBS_HPP #define SHARE_VM_GC_IMPLEMENTATION_G1_G1SATBCARDTABLEMODREFBS_HPP +#include "gc_implementation/g1/g1RegionToSpaceMapper.hpp" #include "memory/cardTableModRefBS.hpp" #include "memory/memRegion.hpp" #include "oops/oop.inline.hpp" @@ -33,6 +34,7 @@ #if INCLUDE_ALL_GCS class DirtyCardQueueSet; +class G1SATBCardTableLoggingModRefBS; // This barrier is specialized to use a logging barrier to support // snapshot-at-the-beginning marking. @@ -126,18 +128,40 @@ jbyte val = _byte_map[card_index]; return (val & (clean_card_mask_val() | deferred_card_val())) == deferred_card_val(); } +}; +class G1SATBCardTableLoggingModRefBSChangedListener : public G1MappingChangedListener { + private: + G1SATBCardTableLoggingModRefBS* _card_table; + public: + G1SATBCardTableLoggingModRefBSChangedListener() : _card_table(NULL) { } + + void set_card_table(G1SATBCardTableLoggingModRefBS* card_table) { _card_table = card_table; } + + virtual void on_commit(uint start_idx, size_t num_regions); }; // Adds card-table logging to the post-barrier. // Usual invariant: all dirty cards are logged in the DirtyCardQueueSet. class G1SATBCardTableLoggingModRefBS: public G1SATBCardTableModRefBS { + friend class G1SATBCardTableLoggingModRefBSChangedListener; private: + G1SATBCardTableLoggingModRefBSChangedListener _listener; DirtyCardQueueSet& _dcqs; public: + static size_t compute_size(size_t mem_region_size_in_words) { + size_t number_of_slots = (mem_region_size_in_words / card_size_in_words); + return ReservedSpace::allocation_align_size_up(number_of_slots); + } + G1SATBCardTableLoggingModRefBS(MemRegion whole_heap, int max_covered_regions); + virtual void initialize() { } + virtual void initialize(G1RegionToSpaceMapper* mapper); + + virtual void resize_covered_region(MemRegion new_region) { ShouldNotReachHere(); } + bool is_a(BarrierSet::Name bsn) { return bsn == BarrierSet::G1SATBCTLogging || G1SATBCardTableModRefBS::is_a(bsn); @@ -154,8 +178,6 @@ void write_region_work(MemRegion mr) { invalidate(mr); } void write_ref_array_work(MemRegion mr) { invalidate(mr); } - - };
--- a/src/share/vm/gc_implementation/g1/heapRegion.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegion.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -344,11 +344,6 @@ return low; } -#ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away -#pragma warning( disable:4355 ) // 'this' : used in base member initializer list -#endif // _MSC_VER - - HeapRegion::HeapRegion(uint hrs_index, G1BlockOffsetSharedArray* sharedOffsetArray, MemRegion mr) : @@ -360,7 +355,7 @@ _claimed(InitialClaimValue), _evacuation_failed(false), _prev_marked_bytes(0), _next_marked_bytes(0), _gc_efficiency(0.0), _young_type(NotYoung), _next_young_region(NULL), - _next_dirty_cards_region(NULL), _next(NULL), _prev(NULL), _pending_removal(false), + _next_dirty_cards_region(NULL), _next(NULL), _prev(NULL), #ifdef ASSERT _containing_set(NULL), #endif // ASSERT @@ -369,14 +364,20 @@ _predicted_bytes_to_copy(0) { _rem_set = new HeapRegionRemSet(sharedOffsetArray, this); + assert(HeapRegionRemSet::num_par_rem_sets() > 0, "Invariant."); + + initialize(mr); +} + +void HeapRegion::initialize(MemRegion mr, bool clear_space, bool mangle_space) { + assert(_rem_set->is_empty(), "Remembered set must be empty"); + + G1OffsetTableContigSpace::initialize(mr, clear_space, mangle_space); + _orig_end = mr.end(); - // Note that initialize() will set the start of the unmarked area of the - // region. hr_clear(false /*par*/, false /*clear_space*/); set_top(bottom()); record_top_and_timestamp(); - - assert(HeapRegionRemSet::num_par_rem_sets() > 0, "Invariant."); } CompactibleSpace* HeapRegion::next_compaction_space() const { @@ -907,7 +908,7 @@ } // If it returns false, verify_for_object() will output the - // appropriate messasge. + // appropriate message. if (do_bot_verify && !g1->is_obj_dead(obj, this) && !_offsets.verify_for_object(p, obj_size)) { @@ -1038,8 +1039,7 @@ set_top(bottom()); set_saved_mark_word(bottom()); CompactibleSpace::clear(mangle_space); - _offsets.zero_bottom_entry(); - _offsets.initialize_threshold(); + reset_bot(); } void G1OffsetTableContigSpace::set_bottom(HeapWord* new_bottom) { @@ -1129,9 +1129,11 @@ _gc_time_stamp(0) { _offsets.set_space(this); - // false ==> we'll do the clearing if there's clearing to be done. - CompactibleSpace::initialize(mr, false, SpaceDecorator::Mangle); +} + +void G1OffsetTableContigSpace::initialize(MemRegion mr, bool clear_space, bool mangle_space) { + CompactibleSpace::initialize(mr, clear_space, mangle_space); _top = bottom(); - _offsets.zero_bottom_entry(); - _offsets.initialize_threshold(); + reset_bot(); } +
--- a/src/share/vm/gc_implementation/g1/heapRegion.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegion.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -62,7 +62,7 @@ p2i((_hr_)->bottom()), p2i((_hr_)->top()), p2i((_hr_)->end()) // sentinel value for hrs_index -#define G1_NULL_HRS_INDEX ((uint) -1) +#define G1_NO_HRS_INDEX ((uint) -1) // A dirty card to oop closure for heap regions. It // knows how to get the G1 heap and how to use the bitmap @@ -146,6 +146,9 @@ HeapWord* top() const { return _top; } protected: + // Reset the G1OffsetTableContigSpace. + virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space); + HeapWord** top_addr() { return &_top; } // Allocation helpers (return NULL if full). inline HeapWord* allocate_impl(size_t word_size, HeapWord* end_value); @@ -200,8 +203,7 @@ virtual void print() const; void reset_bot() { - _offsets.zero_bottom_entry(); - _offsets.initialize_threshold(); + _offsets.reset_bot(); } void update_bot_for_object(HeapWord* start, size_t word_size) { @@ -264,7 +266,6 @@ #ifdef ASSERT HeapRegionSetBase* _containing_set; #endif // ASSERT - bool _pending_removal; // For parallel heapRegion traversal. jint _claimed; @@ -333,6 +334,12 @@ G1BlockOffsetSharedArray* sharedOffsetArray, MemRegion mr); + // Initializing the HeapRegion not only resets the data structure, but also + // resets the BOT for that heap region. + // The default values for clear_space means that we will do the clearing if + // there's clearing to be done ourselves. We also always mangle the space. + virtual void initialize(MemRegion mr, bool clear_space = false, bool mangle_space = SpaceDecorator::Mangle); + static int LogOfHRGrainBytes; static int LogOfHRGrainWords; @@ -553,26 +560,6 @@ // to provide a dummy version of it. #endif // ASSERT - // If we want to remove regions from a list in bulk we can simply tag - // them with the pending_removal tag and call the - // remove_all_pending() method on the list. - - bool pending_removal() { return _pending_removal; } - - void set_pending_removal(bool pending_removal) { - if (pending_removal) { - assert(!_pending_removal && containing_set() != NULL, - "can only set pending removal to true if it's false and " - "the region belongs to a region set"); - } else { - assert( _pending_removal && containing_set() == NULL, - "can only set pending removal to false if it's true and " - "the region does not belong to a region set"); - } - - _pending_removal = pending_removal; - } - HeapRegion* get_next_young_region() { return _next_young_region; } void set_next_young_region(HeapRegion* hr) { _next_young_region = hr;
--- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -372,17 +372,17 @@ _max_regions, &_static_mem_size); - for (uint i = 0; i < n_par_rs; i++) { - for (uint j = 0; j < _max_regions; j++) { - set(i, j, InvalidCard); - } - } + invalidate(0, _max_regions); } -void FromCardCache::shrink(uint new_num_regions) { +void FromCardCache::invalidate(uint start_idx, size_t new_num_regions) { + guarantee((size_t)start_idx + new_num_regions <= max_uintx, + err_msg("Trying to invalidate beyond maximum region, from %u size "SIZE_FORMAT, + start_idx, new_num_regions)); for (uint i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { - assert(new_num_regions <= _max_regions, "Must be within max."); - for (uint j = new_num_regions; j < _max_regions; j++) { + uint end_idx = (start_idx + (uint)new_num_regions); + assert(end_idx <= _max_regions, "Must be within max."); + for (uint j = start_idx; j < end_idx; j++) { set(i, j, InvalidCard); } } @@ -406,12 +406,12 @@ } } -void OtherRegionsTable::init_from_card_cache(uint max_regions) { +void OtherRegionsTable::initialize(uint max_regions) { FromCardCache::initialize(HeapRegionRemSet::num_par_rem_sets(), max_regions); } -void OtherRegionsTable::shrink_from_card_cache(uint new_num_regions) { - FromCardCache::shrink(new_num_regions); +void OtherRegionsTable::invalidate(uint start_idx, size_t num_regions) { + FromCardCache::invalidate(start_idx, num_regions); } void OtherRegionsTable::print_from_card_cache() { @@ -802,7 +802,6 @@ bool OtherRegionsTable::contains_reference_locked(OopOrNarrowOopStar from) const { HeapRegion* hr = _g1h->heap_region_containing_raw(from); - if (hr == NULL) return false; RegionIdx_t hr_ind = (RegionIdx_t) hr->hrs_index(); // Is this region in the coarse map? if (_coarse_map.at(hr_ind)) return true; @@ -840,8 +839,8 @@ HeapRegionRemSet::HeapRegionRemSet(G1BlockOffsetSharedArray* bosa, HeapRegion* hr) : _bosa(bosa), - _m(Mutex::leaf, FormatBuffer<128>("HeapRegionRemSet lock #"UINT32_FORMAT, hr->hrs_index()), true), - _code_roots(), _other_regions(hr, &_m) { + _m(Mutex::leaf, FormatBuffer<128>("HeapRegionRemSet lock #%u", hr->hrs_index()), true), + _code_roots(), _other_regions(hr, &_m), _iter_state(Unclaimed), _iter_claimed(0) { reset_for_par_iteration(); }
--- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -84,7 +84,7 @@ static void initialize(uint n_par_rs, uint max_num_regions); - static void shrink(uint new_num_regions); + static void invalidate(uint start_idx, size_t num_regions); static void print(outputStream* out = gclog_or_tty) PRODUCT_RETURN; @@ -213,11 +213,11 @@ // Declare the heap size (in # of regions) to the OtherRegionsTable. // (Uses it to initialize from_card_cache). - static void init_from_card_cache(uint max_regions); + static void initialize(uint max_regions); - // Declares that only regions i s.t. 0 <= i < new_n_regs are in use. - // Make sure any entries for higher regions are invalid. - static void shrink_from_card_cache(uint new_num_regions); + // Declares that regions between start_idx <= i < start_idx + num_regions are + // not in use. Make sure that any entries for these regions are invalid. + static void invalidate(uint start_idx, size_t num_regions); static void print_from_card_cache(); }; @@ -404,12 +404,11 @@ // Declare the heap size (in # of regions) to the HeapRegionRemSet(s). // (Uses it to initialize from_card_cache). static void init_heap(uint max_regions) { - OtherRegionsTable::init_from_card_cache(max_regions); + OtherRegionsTable::initialize(max_regions); } - // Declares that only regions i s.t. 0 <= i < new_n_regs are in use. - static void shrink_heap(uint new_n_regs) { - OtherRegionsTable::shrink_from_card_cache(new_n_regs); + static void invalidate(uint start_idx, uint num_regions) { + OtherRegionsTable::invalidate(start_idx, num_regions); } #ifndef PRODUCT
--- a/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -25,163 +25,204 @@ #include "precompiled.hpp" #include "gc_implementation/g1/heapRegion.hpp" #include "gc_implementation/g1/heapRegionSeq.inline.hpp" -#include "gc_implementation/g1/heapRegionSet.hpp" +#include "gc_implementation/g1/heapRegionSet.inline.hpp" #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" +#include "gc_implementation/g1/concurrentG1Refine.hpp" #include "memory/allocation.hpp" -// Private +void HeapRegionSeq::initialize(G1RegionToSpaceMapper* heap_storage, + G1RegionToSpaceMapper* prev_bitmap, + G1RegionToSpaceMapper* next_bitmap, + G1RegionToSpaceMapper* bot, + G1RegionToSpaceMapper* cardtable, + G1RegionToSpaceMapper* card_counts) { + _allocated_heapregions_length = 0; + + _heap_mapper = heap_storage; + + _prev_bitmap_mapper = prev_bitmap; + _next_bitmap_mapper = next_bitmap; + + _bot_mapper = bot; + _cardtable_mapper = cardtable; + + _card_counts_mapper = card_counts; + + MemRegion reserved = heap_storage->reserved(); + _regions.initialize(reserved.start(), reserved.end(), HeapRegion::GrainBytes); + + _available_map.resize(_regions.length(), false); + _available_map.clear(); +} -uint HeapRegionSeq::find_contiguous_from(uint from, uint num) { - uint len = length(); - assert(num > 1, "use this only for sequences of length 2 or greater"); - assert(from <= len, - err_msg("from: %u should be valid and <= than %u", from, len)); +bool HeapRegionSeq::is_available(uint region) const { + return _available_map.at(region); +} + +#ifdef ASSERT +bool HeapRegionSeq::is_free(HeapRegion* hr) const { + return _free_list.contains(hr); +} +#endif + +HeapRegion* HeapRegionSeq::new_heap_region(uint hrs_index) { + HeapWord* bottom = G1CollectedHeap::heap()->bottom_addr_for_region(hrs_index); + MemRegion mr(bottom, bottom + HeapRegion::GrainWords); + assert(reserved().contains(mr), "invariant"); + return new HeapRegion(hrs_index, G1CollectedHeap::heap()->bot_shared(), mr); +} + +void HeapRegionSeq::commit_regions(uint index, size_t num_regions) { + guarantee(num_regions > 0, "Must commit more than zero regions"); + guarantee(_num_committed + num_regions <= max_length(), "Cannot commit more than the maximum amount of regions"); + + _num_committed += (uint)num_regions; + + _heap_mapper->commit_regions(index, num_regions); + + // Also commit auxiliary data + _prev_bitmap_mapper->commit_regions(index, num_regions); + _next_bitmap_mapper->commit_regions(index, num_regions); - uint curr = from; - uint first = G1_NULL_HRS_INDEX; - uint num_so_far = 0; - while (curr < len && num_so_far < num) { - if (at(curr)->is_empty()) { - if (first == G1_NULL_HRS_INDEX) { - first = curr; - num_so_far = 1; - } else { - num_so_far += 1; - } - } else { - first = G1_NULL_HRS_INDEX; - num_so_far = 0; + _bot_mapper->commit_regions(index, num_regions); + _cardtable_mapper->commit_regions(index, num_regions); + + _card_counts_mapper->commit_regions(index, num_regions); +} + +void HeapRegionSeq::uncommit_regions(uint start, size_t num_regions) { + guarantee(num_regions >= 1, err_msg("Need to specify at least one region to uncommit, tried to uncommit zero regions at %u", start)); + guarantee(_num_committed >= num_regions, "pre-condition"); + + // Print before uncommitting. + if (G1CollectedHeap::heap()->hr_printer()->is_active()) { + for (uint i = start; i < start + num_regions; i++) { + HeapRegion* hr = at(i); + G1CollectedHeap::heap()->hr_printer()->uncommit(hr->bottom(), hr->end()); } - curr += 1; } - assert(num_so_far <= num, "post-condition"); - if (num_so_far == num) { - // we found enough space for the humongous object - assert(from <= first && first < len, "post-condition"); - assert(first < curr && (curr - first) == num, "post-condition"); - for (uint i = first; i < first + num; ++i) { - assert(at(i)->is_empty(), "post-condition"); + + _num_committed -= (uint)num_regions; + + _available_map.par_clear_range(start, start + num_regions, BitMap::unknown_range); + _heap_mapper->uncommit_regions(start, num_regions); + + // Also uncommit auxiliary data + _prev_bitmap_mapper->uncommit_regions(start, num_regions); + _next_bitmap_mapper->uncommit_regions(start, num_regions); + + _bot_mapper->uncommit_regions(start, num_regions); + _cardtable_mapper->uncommit_regions(start, num_regions); + + _card_counts_mapper->uncommit_regions(start, num_regions); +} + +void HeapRegionSeq::make_regions_available(uint start, uint num_regions) { + guarantee(num_regions > 0, "No point in calling this for zero regions"); + commit_regions(start, num_regions); + for (uint i = start; i < start + num_regions; i++) { + if (_regions.get_by_index(i) == NULL) { + HeapRegion* new_hr = new_heap_region(i); + _regions.set_by_index(i, new_hr); + _allocated_heapregions_length = MAX2(_allocated_heapregions_length, i + 1); } - return first; - } else { - // we failed to find enough space for the humongous object - return G1_NULL_HRS_INDEX; + } + + _available_map.par_set_range(start, start + num_regions, BitMap::unknown_range); + + for (uint i = start; i < start + num_regions; i++) { + assert(is_available(i), err_msg("Just made region %u available but is apparently not.", i)); + HeapRegion* hr = at(i); + if (G1CollectedHeap::heap()->hr_printer()->is_active()) { + G1CollectedHeap::heap()->hr_printer()->commit(hr->bottom(), hr->end()); + } + HeapWord* bottom = G1CollectedHeap::heap()->bottom_addr_for_region(i); + MemRegion mr(bottom, bottom + HeapRegion::GrainWords); + + hr->initialize(mr); + insert_into_free_list(at(i)); } } -// Public +uint HeapRegionSeq::expand_by(uint num_regions) { + return expand_at(0, num_regions); +} + +uint HeapRegionSeq::expand_at(uint start, uint num_regions) { + if (num_regions == 0) { + return 0; + } + + uint cur = start; + uint idx_last_found = 0; + uint num_last_found = 0; -void HeapRegionSeq::initialize(HeapWord* bottom, HeapWord* end) { - assert((uintptr_t) bottom % HeapRegion::GrainBytes == 0, - "bottom should be heap region aligned"); - assert((uintptr_t) end % HeapRegion::GrainBytes == 0, - "end should be heap region aligned"); + uint expanded = 0; - _next_search_index = 0; - _allocated_length = 0; + while (expanded < num_regions && + (num_last_found = find_unavailable_from_idx(cur, &idx_last_found)) > 0) { + uint to_expand = MIN2(num_regions - expanded, num_last_found); + make_regions_available(idx_last_found, to_expand); + expanded += to_expand; + cur = idx_last_found + num_last_found + 1; + } - _regions.initialize(bottom, end, HeapRegion::GrainBytes); + verify_optional(); + return expanded; } -MemRegion HeapRegionSeq::expand_by(HeapWord* old_end, - HeapWord* new_end, - FreeRegionList* list) { - assert(old_end < new_end, "don't call it otherwise"); - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - - HeapWord* next_bottom = old_end; - assert(heap_bottom() <= next_bottom, "invariant"); - while (next_bottom < new_end) { - assert(next_bottom < heap_end(), "invariant"); - uint index = length(); - - assert(index < max_length(), "otherwise we cannot expand further"); - if (index == 0) { - // We have not allocated any regions so far - assert(next_bottom == heap_bottom(), "invariant"); - } else { - // next_bottom should match the end of the last/previous region - assert(next_bottom == at(index - 1)->end(), "invariant"); - } +uint HeapRegionSeq::find_contiguous(size_t num, bool empty_only) { + uint found = 0; + size_t length_found = 0; + uint cur = 0; - if (index == _allocated_length) { - // We have to allocate a new HeapRegion. - HeapRegion* new_hr = g1h->new_heap_region(index, next_bottom); - if (new_hr == NULL) { - // allocation failed, we bail out and return what we have done so far - return MemRegion(old_end, next_bottom); - } - assert(_regions.get_by_index(index) == NULL, "invariant"); - _regions.set_by_index(index, new_hr); - increment_allocated_length(); + while (length_found < num && cur < max_length()) { + HeapRegion* hr = _regions.get_by_index(cur); + if ((!empty_only && !is_available(cur)) || (is_available(cur) && hr != NULL && hr->is_empty())) { + // This region is a potential candidate for allocation into. + length_found++; + } else { + // This region is not a candidate. The next region is the next possible one. + found = cur + 1; + length_found = 0; } - // Have to increment the length first, otherwise we will get an - // assert failure at(index) below. - increment_length(); - HeapRegion* hr = at(index); - list->add_as_tail(hr); + cur++; + } - next_bottom = hr->end(); + if (length_found == num) { + for (uint i = found; i < (found + num); i++) { + HeapRegion* hr = _regions.get_by_index(i); + // sanity check + guarantee((!empty_only && !is_available(i)) || (is_available(i) && hr != NULL && hr->is_empty()), + err_msg("Found region sequence starting at " UINT32_FORMAT ", length " SIZE_FORMAT + " that is not empty at " UINT32_FORMAT ". Hr is " PTR_FORMAT, found, num, i, p2i(hr))); + } + return found; + } else { + return G1_NO_HRS_INDEX; } - assert(next_bottom == new_end, "post-condition"); - return MemRegion(old_end, next_bottom); } -uint HeapRegionSeq::free_suffix() { - uint res = 0; - uint index = length(); - while (index > 0) { - index -= 1; - if (!at(index)->is_empty()) { - break; +HeapRegion* HeapRegionSeq::next_region_in_heap(const HeapRegion* r) const { + guarantee(r != NULL, "Start region must be a valid region"); + guarantee(is_available(r->hrs_index()), err_msg("Trying to iterate starting from region %u which is not in the heap", r->hrs_index())); + for (uint i = r->hrs_index() + 1; i < _allocated_heapregions_length; i++) { + HeapRegion* hr = _regions.get_by_index(i); + if (is_available(i)) { + return hr; } - res += 1; } - return res; -} - -uint HeapRegionSeq::find_contiguous(uint num) { - assert(num > 1, "use this only for sequences of length 2 or greater"); - assert(_next_search_index <= length(), - err_msg("_next_search_index: %u should be valid and <= than %u", - _next_search_index, length())); - - uint start = _next_search_index; - uint res = find_contiguous_from(start, num); - if (res == G1_NULL_HRS_INDEX && start > 0) { - // Try starting from the beginning. If _next_search_index was 0, - // no point in doing this again. - res = find_contiguous_from(0, num); - } - if (res != G1_NULL_HRS_INDEX) { - assert(res < length(), err_msg("res: %u should be valid", res)); - _next_search_index = res + num; - assert(_next_search_index <= length(), - err_msg("_next_search_index: %u should be valid and <= than %u", - _next_search_index, length())); - } - return res; + return NULL; } void HeapRegionSeq::iterate(HeapRegionClosure* blk) const { - iterate_from((HeapRegion*) NULL, blk); -} - -void HeapRegionSeq::iterate_from(HeapRegion* hr, HeapRegionClosure* blk) const { - uint hr_index = 0; - if (hr != NULL) { - hr_index = hr->hrs_index(); - } + uint len = max_length(); - uint len = length(); - for (uint i = hr_index; i < len; i += 1) { - bool res = blk->doHeapRegion(at(i)); - if (res) { - blk->incomplete(); - return; + for (uint i = 0; i < len; i++) { + if (!is_available(i)) { + continue; } - } - for (uint i = 0; i < hr_index; i += 1) { + guarantee(at(i) != NULL, err_msg("Tried to access region %u that has a NULL HeapRegion*", i)); bool res = blk->doHeapRegion(at(i)); if (res) { blk->incomplete(); @@ -190,72 +231,220 @@ } } +uint HeapRegionSeq::find_unavailable_from_idx(uint start_idx, uint* res_idx) const { + guarantee(res_idx != NULL, "checking"); + guarantee(start_idx <= (max_length() + 1), "checking"); + + uint num_regions = 0; + + uint cur = start_idx; + while (cur < max_length() && is_available(cur)) { + cur++; + } + if (cur == max_length()) { + return num_regions; + } + *res_idx = cur; + while (cur < max_length() && !is_available(cur)) { + cur++; + } + num_regions = cur - *res_idx; +#ifdef ASSERT + for (uint i = *res_idx; i < (*res_idx + num_regions); i++) { + assert(!is_available(i), "just checking"); + } + assert(cur == max_length() || num_regions == 0 || is_available(cur), + err_msg("The region at the current position %u must be available or at the end of the heap.", cur)); +#endif + return num_regions; +} + +uint HeapRegionSeq::start_region_for_worker(uint worker_i, uint num_workers, uint num_regions) const { + return num_regions * worker_i / num_workers; +} + +void HeapRegionSeq::par_iterate(HeapRegionClosure* blk, uint worker_id, uint num_workers, jint claim_value) const { + const uint start_index = start_region_for_worker(worker_id, num_workers, _allocated_heapregions_length); + + // Every worker will actually look at all regions, skipping over regions that + // are currently not committed. + // This also (potentially) iterates over regions newly allocated during GC. This + // is no problem except for some extra work. + for (uint count = 0; count < _allocated_heapregions_length; count++) { + const uint index = (start_index + count) % _allocated_heapregions_length; + assert(0 <= index && index < _allocated_heapregions_length, "sanity"); + // Skip over unavailable regions + if (!is_available(index)) { + continue; + } + HeapRegion* r = _regions.get_by_index(index); + // We'll ignore "continues humongous" regions (we'll process them + // when we come across their corresponding "start humongous" + // region) and regions already claimed. + if (r->claim_value() == claim_value || r->continuesHumongous()) { + continue; + } + // OK, try to claim it + if (!r->claimHeapRegion(claim_value)) { + continue; + } + // Success! + if (r->startsHumongous()) { + // If the region is "starts humongous" we'll iterate over its + // "continues humongous" first; in fact we'll do them + // first. The order is important. In one case, calling the + // closure on the "starts humongous" region might de-allocate + // and clear all its "continues humongous" regions and, as a + // result, we might end up processing them twice. So, we'll do + // them first (note: most closures will ignore them anyway) and + // then we'll do the "starts humongous" region. + for (uint ch_index = index + 1; ch_index < index + r->region_num(); ch_index++) { + HeapRegion* chr = _regions.get_by_index(ch_index); + + assert(chr->continuesHumongous(), "Must be humongous region"); + assert(chr->humongous_start_region() == r, + err_msg("Must work on humongous continuation of the original start region " + PTR_FORMAT ", but is " PTR_FORMAT, p2i(r), p2i(chr))); + assert(chr->claim_value() != claim_value, + "Must not have been claimed yet because claiming of humongous continuation first claims the start region"); + + bool claim_result = chr->claimHeapRegion(claim_value); + // We should always be able to claim it; no one else should + // be trying to claim this region. + guarantee(claim_result, "We should always be able to claim the continuesHumongous part of the humongous object"); + + bool res2 = blk->doHeapRegion(chr); + if (res2) { + return; + } + + // Right now, this holds (i.e., no closure that actually + // does something with "continues humongous" regions + // clears them). We might have to weaken it in the future, + // but let's leave these two asserts here for extra safety. + assert(chr->continuesHumongous(), "should still be the case"); + assert(chr->humongous_start_region() == r, "sanity"); + } + } + + bool res = blk->doHeapRegion(r); + if (res) { + return; + } + } +} + uint HeapRegionSeq::shrink_by(uint num_regions_to_remove) { - // Reset this in case it's currently pointing into the regions that - // we just removed. - _next_search_index = 0; - assert(length() > 0, "the region sequence should not be empty"); - assert(length() <= _allocated_length, "invariant"); - assert(_allocated_length > 0, "we should have at least one region committed"); + assert(length() <= _allocated_heapregions_length, "invariant"); + assert(_allocated_heapregions_length > 0, "we should have at least one region committed"); assert(num_regions_to_remove < length(), "We should never remove all regions"); - uint i = 0; - for (; i < num_regions_to_remove; i++) { - HeapRegion* cur = at(length() - 1); + if (num_regions_to_remove == 0) { + return 0; + } + + uint removed = 0; + uint cur = _allocated_heapregions_length - 1; + uint idx_last_found = 0; + uint num_last_found = 0; - if (!cur->is_empty()) { - // We have to give up if the region can not be moved - break; + while ((removed < num_regions_to_remove) && + (num_last_found = find_empty_from_idx_reverse(cur, &idx_last_found)) > 0) { + // Only allow uncommit from the end of the heap. + if ((idx_last_found + num_last_found) != _allocated_heapregions_length) { + return 0; + } + uint to_remove = MIN2(num_regions_to_remove - removed, num_last_found); + + uncommit_regions(idx_last_found + num_last_found - to_remove, to_remove); + + cur -= num_last_found; + removed += to_remove; } - assert(!cur->isHumongous(), "Humongous regions should not be empty"); - decrement_length(); - } - return i; + verify_optional(); + + return removed; } -#ifndef PRODUCT -void HeapRegionSeq::verify_optional() { - guarantee(length() <= _allocated_length, +uint HeapRegionSeq::find_empty_from_idx_reverse(uint start_idx, uint* res_idx) const { + guarantee(start_idx < _allocated_heapregions_length, "checking"); + guarantee(res_idx != NULL, "checking"); + + uint num_regions_found = 0; + + jlong cur = start_idx; + while (cur != -1 && !(is_available(cur) && at(cur)->is_empty())) { + cur--; + } + if (cur == -1) { + return num_regions_found; + } + jlong old_cur = cur; + // cur indexes the first empty region + while (cur != -1 && is_available(cur) && at(cur)->is_empty()) { + cur--; + } + *res_idx = cur + 1; + num_regions_found = old_cur - cur; + +#ifdef ASSERT + for (uint i = *res_idx; i < (*res_idx + num_regions_found); i++) { + assert(at(i)->is_empty(), "just checking"); + } +#endif + return num_regions_found; +} + +void HeapRegionSeq::verify() { + guarantee(length() <= _allocated_heapregions_length, err_msg("invariant: _length: %u _allocated_length: %u", - length(), _allocated_length)); - guarantee(_allocated_length <= max_length(), + length(), _allocated_heapregions_length)); + guarantee(_allocated_heapregions_length <= max_length(), err_msg("invariant: _allocated_length: %u _max_length: %u", - _allocated_length, max_length())); - guarantee(_next_search_index <= length(), - err_msg("invariant: _next_search_index: %u _length: %u", - _next_search_index, length())); + _allocated_heapregions_length, max_length())); + bool prev_committed = true; + uint num_committed = 0; HeapWord* prev_end = heap_bottom(); - for (uint i = 0; i < _allocated_length; i += 1) { + for (uint i = 0; i < _allocated_heapregions_length; i++) { + if (!is_available(i)) { + prev_committed = false; + continue; + } + num_committed++; HeapRegion* hr = _regions.get_by_index(i); guarantee(hr != NULL, err_msg("invariant: i: %u", i)); - guarantee(hr->bottom() == prev_end, + guarantee(!prev_committed || hr->bottom() == prev_end, err_msg("invariant i: %u "HR_FORMAT" prev_end: "PTR_FORMAT, i, HR_FORMAT_PARAMS(hr), p2i(prev_end))); guarantee(hr->hrs_index() == i, err_msg("invariant: i: %u hrs_index(): %u", i, hr->hrs_index())); - if (i < length()) { - // Asserts will fire if i is >= _length - HeapWord* addr = hr->bottom(); - guarantee(addr_to_region(addr) == hr, "sanity"); - guarantee(addr_to_region_unsafe(addr) == hr, "sanity"); - } else { - guarantee(hr->is_empty(), "sanity"); - guarantee(!hr->isHumongous(), "sanity"); - // using assert instead of guarantee here since containing_set() - // is only available in non-product builds. - assert(hr->containing_set() == NULL, "sanity"); - } + // Asserts will fire if i is >= _length + HeapWord* addr = hr->bottom(); + guarantee(addr_to_region(addr) == hr, "sanity"); + // We cannot check whether the region is part of a particular set: at the time + // this method may be called, we have only completed allocation of the regions, + // but not put into a region set. + prev_committed = true; if (hr->startsHumongous()) { prev_end = hr->orig_end(); } else { prev_end = hr->end(); } } - for (uint i = _allocated_length; i < max_length(); i += 1) { + for (uint i = _allocated_heapregions_length; i < max_length(); i++) { guarantee(_regions.get_by_index(i) == NULL, err_msg("invariant i: %u", i)); } + + guarantee(num_committed == _num_committed, err_msg("Found %u committed regions, but should be %u", num_committed, _num_committed)); + _free_list.verify(); +} + +#ifndef PRODUCT +void HeapRegionSeq::verify_optional() { + verify(); } #endif // PRODUCT +
--- a/src/share/vm/gc_implementation/g1/heapRegionSeq.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegionSeq.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -26,6 +26,8 @@ #define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSEQ_HPP #include "gc_implementation/g1/g1BiasedArray.hpp" +#include "gc_implementation/g1/g1RegionToSpaceMapper.hpp" +#include "gc_implementation/g1/heapRegionSet.hpp" class HeapRegion; class HeapRegionClosure; @@ -33,16 +35,20 @@ class G1HeapRegionTable : public G1BiasedMappedArray<HeapRegion*> { protected: - virtual HeapRegion* default_value() const { return NULL; } + virtual HeapRegion* default_value() const { return NULL; } }; -// This class keeps track of the region metadata (i.e., HeapRegion -// instances). They are kept in the _regions array in address -// order. A region's index in the array corresponds to its index in -// the heap (i.e., 0 is the region at the bottom of the heap, 1 is -// the one after it, etc.). Two regions that are consecutive in the -// array should also be adjacent in the address space (i.e., -// region(i).end() == region(i+1).bottom(). +// This class keeps track of the actual heap memory, auxiliary data +// and its metadata (i.e., HeapRegion instances) and the list of free regions. +// +// This allows maximum flexibility for deciding what to commit or uncommit given +// a request from outside. +// +// HeapRegions are kept in the _regions array in address order. A region's +// index in the array corresponds to its index in the heap (i.e., 0 is the +// region at the bottom of the heap, 1 is the one after it, etc.). Two +// regions that are consecutive in the array should also be adjacent in the +// address space (i.e., region(i).end() == region(i+1).bottom(). // // We create a HeapRegion when we commit the region's address space // for the first time. When we uncommit the address space of a @@ -51,56 +57,94 @@ // // We keep track of three lengths: // -// * _committed_length (returned by length()) is the number of currently -// committed regions. -// * _allocated_length (not exposed outside this class) is the -// number of regions for which we have HeapRegions. +// * _num_committed (returned by length()) is the number of currently +// committed regions. These may not be contiguous. +// * _allocated_heapregions_length (not exposed outside this class) is the +// number of regions+1 for which we have HeapRegions. // * max_length() returns the maximum number of regions the heap can have. // -// and maintain that: _committed_length <= _allocated_length <= max_length() class HeapRegionSeq: public CHeapObj<mtGC> { friend class VMStructs; G1HeapRegionTable _regions; - // The number of regions committed in the heap. - uint _committed_length; + G1RegionToSpaceMapper* _heap_mapper; + G1RegionToSpaceMapper* _prev_bitmap_mapper; + G1RegionToSpaceMapper* _next_bitmap_mapper; + G1RegionToSpaceMapper* _bot_mapper; + G1RegionToSpaceMapper* _cardtable_mapper; + G1RegionToSpaceMapper* _card_counts_mapper; - // A hint for which index to start searching from for humongous - // allocations. - uint _next_search_index; + FreeRegionList _free_list; + + // Each bit in this bitmap indicates that the corresponding region is available + // for allocation. + BitMap _available_map; + + // The number of regions committed in the heap. + uint _num_committed; - // The number of regions for which we have allocated HeapRegions for. - uint _allocated_length; + // Internal only. The highest heap region +1 we allocated a HeapRegion instance for. + uint _allocated_heapregions_length; + + HeapWord* heap_bottom() const { return _regions.bottom_address_mapped(); } + HeapWord* heap_end() const {return _regions.end_address_mapped(); } + + void make_regions_available(uint index, uint num_regions = 1); - // Find a contiguous set of empty regions of length num, starting - // from the given index. - uint find_contiguous_from(uint from, uint num); + // Pass down commit calls to the VirtualSpace. + void commit_regions(uint index, size_t num_regions = 1); + void uncommit_regions(uint index, size_t num_regions = 1); - void increment_allocated_length() { - assert(_allocated_length < max_length(), "pre-condition"); - _allocated_length++; - } + // Notify other data structures about change in the heap layout. + void update_committed_space(HeapWord* old_end, HeapWord* new_end); + // Calculate the starting region for each worker during parallel iteration so + // that they do not all start from the same region. + uint start_region_for_worker(uint worker_i, uint num_workers, uint num_regions) const; - void increment_length() { - assert(length() < max_length(), "pre-condition"); - _committed_length++; - } + // Find a contiguous set of empty or uncommitted regions of length num and return + // the index of the first region or G1_NO_HRS_INDEX if the search was unsuccessful. + // If only_empty is true, only empty regions are considered. + // Searches from bottom to top of the heap, doing a first-fit. + uint find_contiguous(size_t num, bool only_empty); + // Finds the next sequence of unavailable regions starting from start_idx. Returns the + // length of the sequence found. If this result is zero, no such sequence could be found, + // otherwise res_idx indicates the start index of these regions. + uint find_unavailable_from_idx(uint start_idx, uint* res_idx) const; + // Finds the next sequence of empty regions starting from start_idx, going backwards in + // the heap. Returns the length of the sequence found. If this value is zero, no + // sequence could be found, otherwise res_idx contains the start index of this range. + uint find_empty_from_idx_reverse(uint start_idx, uint* res_idx) const; + // Allocate a new HeapRegion for the given index. + HeapRegion* new_heap_region(uint hrs_index); +#ifdef ASSERT +public: + bool is_free(HeapRegion* hr) const; +#endif + // Returns whether the given region is available for allocation. + bool is_available(uint region) const; - void decrement_length() { - assert(length() > 0, "pre-condition"); - _committed_length--; - } + public: + // Empty constructor, we'll initialize it with the initialize() method. + HeapRegionSeq() : _regions(), _heap_mapper(NULL), _num_committed(0), + _next_bitmap_mapper(NULL), _prev_bitmap_mapper(NULL), _bot_mapper(NULL), + _allocated_heapregions_length(0), _available_map(), + _free_list("Free list", new MasterFreeRegionListMtSafeChecker()) + { } - HeapWord* heap_bottom() const { return _regions.bottom_address_mapped(); } - HeapWord* heap_end() const {return _regions.end_address_mapped(); } + void initialize(G1RegionToSpaceMapper* heap_storage, + G1RegionToSpaceMapper* prev_bitmap, + G1RegionToSpaceMapper* next_bitmap, + G1RegionToSpaceMapper* bot, + G1RegionToSpaceMapper* cardtable, + G1RegionToSpaceMapper* card_counts); - public: - // Empty contructor, we'll initialize it with the initialize() method. - HeapRegionSeq() : _regions(), _committed_length(0), _next_search_index(0), _allocated_length(0) { } - - void initialize(HeapWord* bottom, HeapWord* end); + // Return the "dummy" region used for G1AllocRegion. This is currently a hardwired + // new HeapRegion that owns HeapRegion at index 0. Since at the moment we commit + // the heap from the lowest address, this region (and its associated data + // structures) are available and we do not need to check further. + HeapRegion* get_dummy_region() { return new_heap_region(0); } // Return the HeapRegion at the given index. Assume that the index // is valid. @@ -110,47 +154,83 @@ // HeapRegion, otherwise return NULL. inline HeapRegion* addr_to_region(HeapWord* addr) const; - // Return the HeapRegion that corresponds to the given - // address. Assume the address is valid. - inline HeapRegion* addr_to_region_unsafe(HeapWord* addr) const; + // Insert the given region into the free region list. + inline void insert_into_free_list(HeapRegion* hr); + + // Insert the given region list into the global free region list. + void insert_list_into_free_list(FreeRegionList* list) { + _free_list.add_ordered(list); + } + + HeapRegion* allocate_free_region(bool is_old) { + HeapRegion* hr = _free_list.remove_region(is_old); + + if (hr != NULL) { + assert(hr->next() == NULL, "Single region should not have next"); + assert(is_available(hr->hrs_index()), "Must be committed"); + } + return hr; + } + + inline void allocate_free_regions_starting_at(uint first, uint num_regions); + + // Remove all regions from the free list. + void remove_all_free_regions() { + _free_list.remove_all(); + } + + // Return the number of committed free regions in the heap. + uint num_free_regions() const { + return _free_list.length(); + } + + size_t total_capacity_bytes() const { + return num_free_regions() * HeapRegion::GrainBytes; + } + + // Return the number of available (uncommitted) regions. + uint available() const { return max_length() - length(); } // Return the number of regions that have been committed in the heap. - uint length() const { return _committed_length; } + uint length() const { return _num_committed; } // Return the maximum number of regions in the heap. uint max_length() const { return (uint)_regions.length(); } - // Expand the sequence to reflect that the heap has grown from - // old_end to new_end. Either create new HeapRegions, or re-use - // existing ones, and return them in the given list. Returns the - // memory region that covers the newly-created regions. If a - // HeapRegion allocation fails, the result memory region might be - // smaller than the desired one. - MemRegion expand_by(HeapWord* old_end, HeapWord* new_end, - FreeRegionList* list); + MemRegion reserved() const { return MemRegion(heap_bottom(), heap_end()); } + + // Expand the sequence to reflect that the heap has grown. Either create new + // HeapRegions, or re-use existing ones. Returns the number of regions the + // sequence was expanded by. If a HeapRegion allocation fails, the resulting + // number of regions might be smaller than what's desired. + uint expand_by(uint num_regions); - // Return the number of contiguous regions at the end of the sequence - // that are available for allocation. - uint free_suffix(); + // Makes sure that the regions from start to start+num_regions-1 are available + // for allocation. Returns the number of regions that were committed to achieve + // this. + uint expand_at(uint start, uint num_regions); - // Find a contiguous set of empty regions of length num and return - // the index of the first region or G1_NULL_HRS_INDEX if the - // search was unsuccessful. - uint find_contiguous(uint num); + // Find a contiguous set of empty regions of length num. Returns the start index of + // that set, or G1_NO_HRS_INDEX. + uint find_contiguous_only_empty(size_t num) { return find_contiguous(num, true); } + // Find a contiguous set of empty or unavailable regions of length num. Returns the + // start index of that set, or G1_NO_HRS_INDEX. + uint find_contiguous_empty_or_unavailable(size_t num) { return find_contiguous(num, false); } + + HeapRegion* next_region_in_heap(const HeapRegion* r) const; // Apply blk->doHeapRegion() on all committed regions in address order, // terminating the iteration early if doHeapRegion() returns true. void iterate(HeapRegionClosure* blk) const; - // As above, but start the iteration from hr and loop around. If hr - // is NULL, we start from the first region in the heap. - void iterate_from(HeapRegion* hr, HeapRegionClosure* blk) const; + void par_iterate(HeapRegionClosure* blk, uint worker_id, uint no_of_par_workers, jint claim_value) const; - // Tag as uncommitted as many regions that are completely free as - // possible, up to num_regions_to_remove, from the suffix of the committed - // sequence. Return the actual number of removed regions. + // Uncommit up to num_regions_to_remove regions that are completely free. + // Return the actual number of uncommitted regions. uint shrink_by(uint num_regions_to_remove); + void verify(); + // Do some sanity checking. void verify_optional() PRODUCT_RETURN; };
--- a/src/share/vm/gc_implementation/g1/heapRegionSeq.inline.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegionSeq.inline.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -27,28 +27,32 @@ #include "gc_implementation/g1/heapRegion.hpp" #include "gc_implementation/g1/heapRegionSeq.hpp" +#include "gc_implementation/g1/heapRegionSet.inline.hpp" -inline HeapRegion* HeapRegionSeq::addr_to_region_unsafe(HeapWord* addr) const { +inline HeapRegion* HeapRegionSeq::addr_to_region(HeapWord* addr) const { + assert(addr < heap_end(), + err_msg("addr: "PTR_FORMAT" end: "PTR_FORMAT, p2i(addr), p2i(heap_end()))); + assert(addr >= heap_bottom(), + err_msg("addr: "PTR_FORMAT" bottom: "PTR_FORMAT, p2i(addr), p2i(heap_bottom()))); + HeapRegion* hr = _regions.get_by_address(addr); - assert(hr != NULL, "invariant"); return hr; } -inline HeapRegion* HeapRegionSeq::addr_to_region(HeapWord* addr) const { - if (addr != NULL && addr < heap_end()) { - assert(addr >= heap_bottom(), - err_msg("addr: " PTR_FORMAT " bottom: " PTR_FORMAT, p2i(addr), p2i(heap_bottom()))); - return addr_to_region_unsafe(addr); - } - return NULL; -} - inline HeapRegion* HeapRegionSeq::at(uint index) const { - assert(index < length(), "pre-condition"); + assert(is_available(index), "pre-condition"); HeapRegion* hr = _regions.get_by_index(index); assert(hr != NULL, "sanity"); assert(hr->hrs_index() == index, "sanity"); return hr; } +inline void HeapRegionSeq::insert_into_free_list(HeapRegion* hr) { + _free_list.add_ordered(hr); +} + +inline void HeapRegionSeq::allocate_free_regions_starting_at(uint first, uint num_regions) { + _free_list.remove_starting_at(at(first), num_regions); +} + #endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSEQ_INLINE_HPP
--- a/src/share/vm/gc_implementation/g1/heapRegionSet.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegionSet.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -23,6 +23,7 @@ */ #include "precompiled.hpp" +#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" #include "gc_implementation/g1/heapRegionRemSet.hpp" #include "gc_implementation/g1/heapRegionSet.inline.hpp" @@ -67,7 +68,7 @@ // Do the basic verification first before we do the checks over the regions. HeapRegionSetBase::verify(); - _verify_in_progress = true; + _verify_in_progress = true; } void HeapRegionSetBase::verify_end() { @@ -103,62 +104,7 @@ } void FreeRegionList::fill_in_ext_msg_extra(hrs_ext_msg* msg) { - msg->append(" hd: "PTR_FORMAT" tl: "PTR_FORMAT, head(), tail()); -} - -void FreeRegionList::add_as_head_or_tail(FreeRegionList* from_list, bool as_head) { - check_mt_safety(); - from_list->check_mt_safety(); - - verify_optional(); - from_list->verify_optional(); - - if (from_list->is_empty()) { - return; - } - -#ifdef ASSERT - FreeRegionListIterator iter(from_list); - while (iter.more_available()) { - HeapRegion* hr = iter.get_next(); - // In set_containing_set() we check that we either set the value - // from NULL to non-NULL or vice versa to catch bugs. So, we have - // to NULL it first before setting it to the value. - hr->set_containing_set(NULL); - hr->set_containing_set(this); - } -#endif // ASSERT - - if (_head == NULL) { - assert(length() == 0 && _tail == NULL, hrs_ext_msg(this, "invariant")); - _head = from_list->_head; - _tail = from_list->_tail; - } else { - assert(length() > 0 && _tail != NULL, hrs_ext_msg(this, "invariant")); - if (as_head) { - from_list->_tail->set_next(_head); - _head->set_prev(from_list->_tail); - _head = from_list->_head; - } else { - _tail->set_next(from_list->_head); - from_list->_head->set_prev(_tail); - _tail = from_list->_tail; - } - } - - _count.increment(from_list->length(), from_list->total_capacity_bytes()); - from_list->clear(); - - verify_optional(); - from_list->verify_optional(); -} - -void FreeRegionList::add_as_head(FreeRegionList* from_list) { - add_as_head_or_tail(from_list, true /* as_head */); -} - -void FreeRegionList::add_as_tail(FreeRegionList* from_list) { - add_as_head_or_tail(from_list, false /* as_head */); + msg->append(" hd: "PTR_FORMAT" tl: "PTR_FORMAT, _head, _tail); } void FreeRegionList::remove_all() { @@ -191,11 +137,6 @@ return; } - if (is_empty()) { - add_as_head(from_list); - return; - } - #ifdef ASSERT FreeRegionListIterator iter(from_list); while (iter.more_available()) { @@ -208,39 +149,45 @@ } #endif // ASSERT - HeapRegion* curr_to = _head; - HeapRegion* curr_from = from_list->_head; + if (is_empty()) { + assert(length() == 0 && _tail == NULL, hrs_ext_msg(this, "invariant")); + _head = from_list->_head; + _tail = from_list->_tail; + } else { + HeapRegion* curr_to = _head; + HeapRegion* curr_from = from_list->_head; + + while (curr_from != NULL) { + while (curr_to != NULL && curr_to->hrs_index() < curr_from->hrs_index()) { + curr_to = curr_to->next(); + } - while (curr_from != NULL) { - while (curr_to != NULL && curr_to->hrs_index() < curr_from->hrs_index()) { - curr_to = curr_to->next(); + if (curr_to == NULL) { + // The rest of the from list should be added as tail + _tail->set_next(curr_from); + curr_from->set_prev(_tail); + curr_from = NULL; + } else { + HeapRegion* next_from = curr_from->next(); + + curr_from->set_next(curr_to); + curr_from->set_prev(curr_to->prev()); + if (curr_to->prev() == NULL) { + _head = curr_from; + } else { + curr_to->prev()->set_next(curr_from); + } + curr_to->set_prev(curr_from); + + curr_from = next_from; + } } - if (curr_to == NULL) { - // The rest of the from list should be added as tail - _tail->set_next(curr_from); - curr_from->set_prev(_tail); - curr_from = NULL; - } else { - HeapRegion* next_from = curr_from->next(); - - curr_from->set_next(curr_to); - curr_from->set_prev(curr_to->prev()); - if (curr_to->prev() == NULL) { - _head = curr_from; - } else { - curr_to->prev()->set_next(curr_from); - } - curr_to->set_prev(curr_from); - - curr_from = next_from; + if (_tail->hrs_index() < from_list->_tail->hrs_index()) { + _tail = from_list->_tail; } } - if (_tail->hrs_index() < from_list->_tail->hrs_index()) { - _tail = from_list->_tail; - } - _count.increment(from_list->length(), from_list->total_capacity_bytes()); from_list->clear(); @@ -248,68 +195,59 @@ from_list->verify_optional(); } -void FreeRegionList::remove_all_pending(uint target_count) { +void FreeRegionList::remove_starting_at(HeapRegion* first, uint num_regions) { check_mt_safety(); - assert(target_count > 1, hrs_ext_msg(this, "pre-condition")); + assert(num_regions >= 1, hrs_ext_msg(this, "pre-condition")); assert(!is_empty(), hrs_ext_msg(this, "pre-condition")); verify_optional(); DEBUG_ONLY(uint old_length = length();) - HeapRegion* curr = _head; + HeapRegion* curr = first; uint count = 0; - while (curr != NULL) { + while (count < num_regions) { verify_region(curr); HeapRegion* next = curr->next(); HeapRegion* prev = curr->prev(); - if (curr->pending_removal()) { - assert(count < target_count, - hrs_err_msg("[%s] should not come across more regions " - "pending for removal than target_count: %u", - name(), target_count)); + assert(count < num_regions, + hrs_err_msg("[%s] should not come across more regions " + "pending for removal than num_regions: %u", + name(), num_regions)); - if (prev == NULL) { - assert(_head == curr, hrs_ext_msg(this, "invariant")); - _head = next; - } else { - assert(_head != curr, hrs_ext_msg(this, "invariant")); - prev->set_next(next); - } - if (next == NULL) { - assert(_tail == curr, hrs_ext_msg(this, "invariant")); - _tail = prev; - } else { - assert(_tail != curr, hrs_ext_msg(this, "invariant")); - next->set_prev(prev); - } - if (_last = curr) { - _last = NULL; - } + if (prev == NULL) { + assert(_head == curr, hrs_ext_msg(this, "invariant")); + _head = next; + } else { + assert(_head != curr, hrs_ext_msg(this, "invariant")); + prev->set_next(next); + } + if (next == NULL) { + assert(_tail == curr, hrs_ext_msg(this, "invariant")); + _tail = prev; + } else { + assert(_tail != curr, hrs_ext_msg(this, "invariant")); + next->set_prev(prev); + } + if (_last = curr) { + _last = NULL; + } - curr->set_next(NULL); - curr->set_prev(NULL); - remove(curr); - curr->set_pending_removal(false); - - count += 1; + curr->set_next(NULL); + curr->set_prev(NULL); + remove(curr); - // If we have come across the target number of regions we can - // just bail out. However, for debugging purposes, we can just - // carry on iterating to make sure there are not more regions - // tagged with pending removal. - DEBUG_ONLY(if (count == target_count) break;) - } + count++; curr = next; } - assert(count == target_count, - hrs_err_msg("[%s] count: %u should be == target_count: %u", - name(), count, target_count)); - assert(length() + target_count == old_length, + assert(count == num_regions, + hrs_err_msg("[%s] count: %u should be == num_regions: %u", + name(), count, num_regions)); + assert(length() + num_regions == old_length, hrs_err_msg("[%s] new length should be consistent " - "new length: %u old length: %u target_count: %u", - name(), length(), old_length, target_count)); + "new length: %u old length: %u num_regions: %u", + name(), length(), old_length, num_regions)); verify_optional(); } @@ -348,10 +286,12 @@ hr->print_on(out); } } + + out->cr(); } void FreeRegionList::verify_list() { - HeapRegion* curr = head(); + HeapRegion* curr = _head; HeapRegion* prev1 = NULL; HeapRegion* prev0 = NULL; uint count = 0; @@ -379,7 +319,7 @@ curr = curr->next(); } - guarantee(tail() == prev0, err_msg("Expected %s to end with %u but it ended with %u.", name(), tail()->hrs_index(), prev0->hrs_index())); + guarantee(_tail == prev0, err_msg("Expected %s to end with %u but it ended with %u.", name(), _tail->hrs_index(), prev0->hrs_index())); guarantee(_tail == NULL || _tail->next() == NULL, "_tail should not have a next"); guarantee(length() == count, err_msg("%s count mismatch. Expected %u, actual %u.", name(), length(), count)); guarantee(total_capacity_bytes() == capacity, err_msg("%s capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, @@ -463,3 +403,41 @@ "master humongous set MT safety protocol outside a safepoint"); } } + +void FreeRegionList_test() { + FreeRegionList l("test"); + + const uint num_regions_in_test = 5; + // Create a fake heap. It does not need to be valid, as the HeapRegion constructor + // does not access it. + MemRegion heap(NULL, num_regions_in_test * HeapRegion::GrainWords); + // Allocate a fake BOT because the HeapRegion constructor initializes + // the BOT. + size_t bot_size = G1BlockOffsetSharedArray::compute_size(heap.word_size()); + HeapWord* bot_data = NEW_C_HEAP_ARRAY(HeapWord, bot_size, mtGC); + ReservedSpace bot_rs(G1BlockOffsetSharedArray::compute_size(heap.word_size())); + G1RegionToSpaceMapper* bot_storage = + G1RegionToSpaceMapper::create_mapper(bot_rs, + os::vm_page_size(), + HeapRegion::GrainBytes, + G1BlockOffsetSharedArray::N_bytes, + mtGC); + G1BlockOffsetSharedArray oa(heap, bot_storage); + bot_storage->commit_regions(0, num_regions_in_test); + HeapRegion hr0(0, &oa, heap); + HeapRegion hr1(1, &oa, heap); + HeapRegion hr2(2, &oa, heap); + HeapRegion hr3(3, &oa, heap); + HeapRegion hr4(4, &oa, heap); + l.add_ordered(&hr1); + l.add_ordered(&hr0); + l.add_ordered(&hr3); + l.add_ordered(&hr4); + l.add_ordered(&hr2); + assert(l.length() == num_regions_in_test, "wrong length"); + l.verify_list(); + + bot_storage->uncommit_regions(0, num_regions_in_test); + delete bot_storage; + FREE_C_HEAP_ARRAY(HeapWord, bot_data, mtGC); +}
--- a/src/share/vm/gc_implementation/g1/heapRegionSet.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegionSet.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -162,7 +162,7 @@ // diagnosing failures. class hrs_ext_msg : public hrs_err_msg { public: - hrs_ext_msg(HeapRegionSetBase* set, const char* message) : hrs_err_msg("%s","") { + hrs_ext_msg(HeapRegionSetBase* set, const char* message) : hrs_err_msg("%s", "") { set->fill_in_ext_msg(this, message); } }; @@ -192,13 +192,9 @@ }; // A set that links all the regions added to it in a doubly-linked -// list. We should try to avoid doing operations that iterate over +// sorted list. We should try to avoid doing operations that iterate over // such lists in performance critical paths. Typically we should -// add / remove one region at a time or concatenate two lists. There are -// two ways to treat your lists, ordered and un-ordered. All un-ordered -// operations are done in constant time. To keep a list ordered only use -// add_ordered() to add elements to the list. If a list is not ordered -// from start, there is no way to sort it later. +// add / remove one region at a time or concatenate two lists. class FreeRegionListIterator; @@ -210,13 +206,13 @@ HeapRegion* _tail; // _last is used to keep track of where we added an element the last - // time in ordered lists. It helps to improve performance when adding - // several ordered items in a row. + // time. It helps to improve performance when adding several ordered items in a row. HeapRegion* _last; static uint _unrealistically_long_length; - void add_as_head_or_tail(FreeRegionList* from_list, bool as_head); + inline HeapRegion* remove_from_head_impl(); + inline HeapRegion* remove_from_tail_impl(); protected: virtual void fill_in_ext_msg_extra(hrs_ext_msg* msg); @@ -232,8 +228,11 @@ void verify_list(); - HeapRegion* head() { return _head; } - HeapRegion* tail() { return _tail; } +#ifdef ASSERT + bool contains(HeapRegion* hr) const { + return hr->containing_set() == this; + } +#endif static void set_unrealistically_long_length(uint len); @@ -242,55 +241,20 @@ // is determined by hrs_index. inline void add_ordered(HeapRegion* hr); - // It adds hr to the list as the new head. The region should not be - // a member of another set. - inline void add_as_head(HeapRegion* hr); - - // It adds hr to the list as the new tail. The region should not be - // a member of another set. - inline void add_as_tail(HeapRegion* hr); - - // It removes and returns the head of the list. It assumes that the - // list is not empty so it will return a non-NULL value. - inline HeapRegion* remove_head(); - - // Convenience method. - inline HeapRegion* remove_head_or_null(); - - // Removes and returns the last element (_tail) of the list. It assumes - // that the list isn't empty so that it can return a non-NULL value. - inline HeapRegion* remove_tail(); - - // Convenience method - inline HeapRegion* remove_tail_or_null(); - // Removes from head or tail based on the given argument. - inline HeapRegion* remove_region(bool from_head); + HeapRegion* remove_region(bool from_head); // Merge two ordered lists. The result is also ordered. The order is // determined by hrs_index. void add_ordered(FreeRegionList* from_list); - // It moves the regions from from_list to this list and empties - // from_list. The new regions will appear in the same order as they - // were in from_list and be linked in the beginning of this list. - void add_as_head(FreeRegionList* from_list); - - // It moves the regions from from_list to this list and empties - // from_list. The new regions will appear in the same order as they - // were in from_list and be linked in the end of this list. - void add_as_tail(FreeRegionList* from_list); - // It empties the list by removing all regions from it. void remove_all(); - // It removes all regions in the list that are pending for removal - // (i.e., they have been tagged with "pending_removal"). The list - // must not be empty, target_count should reflect the exact number - // of regions that are pending for removal in the list, and - // target_count should be > 1 (currently, we never need to remove a - // single region using this). - void remove_all_pending(uint target_count); + // Remove all (contiguous) regions from first to first + num_regions -1 from + // this list. + // Num_regions must be > 1. + void remove_starting_at(HeapRegion* first, uint num_regions); virtual void verify(); @@ -298,7 +262,7 @@ }; // Iterator class that provides a convenient way to iterate over the -// regions of a HeapRegionLinkedList instance. +// regions of a FreeRegionList. class FreeRegionListIterator : public StackObj { private: @@ -324,7 +288,7 @@ } FreeRegionListIterator(FreeRegionList* list) : _curr(NULL), _list(list) { - _curr = list->head(); + _curr = list->_head; } };
--- a/src/share/vm/gc_implementation/g1/heapRegionSet.inline.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegionSet.inline.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -30,7 +30,8 @@ inline void HeapRegionSetBase::add(HeapRegion* hr) { check_mt_safety(); assert(hr->containing_set() == NULL, hrs_ext_msg(this, "should not already have a containing set %u")); - assert(hr->next() == NULL && hr->prev() == NULL, hrs_ext_msg(this, "should not already be linked")); + assert(hr->next() == NULL, hrs_ext_msg(this, "should not already be linked")); + assert(hr->prev() == NULL, hrs_ext_msg(this, "should not already be linked")); _count.increment(1u, hr->capacity()); hr->set_containing_set(this); @@ -40,7 +41,8 @@ inline void HeapRegionSetBase::remove(HeapRegion* hr) { check_mt_safety(); verify_region(hr); - assert(hr->next() == NULL && hr->prev() == NULL, hrs_ext_msg(this, "should already be unlinked")); + assert(hr->next() == NULL, hrs_ext_msg(this, "should already be unlinked")); + assert(hr->prev() == NULL, hrs_ext_msg(this, "should already be unlinked")); hr->set_containing_set(NULL); assert(_count.length() > 0, hrs_ext_msg(this, "pre-condition")); @@ -48,8 +50,7 @@ } inline void FreeRegionList::add_ordered(HeapRegion* hr) { - check_mt_safety(); - assert((length() == 0 && _head == NULL && _tail == NULL) || + assert((length() == 0 && _head == NULL && _tail == NULL && _last == NULL) || (length() > 0 && _head != NULL && _tail != NULL), hrs_ext_msg(this, "invariant")); // add() will verify the region and check mt safety. @@ -95,55 +96,48 @@ _last = hr; } -inline void FreeRegionList::add_as_head(HeapRegion* hr) { - assert((length() == 0 && _head == NULL && _tail == NULL) || - (length() > 0 && _head != NULL && _tail != NULL), - hrs_ext_msg(this, "invariant")); - // add() will verify the region and check mt safety. - add(hr); - - // Now link the region. - if (_head != NULL) { - hr->set_next(_head); - _head->set_prev(hr); - } else { - _tail = hr; - } - _head = hr; -} - -inline void FreeRegionList::add_as_tail(HeapRegion* hr) { - check_mt_safety(); - assert((length() == 0 && _head == NULL && _tail == NULL) || - (length() > 0 && _head != NULL && _tail != NULL), - hrs_ext_msg(this, "invariant")); - // add() will verify the region and check mt safety. - add(hr); - - // Now link the region. - if (_tail != NULL) { - _tail->set_next(hr); - hr->set_prev(_tail); - } else { - _head = hr; - } - _tail = hr; -} - -inline HeapRegion* FreeRegionList::remove_head() { - assert(!is_empty(), hrs_ext_msg(this, "the list should not be empty")); - assert(length() > 0 && _head != NULL && _tail != NULL, - hrs_ext_msg(this, "invariant")); - - // We need to unlink it first. - HeapRegion* hr = _head; - _head = hr->next(); +inline HeapRegion* FreeRegionList::remove_from_head_impl() { + HeapRegion* result = _head; + _head = result->next(); if (_head == NULL) { _tail = NULL; } else { _head->set_prev(NULL); } - hr->set_next(NULL); + result->set_next(NULL); + return result; +} + +inline HeapRegion* FreeRegionList::remove_from_tail_impl() { + HeapRegion* result = _tail; + + _tail = result->prev(); + if (_tail == NULL) { + _head = NULL; + } else { + _tail->set_next(NULL); + } + result->set_prev(NULL); + return result; +} + +inline HeapRegion* FreeRegionList::remove_region(bool from_head) { + check_mt_safety(); + verify_optional(); + + if (is_empty()) { + return NULL; + } + assert(length() > 0 && _head != NULL && _tail != NULL, + hrs_ext_msg(this, "invariant")); + + HeapRegion* hr; + + if (from_head) { + hr = remove_from_head_impl(); + } else { + hr = remove_from_tail_impl(); + } if (_last == hr) { _last = NULL; @@ -154,56 +148,5 @@ return hr; } -inline HeapRegion* FreeRegionList::remove_head_or_null() { - check_mt_safety(); - if (!is_empty()) { - return remove_head(); - } else { - return NULL; - } -} - -inline HeapRegion* FreeRegionList::remove_tail() { - assert(!is_empty(), hrs_ext_msg(this, "The list should not be empty")); - assert(length() > 0 && _head != NULL && _tail != NULL, - hrs_ext_msg(this, "invariant")); - - // We need to unlink it first - HeapRegion* hr = _tail; - - _tail = hr->prev(); - if (_tail == NULL) { - _head = NULL; - } else { - _tail->set_next(NULL); - } - hr->set_prev(NULL); +#endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSET_INLINE_HPP - if (_last == hr) { - _last = NULL; - } - - // remove() will verify the region and check mt safety. - remove(hr); - return hr; -} - -inline HeapRegion* FreeRegionList::remove_tail_or_null() { - check_mt_safety(); - - if (!is_empty()) { - return remove_tail(); - } else { - return NULL; - } -} - -inline HeapRegion* FreeRegionList::remove_region(bool from_head) { - if (from_head) { - return remove_head_or_null(); - } else { - return remove_tail_or_null(); - } -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSET_INLINE_HPP
--- a/src/share/vm/gc_implementation/g1/vmStructs_g1.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/g1/vmStructs_g1.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -43,10 +43,9 @@ nonstatic_field(G1HeapRegionTable, _shift_by, uint) \ \ nonstatic_field(HeapRegionSeq, _regions, G1HeapRegionTable) \ - nonstatic_field(HeapRegionSeq, _committed_length, uint) \ + nonstatic_field(HeapRegionSeq, _num_committed, uint) \ \ nonstatic_field(G1CollectedHeap, _hrs, HeapRegionSeq) \ - nonstatic_field(G1CollectedHeap, _g1_committed, MemRegion) \ nonstatic_field(G1CollectedHeap, _summary_bytes_used, size_t) \ nonstatic_field(G1CollectedHeap, _g1mm, G1MonitoringSupport*) \ nonstatic_field(G1CollectedHeap, _old_set, HeapRegionSetBase) \
--- a/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -78,6 +78,7 @@ (HeapWord*)(heap_rs.base() + heap_rs.size())); CardTableExtension* const barrier_set = new CardTableExtension(_reserved, 3); + barrier_set->initialize(); _barrier_set = barrier_set; oopDesc::set_bs(_barrier_set); if (_barrier_set == NULL) {
--- a/src/share/vm/memory/cardTableModRefBS.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/memory/cardTableModRefBS.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -44,13 +44,6 @@ // enumerate ref fields that have been modified (since the last // enumeration.) -size_t CardTableModRefBS::cards_required(size_t covered_words) -{ - // Add one for a guard card, used to detect errors. - const size_t words = align_size_up(covered_words, card_size_in_words); - return words / card_size_in_words + 1; -} - size_t CardTableModRefBS::compute_byte_map_size() { assert(_guard_index == cards_required(_whole_heap.word_size()) - 1, @@ -64,27 +57,50 @@ int max_covered_regions): ModRefBarrierSet(max_covered_regions), _whole_heap(whole_heap), - _guard_index(cards_required(whole_heap.word_size()) - 1), - _last_valid_index(_guard_index - 1), + _guard_index(0), + _guard_region(), + _last_valid_index(0), _page_size(os::vm_page_size()), - _byte_map_size(compute_byte_map_size()) + _byte_map_size(0), + _covered(NULL), + _committed(NULL), + _cur_covered_regions(0), + _byte_map(NULL), + byte_map_base(NULL), + // LNC functionality + _lowest_non_clean(NULL), + _lowest_non_clean_chunk_size(NULL), + _lowest_non_clean_base_chunk_index(NULL), + _last_LNC_resizing_collection(NULL) { _kind = BarrierSet::CardTableModRef; + assert((uintptr_t(_whole_heap.start()) & (card_size - 1)) == 0, "heap must start at card boundary"); + assert((uintptr_t(_whole_heap.end()) & (card_size - 1)) == 0, "heap must end at card boundary"); + + assert(card_size <= 512, "card_size must be less than 512"); // why? + + _covered = new MemRegion[_max_covered_regions]; + if (_covered == NULL) { + vm_exit_during_initialization("Could not allocate card table covered region set."); + } +} + +void CardTableModRefBS::initialize() { + _guard_index = cards_required(_whole_heap.word_size()) - 1; + _last_valid_index = _guard_index - 1; + + _byte_map_size = compute_byte_map_size(); + HeapWord* low_bound = _whole_heap.start(); HeapWord* high_bound = _whole_heap.end(); - assert((uintptr_t(low_bound) & (card_size - 1)) == 0, "heap must start at card boundary"); - assert((uintptr_t(high_bound) & (card_size - 1)) == 0, "heap must end at card boundary"); - assert(card_size <= 512, "card_size must be less than 512"); // why? - - _covered = new MemRegion[max_covered_regions]; - _committed = new MemRegion[max_covered_regions]; - if (_covered == NULL || _committed == NULL) { - vm_exit_during_initialization("couldn't alloc card table covered region set."); + _cur_covered_regions = 0; + _committed = new MemRegion[_max_covered_regions]; + if (_committed == NULL) { + vm_exit_during_initialization("Could not allocate card table committed region set."); } - _cur_covered_regions = 0; const size_t rs_align = _page_size == (size_t) os::vm_page_size() ? 0 : MAX2(_page_size, (size_t) os::vm_allocation_granularity()); ReservedSpace heap_rs(_byte_map_size, rs_align, false); @@ -114,20 +130,20 @@ !ExecMem, "card table last card"); *guard_card = last_card; - _lowest_non_clean = - NEW_C_HEAP_ARRAY(CardArr, max_covered_regions, mtGC); + _lowest_non_clean = + NEW_C_HEAP_ARRAY(CardArr, _max_covered_regions, mtGC); _lowest_non_clean_chunk_size = - NEW_C_HEAP_ARRAY(size_t, max_covered_regions, mtGC); + NEW_C_HEAP_ARRAY(size_t, _max_covered_regions, mtGC); _lowest_non_clean_base_chunk_index = - NEW_C_HEAP_ARRAY(uintptr_t, max_covered_regions, mtGC); + NEW_C_HEAP_ARRAY(uintptr_t, _max_covered_regions, mtGC); _last_LNC_resizing_collection = - NEW_C_HEAP_ARRAY(int, max_covered_regions, mtGC); + NEW_C_HEAP_ARRAY(int, _max_covered_regions, mtGC); if (_lowest_non_clean == NULL || _lowest_non_clean_chunk_size == NULL || _lowest_non_clean_base_chunk_index == NULL || _last_LNC_resizing_collection == NULL) vm_exit_during_initialization("couldn't allocate an LNC array."); - for (int i = 0; i < max_covered_regions; i++) { + for (int i = 0; i < _max_covered_regions; i++) { _lowest_non_clean[i] = NULL; _lowest_non_clean_chunk_size[i] = 0; _last_LNC_resizing_collection[i] = -1; @@ -650,7 +666,7 @@ jbyte val, bool val_equals) { jbyte* start = byte_for(mr.start()); jbyte* end = byte_for(mr.last()); - bool failures = false; + bool failures = false; for (jbyte* curr = start; curr <= end; ++curr) { jbyte curr_val = *curr; bool failed = (val_equals) ? (curr_val != val) : (curr_val == val);
--- a/src/share/vm/memory/cardTableModRefBS.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/memory/cardTableModRefBS.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -96,12 +96,12 @@ // The declaration order of these const fields is important; see the // constructor before changing. const MemRegion _whole_heap; // the region covered by the card table - const size_t _guard_index; // index of very last element in the card + size_t _guard_index; // index of very last element in the card // table; it is set to a guard value // (last_card) and should never be modified - const size_t _last_valid_index; // index of the last valid element + size_t _last_valid_index; // index of the last valid element const size_t _page_size; // page size used when mapping _byte_map - const size_t _byte_map_size; // in bytes + size_t _byte_map_size; // in bytes jbyte* _byte_map; // the card marking array int _cur_covered_regions; @@ -123,7 +123,12 @@ protected: // Initialization utilities; covered_words is the size of the covered region // in, um, words. - inline size_t cards_required(size_t covered_words); + inline size_t cards_required(size_t covered_words) { + // Add one for a guard card, used to detect errors. + const size_t words = align_size_up(covered_words, card_size_in_words); + return words / card_size_in_words + 1; + } + inline size_t compute_byte_map_size(); // Finds and return the index of the region, if any, to which the given @@ -137,7 +142,7 @@ int find_covering_region_containing(HeapWord* addr); // Resize one of the regions covered by the remembered set. - void resize_covered_region(MemRegion new_region); + virtual void resize_covered_region(MemRegion new_region); // Returns the leftmost end of a committed region corresponding to a // covered region before covered region "ind", or else "NULL" if "ind" is @@ -282,6 +287,8 @@ CardTableModRefBS(MemRegion whole_heap, int max_covered_regions); ~CardTableModRefBS(); + virtual void initialize(); + // *** Barrier set functions. bool has_write_ref_pre_barrier() { return false; }
--- a/src/share/vm/memory/cardTableRS.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/memory/cardTableRS.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -53,6 +53,7 @@ #else _ct_bs = new CardTableModRefBSForCTRS(whole_heap, max_covered_regions); #endif + _ct_bs->initialize(); set_bs(_ct_bs); _last_cur_val_in_gen = NEW_C_HEAP_ARRAY3(jbyte, GenCollectedHeap::max_gens + 1, mtGC, 0, AllocFailStrategy::RETURN_NULL);
--- a/src/share/vm/opto/c2_globals.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/opto/c2_globals.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -653,9 +653,6 @@ product(bool, UseMathExactIntrinsics, true, \ "Enables intrinsification of various java.lang.Math functions") \ \ - experimental(bool, ReplaceInParentMaps, false, \ - "Propagate type improvements in callers of inlinee if possible") \ - \ product(bool, UseTypeSpeculation, true, \ "Speculatively propagate types from profiles") \ \
--- a/src/share/vm/opto/callGenerator.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/opto/callGenerator.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -63,12 +63,12 @@ } virtual bool is_parse() const { return true; } - virtual JVMState* generate(JVMState* jvms, Parse* parent_parser); + virtual JVMState* generate(JVMState* jvms); int is_osr() { return _is_osr; } }; -JVMState* ParseGenerator::generate(JVMState* jvms, Parse* parent_parser) { +JVMState* ParseGenerator::generate(JVMState* jvms) { Compile* C = Compile::current(); if (is_osr()) { @@ -80,7 +80,7 @@ return NULL; // bailing out of the compile; do not try to parse } - Parse parser(jvms, method(), _expected_uses, parent_parser); + Parse parser(jvms, method(), _expected_uses); // Grab signature for matching/allocation #ifdef ASSERT if (parser.tf() != (parser.depth() == 1 ? C->tf() : tf())) { @@ -119,12 +119,12 @@ _separate_io_proj(separate_io_proj) { } - virtual JVMState* generate(JVMState* jvms, Parse* parent_parser); + virtual JVMState* generate(JVMState* jvms); CallStaticJavaNode* call_node() const { return _call_node; } }; -JVMState* DirectCallGenerator::generate(JVMState* jvms, Parse* parent_parser) { +JVMState* DirectCallGenerator::generate(JVMState* jvms) { GraphKit kit(jvms); bool is_static = method()->is_static(); address target = is_static ? SharedRuntime::get_resolve_static_call_stub() @@ -171,10 +171,10 @@ vtable_index >= 0, "either invalid or usable"); } virtual bool is_virtual() const { return true; } - virtual JVMState* generate(JVMState* jvms, Parse* parent_parser); + virtual JVMState* generate(JVMState* jvms); }; -JVMState* VirtualCallGenerator::generate(JVMState* jvms, Parse* parent_parser) { +JVMState* VirtualCallGenerator::generate(JVMState* jvms) { GraphKit kit(jvms); Node* receiver = kit.argument(0); @@ -276,7 +276,7 @@ // Convert the CallStaticJava into an inline virtual void do_late_inline(); - virtual JVMState* generate(JVMState* jvms, Parse* parent_parser) { + virtual JVMState* generate(JVMState* jvms) { Compile *C = Compile::current(); C->print_inlining_skip(this); @@ -290,7 +290,7 @@ // that the late inlining logic can distinguish between fall // through and exceptional uses of the memory and io projections // as is done for allocations and macro expansion. - return DirectCallGenerator::generate(jvms, parent_parser); + return DirectCallGenerator::generate(jvms); } virtual void print_inlining_late(const char* msg) { @@ -389,7 +389,7 @@ } // Now perform the inling using the synthesized JVMState - JVMState* new_jvms = _inline_cg->generate(jvms, NULL); + JVMState* new_jvms = _inline_cg->generate(jvms); if (new_jvms == NULL) return; // no change if (C->failing()) return; @@ -407,7 +407,7 @@ C->env()->notice_inlined_method(_inline_cg->method()); C->set_inlining_progress(true); - kit.replace_call(call, result); + kit.replace_call(call, result, true); } @@ -429,8 +429,8 @@ virtual bool is_mh_late_inline() const { return true; } - virtual JVMState* generate(JVMState* jvms, Parse* parent_parser) { - JVMState* new_jvms = LateInlineCallGenerator::generate(jvms, parent_parser); + virtual JVMState* generate(JVMState* jvms) { + JVMState* new_jvms = LateInlineCallGenerator::generate(jvms); if (_input_not_const) { // inlining won't be possible so no need to enqueue right now. call_node()->set_generator(this); @@ -477,13 +477,13 @@ LateInlineStringCallGenerator(ciMethod* method, CallGenerator* inline_cg) : LateInlineCallGenerator(method, inline_cg) {} - virtual JVMState* generate(JVMState* jvms, Parse* parent_parser) { + virtual JVMState* generate(JVMState* jvms) { Compile *C = Compile::current(); C->print_inlining_skip(this); C->add_string_late_inline(this); - JVMState* new_jvms = DirectCallGenerator::generate(jvms, parent_parser); + JVMState* new_jvms = DirectCallGenerator::generate(jvms); return new_jvms; } @@ -500,13 +500,13 @@ LateInlineBoxingCallGenerator(ciMethod* method, CallGenerator* inline_cg) : LateInlineCallGenerator(method, inline_cg) {} - virtual JVMState* generate(JVMState* jvms, Parse* parent_parser) { + virtual JVMState* generate(JVMState* jvms) { Compile *C = Compile::current(); C->print_inlining_skip(this); C->add_boxing_late_inline(this); - JVMState* new_jvms = DirectCallGenerator::generate(jvms, parent_parser); + JVMState* new_jvms = DirectCallGenerator::generate(jvms); return new_jvms; } }; @@ -542,7 +542,7 @@ virtual bool is_virtual() const { return _is_virtual; } virtual bool is_deferred() const { return true; } - virtual JVMState* generate(JVMState* jvms, Parse* parent_parser); + virtual JVMState* generate(JVMState* jvms); }; @@ -552,12 +552,12 @@ return new WarmCallGenerator(ci, if_cold, if_hot); } -JVMState* WarmCallGenerator::generate(JVMState* jvms, Parse* parent_parser) { +JVMState* WarmCallGenerator::generate(JVMState* jvms) { Compile* C = Compile::current(); if (C->log() != NULL) { C->log()->elem("warm_call bci='%d'", jvms->bci()); } - jvms = _if_cold->generate(jvms, parent_parser); + jvms = _if_cold->generate(jvms); if (jvms != NULL) { Node* m = jvms->map()->control(); if (m->is_CatchProj()) m = m->in(0); else m = C->top(); @@ -618,7 +618,7 @@ virtual bool is_inline() const { return _if_hit->is_inline(); } virtual bool is_deferred() const { return _if_hit->is_deferred(); } - virtual JVMState* generate(JVMState* jvms, Parse* parent_parser); + virtual JVMState* generate(JVMState* jvms); }; @@ -630,7 +630,7 @@ } -JVMState* PredictedCallGenerator::generate(JVMState* jvms, Parse* parent_parser) { +JVMState* PredictedCallGenerator::generate(JVMState* jvms) { GraphKit kit(jvms); PhaseGVN& gvn = kit.gvn(); // We need an explicit receiver null_check before checking its type. @@ -648,6 +648,10 @@ return kit.transfer_exceptions_into_jvms(); } + // Make a copy of the replaced nodes in case we need to restore them + ReplacedNodes replaced_nodes = kit.map()->replaced_nodes(); + replaced_nodes.clone(); + Node* exact_receiver = receiver; // will get updated in place... Node* slow_ctl = kit.type_check_receiver(receiver, _predicted_receiver, _hit_prob, @@ -658,7 +662,7 @@ { PreserveJVMState pjvms(&kit); kit.set_control(slow_ctl); if (!kit.stopped()) { - slow_jvms = _if_missed->generate(kit.sync_jvms(), parent_parser); + slow_jvms = _if_missed->generate(kit.sync_jvms()); if (kit.failing()) return NULL; // might happen because of NodeCountInliningCutoff assert(slow_jvms != NULL, "must be"); @@ -679,12 +683,12 @@ kit.replace_in_map(receiver, exact_receiver); // Make the hot call: - JVMState* new_jvms = _if_hit->generate(kit.sync_jvms(), parent_parser); + JVMState* new_jvms = _if_hit->generate(kit.sync_jvms()); if (new_jvms == NULL) { // Inline failed, so make a direct call. assert(_if_hit->is_inline(), "must have been a failed inline"); CallGenerator* cg = CallGenerator::for_direct_call(_if_hit->method()); - new_jvms = cg->generate(kit.sync_jvms(), parent_parser); + new_jvms = cg->generate(kit.sync_jvms()); } kit.add_exception_states_from(new_jvms); kit.set_jvms(new_jvms); @@ -701,6 +705,11 @@ return kit.transfer_exceptions_into_jvms(); } + // There are 2 branches and the replaced nodes are only valid on + // one: restore the replaced nodes to what they were before the + // branch. + kit.map()->set_replaced_nodes(replaced_nodes); + // Finish the diamond. kit.C->set_has_split_ifs(true); // Has chance for split-if optimization RegionNode* region = new (kit.C) RegionNode(3); @@ -891,7 +900,7 @@ virtual bool is_inlined() const { return true; } virtual bool is_intrinsic() const { return true; } - virtual JVMState* generate(JVMState* jvms, Parse* parent_parser); + virtual JVMState* generate(JVMState* jvms); }; @@ -901,7 +910,7 @@ } -JVMState* PredicatedIntrinsicGenerator::generate(JVMState* jvms, Parse* parent_parser) { +JVMState* PredicatedIntrinsicGenerator::generate(JVMState* jvms) { // The code we want to generate here is: // if (receiver == NULL) // uncommon_Trap @@ -961,7 +970,7 @@ if (!kit.stopped()) { PreserveJVMState pjvms(&kit); // Generate intrinsic code: - JVMState* new_jvms = _intrinsic->generate(kit.sync_jvms(), parent_parser); + JVMState* new_jvms = _intrinsic->generate(kit.sync_jvms()); if (new_jvms == NULL) { // Intrinsic failed, use normal compilation path for this predicate. slow_region->add_req(kit.control()); @@ -986,7 +995,7 @@ PreserveJVMState pjvms(&kit); // Generate normal compilation code: kit.set_control(gvn.transform(slow_region)); - JVMState* new_jvms = _cg->generate(kit.sync_jvms(), parent_parser); + JVMState* new_jvms = _cg->generate(kit.sync_jvms()); if (kit.failing()) return NULL; // might happen because of NodeCountInliningCutoff assert(new_jvms != NULL, "must be"); @@ -1093,7 +1102,7 @@ virtual bool is_virtual() const { ShouldNotReachHere(); return false; } virtual bool is_trap() const { return true; } - virtual JVMState* generate(JVMState* jvms, Parse* parent_parser); + virtual JVMState* generate(JVMState* jvms); }; @@ -1105,7 +1114,7 @@ } -JVMState* UncommonTrapCallGenerator::generate(JVMState* jvms, Parse* parent_parser) { +JVMState* UncommonTrapCallGenerator::generate(JVMState* jvms) { GraphKit kit(jvms); // Take the trap with arguments pushed on the stack. (Cf. null_check_receiver). int nargs = method()->arg_size();
--- a/src/share/vm/opto/callGenerator.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/opto/callGenerator.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -31,8 +31,6 @@ #include "opto/type.hpp" #include "runtime/deoptimization.hpp" -class Parse; - //---------------------------CallGenerator------------------------------------- // The subclasses of this class handle generation of ideal nodes for // call sites and method entry points. @@ -112,7 +110,7 @@ // // If the result is NULL, it means that this CallGenerator was unable // to handle the given call, and another CallGenerator should be consulted. - virtual JVMState* generate(JVMState* jvms, Parse* parent_parser) = 0; + virtual JVMState* generate(JVMState* jvms) = 0; // How to generate a call site that is inlined: static CallGenerator* for_inline(ciMethod* m, float expected_uses = -1);
--- a/src/share/vm/opto/callnode.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/opto/callnode.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -1089,6 +1089,7 @@ #ifndef PRODUCT void SafePointNode::dump_spec(outputStream *st) const { st->print(" SafePoint "); + _replaced_nodes.dump(st); } #endif
--- a/src/share/vm/opto/callnode.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/opto/callnode.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -30,6 +30,7 @@ #include "opto/multnode.hpp" #include "opto/opcodes.hpp" #include "opto/phaseX.hpp" +#include "opto/replacednodes.hpp" #include "opto/type.hpp" // Portions of code courtesy of Clifford Click @@ -335,6 +336,7 @@ OopMap* _oop_map; // Array of OopMap info (8-bit char) for GC JVMState* const _jvms; // Pointer to list of JVM State objects const TypePtr* _adr_type; // What type of memory does this node produce? + ReplacedNodes _replaced_nodes; // During parsing: list of pair of nodes from calls to GraphKit::replace_in_map() // Many calls take *all* of memory as input, // but some produce a limited subset of that memory as output. @@ -426,6 +428,37 @@ void set_next_exception(SafePointNode* n); bool has_exceptions() const { return next_exception() != NULL; } + // Helper methods to operate on replaced nodes + ReplacedNodes replaced_nodes() const { + return _replaced_nodes; + } + + void set_replaced_nodes(ReplacedNodes replaced_nodes) { + _replaced_nodes = replaced_nodes; + } + + void clone_replaced_nodes() { + _replaced_nodes.clone(); + } + void record_replaced_node(Node* initial, Node* improved) { + _replaced_nodes.record(initial, improved); + } + void transfer_replaced_nodes_from(SafePointNode* sfpt, uint idx = 0) { + _replaced_nodes.transfer_from(sfpt->_replaced_nodes, idx); + } + void delete_replaced_nodes() { + _replaced_nodes.reset(); + } + void apply_replaced_nodes() { + _replaced_nodes.apply(this); + } + void merge_replaced_nodes_with(SafePointNode* sfpt) { + _replaced_nodes.merge_with(sfpt->_replaced_nodes); + } + bool has_replaced_nodes() const { + return !_replaced_nodes.is_empty(); + } + // Standard Node stuff virtual int Opcode() const; virtual bool pinned() const { return true; }
--- a/src/share/vm/opto/compile.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/opto/compile.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -391,6 +391,11 @@ uint next = 0; while (next < useful.size()) { Node *n = useful.at(next++); + if (n->is_SafePoint()) { + // We're done with a parsing phase. Replaced nodes are not valid + // beyond that point. + n->as_SafePoint()->delete_replaced_nodes(); + } // Use raw traversal of out edges since this code removes out edges int max = n->outcnt(); for (int j = 0; j < max; ++j) { @@ -670,7 +675,6 @@ _inlining_incrementally(false), _print_inlining_list(NULL), _print_inlining_idx(0), - _preserve_jvm_state(0), _interpreter_frame_size(0) { C = this; @@ -782,7 +786,7 @@ return; } JVMState* jvms = build_start_state(start(), tf()); - if ((jvms = cg->generate(jvms, NULL)) == NULL) { + if ((jvms = cg->generate(jvms)) == NULL) { record_method_not_compilable("method parse failed"); return; } @@ -977,7 +981,6 @@ _inlining_incrementally(false), _print_inlining_list(NULL), _print_inlining_idx(0), - _preserve_jvm_state(0), _allowed_reasons(0), _interpreter_frame_size(0) { C = this; @@ -1910,6 +1913,8 @@ for_igvn()->clear(); gvn->replace_with(&igvn); + _late_inlines_pos = _late_inlines.length(); + while (_boxing_late_inlines.length() > 0) { CallGenerator* cg = _boxing_late_inlines.pop(); cg->do_late_inline(); @@ -1973,8 +1978,8 @@ if (live_nodes() > (uint)LiveNodeCountInliningCutoff) { if (low_live_nodes < (uint)LiveNodeCountInliningCutoff * 8 / 10) { // PhaseIdealLoop is expensive so we only try it once we are - // out of loop and we only try it again if the previous helped - // got the number of nodes down significantly + // out of live nodes and we only try it again if the previous + // helped got the number of nodes down significantly PhaseIdealLoop ideal_loop( igvn, false, true ); if (failing()) return; low_live_nodes = live_nodes(); @@ -2066,6 +2071,10 @@ // Inline valueOf() methods now. inline_boxing_calls(igvn); + if (AlwaysIncrementalInline) { + inline_incrementally(igvn); + } + print_method(PHASE_INCREMENTAL_BOXING_INLINE, 2); if (failing()) return;
--- a/src/share/vm/opto/compile.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/opto/compile.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -429,9 +429,6 @@ // Remove the speculative part of types and clean up the graph void remove_speculative_types(PhaseIterGVN &igvn); - // Are we within a PreserveJVMState block? - int _preserve_jvm_state; - void* _replay_inline_data; // Pointer to data loaded from file public: @@ -1196,21 +1193,6 @@ // Auxiliary method for randomized fuzzing/stressing static bool randomized_select(int count); - - // enter a PreserveJVMState block - void inc_preserve_jvm_state() { - _preserve_jvm_state++; - } - - // exit a PreserveJVMState block - void dec_preserve_jvm_state() { - _preserve_jvm_state--; - assert(_preserve_jvm_state >= 0, "_preserve_jvm_state shouldn't be negative"); - } - - bool has_preserve_jvm_state() const { - return _preserve_jvm_state > 0; - } }; #endif // SHARE_VM_OPTO_COMPILE_HPP
--- a/src/share/vm/opto/doCall.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/opto/doCall.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -523,7 +523,7 @@ // because exceptions don't return to the call site.) profile_call(receiver); - JVMState* new_jvms = cg->generate(jvms, this); + JVMState* new_jvms = cg->generate(jvms); if (new_jvms == NULL) { // When inlining attempt fails (e.g., too many arguments), // it may contaminate the current compile state, making it @@ -537,7 +537,7 @@ // intrinsic was expecting to optimize. Should always be possible to // get a normal java call that may inline in that case cg = C->call_generator(cg->method(), vtable_index, call_does_dispatch, jvms, try_inline, prof_factor(), speculative_receiver_type, /* allow_intrinsics= */ false); - if ((new_jvms = cg->generate(jvms, this)) == NULL) { + if ((new_jvms = cg->generate(jvms)) == NULL) { guarantee(failing(), "call failed to generate: calls should work"); return; }
--- a/src/share/vm/opto/graphKit.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/opto/graphKit.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -428,6 +428,7 @@ } } } + phi_map->merge_replaced_nodes_with(ex_map); } //--------------------------use_exception_state-------------------------------- @@ -641,7 +642,6 @@ _map = kit->map(); // preserve the map _sp = kit->sp(); kit->set_map(clone_map ? kit->clone_map() : NULL); - Compile::current()->inc_preserve_jvm_state(); #ifdef ASSERT _bci = kit->bci(); Parse* parser = kit->is_Parse(); @@ -659,7 +659,6 @@ #endif kit->set_map(_map); kit->set_sp(_sp); - Compile::current()->dec_preserve_jvm_state(); } @@ -1398,60 +1397,17 @@ // on the map. This includes locals, stack, and monitors // of the current (innermost) JVM state. - if (!ReplaceInParentMaps) { - return; - } - - // PreserveJVMState doesn't do a deep copy so we can't modify - // parents - if (Compile::current()->has_preserve_jvm_state()) { + // don't let inconsistent types from profiling escape this + // method + + const Type* told = _gvn.type(old); + const Type* tnew = _gvn.type(neww); + + if (!tnew->higher_equal(told)) { return; } - Parse* parser = is_Parse(); - bool progress = true; - Node* ctrl = map()->in(0); - // Follow the chain of parsers and see whether the update can be - // done in the map of callers. We can do the replace for a caller if - // the current control post dominates the control of a caller. - while (parser != NULL && parser->caller() != NULL && progress) { - progress = false; - Node* parent_map = parser->caller()->map(); - assert(parser->exits().map()->jvms()->depth() == parser->caller()->depth(), "map mismatch"); - - Node* parent_ctrl = parent_map->in(0); - - while (parent_ctrl->is_Region()) { - Node* n = parent_ctrl->as_Region()->is_copy(); - if (n == NULL) { - break; - } - parent_ctrl = n; - } - - for (;;) { - if (ctrl == parent_ctrl) { - // update the map of the exits which is the one that will be - // used when compilation resume after inlining - parser->exits().map()->replace_edge(old, neww); - progress = true; - break; - } - if (ctrl->is_Proj() && ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) { - ctrl = ctrl->in(0)->in(0); - } else if (ctrl->is_Region()) { - Node* n = ctrl->as_Region()->is_copy(); - if (n == NULL) { - break; - } - ctrl = n; - } else { - break; - } - } - - parser = parser->parent_parser(); - } + map()->record_replaced_node(old, neww); } @@ -1855,12 +1811,16 @@ // Replace the call with the current state of the kit. -void GraphKit::replace_call(CallNode* call, Node* result) { +void GraphKit::replace_call(CallNode* call, Node* result, bool do_replaced_nodes) { JVMState* ejvms = NULL; if (has_exceptions()) { ejvms = transfer_exceptions_into_jvms(); } + ReplacedNodes replaced_nodes = map()->replaced_nodes(); + ReplacedNodes replaced_nodes_exception; + Node* ex_ctl = top(); + SafePointNode* final_state = stop(); // Find all the needed outputs of this call @@ -1877,6 +1837,10 @@ C->gvn_replace_by(callprojs.fallthrough_catchproj, final_ctl); } if (callprojs.fallthrough_memproj != NULL) { + if (final_mem->is_MergeMem()) { + // Parser's exits MergeMem was not transformed but may be optimized + final_mem = _gvn.transform(final_mem); + } C->gvn_replace_by(callprojs.fallthrough_memproj, final_mem); } if (callprojs.fallthrough_ioproj != NULL) { @@ -1908,10 +1872,13 @@ // Load my combined exception state into the kit, with all phis transformed: SafePointNode* ex_map = ekit.combine_and_pop_all_exception_states(); + replaced_nodes_exception = ex_map->replaced_nodes(); Node* ex_oop = ekit.use_exception_state(ex_map); + if (callprojs.catchall_catchproj != NULL) { C->gvn_replace_by(callprojs.catchall_catchproj, ekit.control()); + ex_ctl = ekit.control(); } if (callprojs.catchall_memproj != NULL) { C->gvn_replace_by(callprojs.catchall_memproj, ekit.reset_memory()); @@ -1944,6 +1911,13 @@ _gvn.transform(wl.pop()); } } + + if (callprojs.fallthrough_catchproj != NULL && !final_ctl->is_top() && do_replaced_nodes) { + replaced_nodes.apply(C, final_ctl); + } + if (!ex_ctl->is_top() && do_replaced_nodes) { + replaced_nodes_exception.apply(C, ex_ctl); + } }
--- a/src/share/vm/opto/graphKit.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/opto/graphKit.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -685,7 +685,7 @@ // Replace the call with the current state of the kit. Requires // that the call was generated with separate io_projs so that // exceptional control flow can be handled properly. - void replace_call(CallNode* call, Node* result); + void replace_call(CallNode* call, Node* result, bool do_replaced_nodes = false); // helper functions for statistics void increment_counter(address counter_addr); // increment a debug counter
--- a/src/share/vm/opto/ifnode.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/opto/ifnode.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -503,7 +503,7 @@ jint off = 0; if (l->is_top()) { return 0; - } else if (l->is_Add()) { + } else if (l->Opcode() == Op_AddI) { if ((off = l->in(1)->find_int_con(0)) != 0) { ind = l->in(2); } else if ((off = l->in(2)->find_int_con(0)) != 0) {
--- a/src/share/vm/opto/library_call.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/opto/library_call.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -66,7 +66,7 @@ virtual bool is_predicated() const { return _predicates_count > 0; } virtual int predicates_count() const { return _predicates_count; } virtual bool does_virtual_dispatch() const { return _does_virtual_dispatch; } - virtual JVMState* generate(JVMState* jvms, Parse* parent_parser); + virtual JVMState* generate(JVMState* jvms); virtual Node* generate_predicate(JVMState* jvms, int predicate); vmIntrinsics::ID intrinsic_id() const { return _intrinsic_id; } }; @@ -614,7 +614,7 @@ // Nothing to do here. } -JVMState* LibraryIntrinsic::generate(JVMState* jvms, Parse* parent_parser) { +JVMState* LibraryIntrinsic::generate(JVMState* jvms) { LibraryCallKit kit(jvms, this); Compile* C = kit.C; int nodes = C->unique();
--- a/src/share/vm/opto/node.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/opto/node.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -527,6 +527,9 @@ if (n->is_Call()) { n->as_Call()->clone_jvms(C); } + if (n->is_SafePoint()) { + n->as_SafePoint()->clone_replaced_nodes(); + } return n; // Return the clone } @@ -622,6 +625,9 @@ if (is_expensive()) { compile->remove_expensive_node(this); } + if (is_SafePoint()) { + as_SafePoint()->delete_replaced_nodes(); + } #ifdef ASSERT // We will not actually delete the storage, but we'll make the node unusable. *(address*)this = badAddress; // smash the C++ vtbl, probably
--- a/src/share/vm/opto/parse.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/opto/parse.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -357,12 +357,13 @@ int _est_switch_depth; // Debugging SwitchRanges. #endif - // parser for the caller of the method of this object - Parse* const _parent; + bool _first_return; // true if return is the first to be parsed + bool _replaced_nodes_for_exceptions; // needs processing of replaced nodes in exception paths? + uint _new_idx; // any node with _idx above were new during this parsing. Used to trim the replaced nodes list. public: // Constructor - Parse(JVMState* caller, ciMethod* parse_method, float expected_uses, Parse* parent); + Parse(JVMState* caller, ciMethod* parse_method, float expected_uses); virtual Parse* is_Parse() const { return (Parse*)this; } @@ -419,8 +420,6 @@ return block()->successor_for_bci(bci); } - Parse* parent_parser() const { return _parent; } - private: // Create a JVMS & map for the initial state of this method. SafePointNode* create_entry_map();
--- a/src/share/vm/opto/parse1.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/opto/parse1.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -381,8 +381,8 @@ //------------------------------Parse------------------------------------------ // Main parser constructor. -Parse::Parse(JVMState* caller, ciMethod* parse_method, float expected_uses, Parse* parent) - : _exits(caller), _parent(parent) +Parse::Parse(JVMState* caller, ciMethod* parse_method, float expected_uses) + : _exits(caller) { // Init some variables _caller = caller; @@ -395,6 +395,9 @@ _entry_bci = InvocationEntryBci; _tf = NULL; _block = NULL; + _first_return = true; + _replaced_nodes_for_exceptions = false; + _new_idx = C->unique(); debug_only(_block_count = -1); debug_only(_blocks = (Block*)-1); #ifndef PRODUCT @@ -895,6 +898,10 @@ for (uint i = 0; i < TypeFunc::Parms; i++) { caller.map()->set_req(i, ex_map->in(i)); } + if (ex_map->has_replaced_nodes()) { + _replaced_nodes_for_exceptions = true; + } + caller.map()->transfer_replaced_nodes_from(ex_map, _new_idx); // ...and the exception: Node* ex_oop = saved_ex_oop(ex_map); SafePointNode* caller_ex_map = caller.make_exception_state(ex_oop); @@ -963,7 +970,7 @@ bool do_synch = method()->is_synchronized() && GenerateSynchronizationCode; // record exit from a method if compiled while Dtrace is turned on. - if (do_synch || C->env()->dtrace_method_probes()) { + if (do_synch || C->env()->dtrace_method_probes() || _replaced_nodes_for_exceptions) { // First move the exception list out of _exits: GraphKit kit(_exits.transfer_exceptions_into_jvms()); SafePointNode* normal_map = kit.map(); // keep this guy safe @@ -988,6 +995,9 @@ if (C->env()->dtrace_method_probes()) { kit.make_dtrace_method_exit(method()); } + if (_replaced_nodes_for_exceptions) { + kit.map()->apply_replaced_nodes(); + } // Done with exception-path processing. ex_map = kit.make_exception_state(ex_oop); assert(ex_jvms->same_calls_as(ex_map->jvms()), "sanity"); @@ -1007,6 +1017,7 @@ _exits.add_exception_state(ex_map); } } + _exits.map()->apply_replaced_nodes(); } //-----------------------------create_entry_map------------------------------- @@ -1021,6 +1032,9 @@ return NULL; } + // clear current replaced nodes that are of no use from here on (map was cloned in build_exits). + _caller->map()->delete_replaced_nodes(); + // If this is an inlined method, we may have to do a receiver null check. if (_caller->has_method() && is_normal_parse() && !method()->is_static()) { GraphKit kit(_caller); @@ -1044,6 +1058,8 @@ SafePointNode* inmap = _caller->map(); assert(inmap != NULL, "must have inmap"); + // In case of null check on receiver above + map()->transfer_replaced_nodes_from(inmap, _new_idx); uint i; @@ -1673,6 +1689,8 @@ set_control(r->nonnull_req()); } + map()->merge_replaced_nodes_with(newin); + // newin has been subsumed into the lazy merge, and is now dead. set_block(save_block); @@ -2077,6 +2095,13 @@ phi->add_req(value); } + if (_first_return) { + _exits.map()->transfer_replaced_nodes_from(map(), _new_idx); + _first_return = false; + } else { + _exits.map()->merge_replaced_nodes_with(map()); + } + stop_and_kill_map(); // This CFG path dies here }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/opto/replacednodes.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "opto/cfgnode.hpp" +#include "opto/phaseX.hpp" +#include "opto/replacednodes.hpp" + +void ReplacedNodes::allocate_if_necessary() { + if (_replaced_nodes == NULL) { + _replaced_nodes = new GrowableArray<ReplacedNode>(); + } +} + +bool ReplacedNodes::is_empty() const { + return _replaced_nodes == NULL || _replaced_nodes->length() == 0; +} + +bool ReplacedNodes::has_node(const ReplacedNode& r) const { + return _replaced_nodes->find(r) != -1; +} + +bool ReplacedNodes::has_target_node(Node* n) const { + for (int i = 0; i < _replaced_nodes->length(); i++) { + if (_replaced_nodes->at(i).improved() == n) { + return true; + } + } + return false; +} + +// Record replaced node if not seen before +void ReplacedNodes::record(Node* initial, Node* improved) { + allocate_if_necessary(); + ReplacedNode r(initial, improved); + if (!has_node(r)) { + _replaced_nodes->push(r); + } +} + +// Copy replaced nodes from one map to another. idx is used to +// identify nodes that are too new to be of interest in the target +// node list. +void ReplacedNodes::transfer_from(const ReplacedNodes& other, uint idx) { + if (other.is_empty()) { + return; + } + allocate_if_necessary(); + for (int i = 0; i < other._replaced_nodes->length(); i++) { + ReplacedNode replaced = other._replaced_nodes->at(i); + // Only transfer the nodes that can actually be useful + if (!has_node(replaced) && (replaced.initial()->_idx < idx || has_target_node(replaced.initial()))) { + _replaced_nodes->push(replaced); + } + } +} + +void ReplacedNodes::clone() { + if (_replaced_nodes != NULL) { + GrowableArray<ReplacedNode>* replaced_nodes_clone = new GrowableArray<ReplacedNode>(); + replaced_nodes_clone->appendAll(_replaced_nodes); + _replaced_nodes = replaced_nodes_clone; + } +} + +void ReplacedNodes::reset() { + if (_replaced_nodes != NULL) { + _replaced_nodes->clear(); + } +} + +// Perfom node replacement (used when returning to caller) +void ReplacedNodes::apply(Node* n) { + if (is_empty()) { + return; + } + for (int i = 0; i < _replaced_nodes->length(); i++) { + ReplacedNode replaced = _replaced_nodes->at(i); + n->replace_edge(replaced.initial(), replaced.improved()); + } +} + +static void enqueue_use(Node* n, Node* use, Unique_Node_List& work) { + if (use->is_Phi()) { + Node* r = use->in(0); + assert(r->is_Region(), "Phi should have Region"); + for (uint i = 1; i < use->req(); i++) { + if (use->in(i) == n) { + work.push(r->in(i)); + } + } + } else { + work.push(use); + } +} + +// Perfom node replacement following late inlining +void ReplacedNodes::apply(Compile* C, Node* ctl) { + // ctl is the control on exit of the method that was late inlined + if (is_empty()) { + return; + } + for (int i = 0; i < _replaced_nodes->length(); i++) { + ReplacedNode replaced = _replaced_nodes->at(i); + Node* initial = replaced.initial(); + Node* improved = replaced.improved(); + assert (ctl != NULL && !ctl->is_top(), "replaced node should have actual control"); + + ResourceMark rm; + Unique_Node_List work; + // Go over all the uses of the node that is considered for replacement... + for (DUIterator j = initial->outs(); initial->has_out(j); j++) { + Node* use = initial->out(j); + + if (use == improved || use->outcnt() == 0) { + continue; + } + work.clear(); + enqueue_use(initial, use, work); + bool replace = true; + // Check that this use is dominated by ctl. Go ahead with the + // replacement if it is. + while (work.size() != 0 && replace) { + Node* n = work.pop(); + if (use->outcnt() == 0) { + continue; + } + if (n->is_CFG() || (n->in(0) != NULL && !n->in(0)->is_top())) { + int depth = 0; + Node *m = n; + if (!n->is_CFG()) { + n = n->in(0); + } + assert(n->is_CFG(), "should be CFG now"); + while(n != ctl) { + n = IfNode::up_one_dom(n); + depth++; + // limit search depth + if (depth >= 100 || n == NULL) { + replace = false; + break; + } + } + } else { + for (DUIterator k = n->outs(); n->has_out(k); k++) { + enqueue_use(n, n->out(k), work); + } + } + } + if (replace) { + bool is_in_table = C->initial_gvn()->hash_delete(use); + int replaced = use->replace_edge(initial, improved); + if (is_in_table) { + C->initial_gvn()->hash_find_insert(use); + } + C->record_for_igvn(use); + + assert(replaced > 0, "inconsistent"); + --j; + } + } + } +} + +void ReplacedNodes::dump(outputStream *st) const { + if (!is_empty()) { + tty->print("replaced nodes: "); + for (int i = 0; i < _replaced_nodes->length(); i++) { + tty->print("%d->%d", _replaced_nodes->at(i).initial()->_idx, _replaced_nodes->at(i).improved()->_idx); + if (i < _replaced_nodes->length()-1) { + tty->print(","); + } + } + } +} + +// Merge 2 list of replaced node at a point where control flow paths merge +void ReplacedNodes::merge_with(const ReplacedNodes& other) { + if (is_empty()) { + return; + } + if (other.is_empty()) { + reset(); + return; + } + int shift = 0; + int len = _replaced_nodes->length(); + for (int i = 0; i < len; i++) { + if (!other.has_node(_replaced_nodes->at(i))) { + shift++; + } else if (shift > 0) { + _replaced_nodes->at_put(i-shift, _replaced_nodes->at(i)); + } + } + if (shift > 0) { + _replaced_nodes->trunc_to(len - shift); + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/opto/replacednodes.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_OPTO_REPLACEDNODES_HPP +#define SHARE_VM_OPTO_REPLACEDNODES_HPP + +#include "opto/connode.hpp" + +// During parsing, when a node is "improved", +// GraphKit::replace_in_map() is called to update the current map so +// that the improved node is used from that point +// on. GraphKit::replace_in_map() doesn't operate on the callers maps +// and so some optimization opportunities may be lost. The +// ReplacedNodes class addresses that problem. +// +// A ReplacedNodes object is a list of pair of nodes. Every +// SafePointNode carries a ReplacedNodes object. Every time +// GraphKit::replace_in_map() is called, a new pair of nodes is pushed +// on the list of replaced nodes. When control flow paths merge, their +// replaced nodes are also merged. When parsing exits a method to +// return to a caller, the replaced nodes on the exit path are used to +// update the caller's map. +class ReplacedNodes VALUE_OBJ_CLASS_SPEC { + private: + class ReplacedNode VALUE_OBJ_CLASS_SPEC { + private: + Node* _initial; + Node* _improved; + public: + ReplacedNode() : _initial(NULL), _improved(NULL) {} + ReplacedNode(Node* initial, Node* improved) : _initial(initial), _improved(improved) {} + Node* initial() const { return _initial; } + Node* improved() const { return _improved; } + + bool operator==(const ReplacedNode& other) { + return _initial == other._initial && _improved == other._improved; + } + }; + GrowableArray<ReplacedNode>* _replaced_nodes; + + void allocate_if_necessary(); + bool has_node(const ReplacedNode& r) const; + bool has_target_node(Node* n) const; + + public: + ReplacedNodes() + : _replaced_nodes(NULL) {} + + void clone(); + void record(Node* initial, Node* improved); + void transfer_from(const ReplacedNodes& other, uint idx); + void reset(); + void apply(Node* n); + void merge_with(const ReplacedNodes& other); + bool is_empty() const; + void dump(outputStream *st) const; + void apply(Compile* C, Node* ctl); +}; + +#endif // SHARE_VM_OPTO_REPLACEDNODES_HPP
--- a/src/share/vm/prims/jni.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/prims/jni.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -292,15 +292,6 @@ "Bug in native code: jfieldID offset must address interior of object"); } -// Pick a reasonable higher bound for local capacity requested -// for EnsureLocalCapacity and PushLocalFrame. We don't want it too -// high because a test (or very unusual application) may try to allocate -// that many handles and run out of swap space. An implementation is -// permitted to allocate more handles than the ensured capacity, so this -// value is set high enough to prevent compatibility problems. -const int MAX_REASONABLE_LOCAL_CAPACITY = 4*K; - - // Wrapper to trace JNI functions #ifdef ASSERT @@ -880,7 +871,8 @@ env, capacity); #endif /* USDT2 */ //%note jni_11 - if (capacity < 0 || capacity > MAX_REASONABLE_LOCAL_CAPACITY) { + if (capacity < 0 || + ((MaxJNILocalCapacity > 0) && (capacity > MaxJNILocalCapacity))) { #ifndef USDT2 DTRACE_PROBE1(hotspot_jni, PushLocalFrame__return, JNI_ERR); #else /* USDT2 */ @@ -1039,7 +1031,8 @@ env, capacity); #endif /* USDT2 */ jint ret; - if (capacity >= 0 && capacity <= MAX_REASONABLE_LOCAL_CAPACITY) { + if (capacity >= 0 && + ((MaxJNILocalCapacity <= 0) || (capacity <= MaxJNILocalCapacity))) { ret = JNI_OK; } else { ret = JNI_ERR; @@ -5089,6 +5082,7 @@ void TestG1BiasedArray_test(); void TestBufferingOopClosure_test(); void TestCodeCacheRemSet_test(); +void FreeRegionList_test(); #endif void execute_internal_vm_tests() { @@ -5119,6 +5113,9 @@ run_unit_test(HeapRegionRemSet::test_prt()); run_unit_test(TestBufferingOopClosure_test()); run_unit_test(TestCodeCacheRemSet_test()); + if (UseG1GC) { + run_unit_test(FreeRegionList_test()); + } #endif tty->print_cr("All internal VM tests passed"); }
--- a/src/share/vm/prims/whitebox.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/prims/whitebox.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -231,7 +231,7 @@ WB_ENTRY(jlong, WB_G1NumFreeRegions(JNIEnv* env, jobject o)) G1CollectedHeap* g1 = G1CollectedHeap::heap(); - size_t nr = g1->free_regions(); + size_t nr = g1->num_free_regions(); return (jlong)nr; WB_END
--- a/src/share/vm/runtime/arguments.cpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/runtime/arguments.cpp Thu Aug 21 14:42:54 2014 -0700 @@ -3792,10 +3792,6 @@ // nothing to use the profiling, turn if off FLAG_SET_DEFAULT(TypeProfileLevel, 0); } - if (UseTypeSpeculation && FLAG_IS_DEFAULT(ReplaceInParentMaps)) { - // Doing the replace in parent maps helps speculation - FLAG_SET_DEFAULT(ReplaceInParentMaps, true); - } #endif if (PrintAssembly && FLAG_IS_DEFAULT(DebugNonSafepoints)) {
--- a/src/share/vm/runtime/globals.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/runtime/globals.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -1216,6 +1216,11 @@ product(bool, UseFastJNIAccessors, true, \ "Use optimized versions of Get<Primitive>Field") \ \ + product(intx, MaxJNILocalCapacity, 65536, \ + "Maximum allowable local JNI handle capacity to " \ + "EnsureLocalCapacity() and PushLocalFrame(), " \ + "where <= 0 is unlimited, default: 65536") \ + \ product(bool, EagerXrunInit, false, \ "Eagerly initialize -Xrun libraries; allows startup profiling, " \ "but not all -Xrun libraries may support the state of the VM " \ @@ -1940,6 +1945,10 @@ "not just one of the generations (e.g., G1). A value of 0 " \ "denotes 'do constant GC cycles'.") \ \ + manageable(intx, CMSTriggerInterval, -1, \ + "Commence a CMS collection cycle (at least) every so many " \ + "milliseconds (0 permanently, -1 disabled)") \ + \ product(bool, UseCMSInitiatingOccupancyOnly, false, \ "Only use occupancy as a criterion for starting a CMS collection")\ \
--- a/src/share/vm/utilities/growableArray.hpp Thu Aug 21 11:43:17 2014 -0700 +++ b/src/share/vm/utilities/growableArray.hpp Thu Aug 21 14:42:54 2014 -0700 @@ -349,6 +349,7 @@ // inserts the given element before the element at index i void insert_before(const int idx, const E& elem) { + assert(0 <= idx && idx <= _len, "illegal index"); check_nesting(); if (_len == _max) grow(_len); for (int j = _len - 1; j >= idx; j--) { @@ -360,7 +361,7 @@ void appendAll(const GrowableArray<E>* l) { for (int i = 0; i < l->_len; i++) { - raw_at_put_grow(_len, l->_data[i], 0); + raw_at_put_grow(_len, l->_data[i], E()); } }
--- a/test/gc/class_unloading/TestCMSClassUnloadingEnabledHWM.java Thu Aug 21 11:43:17 2014 -0700 +++ b/test/gc/class_unloading/TestCMSClassUnloadingEnabledHWM.java Thu Aug 21 14:42:54 2014 -0700 @@ -45,6 +45,7 @@ private static OutputAnalyzer run(boolean enableUnloading) throws Exception { ProcessBuilder pb = ProcessTools.createJavaProcessBuilder( "-Xbootclasspath/a:.", + "-XX:+UnlockDiagnosticVMOptions", "-XX:+WhiteBoxAPI", "-XX:MetaspaceSize=" + MetaspaceSize, "-Xmn" + YoungGenSize,
--- a/test/gc/class_unloading/TestG1ClassUnloadingHWM.java Thu Aug 21 11:43:17 2014 -0700 +++ b/test/gc/class_unloading/TestG1ClassUnloadingHWM.java Thu Aug 21 14:42:54 2014 -0700 @@ -45,6 +45,7 @@ private static OutputAnalyzer run(boolean enableUnloading) throws Exception { ProcessBuilder pb = ProcessTools.createJavaProcessBuilder( "-Xbootclasspath/a:.", + "-XX:+UnlockDiagnosticVMOptions", "-XX:+WhiteBoxAPI", "-XX:MetaspaceSize=" + MetaspaceSize, "-Xmn" + YoungGenSize,
--- a/test/gc/g1/TestEagerReclaimHumongousRegions2.java Thu Aug 21 11:43:17 2014 -0700 +++ b/test/gc/g1/TestEagerReclaimHumongousRegions2.java Thu Aug 21 14:42:54 2014 -0700 @@ -46,6 +46,8 @@ } class ReclaimRegionFast { + public static final long MAX_MILLIS_FOR_RUN = 50 * 1000; // The maximum runtime for the actual test. + public static final int M = 1024*1024; public static LinkedList<Object> garbageList = new LinkedList<Object>(); @@ -83,7 +85,14 @@ Object ref_from_stack = large1; + long start_millis = System.currentTimeMillis(); + for (int i = 0; i < 20; i++) { + long current_millis = System.currentTimeMillis(); + if ((current_millis - start_millis) > MAX_MILLIS_FOR_RUN) { + System.out.println("Finishing test because maximum runtime exceeded"); + break; + } // A set of large objects that will be reclaimed eagerly - and hopefully marked. large1 = new int[M - 20]; large2 = new int[M - 20];