# HG changeset patch # User tschatzl # Date 1406099012 -7200 # Node ID a3953c77756502893d7e8404f9c217c1204ed697 # Parent 3f2894c5052eb997daa2b61d23c13ec56669ff10 8027959: Early reclamation of large objects in G1 Summary: Try to reclaim humongous objects at every young collection after doing a conservative estimate of its liveness. Reviewed-by: brutisso, mgerdin diff -r 3f2894c5052e -r a3953c777565 src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Mon Jul 21 10:00:31 2014 +0200 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Wed Jul 23 09:03:32 2014 +0200 @@ -1928,6 +1928,8 @@ _secondary_free_list("Secondary Free List", new SecondaryFreeRegionListMtSafeChecker()), _old_set("Old Set", false /* humongous */, new OldRegionSetMtSafeChecker()), _humongous_set("Master Humongous Set", true /* humongous */, new HumongousRegionSetMtSafeChecker()), + _humongous_is_live(), + _has_humongous_reclaim_candidates(false), _free_regions_coming(false), _young_list(new YoungList(this)), _gc_time_stamp(0), @@ -2084,6 +2086,7 @@ _g1h = this; _in_cset_fast_test.initialize(_g1_reserved.start(), _g1_reserved.end(), HeapRegion::GrainBytes); + _humongous_is_live.initialize(_g1_reserved.start(), _g1_reserved.end(), HeapRegion::GrainBytes); // Create the ConcurrentMark data structure and thread. // (Must do this late, so that "max_regions" is defined.) @@ -2179,6 +2182,11 @@ } } +void G1CollectedHeap::clear_humongous_is_live_table() { + guarantee(G1ReclaimDeadHumongousObjectsAtYoungGC, "Should only be called if true"); + _humongous_is_live.clear(); +} + size_t G1CollectedHeap::conservative_max_heap_alignment() { return HeapRegion::max_region_size(); } @@ -3797,6 +3805,61 @@ return g1_rem_set()->cardsScanned(); } +bool G1CollectedHeap::humongous_region_is_always_live(uint index) { + HeapRegion* region = region_at(index); + assert(region->startsHumongous(), "Must start a humongous object"); + return oop(region->bottom())->is_objArray() || !region->rem_set()->is_empty(); +} + +class RegisterHumongousWithInCSetFastTestClosure : public HeapRegionClosure { + private: + size_t _total_humongous; + size_t _candidate_humongous; + public: + RegisterHumongousWithInCSetFastTestClosure() : _total_humongous(0), _candidate_humongous(0) { + } + + virtual bool doHeapRegion(HeapRegion* r) { + if (!r->startsHumongous()) { + return false; + } + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + uint region_idx = r->hrs_index(); + bool is_candidate = !g1h->humongous_region_is_always_live(region_idx); + // Is_candidate already filters out humongous regions with some remembered set. + // This will not lead to humongous object that we mistakenly keep alive because + // during young collection the remembered sets will only be added to. + if (is_candidate) { + g1h->register_humongous_region_with_in_cset_fast_test(region_idx); + _candidate_humongous++; + } + _total_humongous++; + + return false; + } + + size_t total_humongous() const { return _total_humongous; } + size_t candidate_humongous() const { return _candidate_humongous; } +}; + +void G1CollectedHeap::register_humongous_regions_with_in_cset_fast_test() { + if (!G1ReclaimDeadHumongousObjectsAtYoungGC) { + g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(0, 0); + return; + } + + RegisterHumongousWithInCSetFastTestClosure cl; + heap_region_iterate(&cl); + g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(cl.total_humongous(), + cl.candidate_humongous()); + _has_humongous_reclaim_candidates = cl.candidate_humongous() > 0; + + if (_has_humongous_reclaim_candidates) { + clear_humongous_is_live_table(); + } +} + void G1CollectedHeap::setup_surviving_young_words() { assert(_surviving_young_words == NULL, "pre-condition"); @@ -4083,6 +4146,8 @@ g1_policy()->finalize_cset(target_pause_time_ms, evacuation_info); + register_humongous_regions_with_in_cset_fast_test(); + _cm->note_start_of_gc(); // We should not verify the per-thread SATB buffers given that // we have not filtered them yet (we'll do so during the @@ -4133,6 +4198,9 @@ true /* verify_fingers */); free_collection_set(g1_policy()->collection_set(), evacuation_info); + + eagerly_reclaim_humongous_regions(); + g1_policy()->clear_collection_set(); cleanup_surviving_young_words(); @@ -4644,7 +4712,9 @@ assert(_worker_id == _par_scan_state->queue_num(), "sanity"); - if (_g1->in_cset_fast_test(obj)) { + G1CollectedHeap::in_cset_state_t state = _g1->in_cset_state(obj); + + if (state == G1CollectedHeap::InCSet) { oop forwardee; if (obj->is_forwarded()) { forwardee = obj->forwardee(); @@ -4663,6 +4733,9 @@ do_klass_barrier(p, forwardee); } } else { + if (state == G1CollectedHeap::IsHumongous) { + _g1->set_humongous_is_live(obj); + } // The object is not in collection set. If we're a root scanning // closure during an initial mark pause then attempt to mark the object. if (do_mark_object == G1MarkFromRoot) { @@ -5492,12 +5565,21 @@ public: G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {} void do_oop(narrowOop* p) { guarantee(false, "Not needed"); } - void do_oop( oop* p) { + void do_oop(oop* p) { oop obj = *p; - if (_g1->obj_in_cs(obj)) { + G1CollectedHeap::in_cset_state_t cset_state = _g1->in_cset_state(obj); + if (obj == NULL || cset_state == G1CollectedHeap::InNeither) { + return; + } + if (cset_state == G1CollectedHeap::InCSet) { assert( obj->is_forwarded(), "invariant" ); *p = obj->forwardee(); + } else { + assert(!obj->is_forwarded(), "invariant" ); + assert(cset_state == G1CollectedHeap::IsHumongous, + err_msg("Only allowed InCSet state is IsHumongous, but is %d", cset_state)); + _g1->set_humongous_is_live(obj); } } }; @@ -5527,7 +5609,7 @@ template void do_oop_work(T* p) { oop obj = oopDesc::load_decode_heap_oop(p); - if (_g1h->obj_in_cs(obj)) { + if (_g1h->is_in_cset_or_humongous(obj)) { // If the referent object has been forwarded (either copied // to a new location or to itself in the event of an // evacuation failure) then we need to update the reference @@ -5552,10 +5634,10 @@ assert(!Metaspace::contains((const void*)p), err_msg("Unexpectedly found a pointer from metadata: " PTR_FORMAT, p)); - _copy_non_heap_obj_cl->do_oop(p); - } + _copy_non_heap_obj_cl->do_oop(p); } } + } }; // Serial drain queue closure. Called as the 'complete_gc' @@ -6477,6 +6559,147 @@ policy->phase_times()->record_non_young_free_cset_time_ms(non_young_time_ms); } +class G1FreeHumongousRegionClosure : public HeapRegionClosure { + private: + FreeRegionList* _free_region_list; + HeapRegionSet* _proxy_set; + HeapRegionSetCount _humongous_regions_removed; + size_t _freed_bytes; + public: + + G1FreeHumongousRegionClosure(FreeRegionList* free_region_list) : + _free_region_list(free_region_list), _humongous_regions_removed(), _freed_bytes(0) { + } + + virtual bool doHeapRegion(HeapRegion* r) { + if (!r->startsHumongous()) { + return false; + } + + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + // The following checks whether the humongous object is live are sufficient. + // The main additional check (in addition to having a reference from the roots + // or the young gen) is whether the humongous object has a remembered set entry. + // + // A humongous object cannot be live if there is no remembered set for it + // because: + // - there can be no references from within humongous starts regions referencing + // the object because we never allocate other objects into them. + // (I.e. there are no intra-region references that may be missed by the + // remembered set) + // - as soon there is a remembered set entry to the humongous starts region + // (i.e. it has "escaped" to an old object) this remembered set entry will stay + // until the end of a concurrent mark. + // + // It is not required to check whether the object has been found dead by marking + // or not, in fact it would prevent reclamation within a concurrent cycle, as + // all objects allocated during that time are considered live. + // SATB marking is even more conservative than the remembered set. + // So if at this point in the collection there is no remembered set entry, + // nobody has a reference to it. + // At the start of collection we flush all refinement logs, and remembered sets + // are completely up-to-date wrt to references to the humongous object. + // + // Other implementation considerations: + // - never consider object arrays: while they are a valid target, they have not + // been observed to be used as temporary objects. + // - they would also pose considerable effort for cleaning up the the remembered + // sets. + // While this cleanup is not strictly necessary to be done (or done instantly), + // given that their occurrence is very low, this saves us this additional + // complexity. + uint region_idx = r->hrs_index(); + if (g1h->humongous_is_live(region_idx) || + g1h->humongous_region_is_always_live(region_idx)) { + + if (G1TraceReclaimDeadHumongousObjectsAtYoungGC) { + gclog_or_tty->print_cr("Live humongous %d region %d with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is dead-bitmap %d live-other %d obj array %d", + r->isHumongous(), + region_idx, + r->rem_set()->occupied(), + r->rem_set()->strong_code_roots_list_length(), + g1h->mark_in_progress() && !g1h->g1_policy()->during_initial_mark_pause(), + g1h->humongous_is_live(region_idx), + oop(r->bottom())->is_objArray() + ); + } + + return false; + } + + guarantee(!((oop)(r->bottom()))->is_objArray(), + err_msg("Eagerly reclaiming object arrays is not supported, but the object "PTR_FORMAT" is.", + r->bottom())); + + if (G1TraceReclaimDeadHumongousObjectsAtYoungGC) { + gclog_or_tty->print_cr("Reclaim humongous region %d start "PTR_FORMAT" region %d length "UINT32_FORMAT" with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is dead-bitmap %d live-other %d obj array %d", + r->isHumongous(), + r->bottom(), + region_idx, + r->region_num(), + r->rem_set()->occupied(), + r->rem_set()->strong_code_roots_list_length(), + g1h->mark_in_progress() && !g1h->g1_policy()->during_initial_mark_pause(), + g1h->humongous_is_live(region_idx), + oop(r->bottom())->is_objArray() + ); + } + _freed_bytes += r->used(); + r->set_containing_set(NULL); + _humongous_regions_removed.increment(1u, r->capacity()); + g1h->free_humongous_region(r, _free_region_list, false); + + return false; + } + + HeapRegionSetCount& humongous_free_count() { + return _humongous_regions_removed; + } + + size_t bytes_freed() const { + return _freed_bytes; + } + + size_t humongous_reclaimed() const { + return _humongous_regions_removed.length(); + } +}; + +void G1CollectedHeap::eagerly_reclaim_humongous_regions() { + assert_at_safepoint(true); + + if (!G1ReclaimDeadHumongousObjectsAtYoungGC || !_has_humongous_reclaim_candidates) { + g1_policy()->phase_times()->record_fast_reclaim_humongous_time_ms(0.0, 0); + return; + } + + double start_time = os::elapsedTime(); + + FreeRegionList local_cleanup_list("Local Humongous Cleanup List"); + + G1FreeHumongousRegionClosure cl(&local_cleanup_list); + heap_region_iterate(&cl); + + HeapRegionSetCount empty_set; + remove_from_old_sets(empty_set, cl.humongous_free_count()); + + G1HRPrinter* hr_printer = _g1h->hr_printer(); + if (hr_printer->is_active()) { + FreeRegionListIterator iter(&local_cleanup_list); + while (iter.more_available()) { + HeapRegion* hr = iter.get_next(); + hr_printer->cleanup(hr); + } + } + + prepend_to_freelist(&local_cleanup_list); + decrement_summary_bytes(cl.bytes_freed()); + + g1_policy()->phase_times()->record_fast_reclaim_humongous_time_ms((os::elapsedTime() - start_time) * 1000.0, + cl.humongous_reclaimed()); +} + // This routine is similar to the above but does not record // any policy statistics or update free lists; we are abandoning // the current incremental collection set in preparation of a diff -r 3f2894c5052e -r a3953c777565 src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Mon Jul 21 10:00:31 2014 +0200 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Wed Jul 23 09:03:32 2014 +0200 @@ -197,16 +197,6 @@ bool do_object_b(oop p); }; -// Instances of this class are used for quick tests on whether a reference points -// into the collection set. Each of the array's elements denotes whether the -// corresponding region is in the collection set. -class G1FastCSetBiasedMappedArray : public G1BiasedMappedArray { - protected: - bool default_value() const { return false; } - public: - void clear() { G1BiasedMappedArray::clear(); } -}; - class RefineCardTableEntryClosure; class G1CollectedHeap : public SharedHeap { @@ -237,6 +227,7 @@ friend class EvacPopObjClosure; friend class G1ParCleanupCTTask; + friend class G1FreeHumongousRegionClosure; // Other related classes. friend class G1MarkSweep; @@ -267,6 +258,9 @@ // It keeps track of the humongous regions. HeapRegionSet _humongous_set; + void clear_humongous_is_live_table(); + void eagerly_reclaim_humongous_regions(); + // The number of regions we could create by expansion. uint _expansion_regions; @@ -367,10 +361,25 @@ // than the current allocation region. size_t _summary_bytes_used; - // This array is used for a quick test on whether a reference points into - // the collection set or not. Each of the array's elements denotes whether the - // corresponding region is in the collection set or not. - G1FastCSetBiasedMappedArray _in_cset_fast_test; + // Records whether the region at the given index is kept live by roots or + // references from the young generation. + class HumongousIsLiveBiasedMappedArray : public G1BiasedMappedArray { + protected: + bool default_value() const { return false; } + public: + void clear() { G1BiasedMappedArray::clear(); } + void set_live(uint region) { + set_by_index(region, true); + } + bool is_live(uint region) { + return get_by_index(region); + } + }; + + HumongousIsLiveBiasedMappedArray _humongous_is_live; + // Stores whether during humongous object registration we found candidate regions. + // If not, we can skip a few steps. + bool _has_humongous_reclaim_candidates; volatile unsigned _gc_time_stamp; @@ -690,10 +699,24 @@ virtual void gc_prologue(bool full); virtual void gc_epilogue(bool full); + inline void set_humongous_is_live(oop obj); + + bool humongous_is_live(uint region) { + return _humongous_is_live.is_live(region); + } + + // Returns whether the given region (which must be a humongous (start) region) + // is to be considered conservatively live regardless of any other conditions. + bool humongous_region_is_always_live(uint index); + // Register the given region to be part of the collection set. + inline void register_humongous_region_with_in_cset_fast_test(uint index); + // Register regions with humongous objects (actually on the start region) in + // the in_cset_fast_test table. + void register_humongous_regions_with_in_cset_fast_test(); // We register a region with the fast "in collection set" test. We // simply set to true the array slot corresponding to this region. void register_region_with_in_cset_fast_test(HeapRegion* r) { - _in_cset_fast_test.set_by_index(r->hrs_index(), true); + _in_cset_fast_test.set_in_cset(r->hrs_index()); } // This is a fast test on whether a reference points into the @@ -1292,9 +1315,61 @@ virtual bool is_in(const void* p) const; // Return "TRUE" iff the given object address is within the collection - // set. + // set. Slow implementation. inline bool obj_in_cs(oop obj); + inline bool is_in_cset(oop obj); + + inline bool is_in_cset_or_humongous(const oop obj); + + enum in_cset_state_t { + InNeither, // neither in collection set nor humongous + InCSet, // region is in collection set only + IsHumongous // region is a humongous start region + }; + private: + // Instances of this class are used for quick tests on whether a reference points + // into the collection set or is a humongous object (points into a humongous + // object). + // Each of the array's elements denotes whether the corresponding region is in + // the collection set or a humongous region. + // We use this to quickly reclaim humongous objects: by making a humongous region + // succeed this test, we sort-of add it to the collection set. During the reference + // iteration closures, when we see a humongous region, we simply mark it as + // referenced, i.e. live. + class G1FastCSetBiasedMappedArray : public G1BiasedMappedArray { + protected: + char default_value() const { return G1CollectedHeap::InNeither; } + public: + void set_humongous(uintptr_t index) { + assert(get_by_index(index) != InCSet, "Should not overwrite InCSet values"); + set_by_index(index, G1CollectedHeap::IsHumongous); + } + + void clear_humongous(uintptr_t index) { + set_by_index(index, G1CollectedHeap::InNeither); + } + + void set_in_cset(uintptr_t index) { + assert(get_by_index(index) != G1CollectedHeap::IsHumongous, "Should not overwrite IsHumongous value"); + set_by_index(index, G1CollectedHeap::InCSet); + } + + bool is_in_cset_or_humongous(HeapWord* addr) const { return get_by_address(addr) != G1CollectedHeap::InNeither; } + bool is_in_cset(HeapWord* addr) const { return get_by_address(addr) == G1CollectedHeap::InCSet; } + G1CollectedHeap::in_cset_state_t at(HeapWord* addr) const { return (G1CollectedHeap::in_cset_state_t)get_by_address(addr); } + void clear() { G1BiasedMappedArray::clear(); } + }; + + // This array is used for a quick test on whether a reference points into + // the collection set or not. Each of the array's elements denotes whether the + // corresponding region is in the collection set or not. + G1FastCSetBiasedMappedArray _in_cset_fast_test; + + public: + + inline in_cset_state_t in_cset_state(const oop obj); + // Return "TRUE" iff the given object address is in the reserved // region of g1. bool is_in_g1_reserved(const void* p) const { @@ -1349,6 +1424,10 @@ // Return the region with the given index. It assumes the index is valid. inline HeapRegion* region_at(uint index) const; + // Calculate the region index of the given address. Given address must be + // within the heap. + inline uint addr_to_region(HeapWord* addr) const; + // Divide the heap region sequence into "chunks" of some size (the number // of regions divided by the number of parallel threads times some // overpartition factor, currently 4). Assumes that this will be called diff -r 3f2894c5052e -r a3953c777565 src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp Mon Jul 21 10:00:31 2014 +0200 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp Wed Jul 23 09:03:32 2014 +0200 @@ -40,6 +40,13 @@ // Return the region with the given index. It assumes the index is valid. inline HeapRegion* G1CollectedHeap::region_at(uint index) const { return _hrs.at(index); } +inline uint G1CollectedHeap::addr_to_region(HeapWord* addr) const { + assert(is_in_reserved(addr), + err_msg("Cannot calculate region index for address "PTR_FORMAT" that is outside of the heap ["PTR_FORMAT", "PTR_FORMAT")", + p2i(addr), p2i(_reserved.start()), p2i(_reserved.end()))); + return (uint)(pointer_delta(addr, _reserved.start(), sizeof(uint8_t)) >> HeapRegion::LogOfHRGrainBytes); +} + template inline HeapRegion* G1CollectedHeap::heap_region_containing(const T addr) const { @@ -172,12 +179,11 @@ return _cm->nextMarkBitMap()->isMarked((HeapWord *)obj); } - // This is a fast test on whether a reference points into the // collection set or not. Assume that the reference // points into the heap. -inline bool G1CollectedHeap::in_cset_fast_test(oop obj) { - bool ret = _in_cset_fast_test.get_by_address((HeapWord*)obj); +inline bool G1CollectedHeap::is_in_cset(oop obj) { + bool ret = _in_cset_fast_test.is_in_cset((HeapWord*)obj); // let's make sure the result is consistent with what the slower // test returns assert( ret || !obj_in_cs(obj), "sanity"); @@ -185,6 +191,18 @@ return ret; } +bool G1CollectedHeap::is_in_cset_or_humongous(const oop obj) { + return _in_cset_fast_test.is_in_cset_or_humongous((HeapWord*)obj); +} + +G1CollectedHeap::in_cset_state_t G1CollectedHeap::in_cset_state(const oop obj) { + return _in_cset_fast_test.at((HeapWord*)obj); +} + +void G1CollectedHeap::register_humongous_region_with_in_cset_fast_test(uint index) { + _in_cset_fast_test.set_humongous(index); +} + #ifndef PRODUCT // Support for G1EvacuationFailureALot @@ -290,4 +308,22 @@ else return is_obj_ill(obj, hr); } +inline void G1CollectedHeap::set_humongous_is_live(oop obj) { + uint region = addr_to_region((HeapWord*)obj); + // We not only set the "live" flag in the humongous_is_live table, but also + // reset the entry in the _in_cset_fast_test table so that subsequent references + // to the same humongous object do not go into the slow path again. + // This is racy, as multiple threads may at the same time enter here, but this + // is benign. + // During collection we only ever set the "live" flag, and only ever clear the + // entry in the in_cset_fast_table. + // We only ever evaluate the contents of these tables (in the VM thread) after + // having synchronized the worker threads with the VM thread, or in the same + // thread (i.e. within the VM thread). + if (!_humongous_is_live.is_live(region)) { + _humongous_is_live.set_live(region); + _in_cset_fast_test.clear_humongous(region); + } +} + #endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1COLLECTEDHEAP_INLINE_HPP diff -r 3f2894c5052e -r a3953c777565 src/share/vm/gc_implementation/g1/g1GCPhaseTimes.cpp --- a/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.cpp Mon Jul 21 10:00:31 2014 +0200 +++ b/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.cpp Wed Jul 23 09:03:32 2014 +0200 @@ -254,6 +254,10 @@ LineBuffer(level).append_and_print_cr("[%s: %.1lf ms]", str, value); } +void G1GCPhaseTimes::print_stats(int level, const char* str, size_t value) { + LineBuffer(level).append_and_print_cr("[%s: "SIZE_FORMAT"]", str, value); +} + void G1GCPhaseTimes::print_stats(int level, const char* str, double value, uint workers) { LineBuffer(level).append_and_print_cr("[%s: %.1lf ms, GC Workers: " UINT32_FORMAT "]", str, value, workers); } @@ -356,6 +360,14 @@ _last_redirty_logged_cards_processed_cards.print(3, "Redirtied Cards"); } } + if (G1ReclaimDeadHumongousObjectsAtYoungGC) { + print_stats(2, "Humongous Reclaim", _cur_fast_reclaim_humongous_time_ms); + if (G1Log::finest()) { + print_stats(3, "Humongous Total", _cur_fast_reclaim_humongous_total); + print_stats(3, "Humongous Candidate", _cur_fast_reclaim_humongous_candidates); + print_stats(3, "Humongous Reclaimed", _cur_fast_reclaim_humongous_reclaimed); + } + } print_stats(2, "Free CSet", (_recorded_young_free_cset_time_ms + _recorded_non_young_free_cset_time_ms)); diff -r 3f2894c5052e -r a3953c777565 src/share/vm/gc_implementation/g1/g1GCPhaseTimes.hpp --- a/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.hpp Mon Jul 21 10:00:31 2014 +0200 +++ b/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.hpp Wed Jul 23 09:03:32 2014 +0200 @@ -157,11 +157,17 @@ double _recorded_young_free_cset_time_ms; double _recorded_non_young_free_cset_time_ms; + double _cur_fast_reclaim_humongous_time_ms; + size_t _cur_fast_reclaim_humongous_total; + size_t _cur_fast_reclaim_humongous_candidates; + size_t _cur_fast_reclaim_humongous_reclaimed; + double _cur_verify_before_time_ms; double _cur_verify_after_time_ms; // Helper methods for detailed logging void print_stats(int level, const char* str, double value); + void print_stats(int level, const char* str, size_t value); void print_stats(int level, const char* str, double value, uint workers); public: @@ -282,6 +288,16 @@ _recorded_non_young_free_cset_time_ms = time_ms; } + void record_fast_reclaim_humongous_stats(size_t total, size_t candidates) { + _cur_fast_reclaim_humongous_total = total; + _cur_fast_reclaim_humongous_candidates = candidates; + } + + void record_fast_reclaim_humongous_time_ms(double value, size_t reclaimed) { + _cur_fast_reclaim_humongous_time_ms = value; + _cur_fast_reclaim_humongous_reclaimed = reclaimed; + } + void record_young_cset_choice_time_ms(double time_ms) { _recorded_young_cset_choice_time_ms = time_ms; } @@ -348,6 +364,10 @@ return _recorded_non_young_free_cset_time_ms; } + double fast_reclaim_humongous_time_ms() { + return _cur_fast_reclaim_humongous_time_ms; + } + double average_last_update_rs_time() { return _last_update_rs_times_ms.average(); } diff -r 3f2894c5052e -r a3953c777565 src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp --- a/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp Mon Jul 21 10:00:31 2014 +0200 +++ b/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp Wed Jul 23 09:03:32 2014 +0200 @@ -44,7 +44,7 @@ inline void FilterIntoCSClosure::do_oop_nv(T* p) { T heap_oop = oopDesc::load_heap_oop(p); if (!oopDesc::is_null(heap_oop) && - _g1->obj_in_cs(oopDesc::decode_heap_oop_not_null(heap_oop))) { + _g1->is_in_cset_or_humongous(oopDesc::decode_heap_oop_not_null(heap_oop))) { _oc->do_oop(p); } } @@ -67,7 +67,8 @@ if (!oopDesc::is_null(heap_oop)) { oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if (_g1->in_cset_fast_test(obj)) { + G1CollectedHeap::in_cset_state_t state = _g1->in_cset_state(obj); + if (state == G1CollectedHeap::InCSet) { // We're not going to even bother checking whether the object is // already forwarded or not, as this usually causes an immediate // stall. We'll try to prefetch the object (for write, given that @@ -86,6 +87,9 @@ _par_scan_state->push_on_queue(p); } else { + if (state == G1CollectedHeap::IsHumongous) { + _g1->set_humongous_is_live(obj); + } _par_scan_state->update_rs(_from, p, _worker_id); } } @@ -97,12 +101,14 @@ if (!oopDesc::is_null(heap_oop)) { oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if (_g1->in_cset_fast_test(obj)) { + if (_g1->is_in_cset_or_humongous(obj)) { Prefetch::write(obj->mark_addr(), 0); Prefetch::read(obj->mark_addr(), (HeapWordSize*2)); // Place on the references queue _par_scan_state->push_on_queue(p); + } else { + assert(!_g1->obj_in_cs(obj), "checking"); } } } diff -r 3f2894c5052e -r a3953c777565 src/share/vm/gc_implementation/g1/g1ParScanThreadState.inline.hpp --- a/src/share/vm/gc_implementation/g1/g1ParScanThreadState.inline.hpp Mon Jul 21 10:00:31 2014 +0200 +++ b/src/share/vm/gc_implementation/g1/g1ParScanThreadState.inline.hpp Wed Jul 23 09:03:32 2014 +0200 @@ -52,15 +52,20 @@ // set, due to (benign) races in the claim mechanism during RSet scanning more // than one thread might claim the same card. So the same card may be // processed multiple times. So redo this check. - if (_g1h->in_cset_fast_test(obj)) { + G1CollectedHeap::in_cset_state_t in_cset_state = _g1h->in_cset_state(obj); + if (in_cset_state == G1CollectedHeap::InCSet) { oop forwardee; if (obj->is_forwarded()) { forwardee = obj->forwardee(); } else { forwardee = copy_to_survivor_space(obj); } - assert(forwardee != NULL, "forwardee should not be NULL"); oopDesc::encode_store_heap_oop(p, forwardee); + } else if (in_cset_state == G1CollectedHeap::IsHumongous) { + _g1h->set_humongous_is_live(obj); + } else { + assert(in_cset_state == G1CollectedHeap::InNeither, + err_msg("In_cset_state must be InNeither here, but is %d", in_cset_state)); } assert(obj != NULL, "Must be"); diff -r 3f2894c5052e -r a3953c777565 src/share/vm/gc_implementation/g1/g1_globals.hpp --- a/src/share/vm/gc_implementation/g1/g1_globals.hpp Mon Jul 21 10:00:31 2014 +0200 +++ b/src/share/vm/gc_implementation/g1/g1_globals.hpp Wed Jul 23 09:03:32 2014 +0200 @@ -289,6 +289,13 @@ "The amount of code root chunks that should be kept at most " \ "as percentage of already allocated.") \ \ + experimental(bool, G1ReclaimDeadHumongousObjectsAtYoungGC, true, \ + "Try to reclaim dead large objects at every young GC.") \ + \ + experimental(bool, G1TraceReclaimDeadHumongousObjectsAtYoungGC, false, \ + "Print some information about large object liveness " \ + "at every young GC.") \ + \ experimental(uintx, G1OldCSetRegionThresholdPercent, 10, \ "An upper bound for the number of old CSet regions expressed " \ "as a percentage of the heap size.") \ diff -r 3f2894c5052e -r a3953c777565 src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp --- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp Mon Jul 21 10:00:31 2014 +0200 +++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp Wed Jul 23 09:03:32 2014 +0200 @@ -694,6 +694,9 @@ clear_fcc(); } +bool OtherRegionsTable::is_empty() const { + return occ_sparse() == 0 && occ_coarse() == 0 && _first_all_fine_prts == NULL; +} size_t OtherRegionsTable::occupied() const { size_t sum = occ_fine(); diff -r 3f2894c5052e -r a3953c777565 src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp --- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp Mon Jul 21 10:00:31 2014 +0200 +++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp Wed Jul 23 09:03:32 2014 +0200 @@ -185,6 +185,9 @@ // objects. void scrub(CardTableModRefBS* ctbs, BitMap* region_bm, BitMap* card_bm); + // Returns whether this remembered set (and all sub-sets) contain no entries. + bool is_empty() const; + size_t occupied() const; size_t occ_fine() const; size_t occ_coarse() const; @@ -269,6 +272,10 @@ return _other_regions.hr(); } + bool is_empty() const { + return (strong_code_roots_list_length() == 0) && _other_regions.is_empty(); + } + size_t occupied() { MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); return occupied_locked(); @@ -375,7 +382,7 @@ void strong_code_roots_do(CodeBlobClosure* blk) const; // Returns the number of elements in the strong code roots list - size_t strong_code_roots_list_length() { + size_t strong_code_roots_list_length() const { return _code_roots.length(); } diff -r 3f2894c5052e -r a3953c777565 test/gc/g1/TestEagerReclaimHumongousRegions.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/gc/g1/TestEagerReclaimHumongousRegions.java Wed Jul 23 09:03:32 2014 +0200 @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test TestEagerReclaimHumongousRegions + * @bug 8027959 + * @summary Test to make sure that eager reclaim of humongous objects work. We simply try to fill + * up the heap with humongous objects that should be eagerly reclaimable to avoid Full GC. + * @key gc + * @library /testlibrary + */ + +import java.util.regex.Pattern; +import java.util.regex.Matcher; +import java.util.LinkedList; + +import com.oracle.java.testlibrary.OutputAnalyzer; +import com.oracle.java.testlibrary.ProcessTools; +import com.oracle.java.testlibrary.Asserts; + +class ReclaimRegionFast { + public static final int M = 1024*1024; + + public static LinkedList garbageList = new LinkedList(); + + public static void genGarbage() { + for (int i = 0; i < 32*1024; i++) { + garbageList.add(new int[100]); + } + garbageList.clear(); + } + + // A large object referenced by a static. + static int[] filler = new int[10 * M]; + + public static void main(String[] args) { + + int[] large = new int[M]; + + Object ref_from_stack = large; + + for (int i = 0; i < 100; i++) { + // A large object that will be reclaimed eagerly. + large = new int[6*M]; + genGarbage(); + // Make sure that the compiler cannot completely remove + // the allocation of the large object until here. + System.out.println(large); + } + + // Keep the reference to the first object alive. + System.out.println(ref_from_stack); + } +} + +public class TestEagerReclaimHumongousRegions { + public static void main(String[] args) throws Exception { + ProcessBuilder pb = ProcessTools.createJavaProcessBuilder( + "-XX:+UseG1GC", + "-Xms128M", + "-Xmx128M", + "-Xmn16M", + "-XX:+PrintGC", + ReclaimRegionFast.class.getName()); + + Pattern p = Pattern.compile("Full GC"); + + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + + int found = 0; + Matcher m = p.matcher(output.getStdout()); + while (m.find()) { found++; } + System.out.println("Issued " + found + " Full GCs"); + Asserts.assertLT(found, 10, "Found that " + found + " Full GCs were issued. This is larger than the bound. Eager reclaim seems to not work at all"); + + output.shouldHaveExitValue(0); + } +} diff -r 3f2894c5052e -r a3953c777565 test/gc/g1/TestGCLogMessages.java --- a/test/gc/g1/TestGCLogMessages.java Mon Jul 21 10:00:31 2014 +0200 +++ b/test/gc/g1/TestGCLogMessages.java Wed Jul 23 09:03:32 2014 +0200 @@ -23,7 +23,7 @@ /* * @test TestGCLogMessages - * @bug 8035406 8027295 8035398 8019342 + * @bug 8035406 8027295 8035398 8019342 8027959 * @summary Ensure that the PrintGCDetails output for a minor GC with G1 * includes the expected necessary messages. * @key gc @@ -54,6 +54,7 @@ output.shouldNotContain("[String Dedup Fixup"); output.shouldNotContain("[Young Free CSet"); output.shouldNotContain("[Non-Young Free CSet"); + output.shouldNotContain("[Humongous Reclaim"); output.shouldHaveExitValue(0); pb = ProcessTools.createJavaProcessBuilder("-XX:+UseG1GC", @@ -71,6 +72,10 @@ output.shouldContain("[String Dedup Fixup"); output.shouldNotContain("[Young Free CSet"); output.shouldNotContain("[Non-Young Free CSet"); + output.shouldContain("[Humongous Reclaim"); + output.shouldNotContain("[Humongous Total"); + output.shouldNotContain("[Humongous Candidate"); + output.shouldNotContain("[Humongous Reclaimed"); output.shouldHaveExitValue(0); pb = ProcessTools.createJavaProcessBuilder("-XX:+UseG1GC", @@ -90,6 +95,10 @@ output.shouldContain("[String Dedup Fixup"); output.shouldContain("[Young Free CSet"); output.shouldContain("[Non-Young Free CSet"); + output.shouldContain("[Humongous Reclaim"); + output.shouldContain("[Humongous Total"); + output.shouldContain("[Humongous Candidate"); + output.shouldContain("[Humongous Reclaimed"); output.shouldHaveExitValue(0); }