graal-compiler: src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp comparison

comparison src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp @ 12355:cefad50507d8

Merge with hs25-b53

author	Gilles Duboscq <duboscq@ssw.jku.at>
date	Fri, 11 Oct 2013 10:38:03 +0200
parents	d0aeaf72c7bd 798522662fcd
children	096c224171c4

comparison

equal deleted inserted replaced

-:ccb4f2af2319
+:cefad50507d8
 * questions.
 *
 */
 #include "precompiled.hpp"
+#include "code/codeCache.hpp"
 #include "code/icBuffer.hpp"
 #include "gc_implementation/g1/bufferingOopClosure.hpp"
 #include "gc_implementation/g1/concurrentG1Refine.hpp"
 #include "gc_implementation/g1/concurrentG1RefineThread.hpp"
 #include "gc_implementation/g1/concurrentMarkThread.inline.hpp"
 G1CollectedHeap* _g1h;
 CardTableModRefBS* _ctbs;
 int _histo[256];
 public:
 ClearLoggedCardTableEntryClosure() :
-_calls(0)
+_calls(0), _g1h(G1CollectedHeap::heap()), _ctbs(_g1h->g1_barrier_set())
 {
-_g1h = G1CollectedHeap::heap();
-_ctbs = (CardTableModRefBS*)_g1h->barrier_set();
 for (int i = 0; i < 256; i++) _histo[i] = 0;
 }
 bool do_card_ptr(jbyte* card_ptr, int worker_i) {
 if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {
 _calls++;
 int _calls;
 G1CollectedHeap* _g1h;
 CardTableModRefBS* _ctbs;
 public:
 RedirtyLoggedCardTableEntryClosure() :
-_calls(0)
+_calls(0), _g1h(G1CollectedHeap::heap()), _ctbs(_g1h->g1_barrier_set()) {}
-{
-_g1h = G1CollectedHeap::heap();
-_ctbs = (CardTableModRefBS*)_g1h->barrier_set();
-}
 bool do_card_ptr(jbyte* card_ptr, int worker_i) {
 if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {
 _calls++;
 *card_ptr = 0;
 }
 }
 }
 void G1CollectedHeap::check_ct_logs_at_safepoint() {
 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
-CardTableModRefBS* ct_bs = (CardTableModRefBS*)barrier_set();
+CardTableModRefBS* ct_bs = g1_barrier_set();
 // Count the dirty cards at the start.
 CountNonCleanMemRegionClosure count1(this);
 ct_bs->mod_card_iterate(&count1);
 int orig_count = count1.n();
 }
 }
 if (should_try_gc) {
 bool succeeded;
-result = do_collection_pause(word_size, gc_count_before, &succeeded);
+result = do_collection_pause(word_size, gc_count_before, &succeeded,
+GCCause::_g1_inc_collection_pause);
 if (result != NULL) {
 assert(succeeded, "only way to get back a non-NULL result");
 return result;
 }
 // If we failed to allocate the humongous object, we should try to
 // do a collection pause (if we're allowed) in case it reclaims
 // enough space for the allocation to succeed after the pause.
 bool succeeded;
-result = do_collection_pause(word_size, gc_count_before, &succeeded);
+result = do_collection_pause(word_size, gc_count_before, &succeeded,
+GCCause::_g1_humongous_allocation);
 if (result != NULL) {
 assert(succeeded, "only way to get back a non-NULL result");
 return result;
 }
 class PostMCRemSetClearClosure: public HeapRegionClosure {
 G1CollectedHeap* _g1h;
 ModRefBarrierSet* _mr_bs;
 public:
 PostMCRemSetClearClosure(G1CollectedHeap* g1h, ModRefBarrierSet* mr_bs) :
-_g1h(g1h), _mr_bs(mr_bs) { }
+_g1h(g1h), _mr_bs(mr_bs) {}
 bool doHeapRegion(HeapRegion* r) {
+HeapRegionRemSet* hrrs = r->rem_set();
 if (r->continuesHumongous()) {
+// We'll assert that the strong code root list and RSet is empty
+assert(hrrs->strong_code_roots_list_length() == 0, "sanity");
+assert(hrrs->occupied() == 0, "RSet should be empty");
 return false;
 }
 _g1h->reset_gc_time_stamps(r);
-HeapRegionRemSet* hrrs = r->rem_set();
+hrrs->clear();
-if (hrrs != NULL) hrrs->clear();
 // You might think here that we could clear just the cards
 // corresponding to the used region.  But no: if we leave a dirty card
 // in a region we might allocate into, then it would prevent that card
 // from being enqueued, and cause it to be missed.
 // Re: the performance cost: we shouldn't be doing full GC anyway!
 _mr_bs->clear(MemRegion(r->bottom(), r->end()));
 return false;
 }
 };
 void G1CollectedHeap::clear_rsets_post_compaction() {
-PostMCRemSetClearClosure rs_clear(this, mr_bs());
+PostMCRemSetClearClosure rs_clear(this, g1_barrier_set());
 heap_region_iterate(&rs_clear);
 }
 class RebuildRSOutOfRegionClosure: public HeapRegionClosure {
 G1CollectedHeap*   _g1h;
 void G1CollectedHeap::print_hrs_post_compaction() {
 PostCompactionPrinterClosure cl(hr_printer());
 heap_region_iterate(&cl);
 }
-double G1CollectedHeap::verify(bool guard, const char* msg) {
-double verify_time_ms = 0.0;
-if (guard && total_collections() >= VerifyGCStartAt) {
-double verify_start = os::elapsedTime();
-HandleMark hm;  // Discard invalid handles created during verification
-prepare_for_verify();
-Universe::verify(VerifyOption_G1UsePrevMarking, msg);
-verify_time_ms = (os::elapsedTime() - verify_start) * 1000;
-}
-return verify_time_ms;
-}
-void G1CollectedHeap::verify_before_gc() {
-double verify_time_ms = verify(VerifyBeforeGC, " VerifyBeforeGC:");
-g1_policy()->phase_times()->record_verify_before_time_ms(verify_time_ms);
-}
-void G1CollectedHeap::verify_after_gc() {
-double verify_time_ms = verify(VerifyAfterGC, " VerifyAfterGC:");
-g1_policy()->phase_times()->record_verify_after_time_ms(verify_time_ms);
-}
 bool G1CollectedHeap::do_collection(bool explicit_gc,
 bool clear_all_soft_refs,
 size_t word_size) {
 assert_at_safepoint(true /* should_be_vm_thread */);
 assert(!ref_processor_stw()->discovery_enabled(), "Postcondition");
 ref_processor_stw()->verify_no_references_recorded();
 // Delete metaspaces for unloaded class loaders and clean up loader_data graph
 ClassLoaderDataGraph::purge();
 MetaspaceAux::verify_metrics();
 // Note: since we've just done a full GC, concurrent
 // marking is no longer active. Therefore we need not
 // re-enable reference discovery for the CM ref processor.
 // That will be done at the start of the next marking cycle.
 } else {
 RebuildRSOutOfRegionClosure rebuild_rs(this);
 heap_region_iterate(&rebuild_rs);
 }
+// Rebuild the strong code root lists for each region
+rebuild_strong_code_roots();
 if (true) { // FIXME
 MetaspaceGC::compute_new_size();
 }
 #ifdef TRACESPINNING
 // Tell the hot card cache about the update
 _cg1r->hot_card_cache()->resize_card_counts(capacity());
 }
 bool G1CollectedHeap::expand(size_t expand_bytes) {
-size_t old_mem_size = _g1_storage.committed_size();
 size_t aligned_expand_bytes = ReservedSpace::page_align_size_up(expand_bytes);
 aligned_expand_bytes = align_size_up(aligned_expand_bytes,
 HeapRegion::GrainBytes);
 ergo_verbose2(ErgoHeapSizing,
 "expand the heap",
 ergo_format_byte("requested expansion amount")
 ergo_format_byte("attempted expansion amount"),
 expand_bytes, aligned_expand_bytes);
+if (_g1_storage.uncommitted_size() == 0) {
+ergo_verbose0(ErgoHeapSizing,
+"did not expand the heap",
+ergo_format_reason("heap already fully expanded"));
+return false;
+}
 // First commit the memory.
 HeapWord* old_end = (HeapWord*) _g1_storage.high();
 bool successful = _g1_storage.expand_by(aligned_expand_bytes);
 if (successful) {
 }
 return successful;
 }
 void G1CollectedHeap::shrink_helper(size_t shrink_bytes) {
-size_t old_mem_size = _g1_storage.committed_size();
 size_t aligned_shrink_bytes =
 ReservedSpace::page_align_size_down(shrink_bytes);
 aligned_shrink_bytes = align_size_down(aligned_shrink_bytes,
 HeapRegion::GrainBytes);
 uint num_regions_to_remove = (uint)(shrink_bytes / HeapRegion::GrainBytes);
 // HeapWordSize).
 guarantee(HeapWordSize == wordSize, "HeapWordSize must equal wordSize");
 size_t init_byte_size = collector_policy()->initial_heap_byte_size();
 size_t max_byte_size = collector_policy()->max_heap_byte_size();
+size_t heap_alignment = collector_policy()->max_alignment();
 // Ensure that the sizes are properly aligned.
 Universe::check_alignment(init_byte_size, HeapRegion::GrainBytes, "g1 heap");
 Universe::check_alignment(max_byte_size, HeapRegion::GrainBytes, "g1 heap");
+Universe::check_alignment(max_byte_size, heap_alignment, "g1 heap");
 _cg1r = new ConcurrentG1Refine(this);
 // Reserve the maximum.
 // base of the reserved heap may end up differing from the
 // address that was requested (i.e. the preferred heap base).
 // If this happens then we could end up using a non-optimal
 // compressed oops mode.
-// Since max_byte_size is aligned to the size of a heap region (checked
-// above).
-Universe::check_alignment(max_byte_size, HeapRegion::GrainBytes, "g1 heap");
 ReservedSpace heap_rs = Universe::reserve_heap(max_byte_size,
-HeapRegion::GrainBytes);
+heap_alignment);
 // It is important to do this in a way such that concurrent readers can't
 // temporarily think something is in the heap.  (I've actually seen this
 // happen in asserts: DLD.)
 _reserved.set_word_size(0);
 _expansion_regions = (uint) (max_byte_size / HeapRegion::GrainBytes);
 // Create the gen rem set (and barrier set) for the entire reserved region.
 _rem_set = collector_policy()->create_rem_set(_reserved, 2);
 set_barrier_set(rem_set()->bs());
-if (barrier_set()->is_a(BarrierSet::ModRef)) {
+if (!barrier_set()->is_a(BarrierSet::G1SATBCTLogging)) {
-_mr_bs = (ModRefBarrierSet*)_barrier_set;
+vm_exit_during_initialization("G1 requires a G1SATBLoggingCardTableModRefBS");
-} else {
-vm_exit_during_initialization("G1 requires a mod ref bs.");
 return JNI_ENOMEM;
 }
 // Also create a G1 rem set.
-if (mr_bs()->is_a(BarrierSet::CardTableModRef)) {
+_g1_rem_set = new G1RemSet(this, g1_barrier_set());
-_g1_rem_set = new G1RemSet(this, (CardTableModRefBS*)mr_bs());
-} else {
-vm_exit_during_initialization("G1 requires a cardtable mod ref bs.");
-return JNI_ENOMEM;
-}
 // Carve out the G1 part of the heap.
 ReservedSpace g1_rs   = heap_rs.first_part(max_byte_size);
 _g1_reserved = MemRegion((HeapWord*)g1_rs.base(),
 g1_rs.size()/HeapWordSize);
 _g1_storage.initialize(g1_rs, 0);
 _g1_committed = MemRegion((HeapWord*)_g1_storage.low(), (size_t) 0);
 _hrs.initialize((HeapWord*) _g1_reserved.start(),
-(HeapWord*) _g1_reserved.end(),
+(HeapWord*) _g1_reserved.end());
-_expansion_regions);
+assert(_hrs.max_length() == _expansion_regions,
+err_msg("max length: %u expansion regions: %u",
+_hrs.max_length(), _expansion_regions));
 // Do later initialization work for concurrent refinement.
 _cg1r->init();
 // 6843694 - ensure that the maximum region index can fit
 // Do create of the monitoring and management support so that
 // values in the heap have been properly initialized.
 _g1mm = new G1MonitoringSupport(this);
 return JNI_OK;
+}
+size_t G1CollectedHeap::conservative_max_heap_alignment() {
+return HeapRegion::max_region_size();
 }
 void G1CollectedHeap::ref_processing_init() {
 // Reference processing in G1 currently works as follows:
 //
 trace_heap_before_gc(_gc_tracer_cm);
 }
 void G1CollectedHeap::register_concurrent_cycle_end() {
 if (_concurrent_cycle_started) {
-_gc_timer_cm->register_gc_end(os::elapsed_counter());
 if (_cm->has_aborted()) {
 _gc_tracer_cm->report_concurrent_mode_failure();
 }
+_gc_timer_cm->register_gc_end(os::elapsed_counter());
 _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions());
 _concurrent_cycle_started = false;
 }
 }
 case VerifyOption_G1UseMarkWord:    return "NONE";
 default:                            ShouldNotReachHere();
 }
 return NULL; // keep some compilers happy
 }
+// TODO: VerifyRootsClosure extends OopsInGenClosure so that we can
+//       pass it as the perm_blk to SharedHeap::process_strong_roots.
+//       When process_strong_roots stop calling perm_blk->younger_refs_iterate
+//       we can change this closure to extend the simpler OopClosure.
+class VerifyRootsClosure: public OopsInGenClosure {
+private:
+G1CollectedHeap* _g1h;
+VerifyOption     _vo;
+bool             _failures;
+public:
+// _vo == UsePrevMarking -> use "prev" marking information,
+// _vo == UseNextMarking -> use "next" marking information,
+// _vo == UseMarkWord    -> use mark word from object header.
+VerifyRootsClosure(VerifyOption vo) :
+_g1h(G1CollectedHeap::heap()),
+_vo(vo),
+_failures(false) { }
+bool failures() { return _failures; }
+template <class T> void do_oop_nv(T* p) {
+T heap_oop = oopDesc::load_heap_oop(p);
+if (!oopDesc::is_null(heap_oop)) {
+oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
+if (_g1h->is_obj_dead_cond(obj, _vo)) {
+gclog_or_tty->print_cr("Root location "PTR_FORMAT" "
+"points to dead obj "PTR_FORMAT, p, (void*) obj);
+if (_vo == VerifyOption_G1UseMarkWord) {
+gclog_or_tty->print_cr("  Mark word: "PTR_FORMAT, (void*)(obj->mark()));
+}
+obj->print_on(gclog_or_tty);
+_failures = true;
+}
+}
+}
+void do_oop(oop* p)       { do_oop_nv(p); }
+void do_oop(narrowOop* p) { do_oop_nv(p); }
+};
+class G1VerifyCodeRootOopClosure: public OopsInGenClosure {
+G1CollectedHeap* _g1h;
+OopClosure* _root_cl;
+nmethod* _nm;
+VerifyOption _vo;
+bool _failures;
+template <class T> void do_oop_work(T* p) {
+// First verify that this root is live
+_root_cl->do_oop(p);
+if (!G1VerifyHeapRegionCodeRoots) {
+// We're not verifying the code roots attached to heap region.
+return;
+}
+// Don't check the code roots during marking verification in a full GC
+if (_vo == VerifyOption_G1UseMarkWord) {
+return;
+}
+// Now verify that the current nmethod (which contains p) is
+// in the code root list of the heap region containing the
+// object referenced by p.
+T heap_oop = oopDesc::load_heap_oop(p);
+if (!oopDesc::is_null(heap_oop)) {
+oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
+// Now fetch the region containing the object
+HeapRegion* hr = _g1h->heap_region_containing(obj);
+HeapRegionRemSet* hrrs = hr->rem_set();
+// Verify that the strong code root list for this region
+// contains the nmethod
+if (!hrrs->strong_code_roots_list_contains(_nm)) {
+gclog_or_tty->print_cr("Code root location "PTR_FORMAT" "
+"from nmethod "PTR_FORMAT" not in strong "
+"code roots for region ["PTR_FORMAT","PTR_FORMAT")",
+p, _nm, hr->bottom(), hr->end());
+_failures = true;
+}
+}
+}
+public:
+G1VerifyCodeRootOopClosure(G1CollectedHeap* g1h, OopClosure* root_cl, VerifyOption vo):
+_g1h(g1h), _root_cl(root_cl), _vo(vo), _nm(NULL), _failures(false) {}
+void do_oop(oop* p) { do_oop_work(p); }
+void do_oop(narrowOop* p) { do_oop_work(p); }
+void set_nmethod(nmethod* nm) { _nm = nm; }
+bool failures() { return _failures; }
+};
+class G1VerifyCodeRootBlobClosure: public CodeBlobClosure {
+G1VerifyCodeRootOopClosure* _oop_cl;
+public:
+G1VerifyCodeRootBlobClosure(G1VerifyCodeRootOopClosure* oop_cl):
+_oop_cl(oop_cl) {}
+void do_code_blob(CodeBlob* cb) {
+nmethod* nm = cb->as_nmethod_or_null();
+if (nm != NULL) {
+_oop_cl->set_nmethod(nm);
+nm->oops_do(_oop_cl);
+}
+}
+};
+class YoungRefCounterClosure : public OopClosure {
+G1CollectedHeap* _g1h;
+int              _count;
+public:
+YoungRefCounterClosure(G1CollectedHeap* g1h) : _g1h(g1h), _count(0) {}
+void do_oop(oop* p)       { if (_g1h->is_in_young(*p)) { _count++; } }
+void do_oop(narrowOop* p) { ShouldNotReachHere(); }
+int count() { return _count; }
+void reset_count() { _count = 0; };
+};
+class VerifyKlassClosure: public KlassClosure {
+YoungRefCounterClosure _young_ref_counter_closure;
+OopClosure *_oop_closure;
+public:
+VerifyKlassClosure(G1CollectedHeap* g1h, OopClosure* cl) : _young_ref_counter_closure(g1h), _oop_closure(cl) {}
+void do_klass(Klass* k) {
+k->oops_do(_oop_closure);
+_young_ref_counter_closure.reset_count();
+k->oops_do(&_young_ref_counter_closure);
+if (_young_ref_counter_closure.count() > 0) {
+guarantee(k->has_modified_oops(), err_msg("Klass %p, has young refs but is not dirty.", k));
+}
+}
+};
 class VerifyLivenessOopClosure: public OopClosure {
 G1CollectedHeap* _g1h;
 VerifyOption _vo;
 public:
 }
 return false; // stop the region iteration if we hit a failure
 }
 };
-class YoungRefCounterClosure : public OopClosure {
+// This is the task used for parallel verification of the heap regions
-G1CollectedHeap* _g1h;
-int              _count;
-public:
-YoungRefCounterClosure(G1CollectedHeap* g1h) : _g1h(g1h), _count(0) {}
-void do_oop(oop* p)       { if (_g1h->is_in_young(*p)) { _count++; } }
-void do_oop(narrowOop* p) { ShouldNotReachHere(); }
-int count() { return _count; }
-void reset_count() { _count = 0; };
-};
-class VerifyKlassClosure: public KlassClosure {
-YoungRefCounterClosure _young_ref_counter_closure;
-OopClosure *_oop_closure;
-public:
-VerifyKlassClosure(G1CollectedHeap* g1h, OopClosure* cl) : _young_ref_counter_closure(g1h), _oop_closure(cl) {}
-void do_klass(Klass* k) {
-k->oops_do(_oop_closure);
-_young_ref_counter_closure.reset_count();
-k->oops_do(&_young_ref_counter_closure);
-if (_young_ref_counter_closure.count() > 0) {
-guarantee(k->has_modified_oops(), err_msg("Klass %p, has young refs but is not dirty.", k));
-}
-}
-};
-// TODO: VerifyRootsClosure extends OopsInGenClosure so that we can
-//       pass it as the perm_blk to SharedHeap::process_strong_roots.
-//       When process_strong_roots stop calling perm_blk->younger_refs_iterate
-//       we can change this closure to extend the simpler OopClosure.
-class VerifyRootsClosure: public OopsInGenClosure {
-private:
-G1CollectedHeap* _g1h;
-VerifyOption     _vo;
-bool             _failures;
-public:
-// _vo == UsePrevMarking -> use "prev" marking information,
-// _vo == UseNextMarking -> use "next" marking information,
-// _vo == UseMarkWord    -> use mark word from object header.
-VerifyRootsClosure(VerifyOption vo) :
-_g1h(G1CollectedHeap::heap()),
-_vo(vo),
-_failures(false) { }
-bool failures() { return _failures; }
-template <class T> void do_oop_nv(T* p) {
-T heap_oop = oopDesc::load_heap_oop(p);
-if (!oopDesc::is_null(heap_oop)) {
-oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
-if (_g1h->is_obj_dead_cond(obj, _vo)) {
-gclog_or_tty->print_cr("Root location "PTR_FORMAT" "
-"points to dead obj "PTR_FORMAT, p, (void*) obj);
-if (_vo == VerifyOption_G1UseMarkWord) {
-gclog_or_tty->print_cr("  Mark word: "PTR_FORMAT, (void*)(obj->mark()));
-}
-obj->print_on(gclog_or_tty);
-_failures = true;
-}
-}
-}
-void do_oop(oop* p)       { do_oop_nv(p); }
-void do_oop(narrowOop* p) { do_oop_nv(p); }
-};
-// This is the task used for parallel heap verification.
 class G1ParVerifyTask: public AbstractGangTask {
 private:
 G1CollectedHeap* _g1h;
 VerifyOption     _vo;
 _failures = true;
 }
 }
 };
-void G1CollectedHeap::verify(bool silent) {
+void G1CollectedHeap::verify(bool silent, VerifyOption vo) {
-verify(silent, VerifyOption_G1UsePrevMarking);
-}
-void G1CollectedHeap::verify(bool silent,
-VerifyOption vo) {
 if (SafepointSynchronize::is_at_safepoint()) {
+assert(Thread::current()->is_VM_thread(),
+"Expected to be executed serially by the VM thread at this point");
 if (!silent) { gclog_or_tty->print("Roots "); }
 VerifyRootsClosure rootsCl(vo);
+G1VerifyCodeRootOopClosure codeRootsCl(this, &rootsCl, vo);
-assert(Thread::current()->is_VM_thread(),
+G1VerifyCodeRootBlobClosure blobsCl(&codeRootsCl);
-"Expected to be executed serially by the VM thread at this point");
-CodeBlobToOopClosure blobsCl(&rootsCl, /*do_marking=*/ false);
 VerifyKlassClosure klassCl(this, &rootsCl);
 // We apply the relevant closures to all the oops in the
 // system dictionary, the string table and the code cache.
 const int so = SO_AllClasses | SO_Strings | SO_CodeCache;
 &rootsCl,
 &blobsCl,
 &klassCl
 );
-bool failures = rootsCl.failures();
+bool failures = rootsCl.failures() || codeRootsCl.failures();
 if (vo != VerifyOption_G1UseMarkWord) {
 // If we're verifying during a full GC then the region sets
 // will have been torn down at the start of the GC. Therefore
 // verifying the region sets will fail. So we only verify
 guarantee(!failures, "there should not have been any failures");
 } else {
 if (!silent)
 gclog_or_tty->print("(SKIPPING roots, heapRegionSets, heapRegions, remset) ");
 }
+}
+void G1CollectedHeap::verify(bool silent) {
+verify(silent, VerifyOption_G1UsePrevMarking);
+}
+double G1CollectedHeap::verify(bool guard, const char* msg) {
+double verify_time_ms = 0.0;
+if (guard && total_collections() >= VerifyGCStartAt) {
+double verify_start = os::elapsedTime();
+HandleMark hm;  // Discard invalid handles created during verification
+prepare_for_verify();
+Universe::verify(VerifyOption_G1UsePrevMarking, msg);
+verify_time_ms = (os::elapsedTime() - verify_start) * 1000;
+}
+return verify_time_ms;
+}
+void G1CollectedHeap::verify_before_gc() {
+double verify_time_ms = verify(VerifyBeforeGC, " VerifyBeforeGC:");
+g1_policy()->phase_times()->record_verify_before_time_ms(verify_time_ms);
+}
+void G1CollectedHeap::verify_after_gc() {
+double verify_time_ms = verify(VerifyAfterGC, " VerifyAfterGC:");
+g1_policy()->phase_times()->record_verify_after_time_ms(verify_time_ms);
 }
 class PrintRegionClosure: public HeapRegionClosure {
 outputStream* _st;
 public:
 void G1CollectedHeap::gc_prologue(bool full /* Ignored */) {
 // always_do_update_barrier = false;
 assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer");
 // Fill TLAB's and such
 ensure_parsability(true);
+if (G1SummarizeRSetStats && (G1SummarizeRSetStatsPeriod > 0) &&
+(total_collections() % G1SummarizeRSetStatsPeriod == 0)) {
+g1_rem_set()->print_periodic_summary_info("Before GC RS summary");
+}
 }
 void G1CollectedHeap::gc_epilogue(bool full /* Ignored */) {
 if (G1SummarizeRSetStats &&
 (G1SummarizeRSetStatsPeriod > 0) &&
 // we are at the end of the GC. Total collections has already been increased.
 ((total_collections() - 1) % G1SummarizeRSetStatsPeriod == 0)) {
-g1_rem_set()->print_periodic_summary_info();
+g1_rem_set()->print_periodic_summary_info("After GC RS summary");
 }
 // FIXME: what is this about?
 // I'm ignoring the "fill_newgen()" call if "alloc_event_enabled"
 // is set.
 Universe::update_heap_info_at_gc();
 }
 HeapWord* G1CollectedHeap::do_collection_pause(size_t word_size,
 unsigned int gc_count_before,
-bool* succeeded) {
+bool* succeeded,
+GCCause::Cause gc_cause) {
 assert_heap_not_locked_and_not_at_safepoint();
 g1_policy()->record_stop_world_start();
 VM_G1IncCollectionPause op(gc_count_before,
 word_size,
 false, /* should_initiate_conc_mark */
 g1_policy()->max_pause_time_ms(),
-GCCause::_g1_inc_collection_pause);
+gc_cause);
 VMThread::execute(&op);
 HeapWord* result = op.result();
 bool ret_succeeded = op.prologue_succeeded() && op.pause_succeeded();
 assert(result == NULL || ret_succeeded,
 // get entries from the secondary_free_list.
 if (!G1StressConcRegionFreeing) {
 append_secondary_free_list_if_not_empty_with_lock();
 }
-assert(check_young_list_well_formed(),
+assert(check_young_list_well_formed(), "young list should be well formed");
-"young list should be well formed");
+assert(check_heap_region_claim_values(HeapRegion::InitialClaimValue),
+"sanity check");
 // Don't dynamically change the number of GC threads this early.  A value of
 // 0 is used to indicate serial work.  When parallel work is done,
 // it will be set.
 G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, uint queue_num)
 : _g1h(g1h),
 _refs(g1h->task_queue(queue_num)),
 _dcq(&g1h->dirty_card_queue_set()),
-_ct_bs((CardTableModRefBS*)_g1h->barrier_set()),
+_ct_bs(g1h->g1_barrier_set()),
 _g1_rem(g1h->g1_rem_set()),
 _hash_seed(17), _queue_num(queue_num),
 _term_attempts(0),
 _surviving_alloc_buffer(g1h->desired_plab_sz(GCAllocForSurvived)),
 _tenured_alloc_buffer(g1h->desired_plab_sz(GCAllocForTenured)),
 assert(ref != NULL, "invariant");
 assert(UseCompressedOops, "sanity");
 assert(!has_partial_array_mask(ref), err_msg("ref=" PTR_FORMAT, ref));
 oop p = oopDesc::load_decode_heap_oop(ref);
 assert(_g1h->is_in_g1_reserved(p),
-err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, ref, intptr_t(p)));
+err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, ref, (void *)p));
 return true;
 }
 bool G1ParScanThreadState::verify_ref(oop* ref) const {
 assert(ref != NULL, "invariant");
 if (has_partial_array_mask(ref)) {
 // Must be in the collection set--it's already been copied.
 oop p = clear_partial_array_mask(ref);
 assert(_g1h->obj_in_cs(p),
-err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, ref, intptr_t(p)));
+err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, ref, (void *)p));
 } else {
 oop p = oopDesc::load_decode_heap_oop(ref);
 assert(_g1h->is_in_g1_reserved(p),
-err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, ref, intptr_t(p)));
+err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, ref, (void *)p));
 }
 return true;
 }
 bool G1ParScanThreadState::verify_task(StarTask ref) const {
 scan_klasses_cl = &scan_mark_klasses_cl_s;
 }
 G1ParPushHeapRSClosure          push_heap_rs_cl(_g1h, &pss);
-int so = SharedHeap::SO_AllClasses | SharedHeap::SO_Strings | SharedHeap::SO_CodeCache;
+// Don't scan the scavengable methods in the code cache as part
+// of strong root scanning. The code roots that point into a
+// region in the collection set are scanned when we scan the
+// region's RSet.
+int so = SharedHeap::SO_AllClasses | SharedHeap::SO_Strings;
 pss.start_strong_roots();
 _g1h->g1_process_strong_roots(/* is scavenging */ true,
 SharedHeap::ScanningOption(so),
 scan_root_cl,
 }
 };
 // *** Common G1 Evacuation Stuff
-// Closures that support the filtering of CodeBlobs scanned during
-// external root scanning.
-// Closure applied to reference fields in code blobs (specifically nmethods)
-// to determine whether an nmethod contains references that point into
-// the collection set. Used as a predicate when walking code roots so
-// that only nmethods that point into the collection set are added to the
-// 'marked' list.
-class G1FilteredCodeBlobToOopClosure : public CodeBlobToOopClosure {
-class G1PointsIntoCSOopClosure : public OopClosure {
-G1CollectedHeap* _g1;
-bool _points_into_cs;
-public:
-G1PointsIntoCSOopClosure(G1CollectedHeap* g1) :
-_g1(g1), _points_into_cs(false) { }
-bool points_into_cs() const { return _points_into_cs; }
-template <class T>
-void do_oop_nv(T* p) {
-if (!_points_into_cs) {
-T heap_oop = oopDesc::load_heap_oop(p);
-if (!oopDesc::is_null(heap_oop) &&
-_g1->in_cset_fast_test(oopDesc::decode_heap_oop_not_null(heap_oop))) {
-_points_into_cs = true;
-}
-}
-}
-virtual void do_oop(oop* p)        { do_oop_nv(p); }
-virtual void do_oop(narrowOop* p)  { do_oop_nv(p); }
-};
-G1CollectedHeap* _g1;
-public:
-G1FilteredCodeBlobToOopClosure(G1CollectedHeap* g1, OopClosure* cl) :
-CodeBlobToOopClosure(cl, true), _g1(g1) { }
-virtual void do_code_blob(CodeBlob* cb) {
-nmethod* nm = cb->as_nmethod_or_null();
-if (nm != NULL && !(nm->test_oops_do_mark())) {
-G1PointsIntoCSOopClosure predicate_cl(_g1);
-nm->oops_do(&predicate_cl);
-if (predicate_cl.points_into_cs()) {
-// At least one of the reference fields or the oop relocations
-// in the nmethod points into the collection set. We have to
-// 'mark' this nmethod.
-// Note: Revisit the following if CodeBlobToOopClosure::do_code_blob()
-// or MarkingCodeBlobClosure::do_code_blob() change.
-if (!nm->test_set_oops_do_mark()) {
-do_newly_marked_nmethod(nm);
-}
-}
-}
-}
-};
 // This method is run in a GC worker.
 void
 G1CollectedHeap::
 g1_process_strong_roots(bool is_scavenging,
 double ext_roots_start = os::elapsedTime();
 double closure_app_time_sec = 0.0;
 BufferingOopClosure buf_scan_non_heap_roots(scan_non_heap_roots);
-// Walk the code cache w/o buffering, because StarTask cannot handle
+assert(so & SO_CodeCache || scan_rs != NULL, "must scan code roots somehow");
-// unaligned oop locations.
+// Walk the code cache/strong code roots w/o buffering, because StarTask
-G1FilteredCodeBlobToOopClosure eager_scan_cs_code_roots(this, scan_non_heap_roots);
+// cannot handle unaligned oop locations.
+CodeBlobToOopClosure eager_scan_code_roots(scan_non_heap_roots, true /* do_marking */);
-// Scan all code roots from stack
-CodeBlobToOopClosure eager_scan_all_code_roots(scan_non_heap_roots, true);
-CodeBlobToOopClosure* blobs = &eager_scan_cs_code_roots;
-if (UseNewCode && g1_policy()->during_initial_mark_pause()) {
-// during initial-mark we need to take care to follow all code roots
-blobs = &eager_scan_all_code_roots;
-}
 process_strong_roots(false, // no scoping; this is parallel code
 is_scavenging, so,
 &buf_scan_non_heap_roots,
-blobs,
+&eager_scan_code_roots,
 scan_klasses
 );
 // Now the CM ref_processor roots.
 if (!_process_strong_tasks->is_task_claimed(G1H_PS_refProcessor_oops_do)) {
 satb_filtering_ms = (os::elapsedTime() - satb_filter_start) * 1000.0;
 }
 }
 g1_policy()->phase_times()->record_satb_filtering_time(worker_i, satb_filtering_ms);
+// If this is an initial mark pause, and we're not scanning
+// the entire code cache, we need to mark the oops in the
+// strong code root lists for the regions that are not in
+// the collection set.
+// Note all threads participate in this set of root tasks.
+double mark_strong_code_roots_ms = 0.0;
+if (g1_policy()->during_initial_mark_pause() && !(so & SO_CodeCache)) {
+double mark_strong_roots_start = os::elapsedTime();
+mark_strong_code_roots(worker_i);
+mark_strong_code_roots_ms = (os::elapsedTime() - mark_strong_roots_start) * 1000.0;
+}
+g1_policy()->phase_times()->record_strong_code_root_mark_time(worker_i, mark_strong_code_roots_ms);
 // Now scan the complement of the collection set.
 if (scan_rs != NULL) {
-g1_rem_set()->oops_into_collection_set_do(scan_rs, worker_i);
+g1_rem_set()->oops_into_collection_set_do(scan_rs, &eager_scan_code_roots, worker_i);
 }
 _process_strong_tasks->all_tasks_completed();
 }
 void
 // objects (and their reachable sub-graphs) that were
 // not copied during the pause.
 process_discovered_references(n_workers);
 // Weak root processing.
-// Note: when JSR 292 is enabled and code blobs can contain
-// non-perm oops then we will need to process the code blobs
-// here too.
 {
 G1STWIsAliveClosure is_alive(this);
 G1KeepAliveClosure keep_alive(this);
 JNIHandles::weak_oops_do(&is_alive, &keep_alive);
 }
 // Reset and re-enable the hot card cache.
 // Note the counts for the cards in the regions in the
 // collection set are reset when the collection set is freed.
 hot_card_cache->reset_hot_cache();
 hot_card_cache->set_use_cache(true);
+// Migrate the strong code roots attached to each region in
+// the collection set. Ideally we would like to do this
+// after we have finished the scanning/evacuation of the
+// strong code roots for a particular heap region.
+migrate_strong_code_roots();
+if (g1_policy()->during_initial_mark_pause()) {
+// Reset the claim values set during marking the strong code roots
+reset_heap_region_claim_values();
+}
 finalize_for_evac_failure();
 if (evacuation_failed()) {
 remove_self_forwarding_pointers();
 _humongous_set.update_from_proxy(humongous_proxy_set);
 }
 }
 class G1ParCleanupCTTask : public AbstractGangTask {
-CardTableModRefBS* _ct_bs;
+G1SATBCardTableModRefBS* _ct_bs;
 G1CollectedHeap* _g1h;
 HeapRegion* volatile _su_head;
 public:
-G1ParCleanupCTTask(CardTableModRefBS* ct_bs,
+G1ParCleanupCTTask(G1SATBCardTableModRefBS* ct_bs,
 G1CollectedHeap* g1h) :
 AbstractGangTask("G1 Par Cleanup CT Task"),
 _ct_bs(ct_bs), _g1h(g1h) { }
 void work(uint worker_id) {
 };
 #ifndef PRODUCT
 class G1VerifyCardTableCleanup: public HeapRegionClosure {
 G1CollectedHeap* _g1h;
-CardTableModRefBS* _ct_bs;
+G1SATBCardTableModRefBS* _ct_bs;
 public:
-G1VerifyCardTableCleanup(G1CollectedHeap* g1h, CardTableModRefBS* ct_bs)
+G1VerifyCardTableCleanup(G1CollectedHeap* g1h, G1SATBCardTableModRefBS* ct_bs)
 : _g1h(g1h), _ct_bs(ct_bs) { }
 virtual bool doHeapRegion(HeapRegion* r) {
 if (r->is_survivor()) {
 _g1h->verify_dirty_region(r);
 } else {
 }
 };
 void G1CollectedHeap::verify_not_dirty_region(HeapRegion* hr) {
 // All of the region should be clean.
-CardTableModRefBS* ct_bs = (CardTableModRefBS*)barrier_set();
+G1SATBCardTableModRefBS* ct_bs = g1_barrier_set();
 MemRegion mr(hr->bottom(), hr->end());
 ct_bs->verify_not_dirty_region(mr);
 }
 void G1CollectedHeap::verify_dirty_region(HeapRegion* hr) {
 // retires each region and replaces it with a new one will do a
 // maximal allocation to fill in [pre_dummy_top(),end()] but will
 // not dirty that area (one less thing to have to do while holding
 // a lock). So we can only verify that [bottom(),pre_dummy_top()]
 // is dirty.
-CardTableModRefBS* ct_bs = (CardTableModRefBS*) barrier_set();
+G1SATBCardTableModRefBS* ct_bs = g1_barrier_set();
 MemRegion mr(hr->bottom(), hr->pre_dummy_top());
 ct_bs->verify_dirty_region(mr);
 }
 void G1CollectedHeap::verify_dirty_young_list(HeapRegion* head) {
-CardTableModRefBS* ct_bs = (CardTableModRefBS*) barrier_set();
+G1SATBCardTableModRefBS* ct_bs = g1_barrier_set();
 for (HeapRegion* hr = head; hr != NULL; hr = hr->get_next_young_region()) {
 verify_dirty_region(hr);
 }
 }
 verify_dirty_young_list(_young_list->first_region());
 }
 #endif
 void G1CollectedHeap::cleanUpCardTable() {
-CardTableModRefBS* ct_bs = (CardTableModRefBS*) (barrier_set());
+G1SATBCardTableModRefBS* ct_bs = g1_barrier_set();
 double start = os::elapsedTime();
 {
 // Iterate over the dirty cards region list.
 G1ParCleanupCTTask cleanup_task(ct_bs, this);
 _old_set.verify_end();
 _humongous_set.verify_end();
 _free_list.verify_end();
 }
+// Optimized nmethod scanning
+class RegisterNMethodOopClosure: public OopClosure {
+G1CollectedHeap* _g1h;
+nmethod* _nm;
+template <class T> void do_oop_work(T* p) {
+T heap_oop = oopDesc::load_heap_oop(p);
+if (!oopDesc::is_null(heap_oop)) {
+oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
+HeapRegion* hr = _g1h->heap_region_containing(obj);
+assert(!hr->isHumongous(), "code root in humongous region?");
+// HeapRegion::add_strong_code_root() avoids adding duplicate
+// entries but having duplicates is  OK since we "mark" nmethods
+// as visited when we scan the strong code root lists during the GC.
+hr->add_strong_code_root(_nm);
+assert(hr->rem_set()->strong_code_roots_list_contains(_nm), "add failed?");
+}
+}
+public:
+RegisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) :
+_g1h(g1h), _nm(nm) {}
+void do_oop(oop* p)       { do_oop_work(p); }
+void do_oop(narrowOop* p) { do_oop_work(p); }
+};
+class UnregisterNMethodOopClosure: public OopClosure {
+G1CollectedHeap* _g1h;
+nmethod* _nm;
+template <class T> void do_oop_work(T* p) {
+T heap_oop = oopDesc::load_heap_oop(p);
+if (!oopDesc::is_null(heap_oop)) {
+oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
+HeapRegion* hr = _g1h->heap_region_containing(obj);
+assert(!hr->isHumongous(), "code root in humongous region?");
+hr->remove_strong_code_root(_nm);
+assert(!hr->rem_set()->strong_code_roots_list_contains(_nm), "remove failed?");
+}
+}
+public:
+UnregisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) :
+_g1h(g1h), _nm(nm) {}
+void do_oop(oop* p)       { do_oop_work(p); }
+void do_oop(narrowOop* p) { do_oop_work(p); }
+};
+void G1CollectedHeap::register_nmethod(nmethod* nm) {
+CollectedHeap::register_nmethod(nm);
+guarantee(nm != NULL, "sanity");
+RegisterNMethodOopClosure reg_cl(this, nm);
+nm->oops_do(&reg_cl);
+}
+void G1CollectedHeap::unregister_nmethod(nmethod* nm) {
+CollectedHeap::unregister_nmethod(nm);
+guarantee(nm != NULL, "sanity");
+UnregisterNMethodOopClosure reg_cl(this, nm);
+nm->oops_do(&reg_cl, true);
+}
+class MigrateCodeRootsHeapRegionClosure: public HeapRegionClosure {
+public:
+bool doHeapRegion(HeapRegion *hr) {
+assert(!hr->isHumongous(), "humongous region in collection set?");
+hr->migrate_strong_code_roots();
+return false;
+}
+};
+void G1CollectedHeap::migrate_strong_code_roots() {
+MigrateCodeRootsHeapRegionClosure cl;
+double migrate_start = os::elapsedTime();
+collection_set_iterate(&cl);
+double migration_time_ms = (os::elapsedTime() - migrate_start) * 1000.0;
+g1_policy()->phase_times()->record_strong_code_root_migration_time(migration_time_ms);
+}
+// Mark all the code roots that point into regions *not* in the
+// collection set.
+//
+// Note we do not want to use a "marking" CodeBlobToOopClosure while
+// walking the the code roots lists of regions not in the collection
+// set. Suppose we have an nmethod (M) that points to objects in two
+// separate regions - one in the collection set (R1) and one not (R2).
+// Using a "marking" CodeBlobToOopClosure here would result in "marking"
+// nmethod M when walking the code roots for R1. When we come to scan
+// the code roots for R2, we would see that M is already marked and it
+// would be skipped and the objects in R2 that are referenced from M
+// would not be evacuated.
+class MarkStrongCodeRootCodeBlobClosure: public CodeBlobClosure {
+class MarkStrongCodeRootOopClosure: public OopClosure {
+ConcurrentMark* _cm;
+HeapRegion* _hr;
+uint _worker_id;
+template <class T> void do_oop_work(T* p) {
+T heap_oop = oopDesc::load_heap_oop(p);
+if (!oopDesc::is_null(heap_oop)) {
+oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
+// Only mark objects in the region (which is assumed
+// to be not in the collection set).
+if (_hr->is_in(obj)) {
+_cm->grayRoot(obj, (size_t) obj->size(), _worker_id);
+}
+}
+}
+public:
+MarkStrongCodeRootOopClosure(ConcurrentMark* cm, HeapRegion* hr, uint worker_id) :
+_cm(cm), _hr(hr), _worker_id(worker_id) {
+assert(!_hr->in_collection_set(), "sanity");
+}
+void do_oop(narrowOop* p) { do_oop_work(p); }
+void do_oop(oop* p)       { do_oop_work(p); }
+};
+MarkStrongCodeRootOopClosure _oop_cl;
+public:
+MarkStrongCodeRootCodeBlobClosure(ConcurrentMark* cm, HeapRegion* hr, uint worker_id):
+_oop_cl(cm, hr, worker_id) {}
+void do_code_blob(CodeBlob* cb) {
+nmethod* nm = (cb == NULL) ? NULL : cb->as_nmethod_or_null();
+if (nm != NULL) {
+nm->oops_do(&_oop_cl);
+}
+}
+};
+class MarkStrongCodeRootsHRClosure: public HeapRegionClosure {
+G1CollectedHeap* _g1h;
+uint _worker_id;
+public:
+MarkStrongCodeRootsHRClosure(G1CollectedHeap* g1h, uint worker_id) :
+_g1h(g1h), _worker_id(worker_id) {}
+bool doHeapRegion(HeapRegion *hr) {
+HeapRegionRemSet* hrrs = hr->rem_set();
+if (hr->isHumongous()) {
+// Code roots should never be attached to a humongous region
+assert(hrrs->strong_code_roots_list_length() == 0, "sanity");
+return false;
+}
+if (hr->in_collection_set()) {
+// Don't mark code roots into regions in the collection set here.
+// They will be marked when we scan them.
+return false;
+}
+MarkStrongCodeRootCodeBlobClosure cb_cl(_g1h->concurrent_mark(), hr, _worker_id);
+hr->strong_code_roots_do(&cb_cl);
+return false;
+}
+};
+void G1CollectedHeap::mark_strong_code_roots(uint worker_id) {
+MarkStrongCodeRootsHRClosure cl(this, worker_id);
+if (G1CollectedHeap::use_parallel_gc_threads()) {
+heap_region_par_iterate_chunked(&cl,
+worker_id,
+workers()->active_workers(),
+HeapRegion::ParMarkRootClaimValue);
+} else {
+heap_region_iterate(&cl);
+}
+}
+class RebuildStrongCodeRootClosure: public CodeBlobClosure {
+G1CollectedHeap* _g1h;
+public:
+RebuildStrongCodeRootClosure(G1CollectedHeap* g1h) :
+_g1h(g1h) {}
+void do_code_blob(CodeBlob* cb) {
+nmethod* nm = (cb != NULL) ? cb->as_nmethod_or_null() : NULL;
+if (nm == NULL) {
+return;
+}
+if (ScavengeRootsInCode && nm->detect_scavenge_root_oops()) {
+_g1h->register_nmethod(nm);
+}
+}
+};
+void G1CollectedHeap::rebuild_strong_code_roots() {
+RebuildStrongCodeRootClosure blob_cl(this);
+CodeCache::blobs_do(&blob_cl);
+}

Mercurial > hg > graal-compiler

comparison src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp @ 12355:cefad50507d8