comparison src/share/vm/gc_implementation/g1/g1RemSet.cpp @ 1705:2d160770d2e5

6814437: G1: remove the _new_refs array Summary: The per-worker _new_refs array is used to hold references that point into the collection set. It is populated during RSet updating and subsequently processed. In the event of an evacuation failure it processed again to recreate the RSets of regions in the collection set. Remove the per-worker _new_refs array by processing the references directly. Use a DirtyCardQueue to hold the cards containing the references so that the RSets of regions in the collection set can be recreated when handling an evacuation failure. Reviewed-by: iveresov, jmasa, tonyp
author johnc
date Mon, 02 Aug 2010 12:51:43 -0700
parents 5cbac8938c4c
children a03ae377b2e8
comparison
equal deleted inserted replaced
1704:63f4675ac87d 1705:2d160770d2e5
120 }; 120 };
121 121
122 HRInto_G1RemSet::HRInto_G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs) 122 HRInto_G1RemSet::HRInto_G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs)
123 : G1RemSet(g1), _ct_bs(ct_bs), _g1p(_g1->g1_policy()), 123 : G1RemSet(g1), _ct_bs(ct_bs), _g1p(_g1->g1_policy()),
124 _cg1r(g1->concurrent_g1_refine()), 124 _cg1r(g1->concurrent_g1_refine()),
125 _par_traversal_in_progress(false), _new_refs(NULL), 125 _par_traversal_in_progress(false),
126 _cset_rs_update_cl(NULL),
126 _cards_scanned(NULL), _total_cards_scanned(0) 127 _cards_scanned(NULL), _total_cards_scanned(0)
127 { 128 {
128 _seq_task = new SubTasksDone(NumSeqTasks); 129 _seq_task = new SubTasksDone(NumSeqTasks);
129 guarantee(n_workers() > 0, "There should be some workers"); 130 guarantee(n_workers() > 0, "There should be some workers");
130 _new_refs = NEW_C_HEAP_ARRAY(GrowableArray<OopOrNarrowOopStar>*, n_workers()); 131 _cset_rs_update_cl = NEW_C_HEAP_ARRAY(OopsInHeapRegionClosure*, n_workers());
131 for (uint i = 0; i < n_workers(); i++) { 132 for (uint i = 0; i < n_workers(); i++) {
132 _new_refs[i] = new (ResourceObj::C_HEAP) GrowableArray<OopOrNarrowOopStar>(8192,true); 133 _cset_rs_update_cl[i] = NULL;
133 } 134 }
134 } 135 }
135 136
136 HRInto_G1RemSet::~HRInto_G1RemSet() { 137 HRInto_G1RemSet::~HRInto_G1RemSet() {
137 delete _seq_task; 138 delete _seq_task;
138 for (uint i = 0; i < n_workers(); i++) { 139 for (uint i = 0; i < n_workers(); i++) {
139 delete _new_refs[i]; 140 assert(_cset_rs_update_cl[i] == NULL, "it should be");
140 } 141 }
141 FREE_C_HEAP_ARRAY(GrowableArray<OopOrNarrowOopStar>*, _new_refs); 142 FREE_C_HEAP_ARRAY(OopsInHeapRegionClosure*, _cset_rs_update_cl);
142 } 143 }
143 144
144 void CountNonCleanMemRegionClosure::do_MemRegion(MemRegion mr) { 145 void CountNonCleanMemRegionClosure::do_MemRegion(MemRegion mr) {
145 if (_g1->is_in_g1_reserved(mr.start())) { 146 if (_g1->is_in_g1_reserved(mr.start())) {
146 _n += (int) ((mr.byte_size() / CardTableModRefBS::card_size)); 147 _n += (int) ((mr.byte_size() / CardTableModRefBS::card_size));
304 _cards_scanned[worker_i] = scanRScl.cards_done(); 305 _cards_scanned[worker_i] = scanRScl.cards_done();
305 306
306 _g1p->record_scan_rs_time(worker_i, scan_rs_time_sec * 1000.0); 307 _g1p->record_scan_rs_time(worker_i, scan_rs_time_sec * 1000.0);
307 } 308 }
308 309
309 void HRInto_G1RemSet::updateRS(int worker_i) { 310 // Closure used for updating RSets and recording references that
310 ConcurrentG1Refine* cg1r = _g1->concurrent_g1_refine(); 311 // point into the collection set. Only called during an
311 312 // evacuation pause.
313
314 class RefineRecordRefsIntoCSCardTableEntryClosure: public CardTableEntryClosure {
315 G1RemSet* _g1rs;
316 DirtyCardQueue* _into_cset_dcq;
317 public:
318 RefineRecordRefsIntoCSCardTableEntryClosure(G1CollectedHeap* g1h,
319 DirtyCardQueue* into_cset_dcq) :
320 _g1rs(g1h->g1_rem_set()), _into_cset_dcq(into_cset_dcq)
321 {}
322 bool do_card_ptr(jbyte* card_ptr, int worker_i) {
323 // The only time we care about recording cards that
324 // contain references that point into the collection set
325 // is during RSet updating within an evacuation pause.
326 // In this case worker_i should be the id of a GC worker thread.
327 assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause");
328 assert(worker_i < (int) DirtyCardQueueSet::num_par_ids(), "should be a GC worker");
329
330 if (_g1rs->concurrentRefineOneCard(card_ptr, worker_i, true)) {
331 // 'card_ptr' contains references that point into the collection
332 // set. We need to record the card in the DCQS
333 // (G1CollectedHeap::into_cset_dirty_card_queue_set())
334 // that's used for that purpose.
335 //
336 // Enqueue the card
337 _into_cset_dcq->enqueue(card_ptr);
338 }
339 return true;
340 }
341 };
342
343 void HRInto_G1RemSet::updateRS(DirtyCardQueue* into_cset_dcq, int worker_i) {
312 double start = os::elapsedTime(); 344 double start = os::elapsedTime();
313 // Apply the appropriate closure to all remaining log entries. 345 // Apply the given closure to all remaining log entries.
314 _g1->iterate_dirty_card_closure(false, worker_i); 346 RefineRecordRefsIntoCSCardTableEntryClosure into_cset_update_rs_cl(_g1, into_cset_dcq);
347 _g1->iterate_dirty_card_closure(&into_cset_update_rs_cl, into_cset_dcq, false, worker_i);
348
315 // Now there should be no dirty cards. 349 // Now there should be no dirty cards.
316 if (G1RSLogCheckCardTable) { 350 if (G1RSLogCheckCardTable) {
317 CountNonCleanMemRegionClosure cl(_g1); 351 CountNonCleanMemRegionClosure cl(_g1);
318 _ct_bs->mod_card_iterate(&cl); 352 _ct_bs->mod_card_iterate(&cl);
319 // XXX This isn't true any more: keeping cards of young regions 353 // XXX This isn't true any more: keeping cards of young regions
403 } 437 }
404 gclog_or_tty->print_cr(" > %8d %8d", (1 << (MIN+mx-2))+1, _histo[mx-1]); 438 gclog_or_tty->print_cr(" > %8d %8d", (1 << (MIN+mx-2))+1, _histo[mx-1]);
405 } 439 }
406 }; 440 };
407 441
408 template <class T> void
409 HRInto_G1RemSet::scanNewRefsRS_work(OopsInHeapRegionClosure* oc,
410 int worker_i) {
411 double scan_new_refs_start_sec = os::elapsedTime();
412 G1CollectedHeap* g1h = G1CollectedHeap::heap();
413 CardTableModRefBS* ct_bs = (CardTableModRefBS*) (g1h->barrier_set());
414 for (int i = 0; i < _new_refs[worker_i]->length(); i++) {
415 T* p = (T*) _new_refs[worker_i]->at(i);
416 oop obj = oopDesc::load_decode_heap_oop(p);
417 // *p was in the collection set when p was pushed on "_new_refs", but
418 // another thread may have processed this location from an RS, so it
419 // might not point into the CS any longer. If so, it's obviously been
420 // processed, and we don't need to do anything further.
421 if (g1h->obj_in_cs(obj)) {
422 HeapRegion* r = g1h->heap_region_containing(p);
423
424 DEBUG_ONLY(HeapRegion* to = g1h->heap_region_containing(obj));
425 oc->set_region(r);
426 // If "p" has already been processed concurrently, this is
427 // idempotent.
428 oc->do_oop(p);
429 }
430 }
431 double scan_new_refs_time_ms = (os::elapsedTime() - scan_new_refs_start_sec) * 1000.0;
432 _g1p->record_scan_new_refs_time(worker_i, scan_new_refs_time_ms);
433 }
434
435 void HRInto_G1RemSet::cleanupHRRS() { 442 void HRInto_G1RemSet::cleanupHRRS() {
436 HeapRegionRemSet::cleanup(); 443 HeapRegionRemSet::cleanup();
437 } 444 }
438 445
439 void 446 void
455 count_cl.n(), (float)count_cl.tot()/(float)count_cl.n(), 462 count_cl.n(), (float)count_cl.tot()/(float)count_cl.n(),
456 count_cl.mx(), count_cl.mxr()); 463 count_cl.mx(), count_cl.mxr());
457 count_cl.print_histo(); 464 count_cl.print_histo();
458 } 465 }
459 466
467 // We cache the value of 'oc' closure into the appropriate slot in the
468 // _cset_rs_update_cl for this worker
469 assert(worker_i < (int)n_workers(), "sanity");
470 _cset_rs_update_cl[worker_i] = oc;
471
472 // A DirtyCardQueue that is used to hold cards containing references
473 // that point into the collection set. This DCQ is associated with a
474 // special DirtyCardQueueSet (see g1CollectedHeap.hpp). Under normal
475 // circumstances (i.e. the pause successfully completes), these cards
476 // are just discarded (there's no need to update the RSets of regions
477 // that were in the collection set - after the pause these regions
478 // are wholly 'free' of live objects. In the event of an evacuation
479 // failure the cards/buffers in this queue set are:
480 // * passed to the DirtyCardQueueSet that is used to manage deferred
481 // RSet updates, or
482 // * scanned for references that point into the collection set
483 // and the RSet of the corresponding region in the collection set
484 // is updated immediately.
485 DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set());
486
460 if (ParallelGCThreads > 0) { 487 if (ParallelGCThreads > 0) {
461 // The two flags below were introduced temporarily to serialize 488 // The two flags below were introduced temporarily to serialize
462 // the updating and scanning of remembered sets. There are some 489 // the updating and scanning of remembered sets. There are some
463 // race conditions when these two operations are done in parallel 490 // race conditions when these two operations are done in parallel
464 // and they are causing failures. When we resolve said race 491 // and they are causing failures. When we resolve said race
465 // conditions, we'll revert back to parallel remembered set 492 // conditions, we'll revert back to parallel remembered set
466 // updating and scanning. See CRs 6677707 and 6677708. 493 // updating and scanning. See CRs 6677707 and 6677708.
467 if (G1UseParallelRSetUpdating || (worker_i == 0)) { 494 if (G1UseParallelRSetUpdating || (worker_i == 0)) {
468 updateRS(worker_i); 495 updateRS(&into_cset_dcq, worker_i);
469 scanNewRefsRS(oc, worker_i);
470 } else { 496 } else {
471 _g1p->record_update_rs_processed_buffers(worker_i, 0.0); 497 _g1p->record_update_rs_processed_buffers(worker_i, 0.0);
472 _g1p->record_update_rs_time(worker_i, 0.0); 498 _g1p->record_update_rs_time(worker_i, 0.0);
473 _g1p->record_scan_new_refs_time(worker_i, 0.0);
474 } 499 }
475 if (G1UseParallelRSetScanning || (worker_i == 0)) { 500 if (G1UseParallelRSetScanning || (worker_i == 0)) {
476 scanRS(oc, worker_i); 501 scanRS(oc, worker_i);
477 } else { 502 } else {
478 _g1p->record_scan_rs_time(worker_i, 0.0); 503 _g1p->record_scan_rs_time(worker_i, 0.0);
479 } 504 }
480 } else { 505 } else {
481 assert(worker_i == 0, "invariant"); 506 assert(worker_i == 0, "invariant");
482 updateRS(0); 507 updateRS(&into_cset_dcq, 0);
483 scanNewRefsRS(oc, 0);
484 scanRS(oc, 0); 508 scanRS(oc, 0);
485 } 509 }
510
511 // We now clear the cached values of _cset_rs_update_cl for this worker
512 _cset_rs_update_cl[worker_i] = NULL;
486 } 513 }
487 514
488 void HRInto_G1RemSet:: 515 void HRInto_G1RemSet::
489 prepare_for_oops_into_collection_set_do() { 516 prepare_for_oops_into_collection_set_do() {
490 #if G1_REM_SET_LOGGING 517 #if G1_REM_SET_LOGGING
517 hrrs->init_for_par_iteration(); 544 hrrs->init_for_par_iteration();
518 return false; 545 return false;
519 } 546 }
520 }; 547 };
521 548
522 class UpdateRSetOopsIntoCSImmediate : public OopClosure { 549 // This closure, applied to a DirtyCardQueueSet, is used to immediately
523 G1CollectedHeap* _g1; 550 // update the RSets for the regions in the CSet. For each card it iterates
524 public: 551 // through the oops which coincide with that card. It scans the reference
525 UpdateRSetOopsIntoCSImmediate(G1CollectedHeap* g1) : _g1(g1) { } 552 // fields in each oop; when it finds an oop that points into the collection
526 virtual void do_oop(narrowOop* p) { do_oop_work(p); } 553 // set, the RSet for the region containing the referenced object is updated.
527 virtual void do_oop( oop* p) { do_oop_work(p); } 554 // Note: _par_traversal_in_progress in the G1RemSet must be FALSE; otherwise
528 template <class T> void do_oop_work(T* p) { 555 // the UpdateRSetImmediate closure will cause cards to be enqueued on to
529 HeapRegion* to = _g1->heap_region_containing(oopDesc::load_decode_heap_oop(p)); 556 // the DCQS that we're iterating over, causing an infinite loop.
530 if (to->in_collection_set()) { 557 class UpdateRSetCardTableEntryIntoCSetClosure: public CardTableEntryClosure {
531 to->rem_set()->add_reference(p, 0);
532 }
533 }
534 };
535
536 class UpdateRSetOopsIntoCSDeferred : public OopClosure {
537 G1CollectedHeap* _g1; 558 G1CollectedHeap* _g1;
538 CardTableModRefBS* _ct_bs; 559 CardTableModRefBS* _ct_bs;
539 DirtyCardQueue* _dcq; 560 public:
540 public: 561 UpdateRSetCardTableEntryIntoCSetClosure(G1CollectedHeap* g1,
541 UpdateRSetOopsIntoCSDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) : 562 CardTableModRefBS* bs):
542 _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) { } 563 _g1(g1), _ct_bs(bs)
543 virtual void do_oop(narrowOop* p) { do_oop_work(p); } 564 { }
544 virtual void do_oop( oop* p) { do_oop_work(p); } 565
545 template <class T> void do_oop_work(T* p) { 566 bool do_card_ptr(jbyte* card_ptr, int worker_i) {
546 oop obj = oopDesc::load_decode_heap_oop(p); 567 // Construct the region representing the card.
547 if (_g1->obj_in_cs(obj)) { 568 HeapWord* start = _ct_bs->addr_for(card_ptr);
548 size_t card_index = _ct_bs->index_for(p); 569 // And find the region containing it.
549 if (_ct_bs->mark_card_deferred(card_index)) { 570 HeapRegion* r = _g1->heap_region_containing(start);
550 _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index)); 571 assert(r != NULL, "unexpected null");
551 } 572
552 } 573 // Scan oops in the card looking for references into the collection set
553 } 574 HeapWord* end = _ct_bs->addr_for(card_ptr + 1);
554 }; 575 MemRegion scanRegion(start, end);
555 576
556 template <class T> void HRInto_G1RemSet::new_refs_iterate_work(OopClosure* cl) { 577 UpdateRSetImmediate update_rs_cl(_g1->g1_rem_set());
557 for (size_t i = 0; i < n_workers(); i++) { 578 FilterIntoCSClosure update_rs_cset_oop_cl(NULL, _g1, &update_rs_cl);
558 for (int j = 0; j < _new_refs[i]->length(); j++) { 579 FilterOutOfRegionClosure filter_then_update_rs_cset_oop_cl(r, &update_rs_cset_oop_cl);
559 T* p = (T*) _new_refs[i]->at(j); 580
560 cl->do_oop(p); 581 // We can pass false as the "filter_young" parameter here as:
561 } 582 // * we should be in a STW pause,
562 } 583 // * the DCQS to which this closure is applied is used to hold
563 } 584 // references that point into the collection set from the prior
585 // RSet updating,
586 // * the post-write barrier shouldn't be logging updates to young
587 // regions (but there is a situation where this can happen - see
588 // the comment in HRInto_G1RemSet::concurrentRefineOneCard below -
589 // that should not be applicable here), and
590 // * during actual RSet updating, the filtering of cards in young
591 // regions in HeapRegion::oops_on_card_seq_iterate_careful is
592 // employed.
593 // As a result, when this closure is applied to "refs into cset"
594 // DCQS, we shouldn't see any cards in young regions.
595 update_rs_cl.set_region(r);
596 HeapWord* stop_point =
597 r->oops_on_card_seq_iterate_careful(scanRegion,
598 &filter_then_update_rs_cset_oop_cl,
599 false /* filter_young */);
600
601 // Since this is performed in the event of an evacuation failure, we
602 // we shouldn't see a non-null stop point
603 assert(stop_point == NULL, "saw an unallocated region");
604 return true;
605 }
606 };
564 607
565 void HRInto_G1RemSet::cleanup_after_oops_into_collection_set_do() { 608 void HRInto_G1RemSet::cleanup_after_oops_into_collection_set_do() {
566 guarantee( _cards_scanned != NULL, "invariant" ); 609 guarantee( _cards_scanned != NULL, "invariant" );
567 _total_cards_scanned = 0; 610 _total_cards_scanned = 0;
568 for (uint i = 0; i < n_workers(); ++i) 611 for (uint i = 0; i < n_workers(); ++i)
582 625
583 if (ParallelGCThreads > 0) { 626 if (ParallelGCThreads > 0) {
584 set_par_traversal(false); 627 set_par_traversal(false);
585 } 628 }
586 629
630 DirtyCardQueueSet& into_cset_dcqs = _g1->into_cset_dirty_card_queue_set();
631 int into_cset_n_buffers = into_cset_dcqs.completed_buffers_num();
632
587 if (_g1->evacuation_failed()) { 633 if (_g1->evacuation_failed()) {
588 // Restore remembered sets for the regions pointing into 634 // Restore remembered sets for the regions pointing into the collection set.
589 // the collection set. 635
590 if (G1DeferredRSUpdate) { 636 if (G1DeferredRSUpdate) {
591 DirtyCardQueue dcq(&_g1->dirty_card_queue_set()); 637 // If deferred RS updates are enabled then we just need to transfer
592 UpdateRSetOopsIntoCSDeferred deferred_update(_g1, &dcq); 638 // the completed buffers from (a) the DirtyCardQueueSet used to hold
593 new_refs_iterate(&deferred_update); 639 // cards that contain references that point into the collection set
640 // to (b) the DCQS used to hold the deferred RS updates
641 _g1->dirty_card_queue_set().merge_bufferlists(&into_cset_dcqs);
594 } else { 642 } else {
595 UpdateRSetOopsIntoCSImmediate immediate_update(_g1); 643
596 new_refs_iterate(&immediate_update); 644 CardTableModRefBS* bs = (CardTableModRefBS*)_g1->barrier_set();
597 } 645 UpdateRSetCardTableEntryIntoCSetClosure update_rs_cset_immediate(_g1, bs);
598 } 646
599 for (uint i = 0; i < n_workers(); i++) { 647 int n_completed_buffers = 0;
600 _new_refs[i]->clear(); 648 while (into_cset_dcqs.apply_closure_to_completed_buffer(&update_rs_cset_immediate,
601 } 649 0, 0, true)) {
650 n_completed_buffers++;
651 }
652 assert(n_completed_buffers == into_cset_n_buffers, "missed some buffers");
653 }
654 }
655
656 // Free any completed buffers in the DirtyCardQueueSet used to hold cards
657 // which contain references that point into the collection.
658 _g1->into_cset_dirty_card_queue_set().clear();
659 assert(_g1->into_cset_dirty_card_queue_set().completed_buffers_num() == 0,
660 "all buffers should be freed");
661 _g1->into_cset_dirty_card_queue_set().clear_n_completed_buffers();
602 662
603 assert(!_par_traversal_in_progress, "Invariant between iterations."); 663 assert(!_par_traversal_in_progress, "Invariant between iterations.");
604 } 664 }
605 665
606 class UpdateRSObjectClosure: public ObjectClosure { 666 class UpdateRSObjectClosure: public ObjectClosure {
650 } 710 }
651 711
652 712
653 static IntHistogram out_of_histo(50, 50); 713 static IntHistogram out_of_histo(50, 50);
654 714
655 void HRInto_G1RemSet::concurrentRefineOneCard_impl(jbyte* card_ptr, int worker_i) { 715 class TriggerClosure : public OopClosure {
716 bool _trigger;
717 public:
718 TriggerClosure() : _trigger(false) { }
719 bool value() const { return _trigger; }
720 template <class T> void do_oop_nv(T* p) { _trigger = true; }
721 virtual void do_oop(oop* p) { do_oop_nv(p); }
722 virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
723 };
724
725 class InvokeIfNotTriggeredClosure: public OopClosure {
726 TriggerClosure* _t;
727 OopClosure* _oc;
728 public:
729 InvokeIfNotTriggeredClosure(TriggerClosure* t, OopClosure* oc):
730 _t(t), _oc(oc) { }
731 template <class T> void do_oop_nv(T* p) {
732 if (!_t->value()) _oc->do_oop(p);
733 }
734 virtual void do_oop(oop* p) { do_oop_nv(p); }
735 virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
736 };
737
738 class Mux2Closure : public OopClosure {
739 OopClosure* _c1;
740 OopClosure* _c2;
741 public:
742 Mux2Closure(OopClosure *c1, OopClosure *c2) : _c1(c1), _c2(c2) { }
743 template <class T> void do_oop_nv(T* p) {
744 _c1->do_oop(p); _c2->do_oop(p);
745 }
746 virtual void do_oop(oop* p) { do_oop_nv(p); }
747 virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
748 };
749
750 bool HRInto_G1RemSet::concurrentRefineOneCard_impl(jbyte* card_ptr, int worker_i,
751 bool check_for_refs_into_cset) {
656 // Construct the region representing the card. 752 // Construct the region representing the card.
657 HeapWord* start = _ct_bs->addr_for(card_ptr); 753 HeapWord* start = _ct_bs->addr_for(card_ptr);
658 // And find the region containing it. 754 // And find the region containing it.
659 HeapRegion* r = _g1->heap_region_containing(start); 755 HeapRegion* r = _g1->heap_region_containing(start);
660 assert(r != NULL, "unexpected null"); 756 assert(r != NULL, "unexpected null");
667 ct_freq_note_card(_ct_bs->index_for(start)); 763 ct_freq_note_card(_ct_bs->index_for(start));
668 #endif 764 #endif
669 765
670 UpdateRSOopClosure update_rs_oop_cl(this, worker_i); 766 UpdateRSOopClosure update_rs_oop_cl(this, worker_i);
671 update_rs_oop_cl.set_from(r); 767 update_rs_oop_cl.set_from(r);
672 FilterOutOfRegionClosure filter_then_update_rs_oop_cl(r, &update_rs_oop_cl); 768
769 TriggerClosure trigger_cl;
770 FilterIntoCSClosure into_cs_cl(NULL, _g1, &trigger_cl);
771 InvokeIfNotTriggeredClosure invoke_cl(&trigger_cl, &into_cs_cl);
772 Mux2Closure mux(&invoke_cl, &update_rs_oop_cl);
773
774 FilterOutOfRegionClosure filter_then_update_rs_oop_cl(r,
775 (check_for_refs_into_cset ?
776 (OopClosure*)&mux :
777 (OopClosure*)&update_rs_oop_cl));
673 778
674 // Undirty the card. 779 // Undirty the card.
675 *card_ptr = CardTableModRefBS::clean_card_val(); 780 *card_ptr = CardTableModRefBS::clean_card_val();
676 // We must complete this write before we do any of the reads below. 781 // We must complete this write before we do any of the reads below.
677 OrderAccess::storeload(); 782 OrderAccess::storeload();
715 } 820 }
716 } else { 821 } else {
717 out_of_histo.add_entry(filter_then_update_rs_oop_cl.out_of_region()); 822 out_of_histo.add_entry(filter_then_update_rs_oop_cl.out_of_region());
718 _conc_refine_cards++; 823 _conc_refine_cards++;
719 } 824 }
720 } 825
721 826 return trigger_cl.value();
722 void HRInto_G1RemSet::concurrentRefineOneCard(jbyte* card_ptr, int worker_i) { 827 }
828
829 bool HRInto_G1RemSet::concurrentRefineOneCard(jbyte* card_ptr, int worker_i,
830 bool check_for_refs_into_cset) {
723 // If the card is no longer dirty, nothing to do. 831 // If the card is no longer dirty, nothing to do.
724 if (*card_ptr != CardTableModRefBS::dirty_card_val()) return; 832 if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
833 // No need to return that this card contains refs that point
834 // into the collection set.
835 return false;
836 }
725 837
726 // Construct the region representing the card. 838 // Construct the region representing the card.
727 HeapWord* start = _ct_bs->addr_for(card_ptr); 839 HeapWord* start = _ct_bs->addr_for(card_ptr);
728 // And find the region containing it. 840 // And find the region containing it.
729 HeapRegion* r = _g1->heap_region_containing(start); 841 HeapRegion* r = _g1->heap_region_containing(start);
730 if (r == NULL) { 842 if (r == NULL) {
731 guarantee(_g1->is_in_permanent(start), "Or else where?"); 843 guarantee(_g1->is_in_permanent(start), "Or else where?");
732 return; // Not in the G1 heap (might be in perm, for example.) 844 // Again no need to return that this card contains refs that
845 // point into the collection set.
846 return false; // Not in the G1 heap (might be in perm, for example.)
733 } 847 }
734 // Why do we have to check here whether a card is on a young region, 848 // Why do we have to check here whether a card is on a young region,
735 // given that we dirty young regions and, as a result, the 849 // given that we dirty young regions and, as a result, the
736 // post-barrier is supposed to filter them out and never to enqueue 850 // post-barrier is supposed to filter them out and never to enqueue
737 // them? When we allocate a new region as the "allocation region" we 851 // them? When we allocate a new region as the "allocation region" we
741 // allocate objects in the region (after the acquire the lock) 855 // allocate objects in the region (after the acquire the lock)
742 // before all the cards on the region are dirtied. This is unlikely, 856 // before all the cards on the region are dirtied. This is unlikely,
743 // and it doesn't happen often, but it can happen. So, the extra 857 // and it doesn't happen often, but it can happen. So, the extra
744 // check below filters out those cards. 858 // check below filters out those cards.
745 if (r->is_young()) { 859 if (r->is_young()) {
746 return; 860 return false;
747 } 861 }
748 // While we are processing RSet buffers during the collection, we 862 // While we are processing RSet buffers during the collection, we
749 // actually don't want to scan any cards on the collection set, 863 // actually don't want to scan any cards on the collection set,
750 // since we don't want to update remebered sets with entries that 864 // since we don't want to update remebered sets with entries that
751 // point into the collection set, given that live objects from the 865 // point into the collection set, given that live objects from the
754 // involves scanning a card in the collection set and coming across 868 // involves scanning a card in the collection set and coming across
755 // an array that was being chunked and looking malformed. Note, 869 // an array that was being chunked and looking malformed. Note,
756 // however, that if evacuation fails, we have to scan any objects 870 // however, that if evacuation fails, we have to scan any objects
757 // that were not moved and create any missing entries. 871 // that were not moved and create any missing entries.
758 if (r->in_collection_set()) { 872 if (r->in_collection_set()) {
759 return; 873 return false;
760 } 874 }
761 875
762 // Should we defer processing the card? 876 // Should we defer processing the card?
763 // 877 //
764 // Previously the result from the insert_cache call would be 878 // Previously the result from the insert_cache call would be
795 // non-null true card not evicted from _card_counts; card_ptr is 909 // non-null true card not evicted from _card_counts; card_ptr is
796 // currently cold, or caused an eviction from hot 910 // currently cold, or caused an eviction from hot
797 // cache. 911 // cache.
798 // Immediately process res; no need to process card_ptr. 912 // Immediately process res; no need to process card_ptr.
799 913
914
800 jbyte* res = card_ptr; 915 jbyte* res = card_ptr;
801 bool defer = false; 916 bool defer = false;
917
918 // This gets set to true if the card being refined has references
919 // that point into the collection set.
920 bool oops_into_cset = false;
921
802 if (_cg1r->use_cache()) { 922 if (_cg1r->use_cache()) {
803 jbyte* res = _cg1r->cache_insert(card_ptr, &defer); 923 jbyte* res = _cg1r->cache_insert(card_ptr, &defer);
804 if (res != NULL && (res != card_ptr || defer)) { 924 if (res != NULL && (res != card_ptr || defer)) {
805 start = _ct_bs->addr_for(res); 925 start = _ct_bs->addr_for(res);
806 r = _g1->heap_region_containing(start); 926 r = _g1->heap_region_containing(start);
813 // Hence we could see its young type change at any time. 933 // Hence we could see its young type change at any time.
814 // 934 //
815 // Process card pointer we get back from the hot card cache. This 935 // Process card pointer we get back from the hot card cache. This
816 // will check whether the region containing the card is young 936 // will check whether the region containing the card is young
817 // _after_ checking that the region has been allocated from. 937 // _after_ checking that the region has been allocated from.
818 concurrentRefineOneCard_impl(res, worker_i); 938 oops_into_cset = concurrentRefineOneCard_impl(res, worker_i,
939 false /* check_for_refs_into_cset */);
940 // The above call to concurrentRefineOneCard_impl is only
941 // performed if the hot card cache is enabled. This cache is
942 // disabled during an evacuation pause - which is the only
943 // time when we need know if the card contains references
944 // that point into the collection set. Also when the hot card
945 // cache is enabled, this code is executed by the concurrent
946 // refine threads - rather than the GC worker threads - and
947 // concurrentRefineOneCard_impl will return false.
948 assert(!oops_into_cset, "should not see true here");
819 } 949 }
820 } 950 }
821 } 951 }
822 952
823 if (!defer) { 953 if (!defer) {
824 concurrentRefineOneCard_impl(card_ptr, worker_i); 954 oops_into_cset =
825 } 955 concurrentRefineOneCard_impl(card_ptr, worker_i, check_for_refs_into_cset);
956 // We should only be detecting that the card contains references
957 // that point into the collection set if the current thread is
958 // a GC worker thread.
959 assert(!oops_into_cset || SafepointSynchronize::is_at_safepoint(),
960 "invalid result at non safepoint");
961 }
962 return oops_into_cset;
826 } 963 }
827 964
828 class HRRSStatsIter: public HeapRegionClosure { 965 class HRRSStatsIter: public HeapRegionClosure {
829 size_t _occupied; 966 size_t _occupied;
830 size_t _total_mem_sz; 967 size_t _total_mem_sz;
918 gclog_or_tty->print_cr(" Did %d coarsenings.", 1055 gclog_or_tty->print_cr(" Did %d coarsenings.",
919 HeapRegionRemSet::n_coarsenings()); 1056 HeapRegionRemSet::n_coarsenings());
920 1057
921 } 1058 }
922 } 1059 }
1060
923 void HRInto_G1RemSet::prepare_for_verify() { 1061 void HRInto_G1RemSet::prepare_for_verify() {
924 if (G1HRRSFlushLogBuffersOnVerify && 1062 if (G1HRRSFlushLogBuffersOnVerify &&
925 (VerifyBeforeGC || VerifyAfterGC) 1063 (VerifyBeforeGC || VerifyAfterGC)
926 && !_g1->full_collection()) { 1064 && !_g1->full_collection()) {
927 cleanupHRRS(); 1065 cleanupHRRS();
930 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); 1068 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
931 dcqs.concatenate_logs(); 1069 dcqs.concatenate_logs();
932 } 1070 }
933 bool cg1r_use_cache = _cg1r->use_cache(); 1071 bool cg1r_use_cache = _cg1r->use_cache();
934 _cg1r->set_use_cache(false); 1072 _cg1r->set_use_cache(false);
935 updateRS(0); 1073 DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set());
1074 updateRS(&into_cset_dcq, 0);
1075 _g1->into_cset_dirty_card_queue_set().clear();
936 _cg1r->set_use_cache(cg1r_use_cache); 1076 _cg1r->set_use_cache(cg1r_use_cache);
937 1077
938 assert(JavaThread::dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed"); 1078 assert(JavaThread::dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed");
939 } 1079 }
940 } 1080 }