Mercurial > hg > graal-compiler
comparison src/share/vm/gc_implementation/g1/g1RemSet.cpp @ 1705:2d160770d2e5
6814437: G1: remove the _new_refs array
Summary: The per-worker _new_refs array is used to hold references that point into the collection set. It is populated during RSet updating and subsequently processed. In the event of an evacuation failure it processed again to recreate the RSets of regions in the collection set. Remove the per-worker _new_refs array by processing the references directly. Use a DirtyCardQueue to hold the cards containing the references so that the RSets of regions in the collection set can be recreated when handling an evacuation failure.
Reviewed-by: iveresov, jmasa, tonyp
author | johnc |
---|---|
date | Mon, 02 Aug 2010 12:51:43 -0700 |
parents | 5cbac8938c4c |
children | a03ae377b2e8 |
comparison
equal
deleted
inserted
replaced
1704:63f4675ac87d | 1705:2d160770d2e5 |
---|---|
120 }; | 120 }; |
121 | 121 |
122 HRInto_G1RemSet::HRInto_G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs) | 122 HRInto_G1RemSet::HRInto_G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs) |
123 : G1RemSet(g1), _ct_bs(ct_bs), _g1p(_g1->g1_policy()), | 123 : G1RemSet(g1), _ct_bs(ct_bs), _g1p(_g1->g1_policy()), |
124 _cg1r(g1->concurrent_g1_refine()), | 124 _cg1r(g1->concurrent_g1_refine()), |
125 _par_traversal_in_progress(false), _new_refs(NULL), | 125 _par_traversal_in_progress(false), |
126 _cset_rs_update_cl(NULL), | |
126 _cards_scanned(NULL), _total_cards_scanned(0) | 127 _cards_scanned(NULL), _total_cards_scanned(0) |
127 { | 128 { |
128 _seq_task = new SubTasksDone(NumSeqTasks); | 129 _seq_task = new SubTasksDone(NumSeqTasks); |
129 guarantee(n_workers() > 0, "There should be some workers"); | 130 guarantee(n_workers() > 0, "There should be some workers"); |
130 _new_refs = NEW_C_HEAP_ARRAY(GrowableArray<OopOrNarrowOopStar>*, n_workers()); | 131 _cset_rs_update_cl = NEW_C_HEAP_ARRAY(OopsInHeapRegionClosure*, n_workers()); |
131 for (uint i = 0; i < n_workers(); i++) { | 132 for (uint i = 0; i < n_workers(); i++) { |
132 _new_refs[i] = new (ResourceObj::C_HEAP) GrowableArray<OopOrNarrowOopStar>(8192,true); | 133 _cset_rs_update_cl[i] = NULL; |
133 } | 134 } |
134 } | 135 } |
135 | 136 |
136 HRInto_G1RemSet::~HRInto_G1RemSet() { | 137 HRInto_G1RemSet::~HRInto_G1RemSet() { |
137 delete _seq_task; | 138 delete _seq_task; |
138 for (uint i = 0; i < n_workers(); i++) { | 139 for (uint i = 0; i < n_workers(); i++) { |
139 delete _new_refs[i]; | 140 assert(_cset_rs_update_cl[i] == NULL, "it should be"); |
140 } | 141 } |
141 FREE_C_HEAP_ARRAY(GrowableArray<OopOrNarrowOopStar>*, _new_refs); | 142 FREE_C_HEAP_ARRAY(OopsInHeapRegionClosure*, _cset_rs_update_cl); |
142 } | 143 } |
143 | 144 |
144 void CountNonCleanMemRegionClosure::do_MemRegion(MemRegion mr) { | 145 void CountNonCleanMemRegionClosure::do_MemRegion(MemRegion mr) { |
145 if (_g1->is_in_g1_reserved(mr.start())) { | 146 if (_g1->is_in_g1_reserved(mr.start())) { |
146 _n += (int) ((mr.byte_size() / CardTableModRefBS::card_size)); | 147 _n += (int) ((mr.byte_size() / CardTableModRefBS::card_size)); |
304 _cards_scanned[worker_i] = scanRScl.cards_done(); | 305 _cards_scanned[worker_i] = scanRScl.cards_done(); |
305 | 306 |
306 _g1p->record_scan_rs_time(worker_i, scan_rs_time_sec * 1000.0); | 307 _g1p->record_scan_rs_time(worker_i, scan_rs_time_sec * 1000.0); |
307 } | 308 } |
308 | 309 |
309 void HRInto_G1RemSet::updateRS(int worker_i) { | 310 // Closure used for updating RSets and recording references that |
310 ConcurrentG1Refine* cg1r = _g1->concurrent_g1_refine(); | 311 // point into the collection set. Only called during an |
311 | 312 // evacuation pause. |
313 | |
314 class RefineRecordRefsIntoCSCardTableEntryClosure: public CardTableEntryClosure { | |
315 G1RemSet* _g1rs; | |
316 DirtyCardQueue* _into_cset_dcq; | |
317 public: | |
318 RefineRecordRefsIntoCSCardTableEntryClosure(G1CollectedHeap* g1h, | |
319 DirtyCardQueue* into_cset_dcq) : | |
320 _g1rs(g1h->g1_rem_set()), _into_cset_dcq(into_cset_dcq) | |
321 {} | |
322 bool do_card_ptr(jbyte* card_ptr, int worker_i) { | |
323 // The only time we care about recording cards that | |
324 // contain references that point into the collection set | |
325 // is during RSet updating within an evacuation pause. | |
326 // In this case worker_i should be the id of a GC worker thread. | |
327 assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause"); | |
328 assert(worker_i < (int) DirtyCardQueueSet::num_par_ids(), "should be a GC worker"); | |
329 | |
330 if (_g1rs->concurrentRefineOneCard(card_ptr, worker_i, true)) { | |
331 // 'card_ptr' contains references that point into the collection | |
332 // set. We need to record the card in the DCQS | |
333 // (G1CollectedHeap::into_cset_dirty_card_queue_set()) | |
334 // that's used for that purpose. | |
335 // | |
336 // Enqueue the card | |
337 _into_cset_dcq->enqueue(card_ptr); | |
338 } | |
339 return true; | |
340 } | |
341 }; | |
342 | |
343 void HRInto_G1RemSet::updateRS(DirtyCardQueue* into_cset_dcq, int worker_i) { | |
312 double start = os::elapsedTime(); | 344 double start = os::elapsedTime(); |
313 // Apply the appropriate closure to all remaining log entries. | 345 // Apply the given closure to all remaining log entries. |
314 _g1->iterate_dirty_card_closure(false, worker_i); | 346 RefineRecordRefsIntoCSCardTableEntryClosure into_cset_update_rs_cl(_g1, into_cset_dcq); |
347 _g1->iterate_dirty_card_closure(&into_cset_update_rs_cl, into_cset_dcq, false, worker_i); | |
348 | |
315 // Now there should be no dirty cards. | 349 // Now there should be no dirty cards. |
316 if (G1RSLogCheckCardTable) { | 350 if (G1RSLogCheckCardTable) { |
317 CountNonCleanMemRegionClosure cl(_g1); | 351 CountNonCleanMemRegionClosure cl(_g1); |
318 _ct_bs->mod_card_iterate(&cl); | 352 _ct_bs->mod_card_iterate(&cl); |
319 // XXX This isn't true any more: keeping cards of young regions | 353 // XXX This isn't true any more: keeping cards of young regions |
403 } | 437 } |
404 gclog_or_tty->print_cr(" > %8d %8d", (1 << (MIN+mx-2))+1, _histo[mx-1]); | 438 gclog_or_tty->print_cr(" > %8d %8d", (1 << (MIN+mx-2))+1, _histo[mx-1]); |
405 } | 439 } |
406 }; | 440 }; |
407 | 441 |
408 template <class T> void | |
409 HRInto_G1RemSet::scanNewRefsRS_work(OopsInHeapRegionClosure* oc, | |
410 int worker_i) { | |
411 double scan_new_refs_start_sec = os::elapsedTime(); | |
412 G1CollectedHeap* g1h = G1CollectedHeap::heap(); | |
413 CardTableModRefBS* ct_bs = (CardTableModRefBS*) (g1h->barrier_set()); | |
414 for (int i = 0; i < _new_refs[worker_i]->length(); i++) { | |
415 T* p = (T*) _new_refs[worker_i]->at(i); | |
416 oop obj = oopDesc::load_decode_heap_oop(p); | |
417 // *p was in the collection set when p was pushed on "_new_refs", but | |
418 // another thread may have processed this location from an RS, so it | |
419 // might not point into the CS any longer. If so, it's obviously been | |
420 // processed, and we don't need to do anything further. | |
421 if (g1h->obj_in_cs(obj)) { | |
422 HeapRegion* r = g1h->heap_region_containing(p); | |
423 | |
424 DEBUG_ONLY(HeapRegion* to = g1h->heap_region_containing(obj)); | |
425 oc->set_region(r); | |
426 // If "p" has already been processed concurrently, this is | |
427 // idempotent. | |
428 oc->do_oop(p); | |
429 } | |
430 } | |
431 double scan_new_refs_time_ms = (os::elapsedTime() - scan_new_refs_start_sec) * 1000.0; | |
432 _g1p->record_scan_new_refs_time(worker_i, scan_new_refs_time_ms); | |
433 } | |
434 | |
435 void HRInto_G1RemSet::cleanupHRRS() { | 442 void HRInto_G1RemSet::cleanupHRRS() { |
436 HeapRegionRemSet::cleanup(); | 443 HeapRegionRemSet::cleanup(); |
437 } | 444 } |
438 | 445 |
439 void | 446 void |
455 count_cl.n(), (float)count_cl.tot()/(float)count_cl.n(), | 462 count_cl.n(), (float)count_cl.tot()/(float)count_cl.n(), |
456 count_cl.mx(), count_cl.mxr()); | 463 count_cl.mx(), count_cl.mxr()); |
457 count_cl.print_histo(); | 464 count_cl.print_histo(); |
458 } | 465 } |
459 | 466 |
467 // We cache the value of 'oc' closure into the appropriate slot in the | |
468 // _cset_rs_update_cl for this worker | |
469 assert(worker_i < (int)n_workers(), "sanity"); | |
470 _cset_rs_update_cl[worker_i] = oc; | |
471 | |
472 // A DirtyCardQueue that is used to hold cards containing references | |
473 // that point into the collection set. This DCQ is associated with a | |
474 // special DirtyCardQueueSet (see g1CollectedHeap.hpp). Under normal | |
475 // circumstances (i.e. the pause successfully completes), these cards | |
476 // are just discarded (there's no need to update the RSets of regions | |
477 // that were in the collection set - after the pause these regions | |
478 // are wholly 'free' of live objects. In the event of an evacuation | |
479 // failure the cards/buffers in this queue set are: | |
480 // * passed to the DirtyCardQueueSet that is used to manage deferred | |
481 // RSet updates, or | |
482 // * scanned for references that point into the collection set | |
483 // and the RSet of the corresponding region in the collection set | |
484 // is updated immediately. | |
485 DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set()); | |
486 | |
460 if (ParallelGCThreads > 0) { | 487 if (ParallelGCThreads > 0) { |
461 // The two flags below were introduced temporarily to serialize | 488 // The two flags below were introduced temporarily to serialize |
462 // the updating and scanning of remembered sets. There are some | 489 // the updating and scanning of remembered sets. There are some |
463 // race conditions when these two operations are done in parallel | 490 // race conditions when these two operations are done in parallel |
464 // and they are causing failures. When we resolve said race | 491 // and they are causing failures. When we resolve said race |
465 // conditions, we'll revert back to parallel remembered set | 492 // conditions, we'll revert back to parallel remembered set |
466 // updating and scanning. See CRs 6677707 and 6677708. | 493 // updating and scanning. See CRs 6677707 and 6677708. |
467 if (G1UseParallelRSetUpdating || (worker_i == 0)) { | 494 if (G1UseParallelRSetUpdating || (worker_i == 0)) { |
468 updateRS(worker_i); | 495 updateRS(&into_cset_dcq, worker_i); |
469 scanNewRefsRS(oc, worker_i); | |
470 } else { | 496 } else { |
471 _g1p->record_update_rs_processed_buffers(worker_i, 0.0); | 497 _g1p->record_update_rs_processed_buffers(worker_i, 0.0); |
472 _g1p->record_update_rs_time(worker_i, 0.0); | 498 _g1p->record_update_rs_time(worker_i, 0.0); |
473 _g1p->record_scan_new_refs_time(worker_i, 0.0); | |
474 } | 499 } |
475 if (G1UseParallelRSetScanning || (worker_i == 0)) { | 500 if (G1UseParallelRSetScanning || (worker_i == 0)) { |
476 scanRS(oc, worker_i); | 501 scanRS(oc, worker_i); |
477 } else { | 502 } else { |
478 _g1p->record_scan_rs_time(worker_i, 0.0); | 503 _g1p->record_scan_rs_time(worker_i, 0.0); |
479 } | 504 } |
480 } else { | 505 } else { |
481 assert(worker_i == 0, "invariant"); | 506 assert(worker_i == 0, "invariant"); |
482 updateRS(0); | 507 updateRS(&into_cset_dcq, 0); |
483 scanNewRefsRS(oc, 0); | |
484 scanRS(oc, 0); | 508 scanRS(oc, 0); |
485 } | 509 } |
510 | |
511 // We now clear the cached values of _cset_rs_update_cl for this worker | |
512 _cset_rs_update_cl[worker_i] = NULL; | |
486 } | 513 } |
487 | 514 |
488 void HRInto_G1RemSet:: | 515 void HRInto_G1RemSet:: |
489 prepare_for_oops_into_collection_set_do() { | 516 prepare_for_oops_into_collection_set_do() { |
490 #if G1_REM_SET_LOGGING | 517 #if G1_REM_SET_LOGGING |
517 hrrs->init_for_par_iteration(); | 544 hrrs->init_for_par_iteration(); |
518 return false; | 545 return false; |
519 } | 546 } |
520 }; | 547 }; |
521 | 548 |
522 class UpdateRSetOopsIntoCSImmediate : public OopClosure { | 549 // This closure, applied to a DirtyCardQueueSet, is used to immediately |
523 G1CollectedHeap* _g1; | 550 // update the RSets for the regions in the CSet. For each card it iterates |
524 public: | 551 // through the oops which coincide with that card. It scans the reference |
525 UpdateRSetOopsIntoCSImmediate(G1CollectedHeap* g1) : _g1(g1) { } | 552 // fields in each oop; when it finds an oop that points into the collection |
526 virtual void do_oop(narrowOop* p) { do_oop_work(p); } | 553 // set, the RSet for the region containing the referenced object is updated. |
527 virtual void do_oop( oop* p) { do_oop_work(p); } | 554 // Note: _par_traversal_in_progress in the G1RemSet must be FALSE; otherwise |
528 template <class T> void do_oop_work(T* p) { | 555 // the UpdateRSetImmediate closure will cause cards to be enqueued on to |
529 HeapRegion* to = _g1->heap_region_containing(oopDesc::load_decode_heap_oop(p)); | 556 // the DCQS that we're iterating over, causing an infinite loop. |
530 if (to->in_collection_set()) { | 557 class UpdateRSetCardTableEntryIntoCSetClosure: public CardTableEntryClosure { |
531 to->rem_set()->add_reference(p, 0); | |
532 } | |
533 } | |
534 }; | |
535 | |
536 class UpdateRSetOopsIntoCSDeferred : public OopClosure { | |
537 G1CollectedHeap* _g1; | 558 G1CollectedHeap* _g1; |
538 CardTableModRefBS* _ct_bs; | 559 CardTableModRefBS* _ct_bs; |
539 DirtyCardQueue* _dcq; | 560 public: |
540 public: | 561 UpdateRSetCardTableEntryIntoCSetClosure(G1CollectedHeap* g1, |
541 UpdateRSetOopsIntoCSDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) : | 562 CardTableModRefBS* bs): |
542 _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) { } | 563 _g1(g1), _ct_bs(bs) |
543 virtual void do_oop(narrowOop* p) { do_oop_work(p); } | 564 { } |
544 virtual void do_oop( oop* p) { do_oop_work(p); } | 565 |
545 template <class T> void do_oop_work(T* p) { | 566 bool do_card_ptr(jbyte* card_ptr, int worker_i) { |
546 oop obj = oopDesc::load_decode_heap_oop(p); | 567 // Construct the region representing the card. |
547 if (_g1->obj_in_cs(obj)) { | 568 HeapWord* start = _ct_bs->addr_for(card_ptr); |
548 size_t card_index = _ct_bs->index_for(p); | 569 // And find the region containing it. |
549 if (_ct_bs->mark_card_deferred(card_index)) { | 570 HeapRegion* r = _g1->heap_region_containing(start); |
550 _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index)); | 571 assert(r != NULL, "unexpected null"); |
551 } | 572 |
552 } | 573 // Scan oops in the card looking for references into the collection set |
553 } | 574 HeapWord* end = _ct_bs->addr_for(card_ptr + 1); |
554 }; | 575 MemRegion scanRegion(start, end); |
555 | 576 |
556 template <class T> void HRInto_G1RemSet::new_refs_iterate_work(OopClosure* cl) { | 577 UpdateRSetImmediate update_rs_cl(_g1->g1_rem_set()); |
557 for (size_t i = 0; i < n_workers(); i++) { | 578 FilterIntoCSClosure update_rs_cset_oop_cl(NULL, _g1, &update_rs_cl); |
558 for (int j = 0; j < _new_refs[i]->length(); j++) { | 579 FilterOutOfRegionClosure filter_then_update_rs_cset_oop_cl(r, &update_rs_cset_oop_cl); |
559 T* p = (T*) _new_refs[i]->at(j); | 580 |
560 cl->do_oop(p); | 581 // We can pass false as the "filter_young" parameter here as: |
561 } | 582 // * we should be in a STW pause, |
562 } | 583 // * the DCQS to which this closure is applied is used to hold |
563 } | 584 // references that point into the collection set from the prior |
585 // RSet updating, | |
586 // * the post-write barrier shouldn't be logging updates to young | |
587 // regions (but there is a situation where this can happen - see | |
588 // the comment in HRInto_G1RemSet::concurrentRefineOneCard below - | |
589 // that should not be applicable here), and | |
590 // * during actual RSet updating, the filtering of cards in young | |
591 // regions in HeapRegion::oops_on_card_seq_iterate_careful is | |
592 // employed. | |
593 // As a result, when this closure is applied to "refs into cset" | |
594 // DCQS, we shouldn't see any cards in young regions. | |
595 update_rs_cl.set_region(r); | |
596 HeapWord* stop_point = | |
597 r->oops_on_card_seq_iterate_careful(scanRegion, | |
598 &filter_then_update_rs_cset_oop_cl, | |
599 false /* filter_young */); | |
600 | |
601 // Since this is performed in the event of an evacuation failure, we | |
602 // we shouldn't see a non-null stop point | |
603 assert(stop_point == NULL, "saw an unallocated region"); | |
604 return true; | |
605 } | |
606 }; | |
564 | 607 |
565 void HRInto_G1RemSet::cleanup_after_oops_into_collection_set_do() { | 608 void HRInto_G1RemSet::cleanup_after_oops_into_collection_set_do() { |
566 guarantee( _cards_scanned != NULL, "invariant" ); | 609 guarantee( _cards_scanned != NULL, "invariant" ); |
567 _total_cards_scanned = 0; | 610 _total_cards_scanned = 0; |
568 for (uint i = 0; i < n_workers(); ++i) | 611 for (uint i = 0; i < n_workers(); ++i) |
582 | 625 |
583 if (ParallelGCThreads > 0) { | 626 if (ParallelGCThreads > 0) { |
584 set_par_traversal(false); | 627 set_par_traversal(false); |
585 } | 628 } |
586 | 629 |
630 DirtyCardQueueSet& into_cset_dcqs = _g1->into_cset_dirty_card_queue_set(); | |
631 int into_cset_n_buffers = into_cset_dcqs.completed_buffers_num(); | |
632 | |
587 if (_g1->evacuation_failed()) { | 633 if (_g1->evacuation_failed()) { |
588 // Restore remembered sets for the regions pointing into | 634 // Restore remembered sets for the regions pointing into the collection set. |
589 // the collection set. | 635 |
590 if (G1DeferredRSUpdate) { | 636 if (G1DeferredRSUpdate) { |
591 DirtyCardQueue dcq(&_g1->dirty_card_queue_set()); | 637 // If deferred RS updates are enabled then we just need to transfer |
592 UpdateRSetOopsIntoCSDeferred deferred_update(_g1, &dcq); | 638 // the completed buffers from (a) the DirtyCardQueueSet used to hold |
593 new_refs_iterate(&deferred_update); | 639 // cards that contain references that point into the collection set |
640 // to (b) the DCQS used to hold the deferred RS updates | |
641 _g1->dirty_card_queue_set().merge_bufferlists(&into_cset_dcqs); | |
594 } else { | 642 } else { |
595 UpdateRSetOopsIntoCSImmediate immediate_update(_g1); | 643 |
596 new_refs_iterate(&immediate_update); | 644 CardTableModRefBS* bs = (CardTableModRefBS*)_g1->barrier_set(); |
597 } | 645 UpdateRSetCardTableEntryIntoCSetClosure update_rs_cset_immediate(_g1, bs); |
598 } | 646 |
599 for (uint i = 0; i < n_workers(); i++) { | 647 int n_completed_buffers = 0; |
600 _new_refs[i]->clear(); | 648 while (into_cset_dcqs.apply_closure_to_completed_buffer(&update_rs_cset_immediate, |
601 } | 649 0, 0, true)) { |
650 n_completed_buffers++; | |
651 } | |
652 assert(n_completed_buffers == into_cset_n_buffers, "missed some buffers"); | |
653 } | |
654 } | |
655 | |
656 // Free any completed buffers in the DirtyCardQueueSet used to hold cards | |
657 // which contain references that point into the collection. | |
658 _g1->into_cset_dirty_card_queue_set().clear(); | |
659 assert(_g1->into_cset_dirty_card_queue_set().completed_buffers_num() == 0, | |
660 "all buffers should be freed"); | |
661 _g1->into_cset_dirty_card_queue_set().clear_n_completed_buffers(); | |
602 | 662 |
603 assert(!_par_traversal_in_progress, "Invariant between iterations."); | 663 assert(!_par_traversal_in_progress, "Invariant between iterations."); |
604 } | 664 } |
605 | 665 |
606 class UpdateRSObjectClosure: public ObjectClosure { | 666 class UpdateRSObjectClosure: public ObjectClosure { |
650 } | 710 } |
651 | 711 |
652 | 712 |
653 static IntHistogram out_of_histo(50, 50); | 713 static IntHistogram out_of_histo(50, 50); |
654 | 714 |
655 void HRInto_G1RemSet::concurrentRefineOneCard_impl(jbyte* card_ptr, int worker_i) { | 715 class TriggerClosure : public OopClosure { |
716 bool _trigger; | |
717 public: | |
718 TriggerClosure() : _trigger(false) { } | |
719 bool value() const { return _trigger; } | |
720 template <class T> void do_oop_nv(T* p) { _trigger = true; } | |
721 virtual void do_oop(oop* p) { do_oop_nv(p); } | |
722 virtual void do_oop(narrowOop* p) { do_oop_nv(p); } | |
723 }; | |
724 | |
725 class InvokeIfNotTriggeredClosure: public OopClosure { | |
726 TriggerClosure* _t; | |
727 OopClosure* _oc; | |
728 public: | |
729 InvokeIfNotTriggeredClosure(TriggerClosure* t, OopClosure* oc): | |
730 _t(t), _oc(oc) { } | |
731 template <class T> void do_oop_nv(T* p) { | |
732 if (!_t->value()) _oc->do_oop(p); | |
733 } | |
734 virtual void do_oop(oop* p) { do_oop_nv(p); } | |
735 virtual void do_oop(narrowOop* p) { do_oop_nv(p); } | |
736 }; | |
737 | |
738 class Mux2Closure : public OopClosure { | |
739 OopClosure* _c1; | |
740 OopClosure* _c2; | |
741 public: | |
742 Mux2Closure(OopClosure *c1, OopClosure *c2) : _c1(c1), _c2(c2) { } | |
743 template <class T> void do_oop_nv(T* p) { | |
744 _c1->do_oop(p); _c2->do_oop(p); | |
745 } | |
746 virtual void do_oop(oop* p) { do_oop_nv(p); } | |
747 virtual void do_oop(narrowOop* p) { do_oop_nv(p); } | |
748 }; | |
749 | |
750 bool HRInto_G1RemSet::concurrentRefineOneCard_impl(jbyte* card_ptr, int worker_i, | |
751 bool check_for_refs_into_cset) { | |
656 // Construct the region representing the card. | 752 // Construct the region representing the card. |
657 HeapWord* start = _ct_bs->addr_for(card_ptr); | 753 HeapWord* start = _ct_bs->addr_for(card_ptr); |
658 // And find the region containing it. | 754 // And find the region containing it. |
659 HeapRegion* r = _g1->heap_region_containing(start); | 755 HeapRegion* r = _g1->heap_region_containing(start); |
660 assert(r != NULL, "unexpected null"); | 756 assert(r != NULL, "unexpected null"); |
667 ct_freq_note_card(_ct_bs->index_for(start)); | 763 ct_freq_note_card(_ct_bs->index_for(start)); |
668 #endif | 764 #endif |
669 | 765 |
670 UpdateRSOopClosure update_rs_oop_cl(this, worker_i); | 766 UpdateRSOopClosure update_rs_oop_cl(this, worker_i); |
671 update_rs_oop_cl.set_from(r); | 767 update_rs_oop_cl.set_from(r); |
672 FilterOutOfRegionClosure filter_then_update_rs_oop_cl(r, &update_rs_oop_cl); | 768 |
769 TriggerClosure trigger_cl; | |
770 FilterIntoCSClosure into_cs_cl(NULL, _g1, &trigger_cl); | |
771 InvokeIfNotTriggeredClosure invoke_cl(&trigger_cl, &into_cs_cl); | |
772 Mux2Closure mux(&invoke_cl, &update_rs_oop_cl); | |
773 | |
774 FilterOutOfRegionClosure filter_then_update_rs_oop_cl(r, | |
775 (check_for_refs_into_cset ? | |
776 (OopClosure*)&mux : | |
777 (OopClosure*)&update_rs_oop_cl)); | |
673 | 778 |
674 // Undirty the card. | 779 // Undirty the card. |
675 *card_ptr = CardTableModRefBS::clean_card_val(); | 780 *card_ptr = CardTableModRefBS::clean_card_val(); |
676 // We must complete this write before we do any of the reads below. | 781 // We must complete this write before we do any of the reads below. |
677 OrderAccess::storeload(); | 782 OrderAccess::storeload(); |
715 } | 820 } |
716 } else { | 821 } else { |
717 out_of_histo.add_entry(filter_then_update_rs_oop_cl.out_of_region()); | 822 out_of_histo.add_entry(filter_then_update_rs_oop_cl.out_of_region()); |
718 _conc_refine_cards++; | 823 _conc_refine_cards++; |
719 } | 824 } |
720 } | 825 |
721 | 826 return trigger_cl.value(); |
722 void HRInto_G1RemSet::concurrentRefineOneCard(jbyte* card_ptr, int worker_i) { | 827 } |
828 | |
829 bool HRInto_G1RemSet::concurrentRefineOneCard(jbyte* card_ptr, int worker_i, | |
830 bool check_for_refs_into_cset) { | |
723 // If the card is no longer dirty, nothing to do. | 831 // If the card is no longer dirty, nothing to do. |
724 if (*card_ptr != CardTableModRefBS::dirty_card_val()) return; | 832 if (*card_ptr != CardTableModRefBS::dirty_card_val()) { |
833 // No need to return that this card contains refs that point | |
834 // into the collection set. | |
835 return false; | |
836 } | |
725 | 837 |
726 // Construct the region representing the card. | 838 // Construct the region representing the card. |
727 HeapWord* start = _ct_bs->addr_for(card_ptr); | 839 HeapWord* start = _ct_bs->addr_for(card_ptr); |
728 // And find the region containing it. | 840 // And find the region containing it. |
729 HeapRegion* r = _g1->heap_region_containing(start); | 841 HeapRegion* r = _g1->heap_region_containing(start); |
730 if (r == NULL) { | 842 if (r == NULL) { |
731 guarantee(_g1->is_in_permanent(start), "Or else where?"); | 843 guarantee(_g1->is_in_permanent(start), "Or else where?"); |
732 return; // Not in the G1 heap (might be in perm, for example.) | 844 // Again no need to return that this card contains refs that |
845 // point into the collection set. | |
846 return false; // Not in the G1 heap (might be in perm, for example.) | |
733 } | 847 } |
734 // Why do we have to check here whether a card is on a young region, | 848 // Why do we have to check here whether a card is on a young region, |
735 // given that we dirty young regions and, as a result, the | 849 // given that we dirty young regions and, as a result, the |
736 // post-barrier is supposed to filter them out and never to enqueue | 850 // post-barrier is supposed to filter them out and never to enqueue |
737 // them? When we allocate a new region as the "allocation region" we | 851 // them? When we allocate a new region as the "allocation region" we |
741 // allocate objects in the region (after the acquire the lock) | 855 // allocate objects in the region (after the acquire the lock) |
742 // before all the cards on the region are dirtied. This is unlikely, | 856 // before all the cards on the region are dirtied. This is unlikely, |
743 // and it doesn't happen often, but it can happen. So, the extra | 857 // and it doesn't happen often, but it can happen. So, the extra |
744 // check below filters out those cards. | 858 // check below filters out those cards. |
745 if (r->is_young()) { | 859 if (r->is_young()) { |
746 return; | 860 return false; |
747 } | 861 } |
748 // While we are processing RSet buffers during the collection, we | 862 // While we are processing RSet buffers during the collection, we |
749 // actually don't want to scan any cards on the collection set, | 863 // actually don't want to scan any cards on the collection set, |
750 // since we don't want to update remebered sets with entries that | 864 // since we don't want to update remebered sets with entries that |
751 // point into the collection set, given that live objects from the | 865 // point into the collection set, given that live objects from the |
754 // involves scanning a card in the collection set and coming across | 868 // involves scanning a card in the collection set and coming across |
755 // an array that was being chunked and looking malformed. Note, | 869 // an array that was being chunked and looking malformed. Note, |
756 // however, that if evacuation fails, we have to scan any objects | 870 // however, that if evacuation fails, we have to scan any objects |
757 // that were not moved and create any missing entries. | 871 // that were not moved and create any missing entries. |
758 if (r->in_collection_set()) { | 872 if (r->in_collection_set()) { |
759 return; | 873 return false; |
760 } | 874 } |
761 | 875 |
762 // Should we defer processing the card? | 876 // Should we defer processing the card? |
763 // | 877 // |
764 // Previously the result from the insert_cache call would be | 878 // Previously the result from the insert_cache call would be |
795 // non-null true card not evicted from _card_counts; card_ptr is | 909 // non-null true card not evicted from _card_counts; card_ptr is |
796 // currently cold, or caused an eviction from hot | 910 // currently cold, or caused an eviction from hot |
797 // cache. | 911 // cache. |
798 // Immediately process res; no need to process card_ptr. | 912 // Immediately process res; no need to process card_ptr. |
799 | 913 |
914 | |
800 jbyte* res = card_ptr; | 915 jbyte* res = card_ptr; |
801 bool defer = false; | 916 bool defer = false; |
917 | |
918 // This gets set to true if the card being refined has references | |
919 // that point into the collection set. | |
920 bool oops_into_cset = false; | |
921 | |
802 if (_cg1r->use_cache()) { | 922 if (_cg1r->use_cache()) { |
803 jbyte* res = _cg1r->cache_insert(card_ptr, &defer); | 923 jbyte* res = _cg1r->cache_insert(card_ptr, &defer); |
804 if (res != NULL && (res != card_ptr || defer)) { | 924 if (res != NULL && (res != card_ptr || defer)) { |
805 start = _ct_bs->addr_for(res); | 925 start = _ct_bs->addr_for(res); |
806 r = _g1->heap_region_containing(start); | 926 r = _g1->heap_region_containing(start); |
813 // Hence we could see its young type change at any time. | 933 // Hence we could see its young type change at any time. |
814 // | 934 // |
815 // Process card pointer we get back from the hot card cache. This | 935 // Process card pointer we get back from the hot card cache. This |
816 // will check whether the region containing the card is young | 936 // will check whether the region containing the card is young |
817 // _after_ checking that the region has been allocated from. | 937 // _after_ checking that the region has been allocated from. |
818 concurrentRefineOneCard_impl(res, worker_i); | 938 oops_into_cset = concurrentRefineOneCard_impl(res, worker_i, |
939 false /* check_for_refs_into_cset */); | |
940 // The above call to concurrentRefineOneCard_impl is only | |
941 // performed if the hot card cache is enabled. This cache is | |
942 // disabled during an evacuation pause - which is the only | |
943 // time when we need know if the card contains references | |
944 // that point into the collection set. Also when the hot card | |
945 // cache is enabled, this code is executed by the concurrent | |
946 // refine threads - rather than the GC worker threads - and | |
947 // concurrentRefineOneCard_impl will return false. | |
948 assert(!oops_into_cset, "should not see true here"); | |
819 } | 949 } |
820 } | 950 } |
821 } | 951 } |
822 | 952 |
823 if (!defer) { | 953 if (!defer) { |
824 concurrentRefineOneCard_impl(card_ptr, worker_i); | 954 oops_into_cset = |
825 } | 955 concurrentRefineOneCard_impl(card_ptr, worker_i, check_for_refs_into_cset); |
956 // We should only be detecting that the card contains references | |
957 // that point into the collection set if the current thread is | |
958 // a GC worker thread. | |
959 assert(!oops_into_cset || SafepointSynchronize::is_at_safepoint(), | |
960 "invalid result at non safepoint"); | |
961 } | |
962 return oops_into_cset; | |
826 } | 963 } |
827 | 964 |
828 class HRRSStatsIter: public HeapRegionClosure { | 965 class HRRSStatsIter: public HeapRegionClosure { |
829 size_t _occupied; | 966 size_t _occupied; |
830 size_t _total_mem_sz; | 967 size_t _total_mem_sz; |
918 gclog_or_tty->print_cr(" Did %d coarsenings.", | 1055 gclog_or_tty->print_cr(" Did %d coarsenings.", |
919 HeapRegionRemSet::n_coarsenings()); | 1056 HeapRegionRemSet::n_coarsenings()); |
920 | 1057 |
921 } | 1058 } |
922 } | 1059 } |
1060 | |
923 void HRInto_G1RemSet::prepare_for_verify() { | 1061 void HRInto_G1RemSet::prepare_for_verify() { |
924 if (G1HRRSFlushLogBuffersOnVerify && | 1062 if (G1HRRSFlushLogBuffersOnVerify && |
925 (VerifyBeforeGC || VerifyAfterGC) | 1063 (VerifyBeforeGC || VerifyAfterGC) |
926 && !_g1->full_collection()) { | 1064 && !_g1->full_collection()) { |
927 cleanupHRRS(); | 1065 cleanupHRRS(); |
930 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); | 1068 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); |
931 dcqs.concatenate_logs(); | 1069 dcqs.concatenate_logs(); |
932 } | 1070 } |
933 bool cg1r_use_cache = _cg1r->use_cache(); | 1071 bool cg1r_use_cache = _cg1r->use_cache(); |
934 _cg1r->set_use_cache(false); | 1072 _cg1r->set_use_cache(false); |
935 updateRS(0); | 1073 DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set()); |
1074 updateRS(&into_cset_dcq, 0); | |
1075 _g1->into_cset_dirty_card_queue_set().clear(); | |
936 _cg1r->set_use_cache(cg1r_use_cache); | 1076 _cg1r->set_use_cache(cg1r_use_cache); |
937 | 1077 |
938 assert(JavaThread::dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed"); | 1078 assert(JavaThread::dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed"); |
939 } | 1079 } |
940 } | 1080 } |