comparison src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp @ 677:96b229c54d1e

6543938: G1: remove the concept of popularity Reviewed-by: iveresov, tonyp
author apetrusenko
date Wed, 25 Mar 2009 13:10:54 -0700
parents bd441136a5ce
children 4ac7d97e6101
comparison
equal deleted inserted replaced
649:59f139e8a8d1 677:96b229c54d1e
39 // mem_allocate_work, which all such allocation functions call. 39 // mem_allocate_work, which all such allocation functions call.
40 // (Note that this does not apply to TLAB allocation, which is not part 40 // (Note that this does not apply to TLAB allocation, which is not part
41 // of this interface: it is done by clients of this interface.) 41 // of this interface: it is done by clients of this interface.)
42 42
43 // Local to this file. 43 // Local to this file.
44
45 // Finds the first HeapRegion.
46 // No longer used, but might be handy someday.
47
48 class FindFirstRegionClosure: public HeapRegionClosure {
49 HeapRegion* _a_region;
50 public:
51 FindFirstRegionClosure() : _a_region(NULL) {}
52 bool doHeapRegion(HeapRegion* r) {
53 _a_region = r;
54 return true;
55 }
56 HeapRegion* result() { return _a_region; }
57 };
58
59 44
60 class RefineCardTableEntryClosure: public CardTableEntryClosure { 45 class RefineCardTableEntryClosure: public CardTableEntryClosure {
61 SuspendibleThreadSet* _sts; 46 SuspendibleThreadSet* _sts;
62 G1RemSet* _g1rs; 47 G1RemSet* _g1rs;
63 ConcurrentG1Refine* _cg1r; 48 ConcurrentG1Refine* _cg1r;
1205 size_t& freed_regions, 1190 size_t& freed_regions,
1206 UncleanRegionList* list, 1191 UncleanRegionList* list,
1207 bool par) { 1192 bool par) {
1208 assert(!hr->continuesHumongous(), "should have filtered these out"); 1193 assert(!hr->continuesHumongous(), "should have filtered these out");
1209 size_t res = 0; 1194 size_t res = 0;
1210 if (!hr->popular() && hr->used() > 0 && hr->garbage_bytes() == hr->used()) { 1195 if (hr->used() > 0 && hr->garbage_bytes() == hr->used() &&
1211 if (!hr->is_young()) { 1196 !hr->is_young()) {
1212 if (G1PolicyVerbose > 0) 1197 if (G1PolicyVerbose > 0)
1213 gclog_or_tty->print_cr("Freeing empty region "PTR_FORMAT "(" SIZE_FORMAT " bytes)" 1198 gclog_or_tty->print_cr("Freeing empty region "PTR_FORMAT "(" SIZE_FORMAT " bytes)"
1214 " during cleanup", hr, hr->used()); 1199 " during cleanup", hr, hr->used());
1215 free_region_work(hr, pre_used, cleared_h, freed_regions, list, par); 1200 free_region_work(hr, pre_used, cleared_h, freed_regions, list, par);
1216 }
1217 } 1201 }
1218 } 1202 }
1219 1203
1220 // FIXME: both this and shrink could probably be more efficient by 1204 // FIXME: both this and shrink could probably be more efficient by
1221 // doing one "VirtualSpace::expand_by" call rather than several. 1205 // doing one "VirtualSpace::expand_by" call rather than several.
1340 _cg1r(NULL), _czft(NULL), _summary_bytes_used(0), 1324 _cg1r(NULL), _czft(NULL), _summary_bytes_used(0),
1341 _cur_alloc_region(NULL), 1325 _cur_alloc_region(NULL),
1342 _refine_cte_cl(NULL), 1326 _refine_cte_cl(NULL),
1343 _free_region_list(NULL), _free_region_list_size(0), 1327 _free_region_list(NULL), _free_region_list_size(0),
1344 _free_regions(0), 1328 _free_regions(0),
1345 _popular_object_boundary(NULL),
1346 _cur_pop_hr_index(0),
1347 _popular_regions_to_be_evacuated(NULL),
1348 _pop_obj_rc_at_copy(),
1349 _full_collection(false), 1329 _full_collection(false),
1350 _unclean_region_list(), 1330 _unclean_region_list(),
1351 _unclean_regions_coming(false), 1331 _unclean_regions_coming(false),
1352 _young_list(new YoungList(this)), 1332 _young_list(new YoungList(this)),
1353 _gc_time_stamp(0), 1333 _gc_time_stamp(0),
1518 // ...and the concurrent zero-fill thread, if necessary. 1498 // ...and the concurrent zero-fill thread, if necessary.
1519 if (G1ConcZeroFill) { 1499 if (G1ConcZeroFill) {
1520 _czft = new ConcurrentZFThread(); 1500 _czft = new ConcurrentZFThread();
1521 } 1501 }
1522 1502
1523
1524
1525 // Allocate the popular regions; take them off free lists.
1526 size_t pop_byte_size = G1NumPopularRegions * HeapRegion::GrainBytes;
1527 expand(pop_byte_size);
1528 _popular_object_boundary =
1529 _g1_reserved.start() + (G1NumPopularRegions * HeapRegion::GrainWords);
1530 for (int i = 0; i < G1NumPopularRegions; i++) {
1531 HeapRegion* hr = newAllocRegion(HeapRegion::GrainWords);
1532 // assert(hr != NULL && hr->bottom() < _popular_object_boundary,
1533 // "Should be enough, and all should be below boundary.");
1534 hr->set_popular(true);
1535 }
1536 assert(_cur_pop_hr_index == 0, "Start allocating at the first region.");
1537
1538 // Initialize the from_card cache structure of HeapRegionRemSet. 1503 // Initialize the from_card cache structure of HeapRegionRemSet.
1539 HeapRegionRemSet::init_heap(max_regions()); 1504 HeapRegionRemSet::init_heap(max_regions());
1540 1505
1541 // Now expand into the rest of the initial heap size. 1506 // Now expand into the initial heap size.
1542 expand(init_byte_size - pop_byte_size); 1507 expand(init_byte_size);
1543 1508
1544 // Perform any initialization actions delegated to the policy. 1509 // Perform any initialization actions delegated to the policy.
1545 g1_policy()->init(); 1510 g1_policy()->init();
1546 1511
1547 g1_policy()->note_start_of_mark_thread(); 1512 g1_policy()->note_start_of_mark_thread();
1652 1617
1653 #ifndef PRODUCT 1618 #ifndef PRODUCT
1654 class SumUsedRegionsClosure: public HeapRegionClosure { 1619 class SumUsedRegionsClosure: public HeapRegionClosure {
1655 size_t _num; 1620 size_t _num;
1656 public: 1621 public:
1657 // _num is set to 1 to account for the popular region 1622 SumUsedRegionsClosure() : _num(0) {}
1658 SumUsedRegionsClosure() : _num(G1NumPopularRegions) {}
1659 bool doHeapRegion(HeapRegion* r) { 1623 bool doHeapRegion(HeapRegion* r) {
1660 if (r->continuesHumongous() || r->used() > 0 || r->is_gc_alloc_region()) { 1624 if (r->continuesHumongous() || r->used() > 0 || r->is_gc_alloc_region()) {
1661 _num += 1; 1625 _num += 1;
1662 } 1626 }
1663 return false; 1627 return false;
2316 concurrent_mark()->print_summary_info(); 2280 concurrent_mark()->print_summary_info();
2317 } 2281 }
2318 if (SummarizeG1ZFStats) { 2282 if (SummarizeG1ZFStats) {
2319 ConcurrentZFThread::print_summary_info(); 2283 ConcurrentZFThread::print_summary_info();
2320 } 2284 }
2321 if (G1SummarizePopularity) {
2322 print_popularity_summary_info();
2323 }
2324 g1_policy()->print_yg_surv_rate_info(); 2285 g1_policy()->print_yg_surv_rate_info();
2325 2286
2326 GCOverheadReporter::printGCOverhead(); 2287 GCOverheadReporter::printGCOverhead();
2327 2288
2328 SpecializationStats::print(); 2289 SpecializationStats::print();
2493 } 2454 }
2494 2455
2495 // </NEW PREDICTION> 2456 // </NEW PREDICTION>
2496 2457
2497 void 2458 void
2498 G1CollectedHeap::do_collection_pause_at_safepoint(HeapRegion* popular_region) { 2459 G1CollectedHeap::do_collection_pause_at_safepoint() {
2499 char verbose_str[128]; 2460 char verbose_str[128];
2500 sprintf(verbose_str, "GC pause "); 2461 sprintf(verbose_str, "GC pause ");
2501 if (popular_region != NULL) 2462 if (g1_policy()->in_young_gc_mode()) {
2502 strcat(verbose_str, "(popular)");
2503 else if (g1_policy()->in_young_gc_mode()) {
2504 if (g1_policy()->full_young_gcs()) 2463 if (g1_policy()->full_young_gcs())
2505 strcat(verbose_str, "(young)"); 2464 strcat(verbose_str, "(young)");
2506 else 2465 else
2507 strcat(verbose_str, "(partial)"); 2466 strcat(verbose_str, "(partial)");
2508 } 2467 }
2509 bool reset_should_initiate_conc_mark = false;
2510 if (popular_region != NULL && g1_policy()->should_initiate_conc_mark()) {
2511 // we currently do not allow an initial mark phase to be piggy-backed
2512 // on a popular pause
2513 reset_should_initiate_conc_mark = true;
2514 g1_policy()->unset_should_initiate_conc_mark();
2515 }
2516 if (g1_policy()->should_initiate_conc_mark()) 2468 if (g1_policy()->should_initiate_conc_mark())
2517 strcat(verbose_str, " (initial-mark)"); 2469 strcat(verbose_str, " (initial-mark)");
2518 2470
2519 GCCauseSetter x(this, (popular_region == NULL ? 2471 GCCauseSetter x(this, GCCause::_g1_inc_collection_pause);
2520 GCCause::_g1_inc_collection_pause :
2521 GCCause::_g1_pop_region_collection_pause));
2522 2472
2523 // if PrintGCDetails is on, we'll print long statistics information 2473 // if PrintGCDetails is on, we'll print long statistics information
2524 // in the collector policy code, so let's not print this as the output 2474 // in the collector policy code, so let's not print this as the output
2525 // is messy if we do. 2475 // is messy if we do.
2526 gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps); 2476 gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps);
2607 concurrent_mark()->checkpointRootsInitialPre(); 2557 concurrent_mark()->checkpointRootsInitialPre();
2608 } 2558 }
2609 save_marks(); 2559 save_marks();
2610 2560
2611 // We must do this before any possible evacuation that should propagate 2561 // We must do this before any possible evacuation that should propagate
2612 // marks, including evacuation of popular objects in a popular pause. 2562 // marks.
2613 if (mark_in_progress()) { 2563 if (mark_in_progress()) {
2614 double start_time_sec = os::elapsedTime(); 2564 double start_time_sec = os::elapsedTime();
2615 2565
2616 _cm->drainAllSATBBuffers(); 2566 _cm->drainAllSATBBuffers();
2617 double finish_mark_ms = (os::elapsedTime() - start_time_sec) * 1000.0; 2567 double finish_mark_ms = (os::elapsedTime() - start_time_sec) * 1000.0;
2624 // progress, this will be zero. 2574 // progress, this will be zero.
2625 _cm->set_oops_do_bound(); 2575 _cm->set_oops_do_bound();
2626 2576
2627 assert(regions_accounted_for(), "Region leakage."); 2577 assert(regions_accounted_for(), "Region leakage.");
2628 2578
2629 bool abandoned = false;
2630
2631 if (mark_in_progress()) 2579 if (mark_in_progress())
2632 concurrent_mark()->newCSet(); 2580 concurrent_mark()->newCSet();
2633 2581
2634 // Now choose the CS. 2582 // Now choose the CS.
2635 if (popular_region == NULL) { 2583 g1_policy()->choose_collection_set();
2636 g1_policy()->choose_collection_set(); 2584
2637 } else {
2638 // We may be evacuating a single region (for popularity).
2639 g1_policy()->record_popular_pause_preamble_start();
2640 popularity_pause_preamble(popular_region);
2641 g1_policy()->record_popular_pause_preamble_end();
2642 abandoned = (g1_policy()->collection_set() == NULL);
2643 // Now we allow more regions to be added (we have to collect
2644 // all popular regions).
2645 if (!abandoned) {
2646 g1_policy()->choose_collection_set(popular_region);
2647 }
2648 }
2649 // We may abandon a pause if we find no region that will fit in the MMU 2585 // We may abandon a pause if we find no region that will fit in the MMU
2650 // pause. 2586 // pause.
2651 abandoned = (g1_policy()->collection_set() == NULL); 2587 bool abandoned = (g1_policy()->collection_set() == NULL);
2652 2588
2653 // Nothing to do if we were unable to choose a collection set. 2589 // Nothing to do if we were unable to choose a collection set.
2654 if (!abandoned) { 2590 if (!abandoned) {
2655 #if G1_REM_SET_LOGGING 2591 #if G1_REM_SET_LOGGING
2656 gclog_or_tty->print_cr("\nAfter pause, heap:"); 2592 gclog_or_tty->print_cr("\nAfter pause, heap:");
2670 FREE_C_HEAP_ARRAY(bool, _in_cset_fast_test_base); 2606 FREE_C_HEAP_ARRAY(bool, _in_cset_fast_test_base);
2671 // this is more for peace of mind; we're nulling them here and 2607 // this is more for peace of mind; we're nulling them here and
2672 // we're expecting them to be null at the beginning of the next GC 2608 // we're expecting them to be null at the beginning of the next GC
2673 _in_cset_fast_test = NULL; 2609 _in_cset_fast_test = NULL;
2674 _in_cset_fast_test_base = NULL; 2610 _in_cset_fast_test_base = NULL;
2675
2676 if (popular_region != NULL) {
2677 // We have to wait until now, because we don't want the region to
2678 // be rescheduled for pop-evac during RS update.
2679 popular_region->set_popular_pending(false);
2680 }
2681 2611
2682 release_gc_alloc_regions(false /* totally */); 2612 release_gc_alloc_regions(false /* totally */);
2683 2613
2684 cleanup_surviving_young_words(); 2614 cleanup_surviving_young_words();
2685 2615
2722 2652
2723 double end_time_sec = os::elapsedTime(); 2653 double end_time_sec = os::elapsedTime();
2724 double pause_time_ms = (end_time_sec - start_time_sec) * MILLIUNITS; 2654 double pause_time_ms = (end_time_sec - start_time_sec) * MILLIUNITS;
2725 g1_policy()->record_pause_time_ms(pause_time_ms); 2655 g1_policy()->record_pause_time_ms(pause_time_ms);
2726 GCOverheadReporter::recordSTWEnd(end_time_sec); 2656 GCOverheadReporter::recordSTWEnd(end_time_sec);
2727 g1_policy()->record_collection_pause_end(popular_region != NULL, 2657 g1_policy()->record_collection_pause_end(abandoned);
2728 abandoned);
2729 2658
2730 assert(regions_accounted_for(), "Region leakage."); 2659 assert(regions_accounted_for(), "Region leakage.");
2731 2660
2732 if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) { 2661 if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) {
2733 HandleMark hm; // Discard invalid handles created during verification 2662 HandleMark hm; // Discard invalid handles created during verification
2756 2685
2757 gc_epilogue(false); 2686 gc_epilogue(false);
2758 } 2687 }
2759 2688
2760 assert(verify_region_lists(), "Bad region lists."); 2689 assert(verify_region_lists(), "Bad region lists.");
2761
2762 if (reset_should_initiate_conc_mark)
2763 g1_policy()->set_should_initiate_conc_mark();
2764 2690
2765 if (ExitAfterGCNum > 0 && total_collections() == ExitAfterGCNum) { 2691 if (ExitAfterGCNum > 0 && total_collections() == ExitAfterGCNum) {
2766 gclog_or_tty->print_cr("Stopping after GC #%d", ExitAfterGCNum); 2692 gclog_or_tty->print_cr("Stopping after GC #%d", ExitAfterGCNum);
2767 print_tracing_info(); 2693 print_tracing_info();
2768 vm_exit(-1); 2694 vm_exit(-1);
4705 size_t& pre_used, 4631 size_t& pre_used,
4706 size_t& cleared_h_regions, 4632 size_t& cleared_h_regions,
4707 size_t& freed_regions, 4633 size_t& freed_regions,
4708 UncleanRegionList* list, 4634 UncleanRegionList* list,
4709 bool par) { 4635 bool par) {
4710 assert(!hr->popular(), "should not free popular regions");
4711 pre_used += hr->used(); 4636 pre_used += hr->used();
4712 if (hr->isHumongous()) { 4637 if (hr->isHumongous()) {
4713 assert(hr->startsHumongous(), 4638 assert(hr->startsHumongous(),
4714 "Only the start of a humongous region should be freed."); 4639 "Only the start of a humongous region should be freed.");
4715 int ind = _hrs->find(hr); 4640 int ind = _hrs->find(hr);
4789 g1_policy()->record_clear_ct_time( elapsed * 1000.0); 4714 g1_policy()->record_clear_ct_time( elapsed * 1000.0);
4790 } 4715 }
4791 4716
4792 4717
4793 void G1CollectedHeap::do_collection_pause_if_appropriate(size_t word_size) { 4718 void G1CollectedHeap::do_collection_pause_if_appropriate(size_t word_size) {
4794 // First do any popular regions.
4795 HeapRegion* hr;
4796 while ((hr = popular_region_to_evac()) != NULL) {
4797 evac_popular_region(hr);
4798 }
4799 // Now do heuristic pauses.
4800 if (g1_policy()->should_do_collection_pause(word_size)) { 4719 if (g1_policy()->should_do_collection_pause(word_size)) {
4801 do_collection_pause(); 4720 do_collection_pause();
4802 } 4721 }
4803 } 4722 }
4804 4723
5190 class RegionCounter: public HeapRegionClosure { 5109 class RegionCounter: public HeapRegionClosure {
5191 size_t _n; 5110 size_t _n;
5192 public: 5111 public:
5193 RegionCounter() : _n(0) {} 5112 RegionCounter() : _n(0) {}
5194 bool doHeapRegion(HeapRegion* r) { 5113 bool doHeapRegion(HeapRegion* r) {
5195 if (r->is_empty() && !r->popular()) { 5114 if (r->is_empty()) {
5196 assert(!r->isHumongous(), "H regions should not be empty."); 5115 assert(!r->isHumongous(), "H regions should not be empty.");
5197 _n++; 5116 _n++;
5198 } 5117 }
5199 return false; 5118 return false;
5200 } 5119 }
5334 pointer_delta(r->end(), r->top())); 5253 pointer_delta(r->end(), r->top()));
5335 } 5254 }
5336 r->set_zero_fill_allocated(); 5255 r->set_zero_fill_allocated();
5337 } else { 5256 } else {
5338 assert(r->is_empty(), "tautology"); 5257 assert(r->is_empty(), "tautology");
5339 if (r->popular()) { 5258 _n++;
5340 if (r->zero_fill_state() != HeapRegion::Allocated) { 5259 switch (r->zero_fill_state()) {
5341 r->ensure_zero_filled_locked();
5342 r->set_zero_fill_allocated();
5343 }
5344 } else {
5345 _n++;
5346 switch (r->zero_fill_state()) {
5347 case HeapRegion::NotZeroFilled: 5260 case HeapRegion::NotZeroFilled:
5348 case HeapRegion::ZeroFilling: 5261 case HeapRegion::ZeroFilling:
5349 _g1->put_region_on_unclean_list_locked(r); 5262 _g1->put_region_on_unclean_list_locked(r);
5350 break; 5263 break;
5351 case HeapRegion::Allocated: 5264 case HeapRegion::Allocated:
5352 r->set_zero_fill_complete(); 5265 r->set_zero_fill_complete();
5353 // no break; go on to put on free list. 5266 // no break; go on to put on free list.
5354 case HeapRegion::ZeroFilled: 5267 case HeapRegion::ZeroFilled:
5355 _g1->put_free_region_on_list_locked(r); 5268 _g1->put_free_region_on_list_locked(r);
5356 break; 5269 break;
5357 }
5358 } 5270 }
5359 } 5271 }
5360 return false; 5272 return false;
5361 } 5273 }
5362 5274
5398 void G1CollectedHeap::set_used_regions_to_need_zero_fill() { 5310 void G1CollectedHeap::set_used_regions_to_need_zero_fill() {
5399 MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); 5311 MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);
5400 // This needs to go at the end of the full GC. 5312 // This needs to go at the end of the full GC.
5401 UsedRegionsNeedZeroFillSetter rs; 5313 UsedRegionsNeedZeroFillSetter rs;
5402 heap_region_iterate(&rs); 5314 heap_region_iterate(&rs);
5403 }
5404
5405 class CountObjClosure: public ObjectClosure {
5406 size_t _n;
5407 public:
5408 CountObjClosure() : _n(0) {}
5409 void do_object(oop obj) { _n++; }
5410 size_t n() { return _n; }
5411 };
5412
5413 size_t G1CollectedHeap::pop_object_used_objs() {
5414 size_t sum_objs = 0;
5415 for (int i = 0; i < G1NumPopularRegions; i++) {
5416 CountObjClosure cl;
5417 _hrs->at(i)->object_iterate(&cl);
5418 sum_objs += cl.n();
5419 }
5420 return sum_objs;
5421 }
5422
5423 size_t G1CollectedHeap::pop_object_used_bytes() {
5424 size_t sum_bytes = 0;
5425 for (int i = 0; i < G1NumPopularRegions; i++) {
5426 sum_bytes += _hrs->at(i)->used();
5427 }
5428 return sum_bytes;
5429 }
5430
5431
5432 static int nq = 0;
5433
5434 HeapWord* G1CollectedHeap::allocate_popular_object(size_t word_size) {
5435 while (_cur_pop_hr_index < G1NumPopularRegions) {
5436 HeapRegion* cur_pop_region = _hrs->at(_cur_pop_hr_index);
5437 HeapWord* res = cur_pop_region->allocate(word_size);
5438 if (res != NULL) {
5439 // We account for popular objs directly in the used summary:
5440 _summary_bytes_used += (word_size * HeapWordSize);
5441 return res;
5442 }
5443 // Otherwise, try the next region (first making sure that we remember
5444 // the last "top" value as the "next_top_at_mark_start", so that
5445 // objects made popular during markings aren't automatically considered
5446 // live).
5447 cur_pop_region->note_end_of_copying();
5448 // Otherwise, try the next region.
5449 _cur_pop_hr_index++;
5450 }
5451 // XXX: For now !!!
5452 vm_exit_out_of_memory(word_size,
5453 "Not enough pop obj space (To Be Fixed)");
5454 return NULL;
5455 }
5456
5457 class HeapRegionList: public CHeapObj {
5458 public:
5459 HeapRegion* hr;
5460 HeapRegionList* next;
5461 };
5462
5463 void G1CollectedHeap::schedule_popular_region_evac(HeapRegion* r) {
5464 // This might happen during parallel GC, so protect by this lock.
5465 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
5466 // We don't schedule regions whose evacuations are already pending, or
5467 // are already being evacuated.
5468 if (!r->popular_pending() && !r->in_collection_set()) {
5469 r->set_popular_pending(true);
5470 if (G1TracePopularity) {
5471 gclog_or_tty->print_cr("Scheduling region "PTR_FORMAT" "
5472 "["PTR_FORMAT", "PTR_FORMAT") for pop-object evacuation.",
5473 r, r->bottom(), r->end());
5474 }
5475 HeapRegionList* hrl = new HeapRegionList;
5476 hrl->hr = r;
5477 hrl->next = _popular_regions_to_be_evacuated;
5478 _popular_regions_to_be_evacuated = hrl;
5479 }
5480 }
5481
5482 HeapRegion* G1CollectedHeap::popular_region_to_evac() {
5483 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
5484 HeapRegion* res = NULL;
5485 while (_popular_regions_to_be_evacuated != NULL && res == NULL) {
5486 HeapRegionList* hrl = _popular_regions_to_be_evacuated;
5487 _popular_regions_to_be_evacuated = hrl->next;
5488 res = hrl->hr;
5489 // The G1RSPopLimit may have increased, so recheck here...
5490 if (res->rem_set()->occupied() < (size_t) G1RSPopLimit) {
5491 // Hah: don't need to schedule.
5492 if (G1TracePopularity) {
5493 gclog_or_tty->print_cr("Unscheduling region "PTR_FORMAT" "
5494 "["PTR_FORMAT", "PTR_FORMAT") "
5495 "for pop-object evacuation (size %d < limit %d)",
5496 res, res->bottom(), res->end(),
5497 res->rem_set()->occupied(), G1RSPopLimit);
5498 }
5499 res->set_popular_pending(false);
5500 res = NULL;
5501 }
5502 // We do not reset res->popular() here; if we did so, it would allow
5503 // the region to be "rescheduled" for popularity evacuation. Instead,
5504 // this is done in the collection pause, with the world stopped.
5505 // So the invariant is that the regions in the list have the popularity
5506 // boolean set, but having the boolean set does not imply membership
5507 // on the list (though there can at most one such pop-pending region
5508 // not on the list at any time).
5509 delete hrl;
5510 }
5511 return res;
5512 }
5513
5514 void G1CollectedHeap::evac_popular_region(HeapRegion* hr) {
5515 while (true) {
5516 // Don't want to do a GC pause while cleanup is being completed!
5517 wait_for_cleanup_complete();
5518
5519 // Read the GC count while holding the Heap_lock
5520 int gc_count_before = SharedHeap::heap()->total_collections();
5521 g1_policy()->record_stop_world_start();
5522
5523 {
5524 MutexUnlocker mu(Heap_lock); // give up heap lock, execute gets it back
5525 VM_G1PopRegionCollectionPause op(gc_count_before, hr);
5526 VMThread::execute(&op);
5527
5528 // If the prolog succeeded, we didn't do a GC for this.
5529 if (op.prologue_succeeded()) break;
5530 }
5531 // Otherwise we didn't. We should recheck the size, though, since
5532 // the limit may have increased...
5533 if (hr->rem_set()->occupied() < (size_t) G1RSPopLimit) {
5534 hr->set_popular_pending(false);
5535 break;
5536 }
5537 }
5538 }
5539
5540 void G1CollectedHeap::atomic_inc_obj_rc(oop obj) {
5541 Atomic::inc(obj_rc_addr(obj));
5542 }
5543
5544 class CountRCClosure: public OopsInHeapRegionClosure {
5545 G1CollectedHeap* _g1h;
5546 bool _parallel;
5547 public:
5548 CountRCClosure(G1CollectedHeap* g1h) :
5549 _g1h(g1h), _parallel(ParallelGCThreads > 0)
5550 {}
5551 void do_oop(narrowOop* p) {
5552 guarantee(false, "NYI");
5553 }
5554 void do_oop(oop* p) {
5555 oop obj = *p;
5556 assert(obj != NULL, "Precondition.");
5557 if (_parallel) {
5558 // We go sticky at the limit to avoid excess contention.
5559 // If we want to track the actual RC's further, we'll need to keep a
5560 // per-thread hash table or something for the popular objects.
5561 if (_g1h->obj_rc(obj) < G1ObjPopLimit) {
5562 _g1h->atomic_inc_obj_rc(obj);
5563 }
5564 } else {
5565 _g1h->inc_obj_rc(obj);
5566 }
5567 }
5568 };
5569
5570 class EvacPopObjClosure: public ObjectClosure {
5571 G1CollectedHeap* _g1h;
5572 size_t _pop_objs;
5573 size_t _max_rc;
5574 public:
5575 EvacPopObjClosure(G1CollectedHeap* g1h) :
5576 _g1h(g1h), _pop_objs(0), _max_rc(0) {}
5577
5578 void do_object(oop obj) {
5579 size_t rc = _g1h->obj_rc(obj);
5580 _max_rc = MAX2(rc, _max_rc);
5581 if (rc >= (size_t) G1ObjPopLimit) {
5582 _g1h->_pop_obj_rc_at_copy.add((double)rc);
5583 size_t word_sz = obj->size();
5584 HeapWord* new_pop_loc = _g1h->allocate_popular_object(word_sz);
5585 oop new_pop_obj = (oop)new_pop_loc;
5586 Copy::aligned_disjoint_words((HeapWord*)obj, new_pop_loc, word_sz);
5587 obj->forward_to(new_pop_obj);
5588 G1ScanAndBalanceClosure scan_and_balance(_g1h);
5589 new_pop_obj->oop_iterate_backwards(&scan_and_balance);
5590 // preserve "next" mark bit if marking is in progress.
5591 if (_g1h->mark_in_progress() && !_g1h->is_obj_ill(obj)) {
5592 _g1h->concurrent_mark()->markAndGrayObjectIfNecessary(new_pop_obj);
5593 }
5594
5595 if (G1TracePopularity) {
5596 gclog_or_tty->print_cr("Found obj " PTR_FORMAT " of word size " SIZE_FORMAT
5597 " pop (%d), move to " PTR_FORMAT,
5598 (void*) obj, word_sz,
5599 _g1h->obj_rc(obj), (void*) new_pop_obj);
5600 }
5601 _pop_objs++;
5602 }
5603 }
5604 size_t pop_objs() { return _pop_objs; }
5605 size_t max_rc() { return _max_rc; }
5606 };
5607
5608 class G1ParCountRCTask : public AbstractGangTask {
5609 G1CollectedHeap* _g1h;
5610 BitMap _bm;
5611
5612 size_t getNCards() {
5613 return (_g1h->capacity() + G1BlockOffsetSharedArray::N_bytes - 1)
5614 / G1BlockOffsetSharedArray::N_bytes;
5615 }
5616 CountRCClosure _count_rc_closure;
5617 public:
5618 G1ParCountRCTask(G1CollectedHeap* g1h) :
5619 AbstractGangTask("G1 Par RC Count task"),
5620 _g1h(g1h), _bm(getNCards()), _count_rc_closure(g1h)
5621 {}
5622
5623 void work(int i) {
5624 ResourceMark rm;
5625 HandleMark hm;
5626 _g1h->g1_rem_set()->oops_into_collection_set_do(&_count_rc_closure, i);
5627 }
5628 };
5629
5630 void G1CollectedHeap::popularity_pause_preamble(HeapRegion* popular_region) {
5631 // We're evacuating a single region (for popularity).
5632 if (G1TracePopularity) {
5633 gclog_or_tty->print_cr("Doing pop region pause for ["PTR_FORMAT", "PTR_FORMAT")",
5634 popular_region->bottom(), popular_region->end());
5635 }
5636 g1_policy()->set_single_region_collection_set(popular_region);
5637 size_t max_rc;
5638 if (!compute_reference_counts_and_evac_popular(popular_region,
5639 &max_rc)) {
5640 // We didn't evacuate any popular objects.
5641 // We increase the RS popularity limit, to prevent this from
5642 // happening in the future.
5643 if (G1RSPopLimit < (1 << 30)) {
5644 G1RSPopLimit *= 2;
5645 }
5646 // For now, interesting enough for a message:
5647 #if 1
5648 gclog_or_tty->print_cr("In pop region pause for ["PTR_FORMAT", "PTR_FORMAT"), "
5649 "failed to find a pop object (max = %d).",
5650 popular_region->bottom(), popular_region->end(),
5651 max_rc);
5652 gclog_or_tty->print_cr("Increased G1RSPopLimit to %d.", G1RSPopLimit);
5653 #endif // 0
5654 // Also, we reset the collection set to NULL, to make the rest of
5655 // the collection do nothing.
5656 assert(popular_region->next_in_collection_set() == NULL,
5657 "should be single-region.");
5658 popular_region->set_in_collection_set(false);
5659 popular_region->set_popular_pending(false);
5660 g1_policy()->clear_collection_set();
5661 }
5662 }
5663
5664 bool G1CollectedHeap::
5665 compute_reference_counts_and_evac_popular(HeapRegion* popular_region,
5666 size_t* max_rc) {
5667 HeapWord* rc_region_bot;
5668 HeapWord* rc_region_end;
5669
5670 // Set up the reference count region.
5671 HeapRegion* rc_region = newAllocRegion(HeapRegion::GrainWords);
5672 if (rc_region != NULL) {
5673 rc_region_bot = rc_region->bottom();
5674 rc_region_end = rc_region->end();
5675 } else {
5676 rc_region_bot = NEW_C_HEAP_ARRAY(HeapWord, HeapRegion::GrainWords);
5677 if (rc_region_bot == NULL) {
5678 vm_exit_out_of_memory(HeapRegion::GrainWords,
5679 "No space for RC region.");
5680 }
5681 rc_region_end = rc_region_bot + HeapRegion::GrainWords;
5682 }
5683
5684 if (G1TracePopularity)
5685 gclog_or_tty->print_cr("RC region is ["PTR_FORMAT", "PTR_FORMAT")",
5686 rc_region_bot, rc_region_end);
5687 if (rc_region_bot > popular_region->bottom()) {
5688 _rc_region_above = true;
5689 _rc_region_diff =
5690 pointer_delta(rc_region_bot, popular_region->bottom(), 1);
5691 } else {
5692 assert(rc_region_bot < popular_region->bottom(), "Can't be equal.");
5693 _rc_region_above = false;
5694 _rc_region_diff =
5695 pointer_delta(popular_region->bottom(), rc_region_bot, 1);
5696 }
5697 g1_policy()->record_pop_compute_rc_start();
5698 // Count external references.
5699 g1_rem_set()->prepare_for_oops_into_collection_set_do();
5700 if (ParallelGCThreads > 0) {
5701
5702 set_par_threads(workers()->total_workers());
5703 G1ParCountRCTask par_count_rc_task(this);
5704 workers()->run_task(&par_count_rc_task);
5705 set_par_threads(0);
5706
5707 } else {
5708 CountRCClosure count_rc_closure(this);
5709 g1_rem_set()->oops_into_collection_set_do(&count_rc_closure, 0);
5710 }
5711 g1_rem_set()->cleanup_after_oops_into_collection_set_do();
5712 g1_policy()->record_pop_compute_rc_end();
5713
5714 // Now evacuate popular objects.
5715 g1_policy()->record_pop_evac_start();
5716 EvacPopObjClosure evac_pop_obj_cl(this);
5717 popular_region->object_iterate(&evac_pop_obj_cl);
5718 *max_rc = evac_pop_obj_cl.max_rc();
5719
5720 // Make sure the last "top" value of the current popular region is copied
5721 // as the "next_top_at_mark_start", so that objects made popular during
5722 // markings aren't automatically considered live.
5723 HeapRegion* cur_pop_region = _hrs->at(_cur_pop_hr_index);
5724 cur_pop_region->note_end_of_copying();
5725
5726 if (rc_region != NULL) {
5727 free_region(rc_region);
5728 } else {
5729 FREE_C_HEAP_ARRAY(HeapWord, rc_region_bot);
5730 }
5731 g1_policy()->record_pop_evac_end();
5732
5733 return evac_pop_obj_cl.pop_objs() > 0;
5734 }
5735
5736 class CountPopObjInfoClosure: public HeapRegionClosure {
5737 size_t _objs;
5738 size_t _bytes;
5739
5740 class CountObjClosure: public ObjectClosure {
5741 int _n;
5742 public:
5743 CountObjClosure() : _n(0) {}
5744 void do_object(oop obj) { _n++; }
5745 size_t n() { return _n; }
5746 };
5747
5748 public:
5749 CountPopObjInfoClosure() : _objs(0), _bytes(0) {}
5750 bool doHeapRegion(HeapRegion* r) {
5751 _bytes += r->used();
5752 CountObjClosure blk;
5753 r->object_iterate(&blk);
5754 _objs += blk.n();
5755 return false;
5756 }
5757 size_t objs() { return _objs; }
5758 size_t bytes() { return _bytes; }
5759 };
5760
5761
5762 void G1CollectedHeap::print_popularity_summary_info() const {
5763 CountPopObjInfoClosure blk;
5764 for (int i = 0; i <= _cur_pop_hr_index; i++) {
5765 blk.doHeapRegion(_hrs->at(i));
5766 }
5767 gclog_or_tty->print_cr("\nPopular objects: %d objs, %d bytes.",
5768 blk.objs(), blk.bytes());
5769 gclog_or_tty->print_cr(" RC at copy = [avg = %5.2f, max = %5.2f, sd = %5.2f].",
5770 _pop_obj_rc_at_copy.avg(),
5771 _pop_obj_rc_at_copy.maximum(),
5772 _pop_obj_rc_at_copy.sd());
5773 } 5315 }
5774 5316
5775 void G1CollectedHeap::set_refine_cte_cl_concurrency(bool concurrent) { 5317 void G1CollectedHeap::set_refine_cte_cl_concurrency(bool concurrent) {
5776 _refine_cte_cl->set_concurrent(concurrent); 5318 _refine_cte_cl->set_concurrent(concurrent);
5777 } 5319 }
5843 // TODO: regions accounting for young/survivor/tenured 5385 // TODO: regions accounting for young/survivor/tenured
5844 return true; 5386 return true;
5845 } 5387 }
5846 5388
5847 bool G1CollectedHeap::print_region_accounting_info() { 5389 bool G1CollectedHeap::print_region_accounting_info() {
5848 gclog_or_tty->print_cr("P regions: %d.", G1NumPopularRegions);
5849 gclog_or_tty->print_cr("Free regions: %d (count: %d count list %d) (clean: %d unclean: %d).", 5390 gclog_or_tty->print_cr("Free regions: %d (count: %d count list %d) (clean: %d unclean: %d).",
5850 free_regions(), 5391 free_regions(),
5851 count_free_regions(), count_free_regions_list(), 5392 count_free_regions(), count_free_regions_list(),
5852 _free_region_list_size, _unclean_region_list.sz()); 5393 _free_region_list_size, _unclean_region_list.sz());
5853 gclog_or_tty->print_cr("cur_alloc: %d.", 5394 gclog_or_tty->print_cr("cur_alloc: %d.",