Mercurial > hg > truffle
comparison src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp @ 677:96b229c54d1e
6543938: G1: remove the concept of popularity
Reviewed-by: iveresov, tonyp
author | apetrusenko |
---|---|
date | Wed, 25 Mar 2009 13:10:54 -0700 |
parents | bd441136a5ce |
children | 4ac7d97e6101 |
comparison
equal
deleted
inserted
replaced
649:59f139e8a8d1 | 677:96b229c54d1e |
---|---|
39 // mem_allocate_work, which all such allocation functions call. | 39 // mem_allocate_work, which all such allocation functions call. |
40 // (Note that this does not apply to TLAB allocation, which is not part | 40 // (Note that this does not apply to TLAB allocation, which is not part |
41 // of this interface: it is done by clients of this interface.) | 41 // of this interface: it is done by clients of this interface.) |
42 | 42 |
43 // Local to this file. | 43 // Local to this file. |
44 | |
45 // Finds the first HeapRegion. | |
46 // No longer used, but might be handy someday. | |
47 | |
48 class FindFirstRegionClosure: public HeapRegionClosure { | |
49 HeapRegion* _a_region; | |
50 public: | |
51 FindFirstRegionClosure() : _a_region(NULL) {} | |
52 bool doHeapRegion(HeapRegion* r) { | |
53 _a_region = r; | |
54 return true; | |
55 } | |
56 HeapRegion* result() { return _a_region; } | |
57 }; | |
58 | |
59 | 44 |
60 class RefineCardTableEntryClosure: public CardTableEntryClosure { | 45 class RefineCardTableEntryClosure: public CardTableEntryClosure { |
61 SuspendibleThreadSet* _sts; | 46 SuspendibleThreadSet* _sts; |
62 G1RemSet* _g1rs; | 47 G1RemSet* _g1rs; |
63 ConcurrentG1Refine* _cg1r; | 48 ConcurrentG1Refine* _cg1r; |
1205 size_t& freed_regions, | 1190 size_t& freed_regions, |
1206 UncleanRegionList* list, | 1191 UncleanRegionList* list, |
1207 bool par) { | 1192 bool par) { |
1208 assert(!hr->continuesHumongous(), "should have filtered these out"); | 1193 assert(!hr->continuesHumongous(), "should have filtered these out"); |
1209 size_t res = 0; | 1194 size_t res = 0; |
1210 if (!hr->popular() && hr->used() > 0 && hr->garbage_bytes() == hr->used()) { | 1195 if (hr->used() > 0 && hr->garbage_bytes() == hr->used() && |
1211 if (!hr->is_young()) { | 1196 !hr->is_young()) { |
1212 if (G1PolicyVerbose > 0) | 1197 if (G1PolicyVerbose > 0) |
1213 gclog_or_tty->print_cr("Freeing empty region "PTR_FORMAT "(" SIZE_FORMAT " bytes)" | 1198 gclog_or_tty->print_cr("Freeing empty region "PTR_FORMAT "(" SIZE_FORMAT " bytes)" |
1214 " during cleanup", hr, hr->used()); | 1199 " during cleanup", hr, hr->used()); |
1215 free_region_work(hr, pre_used, cleared_h, freed_regions, list, par); | 1200 free_region_work(hr, pre_used, cleared_h, freed_regions, list, par); |
1216 } | |
1217 } | 1201 } |
1218 } | 1202 } |
1219 | 1203 |
1220 // FIXME: both this and shrink could probably be more efficient by | 1204 // FIXME: both this and shrink could probably be more efficient by |
1221 // doing one "VirtualSpace::expand_by" call rather than several. | 1205 // doing one "VirtualSpace::expand_by" call rather than several. |
1340 _cg1r(NULL), _czft(NULL), _summary_bytes_used(0), | 1324 _cg1r(NULL), _czft(NULL), _summary_bytes_used(0), |
1341 _cur_alloc_region(NULL), | 1325 _cur_alloc_region(NULL), |
1342 _refine_cte_cl(NULL), | 1326 _refine_cte_cl(NULL), |
1343 _free_region_list(NULL), _free_region_list_size(0), | 1327 _free_region_list(NULL), _free_region_list_size(0), |
1344 _free_regions(0), | 1328 _free_regions(0), |
1345 _popular_object_boundary(NULL), | |
1346 _cur_pop_hr_index(0), | |
1347 _popular_regions_to_be_evacuated(NULL), | |
1348 _pop_obj_rc_at_copy(), | |
1349 _full_collection(false), | 1329 _full_collection(false), |
1350 _unclean_region_list(), | 1330 _unclean_region_list(), |
1351 _unclean_regions_coming(false), | 1331 _unclean_regions_coming(false), |
1352 _young_list(new YoungList(this)), | 1332 _young_list(new YoungList(this)), |
1353 _gc_time_stamp(0), | 1333 _gc_time_stamp(0), |
1518 // ...and the concurrent zero-fill thread, if necessary. | 1498 // ...and the concurrent zero-fill thread, if necessary. |
1519 if (G1ConcZeroFill) { | 1499 if (G1ConcZeroFill) { |
1520 _czft = new ConcurrentZFThread(); | 1500 _czft = new ConcurrentZFThread(); |
1521 } | 1501 } |
1522 | 1502 |
1523 | |
1524 | |
1525 // Allocate the popular regions; take them off free lists. | |
1526 size_t pop_byte_size = G1NumPopularRegions * HeapRegion::GrainBytes; | |
1527 expand(pop_byte_size); | |
1528 _popular_object_boundary = | |
1529 _g1_reserved.start() + (G1NumPopularRegions * HeapRegion::GrainWords); | |
1530 for (int i = 0; i < G1NumPopularRegions; i++) { | |
1531 HeapRegion* hr = newAllocRegion(HeapRegion::GrainWords); | |
1532 // assert(hr != NULL && hr->bottom() < _popular_object_boundary, | |
1533 // "Should be enough, and all should be below boundary."); | |
1534 hr->set_popular(true); | |
1535 } | |
1536 assert(_cur_pop_hr_index == 0, "Start allocating at the first region."); | |
1537 | |
1538 // Initialize the from_card cache structure of HeapRegionRemSet. | 1503 // Initialize the from_card cache structure of HeapRegionRemSet. |
1539 HeapRegionRemSet::init_heap(max_regions()); | 1504 HeapRegionRemSet::init_heap(max_regions()); |
1540 | 1505 |
1541 // Now expand into the rest of the initial heap size. | 1506 // Now expand into the initial heap size. |
1542 expand(init_byte_size - pop_byte_size); | 1507 expand(init_byte_size); |
1543 | 1508 |
1544 // Perform any initialization actions delegated to the policy. | 1509 // Perform any initialization actions delegated to the policy. |
1545 g1_policy()->init(); | 1510 g1_policy()->init(); |
1546 | 1511 |
1547 g1_policy()->note_start_of_mark_thread(); | 1512 g1_policy()->note_start_of_mark_thread(); |
1652 | 1617 |
1653 #ifndef PRODUCT | 1618 #ifndef PRODUCT |
1654 class SumUsedRegionsClosure: public HeapRegionClosure { | 1619 class SumUsedRegionsClosure: public HeapRegionClosure { |
1655 size_t _num; | 1620 size_t _num; |
1656 public: | 1621 public: |
1657 // _num is set to 1 to account for the popular region | 1622 SumUsedRegionsClosure() : _num(0) {} |
1658 SumUsedRegionsClosure() : _num(G1NumPopularRegions) {} | |
1659 bool doHeapRegion(HeapRegion* r) { | 1623 bool doHeapRegion(HeapRegion* r) { |
1660 if (r->continuesHumongous() || r->used() > 0 || r->is_gc_alloc_region()) { | 1624 if (r->continuesHumongous() || r->used() > 0 || r->is_gc_alloc_region()) { |
1661 _num += 1; | 1625 _num += 1; |
1662 } | 1626 } |
1663 return false; | 1627 return false; |
2316 concurrent_mark()->print_summary_info(); | 2280 concurrent_mark()->print_summary_info(); |
2317 } | 2281 } |
2318 if (SummarizeG1ZFStats) { | 2282 if (SummarizeG1ZFStats) { |
2319 ConcurrentZFThread::print_summary_info(); | 2283 ConcurrentZFThread::print_summary_info(); |
2320 } | 2284 } |
2321 if (G1SummarizePopularity) { | |
2322 print_popularity_summary_info(); | |
2323 } | |
2324 g1_policy()->print_yg_surv_rate_info(); | 2285 g1_policy()->print_yg_surv_rate_info(); |
2325 | 2286 |
2326 GCOverheadReporter::printGCOverhead(); | 2287 GCOverheadReporter::printGCOverhead(); |
2327 | 2288 |
2328 SpecializationStats::print(); | 2289 SpecializationStats::print(); |
2493 } | 2454 } |
2494 | 2455 |
2495 // </NEW PREDICTION> | 2456 // </NEW PREDICTION> |
2496 | 2457 |
2497 void | 2458 void |
2498 G1CollectedHeap::do_collection_pause_at_safepoint(HeapRegion* popular_region) { | 2459 G1CollectedHeap::do_collection_pause_at_safepoint() { |
2499 char verbose_str[128]; | 2460 char verbose_str[128]; |
2500 sprintf(verbose_str, "GC pause "); | 2461 sprintf(verbose_str, "GC pause "); |
2501 if (popular_region != NULL) | 2462 if (g1_policy()->in_young_gc_mode()) { |
2502 strcat(verbose_str, "(popular)"); | |
2503 else if (g1_policy()->in_young_gc_mode()) { | |
2504 if (g1_policy()->full_young_gcs()) | 2463 if (g1_policy()->full_young_gcs()) |
2505 strcat(verbose_str, "(young)"); | 2464 strcat(verbose_str, "(young)"); |
2506 else | 2465 else |
2507 strcat(verbose_str, "(partial)"); | 2466 strcat(verbose_str, "(partial)"); |
2508 } | 2467 } |
2509 bool reset_should_initiate_conc_mark = false; | |
2510 if (popular_region != NULL && g1_policy()->should_initiate_conc_mark()) { | |
2511 // we currently do not allow an initial mark phase to be piggy-backed | |
2512 // on a popular pause | |
2513 reset_should_initiate_conc_mark = true; | |
2514 g1_policy()->unset_should_initiate_conc_mark(); | |
2515 } | |
2516 if (g1_policy()->should_initiate_conc_mark()) | 2468 if (g1_policy()->should_initiate_conc_mark()) |
2517 strcat(verbose_str, " (initial-mark)"); | 2469 strcat(verbose_str, " (initial-mark)"); |
2518 | 2470 |
2519 GCCauseSetter x(this, (popular_region == NULL ? | 2471 GCCauseSetter x(this, GCCause::_g1_inc_collection_pause); |
2520 GCCause::_g1_inc_collection_pause : | |
2521 GCCause::_g1_pop_region_collection_pause)); | |
2522 | 2472 |
2523 // if PrintGCDetails is on, we'll print long statistics information | 2473 // if PrintGCDetails is on, we'll print long statistics information |
2524 // in the collector policy code, so let's not print this as the output | 2474 // in the collector policy code, so let's not print this as the output |
2525 // is messy if we do. | 2475 // is messy if we do. |
2526 gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps); | 2476 gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps); |
2607 concurrent_mark()->checkpointRootsInitialPre(); | 2557 concurrent_mark()->checkpointRootsInitialPre(); |
2608 } | 2558 } |
2609 save_marks(); | 2559 save_marks(); |
2610 | 2560 |
2611 // We must do this before any possible evacuation that should propagate | 2561 // We must do this before any possible evacuation that should propagate |
2612 // marks, including evacuation of popular objects in a popular pause. | 2562 // marks. |
2613 if (mark_in_progress()) { | 2563 if (mark_in_progress()) { |
2614 double start_time_sec = os::elapsedTime(); | 2564 double start_time_sec = os::elapsedTime(); |
2615 | 2565 |
2616 _cm->drainAllSATBBuffers(); | 2566 _cm->drainAllSATBBuffers(); |
2617 double finish_mark_ms = (os::elapsedTime() - start_time_sec) * 1000.0; | 2567 double finish_mark_ms = (os::elapsedTime() - start_time_sec) * 1000.0; |
2624 // progress, this will be zero. | 2574 // progress, this will be zero. |
2625 _cm->set_oops_do_bound(); | 2575 _cm->set_oops_do_bound(); |
2626 | 2576 |
2627 assert(regions_accounted_for(), "Region leakage."); | 2577 assert(regions_accounted_for(), "Region leakage."); |
2628 | 2578 |
2629 bool abandoned = false; | |
2630 | |
2631 if (mark_in_progress()) | 2579 if (mark_in_progress()) |
2632 concurrent_mark()->newCSet(); | 2580 concurrent_mark()->newCSet(); |
2633 | 2581 |
2634 // Now choose the CS. | 2582 // Now choose the CS. |
2635 if (popular_region == NULL) { | 2583 g1_policy()->choose_collection_set(); |
2636 g1_policy()->choose_collection_set(); | 2584 |
2637 } else { | |
2638 // We may be evacuating a single region (for popularity). | |
2639 g1_policy()->record_popular_pause_preamble_start(); | |
2640 popularity_pause_preamble(popular_region); | |
2641 g1_policy()->record_popular_pause_preamble_end(); | |
2642 abandoned = (g1_policy()->collection_set() == NULL); | |
2643 // Now we allow more regions to be added (we have to collect | |
2644 // all popular regions). | |
2645 if (!abandoned) { | |
2646 g1_policy()->choose_collection_set(popular_region); | |
2647 } | |
2648 } | |
2649 // We may abandon a pause if we find no region that will fit in the MMU | 2585 // We may abandon a pause if we find no region that will fit in the MMU |
2650 // pause. | 2586 // pause. |
2651 abandoned = (g1_policy()->collection_set() == NULL); | 2587 bool abandoned = (g1_policy()->collection_set() == NULL); |
2652 | 2588 |
2653 // Nothing to do if we were unable to choose a collection set. | 2589 // Nothing to do if we were unable to choose a collection set. |
2654 if (!abandoned) { | 2590 if (!abandoned) { |
2655 #if G1_REM_SET_LOGGING | 2591 #if G1_REM_SET_LOGGING |
2656 gclog_or_tty->print_cr("\nAfter pause, heap:"); | 2592 gclog_or_tty->print_cr("\nAfter pause, heap:"); |
2670 FREE_C_HEAP_ARRAY(bool, _in_cset_fast_test_base); | 2606 FREE_C_HEAP_ARRAY(bool, _in_cset_fast_test_base); |
2671 // this is more for peace of mind; we're nulling them here and | 2607 // this is more for peace of mind; we're nulling them here and |
2672 // we're expecting them to be null at the beginning of the next GC | 2608 // we're expecting them to be null at the beginning of the next GC |
2673 _in_cset_fast_test = NULL; | 2609 _in_cset_fast_test = NULL; |
2674 _in_cset_fast_test_base = NULL; | 2610 _in_cset_fast_test_base = NULL; |
2675 | |
2676 if (popular_region != NULL) { | |
2677 // We have to wait until now, because we don't want the region to | |
2678 // be rescheduled for pop-evac during RS update. | |
2679 popular_region->set_popular_pending(false); | |
2680 } | |
2681 | 2611 |
2682 release_gc_alloc_regions(false /* totally */); | 2612 release_gc_alloc_regions(false /* totally */); |
2683 | 2613 |
2684 cleanup_surviving_young_words(); | 2614 cleanup_surviving_young_words(); |
2685 | 2615 |
2722 | 2652 |
2723 double end_time_sec = os::elapsedTime(); | 2653 double end_time_sec = os::elapsedTime(); |
2724 double pause_time_ms = (end_time_sec - start_time_sec) * MILLIUNITS; | 2654 double pause_time_ms = (end_time_sec - start_time_sec) * MILLIUNITS; |
2725 g1_policy()->record_pause_time_ms(pause_time_ms); | 2655 g1_policy()->record_pause_time_ms(pause_time_ms); |
2726 GCOverheadReporter::recordSTWEnd(end_time_sec); | 2656 GCOverheadReporter::recordSTWEnd(end_time_sec); |
2727 g1_policy()->record_collection_pause_end(popular_region != NULL, | 2657 g1_policy()->record_collection_pause_end(abandoned); |
2728 abandoned); | |
2729 | 2658 |
2730 assert(regions_accounted_for(), "Region leakage."); | 2659 assert(regions_accounted_for(), "Region leakage."); |
2731 | 2660 |
2732 if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) { | 2661 if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) { |
2733 HandleMark hm; // Discard invalid handles created during verification | 2662 HandleMark hm; // Discard invalid handles created during verification |
2756 | 2685 |
2757 gc_epilogue(false); | 2686 gc_epilogue(false); |
2758 } | 2687 } |
2759 | 2688 |
2760 assert(verify_region_lists(), "Bad region lists."); | 2689 assert(verify_region_lists(), "Bad region lists."); |
2761 | |
2762 if (reset_should_initiate_conc_mark) | |
2763 g1_policy()->set_should_initiate_conc_mark(); | |
2764 | 2690 |
2765 if (ExitAfterGCNum > 0 && total_collections() == ExitAfterGCNum) { | 2691 if (ExitAfterGCNum > 0 && total_collections() == ExitAfterGCNum) { |
2766 gclog_or_tty->print_cr("Stopping after GC #%d", ExitAfterGCNum); | 2692 gclog_or_tty->print_cr("Stopping after GC #%d", ExitAfterGCNum); |
2767 print_tracing_info(); | 2693 print_tracing_info(); |
2768 vm_exit(-1); | 2694 vm_exit(-1); |
4705 size_t& pre_used, | 4631 size_t& pre_used, |
4706 size_t& cleared_h_regions, | 4632 size_t& cleared_h_regions, |
4707 size_t& freed_regions, | 4633 size_t& freed_regions, |
4708 UncleanRegionList* list, | 4634 UncleanRegionList* list, |
4709 bool par) { | 4635 bool par) { |
4710 assert(!hr->popular(), "should not free popular regions"); | |
4711 pre_used += hr->used(); | 4636 pre_used += hr->used(); |
4712 if (hr->isHumongous()) { | 4637 if (hr->isHumongous()) { |
4713 assert(hr->startsHumongous(), | 4638 assert(hr->startsHumongous(), |
4714 "Only the start of a humongous region should be freed."); | 4639 "Only the start of a humongous region should be freed."); |
4715 int ind = _hrs->find(hr); | 4640 int ind = _hrs->find(hr); |
4789 g1_policy()->record_clear_ct_time( elapsed * 1000.0); | 4714 g1_policy()->record_clear_ct_time( elapsed * 1000.0); |
4790 } | 4715 } |
4791 | 4716 |
4792 | 4717 |
4793 void G1CollectedHeap::do_collection_pause_if_appropriate(size_t word_size) { | 4718 void G1CollectedHeap::do_collection_pause_if_appropriate(size_t word_size) { |
4794 // First do any popular regions. | |
4795 HeapRegion* hr; | |
4796 while ((hr = popular_region_to_evac()) != NULL) { | |
4797 evac_popular_region(hr); | |
4798 } | |
4799 // Now do heuristic pauses. | |
4800 if (g1_policy()->should_do_collection_pause(word_size)) { | 4719 if (g1_policy()->should_do_collection_pause(word_size)) { |
4801 do_collection_pause(); | 4720 do_collection_pause(); |
4802 } | 4721 } |
4803 } | 4722 } |
4804 | 4723 |
5190 class RegionCounter: public HeapRegionClosure { | 5109 class RegionCounter: public HeapRegionClosure { |
5191 size_t _n; | 5110 size_t _n; |
5192 public: | 5111 public: |
5193 RegionCounter() : _n(0) {} | 5112 RegionCounter() : _n(0) {} |
5194 bool doHeapRegion(HeapRegion* r) { | 5113 bool doHeapRegion(HeapRegion* r) { |
5195 if (r->is_empty() && !r->popular()) { | 5114 if (r->is_empty()) { |
5196 assert(!r->isHumongous(), "H regions should not be empty."); | 5115 assert(!r->isHumongous(), "H regions should not be empty."); |
5197 _n++; | 5116 _n++; |
5198 } | 5117 } |
5199 return false; | 5118 return false; |
5200 } | 5119 } |
5334 pointer_delta(r->end(), r->top())); | 5253 pointer_delta(r->end(), r->top())); |
5335 } | 5254 } |
5336 r->set_zero_fill_allocated(); | 5255 r->set_zero_fill_allocated(); |
5337 } else { | 5256 } else { |
5338 assert(r->is_empty(), "tautology"); | 5257 assert(r->is_empty(), "tautology"); |
5339 if (r->popular()) { | 5258 _n++; |
5340 if (r->zero_fill_state() != HeapRegion::Allocated) { | 5259 switch (r->zero_fill_state()) { |
5341 r->ensure_zero_filled_locked(); | |
5342 r->set_zero_fill_allocated(); | |
5343 } | |
5344 } else { | |
5345 _n++; | |
5346 switch (r->zero_fill_state()) { | |
5347 case HeapRegion::NotZeroFilled: | 5260 case HeapRegion::NotZeroFilled: |
5348 case HeapRegion::ZeroFilling: | 5261 case HeapRegion::ZeroFilling: |
5349 _g1->put_region_on_unclean_list_locked(r); | 5262 _g1->put_region_on_unclean_list_locked(r); |
5350 break; | 5263 break; |
5351 case HeapRegion::Allocated: | 5264 case HeapRegion::Allocated: |
5352 r->set_zero_fill_complete(); | 5265 r->set_zero_fill_complete(); |
5353 // no break; go on to put on free list. | 5266 // no break; go on to put on free list. |
5354 case HeapRegion::ZeroFilled: | 5267 case HeapRegion::ZeroFilled: |
5355 _g1->put_free_region_on_list_locked(r); | 5268 _g1->put_free_region_on_list_locked(r); |
5356 break; | 5269 break; |
5357 } | |
5358 } | 5270 } |
5359 } | 5271 } |
5360 return false; | 5272 return false; |
5361 } | 5273 } |
5362 | 5274 |
5398 void G1CollectedHeap::set_used_regions_to_need_zero_fill() { | 5310 void G1CollectedHeap::set_used_regions_to_need_zero_fill() { |
5399 MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); | 5311 MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); |
5400 // This needs to go at the end of the full GC. | 5312 // This needs to go at the end of the full GC. |
5401 UsedRegionsNeedZeroFillSetter rs; | 5313 UsedRegionsNeedZeroFillSetter rs; |
5402 heap_region_iterate(&rs); | 5314 heap_region_iterate(&rs); |
5403 } | |
5404 | |
5405 class CountObjClosure: public ObjectClosure { | |
5406 size_t _n; | |
5407 public: | |
5408 CountObjClosure() : _n(0) {} | |
5409 void do_object(oop obj) { _n++; } | |
5410 size_t n() { return _n; } | |
5411 }; | |
5412 | |
5413 size_t G1CollectedHeap::pop_object_used_objs() { | |
5414 size_t sum_objs = 0; | |
5415 for (int i = 0; i < G1NumPopularRegions; i++) { | |
5416 CountObjClosure cl; | |
5417 _hrs->at(i)->object_iterate(&cl); | |
5418 sum_objs += cl.n(); | |
5419 } | |
5420 return sum_objs; | |
5421 } | |
5422 | |
5423 size_t G1CollectedHeap::pop_object_used_bytes() { | |
5424 size_t sum_bytes = 0; | |
5425 for (int i = 0; i < G1NumPopularRegions; i++) { | |
5426 sum_bytes += _hrs->at(i)->used(); | |
5427 } | |
5428 return sum_bytes; | |
5429 } | |
5430 | |
5431 | |
5432 static int nq = 0; | |
5433 | |
5434 HeapWord* G1CollectedHeap::allocate_popular_object(size_t word_size) { | |
5435 while (_cur_pop_hr_index < G1NumPopularRegions) { | |
5436 HeapRegion* cur_pop_region = _hrs->at(_cur_pop_hr_index); | |
5437 HeapWord* res = cur_pop_region->allocate(word_size); | |
5438 if (res != NULL) { | |
5439 // We account for popular objs directly in the used summary: | |
5440 _summary_bytes_used += (word_size * HeapWordSize); | |
5441 return res; | |
5442 } | |
5443 // Otherwise, try the next region (first making sure that we remember | |
5444 // the last "top" value as the "next_top_at_mark_start", so that | |
5445 // objects made popular during markings aren't automatically considered | |
5446 // live). | |
5447 cur_pop_region->note_end_of_copying(); | |
5448 // Otherwise, try the next region. | |
5449 _cur_pop_hr_index++; | |
5450 } | |
5451 // XXX: For now !!! | |
5452 vm_exit_out_of_memory(word_size, | |
5453 "Not enough pop obj space (To Be Fixed)"); | |
5454 return NULL; | |
5455 } | |
5456 | |
5457 class HeapRegionList: public CHeapObj { | |
5458 public: | |
5459 HeapRegion* hr; | |
5460 HeapRegionList* next; | |
5461 }; | |
5462 | |
5463 void G1CollectedHeap::schedule_popular_region_evac(HeapRegion* r) { | |
5464 // This might happen during parallel GC, so protect by this lock. | |
5465 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); | |
5466 // We don't schedule regions whose evacuations are already pending, or | |
5467 // are already being evacuated. | |
5468 if (!r->popular_pending() && !r->in_collection_set()) { | |
5469 r->set_popular_pending(true); | |
5470 if (G1TracePopularity) { | |
5471 gclog_or_tty->print_cr("Scheduling region "PTR_FORMAT" " | |
5472 "["PTR_FORMAT", "PTR_FORMAT") for pop-object evacuation.", | |
5473 r, r->bottom(), r->end()); | |
5474 } | |
5475 HeapRegionList* hrl = new HeapRegionList; | |
5476 hrl->hr = r; | |
5477 hrl->next = _popular_regions_to_be_evacuated; | |
5478 _popular_regions_to_be_evacuated = hrl; | |
5479 } | |
5480 } | |
5481 | |
5482 HeapRegion* G1CollectedHeap::popular_region_to_evac() { | |
5483 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); | |
5484 HeapRegion* res = NULL; | |
5485 while (_popular_regions_to_be_evacuated != NULL && res == NULL) { | |
5486 HeapRegionList* hrl = _popular_regions_to_be_evacuated; | |
5487 _popular_regions_to_be_evacuated = hrl->next; | |
5488 res = hrl->hr; | |
5489 // The G1RSPopLimit may have increased, so recheck here... | |
5490 if (res->rem_set()->occupied() < (size_t) G1RSPopLimit) { | |
5491 // Hah: don't need to schedule. | |
5492 if (G1TracePopularity) { | |
5493 gclog_or_tty->print_cr("Unscheduling region "PTR_FORMAT" " | |
5494 "["PTR_FORMAT", "PTR_FORMAT") " | |
5495 "for pop-object evacuation (size %d < limit %d)", | |
5496 res, res->bottom(), res->end(), | |
5497 res->rem_set()->occupied(), G1RSPopLimit); | |
5498 } | |
5499 res->set_popular_pending(false); | |
5500 res = NULL; | |
5501 } | |
5502 // We do not reset res->popular() here; if we did so, it would allow | |
5503 // the region to be "rescheduled" for popularity evacuation. Instead, | |
5504 // this is done in the collection pause, with the world stopped. | |
5505 // So the invariant is that the regions in the list have the popularity | |
5506 // boolean set, but having the boolean set does not imply membership | |
5507 // on the list (though there can at most one such pop-pending region | |
5508 // not on the list at any time). | |
5509 delete hrl; | |
5510 } | |
5511 return res; | |
5512 } | |
5513 | |
5514 void G1CollectedHeap::evac_popular_region(HeapRegion* hr) { | |
5515 while (true) { | |
5516 // Don't want to do a GC pause while cleanup is being completed! | |
5517 wait_for_cleanup_complete(); | |
5518 | |
5519 // Read the GC count while holding the Heap_lock | |
5520 int gc_count_before = SharedHeap::heap()->total_collections(); | |
5521 g1_policy()->record_stop_world_start(); | |
5522 | |
5523 { | |
5524 MutexUnlocker mu(Heap_lock); // give up heap lock, execute gets it back | |
5525 VM_G1PopRegionCollectionPause op(gc_count_before, hr); | |
5526 VMThread::execute(&op); | |
5527 | |
5528 // If the prolog succeeded, we didn't do a GC for this. | |
5529 if (op.prologue_succeeded()) break; | |
5530 } | |
5531 // Otherwise we didn't. We should recheck the size, though, since | |
5532 // the limit may have increased... | |
5533 if (hr->rem_set()->occupied() < (size_t) G1RSPopLimit) { | |
5534 hr->set_popular_pending(false); | |
5535 break; | |
5536 } | |
5537 } | |
5538 } | |
5539 | |
5540 void G1CollectedHeap::atomic_inc_obj_rc(oop obj) { | |
5541 Atomic::inc(obj_rc_addr(obj)); | |
5542 } | |
5543 | |
5544 class CountRCClosure: public OopsInHeapRegionClosure { | |
5545 G1CollectedHeap* _g1h; | |
5546 bool _parallel; | |
5547 public: | |
5548 CountRCClosure(G1CollectedHeap* g1h) : | |
5549 _g1h(g1h), _parallel(ParallelGCThreads > 0) | |
5550 {} | |
5551 void do_oop(narrowOop* p) { | |
5552 guarantee(false, "NYI"); | |
5553 } | |
5554 void do_oop(oop* p) { | |
5555 oop obj = *p; | |
5556 assert(obj != NULL, "Precondition."); | |
5557 if (_parallel) { | |
5558 // We go sticky at the limit to avoid excess contention. | |
5559 // If we want to track the actual RC's further, we'll need to keep a | |
5560 // per-thread hash table or something for the popular objects. | |
5561 if (_g1h->obj_rc(obj) < G1ObjPopLimit) { | |
5562 _g1h->atomic_inc_obj_rc(obj); | |
5563 } | |
5564 } else { | |
5565 _g1h->inc_obj_rc(obj); | |
5566 } | |
5567 } | |
5568 }; | |
5569 | |
5570 class EvacPopObjClosure: public ObjectClosure { | |
5571 G1CollectedHeap* _g1h; | |
5572 size_t _pop_objs; | |
5573 size_t _max_rc; | |
5574 public: | |
5575 EvacPopObjClosure(G1CollectedHeap* g1h) : | |
5576 _g1h(g1h), _pop_objs(0), _max_rc(0) {} | |
5577 | |
5578 void do_object(oop obj) { | |
5579 size_t rc = _g1h->obj_rc(obj); | |
5580 _max_rc = MAX2(rc, _max_rc); | |
5581 if (rc >= (size_t) G1ObjPopLimit) { | |
5582 _g1h->_pop_obj_rc_at_copy.add((double)rc); | |
5583 size_t word_sz = obj->size(); | |
5584 HeapWord* new_pop_loc = _g1h->allocate_popular_object(word_sz); | |
5585 oop new_pop_obj = (oop)new_pop_loc; | |
5586 Copy::aligned_disjoint_words((HeapWord*)obj, new_pop_loc, word_sz); | |
5587 obj->forward_to(new_pop_obj); | |
5588 G1ScanAndBalanceClosure scan_and_balance(_g1h); | |
5589 new_pop_obj->oop_iterate_backwards(&scan_and_balance); | |
5590 // preserve "next" mark bit if marking is in progress. | |
5591 if (_g1h->mark_in_progress() && !_g1h->is_obj_ill(obj)) { | |
5592 _g1h->concurrent_mark()->markAndGrayObjectIfNecessary(new_pop_obj); | |
5593 } | |
5594 | |
5595 if (G1TracePopularity) { | |
5596 gclog_or_tty->print_cr("Found obj " PTR_FORMAT " of word size " SIZE_FORMAT | |
5597 " pop (%d), move to " PTR_FORMAT, | |
5598 (void*) obj, word_sz, | |
5599 _g1h->obj_rc(obj), (void*) new_pop_obj); | |
5600 } | |
5601 _pop_objs++; | |
5602 } | |
5603 } | |
5604 size_t pop_objs() { return _pop_objs; } | |
5605 size_t max_rc() { return _max_rc; } | |
5606 }; | |
5607 | |
5608 class G1ParCountRCTask : public AbstractGangTask { | |
5609 G1CollectedHeap* _g1h; | |
5610 BitMap _bm; | |
5611 | |
5612 size_t getNCards() { | |
5613 return (_g1h->capacity() + G1BlockOffsetSharedArray::N_bytes - 1) | |
5614 / G1BlockOffsetSharedArray::N_bytes; | |
5615 } | |
5616 CountRCClosure _count_rc_closure; | |
5617 public: | |
5618 G1ParCountRCTask(G1CollectedHeap* g1h) : | |
5619 AbstractGangTask("G1 Par RC Count task"), | |
5620 _g1h(g1h), _bm(getNCards()), _count_rc_closure(g1h) | |
5621 {} | |
5622 | |
5623 void work(int i) { | |
5624 ResourceMark rm; | |
5625 HandleMark hm; | |
5626 _g1h->g1_rem_set()->oops_into_collection_set_do(&_count_rc_closure, i); | |
5627 } | |
5628 }; | |
5629 | |
5630 void G1CollectedHeap::popularity_pause_preamble(HeapRegion* popular_region) { | |
5631 // We're evacuating a single region (for popularity). | |
5632 if (G1TracePopularity) { | |
5633 gclog_or_tty->print_cr("Doing pop region pause for ["PTR_FORMAT", "PTR_FORMAT")", | |
5634 popular_region->bottom(), popular_region->end()); | |
5635 } | |
5636 g1_policy()->set_single_region_collection_set(popular_region); | |
5637 size_t max_rc; | |
5638 if (!compute_reference_counts_and_evac_popular(popular_region, | |
5639 &max_rc)) { | |
5640 // We didn't evacuate any popular objects. | |
5641 // We increase the RS popularity limit, to prevent this from | |
5642 // happening in the future. | |
5643 if (G1RSPopLimit < (1 << 30)) { | |
5644 G1RSPopLimit *= 2; | |
5645 } | |
5646 // For now, interesting enough for a message: | |
5647 #if 1 | |
5648 gclog_or_tty->print_cr("In pop region pause for ["PTR_FORMAT", "PTR_FORMAT"), " | |
5649 "failed to find a pop object (max = %d).", | |
5650 popular_region->bottom(), popular_region->end(), | |
5651 max_rc); | |
5652 gclog_or_tty->print_cr("Increased G1RSPopLimit to %d.", G1RSPopLimit); | |
5653 #endif // 0 | |
5654 // Also, we reset the collection set to NULL, to make the rest of | |
5655 // the collection do nothing. | |
5656 assert(popular_region->next_in_collection_set() == NULL, | |
5657 "should be single-region."); | |
5658 popular_region->set_in_collection_set(false); | |
5659 popular_region->set_popular_pending(false); | |
5660 g1_policy()->clear_collection_set(); | |
5661 } | |
5662 } | |
5663 | |
5664 bool G1CollectedHeap:: | |
5665 compute_reference_counts_and_evac_popular(HeapRegion* popular_region, | |
5666 size_t* max_rc) { | |
5667 HeapWord* rc_region_bot; | |
5668 HeapWord* rc_region_end; | |
5669 | |
5670 // Set up the reference count region. | |
5671 HeapRegion* rc_region = newAllocRegion(HeapRegion::GrainWords); | |
5672 if (rc_region != NULL) { | |
5673 rc_region_bot = rc_region->bottom(); | |
5674 rc_region_end = rc_region->end(); | |
5675 } else { | |
5676 rc_region_bot = NEW_C_HEAP_ARRAY(HeapWord, HeapRegion::GrainWords); | |
5677 if (rc_region_bot == NULL) { | |
5678 vm_exit_out_of_memory(HeapRegion::GrainWords, | |
5679 "No space for RC region."); | |
5680 } | |
5681 rc_region_end = rc_region_bot + HeapRegion::GrainWords; | |
5682 } | |
5683 | |
5684 if (G1TracePopularity) | |
5685 gclog_or_tty->print_cr("RC region is ["PTR_FORMAT", "PTR_FORMAT")", | |
5686 rc_region_bot, rc_region_end); | |
5687 if (rc_region_bot > popular_region->bottom()) { | |
5688 _rc_region_above = true; | |
5689 _rc_region_diff = | |
5690 pointer_delta(rc_region_bot, popular_region->bottom(), 1); | |
5691 } else { | |
5692 assert(rc_region_bot < popular_region->bottom(), "Can't be equal."); | |
5693 _rc_region_above = false; | |
5694 _rc_region_diff = | |
5695 pointer_delta(popular_region->bottom(), rc_region_bot, 1); | |
5696 } | |
5697 g1_policy()->record_pop_compute_rc_start(); | |
5698 // Count external references. | |
5699 g1_rem_set()->prepare_for_oops_into_collection_set_do(); | |
5700 if (ParallelGCThreads > 0) { | |
5701 | |
5702 set_par_threads(workers()->total_workers()); | |
5703 G1ParCountRCTask par_count_rc_task(this); | |
5704 workers()->run_task(&par_count_rc_task); | |
5705 set_par_threads(0); | |
5706 | |
5707 } else { | |
5708 CountRCClosure count_rc_closure(this); | |
5709 g1_rem_set()->oops_into_collection_set_do(&count_rc_closure, 0); | |
5710 } | |
5711 g1_rem_set()->cleanup_after_oops_into_collection_set_do(); | |
5712 g1_policy()->record_pop_compute_rc_end(); | |
5713 | |
5714 // Now evacuate popular objects. | |
5715 g1_policy()->record_pop_evac_start(); | |
5716 EvacPopObjClosure evac_pop_obj_cl(this); | |
5717 popular_region->object_iterate(&evac_pop_obj_cl); | |
5718 *max_rc = evac_pop_obj_cl.max_rc(); | |
5719 | |
5720 // Make sure the last "top" value of the current popular region is copied | |
5721 // as the "next_top_at_mark_start", so that objects made popular during | |
5722 // markings aren't automatically considered live. | |
5723 HeapRegion* cur_pop_region = _hrs->at(_cur_pop_hr_index); | |
5724 cur_pop_region->note_end_of_copying(); | |
5725 | |
5726 if (rc_region != NULL) { | |
5727 free_region(rc_region); | |
5728 } else { | |
5729 FREE_C_HEAP_ARRAY(HeapWord, rc_region_bot); | |
5730 } | |
5731 g1_policy()->record_pop_evac_end(); | |
5732 | |
5733 return evac_pop_obj_cl.pop_objs() > 0; | |
5734 } | |
5735 | |
5736 class CountPopObjInfoClosure: public HeapRegionClosure { | |
5737 size_t _objs; | |
5738 size_t _bytes; | |
5739 | |
5740 class CountObjClosure: public ObjectClosure { | |
5741 int _n; | |
5742 public: | |
5743 CountObjClosure() : _n(0) {} | |
5744 void do_object(oop obj) { _n++; } | |
5745 size_t n() { return _n; } | |
5746 }; | |
5747 | |
5748 public: | |
5749 CountPopObjInfoClosure() : _objs(0), _bytes(0) {} | |
5750 bool doHeapRegion(HeapRegion* r) { | |
5751 _bytes += r->used(); | |
5752 CountObjClosure blk; | |
5753 r->object_iterate(&blk); | |
5754 _objs += blk.n(); | |
5755 return false; | |
5756 } | |
5757 size_t objs() { return _objs; } | |
5758 size_t bytes() { return _bytes; } | |
5759 }; | |
5760 | |
5761 | |
5762 void G1CollectedHeap::print_popularity_summary_info() const { | |
5763 CountPopObjInfoClosure blk; | |
5764 for (int i = 0; i <= _cur_pop_hr_index; i++) { | |
5765 blk.doHeapRegion(_hrs->at(i)); | |
5766 } | |
5767 gclog_or_tty->print_cr("\nPopular objects: %d objs, %d bytes.", | |
5768 blk.objs(), blk.bytes()); | |
5769 gclog_or_tty->print_cr(" RC at copy = [avg = %5.2f, max = %5.2f, sd = %5.2f].", | |
5770 _pop_obj_rc_at_copy.avg(), | |
5771 _pop_obj_rc_at_copy.maximum(), | |
5772 _pop_obj_rc_at_copy.sd()); | |
5773 } | 5315 } |
5774 | 5316 |
5775 void G1CollectedHeap::set_refine_cte_cl_concurrency(bool concurrent) { | 5317 void G1CollectedHeap::set_refine_cte_cl_concurrency(bool concurrent) { |
5776 _refine_cte_cl->set_concurrent(concurrent); | 5318 _refine_cte_cl->set_concurrent(concurrent); |
5777 } | 5319 } |
5843 // TODO: regions accounting for young/survivor/tenured | 5385 // TODO: regions accounting for young/survivor/tenured |
5844 return true; | 5386 return true; |
5845 } | 5387 } |
5846 | 5388 |
5847 bool G1CollectedHeap::print_region_accounting_info() { | 5389 bool G1CollectedHeap::print_region_accounting_info() { |
5848 gclog_or_tty->print_cr("P regions: %d.", G1NumPopularRegions); | |
5849 gclog_or_tty->print_cr("Free regions: %d (count: %d count list %d) (clean: %d unclean: %d).", | 5390 gclog_or_tty->print_cr("Free regions: %d (count: %d count list %d) (clean: %d unclean: %d).", |
5850 free_regions(), | 5391 free_regions(), |
5851 count_free_regions(), count_free_regions_list(), | 5392 count_free_regions(), count_free_regions_list(), |
5852 _free_region_list_size, _unclean_region_list.sz()); | 5393 _free_region_list_size, _unclean_region_list.sz()); |
5853 gclog_or_tty->print_cr("cur_alloc: %d.", | 5394 gclog_or_tty->print_cr("cur_alloc: %d.", |