Mercurial > hg > truffle
comparison src/share/vm/gc_implementation/g1/concurrentMark.cpp @ 6862:8a5ea0a9ccc4
7127708: G1: change task num types from int to uint in concurrent mark
Summary: Change the type of various task num fields, parameters etc to unsigned and rename them to be more consistent with the other collectors. Code changes were also reviewed by Vitaly Davidovich.
Reviewed-by: johnc
Contributed-by: Kaushik Srenevasan <kaushik@twitter.com>
author | johnc |
---|---|
date | Sat, 06 Oct 2012 01:17:44 -0700 |
parents | 988bf00cc564 |
children | 4202510ee0fe |
comparison
equal
deleted
inserted
replaced
6830:81e878c53615 | 6862:8a5ea0a9ccc4 |
---|---|
424 _nextMarkBitMap(&_markBitMap2), | 424 _nextMarkBitMap(&_markBitMap2), |
425 | 425 |
426 _markStack(this), | 426 _markStack(this), |
427 // _finger set in set_non_marking_state | 427 // _finger set in set_non_marking_state |
428 | 428 |
429 _max_task_num(MAX2((uint)ParallelGCThreads, 1U)), | 429 _max_worker_id(MAX2((uint)ParallelGCThreads, 1U)), |
430 // _active_tasks set in set_non_marking_state | 430 // _active_tasks set in set_non_marking_state |
431 // _tasks set inside the constructor | 431 // _tasks set inside the constructor |
432 _task_queues(new CMTaskQueueSet((int) _max_task_num)), | 432 _task_queues(new CMTaskQueueSet((int) _max_worker_id)), |
433 _terminator(ParallelTaskTerminator((int) _max_task_num, _task_queues)), | 433 _terminator(ParallelTaskTerminator((int) _max_worker_id, _task_queues)), |
434 | 434 |
435 _has_overflown(false), | 435 _has_overflown(false), |
436 _concurrent(false), | 436 _concurrent(false), |
437 _has_aborted(false), | 437 _has_aborted(false), |
438 _restart_for_overflow(false), | 438 _restart_for_overflow(false), |
479 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set(); | 479 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set(); |
480 satb_qs.set_buffer_size(G1SATBBufferSize); | 480 satb_qs.set_buffer_size(G1SATBBufferSize); |
481 | 481 |
482 _root_regions.init(_g1h, this); | 482 _root_regions.init(_g1h, this); |
483 | 483 |
484 _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_task_num, mtGC); | 484 _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_worker_id, mtGC); |
485 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_task_num, mtGC); | 485 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_worker_id, mtGC); |
486 | 486 |
487 _count_card_bitmaps = NEW_C_HEAP_ARRAY(BitMap, _max_task_num, mtGC); | 487 _count_card_bitmaps = NEW_C_HEAP_ARRAY(BitMap, _max_worker_id, mtGC); |
488 _count_marked_bytes = NEW_C_HEAP_ARRAY(size_t*, _max_task_num, mtGC); | 488 _count_marked_bytes = NEW_C_HEAP_ARRAY(size_t*, _max_worker_id, mtGC); |
489 | 489 |
490 BitMap::idx_t card_bm_size = _card_bm.size(); | 490 BitMap::idx_t card_bm_size = _card_bm.size(); |
491 | 491 |
492 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail | 492 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail |
493 _active_tasks = _max_task_num; | 493 _active_tasks = _max_worker_id; |
494 for (int i = 0; i < (int) _max_task_num; ++i) { | 494 for (uint i = 0; i < _max_worker_id; ++i) { |
495 CMTaskQueue* task_queue = new CMTaskQueue(); | 495 CMTaskQueue* task_queue = new CMTaskQueue(); |
496 task_queue->initialize(); | 496 task_queue->initialize(); |
497 _task_queues->register_queue(i, task_queue); | 497 _task_queues->register_queue(i, task_queue); |
498 | 498 |
499 _count_card_bitmaps[i] = BitMap(card_bm_size, false); | 499 _count_card_bitmaps[i] = BitMap(card_bm_size, false); |
636 } | 636 } |
637 | 637 |
638 // We do reset all of them, since different phases will use | 638 // We do reset all of them, since different phases will use |
639 // different number of active threads. So, it's easiest to have all | 639 // different number of active threads. So, it's easiest to have all |
640 // of them ready. | 640 // of them ready. |
641 for (int i = 0; i < (int) _max_task_num; ++i) { | 641 for (uint i = 0; i < _max_worker_id; ++i) { |
642 _tasks[i]->reset(_nextMarkBitMap); | 642 _tasks[i]->reset(_nextMarkBitMap); |
643 } | 643 } |
644 | 644 |
645 // we need this to make sure that the flag is on during the evac | 645 // we need this to make sure that the flag is on during the evac |
646 // pause with initial mark piggy-backed | 646 // pause with initial mark piggy-backed |
647 set_concurrent_marking_in_progress(); | 647 set_concurrent_marking_in_progress(); |
648 } | 648 } |
649 | 649 |
650 void ConcurrentMark::set_phase(uint active_tasks, bool concurrent) { | 650 void ConcurrentMark::set_phase(uint active_tasks, bool concurrent) { |
651 assert(active_tasks <= _max_task_num, "we should not have more"); | 651 assert(active_tasks <= _max_worker_id, "we should not have more"); |
652 | 652 |
653 _active_tasks = active_tasks; | 653 _active_tasks = active_tasks; |
654 // Need to update the three data structures below according to the | 654 // Need to update the three data structures below according to the |
655 // number of active threads for this phase. | 655 // number of active threads for this phase. |
656 _terminator = ParallelTaskTerminator((int) active_tasks, _task_queues); | 656 _terminator = ParallelTaskTerminator((int) active_tasks, _task_queues); |
657 _first_overflow_barrier_sync.set_n_workers((int) active_tasks); | 657 _first_overflow_barrier_sync.set_n_workers((int) active_tasks); |
658 _second_overflow_barrier_sync.set_n_workers((int) active_tasks); | 658 _second_overflow_barrier_sync.set_n_workers((int) active_tasks); |
659 | 659 |
660 _concurrent = concurrent; | 660 _concurrent = concurrent; |
661 // We propagate this to all tasks, not just the active ones. | 661 // We propagate this to all tasks, not just the active ones. |
662 for (int i = 0; i < (int) _max_task_num; ++i) | 662 for (uint i = 0; i < _max_worker_id; ++i) |
663 _tasks[i]->set_concurrent(concurrent); | 663 _tasks[i]->set_concurrent(concurrent); |
664 | 664 |
665 if (concurrent) { | 665 if (concurrent) { |
666 set_concurrent_marking_in_progress(); | 666 set_concurrent_marking_in_progress(); |
667 } else { | 667 } else { |
816 * could occur. This is actually safe, since the entering the sync | 816 * could occur. This is actually safe, since the entering the sync |
817 * barrier is one of the last things do_marking_step() does, and it | 817 * barrier is one of the last things do_marking_step() does, and it |
818 * doesn't manipulate any data structures afterwards. | 818 * doesn't manipulate any data structures afterwards. |
819 */ | 819 */ |
820 | 820 |
821 void ConcurrentMark::enter_first_sync_barrier(int task_num) { | 821 void ConcurrentMark::enter_first_sync_barrier(uint worker_id) { |
822 if (verbose_low()) { | 822 if (verbose_low()) { |
823 gclog_or_tty->print_cr("[%d] entering first barrier", task_num); | 823 gclog_or_tty->print_cr("[%u] entering first barrier", worker_id); |
824 } | 824 } |
825 | 825 |
826 if (concurrent()) { | 826 if (concurrent()) { |
827 ConcurrentGCThread::stsLeave(); | 827 ConcurrentGCThread::stsLeave(); |
828 } | 828 } |
832 } | 832 } |
833 // at this point everyone should have synced up and not be doing any | 833 // at this point everyone should have synced up and not be doing any |
834 // more work | 834 // more work |
835 | 835 |
836 if (verbose_low()) { | 836 if (verbose_low()) { |
837 gclog_or_tty->print_cr("[%d] leaving first barrier", task_num); | 837 gclog_or_tty->print_cr("[%u] leaving first barrier", worker_id); |
838 } | 838 } |
839 | 839 |
840 // let task 0 do this | 840 // let the task associated with with worker 0 do this |
841 if (task_num == 0) { | 841 if (worker_id == 0) { |
842 // task 0 is responsible for clearing the global data structures | 842 // task 0 is responsible for clearing the global data structures |
843 // We should be here because of an overflow. During STW we should | 843 // We should be here because of an overflow. During STW we should |
844 // not clear the overflow flag since we rely on it being true when | 844 // not clear the overflow flag since we rely on it being true when |
845 // we exit this method to abort the pause and restart concurent | 845 // we exit this method to abort the pause and restart concurent |
846 // marking. | 846 // marking. |
856 | 856 |
857 // after this, each task should reset its own data structures then | 857 // after this, each task should reset its own data structures then |
858 // then go into the second barrier | 858 // then go into the second barrier |
859 } | 859 } |
860 | 860 |
861 void ConcurrentMark::enter_second_sync_barrier(int task_num) { | 861 void ConcurrentMark::enter_second_sync_barrier(uint worker_id) { |
862 if (verbose_low()) { | 862 if (verbose_low()) { |
863 gclog_or_tty->print_cr("[%d] entering second barrier", task_num); | 863 gclog_or_tty->print_cr("[%u] entering second barrier", worker_id); |
864 } | 864 } |
865 | 865 |
866 if (concurrent()) { | 866 if (concurrent()) { |
867 ConcurrentGCThread::stsLeave(); | 867 ConcurrentGCThread::stsLeave(); |
868 } | 868 } |
871 ConcurrentGCThread::stsJoin(); | 871 ConcurrentGCThread::stsJoin(); |
872 } | 872 } |
873 // at this point everything should be re-initialised and ready to go | 873 // at this point everything should be re-initialised and ready to go |
874 | 874 |
875 if (verbose_low()) { | 875 if (verbose_low()) { |
876 gclog_or_tty->print_cr("[%d] leaving second barrier", task_num); | 876 gclog_or_tty->print_cr("[%u] leaving second barrier", worker_id); |
877 } | 877 } |
878 } | 878 } |
879 | 879 |
880 #ifndef PRODUCT | 880 #ifndef PRODUCT |
881 void ForceOverflowSettings::init() { | 881 void ForceOverflowSettings::init() { |
2111 | 2111 |
2112 template <class T> void do_oop_work(T* p) { | 2112 template <class T> void do_oop_work(T* p) { |
2113 if (!_cm->has_overflown()) { | 2113 if (!_cm->has_overflown()) { |
2114 oop obj = oopDesc::load_decode_heap_oop(p); | 2114 oop obj = oopDesc::load_decode_heap_oop(p); |
2115 if (_cm->verbose_high()) { | 2115 if (_cm->verbose_high()) { |
2116 gclog_or_tty->print_cr("\t[%d] we're looking at location " | 2116 gclog_or_tty->print_cr("\t[%u] we're looking at location " |
2117 "*"PTR_FORMAT" = "PTR_FORMAT, | 2117 "*"PTR_FORMAT" = "PTR_FORMAT, |
2118 _task->task_id(), p, (void*) obj); | 2118 _task->worker_id(), p, (void*) obj); |
2119 } | 2119 } |
2120 | 2120 |
2121 _task->deal_with_reference(obj); | 2121 _task->deal_with_reference(obj); |
2122 _ref_counter--; | 2122 _ref_counter--; |
2123 | 2123 |
2142 } while (_task->has_aborted() && !_cm->has_overflown()); | 2142 } while (_task->has_aborted() && !_cm->has_overflown()); |
2143 _ref_counter = _ref_counter_limit; | 2143 _ref_counter = _ref_counter_limit; |
2144 } | 2144 } |
2145 } else { | 2145 } else { |
2146 if (_cm->verbose_high()) { | 2146 if (_cm->verbose_high()) { |
2147 gclog_or_tty->print_cr("\t[%d] CM Overflow", _task->task_id()); | 2147 gclog_or_tty->print_cr("\t[%u] CM Overflow", _task->worker_id()); |
2148 } | 2148 } |
2149 } | 2149 } |
2150 } | 2150 } |
2151 }; | 2151 }; |
2152 | 2152 |
2158 _cm(cm), _task(task) { } | 2158 _cm(cm), _task(task) { } |
2159 | 2159 |
2160 void do_void() { | 2160 void do_void() { |
2161 do { | 2161 do { |
2162 if (_cm->verbose_high()) { | 2162 if (_cm->verbose_high()) { |
2163 gclog_or_tty->print_cr("\t[%d] Drain: Calling do marking_step", | 2163 gclog_or_tty->print_cr("\t[%u] Drain: Calling do marking_step", |
2164 _task->task_id()); | 2164 _task->worker_id()); |
2165 } | 2165 } |
2166 | 2166 |
2167 // We call CMTask::do_marking_step() to completely drain the local and | 2167 // We call CMTask::do_marking_step() to completely drain the local and |
2168 // global marking stacks. The routine is called in a loop, which we'll | 2168 // global marking stacks. The routine is called in a loop, which we'll |
2169 // exit if there's nothing more to do (i.e. we'completely drained the | 2169 // exit if there's nothing more to do (i.e. we'completely drained the |
2298 g1_drain_mark_stack(this, &_markStack, &g1_keep_alive); | 2298 g1_drain_mark_stack(this, &_markStack, &g1_keep_alive); |
2299 | 2299 |
2300 // We use the work gang from the G1CollectedHeap and we utilize all | 2300 // We use the work gang from the G1CollectedHeap and we utilize all |
2301 // the worker threads. | 2301 // the worker threads. |
2302 uint active_workers = g1h->workers() ? g1h->workers()->active_workers() : 1U; | 2302 uint active_workers = g1h->workers() ? g1h->workers()->active_workers() : 1U; |
2303 active_workers = MAX2(MIN2(active_workers, _max_task_num), 1U); | 2303 active_workers = MAX2(MIN2(active_workers, _max_worker_id), 1U); |
2304 | 2304 |
2305 G1CMRefProcTaskExecutor par_task_executor(g1h, this, | 2305 G1CMRefProcTaskExecutor par_task_executor(g1h, this, |
2306 g1h->workers(), active_workers); | 2306 g1h->workers(), active_workers); |
2307 | 2307 |
2308 if (rp->processing_is_mt()) { | 2308 if (rp->processing_is_mt()) { |
2617 clearRangePrevBitmap(mr); | 2617 clearRangePrevBitmap(mr); |
2618 clearRangeNextBitmap(mr); | 2618 clearRangeNextBitmap(mr); |
2619 } | 2619 } |
2620 | 2620 |
2621 HeapRegion* | 2621 HeapRegion* |
2622 ConcurrentMark::claim_region(int task_num) { | 2622 ConcurrentMark::claim_region(uint worker_id) { |
2623 // "checkpoint" the finger | 2623 // "checkpoint" the finger |
2624 HeapWord* finger = _finger; | 2624 HeapWord* finger = _finger; |
2625 | 2625 |
2626 // _heap_end will not change underneath our feet; it only changes at | 2626 // _heap_end will not change underneath our feet; it only changes at |
2627 // yield points. | 2627 // yield points. |
2655 HeapWord* bottom = curr_region->bottom(); | 2655 HeapWord* bottom = curr_region->bottom(); |
2656 HeapWord* end = curr_region->end(); | 2656 HeapWord* end = curr_region->end(); |
2657 HeapWord* limit = curr_region->next_top_at_mark_start(); | 2657 HeapWord* limit = curr_region->next_top_at_mark_start(); |
2658 | 2658 |
2659 if (verbose_low()) { | 2659 if (verbose_low()) { |
2660 gclog_or_tty->print_cr("[%d] curr_region = "PTR_FORMAT" " | 2660 gclog_or_tty->print_cr("[%u] curr_region = "PTR_FORMAT" " |
2661 "["PTR_FORMAT", "PTR_FORMAT"), " | 2661 "["PTR_FORMAT", "PTR_FORMAT"), " |
2662 "limit = "PTR_FORMAT, | 2662 "limit = "PTR_FORMAT, |
2663 task_num, curr_region, bottom, end, limit); | 2663 worker_id, curr_region, bottom, end, limit); |
2664 } | 2664 } |
2665 | 2665 |
2666 // Is the gap between reading the finger and doing the CAS too long? | 2666 // Is the gap between reading the finger and doing the CAS too long? |
2667 HeapWord* res = (HeapWord*) Atomic::cmpxchg_ptr(end, &_finger, finger); | 2667 HeapWord* res = (HeapWord*) Atomic::cmpxchg_ptr(end, &_finger, finger); |
2668 if (res == finger) { | 2668 if (res == finger) { |
2671 // notice that _finger == end cannot be guaranteed here since, | 2671 // notice that _finger == end cannot be guaranteed here since, |
2672 // someone else might have moved the finger even further | 2672 // someone else might have moved the finger even further |
2673 assert(_finger >= end, "the finger should have moved forward"); | 2673 assert(_finger >= end, "the finger should have moved forward"); |
2674 | 2674 |
2675 if (verbose_low()) { | 2675 if (verbose_low()) { |
2676 gclog_or_tty->print_cr("[%d] we were successful with region = " | 2676 gclog_or_tty->print_cr("[%u] we were successful with region = " |
2677 PTR_FORMAT, task_num, curr_region); | 2677 PTR_FORMAT, worker_id, curr_region); |
2678 } | 2678 } |
2679 | 2679 |
2680 if (limit > bottom) { | 2680 if (limit > bottom) { |
2681 if (verbose_low()) { | 2681 if (verbose_low()) { |
2682 gclog_or_tty->print_cr("[%d] region "PTR_FORMAT" is not empty, " | 2682 gclog_or_tty->print_cr("[%u] region "PTR_FORMAT" is not empty, " |
2683 "returning it ", task_num, curr_region); | 2683 "returning it ", worker_id, curr_region); |
2684 } | 2684 } |
2685 return curr_region; | 2685 return curr_region; |
2686 } else { | 2686 } else { |
2687 assert(limit == bottom, | 2687 assert(limit == bottom, |
2688 "the region limit should be at bottom"); | 2688 "the region limit should be at bottom"); |
2689 if (verbose_low()) { | 2689 if (verbose_low()) { |
2690 gclog_or_tty->print_cr("[%d] region "PTR_FORMAT" is empty, " | 2690 gclog_or_tty->print_cr("[%u] region "PTR_FORMAT" is empty, " |
2691 "returning NULL", task_num, curr_region); | 2691 "returning NULL", worker_id, curr_region); |
2692 } | 2692 } |
2693 // we return NULL and the caller should try calling | 2693 // we return NULL and the caller should try calling |
2694 // claim_region() again. | 2694 // claim_region() again. |
2695 return NULL; | 2695 return NULL; |
2696 } | 2696 } |
2697 } else { | 2697 } else { |
2698 assert(_finger > finger, "the finger should have moved forward"); | 2698 assert(_finger > finger, "the finger should have moved forward"); |
2699 if (verbose_low()) { | 2699 if (verbose_low()) { |
2700 gclog_or_tty->print_cr("[%d] somebody else moved the finger, " | 2700 gclog_or_tty->print_cr("[%u] somebody else moved the finger, " |
2701 "global finger = "PTR_FORMAT", " | 2701 "global finger = "PTR_FORMAT", " |
2702 "our finger = "PTR_FORMAT, | 2702 "our finger = "PTR_FORMAT, |
2703 task_num, _finger, finger); | 2703 worker_id, _finger, finger); |
2704 } | 2704 } |
2705 | 2705 |
2706 // read it again | 2706 // read it again |
2707 finger = _finger; | 2707 finger = _finger; |
2708 } | 2708 } |
2781 // Verify entries on the global mark stack | 2781 // Verify entries on the global mark stack |
2782 cl.set_phase(VerifyNoCSetOopsStack); | 2782 cl.set_phase(VerifyNoCSetOopsStack); |
2783 _markStack.oops_do(&cl); | 2783 _markStack.oops_do(&cl); |
2784 | 2784 |
2785 // Verify entries on the task queues | 2785 // Verify entries on the task queues |
2786 for (int i = 0; i < (int) _max_task_num; i += 1) { | 2786 for (uint i = 0; i < _max_worker_id; i += 1) { |
2787 cl.set_phase(VerifyNoCSetOopsQueues, i); | 2787 cl.set_phase(VerifyNoCSetOopsQueues, i); |
2788 OopTaskQueue* queue = _task_queues->queue(i); | 2788 OopTaskQueue* queue = _task_queues->queue(i); |
2789 queue->oops_do(&cl); | 2789 queue->oops_do(&cl); |
2790 } | 2790 } |
2791 } | 2791 } |
2820 err_msg("global finger: "PTR_FORMAT" region: "HR_FORMAT, | 2820 err_msg("global finger: "PTR_FORMAT" region: "HR_FORMAT, |
2821 global_finger, HR_FORMAT_PARAMS(global_hr))); | 2821 global_finger, HR_FORMAT_PARAMS(global_hr))); |
2822 } | 2822 } |
2823 | 2823 |
2824 // Verify the task fingers | 2824 // Verify the task fingers |
2825 assert(parallel_marking_threads() <= _max_task_num, "sanity"); | 2825 assert(parallel_marking_threads() <= _max_worker_id, "sanity"); |
2826 for (int i = 0; i < (int) parallel_marking_threads(); i += 1) { | 2826 for (int i = 0; i < (int) parallel_marking_threads(); i += 1) { |
2827 CMTask* task = _tasks[i]; | 2827 CMTask* task = _tasks[i]; |
2828 HeapWord* task_finger = task->finger(); | 2828 HeapWord* task_finger = task->finger(); |
2829 if (task_finger != NULL && task_finger < _heap_end) { | 2829 if (task_finger != NULL && task_finger < _heap_end) { |
2830 // See above note on the global finger verification. | 2830 // See above note on the global finger verification. |
2847 } else { | 2847 } else { |
2848 assert(has_overflown(), "pre-condition"); | 2848 assert(has_overflown(), "pre-condition"); |
2849 } | 2849 } |
2850 _finger = _heap_start; | 2850 _finger = _heap_start; |
2851 | 2851 |
2852 for (int i = 0; i < (int)_max_task_num; ++i) { | 2852 for (uint i = 0; i < _max_worker_id; ++i) { |
2853 OopTaskQueue* queue = _task_queues->queue(i); | 2853 OopTaskQueue* queue = _task_queues->queue(i); |
2854 queue->set_empty(); | 2854 queue->set_empty(); |
2855 } | 2855 } |
2856 } | 2856 } |
2857 | 2857 |
2860 class AggregateCountDataHRClosure: public HeapRegionClosure { | 2860 class AggregateCountDataHRClosure: public HeapRegionClosure { |
2861 G1CollectedHeap* _g1h; | 2861 G1CollectedHeap* _g1h; |
2862 ConcurrentMark* _cm; | 2862 ConcurrentMark* _cm; |
2863 CardTableModRefBS* _ct_bs; | 2863 CardTableModRefBS* _ct_bs; |
2864 BitMap* _cm_card_bm; | 2864 BitMap* _cm_card_bm; |
2865 size_t _max_task_num; | 2865 uint _max_worker_id; |
2866 | 2866 |
2867 public: | 2867 public: |
2868 AggregateCountDataHRClosure(G1CollectedHeap* g1h, | 2868 AggregateCountDataHRClosure(G1CollectedHeap* g1h, |
2869 BitMap* cm_card_bm, | 2869 BitMap* cm_card_bm, |
2870 size_t max_task_num) : | 2870 uint max_worker_id) : |
2871 _g1h(g1h), _cm(g1h->concurrent_mark()), | 2871 _g1h(g1h), _cm(g1h->concurrent_mark()), |
2872 _ct_bs((CardTableModRefBS*) (g1h->barrier_set())), | 2872 _ct_bs((CardTableModRefBS*) (g1h->barrier_set())), |
2873 _cm_card_bm(cm_card_bm), _max_task_num(max_task_num) { } | 2873 _cm_card_bm(cm_card_bm), _max_worker_id(max_worker_id) { } |
2874 | 2874 |
2875 bool doHeapRegion(HeapRegion* hr) { | 2875 bool doHeapRegion(HeapRegion* hr) { |
2876 if (hr->continuesHumongous()) { | 2876 if (hr->continuesHumongous()) { |
2877 // We will ignore these here and process them when their | 2877 // We will ignore these here and process them when their |
2878 // associated "starts humongous" region is processed. | 2878 // associated "starts humongous" region is processed. |
2925 | 2925 |
2926 // Aggregate the "stripe" in the count data associated with hr. | 2926 // Aggregate the "stripe" in the count data associated with hr. |
2927 uint hrs_index = hr->hrs_index(); | 2927 uint hrs_index = hr->hrs_index(); |
2928 size_t marked_bytes = 0; | 2928 size_t marked_bytes = 0; |
2929 | 2929 |
2930 for (int i = 0; (size_t)i < _max_task_num; i += 1) { | 2930 for (uint i = 0; i < _max_worker_id; i += 1) { |
2931 size_t* marked_bytes_array = _cm->count_marked_bytes_array_for(i); | 2931 size_t* marked_bytes_array = _cm->count_marked_bytes_array_for(i); |
2932 BitMap* task_card_bm = _cm->count_card_bitmap_for(i); | 2932 BitMap* task_card_bm = _cm->count_card_bitmap_for(i); |
2933 | 2933 |
2934 // Fetch the marked_bytes in this region for task i and | 2934 // Fetch the marked_bytes in this region for task i and |
2935 // add it to the running total for this region. | 2935 // add it to the running total for this region. |
2936 marked_bytes += marked_bytes_array[hrs_index]; | 2936 marked_bytes += marked_bytes_array[hrs_index]; |
2937 | 2937 |
2938 // Now union the bitmaps[0,max_task_num)[start_idx..limit_idx) | 2938 // Now union the bitmaps[0,max_worker_id)[start_idx..limit_idx) |
2939 // into the global card bitmap. | 2939 // into the global card bitmap. |
2940 BitMap::idx_t scan_idx = task_card_bm->get_next_one_offset(start_idx, limit_idx); | 2940 BitMap::idx_t scan_idx = task_card_bm->get_next_one_offset(start_idx, limit_idx); |
2941 | 2941 |
2942 while (scan_idx < limit_idx) { | 2942 while (scan_idx < limit_idx) { |
2943 assert(task_card_bm->at(scan_idx) == true, "should be"); | 2943 assert(task_card_bm->at(scan_idx) == true, "should be"); |
2965 class G1AggregateCountDataTask: public AbstractGangTask { | 2965 class G1AggregateCountDataTask: public AbstractGangTask { |
2966 protected: | 2966 protected: |
2967 G1CollectedHeap* _g1h; | 2967 G1CollectedHeap* _g1h; |
2968 ConcurrentMark* _cm; | 2968 ConcurrentMark* _cm; |
2969 BitMap* _cm_card_bm; | 2969 BitMap* _cm_card_bm; |
2970 size_t _max_task_num; | 2970 uint _max_worker_id; |
2971 int _active_workers; | 2971 int _active_workers; |
2972 | 2972 |
2973 public: | 2973 public: |
2974 G1AggregateCountDataTask(G1CollectedHeap* g1h, | 2974 G1AggregateCountDataTask(G1CollectedHeap* g1h, |
2975 ConcurrentMark* cm, | 2975 ConcurrentMark* cm, |
2976 BitMap* cm_card_bm, | 2976 BitMap* cm_card_bm, |
2977 size_t max_task_num, | 2977 uint max_worker_id, |
2978 int n_workers) : | 2978 int n_workers) : |
2979 AbstractGangTask("Count Aggregation"), | 2979 AbstractGangTask("Count Aggregation"), |
2980 _g1h(g1h), _cm(cm), _cm_card_bm(cm_card_bm), | 2980 _g1h(g1h), _cm(cm), _cm_card_bm(cm_card_bm), |
2981 _max_task_num(max_task_num), | 2981 _max_worker_id(max_worker_id), |
2982 _active_workers(n_workers) { } | 2982 _active_workers(n_workers) { } |
2983 | 2983 |
2984 void work(uint worker_id) { | 2984 void work(uint worker_id) { |
2985 AggregateCountDataHRClosure cl(_g1h, _cm_card_bm, _max_task_num); | 2985 AggregateCountDataHRClosure cl(_g1h, _cm_card_bm, _max_worker_id); |
2986 | 2986 |
2987 if (G1CollectedHeap::use_parallel_gc_threads()) { | 2987 if (G1CollectedHeap::use_parallel_gc_threads()) { |
2988 _g1h->heap_region_par_iterate_chunked(&cl, worker_id, | 2988 _g1h->heap_region_par_iterate_chunked(&cl, worker_id, |
2989 _active_workers, | 2989 _active_workers, |
2990 HeapRegion::AggregateCountClaimValue); | 2990 HeapRegion::AggregateCountClaimValue); |
2999 int n_workers = (G1CollectedHeap::use_parallel_gc_threads() ? | 2999 int n_workers = (G1CollectedHeap::use_parallel_gc_threads() ? |
3000 _g1h->workers()->active_workers() : | 3000 _g1h->workers()->active_workers() : |
3001 1); | 3001 1); |
3002 | 3002 |
3003 G1AggregateCountDataTask g1_par_agg_task(_g1h, this, &_card_bm, | 3003 G1AggregateCountDataTask g1_par_agg_task(_g1h, this, &_card_bm, |
3004 _max_task_num, n_workers); | 3004 _max_worker_id, n_workers); |
3005 | 3005 |
3006 if (G1CollectedHeap::use_parallel_gc_threads()) { | 3006 if (G1CollectedHeap::use_parallel_gc_threads()) { |
3007 assert(_g1h->check_heap_region_claim_values(HeapRegion::InitialClaimValue), | 3007 assert(_g1h->check_heap_region_claim_values(HeapRegion::InitialClaimValue), |
3008 "sanity check"); | 3008 "sanity check"); |
3009 _g1h->set_par_threads(n_workers); | 3009 _g1h->set_par_threads(n_workers); |
3028 // Clear the global region bitmap - it will be filled as part | 3028 // Clear the global region bitmap - it will be filled as part |
3029 // of the final counting task. | 3029 // of the final counting task. |
3030 _region_bm.clear(); | 3030 _region_bm.clear(); |
3031 | 3031 |
3032 uint max_regions = _g1h->max_regions(); | 3032 uint max_regions = _g1h->max_regions(); |
3033 assert(_max_task_num != 0, "unitialized"); | 3033 assert(_max_worker_id > 0, "uninitialized"); |
3034 | 3034 |
3035 for (int i = 0; (size_t) i < _max_task_num; i += 1) { | 3035 for (uint i = 0; i < _max_worker_id; i += 1) { |
3036 BitMap* task_card_bm = count_card_bitmap_for(i); | 3036 BitMap* task_card_bm = count_card_bitmap_for(i); |
3037 size_t* marked_bytes_array = count_marked_bytes_array_for(i); | 3037 size_t* marked_bytes_array = count_marked_bytes_array_for(i); |
3038 | 3038 |
3039 assert(task_card_bm->size() == _card_bm.size(), "size mismatch"); | 3039 assert(task_card_bm->size() == _card_bm.size(), "size mismatch"); |
3040 assert(marked_bytes_array != NULL, "uninitialized"); | 3040 assert(marked_bytes_array != NULL, "uninitialized"); |
3060 _nextMarkBitMap->clearAll(); | 3060 _nextMarkBitMap->clearAll(); |
3061 // Clear the liveness counting data | 3061 // Clear the liveness counting data |
3062 clear_all_count_data(); | 3062 clear_all_count_data(); |
3063 // Empty mark stack | 3063 // Empty mark stack |
3064 clear_marking_state(); | 3064 clear_marking_state(); |
3065 for (int i = 0; i < (int)_max_task_num; ++i) { | 3065 for (uint i = 0; i < _max_worker_id; ++i) { |
3066 _tasks[i]->clear_region_fields(); | 3066 _tasks[i]->clear_region_fields(); |
3067 } | 3067 } |
3068 _has_aborted = true; | 3068 _has_aborted = true; |
3069 | 3069 |
3070 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); | 3070 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); |
3152 #ifndef PRODUCT | 3152 #ifndef PRODUCT |
3153 // for debugging purposes | 3153 // for debugging purposes |
3154 void ConcurrentMark::print_finger() { | 3154 void ConcurrentMark::print_finger() { |
3155 gclog_or_tty->print_cr("heap ["PTR_FORMAT", "PTR_FORMAT"), global finger = "PTR_FORMAT, | 3155 gclog_or_tty->print_cr("heap ["PTR_FORMAT", "PTR_FORMAT"), global finger = "PTR_FORMAT, |
3156 _heap_start, _heap_end, _finger); | 3156 _heap_start, _heap_end, _finger); |
3157 for (int i = 0; i < (int) _max_task_num; ++i) { | 3157 for (uint i = 0; i < _max_worker_id; ++i) { |
3158 gclog_or_tty->print(" %d: "PTR_FORMAT, i, _tasks[i]->finger()); | 3158 gclog_or_tty->print(" %u: "PTR_FORMAT, i, _tasks[i]->finger()); |
3159 } | 3159 } |
3160 gclog_or_tty->print_cr(""); | 3160 gclog_or_tty->print_cr(""); |
3161 } | 3161 } |
3162 #endif | 3162 #endif |
3163 | 3163 |
3164 void CMTask::scan_object(oop obj) { | 3164 void CMTask::scan_object(oop obj) { |
3165 assert(_nextMarkBitMap->isMarked((HeapWord*) obj), "invariant"); | 3165 assert(_nextMarkBitMap->isMarked((HeapWord*) obj), "invariant"); |
3166 | 3166 |
3167 if (_cm->verbose_high()) { | 3167 if (_cm->verbose_high()) { |
3168 gclog_or_tty->print_cr("[%d] we're scanning object "PTR_FORMAT, | 3168 gclog_or_tty->print_cr("[%u] we're scanning object "PTR_FORMAT, |
3169 _task_id, (void*) obj); | 3169 _worker_id, (void*) obj); |
3170 } | 3170 } |
3171 | 3171 |
3172 size_t obj_size = obj->size(); | 3172 size_t obj_size = obj->size(); |
3173 _words_scanned += obj_size; | 3173 _words_scanned += obj_size; |
3174 | 3174 |
3243 "claim_region() should have filtered out continues humongous regions"); | 3243 "claim_region() should have filtered out continues humongous regions"); |
3244 assert(!hr->continuesHumongous(), | 3244 assert(!hr->continuesHumongous(), |
3245 "claim_region() should have filtered out continues humongous regions"); | 3245 "claim_region() should have filtered out continues humongous regions"); |
3246 | 3246 |
3247 if (_cm->verbose_low()) { | 3247 if (_cm->verbose_low()) { |
3248 gclog_or_tty->print_cr("[%d] setting up for region "PTR_FORMAT, | 3248 gclog_or_tty->print_cr("[%u] setting up for region "PTR_FORMAT, |
3249 _task_id, hr); | 3249 _worker_id, hr); |
3250 } | 3250 } |
3251 | 3251 |
3252 _curr_region = hr; | 3252 _curr_region = hr; |
3253 _finger = hr->bottom(); | 3253 _finger = hr->bottom(); |
3254 update_region_limit(); | 3254 update_region_limit(); |
3259 HeapWord* bottom = hr->bottom(); | 3259 HeapWord* bottom = hr->bottom(); |
3260 HeapWord* limit = hr->next_top_at_mark_start(); | 3260 HeapWord* limit = hr->next_top_at_mark_start(); |
3261 | 3261 |
3262 if (limit == bottom) { | 3262 if (limit == bottom) { |
3263 if (_cm->verbose_low()) { | 3263 if (_cm->verbose_low()) { |
3264 gclog_or_tty->print_cr("[%d] found an empty region " | 3264 gclog_or_tty->print_cr("[%u] found an empty region " |
3265 "["PTR_FORMAT", "PTR_FORMAT")", | 3265 "["PTR_FORMAT", "PTR_FORMAT")", |
3266 _task_id, bottom, limit); | 3266 _worker_id, bottom, limit); |
3267 } | 3267 } |
3268 // The region was collected underneath our feet. | 3268 // The region was collected underneath our feet. |
3269 // We set the finger to bottom to ensure that the bitmap | 3269 // We set the finger to bottom to ensure that the bitmap |
3270 // iteration that will follow this will not do anything. | 3270 // iteration that will follow this will not do anything. |
3271 // (this is not a condition that holds when we set the region up, | 3271 // (this is not a condition that holds when we set the region up, |
3292 } | 3292 } |
3293 | 3293 |
3294 void CMTask::giveup_current_region() { | 3294 void CMTask::giveup_current_region() { |
3295 assert(_curr_region != NULL, "invariant"); | 3295 assert(_curr_region != NULL, "invariant"); |
3296 if (_cm->verbose_low()) { | 3296 if (_cm->verbose_low()) { |
3297 gclog_or_tty->print_cr("[%d] giving up region "PTR_FORMAT, | 3297 gclog_or_tty->print_cr("[%u] giving up region "PTR_FORMAT, |
3298 _task_id, _curr_region); | 3298 _worker_id, _curr_region); |
3299 } | 3299 } |
3300 clear_region_fields(); | 3300 clear_region_fields(); |
3301 } | 3301 } |
3302 | 3302 |
3303 void CMTask::clear_region_fields() { | 3303 void CMTask::clear_region_fields() { |
3319 | 3319 |
3320 void CMTask::reset(CMBitMap* nextMarkBitMap) { | 3320 void CMTask::reset(CMBitMap* nextMarkBitMap) { |
3321 guarantee(nextMarkBitMap != NULL, "invariant"); | 3321 guarantee(nextMarkBitMap != NULL, "invariant"); |
3322 | 3322 |
3323 if (_cm->verbose_low()) { | 3323 if (_cm->verbose_low()) { |
3324 gclog_or_tty->print_cr("[%d] resetting", _task_id); | 3324 gclog_or_tty->print_cr("[%u] resetting", _worker_id); |
3325 } | 3325 } |
3326 | 3326 |
3327 _nextMarkBitMap = nextMarkBitMap; | 3327 _nextMarkBitMap = nextMarkBitMap; |
3328 clear_region_fields(); | 3328 clear_region_fields(); |
3329 | 3329 |
3413 double last_interval_ms = curr_time_ms - _interval_start_time_ms; | 3413 double last_interval_ms = curr_time_ms - _interval_start_time_ms; |
3414 _interval_start_time_ms = curr_time_ms; | 3414 _interval_start_time_ms = curr_time_ms; |
3415 _all_clock_intervals_ms.add(last_interval_ms); | 3415 _all_clock_intervals_ms.add(last_interval_ms); |
3416 | 3416 |
3417 if (_cm->verbose_medium()) { | 3417 if (_cm->verbose_medium()) { |
3418 gclog_or_tty->print_cr("[%d] regular clock, interval = %1.2lfms, " | 3418 gclog_or_tty->print_cr("[%u] regular clock, interval = %1.2lfms, " |
3419 "scanned = %d%s, refs reached = %d%s", | 3419 "scanned = %d%s, refs reached = %d%s", |
3420 _task_id, last_interval_ms, | 3420 _worker_id, last_interval_ms, |
3421 _words_scanned, | 3421 _words_scanned, |
3422 (_words_scanned >= _words_scanned_limit) ? " (*)" : "", | 3422 (_words_scanned >= _words_scanned_limit) ? " (*)" : "", |
3423 _refs_reached, | 3423 _refs_reached, |
3424 (_refs_reached >= _refs_reached_limit) ? " (*)" : ""); | 3424 (_refs_reached >= _refs_reached_limit) ? " (*)" : ""); |
3425 } | 3425 } |
3447 // (6) Finally, we check whether there are enough completed STAB | 3447 // (6) Finally, we check whether there are enough completed STAB |
3448 // buffers available for processing. If there are, we abort. | 3448 // buffers available for processing. If there are, we abort. |
3449 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); | 3449 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); |
3450 if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) { | 3450 if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) { |
3451 if (_cm->verbose_low()) { | 3451 if (_cm->verbose_low()) { |
3452 gclog_or_tty->print_cr("[%d] aborting to deal with pending SATB buffers", | 3452 gclog_or_tty->print_cr("[%u] aborting to deal with pending SATB buffers", |
3453 _task_id); | 3453 _worker_id); |
3454 } | 3454 } |
3455 // we do need to process SATB buffers, we'll abort and restart | 3455 // we do need to process SATB buffers, we'll abort and restart |
3456 // the marking task to do so | 3456 // the marking task to do so |
3457 set_has_aborted(); | 3457 set_has_aborted(); |
3458 statsOnly( ++_aborted_satb ); | 3458 statsOnly( ++_aborted_satb ); |
3473 // operation which will increase the per byte scanned cost (i.e. move | 3473 // operation which will increase the per byte scanned cost (i.e. move |
3474 // entries to/from the global stack). It basically tries to decrease the | 3474 // entries to/from the global stack). It basically tries to decrease the |
3475 // scanning limit so that the clock is called earlier. | 3475 // scanning limit so that the clock is called earlier. |
3476 | 3476 |
3477 if (_cm->verbose_medium()) { | 3477 if (_cm->verbose_medium()) { |
3478 gclog_or_tty->print_cr("[%d] decreasing limits", _task_id); | 3478 gclog_or_tty->print_cr("[%u] decreasing limits", _worker_id); |
3479 } | 3479 } |
3480 | 3480 |
3481 _words_scanned_limit = _real_words_scanned_limit - | 3481 _words_scanned_limit = _real_words_scanned_limit - |
3482 3 * words_scanned_period / 4; | 3482 3 * words_scanned_period / 4; |
3483 _refs_reached_limit = _real_refs_reached_limit - | 3483 _refs_reached_limit = _real_refs_reached_limit - |
3501 | 3501 |
3502 statsOnly( ++_global_transfers_to; _local_pops += n ); | 3502 statsOnly( ++_global_transfers_to; _local_pops += n ); |
3503 | 3503 |
3504 if (!_cm->mark_stack_push(buffer, n)) { | 3504 if (!_cm->mark_stack_push(buffer, n)) { |
3505 if (_cm->verbose_low()) { | 3505 if (_cm->verbose_low()) { |
3506 gclog_or_tty->print_cr("[%d] aborting due to global stack overflow", | 3506 gclog_or_tty->print_cr("[%u] aborting due to global stack overflow", |
3507 _task_id); | 3507 _worker_id); |
3508 } | 3508 } |
3509 set_has_aborted(); | 3509 set_has_aborted(); |
3510 } else { | 3510 } else { |
3511 // the transfer was successful | 3511 // the transfer was successful |
3512 | 3512 |
3513 if (_cm->verbose_medium()) { | 3513 if (_cm->verbose_medium()) { |
3514 gclog_or_tty->print_cr("[%d] pushed %d entries to the global stack", | 3514 gclog_or_tty->print_cr("[%u] pushed %d entries to the global stack", |
3515 _task_id, n); | 3515 _worker_id, n); |
3516 } | 3516 } |
3517 statsOnly( int tmp_size = _cm->mark_stack_size(); | 3517 statsOnly( int tmp_size = _cm->mark_stack_size(); |
3518 if (tmp_size > _global_max_size) { | 3518 if (tmp_size > _global_max_size) { |
3519 _global_max_size = tmp_size; | 3519 _global_max_size = tmp_size; |
3520 } | 3520 } |
3537 if (n > 0) { | 3537 if (n > 0) { |
3538 // yes, we did actually pop at least one entry | 3538 // yes, we did actually pop at least one entry |
3539 | 3539 |
3540 statsOnly( ++_global_transfers_from; _global_pops += n ); | 3540 statsOnly( ++_global_transfers_from; _global_pops += n ); |
3541 if (_cm->verbose_medium()) { | 3541 if (_cm->verbose_medium()) { |
3542 gclog_or_tty->print_cr("[%d] popped %d entries from the global stack", | 3542 gclog_or_tty->print_cr("[%u] popped %d entries from the global stack", |
3543 _task_id, n); | 3543 _worker_id, n); |
3544 } | 3544 } |
3545 for (int i = 0; i < n; ++i) { | 3545 for (int i = 0; i < n; ++i) { |
3546 bool success = _task_queue->push(buffer[i]); | 3546 bool success = _task_queue->push(buffer[i]); |
3547 // We only call this when the local queue is empty or under a | 3547 // We only call this when the local queue is empty or under a |
3548 // given target limit. So, we do not expect this push to fail. | 3548 // given target limit. So, we do not expect this push to fail. |
3573 target_size = 0; | 3573 target_size = 0; |
3574 } | 3574 } |
3575 | 3575 |
3576 if (_task_queue->size() > target_size) { | 3576 if (_task_queue->size() > target_size) { |
3577 if (_cm->verbose_high()) { | 3577 if (_cm->verbose_high()) { |
3578 gclog_or_tty->print_cr("[%d] draining local queue, target size = %d", | 3578 gclog_or_tty->print_cr("[%u] draining local queue, target size = %d", |
3579 _task_id, target_size); | 3579 _worker_id, target_size); |
3580 } | 3580 } |
3581 | 3581 |
3582 oop obj; | 3582 oop obj; |
3583 bool ret = _task_queue->pop_local(obj); | 3583 bool ret = _task_queue->pop_local(obj); |
3584 while (ret) { | 3584 while (ret) { |
3585 statsOnly( ++_local_pops ); | 3585 statsOnly( ++_local_pops ); |
3586 | 3586 |
3587 if (_cm->verbose_high()) { | 3587 if (_cm->verbose_high()) { |
3588 gclog_or_tty->print_cr("[%d] popped "PTR_FORMAT, _task_id, | 3588 gclog_or_tty->print_cr("[%u] popped "PTR_FORMAT, _worker_id, |
3589 (void*) obj); | 3589 (void*) obj); |
3590 } | 3590 } |
3591 | 3591 |
3592 assert(_g1h->is_in_g1_reserved((HeapWord*) obj), "invariant" ); | 3592 assert(_g1h->is_in_g1_reserved((HeapWord*) obj), "invariant" ); |
3593 assert(!_g1h->is_on_master_free_list( | 3593 assert(!_g1h->is_on_master_free_list( |
3601 ret = _task_queue->pop_local(obj); | 3601 ret = _task_queue->pop_local(obj); |
3602 } | 3602 } |
3603 } | 3603 } |
3604 | 3604 |
3605 if (_cm->verbose_high()) { | 3605 if (_cm->verbose_high()) { |
3606 gclog_or_tty->print_cr("[%d] drained local queue, size = %d", | 3606 gclog_or_tty->print_cr("[%u] drained local queue, size = %d", |
3607 _task_id, _task_queue->size()); | 3607 _worker_id, _task_queue->size()); |
3608 } | 3608 } |
3609 } | 3609 } |
3610 } | 3610 } |
3611 | 3611 |
3612 void CMTask::drain_global_stack(bool partially) { | 3612 void CMTask::drain_global_stack(bool partially) { |
3629 target_size = 0; | 3629 target_size = 0; |
3630 } | 3630 } |
3631 | 3631 |
3632 if (_cm->mark_stack_size() > target_size) { | 3632 if (_cm->mark_stack_size() > target_size) { |
3633 if (_cm->verbose_low()) { | 3633 if (_cm->verbose_low()) { |
3634 gclog_or_tty->print_cr("[%d] draining global_stack, target size %d", | 3634 gclog_or_tty->print_cr("[%u] draining global_stack, target size %d", |
3635 _task_id, target_size); | 3635 _worker_id, target_size); |
3636 } | 3636 } |
3637 | 3637 |
3638 while (!has_aborted() && _cm->mark_stack_size() > target_size) { | 3638 while (!has_aborted() && _cm->mark_stack_size() > target_size) { |
3639 get_entries_from_global_stack(); | 3639 get_entries_from_global_stack(); |
3640 drain_local_queue(partially); | 3640 drain_local_queue(partially); |
3641 } | 3641 } |
3642 | 3642 |
3643 if (_cm->verbose_low()) { | 3643 if (_cm->verbose_low()) { |
3644 gclog_or_tty->print_cr("[%d] drained global stack, size = %d", | 3644 gclog_or_tty->print_cr("[%u] drained global stack, size = %d", |
3645 _task_id, _cm->mark_stack_size()); | 3645 _worker_id, _cm->mark_stack_size()); |
3646 } | 3646 } |
3647 } | 3647 } |
3648 } | 3648 } |
3649 | 3649 |
3650 // SATB Queue has several assumptions on whether to call the par or | 3650 // SATB Queue has several assumptions on whether to call the par or |
3661 _draining_satb_buffers = true; | 3661 _draining_satb_buffers = true; |
3662 | 3662 |
3663 CMObjectClosure oc(this); | 3663 CMObjectClosure oc(this); |
3664 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); | 3664 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); |
3665 if (G1CollectedHeap::use_parallel_gc_threads()) { | 3665 if (G1CollectedHeap::use_parallel_gc_threads()) { |
3666 satb_mq_set.set_par_closure(_task_id, &oc); | 3666 satb_mq_set.set_par_closure(_worker_id, &oc); |
3667 } else { | 3667 } else { |
3668 satb_mq_set.set_closure(&oc); | 3668 satb_mq_set.set_closure(&oc); |
3669 } | 3669 } |
3670 | 3670 |
3671 // This keeps claiming and applying the closure to completed buffers | 3671 // This keeps claiming and applying the closure to completed buffers |
3672 // until we run out of buffers or we need to abort. | 3672 // until we run out of buffers or we need to abort. |
3673 if (G1CollectedHeap::use_parallel_gc_threads()) { | 3673 if (G1CollectedHeap::use_parallel_gc_threads()) { |
3674 while (!has_aborted() && | 3674 while (!has_aborted() && |
3675 satb_mq_set.par_apply_closure_to_completed_buffer(_task_id)) { | 3675 satb_mq_set.par_apply_closure_to_completed_buffer(_worker_id)) { |
3676 if (_cm->verbose_medium()) { | 3676 if (_cm->verbose_medium()) { |
3677 gclog_or_tty->print_cr("[%d] processed an SATB buffer", _task_id); | 3677 gclog_or_tty->print_cr("[%u] processed an SATB buffer", _worker_id); |
3678 } | 3678 } |
3679 statsOnly( ++_satb_buffers_processed ); | 3679 statsOnly( ++_satb_buffers_processed ); |
3680 regular_clock_call(); | 3680 regular_clock_call(); |
3681 } | 3681 } |
3682 } else { | 3682 } else { |
3683 while (!has_aborted() && | 3683 while (!has_aborted() && |
3684 satb_mq_set.apply_closure_to_completed_buffer()) { | 3684 satb_mq_set.apply_closure_to_completed_buffer()) { |
3685 if (_cm->verbose_medium()) { | 3685 if (_cm->verbose_medium()) { |
3686 gclog_or_tty->print_cr("[%d] processed an SATB buffer", _task_id); | 3686 gclog_or_tty->print_cr("[%u] processed an SATB buffer", _worker_id); |
3687 } | 3687 } |
3688 statsOnly( ++_satb_buffers_processed ); | 3688 statsOnly( ++_satb_buffers_processed ); |
3689 regular_clock_call(); | 3689 regular_clock_call(); |
3690 } | 3690 } |
3691 } | 3691 } |
3692 | 3692 |
3693 if (!concurrent() && !has_aborted()) { | 3693 if (!concurrent() && !has_aborted()) { |
3694 // We should only do this during remark. | 3694 // We should only do this during remark. |
3695 if (G1CollectedHeap::use_parallel_gc_threads()) { | 3695 if (G1CollectedHeap::use_parallel_gc_threads()) { |
3696 satb_mq_set.par_iterate_closure_all_threads(_task_id); | 3696 satb_mq_set.par_iterate_closure_all_threads(_worker_id); |
3697 } else { | 3697 } else { |
3698 satb_mq_set.iterate_closure_all_threads(); | 3698 satb_mq_set.iterate_closure_all_threads(); |
3699 } | 3699 } |
3700 } | 3700 } |
3701 | 3701 |
3704 assert(has_aborted() || | 3704 assert(has_aborted() || |
3705 concurrent() || | 3705 concurrent() || |
3706 satb_mq_set.completed_buffers_num() == 0, "invariant"); | 3706 satb_mq_set.completed_buffers_num() == 0, "invariant"); |
3707 | 3707 |
3708 if (G1CollectedHeap::use_parallel_gc_threads()) { | 3708 if (G1CollectedHeap::use_parallel_gc_threads()) { |
3709 satb_mq_set.set_par_closure(_task_id, NULL); | 3709 satb_mq_set.set_par_closure(_worker_id, NULL); |
3710 } else { | 3710 } else { |
3711 satb_mq_set.set_closure(NULL); | 3711 satb_mq_set.set_closure(NULL); |
3712 } | 3712 } |
3713 | 3713 |
3714 // again, this was a potentially expensive operation, decrease the | 3714 // again, this was a potentially expensive operation, decrease the |
3715 // limits to get the regular clock call early | 3715 // limits to get the regular clock call early |
3716 decrease_limits(); | 3716 decrease_limits(); |
3717 } | 3717 } |
3718 | 3718 |
3719 void CMTask::print_stats() { | 3719 void CMTask::print_stats() { |
3720 gclog_or_tty->print_cr("Marking Stats, task = %d, calls = %d", | 3720 gclog_or_tty->print_cr("Marking Stats, task = %u, calls = %d", |
3721 _task_id, _calls); | 3721 _worker_id, _calls); |
3722 gclog_or_tty->print_cr(" Elapsed time = %1.2lfms, Termination time = %1.2lfms", | 3722 gclog_or_tty->print_cr(" Elapsed time = %1.2lfms, Termination time = %1.2lfms", |
3723 _elapsed_time_ms, _termination_time_ms); | 3723 _elapsed_time_ms, _termination_time_ms); |
3724 gclog_or_tty->print_cr(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms", | 3724 gclog_or_tty->print_cr(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms", |
3725 _step_times_ms.num(), _step_times_ms.avg(), | 3725 _step_times_ms.num(), _step_times_ms.avg(), |
3726 _step_times_ms.sd()); | 3726 _step_times_ms.sd()); |
3864 assert(concurrent() == _cm->concurrent(), "they should be the same"); | 3864 assert(concurrent() == _cm->concurrent(), "they should be the same"); |
3865 | 3865 |
3866 G1CollectorPolicy* g1_policy = _g1h->g1_policy(); | 3866 G1CollectorPolicy* g1_policy = _g1h->g1_policy(); |
3867 assert(_task_queues != NULL, "invariant"); | 3867 assert(_task_queues != NULL, "invariant"); |
3868 assert(_task_queue != NULL, "invariant"); | 3868 assert(_task_queue != NULL, "invariant"); |
3869 assert(_task_queues->queue(_task_id) == _task_queue, "invariant"); | 3869 assert(_task_queues->queue(_worker_id) == _task_queue, "invariant"); |
3870 | 3870 |
3871 assert(!_claimed, | 3871 assert(!_claimed, |
3872 "only one thread should claim this task at any one time"); | 3872 "only one thread should claim this task at any one time"); |
3873 | 3873 |
3874 // OK, this doesn't safeguard again all possible scenarios, as it is | 3874 // OK, this doesn't safeguard again all possible scenarios, as it is |
3896 _draining_satb_buffers = false; | 3896 _draining_satb_buffers = false; |
3897 | 3897 |
3898 ++_calls; | 3898 ++_calls; |
3899 | 3899 |
3900 if (_cm->verbose_low()) { | 3900 if (_cm->verbose_low()) { |
3901 gclog_or_tty->print_cr("[%d] >>>>>>>>>> START, call = %d, " | 3901 gclog_or_tty->print_cr("[%u] >>>>>>>>>> START, call = %d, " |
3902 "target = %1.2lfms >>>>>>>>>>", | 3902 "target = %1.2lfms >>>>>>>>>>", |
3903 _task_id, _calls, _time_target_ms); | 3903 _worker_id, _calls, _time_target_ms); |
3904 } | 3904 } |
3905 | 3905 |
3906 // Set up the bitmap and oop closures. Anything that uses them is | 3906 // Set up the bitmap and oop closures. Anything that uses them is |
3907 // eventually called from this method, so it is OK to allocate these | 3907 // eventually called from this method, so it is OK to allocate these |
3908 // statically. | 3908 // statically. |
3946 // the address where we last found a marked object. If this is a | 3946 // the address where we last found a marked object. If this is a |
3947 // fresh region, _finger points to start(). | 3947 // fresh region, _finger points to start(). |
3948 MemRegion mr = MemRegion(_finger, _region_limit); | 3948 MemRegion mr = MemRegion(_finger, _region_limit); |
3949 | 3949 |
3950 if (_cm->verbose_low()) { | 3950 if (_cm->verbose_low()) { |
3951 gclog_or_tty->print_cr("[%d] we're scanning part " | 3951 gclog_or_tty->print_cr("[%u] we're scanning part " |
3952 "["PTR_FORMAT", "PTR_FORMAT") " | 3952 "["PTR_FORMAT", "PTR_FORMAT") " |
3953 "of region "PTR_FORMAT, | 3953 "of region "PTR_FORMAT, |
3954 _task_id, _finger, _region_limit, _curr_region); | 3954 _worker_id, _finger, _region_limit, _curr_region); |
3955 } | 3955 } |
3956 | 3956 |
3957 // Let's iterate over the bitmap of the part of the | 3957 // Let's iterate over the bitmap of the part of the |
3958 // region that is left. | 3958 // region that is left. |
3959 if (mr.is_empty() || _nextMarkBitMap->iterate(&bitmap_closure, mr)) { | 3959 if (mr.is_empty() || _nextMarkBitMap->iterate(&bitmap_closure, mr)) { |
4005 // Separated the asserts so that we know which one fires. | 4005 // Separated the asserts so that we know which one fires. |
4006 assert(_curr_region == NULL, "invariant"); | 4006 assert(_curr_region == NULL, "invariant"); |
4007 assert(_finger == NULL, "invariant"); | 4007 assert(_finger == NULL, "invariant"); |
4008 assert(_region_limit == NULL, "invariant"); | 4008 assert(_region_limit == NULL, "invariant"); |
4009 if (_cm->verbose_low()) { | 4009 if (_cm->verbose_low()) { |
4010 gclog_or_tty->print_cr("[%d] trying to claim a new region", _task_id); | 4010 gclog_or_tty->print_cr("[%u] trying to claim a new region", _worker_id); |
4011 } | 4011 } |
4012 HeapRegion* claimed_region = _cm->claim_region(_task_id); | 4012 HeapRegion* claimed_region = _cm->claim_region(_worker_id); |
4013 if (claimed_region != NULL) { | 4013 if (claimed_region != NULL) { |
4014 // Yes, we managed to claim one | 4014 // Yes, we managed to claim one |
4015 statsOnly( ++_regions_claimed ); | 4015 statsOnly( ++_regions_claimed ); |
4016 | 4016 |
4017 if (_cm->verbose_low()) { | 4017 if (_cm->verbose_low()) { |
4018 gclog_or_tty->print_cr("[%d] we successfully claimed " | 4018 gclog_or_tty->print_cr("[%u] we successfully claimed " |
4019 "region "PTR_FORMAT, | 4019 "region "PTR_FORMAT, |
4020 _task_id, claimed_region); | 4020 _worker_id, claimed_region); |
4021 } | 4021 } |
4022 | 4022 |
4023 setup_for_region(claimed_region); | 4023 setup_for_region(claimed_region); |
4024 assert(_curr_region == claimed_region, "invariant"); | 4024 assert(_curr_region == claimed_region, "invariant"); |
4025 } | 4025 } |
4042 // tasks might be pushing objects to it concurrently. | 4042 // tasks might be pushing objects to it concurrently. |
4043 assert(_cm->out_of_regions(), | 4043 assert(_cm->out_of_regions(), |
4044 "at this point we should be out of regions"); | 4044 "at this point we should be out of regions"); |
4045 | 4045 |
4046 if (_cm->verbose_low()) { | 4046 if (_cm->verbose_low()) { |
4047 gclog_or_tty->print_cr("[%d] all regions claimed", _task_id); | 4047 gclog_or_tty->print_cr("[%u] all regions claimed", _worker_id); |
4048 } | 4048 } |
4049 | 4049 |
4050 // Try to reduce the number of available SATB buffers so that | 4050 // Try to reduce the number of available SATB buffers so that |
4051 // remark has less work to do. | 4051 // remark has less work to do. |
4052 drain_satb_buffers(); | 4052 drain_satb_buffers(); |
4066 // tasks might be pushing objects to it concurrently. | 4066 // tasks might be pushing objects to it concurrently. |
4067 assert(_cm->out_of_regions() && _task_queue->size() == 0, | 4067 assert(_cm->out_of_regions() && _task_queue->size() == 0, |
4068 "only way to reach here"); | 4068 "only way to reach here"); |
4069 | 4069 |
4070 if (_cm->verbose_low()) { | 4070 if (_cm->verbose_low()) { |
4071 gclog_or_tty->print_cr("[%d] starting to steal", _task_id); | 4071 gclog_or_tty->print_cr("[%u] starting to steal", _worker_id); |
4072 } | 4072 } |
4073 | 4073 |
4074 while (!has_aborted()) { | 4074 while (!has_aborted()) { |
4075 oop obj; | 4075 oop obj; |
4076 statsOnly( ++_steal_attempts ); | 4076 statsOnly( ++_steal_attempts ); |
4077 | 4077 |
4078 if (_cm->try_stealing(_task_id, &_hash_seed, obj)) { | 4078 if (_cm->try_stealing(_worker_id, &_hash_seed, obj)) { |
4079 if (_cm->verbose_medium()) { | 4079 if (_cm->verbose_medium()) { |
4080 gclog_or_tty->print_cr("[%d] stolen "PTR_FORMAT" successfully", | 4080 gclog_or_tty->print_cr("[%u] stolen "PTR_FORMAT" successfully", |
4081 _task_id, (void*) obj); | 4081 _worker_id, (void*) obj); |
4082 } | 4082 } |
4083 | 4083 |
4084 statsOnly( ++_steals ); | 4084 statsOnly( ++_steals ); |
4085 | 4085 |
4086 assert(_nextMarkBitMap->isMarked((HeapWord*) obj), | 4086 assert(_nextMarkBitMap->isMarked((HeapWord*) obj), |
4114 // Separated the asserts so that we know which one fires. | 4114 // Separated the asserts so that we know which one fires. |
4115 assert(_cm->out_of_regions(), "only way to reach here"); | 4115 assert(_cm->out_of_regions(), "only way to reach here"); |
4116 assert(_task_queue->size() == 0, "only way to reach here"); | 4116 assert(_task_queue->size() == 0, "only way to reach here"); |
4117 | 4117 |
4118 if (_cm->verbose_low()) { | 4118 if (_cm->verbose_low()) { |
4119 gclog_or_tty->print_cr("[%d] starting termination protocol", _task_id); | 4119 gclog_or_tty->print_cr("[%u] starting termination protocol", _worker_id); |
4120 } | 4120 } |
4121 | 4121 |
4122 _termination_start_time_ms = os::elapsedVTime() * 1000.0; | 4122 _termination_start_time_ms = os::elapsedVTime() * 1000.0; |
4123 // The CMTask class also extends the TerminatorTerminator class, | 4123 // The CMTask class also extends the TerminatorTerminator class, |
4124 // hence its should_exit_termination() method will also decide | 4124 // hence its should_exit_termination() method will also decide |
4129 termination_end_time_ms - _termination_start_time_ms; | 4129 termination_end_time_ms - _termination_start_time_ms; |
4130 | 4130 |
4131 if (finished) { | 4131 if (finished) { |
4132 // We're all done. | 4132 // We're all done. |
4133 | 4133 |
4134 if (_task_id == 0) { | 4134 if (_worker_id == 0) { |
4135 // let's allow task 0 to do this | 4135 // let's allow task 0 to do this |
4136 if (concurrent()) { | 4136 if (concurrent()) { |
4137 assert(_cm->concurrent_marking_in_progress(), "invariant"); | 4137 assert(_cm->concurrent_marking_in_progress(), "invariant"); |
4138 // we need to set this to false before the next | 4138 // we need to set this to false before the next |
4139 // safepoint. This way we ensure that the marking phase | 4139 // safepoint. This way we ensure that the marking phase |
4151 guarantee(_task_queue->size() == 0, "only way to reach here"); | 4151 guarantee(_task_queue->size() == 0, "only way to reach here"); |
4152 guarantee(!_cm->has_overflown(), "only way to reach here"); | 4152 guarantee(!_cm->has_overflown(), "only way to reach here"); |
4153 guarantee(!_cm->mark_stack_overflow(), "only way to reach here"); | 4153 guarantee(!_cm->mark_stack_overflow(), "only way to reach here"); |
4154 | 4154 |
4155 if (_cm->verbose_low()) { | 4155 if (_cm->verbose_low()) { |
4156 gclog_or_tty->print_cr("[%d] all tasks terminated", _task_id); | 4156 gclog_or_tty->print_cr("[%u] all tasks terminated", _worker_id); |
4157 } | 4157 } |
4158 } else { | 4158 } else { |
4159 // Apparently there's more work to do. Let's abort this task. It | 4159 // Apparently there's more work to do. Let's abort this task. It |
4160 // will restart it and we can hopefully find more things to do. | 4160 // will restart it and we can hopefully find more things to do. |
4161 | 4161 |
4162 if (_cm->verbose_low()) { | 4162 if (_cm->verbose_low()) { |
4163 gclog_or_tty->print_cr("[%d] apparently there is more work to do", | 4163 gclog_or_tty->print_cr("[%u] apparently there is more work to do", |
4164 _task_id); | 4164 _worker_id); |
4165 } | 4165 } |
4166 | 4166 |
4167 set_has_aborted(); | 4167 set_has_aborted(); |
4168 statsOnly( ++_aborted_termination ); | 4168 statsOnly( ++_aborted_termination ); |
4169 } | 4169 } |
4198 // order to do this we have to make sure that all tasks stop | 4198 // order to do this we have to make sure that all tasks stop |
4199 // what they are doing and re-initialise in a safe manner. We | 4199 // what they are doing and re-initialise in a safe manner. We |
4200 // will achieve this with the use of two barrier sync points. | 4200 // will achieve this with the use of two barrier sync points. |
4201 | 4201 |
4202 if (_cm->verbose_low()) { | 4202 if (_cm->verbose_low()) { |
4203 gclog_or_tty->print_cr("[%d] detected overflow", _task_id); | 4203 gclog_or_tty->print_cr("[%u] detected overflow", _worker_id); |
4204 } | 4204 } |
4205 | 4205 |
4206 _cm->enter_first_sync_barrier(_task_id); | 4206 _cm->enter_first_sync_barrier(_worker_id); |
4207 // When we exit this sync barrier we know that all tasks have | 4207 // When we exit this sync barrier we know that all tasks have |
4208 // stopped doing marking work. So, it's now safe to | 4208 // stopped doing marking work. So, it's now safe to |
4209 // re-initialise our data structures. At the end of this method, | 4209 // re-initialise our data structures. At the end of this method, |
4210 // task 0 will clear the global data structures. | 4210 // task 0 will clear the global data structures. |
4211 | 4211 |
4213 | 4213 |
4214 // We clear the local state of this task... | 4214 // We clear the local state of this task... |
4215 clear_region_fields(); | 4215 clear_region_fields(); |
4216 | 4216 |
4217 // ...and enter the second barrier. | 4217 // ...and enter the second barrier. |
4218 _cm->enter_second_sync_barrier(_task_id); | 4218 _cm->enter_second_sync_barrier(_worker_id); |
4219 // At this point everything has bee re-initialised and we're | 4219 // At this point everything has bee re-initialised and we're |
4220 // ready to restart. | 4220 // ready to restart. |
4221 } | 4221 } |
4222 | 4222 |
4223 if (_cm->verbose_low()) { | 4223 if (_cm->verbose_low()) { |
4224 gclog_or_tty->print_cr("[%d] <<<<<<<<<< ABORTING, target = %1.2lfms, " | 4224 gclog_or_tty->print_cr("[%u] <<<<<<<<<< ABORTING, target = %1.2lfms, " |
4225 "elapsed = %1.2lfms <<<<<<<<<<", | 4225 "elapsed = %1.2lfms <<<<<<<<<<", |
4226 _task_id, _time_target_ms, elapsed_time_ms); | 4226 _worker_id, _time_target_ms, elapsed_time_ms); |
4227 if (_cm->has_aborted()) { | 4227 if (_cm->has_aborted()) { |
4228 gclog_or_tty->print_cr("[%d] ========== MARKING ABORTED ==========", | 4228 gclog_or_tty->print_cr("[%u] ========== MARKING ABORTED ==========", |
4229 _task_id); | 4229 _worker_id); |
4230 } | 4230 } |
4231 } | 4231 } |
4232 } else { | 4232 } else { |
4233 if (_cm->verbose_low()) { | 4233 if (_cm->verbose_low()) { |
4234 gclog_or_tty->print_cr("[%d] <<<<<<<<<< FINISHED, target = %1.2lfms, " | 4234 gclog_or_tty->print_cr("[%u] <<<<<<<<<< FINISHED, target = %1.2lfms, " |
4235 "elapsed = %1.2lfms <<<<<<<<<<", | 4235 "elapsed = %1.2lfms <<<<<<<<<<", |
4236 _task_id, _time_target_ms, elapsed_time_ms); | 4236 _worker_id, _time_target_ms, elapsed_time_ms); |
4237 } | 4237 } |
4238 } | 4238 } |
4239 | 4239 |
4240 _claimed = false; | 4240 _claimed = false; |
4241 } | 4241 } |
4242 | 4242 |
4243 CMTask::CMTask(int task_id, | 4243 CMTask::CMTask(uint worker_id, |
4244 ConcurrentMark* cm, | 4244 ConcurrentMark* cm, |
4245 size_t* marked_bytes, | 4245 size_t* marked_bytes, |
4246 BitMap* card_bm, | 4246 BitMap* card_bm, |
4247 CMTaskQueue* task_queue, | 4247 CMTaskQueue* task_queue, |
4248 CMTaskQueueSet* task_queues) | 4248 CMTaskQueueSet* task_queues) |
4249 : _g1h(G1CollectedHeap::heap()), | 4249 : _g1h(G1CollectedHeap::heap()), |
4250 _task_id(task_id), _cm(cm), | 4250 _worker_id(worker_id), _cm(cm), |
4251 _claimed(false), | 4251 _claimed(false), |
4252 _nextMarkBitMap(NULL), _hash_seed(17), | 4252 _nextMarkBitMap(NULL), _hash_seed(17), |
4253 _task_queue(task_queue), | 4253 _task_queue(task_queue), |
4254 _task_queues(task_queues), | 4254 _task_queues(task_queues), |
4255 _cm_oop_closure(NULL), | 4255 _cm_oop_closure(NULL), |