comparison src/share/vm/gc_implementation/g1/concurrentMark.cpp @ 6862:8a5ea0a9ccc4

7127708: G1: change task num types from int to uint in concurrent mark Summary: Change the type of various task num fields, parameters etc to unsigned and rename them to be more consistent with the other collectors. Code changes were also reviewed by Vitaly Davidovich. Reviewed-by: johnc Contributed-by: Kaushik Srenevasan <kaushik@twitter.com>
author johnc
date Sat, 06 Oct 2012 01:17:44 -0700
parents 988bf00cc564
children 4202510ee0fe
comparison
equal deleted inserted replaced
6830:81e878c53615 6862:8a5ea0a9ccc4
424 _nextMarkBitMap(&_markBitMap2), 424 _nextMarkBitMap(&_markBitMap2),
425 425
426 _markStack(this), 426 _markStack(this),
427 // _finger set in set_non_marking_state 427 // _finger set in set_non_marking_state
428 428
429 _max_task_num(MAX2((uint)ParallelGCThreads, 1U)), 429 _max_worker_id(MAX2((uint)ParallelGCThreads, 1U)),
430 // _active_tasks set in set_non_marking_state 430 // _active_tasks set in set_non_marking_state
431 // _tasks set inside the constructor 431 // _tasks set inside the constructor
432 _task_queues(new CMTaskQueueSet((int) _max_task_num)), 432 _task_queues(new CMTaskQueueSet((int) _max_worker_id)),
433 _terminator(ParallelTaskTerminator((int) _max_task_num, _task_queues)), 433 _terminator(ParallelTaskTerminator((int) _max_worker_id, _task_queues)),
434 434
435 _has_overflown(false), 435 _has_overflown(false),
436 _concurrent(false), 436 _concurrent(false),
437 _has_aborted(false), 437 _has_aborted(false),
438 _restart_for_overflow(false), 438 _restart_for_overflow(false),
479 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set(); 479 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
480 satb_qs.set_buffer_size(G1SATBBufferSize); 480 satb_qs.set_buffer_size(G1SATBBufferSize);
481 481
482 _root_regions.init(_g1h, this); 482 _root_regions.init(_g1h, this);
483 483
484 _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_task_num, mtGC); 484 _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_worker_id, mtGC);
485 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_task_num, mtGC); 485 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_worker_id, mtGC);
486 486
487 _count_card_bitmaps = NEW_C_HEAP_ARRAY(BitMap, _max_task_num, mtGC); 487 _count_card_bitmaps = NEW_C_HEAP_ARRAY(BitMap, _max_worker_id, mtGC);
488 _count_marked_bytes = NEW_C_HEAP_ARRAY(size_t*, _max_task_num, mtGC); 488 _count_marked_bytes = NEW_C_HEAP_ARRAY(size_t*, _max_worker_id, mtGC);
489 489
490 BitMap::idx_t card_bm_size = _card_bm.size(); 490 BitMap::idx_t card_bm_size = _card_bm.size();
491 491
492 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail 492 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail
493 _active_tasks = _max_task_num; 493 _active_tasks = _max_worker_id;
494 for (int i = 0; i < (int) _max_task_num; ++i) { 494 for (uint i = 0; i < _max_worker_id; ++i) {
495 CMTaskQueue* task_queue = new CMTaskQueue(); 495 CMTaskQueue* task_queue = new CMTaskQueue();
496 task_queue->initialize(); 496 task_queue->initialize();
497 _task_queues->register_queue(i, task_queue); 497 _task_queues->register_queue(i, task_queue);
498 498
499 _count_card_bitmaps[i] = BitMap(card_bm_size, false); 499 _count_card_bitmaps[i] = BitMap(card_bm_size, false);
636 } 636 }
637 637
638 // We do reset all of them, since different phases will use 638 // We do reset all of them, since different phases will use
639 // different number of active threads. So, it's easiest to have all 639 // different number of active threads. So, it's easiest to have all
640 // of them ready. 640 // of them ready.
641 for (int i = 0; i < (int) _max_task_num; ++i) { 641 for (uint i = 0; i < _max_worker_id; ++i) {
642 _tasks[i]->reset(_nextMarkBitMap); 642 _tasks[i]->reset(_nextMarkBitMap);
643 } 643 }
644 644
645 // we need this to make sure that the flag is on during the evac 645 // we need this to make sure that the flag is on during the evac
646 // pause with initial mark piggy-backed 646 // pause with initial mark piggy-backed
647 set_concurrent_marking_in_progress(); 647 set_concurrent_marking_in_progress();
648 } 648 }
649 649
650 void ConcurrentMark::set_phase(uint active_tasks, bool concurrent) { 650 void ConcurrentMark::set_phase(uint active_tasks, bool concurrent) {
651 assert(active_tasks <= _max_task_num, "we should not have more"); 651 assert(active_tasks <= _max_worker_id, "we should not have more");
652 652
653 _active_tasks = active_tasks; 653 _active_tasks = active_tasks;
654 // Need to update the three data structures below according to the 654 // Need to update the three data structures below according to the
655 // number of active threads for this phase. 655 // number of active threads for this phase.
656 _terminator = ParallelTaskTerminator((int) active_tasks, _task_queues); 656 _terminator = ParallelTaskTerminator((int) active_tasks, _task_queues);
657 _first_overflow_barrier_sync.set_n_workers((int) active_tasks); 657 _first_overflow_barrier_sync.set_n_workers((int) active_tasks);
658 _second_overflow_barrier_sync.set_n_workers((int) active_tasks); 658 _second_overflow_barrier_sync.set_n_workers((int) active_tasks);
659 659
660 _concurrent = concurrent; 660 _concurrent = concurrent;
661 // We propagate this to all tasks, not just the active ones. 661 // We propagate this to all tasks, not just the active ones.
662 for (int i = 0; i < (int) _max_task_num; ++i) 662 for (uint i = 0; i < _max_worker_id; ++i)
663 _tasks[i]->set_concurrent(concurrent); 663 _tasks[i]->set_concurrent(concurrent);
664 664
665 if (concurrent) { 665 if (concurrent) {
666 set_concurrent_marking_in_progress(); 666 set_concurrent_marking_in_progress();
667 } else { 667 } else {
816 * could occur. This is actually safe, since the entering the sync 816 * could occur. This is actually safe, since the entering the sync
817 * barrier is one of the last things do_marking_step() does, and it 817 * barrier is one of the last things do_marking_step() does, and it
818 * doesn't manipulate any data structures afterwards. 818 * doesn't manipulate any data structures afterwards.
819 */ 819 */
820 820
821 void ConcurrentMark::enter_first_sync_barrier(int task_num) { 821 void ConcurrentMark::enter_first_sync_barrier(uint worker_id) {
822 if (verbose_low()) { 822 if (verbose_low()) {
823 gclog_or_tty->print_cr("[%d] entering first barrier", task_num); 823 gclog_or_tty->print_cr("[%u] entering first barrier", worker_id);
824 } 824 }
825 825
826 if (concurrent()) { 826 if (concurrent()) {
827 ConcurrentGCThread::stsLeave(); 827 ConcurrentGCThread::stsLeave();
828 } 828 }
832 } 832 }
833 // at this point everyone should have synced up and not be doing any 833 // at this point everyone should have synced up and not be doing any
834 // more work 834 // more work
835 835
836 if (verbose_low()) { 836 if (verbose_low()) {
837 gclog_or_tty->print_cr("[%d] leaving first barrier", task_num); 837 gclog_or_tty->print_cr("[%u] leaving first barrier", worker_id);
838 } 838 }
839 839
840 // let task 0 do this 840 // let the task associated with with worker 0 do this
841 if (task_num == 0) { 841 if (worker_id == 0) {
842 // task 0 is responsible for clearing the global data structures 842 // task 0 is responsible for clearing the global data structures
843 // We should be here because of an overflow. During STW we should 843 // We should be here because of an overflow. During STW we should
844 // not clear the overflow flag since we rely on it being true when 844 // not clear the overflow flag since we rely on it being true when
845 // we exit this method to abort the pause and restart concurent 845 // we exit this method to abort the pause and restart concurent
846 // marking. 846 // marking.
856 856
857 // after this, each task should reset its own data structures then 857 // after this, each task should reset its own data structures then
858 // then go into the second barrier 858 // then go into the second barrier
859 } 859 }
860 860
861 void ConcurrentMark::enter_second_sync_barrier(int task_num) { 861 void ConcurrentMark::enter_second_sync_barrier(uint worker_id) {
862 if (verbose_low()) { 862 if (verbose_low()) {
863 gclog_or_tty->print_cr("[%d] entering second barrier", task_num); 863 gclog_or_tty->print_cr("[%u] entering second barrier", worker_id);
864 } 864 }
865 865
866 if (concurrent()) { 866 if (concurrent()) {
867 ConcurrentGCThread::stsLeave(); 867 ConcurrentGCThread::stsLeave();
868 } 868 }
871 ConcurrentGCThread::stsJoin(); 871 ConcurrentGCThread::stsJoin();
872 } 872 }
873 // at this point everything should be re-initialised and ready to go 873 // at this point everything should be re-initialised and ready to go
874 874
875 if (verbose_low()) { 875 if (verbose_low()) {
876 gclog_or_tty->print_cr("[%d] leaving second barrier", task_num); 876 gclog_or_tty->print_cr("[%u] leaving second barrier", worker_id);
877 } 877 }
878 } 878 }
879 879
880 #ifndef PRODUCT 880 #ifndef PRODUCT
881 void ForceOverflowSettings::init() { 881 void ForceOverflowSettings::init() {
2111 2111
2112 template <class T> void do_oop_work(T* p) { 2112 template <class T> void do_oop_work(T* p) {
2113 if (!_cm->has_overflown()) { 2113 if (!_cm->has_overflown()) {
2114 oop obj = oopDesc::load_decode_heap_oop(p); 2114 oop obj = oopDesc::load_decode_heap_oop(p);
2115 if (_cm->verbose_high()) { 2115 if (_cm->verbose_high()) {
2116 gclog_or_tty->print_cr("\t[%d] we're looking at location " 2116 gclog_or_tty->print_cr("\t[%u] we're looking at location "
2117 "*"PTR_FORMAT" = "PTR_FORMAT, 2117 "*"PTR_FORMAT" = "PTR_FORMAT,
2118 _task->task_id(), p, (void*) obj); 2118 _task->worker_id(), p, (void*) obj);
2119 } 2119 }
2120 2120
2121 _task->deal_with_reference(obj); 2121 _task->deal_with_reference(obj);
2122 _ref_counter--; 2122 _ref_counter--;
2123 2123
2142 } while (_task->has_aborted() && !_cm->has_overflown()); 2142 } while (_task->has_aborted() && !_cm->has_overflown());
2143 _ref_counter = _ref_counter_limit; 2143 _ref_counter = _ref_counter_limit;
2144 } 2144 }
2145 } else { 2145 } else {
2146 if (_cm->verbose_high()) { 2146 if (_cm->verbose_high()) {
2147 gclog_or_tty->print_cr("\t[%d] CM Overflow", _task->task_id()); 2147 gclog_or_tty->print_cr("\t[%u] CM Overflow", _task->worker_id());
2148 } 2148 }
2149 } 2149 }
2150 } 2150 }
2151 }; 2151 };
2152 2152
2158 _cm(cm), _task(task) { } 2158 _cm(cm), _task(task) { }
2159 2159
2160 void do_void() { 2160 void do_void() {
2161 do { 2161 do {
2162 if (_cm->verbose_high()) { 2162 if (_cm->verbose_high()) {
2163 gclog_or_tty->print_cr("\t[%d] Drain: Calling do marking_step", 2163 gclog_or_tty->print_cr("\t[%u] Drain: Calling do marking_step",
2164 _task->task_id()); 2164 _task->worker_id());
2165 } 2165 }
2166 2166
2167 // We call CMTask::do_marking_step() to completely drain the local and 2167 // We call CMTask::do_marking_step() to completely drain the local and
2168 // global marking stacks. The routine is called in a loop, which we'll 2168 // global marking stacks. The routine is called in a loop, which we'll
2169 // exit if there's nothing more to do (i.e. we'completely drained the 2169 // exit if there's nothing more to do (i.e. we'completely drained the
2298 g1_drain_mark_stack(this, &_markStack, &g1_keep_alive); 2298 g1_drain_mark_stack(this, &_markStack, &g1_keep_alive);
2299 2299
2300 // We use the work gang from the G1CollectedHeap and we utilize all 2300 // We use the work gang from the G1CollectedHeap and we utilize all
2301 // the worker threads. 2301 // the worker threads.
2302 uint active_workers = g1h->workers() ? g1h->workers()->active_workers() : 1U; 2302 uint active_workers = g1h->workers() ? g1h->workers()->active_workers() : 1U;
2303 active_workers = MAX2(MIN2(active_workers, _max_task_num), 1U); 2303 active_workers = MAX2(MIN2(active_workers, _max_worker_id), 1U);
2304 2304
2305 G1CMRefProcTaskExecutor par_task_executor(g1h, this, 2305 G1CMRefProcTaskExecutor par_task_executor(g1h, this,
2306 g1h->workers(), active_workers); 2306 g1h->workers(), active_workers);
2307 2307
2308 if (rp->processing_is_mt()) { 2308 if (rp->processing_is_mt()) {
2617 clearRangePrevBitmap(mr); 2617 clearRangePrevBitmap(mr);
2618 clearRangeNextBitmap(mr); 2618 clearRangeNextBitmap(mr);
2619 } 2619 }
2620 2620
2621 HeapRegion* 2621 HeapRegion*
2622 ConcurrentMark::claim_region(int task_num) { 2622 ConcurrentMark::claim_region(uint worker_id) {
2623 // "checkpoint" the finger 2623 // "checkpoint" the finger
2624 HeapWord* finger = _finger; 2624 HeapWord* finger = _finger;
2625 2625
2626 // _heap_end will not change underneath our feet; it only changes at 2626 // _heap_end will not change underneath our feet; it only changes at
2627 // yield points. 2627 // yield points.
2655 HeapWord* bottom = curr_region->bottom(); 2655 HeapWord* bottom = curr_region->bottom();
2656 HeapWord* end = curr_region->end(); 2656 HeapWord* end = curr_region->end();
2657 HeapWord* limit = curr_region->next_top_at_mark_start(); 2657 HeapWord* limit = curr_region->next_top_at_mark_start();
2658 2658
2659 if (verbose_low()) { 2659 if (verbose_low()) {
2660 gclog_or_tty->print_cr("[%d] curr_region = "PTR_FORMAT" " 2660 gclog_or_tty->print_cr("[%u] curr_region = "PTR_FORMAT" "
2661 "["PTR_FORMAT", "PTR_FORMAT"), " 2661 "["PTR_FORMAT", "PTR_FORMAT"), "
2662 "limit = "PTR_FORMAT, 2662 "limit = "PTR_FORMAT,
2663 task_num, curr_region, bottom, end, limit); 2663 worker_id, curr_region, bottom, end, limit);
2664 } 2664 }
2665 2665
2666 // Is the gap between reading the finger and doing the CAS too long? 2666 // Is the gap between reading the finger and doing the CAS too long?
2667 HeapWord* res = (HeapWord*) Atomic::cmpxchg_ptr(end, &_finger, finger); 2667 HeapWord* res = (HeapWord*) Atomic::cmpxchg_ptr(end, &_finger, finger);
2668 if (res == finger) { 2668 if (res == finger) {
2671 // notice that _finger == end cannot be guaranteed here since, 2671 // notice that _finger == end cannot be guaranteed here since,
2672 // someone else might have moved the finger even further 2672 // someone else might have moved the finger even further
2673 assert(_finger >= end, "the finger should have moved forward"); 2673 assert(_finger >= end, "the finger should have moved forward");
2674 2674
2675 if (verbose_low()) { 2675 if (verbose_low()) {
2676 gclog_or_tty->print_cr("[%d] we were successful with region = " 2676 gclog_or_tty->print_cr("[%u] we were successful with region = "
2677 PTR_FORMAT, task_num, curr_region); 2677 PTR_FORMAT, worker_id, curr_region);
2678 } 2678 }
2679 2679
2680 if (limit > bottom) { 2680 if (limit > bottom) {
2681 if (verbose_low()) { 2681 if (verbose_low()) {
2682 gclog_or_tty->print_cr("[%d] region "PTR_FORMAT" is not empty, " 2682 gclog_or_tty->print_cr("[%u] region "PTR_FORMAT" is not empty, "
2683 "returning it ", task_num, curr_region); 2683 "returning it ", worker_id, curr_region);
2684 } 2684 }
2685 return curr_region; 2685 return curr_region;
2686 } else { 2686 } else {
2687 assert(limit == bottom, 2687 assert(limit == bottom,
2688 "the region limit should be at bottom"); 2688 "the region limit should be at bottom");
2689 if (verbose_low()) { 2689 if (verbose_low()) {
2690 gclog_or_tty->print_cr("[%d] region "PTR_FORMAT" is empty, " 2690 gclog_or_tty->print_cr("[%u] region "PTR_FORMAT" is empty, "
2691 "returning NULL", task_num, curr_region); 2691 "returning NULL", worker_id, curr_region);
2692 } 2692 }
2693 // we return NULL and the caller should try calling 2693 // we return NULL and the caller should try calling
2694 // claim_region() again. 2694 // claim_region() again.
2695 return NULL; 2695 return NULL;
2696 } 2696 }
2697 } else { 2697 } else {
2698 assert(_finger > finger, "the finger should have moved forward"); 2698 assert(_finger > finger, "the finger should have moved forward");
2699 if (verbose_low()) { 2699 if (verbose_low()) {
2700 gclog_or_tty->print_cr("[%d] somebody else moved the finger, " 2700 gclog_or_tty->print_cr("[%u] somebody else moved the finger, "
2701 "global finger = "PTR_FORMAT", " 2701 "global finger = "PTR_FORMAT", "
2702 "our finger = "PTR_FORMAT, 2702 "our finger = "PTR_FORMAT,
2703 task_num, _finger, finger); 2703 worker_id, _finger, finger);
2704 } 2704 }
2705 2705
2706 // read it again 2706 // read it again
2707 finger = _finger; 2707 finger = _finger;
2708 } 2708 }
2781 // Verify entries on the global mark stack 2781 // Verify entries on the global mark stack
2782 cl.set_phase(VerifyNoCSetOopsStack); 2782 cl.set_phase(VerifyNoCSetOopsStack);
2783 _markStack.oops_do(&cl); 2783 _markStack.oops_do(&cl);
2784 2784
2785 // Verify entries on the task queues 2785 // Verify entries on the task queues
2786 for (int i = 0; i < (int) _max_task_num; i += 1) { 2786 for (uint i = 0; i < _max_worker_id; i += 1) {
2787 cl.set_phase(VerifyNoCSetOopsQueues, i); 2787 cl.set_phase(VerifyNoCSetOopsQueues, i);
2788 OopTaskQueue* queue = _task_queues->queue(i); 2788 OopTaskQueue* queue = _task_queues->queue(i);
2789 queue->oops_do(&cl); 2789 queue->oops_do(&cl);
2790 } 2790 }
2791 } 2791 }
2820 err_msg("global finger: "PTR_FORMAT" region: "HR_FORMAT, 2820 err_msg("global finger: "PTR_FORMAT" region: "HR_FORMAT,
2821 global_finger, HR_FORMAT_PARAMS(global_hr))); 2821 global_finger, HR_FORMAT_PARAMS(global_hr)));
2822 } 2822 }
2823 2823
2824 // Verify the task fingers 2824 // Verify the task fingers
2825 assert(parallel_marking_threads() <= _max_task_num, "sanity"); 2825 assert(parallel_marking_threads() <= _max_worker_id, "sanity");
2826 for (int i = 0; i < (int) parallel_marking_threads(); i += 1) { 2826 for (int i = 0; i < (int) parallel_marking_threads(); i += 1) {
2827 CMTask* task = _tasks[i]; 2827 CMTask* task = _tasks[i];
2828 HeapWord* task_finger = task->finger(); 2828 HeapWord* task_finger = task->finger();
2829 if (task_finger != NULL && task_finger < _heap_end) { 2829 if (task_finger != NULL && task_finger < _heap_end) {
2830 // See above note on the global finger verification. 2830 // See above note on the global finger verification.
2847 } else { 2847 } else {
2848 assert(has_overflown(), "pre-condition"); 2848 assert(has_overflown(), "pre-condition");
2849 } 2849 }
2850 _finger = _heap_start; 2850 _finger = _heap_start;
2851 2851
2852 for (int i = 0; i < (int)_max_task_num; ++i) { 2852 for (uint i = 0; i < _max_worker_id; ++i) {
2853 OopTaskQueue* queue = _task_queues->queue(i); 2853 OopTaskQueue* queue = _task_queues->queue(i);
2854 queue->set_empty(); 2854 queue->set_empty();
2855 } 2855 }
2856 } 2856 }
2857 2857
2860 class AggregateCountDataHRClosure: public HeapRegionClosure { 2860 class AggregateCountDataHRClosure: public HeapRegionClosure {
2861 G1CollectedHeap* _g1h; 2861 G1CollectedHeap* _g1h;
2862 ConcurrentMark* _cm; 2862 ConcurrentMark* _cm;
2863 CardTableModRefBS* _ct_bs; 2863 CardTableModRefBS* _ct_bs;
2864 BitMap* _cm_card_bm; 2864 BitMap* _cm_card_bm;
2865 size_t _max_task_num; 2865 uint _max_worker_id;
2866 2866
2867 public: 2867 public:
2868 AggregateCountDataHRClosure(G1CollectedHeap* g1h, 2868 AggregateCountDataHRClosure(G1CollectedHeap* g1h,
2869 BitMap* cm_card_bm, 2869 BitMap* cm_card_bm,
2870 size_t max_task_num) : 2870 uint max_worker_id) :
2871 _g1h(g1h), _cm(g1h->concurrent_mark()), 2871 _g1h(g1h), _cm(g1h->concurrent_mark()),
2872 _ct_bs((CardTableModRefBS*) (g1h->barrier_set())), 2872 _ct_bs((CardTableModRefBS*) (g1h->barrier_set())),
2873 _cm_card_bm(cm_card_bm), _max_task_num(max_task_num) { } 2873 _cm_card_bm(cm_card_bm), _max_worker_id(max_worker_id) { }
2874 2874
2875 bool doHeapRegion(HeapRegion* hr) { 2875 bool doHeapRegion(HeapRegion* hr) {
2876 if (hr->continuesHumongous()) { 2876 if (hr->continuesHumongous()) {
2877 // We will ignore these here and process them when their 2877 // We will ignore these here and process them when their
2878 // associated "starts humongous" region is processed. 2878 // associated "starts humongous" region is processed.
2925 2925
2926 // Aggregate the "stripe" in the count data associated with hr. 2926 // Aggregate the "stripe" in the count data associated with hr.
2927 uint hrs_index = hr->hrs_index(); 2927 uint hrs_index = hr->hrs_index();
2928 size_t marked_bytes = 0; 2928 size_t marked_bytes = 0;
2929 2929
2930 for (int i = 0; (size_t)i < _max_task_num; i += 1) { 2930 for (uint i = 0; i < _max_worker_id; i += 1) {
2931 size_t* marked_bytes_array = _cm->count_marked_bytes_array_for(i); 2931 size_t* marked_bytes_array = _cm->count_marked_bytes_array_for(i);
2932 BitMap* task_card_bm = _cm->count_card_bitmap_for(i); 2932 BitMap* task_card_bm = _cm->count_card_bitmap_for(i);
2933 2933
2934 // Fetch the marked_bytes in this region for task i and 2934 // Fetch the marked_bytes in this region for task i and
2935 // add it to the running total for this region. 2935 // add it to the running total for this region.
2936 marked_bytes += marked_bytes_array[hrs_index]; 2936 marked_bytes += marked_bytes_array[hrs_index];
2937 2937
2938 // Now union the bitmaps[0,max_task_num)[start_idx..limit_idx) 2938 // Now union the bitmaps[0,max_worker_id)[start_idx..limit_idx)
2939 // into the global card bitmap. 2939 // into the global card bitmap.
2940 BitMap::idx_t scan_idx = task_card_bm->get_next_one_offset(start_idx, limit_idx); 2940 BitMap::idx_t scan_idx = task_card_bm->get_next_one_offset(start_idx, limit_idx);
2941 2941
2942 while (scan_idx < limit_idx) { 2942 while (scan_idx < limit_idx) {
2943 assert(task_card_bm->at(scan_idx) == true, "should be"); 2943 assert(task_card_bm->at(scan_idx) == true, "should be");
2965 class G1AggregateCountDataTask: public AbstractGangTask { 2965 class G1AggregateCountDataTask: public AbstractGangTask {
2966 protected: 2966 protected:
2967 G1CollectedHeap* _g1h; 2967 G1CollectedHeap* _g1h;
2968 ConcurrentMark* _cm; 2968 ConcurrentMark* _cm;
2969 BitMap* _cm_card_bm; 2969 BitMap* _cm_card_bm;
2970 size_t _max_task_num; 2970 uint _max_worker_id;
2971 int _active_workers; 2971 int _active_workers;
2972 2972
2973 public: 2973 public:
2974 G1AggregateCountDataTask(G1CollectedHeap* g1h, 2974 G1AggregateCountDataTask(G1CollectedHeap* g1h,
2975 ConcurrentMark* cm, 2975 ConcurrentMark* cm,
2976 BitMap* cm_card_bm, 2976 BitMap* cm_card_bm,
2977 size_t max_task_num, 2977 uint max_worker_id,
2978 int n_workers) : 2978 int n_workers) :
2979 AbstractGangTask("Count Aggregation"), 2979 AbstractGangTask("Count Aggregation"),
2980 _g1h(g1h), _cm(cm), _cm_card_bm(cm_card_bm), 2980 _g1h(g1h), _cm(cm), _cm_card_bm(cm_card_bm),
2981 _max_task_num(max_task_num), 2981 _max_worker_id(max_worker_id),
2982 _active_workers(n_workers) { } 2982 _active_workers(n_workers) { }
2983 2983
2984 void work(uint worker_id) { 2984 void work(uint worker_id) {
2985 AggregateCountDataHRClosure cl(_g1h, _cm_card_bm, _max_task_num); 2985 AggregateCountDataHRClosure cl(_g1h, _cm_card_bm, _max_worker_id);
2986 2986
2987 if (G1CollectedHeap::use_parallel_gc_threads()) { 2987 if (G1CollectedHeap::use_parallel_gc_threads()) {
2988 _g1h->heap_region_par_iterate_chunked(&cl, worker_id, 2988 _g1h->heap_region_par_iterate_chunked(&cl, worker_id,
2989 _active_workers, 2989 _active_workers,
2990 HeapRegion::AggregateCountClaimValue); 2990 HeapRegion::AggregateCountClaimValue);
2999 int n_workers = (G1CollectedHeap::use_parallel_gc_threads() ? 2999 int n_workers = (G1CollectedHeap::use_parallel_gc_threads() ?
3000 _g1h->workers()->active_workers() : 3000 _g1h->workers()->active_workers() :
3001 1); 3001 1);
3002 3002
3003 G1AggregateCountDataTask g1_par_agg_task(_g1h, this, &_card_bm, 3003 G1AggregateCountDataTask g1_par_agg_task(_g1h, this, &_card_bm,
3004 _max_task_num, n_workers); 3004 _max_worker_id, n_workers);
3005 3005
3006 if (G1CollectedHeap::use_parallel_gc_threads()) { 3006 if (G1CollectedHeap::use_parallel_gc_threads()) {
3007 assert(_g1h->check_heap_region_claim_values(HeapRegion::InitialClaimValue), 3007 assert(_g1h->check_heap_region_claim_values(HeapRegion::InitialClaimValue),
3008 "sanity check"); 3008 "sanity check");
3009 _g1h->set_par_threads(n_workers); 3009 _g1h->set_par_threads(n_workers);
3028 // Clear the global region bitmap - it will be filled as part 3028 // Clear the global region bitmap - it will be filled as part
3029 // of the final counting task. 3029 // of the final counting task.
3030 _region_bm.clear(); 3030 _region_bm.clear();
3031 3031
3032 uint max_regions = _g1h->max_regions(); 3032 uint max_regions = _g1h->max_regions();
3033 assert(_max_task_num != 0, "unitialized"); 3033 assert(_max_worker_id > 0, "uninitialized");
3034 3034
3035 for (int i = 0; (size_t) i < _max_task_num; i += 1) { 3035 for (uint i = 0; i < _max_worker_id; i += 1) {
3036 BitMap* task_card_bm = count_card_bitmap_for(i); 3036 BitMap* task_card_bm = count_card_bitmap_for(i);
3037 size_t* marked_bytes_array = count_marked_bytes_array_for(i); 3037 size_t* marked_bytes_array = count_marked_bytes_array_for(i);
3038 3038
3039 assert(task_card_bm->size() == _card_bm.size(), "size mismatch"); 3039 assert(task_card_bm->size() == _card_bm.size(), "size mismatch");
3040 assert(marked_bytes_array != NULL, "uninitialized"); 3040 assert(marked_bytes_array != NULL, "uninitialized");
3060 _nextMarkBitMap->clearAll(); 3060 _nextMarkBitMap->clearAll();
3061 // Clear the liveness counting data 3061 // Clear the liveness counting data
3062 clear_all_count_data(); 3062 clear_all_count_data();
3063 // Empty mark stack 3063 // Empty mark stack
3064 clear_marking_state(); 3064 clear_marking_state();
3065 for (int i = 0; i < (int)_max_task_num; ++i) { 3065 for (uint i = 0; i < _max_worker_id; ++i) {
3066 _tasks[i]->clear_region_fields(); 3066 _tasks[i]->clear_region_fields();
3067 } 3067 }
3068 _has_aborted = true; 3068 _has_aborted = true;
3069 3069
3070 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); 3070 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
3152 #ifndef PRODUCT 3152 #ifndef PRODUCT
3153 // for debugging purposes 3153 // for debugging purposes
3154 void ConcurrentMark::print_finger() { 3154 void ConcurrentMark::print_finger() {
3155 gclog_or_tty->print_cr("heap ["PTR_FORMAT", "PTR_FORMAT"), global finger = "PTR_FORMAT, 3155 gclog_or_tty->print_cr("heap ["PTR_FORMAT", "PTR_FORMAT"), global finger = "PTR_FORMAT,
3156 _heap_start, _heap_end, _finger); 3156 _heap_start, _heap_end, _finger);
3157 for (int i = 0; i < (int) _max_task_num; ++i) { 3157 for (uint i = 0; i < _max_worker_id; ++i) {
3158 gclog_or_tty->print(" %d: "PTR_FORMAT, i, _tasks[i]->finger()); 3158 gclog_or_tty->print(" %u: "PTR_FORMAT, i, _tasks[i]->finger());
3159 } 3159 }
3160 gclog_or_tty->print_cr(""); 3160 gclog_or_tty->print_cr("");
3161 } 3161 }
3162 #endif 3162 #endif
3163 3163
3164 void CMTask::scan_object(oop obj) { 3164 void CMTask::scan_object(oop obj) {
3165 assert(_nextMarkBitMap->isMarked((HeapWord*) obj), "invariant"); 3165 assert(_nextMarkBitMap->isMarked((HeapWord*) obj), "invariant");
3166 3166
3167 if (_cm->verbose_high()) { 3167 if (_cm->verbose_high()) {
3168 gclog_or_tty->print_cr("[%d] we're scanning object "PTR_FORMAT, 3168 gclog_or_tty->print_cr("[%u] we're scanning object "PTR_FORMAT,
3169 _task_id, (void*) obj); 3169 _worker_id, (void*) obj);
3170 } 3170 }
3171 3171
3172 size_t obj_size = obj->size(); 3172 size_t obj_size = obj->size();
3173 _words_scanned += obj_size; 3173 _words_scanned += obj_size;
3174 3174
3243 "claim_region() should have filtered out continues humongous regions"); 3243 "claim_region() should have filtered out continues humongous regions");
3244 assert(!hr->continuesHumongous(), 3244 assert(!hr->continuesHumongous(),
3245 "claim_region() should have filtered out continues humongous regions"); 3245 "claim_region() should have filtered out continues humongous regions");
3246 3246
3247 if (_cm->verbose_low()) { 3247 if (_cm->verbose_low()) {
3248 gclog_or_tty->print_cr("[%d] setting up for region "PTR_FORMAT, 3248 gclog_or_tty->print_cr("[%u] setting up for region "PTR_FORMAT,
3249 _task_id, hr); 3249 _worker_id, hr);
3250 } 3250 }
3251 3251
3252 _curr_region = hr; 3252 _curr_region = hr;
3253 _finger = hr->bottom(); 3253 _finger = hr->bottom();
3254 update_region_limit(); 3254 update_region_limit();
3259 HeapWord* bottom = hr->bottom(); 3259 HeapWord* bottom = hr->bottom();
3260 HeapWord* limit = hr->next_top_at_mark_start(); 3260 HeapWord* limit = hr->next_top_at_mark_start();
3261 3261
3262 if (limit == bottom) { 3262 if (limit == bottom) {
3263 if (_cm->verbose_low()) { 3263 if (_cm->verbose_low()) {
3264 gclog_or_tty->print_cr("[%d] found an empty region " 3264 gclog_or_tty->print_cr("[%u] found an empty region "
3265 "["PTR_FORMAT", "PTR_FORMAT")", 3265 "["PTR_FORMAT", "PTR_FORMAT")",
3266 _task_id, bottom, limit); 3266 _worker_id, bottom, limit);
3267 } 3267 }
3268 // The region was collected underneath our feet. 3268 // The region was collected underneath our feet.
3269 // We set the finger to bottom to ensure that the bitmap 3269 // We set the finger to bottom to ensure that the bitmap
3270 // iteration that will follow this will not do anything. 3270 // iteration that will follow this will not do anything.
3271 // (this is not a condition that holds when we set the region up, 3271 // (this is not a condition that holds when we set the region up,
3292 } 3292 }
3293 3293
3294 void CMTask::giveup_current_region() { 3294 void CMTask::giveup_current_region() {
3295 assert(_curr_region != NULL, "invariant"); 3295 assert(_curr_region != NULL, "invariant");
3296 if (_cm->verbose_low()) { 3296 if (_cm->verbose_low()) {
3297 gclog_or_tty->print_cr("[%d] giving up region "PTR_FORMAT, 3297 gclog_or_tty->print_cr("[%u] giving up region "PTR_FORMAT,
3298 _task_id, _curr_region); 3298 _worker_id, _curr_region);
3299 } 3299 }
3300 clear_region_fields(); 3300 clear_region_fields();
3301 } 3301 }
3302 3302
3303 void CMTask::clear_region_fields() { 3303 void CMTask::clear_region_fields() {
3319 3319
3320 void CMTask::reset(CMBitMap* nextMarkBitMap) { 3320 void CMTask::reset(CMBitMap* nextMarkBitMap) {
3321 guarantee(nextMarkBitMap != NULL, "invariant"); 3321 guarantee(nextMarkBitMap != NULL, "invariant");
3322 3322
3323 if (_cm->verbose_low()) { 3323 if (_cm->verbose_low()) {
3324 gclog_or_tty->print_cr("[%d] resetting", _task_id); 3324 gclog_or_tty->print_cr("[%u] resetting", _worker_id);
3325 } 3325 }
3326 3326
3327 _nextMarkBitMap = nextMarkBitMap; 3327 _nextMarkBitMap = nextMarkBitMap;
3328 clear_region_fields(); 3328 clear_region_fields();
3329 3329
3413 double last_interval_ms = curr_time_ms - _interval_start_time_ms; 3413 double last_interval_ms = curr_time_ms - _interval_start_time_ms;
3414 _interval_start_time_ms = curr_time_ms; 3414 _interval_start_time_ms = curr_time_ms;
3415 _all_clock_intervals_ms.add(last_interval_ms); 3415 _all_clock_intervals_ms.add(last_interval_ms);
3416 3416
3417 if (_cm->verbose_medium()) { 3417 if (_cm->verbose_medium()) {
3418 gclog_or_tty->print_cr("[%d] regular clock, interval = %1.2lfms, " 3418 gclog_or_tty->print_cr("[%u] regular clock, interval = %1.2lfms, "
3419 "scanned = %d%s, refs reached = %d%s", 3419 "scanned = %d%s, refs reached = %d%s",
3420 _task_id, last_interval_ms, 3420 _worker_id, last_interval_ms,
3421 _words_scanned, 3421 _words_scanned,
3422 (_words_scanned >= _words_scanned_limit) ? " (*)" : "", 3422 (_words_scanned >= _words_scanned_limit) ? " (*)" : "",
3423 _refs_reached, 3423 _refs_reached,
3424 (_refs_reached >= _refs_reached_limit) ? " (*)" : ""); 3424 (_refs_reached >= _refs_reached_limit) ? " (*)" : "");
3425 } 3425 }
3447 // (6) Finally, we check whether there are enough completed STAB 3447 // (6) Finally, we check whether there are enough completed STAB
3448 // buffers available for processing. If there are, we abort. 3448 // buffers available for processing. If there are, we abort.
3449 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); 3449 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
3450 if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) { 3450 if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) {
3451 if (_cm->verbose_low()) { 3451 if (_cm->verbose_low()) {
3452 gclog_or_tty->print_cr("[%d] aborting to deal with pending SATB buffers", 3452 gclog_or_tty->print_cr("[%u] aborting to deal with pending SATB buffers",
3453 _task_id); 3453 _worker_id);
3454 } 3454 }
3455 // we do need to process SATB buffers, we'll abort and restart 3455 // we do need to process SATB buffers, we'll abort and restart
3456 // the marking task to do so 3456 // the marking task to do so
3457 set_has_aborted(); 3457 set_has_aborted();
3458 statsOnly( ++_aborted_satb ); 3458 statsOnly( ++_aborted_satb );
3473 // operation which will increase the per byte scanned cost (i.e. move 3473 // operation which will increase the per byte scanned cost (i.e. move
3474 // entries to/from the global stack). It basically tries to decrease the 3474 // entries to/from the global stack). It basically tries to decrease the
3475 // scanning limit so that the clock is called earlier. 3475 // scanning limit so that the clock is called earlier.
3476 3476
3477 if (_cm->verbose_medium()) { 3477 if (_cm->verbose_medium()) {
3478 gclog_or_tty->print_cr("[%d] decreasing limits", _task_id); 3478 gclog_or_tty->print_cr("[%u] decreasing limits", _worker_id);
3479 } 3479 }
3480 3480
3481 _words_scanned_limit = _real_words_scanned_limit - 3481 _words_scanned_limit = _real_words_scanned_limit -
3482 3 * words_scanned_period / 4; 3482 3 * words_scanned_period / 4;
3483 _refs_reached_limit = _real_refs_reached_limit - 3483 _refs_reached_limit = _real_refs_reached_limit -
3501 3501
3502 statsOnly( ++_global_transfers_to; _local_pops += n ); 3502 statsOnly( ++_global_transfers_to; _local_pops += n );
3503 3503
3504 if (!_cm->mark_stack_push(buffer, n)) { 3504 if (!_cm->mark_stack_push(buffer, n)) {
3505 if (_cm->verbose_low()) { 3505 if (_cm->verbose_low()) {
3506 gclog_or_tty->print_cr("[%d] aborting due to global stack overflow", 3506 gclog_or_tty->print_cr("[%u] aborting due to global stack overflow",
3507 _task_id); 3507 _worker_id);
3508 } 3508 }
3509 set_has_aborted(); 3509 set_has_aborted();
3510 } else { 3510 } else {
3511 // the transfer was successful 3511 // the transfer was successful
3512 3512
3513 if (_cm->verbose_medium()) { 3513 if (_cm->verbose_medium()) {
3514 gclog_or_tty->print_cr("[%d] pushed %d entries to the global stack", 3514 gclog_or_tty->print_cr("[%u] pushed %d entries to the global stack",
3515 _task_id, n); 3515 _worker_id, n);
3516 } 3516 }
3517 statsOnly( int tmp_size = _cm->mark_stack_size(); 3517 statsOnly( int tmp_size = _cm->mark_stack_size();
3518 if (tmp_size > _global_max_size) { 3518 if (tmp_size > _global_max_size) {
3519 _global_max_size = tmp_size; 3519 _global_max_size = tmp_size;
3520 } 3520 }
3537 if (n > 0) { 3537 if (n > 0) {
3538 // yes, we did actually pop at least one entry 3538 // yes, we did actually pop at least one entry
3539 3539
3540 statsOnly( ++_global_transfers_from; _global_pops += n ); 3540 statsOnly( ++_global_transfers_from; _global_pops += n );
3541 if (_cm->verbose_medium()) { 3541 if (_cm->verbose_medium()) {
3542 gclog_or_tty->print_cr("[%d] popped %d entries from the global stack", 3542 gclog_or_tty->print_cr("[%u] popped %d entries from the global stack",
3543 _task_id, n); 3543 _worker_id, n);
3544 } 3544 }
3545 for (int i = 0; i < n; ++i) { 3545 for (int i = 0; i < n; ++i) {
3546 bool success = _task_queue->push(buffer[i]); 3546 bool success = _task_queue->push(buffer[i]);
3547 // We only call this when the local queue is empty or under a 3547 // We only call this when the local queue is empty or under a
3548 // given target limit. So, we do not expect this push to fail. 3548 // given target limit. So, we do not expect this push to fail.
3573 target_size = 0; 3573 target_size = 0;
3574 } 3574 }
3575 3575
3576 if (_task_queue->size() > target_size) { 3576 if (_task_queue->size() > target_size) {
3577 if (_cm->verbose_high()) { 3577 if (_cm->verbose_high()) {
3578 gclog_or_tty->print_cr("[%d] draining local queue, target size = %d", 3578 gclog_or_tty->print_cr("[%u] draining local queue, target size = %d",
3579 _task_id, target_size); 3579 _worker_id, target_size);
3580 } 3580 }
3581 3581
3582 oop obj; 3582 oop obj;
3583 bool ret = _task_queue->pop_local(obj); 3583 bool ret = _task_queue->pop_local(obj);
3584 while (ret) { 3584 while (ret) {
3585 statsOnly( ++_local_pops ); 3585 statsOnly( ++_local_pops );
3586 3586
3587 if (_cm->verbose_high()) { 3587 if (_cm->verbose_high()) {
3588 gclog_or_tty->print_cr("[%d] popped "PTR_FORMAT, _task_id, 3588 gclog_or_tty->print_cr("[%u] popped "PTR_FORMAT, _worker_id,
3589 (void*) obj); 3589 (void*) obj);
3590 } 3590 }
3591 3591
3592 assert(_g1h->is_in_g1_reserved((HeapWord*) obj), "invariant" ); 3592 assert(_g1h->is_in_g1_reserved((HeapWord*) obj), "invariant" );
3593 assert(!_g1h->is_on_master_free_list( 3593 assert(!_g1h->is_on_master_free_list(
3601 ret = _task_queue->pop_local(obj); 3601 ret = _task_queue->pop_local(obj);
3602 } 3602 }
3603 } 3603 }
3604 3604
3605 if (_cm->verbose_high()) { 3605 if (_cm->verbose_high()) {
3606 gclog_or_tty->print_cr("[%d] drained local queue, size = %d", 3606 gclog_or_tty->print_cr("[%u] drained local queue, size = %d",
3607 _task_id, _task_queue->size()); 3607 _worker_id, _task_queue->size());
3608 } 3608 }
3609 } 3609 }
3610 } 3610 }
3611 3611
3612 void CMTask::drain_global_stack(bool partially) { 3612 void CMTask::drain_global_stack(bool partially) {
3629 target_size = 0; 3629 target_size = 0;
3630 } 3630 }
3631 3631
3632 if (_cm->mark_stack_size() > target_size) { 3632 if (_cm->mark_stack_size() > target_size) {
3633 if (_cm->verbose_low()) { 3633 if (_cm->verbose_low()) {
3634 gclog_or_tty->print_cr("[%d] draining global_stack, target size %d", 3634 gclog_or_tty->print_cr("[%u] draining global_stack, target size %d",
3635 _task_id, target_size); 3635 _worker_id, target_size);
3636 } 3636 }
3637 3637
3638 while (!has_aborted() && _cm->mark_stack_size() > target_size) { 3638 while (!has_aborted() && _cm->mark_stack_size() > target_size) {
3639 get_entries_from_global_stack(); 3639 get_entries_from_global_stack();
3640 drain_local_queue(partially); 3640 drain_local_queue(partially);
3641 } 3641 }
3642 3642
3643 if (_cm->verbose_low()) { 3643 if (_cm->verbose_low()) {
3644 gclog_or_tty->print_cr("[%d] drained global stack, size = %d", 3644 gclog_or_tty->print_cr("[%u] drained global stack, size = %d",
3645 _task_id, _cm->mark_stack_size()); 3645 _worker_id, _cm->mark_stack_size());
3646 } 3646 }
3647 } 3647 }
3648 } 3648 }
3649 3649
3650 // SATB Queue has several assumptions on whether to call the par or 3650 // SATB Queue has several assumptions on whether to call the par or
3661 _draining_satb_buffers = true; 3661 _draining_satb_buffers = true;
3662 3662
3663 CMObjectClosure oc(this); 3663 CMObjectClosure oc(this);
3664 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); 3664 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
3665 if (G1CollectedHeap::use_parallel_gc_threads()) { 3665 if (G1CollectedHeap::use_parallel_gc_threads()) {
3666 satb_mq_set.set_par_closure(_task_id, &oc); 3666 satb_mq_set.set_par_closure(_worker_id, &oc);
3667 } else { 3667 } else {
3668 satb_mq_set.set_closure(&oc); 3668 satb_mq_set.set_closure(&oc);
3669 } 3669 }
3670 3670
3671 // This keeps claiming and applying the closure to completed buffers 3671 // This keeps claiming and applying the closure to completed buffers
3672 // until we run out of buffers or we need to abort. 3672 // until we run out of buffers or we need to abort.
3673 if (G1CollectedHeap::use_parallel_gc_threads()) { 3673 if (G1CollectedHeap::use_parallel_gc_threads()) {
3674 while (!has_aborted() && 3674 while (!has_aborted() &&
3675 satb_mq_set.par_apply_closure_to_completed_buffer(_task_id)) { 3675 satb_mq_set.par_apply_closure_to_completed_buffer(_worker_id)) {
3676 if (_cm->verbose_medium()) { 3676 if (_cm->verbose_medium()) {
3677 gclog_or_tty->print_cr("[%d] processed an SATB buffer", _task_id); 3677 gclog_or_tty->print_cr("[%u] processed an SATB buffer", _worker_id);
3678 } 3678 }
3679 statsOnly( ++_satb_buffers_processed ); 3679 statsOnly( ++_satb_buffers_processed );
3680 regular_clock_call(); 3680 regular_clock_call();
3681 } 3681 }
3682 } else { 3682 } else {
3683 while (!has_aborted() && 3683 while (!has_aborted() &&
3684 satb_mq_set.apply_closure_to_completed_buffer()) { 3684 satb_mq_set.apply_closure_to_completed_buffer()) {
3685 if (_cm->verbose_medium()) { 3685 if (_cm->verbose_medium()) {
3686 gclog_or_tty->print_cr("[%d] processed an SATB buffer", _task_id); 3686 gclog_or_tty->print_cr("[%u] processed an SATB buffer", _worker_id);
3687 } 3687 }
3688 statsOnly( ++_satb_buffers_processed ); 3688 statsOnly( ++_satb_buffers_processed );
3689 regular_clock_call(); 3689 regular_clock_call();
3690 } 3690 }
3691 } 3691 }
3692 3692
3693 if (!concurrent() && !has_aborted()) { 3693 if (!concurrent() && !has_aborted()) {
3694 // We should only do this during remark. 3694 // We should only do this during remark.
3695 if (G1CollectedHeap::use_parallel_gc_threads()) { 3695 if (G1CollectedHeap::use_parallel_gc_threads()) {
3696 satb_mq_set.par_iterate_closure_all_threads(_task_id); 3696 satb_mq_set.par_iterate_closure_all_threads(_worker_id);
3697 } else { 3697 } else {
3698 satb_mq_set.iterate_closure_all_threads(); 3698 satb_mq_set.iterate_closure_all_threads();
3699 } 3699 }
3700 } 3700 }
3701 3701
3704 assert(has_aborted() || 3704 assert(has_aborted() ||
3705 concurrent() || 3705 concurrent() ||
3706 satb_mq_set.completed_buffers_num() == 0, "invariant"); 3706 satb_mq_set.completed_buffers_num() == 0, "invariant");
3707 3707
3708 if (G1CollectedHeap::use_parallel_gc_threads()) { 3708 if (G1CollectedHeap::use_parallel_gc_threads()) {
3709 satb_mq_set.set_par_closure(_task_id, NULL); 3709 satb_mq_set.set_par_closure(_worker_id, NULL);
3710 } else { 3710 } else {
3711 satb_mq_set.set_closure(NULL); 3711 satb_mq_set.set_closure(NULL);
3712 } 3712 }
3713 3713
3714 // again, this was a potentially expensive operation, decrease the 3714 // again, this was a potentially expensive operation, decrease the
3715 // limits to get the regular clock call early 3715 // limits to get the regular clock call early
3716 decrease_limits(); 3716 decrease_limits();
3717 } 3717 }
3718 3718
3719 void CMTask::print_stats() { 3719 void CMTask::print_stats() {
3720 gclog_or_tty->print_cr("Marking Stats, task = %d, calls = %d", 3720 gclog_or_tty->print_cr("Marking Stats, task = %u, calls = %d",
3721 _task_id, _calls); 3721 _worker_id, _calls);
3722 gclog_or_tty->print_cr(" Elapsed time = %1.2lfms, Termination time = %1.2lfms", 3722 gclog_or_tty->print_cr(" Elapsed time = %1.2lfms, Termination time = %1.2lfms",
3723 _elapsed_time_ms, _termination_time_ms); 3723 _elapsed_time_ms, _termination_time_ms);
3724 gclog_or_tty->print_cr(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms", 3724 gclog_or_tty->print_cr(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
3725 _step_times_ms.num(), _step_times_ms.avg(), 3725 _step_times_ms.num(), _step_times_ms.avg(),
3726 _step_times_ms.sd()); 3726 _step_times_ms.sd());
3864 assert(concurrent() == _cm->concurrent(), "they should be the same"); 3864 assert(concurrent() == _cm->concurrent(), "they should be the same");
3865 3865
3866 G1CollectorPolicy* g1_policy = _g1h->g1_policy(); 3866 G1CollectorPolicy* g1_policy = _g1h->g1_policy();
3867 assert(_task_queues != NULL, "invariant"); 3867 assert(_task_queues != NULL, "invariant");
3868 assert(_task_queue != NULL, "invariant"); 3868 assert(_task_queue != NULL, "invariant");
3869 assert(_task_queues->queue(_task_id) == _task_queue, "invariant"); 3869 assert(_task_queues->queue(_worker_id) == _task_queue, "invariant");
3870 3870
3871 assert(!_claimed, 3871 assert(!_claimed,
3872 "only one thread should claim this task at any one time"); 3872 "only one thread should claim this task at any one time");
3873 3873
3874 // OK, this doesn't safeguard again all possible scenarios, as it is 3874 // OK, this doesn't safeguard again all possible scenarios, as it is
3896 _draining_satb_buffers = false; 3896 _draining_satb_buffers = false;
3897 3897
3898 ++_calls; 3898 ++_calls;
3899 3899
3900 if (_cm->verbose_low()) { 3900 if (_cm->verbose_low()) {
3901 gclog_or_tty->print_cr("[%d] >>>>>>>>>> START, call = %d, " 3901 gclog_or_tty->print_cr("[%u] >>>>>>>>>> START, call = %d, "
3902 "target = %1.2lfms >>>>>>>>>>", 3902 "target = %1.2lfms >>>>>>>>>>",
3903 _task_id, _calls, _time_target_ms); 3903 _worker_id, _calls, _time_target_ms);
3904 } 3904 }
3905 3905
3906 // Set up the bitmap and oop closures. Anything that uses them is 3906 // Set up the bitmap and oop closures. Anything that uses them is
3907 // eventually called from this method, so it is OK to allocate these 3907 // eventually called from this method, so it is OK to allocate these
3908 // statically. 3908 // statically.
3946 // the address where we last found a marked object. If this is a 3946 // the address where we last found a marked object. If this is a
3947 // fresh region, _finger points to start(). 3947 // fresh region, _finger points to start().
3948 MemRegion mr = MemRegion(_finger, _region_limit); 3948 MemRegion mr = MemRegion(_finger, _region_limit);
3949 3949
3950 if (_cm->verbose_low()) { 3950 if (_cm->verbose_low()) {
3951 gclog_or_tty->print_cr("[%d] we're scanning part " 3951 gclog_or_tty->print_cr("[%u] we're scanning part "
3952 "["PTR_FORMAT", "PTR_FORMAT") " 3952 "["PTR_FORMAT", "PTR_FORMAT") "
3953 "of region "PTR_FORMAT, 3953 "of region "PTR_FORMAT,
3954 _task_id, _finger, _region_limit, _curr_region); 3954 _worker_id, _finger, _region_limit, _curr_region);
3955 } 3955 }
3956 3956
3957 // Let's iterate over the bitmap of the part of the 3957 // Let's iterate over the bitmap of the part of the
3958 // region that is left. 3958 // region that is left.
3959 if (mr.is_empty() || _nextMarkBitMap->iterate(&bitmap_closure, mr)) { 3959 if (mr.is_empty() || _nextMarkBitMap->iterate(&bitmap_closure, mr)) {
4005 // Separated the asserts so that we know which one fires. 4005 // Separated the asserts so that we know which one fires.
4006 assert(_curr_region == NULL, "invariant"); 4006 assert(_curr_region == NULL, "invariant");
4007 assert(_finger == NULL, "invariant"); 4007 assert(_finger == NULL, "invariant");
4008 assert(_region_limit == NULL, "invariant"); 4008 assert(_region_limit == NULL, "invariant");
4009 if (_cm->verbose_low()) { 4009 if (_cm->verbose_low()) {
4010 gclog_or_tty->print_cr("[%d] trying to claim a new region", _task_id); 4010 gclog_or_tty->print_cr("[%u] trying to claim a new region", _worker_id);
4011 } 4011 }
4012 HeapRegion* claimed_region = _cm->claim_region(_task_id); 4012 HeapRegion* claimed_region = _cm->claim_region(_worker_id);
4013 if (claimed_region != NULL) { 4013 if (claimed_region != NULL) {
4014 // Yes, we managed to claim one 4014 // Yes, we managed to claim one
4015 statsOnly( ++_regions_claimed ); 4015 statsOnly( ++_regions_claimed );
4016 4016
4017 if (_cm->verbose_low()) { 4017 if (_cm->verbose_low()) {
4018 gclog_or_tty->print_cr("[%d] we successfully claimed " 4018 gclog_or_tty->print_cr("[%u] we successfully claimed "
4019 "region "PTR_FORMAT, 4019 "region "PTR_FORMAT,
4020 _task_id, claimed_region); 4020 _worker_id, claimed_region);
4021 } 4021 }
4022 4022
4023 setup_for_region(claimed_region); 4023 setup_for_region(claimed_region);
4024 assert(_curr_region == claimed_region, "invariant"); 4024 assert(_curr_region == claimed_region, "invariant");
4025 } 4025 }
4042 // tasks might be pushing objects to it concurrently. 4042 // tasks might be pushing objects to it concurrently.
4043 assert(_cm->out_of_regions(), 4043 assert(_cm->out_of_regions(),
4044 "at this point we should be out of regions"); 4044 "at this point we should be out of regions");
4045 4045
4046 if (_cm->verbose_low()) { 4046 if (_cm->verbose_low()) {
4047 gclog_or_tty->print_cr("[%d] all regions claimed", _task_id); 4047 gclog_or_tty->print_cr("[%u] all regions claimed", _worker_id);
4048 } 4048 }
4049 4049
4050 // Try to reduce the number of available SATB buffers so that 4050 // Try to reduce the number of available SATB buffers so that
4051 // remark has less work to do. 4051 // remark has less work to do.
4052 drain_satb_buffers(); 4052 drain_satb_buffers();
4066 // tasks might be pushing objects to it concurrently. 4066 // tasks might be pushing objects to it concurrently.
4067 assert(_cm->out_of_regions() && _task_queue->size() == 0, 4067 assert(_cm->out_of_regions() && _task_queue->size() == 0,
4068 "only way to reach here"); 4068 "only way to reach here");
4069 4069
4070 if (_cm->verbose_low()) { 4070 if (_cm->verbose_low()) {
4071 gclog_or_tty->print_cr("[%d] starting to steal", _task_id); 4071 gclog_or_tty->print_cr("[%u] starting to steal", _worker_id);
4072 } 4072 }
4073 4073
4074 while (!has_aborted()) { 4074 while (!has_aborted()) {
4075 oop obj; 4075 oop obj;
4076 statsOnly( ++_steal_attempts ); 4076 statsOnly( ++_steal_attempts );
4077 4077
4078 if (_cm->try_stealing(_task_id, &_hash_seed, obj)) { 4078 if (_cm->try_stealing(_worker_id, &_hash_seed, obj)) {
4079 if (_cm->verbose_medium()) { 4079 if (_cm->verbose_medium()) {
4080 gclog_or_tty->print_cr("[%d] stolen "PTR_FORMAT" successfully", 4080 gclog_or_tty->print_cr("[%u] stolen "PTR_FORMAT" successfully",
4081 _task_id, (void*) obj); 4081 _worker_id, (void*) obj);
4082 } 4082 }
4083 4083
4084 statsOnly( ++_steals ); 4084 statsOnly( ++_steals );
4085 4085
4086 assert(_nextMarkBitMap->isMarked((HeapWord*) obj), 4086 assert(_nextMarkBitMap->isMarked((HeapWord*) obj),
4114 // Separated the asserts so that we know which one fires. 4114 // Separated the asserts so that we know which one fires.
4115 assert(_cm->out_of_regions(), "only way to reach here"); 4115 assert(_cm->out_of_regions(), "only way to reach here");
4116 assert(_task_queue->size() == 0, "only way to reach here"); 4116 assert(_task_queue->size() == 0, "only way to reach here");
4117 4117
4118 if (_cm->verbose_low()) { 4118 if (_cm->verbose_low()) {
4119 gclog_or_tty->print_cr("[%d] starting termination protocol", _task_id); 4119 gclog_or_tty->print_cr("[%u] starting termination protocol", _worker_id);
4120 } 4120 }
4121 4121
4122 _termination_start_time_ms = os::elapsedVTime() * 1000.0; 4122 _termination_start_time_ms = os::elapsedVTime() * 1000.0;
4123 // The CMTask class also extends the TerminatorTerminator class, 4123 // The CMTask class also extends the TerminatorTerminator class,
4124 // hence its should_exit_termination() method will also decide 4124 // hence its should_exit_termination() method will also decide
4129 termination_end_time_ms - _termination_start_time_ms; 4129 termination_end_time_ms - _termination_start_time_ms;
4130 4130
4131 if (finished) { 4131 if (finished) {
4132 // We're all done. 4132 // We're all done.
4133 4133
4134 if (_task_id == 0) { 4134 if (_worker_id == 0) {
4135 // let's allow task 0 to do this 4135 // let's allow task 0 to do this
4136 if (concurrent()) { 4136 if (concurrent()) {
4137 assert(_cm->concurrent_marking_in_progress(), "invariant"); 4137 assert(_cm->concurrent_marking_in_progress(), "invariant");
4138 // we need to set this to false before the next 4138 // we need to set this to false before the next
4139 // safepoint. This way we ensure that the marking phase 4139 // safepoint. This way we ensure that the marking phase
4151 guarantee(_task_queue->size() == 0, "only way to reach here"); 4151 guarantee(_task_queue->size() == 0, "only way to reach here");
4152 guarantee(!_cm->has_overflown(), "only way to reach here"); 4152 guarantee(!_cm->has_overflown(), "only way to reach here");
4153 guarantee(!_cm->mark_stack_overflow(), "only way to reach here"); 4153 guarantee(!_cm->mark_stack_overflow(), "only way to reach here");
4154 4154
4155 if (_cm->verbose_low()) { 4155 if (_cm->verbose_low()) {
4156 gclog_or_tty->print_cr("[%d] all tasks terminated", _task_id); 4156 gclog_or_tty->print_cr("[%u] all tasks terminated", _worker_id);
4157 } 4157 }
4158 } else { 4158 } else {
4159 // Apparently there's more work to do. Let's abort this task. It 4159 // Apparently there's more work to do. Let's abort this task. It
4160 // will restart it and we can hopefully find more things to do. 4160 // will restart it and we can hopefully find more things to do.
4161 4161
4162 if (_cm->verbose_low()) { 4162 if (_cm->verbose_low()) {
4163 gclog_or_tty->print_cr("[%d] apparently there is more work to do", 4163 gclog_or_tty->print_cr("[%u] apparently there is more work to do",
4164 _task_id); 4164 _worker_id);
4165 } 4165 }
4166 4166
4167 set_has_aborted(); 4167 set_has_aborted();
4168 statsOnly( ++_aborted_termination ); 4168 statsOnly( ++_aborted_termination );
4169 } 4169 }
4198 // order to do this we have to make sure that all tasks stop 4198 // order to do this we have to make sure that all tasks stop
4199 // what they are doing and re-initialise in a safe manner. We 4199 // what they are doing and re-initialise in a safe manner. We
4200 // will achieve this with the use of two barrier sync points. 4200 // will achieve this with the use of two barrier sync points.
4201 4201
4202 if (_cm->verbose_low()) { 4202 if (_cm->verbose_low()) {
4203 gclog_or_tty->print_cr("[%d] detected overflow", _task_id); 4203 gclog_or_tty->print_cr("[%u] detected overflow", _worker_id);
4204 } 4204 }
4205 4205
4206 _cm->enter_first_sync_barrier(_task_id); 4206 _cm->enter_first_sync_barrier(_worker_id);
4207 // When we exit this sync barrier we know that all tasks have 4207 // When we exit this sync barrier we know that all tasks have
4208 // stopped doing marking work. So, it's now safe to 4208 // stopped doing marking work. So, it's now safe to
4209 // re-initialise our data structures. At the end of this method, 4209 // re-initialise our data structures. At the end of this method,
4210 // task 0 will clear the global data structures. 4210 // task 0 will clear the global data structures.
4211 4211
4213 4213
4214 // We clear the local state of this task... 4214 // We clear the local state of this task...
4215 clear_region_fields(); 4215 clear_region_fields();
4216 4216
4217 // ...and enter the second barrier. 4217 // ...and enter the second barrier.
4218 _cm->enter_second_sync_barrier(_task_id); 4218 _cm->enter_second_sync_barrier(_worker_id);
4219 // At this point everything has bee re-initialised and we're 4219 // At this point everything has bee re-initialised and we're
4220 // ready to restart. 4220 // ready to restart.
4221 } 4221 }
4222 4222
4223 if (_cm->verbose_low()) { 4223 if (_cm->verbose_low()) {
4224 gclog_or_tty->print_cr("[%d] <<<<<<<<<< ABORTING, target = %1.2lfms, " 4224 gclog_or_tty->print_cr("[%u] <<<<<<<<<< ABORTING, target = %1.2lfms, "
4225 "elapsed = %1.2lfms <<<<<<<<<<", 4225 "elapsed = %1.2lfms <<<<<<<<<<",
4226 _task_id, _time_target_ms, elapsed_time_ms); 4226 _worker_id, _time_target_ms, elapsed_time_ms);
4227 if (_cm->has_aborted()) { 4227 if (_cm->has_aborted()) {
4228 gclog_or_tty->print_cr("[%d] ========== MARKING ABORTED ==========", 4228 gclog_or_tty->print_cr("[%u] ========== MARKING ABORTED ==========",
4229 _task_id); 4229 _worker_id);
4230 } 4230 }
4231 } 4231 }
4232 } else { 4232 } else {
4233 if (_cm->verbose_low()) { 4233 if (_cm->verbose_low()) {
4234 gclog_or_tty->print_cr("[%d] <<<<<<<<<< FINISHED, target = %1.2lfms, " 4234 gclog_or_tty->print_cr("[%u] <<<<<<<<<< FINISHED, target = %1.2lfms, "
4235 "elapsed = %1.2lfms <<<<<<<<<<", 4235 "elapsed = %1.2lfms <<<<<<<<<<",
4236 _task_id, _time_target_ms, elapsed_time_ms); 4236 _worker_id, _time_target_ms, elapsed_time_ms);
4237 } 4237 }
4238 } 4238 }
4239 4239
4240 _claimed = false; 4240 _claimed = false;
4241 } 4241 }
4242 4242
4243 CMTask::CMTask(int task_id, 4243 CMTask::CMTask(uint worker_id,
4244 ConcurrentMark* cm, 4244 ConcurrentMark* cm,
4245 size_t* marked_bytes, 4245 size_t* marked_bytes,
4246 BitMap* card_bm, 4246 BitMap* card_bm,
4247 CMTaskQueue* task_queue, 4247 CMTaskQueue* task_queue,
4248 CMTaskQueueSet* task_queues) 4248 CMTaskQueueSet* task_queues)
4249 : _g1h(G1CollectedHeap::heap()), 4249 : _g1h(G1CollectedHeap::heap()),
4250 _task_id(task_id), _cm(cm), 4250 _worker_id(worker_id), _cm(cm),
4251 _claimed(false), 4251 _claimed(false),
4252 _nextMarkBitMap(NULL), _hash_seed(17), 4252 _nextMarkBitMap(NULL), _hash_seed(17),
4253 _task_queue(task_queue), 4253 _task_queue(task_queue),
4254 _task_queues(task_queues), 4254 _task_queues(task_queues),
4255 _cm_oop_closure(NULL), 4255 _cm_oop_closure(NULL),