# HG changeset patch # User johnc # Date 1271955758 25200 # Node ID 1316cec51b4dcd5b175d5ce6f91074eea08f8235 # Parent 6ecb6e6de3d6dd661774674ed37d31a9af0937df 6819061: G1: eliminate serial Other times that are proportional to the collection set length 6871109: G1: remove the concept of the scan only prefix Summary: Removed scan only regions and associated code. The young portion of the collection set is now constructed incrementally - when a young region is retired as the current allocation region it is added to the collection set. Reviewed-by: apetrusenko, iveresov, tonyp diff -r 6ecb6e6de3d6 -r 1316cec51b4d src/share/vm/gc_implementation/g1/concurrentG1RefineThread.cpp --- a/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.cpp Mon Apr 19 05:40:21 2010 -0700 +++ b/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.cpp Thu Apr 22 10:02:38 2010 -0700 @@ -1,5 +1,5 @@ /* - * Copyright 2001-2009 Sun Microsystems, Inc. All Rights Reserved. + * Copyright 2001-2010 Sun Microsystems, Inc. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -69,9 +69,9 @@ G1CollectorPolicy* g1p = g1h->g1_policy(); if (g1p->adaptive_young_list_length()) { int regions_visited = 0; - g1h->young_list_rs_length_sampling_init(); - while (g1h->young_list_rs_length_sampling_more()) { - g1h->young_list_rs_length_sampling_next(); + g1h->young_list()->rs_length_sampling_init(); + while (g1h->young_list()->rs_length_sampling_more()) { + g1h->young_list()->rs_length_sampling_next(); ++regions_visited; // we try to yield every time we visit 10 regions @@ -162,6 +162,7 @@ if (_worker_id >= cg1r()->worker_thread_num()) { run_young_rs_sampling(); terminate(); + return; } _vtime_start = os::elapsedVTime(); diff -r 6ecb6e6de3d6 -r 1316cec51b4d src/share/vm/gc_implementation/g1/concurrentMark.hpp --- a/src/share/vm/gc_implementation/g1/concurrentMark.hpp Mon Apr 19 05:40:21 2010 -0700 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.hpp Thu Apr 22 10:02:38 2010 -0700 @@ -1,5 +1,5 @@ /* - * Copyright 2001-2009 Sun Microsystems, Inc. All Rights Reserved. + * Copyright 2001-2010 Sun Microsystems, Inc. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -733,6 +733,19 @@ // to determine whether any heap regions are located above the finger. void registerCSetRegion(HeapRegion* hr); + // Registers the maximum region-end associated with a set of + // regions with CM. Again this is used to determine whether any + // heap regions are located above the finger. + void register_collection_set_finger(HeapWord* max_finger) { + // max_finger is the highest heap region end of the regions currently + // contained in the collection set. If this value is larger than + // _min_finger then we need to gray objects. + // This routine is like registerCSetRegion but for an entire + // collection of regions. + if (max_finger > _min_finger) + _should_gray_objects = true; + } + // Returns "true" if at least one mark has been completed. bool at_least_one_mark_complete() { return _at_least_one_mark_complete; } diff -r 6ecb6e6de3d6 -r 1316cec51b4d src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Mon Apr 19 05:40:21 2010 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Thu Apr 22 10:02:38 2010 -0700 @@ -30,7 +30,7 @@ // turn it on so that the contents of the young list (scan-only / // to-be-collected) are printed at "strategic" points before / during // / after the collection --- this is useful for debugging -#define SCAN_ONLY_VERBOSE 0 +#define YOUNG_LIST_VERBOSE 0 // CURRENT STATUS // This file is under construction. Search for "FIXME". @@ -133,8 +133,7 @@ YoungList::YoungList(G1CollectedHeap* g1h) : _g1h(g1h), _head(NULL), - _scan_only_head(NULL), _scan_only_tail(NULL), _curr_scan_only(NULL), - _length(0), _scan_only_length(0), + _length(0), _last_sampled_rs_lengths(0), _survivor_head(NULL), _survivor_tail(NULL), _survivor_length(0) { @@ -166,48 +165,6 @@ ++_survivor_length; } -HeapRegion* YoungList::pop_region() { - while (_head != NULL) { - assert( length() > 0, "list should not be empty" ); - HeapRegion* ret = _head; - _head = ret->get_next_young_region(); - ret->set_next_young_region(NULL); - --_length; - assert(ret->is_young(), "region should be very young"); - - // Replace 'Survivor' region type with 'Young'. So the region will - // be treated as a young region and will not be 'confused' with - // newly created survivor regions. - if (ret->is_survivor()) { - ret->set_young(); - } - - if (!ret->is_scan_only()) { - return ret; - } - - // scan-only, we'll add it to the scan-only list - if (_scan_only_tail == NULL) { - guarantee( _scan_only_head == NULL, "invariant" ); - - _scan_only_head = ret; - _curr_scan_only = ret; - } else { - guarantee( _scan_only_head != NULL, "invariant" ); - _scan_only_tail->set_next_young_region(ret); - } - guarantee( ret->get_next_young_region() == NULL, "invariant" ); - _scan_only_tail = ret; - - // no need to be tagged as scan-only any more - ret->set_young(); - - ++_scan_only_length; - } - assert( length() == 0, "list should be empty" ); - return NULL; -} - void YoungList::empty_list(HeapRegion* list) { while (list != NULL) { HeapRegion* next = list->get_next_young_region(); @@ -225,12 +182,6 @@ _head = NULL; _length = 0; - empty_list(_scan_only_head); - _scan_only_head = NULL; - _scan_only_tail = NULL; - _scan_only_length = 0; - _curr_scan_only = NULL; - empty_list(_survivor_head); _survivor_head = NULL; _survivor_tail = NULL; @@ -248,11 +199,11 @@ HeapRegion* curr = _head; HeapRegion* last = NULL; while (curr != NULL) { - if (!curr->is_young() || curr->is_scan_only()) { + if (!curr->is_young()) { gclog_or_tty->print_cr("### YOUNG REGION "PTR_FORMAT"-"PTR_FORMAT" " - "incorrectly tagged (%d, %d)", + "incorrectly tagged (y: %d, surv: %d)", curr->bottom(), curr->end(), - curr->is_young(), curr->is_scan_only()); + curr->is_young(), curr->is_survivor()); ret = false; } ++length; @@ -267,47 +218,10 @@ length, _length); } - bool scan_only_ret = true; - length = 0; - curr = _scan_only_head; - last = NULL; - while (curr != NULL) { - if (!curr->is_young() || curr->is_scan_only()) { - gclog_or_tty->print_cr("### SCAN-ONLY REGION "PTR_FORMAT"-"PTR_FORMAT" " - "incorrectly tagged (%d, %d)", - curr->bottom(), curr->end(), - curr->is_young(), curr->is_scan_only()); - scan_only_ret = false; - } - ++length; - last = curr; - curr = curr->get_next_young_region(); - } - scan_only_ret = scan_only_ret && (length == _scan_only_length); - - if ( (last != _scan_only_tail) || - (_scan_only_head == NULL && _scan_only_tail != NULL) || - (_scan_only_head != NULL && _scan_only_tail == NULL) ) { - gclog_or_tty->print_cr("## _scan_only_tail is set incorrectly"); - scan_only_ret = false; - } - - if (_curr_scan_only != NULL && _curr_scan_only != _scan_only_head) { - gclog_or_tty->print_cr("### _curr_scan_only is set incorrectly"); - scan_only_ret = false; - } - - if (!scan_only_ret) { - gclog_or_tty->print_cr("### SCAN-ONLY LIST seems not well formed!"); - gclog_or_tty->print_cr("### list has %d entries, _scan_only_length is %d", - length, _scan_only_length); - } - - return ret && scan_only_ret; + return ret; } -bool YoungList::check_list_empty(bool ignore_scan_only_list, - bool check_sample) { +bool YoungList::check_list_empty(bool check_sample) { bool ret = true; if (_length != 0) { @@ -327,28 +241,7 @@ gclog_or_tty->print_cr("### YOUNG LIST does not seem empty"); } - if (ignore_scan_only_list) - return ret; - - bool scan_only_ret = true; - if (_scan_only_length != 0) { - gclog_or_tty->print_cr("### SCAN-ONLY LIST should have 0 length, not %d", - _scan_only_length); - scan_only_ret = false; - } - if (_scan_only_head != NULL) { - gclog_or_tty->print_cr("### SCAN-ONLY LIST does not have a NULL head"); - scan_only_ret = false; - } - if (_scan_only_tail != NULL) { - gclog_or_tty->print_cr("### SCAN-ONLY LIST does not have a NULL tail"); - scan_only_ret = false; - } - if (!scan_only_ret) { - gclog_or_tty->print_cr("### SCAN-ONLY LIST does not seem empty"); - } - - return ret && scan_only_ret; + return ret; } void @@ -365,7 +258,18 @@ void YoungList::rs_length_sampling_next() { assert( _curr != NULL, "invariant" ); - _sampled_rs_lengths += _curr->rem_set()->occupied(); + size_t rs_length = _curr->rem_set()->occupied(); + + _sampled_rs_lengths += rs_length; + + // The current region may not yet have been added to the + // incremental collection set (it gets added when it is + // retired as the current allocation region). + if (_curr->in_collection_set()) { + // Update the collection set policy information for this region + _g1h->g1_policy()->update_incremental_cset_info(_curr, rs_length); + } + _curr = _curr->get_next_young_region(); if (_curr == NULL) { _last_sampled_rs_lengths = _sampled_rs_lengths; @@ -375,54 +279,46 @@ void YoungList::reset_auxilary_lists() { - // We could have just "moved" the scan-only list to the young list. - // However, the scan-only list is ordered according to the region - // age in descending order, so, by moving one entry at a time, we - // ensure that it is recreated in ascending order. - guarantee( is_empty(), "young list should be empty" ); assert(check_list_well_formed(), "young list should be well formed"); // Add survivor regions to SurvRateGroup. _g1h->g1_policy()->note_start_adding_survivor_regions(); _g1h->g1_policy()->finished_recalculating_age_indexes(true /* is_survivors */); + for (HeapRegion* curr = _survivor_head; curr != NULL; curr = curr->get_next_young_region()) { _g1h->g1_policy()->set_region_survivors(curr); + + // The region is a non-empty survivor so let's add it to + // the incremental collection set for the next evacuation + // pause. + _g1h->g1_policy()->add_region_to_incremental_cset_rhs(curr); } _g1h->g1_policy()->note_stop_adding_survivor_regions(); + _head = _survivor_head; + _length = _survivor_length; if (_survivor_head != NULL) { - _head = _survivor_head; - _length = _survivor_length + _scan_only_length; - _survivor_tail->set_next_young_region(_scan_only_head); - } else { - _head = _scan_only_head; - _length = _scan_only_length; - } - - for (HeapRegion* curr = _scan_only_head; - curr != NULL; - curr = curr->get_next_young_region()) { - curr->recalculate_age_in_surv_rate_group(); - } - _scan_only_head = NULL; - _scan_only_tail = NULL; - _scan_only_length = 0; - _curr_scan_only = NULL; - - _survivor_head = NULL; - _survivor_tail = NULL; - _survivor_length = 0; + assert(_survivor_tail != NULL, "cause it shouldn't be"); + assert(_survivor_length > 0, "invariant"); + _survivor_tail->set_next_young_region(NULL); + } + + // Don't clear the survivor list handles until the start of + // the next evacuation pause - we need it in order to re-tag + // the survivor regions from this evacuation pause as 'young' + // at the start of the next. + _g1h->g1_policy()->finished_recalculating_age_indexes(false /* is_survivors */); assert(check_list_well_formed(), "young list should be well formed"); } void YoungList::print() { - HeapRegion* lists[] = {_head, _scan_only_head, _survivor_head}; - const char* names[] = {"YOUNG", "SCAN-ONLY", "SURVIVOR"}; + HeapRegion* lists[] = {_head, _survivor_head}; + const char* names[] = {"YOUNG", "SURVIVOR"}; for (unsigned int list = 0; list < ARRAY_SIZE(lists); ++list) { gclog_or_tty->print_cr("%s LIST CONTENTS", names[list]); @@ -431,7 +327,7 @@ gclog_or_tty->print_cr(" empty"); while (curr != NULL) { gclog_or_tty->print_cr(" [%08x-%08x], t: %08x, P: %08x, N: %08x, C: %08x, " - "age: %4d, y: %d, s-o: %d, surv: %d", + "age: %4d, y: %d, surv: %d", curr->bottom(), curr->end(), curr->top(), curr->prev_top_at_mark_start(), @@ -439,7 +335,6 @@ curr->top_at_conc_mark_count(), curr->age_in_surv_rate_group_cond(), curr->is_young(), - curr->is_scan_only(), curr->is_survivor()); curr = curr->get_next_young_region(); } @@ -707,6 +602,12 @@ // region below. if (_cur_alloc_region != NULL) { // We're finished with the _cur_alloc_region. + // As we're builing (at least the young portion) of the collection + // set incrementally we'll add the current allocation region to + // the collection set here. + if (_cur_alloc_region->is_young()) { + g1_policy()->add_region_to_incremental_cset_lhs(_cur_alloc_region); + } _summary_bytes_used += _cur_alloc_region->used(); _cur_alloc_region = NULL; } @@ -820,6 +721,12 @@ _free_regions++; free_region(_cur_alloc_region); } else { + // As we're builing (at least the young portion) of the collection + // set incrementally we'll add the current allocation region to + // the collection set here. + if (_cur_alloc_region->is_young()) { + g1_policy()->add_region_to_incremental_cset_lhs(_cur_alloc_region); + } _summary_bytes_used += _cur_alloc_region->used(); } _cur_alloc_region = NULL; @@ -975,6 +882,15 @@ g1_rem_set()->as_HRInto_G1RemSet()->cleanupHRRS(); tear_down_region_lists(); set_used_regions_to_need_zero_fill(); + + // We may have added regions to the current incremental collection + // set between the last GC or pause and now. We need to clear the + // incremental collection set and then start rebuilding it afresh + // after this full GC. + abandon_collection_set(g1_policy()->inc_cset_head()); + g1_policy()->clear_incremental_cset(); + g1_policy()->stop_incremental_cset_building(); + if (g1_policy()->in_young_gc_mode()) { empty_young_list(); g1_policy()->set_full_young_gcs(true); @@ -1058,6 +974,15 @@ perm()->compute_new_size(); } + // Start a new incremental collection set for the next pause + assert(g1_policy()->collection_set() == NULL, "must be"); + g1_policy()->start_incremental_cset_building(); + + // Clear the _cset_fast_test bitmap in anticipation of adding + // regions to the incremental collection set for the next + // evacuation pause. + clear_cset_fast_test(); + double end = os::elapsedTime(); g1_policy()->record_full_collection_end(); @@ -1076,7 +1001,9 @@ if (g1_policy()->in_young_gc_mode()) { _young_list->reset_sampled_info(); - assert( check_young_list_empty(false, false), + // At this point there should be no regions in the + // entire heap tagged as young. + assert( check_young_list_empty(true /* check_heap */), "young list should be empty at this point"); } @@ -1573,6 +1500,20 @@ _g1h = this; + _in_cset_fast_test_length = max_regions(); + _in_cset_fast_test_base = NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length); + + // We're biasing _in_cset_fast_test to avoid subtracting the + // beginning of the heap every time we want to index; basically + // it's the same with what we do with the card table. + _in_cset_fast_test = _in_cset_fast_test_base - + ((size_t) _g1_reserved.start() >> HeapRegion::LogOfHRGrainBytes); + + // Clear the _cset_fast_test bitmap in anticipation of adding + // regions to the incremental collection set for the first + // evacuation pause. + clear_cset_fast_test(); + // Create the ConcurrentMark data structure and thread. // (Must do this late, so that "max_regions" is defined.) _cm = new ConcurrentMark(heap_rs, (int) max_regions()); @@ -2751,25 +2692,19 @@ double start_time_sec = os::elapsedTime(); size_t start_used_bytes = used(); +#if YOUNG_LIST_VERBOSE + gclog_or_tty->print_cr("\nBefore recording pause start.\nYoung_list:"); + _young_list->print(); + g1_policy()->print_collection_set(g1_policy()->inc_cset_head(), gclog_or_tty); +#endif // YOUNG_LIST_VERBOSE + g1_policy()->record_collection_pause_start(start_time_sec, start_used_bytes); - guarantee(_in_cset_fast_test == NULL, "invariant"); - guarantee(_in_cset_fast_test_base == NULL, "invariant"); - _in_cset_fast_test_length = max_regions(); - _in_cset_fast_test_base = - NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length); - memset(_in_cset_fast_test_base, false, - _in_cset_fast_test_length * sizeof(bool)); - // We're biasing _in_cset_fast_test to avoid subtracting the - // beginning of the heap every time we want to index; basically - // it's the same with what we do with the card table. - _in_cset_fast_test = _in_cset_fast_test_base - - ((size_t) _g1_reserved.start() >> HeapRegion::LogOfHRGrainBytes); - -#if SCAN_ONLY_VERBOSE +#if YOUNG_LIST_VERBOSE + gclog_or_tty->print_cr("\nAfter recording pause start.\nYoung_list:"); _young_list->print(); -#endif // SCAN_ONLY_VERBOSE +#endif // YOUNG_LIST_VERBOSE if (g1_policy()->during_initial_mark_pause()) { concurrent_mark()->checkpointRootsInitialPre(); @@ -2796,12 +2731,15 @@ if (mark_in_progress()) concurrent_mark()->newCSet(); - // Now choose the CS. - g1_policy()->choose_collection_set(); - - // We may abandon a pause if we find no region that will fit in the MMU - // pause. - bool abandoned = (g1_policy()->collection_set() == NULL); +#if YOUNG_LIST_VERBOSE + gclog_or_tty->print_cr("\nBefore choosing collection set.\nYoung_list:"); + _young_list->print(); + g1_policy()->print_collection_set(g1_policy()->inc_cset_head(), gclog_or_tty); +#endif // YOUNG_LIST_VERBOSE + + // Now choose the CS. We may abandon a pause if we find no + // region that will fit in the MMU pause. + bool abandoned = g1_policy()->choose_collection_set(); // Nothing to do if we were unable to choose a collection set. if (!abandoned) { @@ -2819,40 +2757,64 @@ // Actually do the work... evacuate_collection_set(); + free_collection_set(g1_policy()->collection_set()); g1_policy()->clear_collection_set(); - FREE_C_HEAP_ARRAY(bool, _in_cset_fast_test_base); - // this is more for peace of mind; we're nulling them here and - // we're expecting them to be null at the beginning of the next GC - _in_cset_fast_test = NULL; - _in_cset_fast_test_base = NULL; - cleanup_surviving_young_words(); + // Start a new incremental collection set for the next pause. + g1_policy()->start_incremental_cset_building(); + + // Clear the _cset_fast_test bitmap in anticipation of adding + // regions to the incremental collection set for the next + // evacuation pause. + clear_cset_fast_test(); + if (g1_policy()->in_young_gc_mode()) { _young_list->reset_sampled_info(); - assert(check_young_list_empty(true), - "young list should be empty"); - -#if SCAN_ONLY_VERBOSE + + // Don't check the whole heap at this point as the + // GC alloc regions from this pause have been tagged + // as survivors and moved on to the survivor list. + // Survivor regions will fail the !is_young() check. + assert(check_young_list_empty(false /* check_heap */), + "young list should be empty"); + +#if YOUNG_LIST_VERBOSE + gclog_or_tty->print_cr("Before recording survivors.\nYoung List:"); _young_list->print(); -#endif // SCAN_ONLY_VERBOSE +#endif // YOUNG_LIST_VERBOSE g1_policy()->record_survivor_regions(_young_list->survivor_length(), _young_list->first_survivor_region(), _young_list->last_survivor_region()); + _young_list->reset_auxilary_lists(); } } else { - if (_in_cset_fast_test != NULL) { - assert(_in_cset_fast_test_base != NULL, "Since _in_cset_fast_test isn't"); - FREE_C_HEAP_ARRAY(bool, _in_cset_fast_test_base); - // this is more for peace of mind; we're nulling them here and - // we're expecting them to be null at the beginning of the next GC - _in_cset_fast_test = NULL; - _in_cset_fast_test_base = NULL; - } + // We have abandoned the current collection. This can only happen + // if we're not doing young or partially young collections, and + // we didn't find an old region that we're able to collect within + // the allowed time. + + assert(g1_policy()->collection_set() == NULL, "should be"); + assert(_young_list->length() == 0, "because it should be"); + + // This should be a no-op. + abandon_collection_set(g1_policy()->inc_cset_head()); + + g1_policy()->clear_incremental_cset(); + g1_policy()->stop_incremental_cset_building(); + + // Start a new incremental collection set for the next pause. + g1_policy()->start_incremental_cset_building(); + + // Clear the _cset_fast_test bitmap in anticipation of adding + // regions to the incremental collection set for the next + // evacuation pause. + clear_cset_fast_test(); + // This looks confusing, because the DPT should really be empty // at this point -- since we have not done any collection work, // there should not be any derived pointers in the table to update; @@ -2886,9 +2848,11 @@ doConcurrentMark(); } -#if SCAN_ONLY_VERBOSE +#if YOUNG_LIST_VERBOSE + gclog_or_tty->print_cr("\nEnd of the pause.\nYoung_list:"); _young_list->print(); -#endif // SCAN_ONLY_VERBOSE + g1_policy()->print_collection_set(g1_policy()->inc_cset_head(), gclog_or_tty); +#endif // YOUNG_LIST_VERBOSE double end_time_sec = os::elapsedTime(); double pause_time_ms = (end_time_sec - start_time_sec) * MILLIUNITS; @@ -4027,16 +3991,13 @@ OopsInHeapRegionClosure *scan_root_cl; OopsInHeapRegionClosure *scan_perm_cl; - OopsInHeapRegionClosure *scan_so_cl; if (_g1h->g1_policy()->during_initial_mark_pause()) { scan_root_cl = &scan_mark_root_cl; scan_perm_cl = &scan_mark_perm_cl; - scan_so_cl = &scan_mark_heap_rs_cl; } else { scan_root_cl = &only_scan_root_cl; scan_perm_cl = &only_scan_perm_cl; - scan_so_cl = &only_scan_heap_rs_cl; } pss.start_strong_roots(); @@ -4044,7 +4005,6 @@ SharedHeap::SO_AllClasses, scan_root_cl, &push_heap_rs_cl, - scan_so_cl, scan_perm_cl, i); pss.end_strong_roots(); @@ -4106,7 +4066,6 @@ SharedHeap::ScanningOption so, OopClosure* scan_non_heap_roots, OopsInHeapRegionClosure* scan_rs, - OopsInHeapRegionClosure* scan_so, OopsInGenClosure* scan_perm, int worker_i) { // First scan the strong roots, including the perm gen. @@ -4126,6 +4085,7 @@ &buf_scan_non_heap_roots, &eager_scan_code_roots, &buf_scan_perm); + // Finish up any enqueued closure apps. buf_scan_non_heap_roots.done(); buf_scan_perm.done(); @@ -4148,9 +4108,6 @@ // XXX What should this be doing in the parallel case? g1_policy()->record_collection_pause_end_CH_strong_roots(); - if (scan_so != NULL) { - scan_scan_only_set(scan_so, worker_i); - } // Now scan the complement of the collection set. if (scan_rs != NULL) { g1_rem_set()->oops_into_collection_set_do(scan_rs, worker_i); @@ -4164,54 +4121,6 @@ } void -G1CollectedHeap::scan_scan_only_region(HeapRegion* r, - OopsInHeapRegionClosure* oc, - int worker_i) { - HeapWord* startAddr = r->bottom(); - HeapWord* endAddr = r->used_region().end(); - - oc->set_region(r); - - HeapWord* p = r->bottom(); - HeapWord* t = r->top(); - guarantee( p == r->next_top_at_mark_start(), "invariant" ); - while (p < t) { - oop obj = oop(p); - p += obj->oop_iterate(oc); - } -} - -void -G1CollectedHeap::scan_scan_only_set(OopsInHeapRegionClosure* oc, - int worker_i) { - double start = os::elapsedTime(); - - BufferingOopsInHeapRegionClosure boc(oc); - - FilterInHeapRegionAndIntoCSClosure scan_only(this, &boc); - FilterAndMarkInHeapRegionAndIntoCSClosure scan_and_mark(this, &boc, concurrent_mark()); - - OopsInHeapRegionClosure *foc; - if (g1_policy()->during_initial_mark_pause()) - foc = &scan_and_mark; - else - foc = &scan_only; - - HeapRegion* hr; - int n = 0; - while ((hr = _young_list->par_get_next_scan_only_region()) != NULL) { - scan_scan_only_region(hr, foc, worker_i); - ++n; - } - boc.done(); - - double closure_app_s = boc.closure_app_seconds(); - g1_policy()->record_obj_copy_time(worker_i, closure_app_s * 1000.0); - double ms = (os::elapsedTime() - start - closure_app_s)*1000.0; - g1_policy()->record_scan_only_time(worker_i, ms, n); -} - -void G1CollectedHeap::g1_process_weak_roots(OopClosure* root_closure, OopClosure* non_root_closure) { CodeBlobToOopClosure roots_in_blobs(root_closure, /*do_marking=*/ false); @@ -4409,17 +4318,14 @@ class G1ParCleanupCTTask : public AbstractGangTask { CardTableModRefBS* _ct_bs; G1CollectedHeap* _g1h; - HeapRegion* volatile _so_head; HeapRegion* volatile _su_head; public: G1ParCleanupCTTask(CardTableModRefBS* ct_bs, G1CollectedHeap* g1h, - HeapRegion* scan_only_list, HeapRegion* survivor_list) : AbstractGangTask("G1 Par Cleanup CT Task"), _ct_bs(ct_bs), _g1h(g1h), - _so_head(scan_only_list), _su_head(survivor_list) { } @@ -4428,14 +4334,13 @@ while (r = _g1h->pop_dirty_cards_region()) { clear_cards(r); } - // Redirty the cards of the scan-only and survivor regions. - dirty_list(&this->_so_head); + // Redirty the cards of the survivor regions. dirty_list(&this->_su_head); } void clear_cards(HeapRegion* r) { - // Cards for Survivor and Scan-Only regions will be dirtied later. - if (!r->is_scan_only() && !r->is_survivor()) { + // Cards for Survivor regions will be dirtied later. + if (!r->is_survivor()) { _ct_bs->clear(MemRegion(r->bottom(), r->end())); } } @@ -4468,7 +4373,7 @@ virtual bool doHeapRegion(HeapRegion* r) { MemRegion mr(r->bottom(), r->end()); - if (r->is_scan_only() || r->is_survivor()) { + if (r->is_survivor()) { _ct_bs->verify_dirty_region(mr); } else { _ct_bs->verify_clean_region(mr); @@ -4484,8 +4389,8 @@ // Iterate over the dirty cards region list. G1ParCleanupCTTask cleanup_task(ct_bs, this, - _young_list->first_scan_only_region(), _young_list->first_survivor_region()); + if (ParallelGCThreads > 0) { set_par_threads(workers()->total_workers()); workers()->run_task(&cleanup_task); @@ -4501,12 +4406,12 @@ } r->set_next_dirty_cards_region(NULL); } - // now, redirty the cards of the scan-only and survivor regions + // now, redirty the cards of the survivor regions // (it seemed faster to do it this way, instead of iterating over // all regions and then clearing / dirtying as appropriate) - dirtyCardsForYoungRegions(ct_bs, _young_list->first_scan_only_region()); dirtyCardsForYoungRegions(ct_bs, _young_list->first_survivor_region()); } + double elapsed = os::elapsedTime() - start; g1_policy()->record_clear_ct_time( elapsed * 1000.0); #ifndef PRODUCT @@ -4527,6 +4432,11 @@ double young_time_ms = 0.0; double non_young_time_ms = 0.0; + // Since the collection set is a superset of the the young list, + // all we need to do to clear the young list is clear its + // head and length, and unlink any young regions in the code below + _young_list->clear(); + G1CollectorPolicy* policy = g1_policy(); double start_sec = os::elapsedTime(); @@ -4570,6 +4480,12 @@ guarantee( (size_t)index < policy->young_cset_length(), "invariant" ); size_t words_survived = _surviving_young_words[index]; cur->record_surv_words_in_group(words_survived); + + // At this point the we have 'popped' cur from the collection set + // (linked via next_in_collection_set()) but it is still in the + // young list (linked via next_young_region()). Clear the + // _next_young_region field. + cur->set_next_young_region(NULL); } else { int index = cur->young_index_in_cset(); guarantee( index == -1, "invariant" ); @@ -4585,7 +4501,6 @@ "Should not have empty regions in a CS."); free_region(cur); } else { - guarantee( !cur->is_scan_only(), "should not be scan only" ); cur->uninstall_surv_rate_group(); if (cur->is_young()) cur->set_young_index_in_cset(-1); @@ -4609,6 +4524,27 @@ policy->record_non_young_free_cset_time_ms(non_young_time_ms); } +// This routine is similar to the above but does not record +// any policy statistics or update free lists; we are abandoning +// the current incremental collection set in preparation of a +// full collection. After the full GC we will start to build up +// the incremental collection set again. +// This is only called when we're doing a full collection +// and is immediately followed by the tearing down of the young list. + +void G1CollectedHeap::abandon_collection_set(HeapRegion* cs_head) { + HeapRegion* cur = cs_head; + + while (cur != NULL) { + HeapRegion* next = cur->next_in_collection_set(); + assert(cur->in_collection_set(), "bad CS"); + cur->set_next_in_collection_set(NULL); + cur->set_in_collection_set(false); + cur->set_young_index_in_cset(-1); + cur = next; + } +} + HeapRegion* G1CollectedHeap::alloc_region_from_unclean_list_locked(bool zero_filled) { assert(ZF_mon->owned_by_self(), "Precondition"); @@ -4975,12 +4911,10 @@ bool success() { return _success; } }; -bool G1CollectedHeap::check_young_list_empty(bool ignore_scan_only_list, - bool check_sample) { - bool ret = true; - - ret = _young_list->check_list_empty(ignore_scan_only_list, check_sample); - if (!ignore_scan_only_list) { +bool G1CollectedHeap::check_young_list_empty(bool check_heap, bool check_sample) { + bool ret = _young_list->check_list_empty(check_sample); + + if (check_heap) { NoYoungRegionsClosure closure; heap_region_iterate(&closure); ret = ret && closure.success(); diff -r 6ecb6e6de3d6 -r 1316cec51b4d src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Mon Apr 19 05:40:21 2010 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Thu Apr 22 10:02:38 2010 -0700 @@ -1,5 +1,5 @@ /* - * Copyright 2001-2009 Sun Microsystems, Inc. All Rights Reserved. + * Copyright 2001-2010 Sun Microsystems, Inc. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -81,33 +81,29 @@ HeapRegion* _head; - HeapRegion* _scan_only_head; - HeapRegion* _scan_only_tail; + HeapRegion* _survivor_head; + HeapRegion* _survivor_tail; + + HeapRegion* _curr; + size_t _length; - size_t _scan_only_length; + size_t _survivor_length; size_t _last_sampled_rs_lengths; size_t _sampled_rs_lengths; - HeapRegion* _curr; - HeapRegion* _curr_scan_only; - HeapRegion* _survivor_head; - HeapRegion* _survivor_tail; - size_t _survivor_length; - - void empty_list(HeapRegion* list); + void empty_list(HeapRegion* list); public: YoungList(G1CollectedHeap* g1h); - void push_region(HeapRegion* hr); - void add_survivor_region(HeapRegion* hr); - HeapRegion* pop_region(); - void empty_list(); - bool is_empty() { return _length == 0; } - size_t length() { return _length; } - size_t scan_only_length() { return _scan_only_length; } - size_t survivor_length() { return _survivor_length; } + void push_region(HeapRegion* hr); + void add_survivor_region(HeapRegion* hr); + + void empty_list(); + bool is_empty() { return _length == 0; } + size_t length() { return _length; } + size_t survivor_length() { return _survivor_length; } void rs_length_sampling_init(); bool rs_length_sampling_more(); @@ -120,22 +116,21 @@ // for development purposes void reset_auxilary_lists(); + void clear() { _head = NULL; _length = 0; } + + void clear_survivors() { + _survivor_head = NULL; + _survivor_tail = NULL; + _survivor_length = 0; + } + HeapRegion* first_region() { return _head; } - HeapRegion* first_scan_only_region() { return _scan_only_head; } HeapRegion* first_survivor_region() { return _survivor_head; } HeapRegion* last_survivor_region() { return _survivor_tail; } - HeapRegion* par_get_next_scan_only_region() { - MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); - HeapRegion* ret = _curr_scan_only; - if (ret != NULL) - _curr_scan_only = ret->get_next_young_region(); - return ret; - } // debugging bool check_list_well_formed(); - bool check_list_empty(bool ignore_scan_only_list, - bool check_sample = true); + bool check_list_empty(bool check_sample = true); void print(); }; @@ -405,8 +400,7 @@ assert(_in_cset_fast_test_base != NULL, "sanity"); assert(r->in_collection_set(), "invariant"); int index = r->hrs_index(); - assert(0 <= (size_t) index && (size_t) index < _in_cset_fast_test_length, - "invariant"); + assert(0 <= index && (size_t) index < _in_cset_fast_test_length, "invariant"); assert(!_in_cset_fast_test_base[index], "invariant"); _in_cset_fast_test_base[index] = true; } @@ -431,6 +425,12 @@ } } + void clear_cset_fast_test() { + assert(_in_cset_fast_test_base != NULL, "sanity"); + memset(_in_cset_fast_test_base, false, + _in_cset_fast_test_length * sizeof(bool)); + } + protected: // Shrink the garbage-first heap by at most the given size (in bytes!). @@ -476,6 +476,10 @@ // regions. void free_collection_set(HeapRegion* cs_head); + // Abandon the current collection set without recording policy + // statistics or updating free lists. + void abandon_collection_set(HeapRegion* cs_head); + // Applies "scan_non_heap_roots" to roots outside the heap, // "scan_rs" to roots inside the heap (having done "set_region" to // indicate the region in which the root resides), and does "scan_perm" @@ -488,16 +492,9 @@ SharedHeap::ScanningOption so, OopClosure* scan_non_heap_roots, OopsInHeapRegionClosure* scan_rs, - OopsInHeapRegionClosure* scan_so, OopsInGenClosure* scan_perm, int worker_i); - void scan_scan_only_set(OopsInHeapRegionClosure* oc, - int worker_i); - void scan_scan_only_region(HeapRegion* hr, - OopsInHeapRegionClosure* oc, - int worker_i); - // Apply "blk" to all the weak roots of the system. These include // JNI weak roots, the code cache, system dictionary, symbol table, // string table, and referents of reachable weak refs. @@ -1136,36 +1133,14 @@ void set_region_short_lived_locked(HeapRegion* hr); // add appropriate methods for any other surv rate groups - void young_list_rs_length_sampling_init() { - _young_list->rs_length_sampling_init(); - } - bool young_list_rs_length_sampling_more() { - return _young_list->rs_length_sampling_more(); - } - void young_list_rs_length_sampling_next() { - _young_list->rs_length_sampling_next(); - } - size_t young_list_sampled_rs_lengths() { - return _young_list->sampled_rs_lengths(); - } - - size_t young_list_length() { return _young_list->length(); } - size_t young_list_scan_only_length() { - return _young_list->scan_only_length(); } - - HeapRegion* pop_region_from_young_list() { - return _young_list->pop_region(); - } - - HeapRegion* young_list_first_region() { - return _young_list->first_region(); - } + YoungList* young_list() { return _young_list; } // debugging bool check_young_list_well_formed() { return _young_list->check_list_well_formed(); } - bool check_young_list_empty(bool ignore_scan_only_list, + + bool check_young_list_empty(bool check_heap, bool check_sample = true); // *** Stuff related to concurrent marking. It's not clear to me that so diff -r 6ecb6e6de3d6 -r 1316cec51b4d src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp --- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Mon Apr 19 05:40:21 2010 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Thu Apr 22 10:02:38 2010 -0700 @@ -1,5 +1,5 @@ /* - * Copyright 2001-2009 Sun Microsystems, Inc. All Rights Reserved. + * Copyright 2001-2010 Sun Microsystems, Inc. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -42,10 +42,6 @@ 0.01, 0.005, 0.005, 0.003, 0.003, 0.002, 0.002, 0.0015 }; -static double cost_per_scan_only_region_ms_defaults[] = { - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 -}; - // all the same static double fully_young_cards_per_entry_ratio_defaults[] = { 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 @@ -125,7 +121,6 @@ _pending_card_diff_seq(new TruncatedSeq(TruncatedSeqLength)), _rs_length_diff_seq(new TruncatedSeq(TruncatedSeqLength)), _cost_per_card_ms_seq(new TruncatedSeq(TruncatedSeqLength)), - _cost_per_scan_only_region_ms_seq(new TruncatedSeq(TruncatedSeqLength)), _fully_young_cards_per_entry_ratio_seq(new TruncatedSeq(TruncatedSeqLength)), _partially_young_cards_per_entry_ratio_seq( new TruncatedSeq(TruncatedSeqLength)), @@ -133,7 +128,6 @@ _partially_young_cost_per_entry_ms_seq(new TruncatedSeq(TruncatedSeqLength)), _cost_per_byte_ms_seq(new TruncatedSeq(TruncatedSeqLength)), _cost_per_byte_ms_during_cm_seq(new TruncatedSeq(TruncatedSeqLength)), - _cost_per_scan_only_region_ms_during_cm_seq(new TruncatedSeq(TruncatedSeqLength)), _constant_other_time_ms_seq(new TruncatedSeq(TruncatedSeqLength)), _young_other_cost_per_region_ms_seq(new TruncatedSeq(TruncatedSeqLength)), _non_young_other_cost_per_region_ms_seq( @@ -186,6 +180,22 @@ _prev_collection_pause_used_at_end_bytes(0), _collection_set(NULL), + _collection_set_size(0), + _collection_set_bytes_used_before(0), + + // Incremental CSet attributes + _inc_cset_build_state(Inactive), + _inc_cset_head(NULL), + _inc_cset_tail(NULL), + _inc_cset_size(0), + _inc_cset_young_index(0), + _inc_cset_bytes_used_before(0), + _inc_cset_max_finger(NULL), + _inc_cset_recorded_young_bytes(0), + _inc_cset_recorded_rs_lengths(0), + _inc_cset_predicted_elapsed_time_ms(0.0), + _inc_cset_predicted_bytes_to_copy(0), + #ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away #pragma warning( disable:4355 ) // 'this' : used in base member initializer list #endif // _MSC_VER @@ -223,8 +233,6 @@ _par_last_ext_root_scan_times_ms = new double[_parallel_gc_threads]; _par_last_mark_stack_scan_times_ms = new double[_parallel_gc_threads]; - _par_last_scan_only_times_ms = new double[_parallel_gc_threads]; - _par_last_scan_only_regions_scanned = new double[_parallel_gc_threads]; _par_last_update_rs_start_times_ms = new double[_parallel_gc_threads]; _par_last_update_rs_times_ms = new double[_parallel_gc_threads]; @@ -254,8 +262,6 @@ _pending_card_diff_seq->add(0.0); _rs_length_diff_seq->add(rs_length_diff_defaults[index]); _cost_per_card_ms_seq->add(cost_per_card_ms_defaults[index]); - _cost_per_scan_only_region_ms_seq->add( - cost_per_scan_only_region_ms_defaults[index]); _fully_young_cards_per_entry_ratio_seq->add( fully_young_cards_per_entry_ratio_defaults[index]); _cost_per_entry_ms_seq->add(cost_per_entry_ms_defaults[index]); @@ -283,7 +289,7 @@ // if G1FixedSurvivorSpaceSize is 0 which means the size is not // fixed, then _max_survivor_regions will be calculated at - // calculate_young_list_target_config during initialization + // calculate_young_list_target_length during initialization _max_survivor_regions = G1FixedSurvivorSpaceSize / HeapRegion::GrainBytes; assert(GCTimeRatio > 0, @@ -357,15 +363,18 @@ set_adaptive_young_list_length(false); _young_list_fixed_length = initial_region_num; } - _free_regions_at_end_of_collection = _g1->free_regions(); - _scan_only_regions_at_end_of_collection = 0; - calculate_young_list_min_length(); - guarantee( _young_list_min_length == 0, "invariant, not enough info" ); - calculate_young_list_target_config(); - } else { + _free_regions_at_end_of_collection = _g1->free_regions(); + calculate_young_list_min_length(); + guarantee( _young_list_min_length == 0, "invariant, not enough info" ); + calculate_young_list_target_length(); + } else { _young_list_fixed_length = 0; _in_young_gc_mode = false; } + + // We may immediately start allocating regions and placing them on the + // collection set list. Initialize the per-collection set info + start_incremental_cset_building(); } // Create the jstat counters for the policy. @@ -385,112 +394,29 @@ double when_ms = _mmu_tracker->when_max_gc_sec(now_sec) * 1000.0; double alloc_rate_ms = predict_alloc_rate_ms(); int min_regions = (int) ceil(alloc_rate_ms * when_ms); - int current_region_num = (int) _g1->young_list_length(); + int current_region_num = (int) _g1->young_list()->length(); _young_list_min_length = min_regions + current_region_num; } } -void G1CollectorPolicy::calculate_young_list_target_config() { +void G1CollectorPolicy::calculate_young_list_target_length() { if (adaptive_young_list_length()) { size_t rs_lengths = (size_t) get_new_prediction(_rs_lengths_seq); - calculate_young_list_target_config(rs_lengths); + calculate_young_list_target_length(rs_lengths); } else { if (full_young_gcs()) _young_list_target_length = _young_list_fixed_length; else _young_list_target_length = _young_list_fixed_length / 2; + _young_list_target_length = MAX2(_young_list_target_length, (size_t)1); - size_t so_length = calculate_optimal_so_length(_young_list_target_length); - guarantee( so_length < _young_list_target_length, "invariant" ); - _young_list_so_prefix_length = so_length; } calculate_survivors_policy(); } -// This method calculate the optimal scan-only set for a fixed young -// gen size. I couldn't work out how to reuse the more elaborate one, -// i.e. calculate_young_list_target_config(rs_length), as the loops are -// fundamentally different (the other one finds a config for different -// S-O lengths, whereas here we need to do the opposite). -size_t G1CollectorPolicy::calculate_optimal_so_length( - size_t young_list_length) { - if (!G1UseScanOnlyPrefix) - return 0; - - if (_all_pause_times_ms->num() < 3) { - // we won't use a scan-only set at the beginning to allow the rest - // of the predictors to warm up - return 0; - } - - if (_cost_per_scan_only_region_ms_seq->num() < 3) { - // then, we'll only set the S-O set to 1 for a little bit of time, - // to get enough information on the scanning cost - return 1; - } - - size_t pending_cards = (size_t) get_new_prediction(_pending_cards_seq); - size_t rs_lengths = (size_t) get_new_prediction(_rs_lengths_seq); - size_t adj_rs_lengths = rs_lengths + predict_rs_length_diff(); - size_t scanned_cards; - if (full_young_gcs()) - scanned_cards = predict_young_card_num(adj_rs_lengths); - else - scanned_cards = predict_non_young_card_num(adj_rs_lengths); - double base_time_ms = predict_base_elapsed_time_ms(pending_cards, - scanned_cards); - - size_t so_length = 0; - double max_gc_eff = 0.0; - for (size_t i = 0; i < young_list_length; ++i) { - double gc_eff = 0.0; - double pause_time_ms = 0.0; - predict_gc_eff(young_list_length, i, base_time_ms, - &gc_eff, &pause_time_ms); - if (gc_eff > max_gc_eff) { - max_gc_eff = gc_eff; - so_length = i; - } - } - - // set it to 95% of the optimal to make sure we sample the "area" - // around the optimal length to get up-to-date survival rate data - return so_length * 950 / 1000; -} - -// This is a really cool piece of code! It finds the best -// target configuration (young length / scan-only prefix length) so -// that GC efficiency is maximized and that we also meet a pause -// time. It's a triple nested loop. These loops are explained below -// from the inside-out :-) -// -// (a) The innermost loop will try to find the optimal young length -// for a fixed S-O length. It uses a binary search to speed up the -// process. We assume that, for a fixed S-O length, as we add more -// young regions to the CSet, the GC efficiency will only go up (I'll -// skip the proof). So, using a binary search to optimize this process -// makes perfect sense. -// -// (b) The middle loop will fix the S-O length before calling the -// innermost one. It will vary it between two parameters, increasing -// it by a given increment. -// -// (c) The outermost loop will call the middle loop three times. -// (1) The first time it will explore all possible S-O length values -// from 0 to as large as it can get, using a coarse increment (to -// quickly "home in" to where the optimal seems to be). -// (2) The second time it will explore the values around the optimal -// that was found by the first iteration using a fine increment. -// (3) Once the optimal config has been determined by the second -// iteration, we'll redo the calculation, but setting the S-O length -// to 95% of the optimal to make sure we sample the "area" -// around the optimal length to get up-to-date survival rate data -// -// Termination conditions for the iterations are several: the pause -// time is over the limit, we do not have enough to-space, etc. - -void G1CollectorPolicy::calculate_young_list_target_config(size_t rs_lengths) { +void G1CollectorPolicy::calculate_young_list_target_length(size_t rs_lengths) { guarantee( adaptive_young_list_length(), "pre-condition" ); + guarantee( !_in_marking_window || !_last_full_young_gc, "invariant" ); double start_time_sec = os::elapsedTime(); size_t min_reserve_perc = MAX2((size_t)2, (size_t)G1ReservePercent); @@ -504,285 +430,80 @@ double survivor_regions_evac_time = predict_survivor_regions_evac_time(); - size_t min_so_length = 0; - size_t max_so_length = 0; - - if (G1UseScanOnlyPrefix) { - if (_all_pause_times_ms->num() < 3) { - // we won't use a scan-only set at the beginning to allow the rest - // of the predictors to warm up - min_so_length = 0; - max_so_length = 0; - } else if (_cost_per_scan_only_region_ms_seq->num() < 3) { - // then, we'll only set the S-O set to 1 for a little bit of time, - // to get enough information on the scanning cost - min_so_length = 1; - max_so_length = 1; - } else if (_in_marking_window || _last_full_young_gc) { - // no S-O prefix during a marking phase either, as at the end - // of the marking phase we'll have to use a very small young - // length target to fill up the rest of the CSet with - // non-young regions and, if we have lots of scan-only regions - // left-over, we will not be able to add any more non-young - // regions. - min_so_length = 0; - max_so_length = 0; - } else { - // this is the common case; we'll never reach the maximum, we - // one of the end conditions will fire well before that - // (hopefully!) - min_so_length = 0; - max_so_length = _free_regions_at_end_of_collection - 1; - } - } else { - // no S-O prefix, as the switch is not set, but we still need to - // do one iteration to calculate the best young target that - // meets the pause time; this way we reuse the same code instead - // of replicating it - min_so_length = 0; - max_so_length = 0; - } - double target_pause_time_ms = _mmu_tracker->max_gc_time() * 1000.0; size_t pending_cards = (size_t) get_new_prediction(_pending_cards_seq); size_t adj_rs_lengths = rs_lengths + predict_rs_length_diff(); - size_t scanned_cards; - if (full_young_gcs()) - scanned_cards = predict_young_card_num(adj_rs_lengths); - else - scanned_cards = predict_non_young_card_num(adj_rs_lengths); - // calculate this once, so that we don't have to recalculate it in - // the innermost loop + size_t scanned_cards = predict_young_card_num(adj_rs_lengths); double base_time_ms = predict_base_elapsed_time_ms(pending_cards, scanned_cards) + survivor_regions_evac_time; + // the result size_t final_young_length = 0; - size_t final_so_length = 0; - double final_gc_eff = 0.0; - // we'll also keep track of how many times we go into the inner loop - // this is for profiling reasons - size_t calculations = 0; - - // this determines which of the three iterations the outer loop is in - typedef enum { - pass_type_coarse, - pass_type_fine, - pass_type_final - } pass_type_t; - - // range of the outer loop's iteration - size_t from_so_length = min_so_length; - size_t to_so_length = max_so_length; - guarantee( from_so_length <= to_so_length, "invariant" ); - - // this will keep the S-O length that's found by the second - // iteration of the outer loop; we'll keep it just in case the third - // iteration fails to find something - size_t fine_so_length = 0; - - // the increment step for the coarse (first) iteration - size_t so_coarse_increments = 5; - - // the common case, we'll start with the coarse iteration - pass_type_t pass = pass_type_coarse; - size_t so_length_incr = so_coarse_increments; - - if (from_so_length == to_so_length) { - // not point in doing the coarse iteration, we'll go directly into - // the fine one (we essentially trying to find the optimal young - // length for a fixed S-O length). - so_length_incr = 1; - pass = pass_type_final; - } else if (to_so_length - from_so_length < 3 * so_coarse_increments) { - // again, the range is too short so no point in foind the coarse - // iteration either - so_length_incr = 1; - pass = pass_type_fine; - } - - bool done = false; - // this is the outermost loop - while (!done) { -#ifdef TRACE_CALC_YOUNG_CONFIG - // leave this in for debugging, just in case - gclog_or_tty->print_cr("searching between " SIZE_FORMAT " and " SIZE_FORMAT - ", incr " SIZE_FORMAT ", pass %s", - from_so_length, to_so_length, so_length_incr, - (pass == pass_type_coarse) ? "coarse" : - (pass == pass_type_fine) ? "fine" : "final"); -#endif // TRACE_CALC_YOUNG_CONFIG - - size_t so_length = from_so_length; - size_t init_free_regions = - MAX2((size_t)0, - _free_regions_at_end_of_collection + - _scan_only_regions_at_end_of_collection - reserve_regions); - - // this determines whether a configuration was found - bool gc_eff_set = false; - // this is the middle loop - while (so_length <= to_so_length) { - // base time, which excludes region-related time; again we - // calculate it once to avoid recalculating it in the - // innermost loop - double base_time_with_so_ms = - base_time_ms + predict_scan_only_time_ms(so_length); - // it's already over the pause target, go around - if (base_time_with_so_ms > target_pause_time_ms) - break; - - size_t starting_young_length = so_length+1; - - // we make sure that the short young length that makes sense - // (one more than the S-O length) is feasible - size_t min_young_length = starting_young_length; - double min_gc_eff; - bool min_ok; - ++calculations; - min_ok = predict_gc_eff(min_young_length, so_length, - base_time_with_so_ms, - init_free_regions, target_pause_time_ms, - &min_gc_eff); - - if (min_ok) { - // the shortest young length is indeed feasible; we'll know - // set up the max young length and we'll do a binary search - // between min_young_length and max_young_length - size_t max_young_length = _free_regions_at_end_of_collection - 1; - double max_gc_eff = 0.0; - bool max_ok = false; - - // the innermost loop! (finally!) - while (max_young_length > min_young_length) { - // we'll make sure that min_young_length is always at a - // feasible config - guarantee( min_ok, "invariant" ); - - ++calculations; - max_ok = predict_gc_eff(max_young_length, so_length, - base_time_with_so_ms, - init_free_regions, target_pause_time_ms, - &max_gc_eff); + + size_t init_free_regions = + MAX2((size_t)0, _free_regions_at_end_of_collection - reserve_regions); + + // if we're still under the pause target... + if (base_time_ms <= target_pause_time_ms) { + // We make sure that the shortest young length that makes sense + // fits within the target pause time. + size_t min_young_length = 1; + + if (predict_will_fit(min_young_length, base_time_ms, + init_free_regions, target_pause_time_ms)) { + // The shortest young length will fit within the target pause time; + // we'll now check whether the absolute maximum number of young + // regions will fit in the target pause time. If not, we'll do + // a binary search between min_young_length and max_young_length + size_t abs_max_young_length = _free_regions_at_end_of_collection - 1; + size_t max_young_length = abs_max_young_length; + + if (max_young_length > min_young_length) { + // Let's check if the initial max young length will fit within the + // target pause. If so then there is no need to search for a maximal + // young length - we'll return the initial maximum + + if (predict_will_fit(max_young_length, base_time_ms, + init_free_regions, target_pause_time_ms)) { + // The maximum young length will satisfy the target pause time. + // We are done so set min young length to this maximum length. + // The code after the loop will then set final_young_length using + // the value cached in the minimum length. + min_young_length = max_young_length; + } else { + // The maximum possible number of young regions will not fit within + // the target pause time so let's search.... size_t diff = (max_young_length - min_young_length) / 2; - if (max_ok) { - min_young_length = max_young_length; - min_gc_eff = max_gc_eff; - min_ok = true; + max_young_length = min_young_length + diff; + + while (max_young_length > min_young_length) { + if (predict_will_fit(max_young_length, base_time_ms, + init_free_regions, target_pause_time_ms)) { + + // The current max young length will fit within the target + // pause time. Note we do not exit the loop here. By setting + // min = max, and then increasing the max below means that + // we will continue searching for an upper bound in the + // range [max..max+diff] + min_young_length = max_young_length; + } + diff = (max_young_length - min_young_length) / 2; + max_young_length = min_young_length + diff; } - max_young_length = min_young_length + diff; + // the above loop found a maximal young length that will fit + // within the target pause time. } - - // the innermost loop found a config - guarantee( min_ok, "invariant" ); - if (min_gc_eff > final_gc_eff) { - // it's the best config so far, so we'll keep it - final_gc_eff = min_gc_eff; - final_young_length = min_young_length; - final_so_length = so_length; - gc_eff_set = true; - } + assert(min_young_length <= abs_max_young_length, "just checking"); } - - // incremental the fixed S-O length and go around - so_length += so_length_incr; + final_young_length = min_young_length; } - - // this is the end of the outermost loop and we need to decide - // what to do during the next iteration - if (pass == pass_type_coarse) { - // we just did the coarse pass (first iteration) - - if (!gc_eff_set) - // we didn't find a feasible config so we'll just bail out; of - // course, it might be the case that we missed it; but I'd say - // it's a bit unlikely - done = true; - else { - // We did find a feasible config with optimal GC eff during - // the first pass. So the second pass we'll only consider the - // S-O lengths around that config with a fine increment. - - guarantee( so_length_incr == so_coarse_increments, "invariant" ); - guarantee( final_so_length >= min_so_length, "invariant" ); - -#ifdef TRACE_CALC_YOUNG_CONFIG - // leave this in for debugging, just in case - gclog_or_tty->print_cr(" coarse pass: SO length " SIZE_FORMAT, - final_so_length); -#endif // TRACE_CALC_YOUNG_CONFIG - - from_so_length = - (final_so_length - min_so_length > so_coarse_increments) ? - final_so_length - so_coarse_increments + 1 : min_so_length; - to_so_length = - (max_so_length - final_so_length > so_coarse_increments) ? - final_so_length + so_coarse_increments - 1 : max_so_length; - - pass = pass_type_fine; - so_length_incr = 1; - } - } else if (pass == pass_type_fine) { - // we just finished the second pass - - if (!gc_eff_set) { - // we didn't find a feasible config (yes, it's possible; - // notice that, sometimes, we go directly into the fine - // iteration and skip the coarse one) so we bail out - done = true; - } else { - // We did find a feasible config with optimal GC eff - guarantee( so_length_incr == 1, "invariant" ); - - if (final_so_length == 0) { - // The config is of an empty S-O set, so we'll just bail out - done = true; - } else { - // we'll go around once more, setting the S-O length to 95% - // of the optimal - size_t new_so_length = 950 * final_so_length / 1000; - -#ifdef TRACE_CALC_YOUNG_CONFIG - // leave this in for debugging, just in case - gclog_or_tty->print_cr(" fine pass: SO length " SIZE_FORMAT - ", setting it to " SIZE_FORMAT, - final_so_length, new_so_length); -#endif // TRACE_CALC_YOUNG_CONFIG - - from_so_length = new_so_length; - to_so_length = new_so_length; - fine_so_length = final_so_length; - - pass = pass_type_final; - } - } - } else if (pass == pass_type_final) { - // we just finished the final (third) pass - - if (!gc_eff_set) - // we didn't find a feasible config, so we'll just use the one - // we found during the second pass, which we saved - final_so_length = fine_so_length; - - // and we're done! - done = true; - } else { - guarantee( false, "should never reach here" ); - } - - // we now go around the outermost loop } + // and we're done! // we should have at least one region in the target young length _young_list_target_length = MAX2((size_t) 1, final_young_length + _recorded_survivor_regions); - if (final_so_length >= final_young_length) - // and we need to ensure that the S-O length is not greater than - // the target young length (this is being a bit careful) - final_so_length = 0; - _young_list_so_prefix_length = final_so_length; - guarantee( !_in_marking_window || !_last_full_young_gc || - _young_list_so_prefix_length == 0, "invariant" ); // let's keep an eye of how long we spend on this calculation // right now, I assume that we'll print it when we need it; we @@ -790,142 +511,91 @@ double end_time_sec = os::elapsedTime(); double elapsed_time_ms = (end_time_sec - start_time_sec) * 1000.0; -#ifdef TRACE_CALC_YOUNG_CONFIG +#ifdef TRACE_CALC_YOUNG_LENGTH // leave this in for debugging, just in case - gclog_or_tty->print_cr("target = %1.1lf ms, young = " SIZE_FORMAT - ", SO = " SIZE_FORMAT ", " - "elapsed %1.2lf ms, calcs: " SIZE_FORMAT " (%s%s) " - SIZE_FORMAT SIZE_FORMAT, + gclog_or_tty->print_cr("target = %1.1lf ms, young = " SIZE_FORMAT ", " + "elapsed %1.2lf ms, (%s%s) " SIZE_FORMAT SIZE_FORMAT, target_pause_time_ms, - _young_list_target_length - _young_list_so_prefix_length, - _young_list_so_prefix_length, + _young_list_target_length elapsed_time_ms, - calculations, full_young_gcs() ? "full" : "partial", during_initial_mark_pause() ? " i-m" : "", _in_marking_window, _in_marking_window_im); -#endif // TRACE_CALC_YOUNG_CONFIG +#endif // TRACE_CALC_YOUNG_LENGTH if (_young_list_target_length < _young_list_min_length) { - // bummer; this means that, if we do a pause when the optimal - // config dictates, we'll violate the pause spacing target (the + // bummer; this means that, if we do a pause when the maximal + // length dictates, we'll violate the pause spacing target (the // min length was calculate based on the application's current // alloc rate); // so, we have to bite the bullet, and allocate the minimum // number. We'll violate our target, but we just can't meet it. - size_t so_length = 0; - // a note further up explains why we do not want an S-O length - // during marking - if (!_in_marking_window && !_last_full_young_gc) - // but we can still try to see whether we can find an optimal - // S-O length - so_length = calculate_optimal_so_length(_young_list_min_length); - -#ifdef TRACE_CALC_YOUNG_CONFIG +#ifdef TRACE_CALC_YOUNG_LENGTH // leave this in for debugging, just in case gclog_or_tty->print_cr("adjusted target length from " - SIZE_FORMAT " to " SIZE_FORMAT - ", SO " SIZE_FORMAT, - _young_list_target_length, _young_list_min_length, - so_length); -#endif // TRACE_CALC_YOUNG_CONFIG - - _young_list_target_length = - MAX2(_young_list_min_length, (size_t)1); - _young_list_so_prefix_length = so_length; + SIZE_FORMAT " to " SIZE_FORMAT, + _young_list_target_length, _young_list_min_length); +#endif // TRACE_CALC_YOUNG_LENGTH + + _young_list_target_length = _young_list_min_length; } } else { // we are in a partially-young mode or we've run out of regions (due // to evacuation failure) -#ifdef TRACE_CALC_YOUNG_CONFIG +#ifdef TRACE_CALC_YOUNG_LENGTH // leave this in for debugging, just in case gclog_or_tty->print_cr("(partial) setting target to " SIZE_FORMAT - ", SO " SIZE_FORMAT, - _young_list_min_length, 0); -#endif // TRACE_CALC_YOUNG_CONFIG - - // we'll do the pause as soon as possible and with no S-O prefix - // (see above for the reasons behind the latter) + _young_list_min_length); +#endif // TRACE_CALC_YOUNG_LENGTH + // we'll do the pause as soon as possible by choosing the minimum _young_list_target_length = MAX2(_young_list_min_length, (size_t) 1); - _young_list_so_prefix_length = 0; } _rs_lengths_prediction = rs_lengths; } -// This is used by: calculate_optimal_so_length(length). It returns -// the GC eff and predicted pause time for a particular config -void -G1CollectorPolicy::predict_gc_eff(size_t young_length, - size_t so_length, - double base_time_ms, - double* ret_gc_eff, - double* ret_pause_time_ms) { - double so_time_ms = predict_scan_only_time_ms(so_length); - double accum_surv_rate_adj = 0.0; - if (so_length > 0) - accum_surv_rate_adj = accum_yg_surv_rate_pred((int)(so_length - 1)); - double accum_surv_rate = - accum_yg_surv_rate_pred((int)(young_length - 1)) - accum_surv_rate_adj; - size_t bytes_to_copy = - (size_t) (accum_surv_rate * (double) HeapRegion::GrainBytes); - double copy_time_ms = predict_object_copy_time_ms(bytes_to_copy); - double young_other_time_ms = - predict_young_other_time_ms(young_length - so_length); - double pause_time_ms = - base_time_ms + so_time_ms + copy_time_ms + young_other_time_ms; - size_t reclaimed_bytes = - (young_length - so_length) * HeapRegion::GrainBytes - bytes_to_copy; - double gc_eff = (double) reclaimed_bytes / pause_time_ms; - - *ret_gc_eff = gc_eff; - *ret_pause_time_ms = pause_time_ms; -} - -// This is used by: calculate_young_list_target_config(rs_length). It -// returns the GC eff of a particular config. It returns false if that -// config violates any of the end conditions of the search in the -// calling method, or true upon success. The end conditions were put -// here since it's called twice and it was best not to replicate them -// in the caller. Also, passing the parameteres avoids having to -// recalculate them in the innermost loop. +// This is used by: calculate_young_list_target_length(rs_length). It +// returns true iff: +// the predicted pause time for the given young list will not overflow +// the target pause time +// and: +// the predicted amount of surviving data will not overflow the +// the amount of free space available for survivor regions. +// bool -G1CollectorPolicy::predict_gc_eff(size_t young_length, - size_t so_length, - double base_time_with_so_ms, - size_t init_free_regions, - double target_pause_time_ms, - double* ret_gc_eff) { - *ret_gc_eff = 0.0; +G1CollectorPolicy::predict_will_fit(size_t young_length, + double base_time_ms, + size_t init_free_regions, + double target_pause_time_ms) { if (young_length >= init_free_regions) // end condition 1: not enough space for the young regions return false; double accum_surv_rate_adj = 0.0; - if (so_length > 0) - accum_surv_rate_adj = accum_yg_surv_rate_pred((int)(so_length - 1)); double accum_surv_rate = accum_yg_surv_rate_pred((int)(young_length - 1)) - accum_surv_rate_adj; + size_t bytes_to_copy = (size_t) (accum_surv_rate * (double) HeapRegion::GrainBytes); + double copy_time_ms = predict_object_copy_time_ms(bytes_to_copy); + double young_other_time_ms = - predict_young_other_time_ms(young_length - so_length); + predict_young_other_time_ms(young_length); + double pause_time_ms = - base_time_with_so_ms + copy_time_ms + young_other_time_ms; + base_time_ms + copy_time_ms + young_other_time_ms; if (pause_time_ms > target_pause_time_ms) // end condition 2: over the target pause time return false; - size_t reclaimed_bytes = - (young_length - so_length) * HeapRegion::GrainBytes - bytes_to_copy; size_t free_bytes = (init_free_regions - young_length) * HeapRegion::GrainBytes; @@ -934,9 +604,6 @@ return false; // success! - double gc_eff = (double) reclaimed_bytes / pause_time_ms; - *ret_gc_eff = gc_eff; - return true; } @@ -953,11 +620,11 @@ void G1CollectorPolicy::check_prediction_validity() { guarantee( adaptive_young_list_length(), "should not call this otherwise" ); - size_t rs_lengths = _g1->young_list_sampled_rs_lengths(); + size_t rs_lengths = _g1->young_list()->sampled_rs_lengths(); if (rs_lengths > _rs_lengths_prediction) { // add 10% to avoid having to recalculate often size_t rs_lengths_prediction = rs_lengths * 1100 / 1000; - calculate_young_list_target_config(rs_lengths_prediction); + calculate_young_list_target_length(rs_lengths_prediction); } } @@ -979,7 +646,7 @@ #ifndef PRODUCT bool G1CollectorPolicy::verify_young_ages() { - HeapRegion* head = _g1->young_list_first_region(); + HeapRegion* head = _g1->young_list()->first_region(); return verify_young_ages(head, _short_lived_surv_rate_group); // also call verify_young_ages on any additional surv rate groups @@ -1056,7 +723,6 @@ _in_marking_window = false; _in_marking_window_im = false; - _short_lived_surv_rate_group->record_scan_only_prefix(0); _short_lived_surv_rate_group->start_adding_regions(); // also call this on any additional surv rate groups @@ -1066,11 +732,10 @@ _prev_region_num_tenured = _region_num_tenured; _free_regions_at_end_of_collection = _g1->free_regions(); - _scan_only_regions_at_end_of_collection = 0; // Reset survivors SurvRateGroup. _survivor_surv_rate_group->reset(); calculate_young_list_min_length(); - calculate_young_list_target_config(); + calculate_young_list_target_length(); } void G1CollectorPolicy::record_before_bytes(size_t bytes) { @@ -1119,8 +784,6 @@ for (int i = 0; i < _parallel_gc_threads; ++i) { _par_last_ext_root_scan_times_ms[i] = -666.0; _par_last_mark_stack_scan_times_ms[i] = -666.0; - _par_last_scan_only_times_ms[i] = -666.0; - _par_last_scan_only_regions_scanned[i] = -666.0; _par_last_update_rs_start_times_ms[i] = -666.0; _par_last_update_rs_times_ms[i] = -666.0; _par_last_update_rs_processed_buffers[i] = -666.0; @@ -1143,47 +806,13 @@ if (in_young_gc_mode()) _last_young_gc_full = false; - // do that for any other surv rate groups _short_lived_surv_rate_group->stop_adding_regions(); - size_t short_lived_so_length = _young_list_so_prefix_length; - _short_lived_surv_rate_group->record_scan_only_prefix(short_lived_so_length); - tag_scan_only(short_lived_so_length); _survivors_age_table.clear(); assert( verify_young_ages(), "region age verification" ); } -void G1CollectorPolicy::tag_scan_only(size_t short_lived_scan_only_length) { - // done in a way that it can be extended for other surv rate groups too... - - HeapRegion* head = _g1->young_list_first_region(); - bool finished_short_lived = (short_lived_scan_only_length == 0); - - if (finished_short_lived) - return; - - for (HeapRegion* curr = head; - curr != NULL; - curr = curr->get_next_young_region()) { - SurvRateGroup* surv_rate_group = curr->surv_rate_group(); - int age = curr->age_in_surv_rate_group(); - - if (surv_rate_group == _short_lived_surv_rate_group) { - if ((size_t)age < short_lived_scan_only_length) - curr->set_scan_only(); - else - finished_short_lived = true; - } - - - if (finished_short_lived) - return; - } - - guarantee( false, "we should never reach here" ); -} - void G1CollectorPolicy::record_mark_closure_time(double mark_closure_time_ms) { _mark_closure_time_ms = mark_closure_time_ms; } @@ -1277,7 +906,7 @@ _last_full_young_gc = true; _in_marking_window = false; if (adaptive_young_list_length()) - calculate_young_list_target_config(); + calculate_young_list_target_length(); } } @@ -1512,6 +1141,7 @@ size_t freed_bytes = _cur_collection_pause_used_at_start_bytes - cur_used_bytes; size_t surviving_bytes = _collection_set_bytes_used_before - freed_bytes; + double survival_fraction = (double)surviving_bytes/ (double)_collection_set_bytes_used_before; @@ -1599,9 +1229,6 @@ double ext_root_scan_time = avg_value(_par_last_ext_root_scan_times_ms); double mark_stack_scan_time = avg_value(_par_last_mark_stack_scan_times_ms); - double scan_only_time = avg_value(_par_last_scan_only_times_ms); - double scan_only_regions_scanned = - sum_of_values(_par_last_scan_only_regions_scanned); double update_rs_time = avg_value(_par_last_update_rs_times_ms); double update_rs_processed_buffers = sum_of_values(_par_last_update_rs_processed_buffers); @@ -1611,7 +1238,7 @@ double parallel_other_time = _cur_collection_par_time_ms - (update_rs_time + ext_root_scan_time + mark_stack_scan_time + - scan_only_time + scan_rs_time + obj_copy_time + termination_time); + scan_rs_time + obj_copy_time + termination_time); if (update_stats) { MainBodySummary* body_summary = summary->main_body_summary(); guarantee(body_summary != NULL, "should not be null!"); @@ -1622,7 +1249,6 @@ body_summary->record_satb_drain_time_ms(0.0); body_summary->record_ext_root_scan_time_ms(ext_root_scan_time); body_summary->record_mark_stack_scan_time_ms(mark_stack_scan_time); - body_summary->record_scan_only_time_ms(scan_only_time); body_summary->record_update_rs_time_ms(update_rs_time); body_summary->record_scan_rs_time_ms(scan_rs_time); body_summary->record_obj_copy_time_ms(obj_copy_time); @@ -1676,7 +1302,7 @@ else other_time_ms -= update_rs_time + - ext_root_scan_time + mark_stack_scan_time + scan_only_time + + ext_root_scan_time + mark_stack_scan_time + scan_rs_time + obj_copy_time; } @@ -1701,9 +1327,6 @@ _par_last_update_rs_processed_buffers, true); print_par_stats(2, "Ext Root Scanning", _par_last_ext_root_scan_times_ms); print_par_stats(2, "Mark Stack Scanning", _par_last_mark_stack_scan_times_ms); - print_par_stats(2, "Scan-Only Scanning", _par_last_scan_only_times_ms); - print_par_buffers(3, "Scan-Only Regions", - _par_last_scan_only_regions_scanned, true); print_par_stats(2, "Scan RS", _par_last_scan_rs_times_ms); print_par_stats(2, "Object Copy", _par_last_obj_copy_times_ms); print_par_stats(2, "Termination", _par_last_termination_times_ms); @@ -1715,7 +1338,6 @@ (int)update_rs_processed_buffers); print_stats(1, "Ext Root Scanning", ext_root_scan_time); print_stats(1, "Mark Stack Scanning", mark_stack_scan_time); - print_stats(1, "Scan-Only Scanning", scan_only_time); print_stats(1, "Scan RS", scan_rs_time); print_stats(1, "Object Copying", obj_copy_time); } @@ -1730,6 +1352,8 @@ } #endif print_stats(1, "Other", other_time_ms); + print_stats(2, "Choose CSet", _recorded_young_cset_choice_time_ms); + for (int i = 0; i < _aux_num; ++i) { if (_cur_aux_times_set[i]) { char buffer[96]; @@ -1815,16 +1439,6 @@ _cost_per_card_ms_seq->add(cost_per_card_ms); } - double cost_per_scan_only_region_ms = 0.0; - if (scan_only_regions_scanned > 0.0) { - cost_per_scan_only_region_ms = - scan_only_time / scan_only_regions_scanned; - if (_in_marking_window_im) - _cost_per_scan_only_region_ms_during_cm_seq->add(cost_per_scan_only_region_ms); - else - _cost_per_scan_only_region_ms_seq->add(cost_per_scan_only_region_ms); - } - size_t cards_scanned = _g1->cards_scanned(); double cost_per_entry_ms = 0.0; @@ -1860,7 +1474,7 @@ } double all_other_time_ms = pause_time_ms - - (update_rs_time + scan_only_time + scan_rs_time + obj_copy_time + + (update_rs_time + scan_rs_time + obj_copy_time + _mark_closure_time_ms + termination_time); double young_other_time_ms = 0.0; @@ -1907,11 +1521,10 @@ if (PREDICTIONS_VERBOSE) { gclog_or_tty->print_cr(""); gclog_or_tty->print_cr("PREDICTIONS %1.4lf %d " - "REGIONS %d %d %d %d " + "REGIONS %d %d %d " "PENDING_CARDS %d %d " "CARDS_SCANNED %d %d " "RS_LENGTHS %d %d " - "SCAN_ONLY_SCAN %1.6lf %1.6lf " "RS_UPDATE %1.6lf %1.6lf RS_SCAN %1.6lf %1.6lf " "SURVIVAL_RATIO %1.6lf %1.6lf " "OBJECT_COPY %1.6lf %1.6lf OTHER_CONSTANT %1.6lf %1.6lf " @@ -1924,12 +1537,10 @@ (last_pause_included_initial_mark) ? 1 : 0, _recorded_region_num, _recorded_young_regions, - _recorded_scan_only_regions, _recorded_non_young_regions, _predicted_pending_cards, _pending_cards, _predicted_cards_scanned, cards_scanned, _predicted_rs_lengths, _max_rs_lengths, - _predicted_scan_only_scan_time_ms, scan_only_time, _predicted_rs_update_time_ms, update_rs_time, _predicted_rs_scan_time_ms, scan_rs_time, _predicted_survival_ratio, survival_ratio, @@ -1954,14 +1565,12 @@ _in_marking_window = new_in_marking_window; _in_marking_window_im = new_in_marking_window_im; _free_regions_at_end_of_collection = _g1->free_regions(); - _scan_only_regions_at_end_of_collection = _g1->young_list_length(); calculate_young_list_min_length(); - calculate_young_list_target_config(); + calculate_young_list_target_length(); // Note that _mmu_tracker->max_gc_time() returns the time in seconds. double update_rs_time_goal_ms = _mmu_tracker->max_gc_time() * MILLIUNITS * G1RSetUpdatingPauseTimePercent / 100.0; adjust_concurrent_refinement(update_rs_time, update_rs_processed_buffers, update_rs_time_goal_ms); - // _target_pause_time_ms = -1.0; @@ -2016,13 +1625,13 @@ guarantee( adjustment == 0 || adjustment == 1, "invariant" ); G1CollectedHeap* g1h = G1CollectedHeap::heap(); - size_t young_num = g1h->young_list_length(); + size_t young_num = g1h->young_list()->length(); if (young_num == 0) return 0.0; young_num += adjustment; size_t pending_cards = predict_pending_cards(); - size_t rs_lengths = g1h->young_list_sampled_rs_lengths() + + size_t rs_lengths = g1h->young_list()->sampled_rs_lengths() + predict_rs_length_diff(); size_t card_num; if (full_young_gcs()) @@ -2106,31 +1715,22 @@ void G1CollectorPolicy::start_recording_regions() { _recorded_rs_lengths = 0; - _recorded_scan_only_regions = 0; _recorded_young_regions = 0; _recorded_non_young_regions = 0; #if PREDICTIONS_VERBOSE - _predicted_rs_lengths = 0; - _predicted_cards_scanned = 0; - _recorded_marked_bytes = 0; _recorded_young_bytes = 0; _predicted_bytes_to_copy = 0; + _predicted_rs_lengths = 0; + _predicted_cards_scanned = 0; #endif // PREDICTIONS_VERBOSE } void -G1CollectorPolicy::record_cset_region(HeapRegion* hr, bool young) { - if (young) { - ++_recorded_young_regions; - } else { - ++_recorded_non_young_regions; - } +G1CollectorPolicy::record_cset_region_info(HeapRegion* hr, bool young) { #if PREDICTIONS_VERBOSE - if (young) { - _recorded_young_bytes += hr->used(); - } else { + if (!young) { _recorded_marked_bytes += hr->max_live_bytes(); } _predicted_bytes_to_copy += predict_bytes_to_copy(hr); @@ -2141,12 +1741,37 @@ } void -G1CollectorPolicy::record_scan_only_regions(size_t scan_only_length) { - _recorded_scan_only_regions = scan_only_length; +G1CollectorPolicy::record_non_young_cset_region(HeapRegion* hr) { + assert(!hr->is_young(), "should not call this"); + ++_recorded_non_young_regions; + record_cset_region_info(hr, false); +} + +void +G1CollectorPolicy::set_recorded_young_regions(size_t n_regions) { + _recorded_young_regions = n_regions; +} + +void G1CollectorPolicy::set_recorded_young_bytes(size_t bytes) { +#if PREDICTIONS_VERBOSE + _recorded_young_bytes = bytes; +#endif // PREDICTIONS_VERBOSE +} + +void G1CollectorPolicy::set_recorded_rs_lengths(size_t rs_lengths) { + _recorded_rs_lengths = rs_lengths; +} + +void G1CollectorPolicy::set_predicted_bytes_to_copy(size_t bytes) { + _predicted_bytes_to_copy = bytes; } void G1CollectorPolicy::end_recording_regions() { + // The _predicted_pause_time_ms field is referenced in code + // not under PREDICTIONS_VERBOSE. Let's initialize it. + _predicted_pause_time_ms = -1.0; + #if PREDICTIONS_VERBOSE _predicted_pending_cards = predict_pending_cards(); _predicted_rs_lengths = _recorded_rs_lengths + predict_rs_length_diff(); @@ -2157,8 +1782,6 @@ predict_non_young_card_num(_predicted_rs_lengths); _recorded_region_num = _recorded_young_regions + _recorded_non_young_regions; - _predicted_scan_only_scan_time_ms = - predict_scan_only_time_ms(_recorded_scan_only_regions); _predicted_rs_update_time_ms = predict_rs_update_time_ms(_g1->pending_card_num()); _predicted_rs_scan_time_ms = @@ -2173,7 +1796,6 @@ predict_non_young_other_time_ms(_recorded_non_young_regions); _predicted_pause_time_ms = - _predicted_scan_only_scan_time_ms + _predicted_rs_update_time_ms + _predicted_rs_scan_time_ms + _predicted_object_copy_time_ms + @@ -2463,8 +2085,6 @@ body_summary->get_ext_root_scan_seq()); print_summary(2, "Mark Stack Scanning", body_summary->get_mark_stack_scan_seq()); - print_summary(2, "Scan-Only Scanning", - body_summary->get_scan_only_seq()); print_summary(2, "Scan RS", body_summary->get_scan_rs_seq()); print_summary(2, "Object Copy", body_summary->get_obj_copy_seq()); print_summary(2, "Termination", body_summary->get_termination_seq()); @@ -2474,7 +2094,6 @@ body_summary->get_update_rs_seq(), body_summary->get_ext_root_scan_seq(), body_summary->get_mark_stack_scan_seq(), - body_summary->get_scan_only_seq(), body_summary->get_scan_rs_seq(), body_summary->get_obj_copy_seq(), body_summary->get_termination_seq() @@ -2492,8 +2111,6 @@ body_summary->get_ext_root_scan_seq()); print_summary(1, "Mark Stack Scanning", body_summary->get_mark_stack_scan_seq()); - print_summary(1, "Scan-Only Scanning", - body_summary->get_scan_only_seq()); print_summary(1, "Scan RS", body_summary->get_scan_rs_seq()); print_summary(1, "Object Copy", body_summary->get_obj_copy_seq()); } @@ -2519,7 +2136,6 @@ body_summary->get_update_rs_seq(), body_summary->get_ext_root_scan_seq(), body_summary->get_mark_stack_scan_seq(), - body_summary->get_scan_only_seq(), body_summary->get_scan_rs_seq(), body_summary->get_obj_copy_seq() }; @@ -2613,7 +2229,7 @@ G1CollectorPolicy::should_add_next_region_to_young_list() { assert(in_young_gc_mode(), "should be in young GC mode"); bool ret; - size_t young_list_length = _g1->young_list_length(); + size_t young_list_length = _g1->young_list()->length(); size_t young_list_max_length = _young_list_target_length; if (G1FixedEdenSize) { young_list_max_length -= _max_survivor_regions; @@ -2676,7 +2292,7 @@ assert(_g1->regions_accounted_for(), "Region leakage!"); double max_pause_time_ms = _mmu_tracker->max_gc_time() * 1000.0; - size_t young_list_length = _g1->young_list_length(); + size_t young_list_length = _g1->young_list()->length(); size_t young_list_max_length = _young_list_target_length; if (G1FixedEdenSize) { young_list_max_length -= _max_survivor_regions; @@ -2685,7 +2301,7 @@ if (in_young_gc_mode()) { if (reached_target_length) { - assert( young_list_length > 0 && _g1->young_list_length() > 0, + assert( young_list_length > 0 && _g1->young_list()->length() > 0, "invariant" ); _target_pause_time_ms = max_pause_time_ms; return true; @@ -2946,10 +2562,12 @@ } } -// Add the heap region to the collection set and return the conservative -// estimate of the number of live bytes. +// Add the heap region at the head of the non-incremental collection set void G1CollectorPolicy:: add_to_collection_set(HeapRegion* hr) { + assert(_inc_cset_build_state == Active, "Precondition"); + assert(!hr->is_young(), "non-incremental add of young region"); + if (G1PrintHeapRegions) { gclog_or_tty->print_cr("added region to cset " "%d:["PTR_FORMAT", "PTR_FORMAT"], " @@ -2961,8 +2579,7 @@ if (_g1->mark_in_progress()) _g1->concurrent_mark()->registerCSetRegion(hr); - assert(!hr->in_collection_set(), - "should not already be in the CSet"); + assert(!hr->in_collection_set(), "should not already be in the CSet"); hr->set_in_collection_set(true); hr->set_next_in_collection_set(_collection_set); _collection_set = hr; @@ -2971,10 +2588,230 @@ _g1->register_region_with_in_cset_fast_test(hr); } -void -G1CollectorPolicy_BestRegionsFirst:: -choose_collection_set() { - double non_young_start_time_sec; +// Initialize the per-collection-set information +void G1CollectorPolicy::start_incremental_cset_building() { + assert(_inc_cset_build_state == Inactive, "Precondition"); + + _inc_cset_head = NULL; + _inc_cset_tail = NULL; + _inc_cset_size = 0; + _inc_cset_bytes_used_before = 0; + + if (in_young_gc_mode()) { + _inc_cset_young_index = 0; + } + + _inc_cset_max_finger = 0; + _inc_cset_recorded_young_bytes = 0; + _inc_cset_recorded_rs_lengths = 0; + _inc_cset_predicted_elapsed_time_ms = 0; + _inc_cset_predicted_bytes_to_copy = 0; + _inc_cset_build_state = Active; +} + +void G1CollectorPolicy::add_to_incremental_cset_info(HeapRegion* hr, size_t rs_length) { + // This routine is used when: + // * adding survivor regions to the incremental cset at the end of an + // evacuation pause, + // * adding the current allocation region to the incremental cset + // when it is retired, and + // * updating existing policy information for a region in the + // incremental cset via young list RSet sampling. + // Therefore this routine may be called at a safepoint by the + // VM thread, or in-between safepoints by mutator threads (when + // retiring the current allocation region) or a concurrent + // refine thread (RSet sampling). + + double region_elapsed_time_ms = predict_region_elapsed_time_ms(hr, true); + size_t used_bytes = hr->used(); + + _inc_cset_recorded_rs_lengths += rs_length; + _inc_cset_predicted_elapsed_time_ms += region_elapsed_time_ms; + + _inc_cset_bytes_used_before += used_bytes; + + // Cache the values we have added to the aggregated informtion + // in the heap region in case we have to remove this region from + // the incremental collection set, or it is updated by the + // rset sampling code + hr->set_recorded_rs_length(rs_length); + hr->set_predicted_elapsed_time_ms(region_elapsed_time_ms); + +#if PREDICTIONS_VERBOSE + size_t bytes_to_copy = predict_bytes_to_copy(hr); + _inc_cset_predicted_bytes_to_copy += bytes_to_copy; + + // Record the number of bytes used in this region + _inc_cset_recorded_young_bytes += used_bytes; + + // Cache the values we have added to the aggregated informtion + // in the heap region in case we have to remove this region from + // the incremental collection set, or it is updated by the + // rset sampling code + hr->set_predicted_bytes_to_copy(bytes_to_copy); +#endif // PREDICTIONS_VERBOSE +} + +void G1CollectorPolicy::remove_from_incremental_cset_info(HeapRegion* hr) { + // This routine is currently only called as part of the updating of + // existing policy information for regions in the incremental cset that + // is performed by the concurrent refine thread(s) as part of young list + // RSet sampling. Therefore we should not be at a safepoint. + + assert(!SafepointSynchronize::is_at_safepoint(), "should not be at safepoint"); + assert(hr->is_young(), "it should be"); + + size_t used_bytes = hr->used(); + size_t old_rs_length = hr->recorded_rs_length(); + double old_elapsed_time_ms = hr->predicted_elapsed_time_ms(); + + // Subtract the old recorded/predicted policy information for + // the given heap region from the collection set info. + _inc_cset_recorded_rs_lengths -= old_rs_length; + _inc_cset_predicted_elapsed_time_ms -= old_elapsed_time_ms; + + _inc_cset_bytes_used_before -= used_bytes; + + // Clear the values cached in the heap region + hr->set_recorded_rs_length(0); + hr->set_predicted_elapsed_time_ms(0); + +#if PREDICTIONS_VERBOSE + size_t old_predicted_bytes_to_copy = hr->predicted_bytes_to_copy(); + _inc_cset_predicted_bytes_to_copy -= old_predicted_bytes_to_copy; + + // Subtract the number of bytes used in this region + _inc_cset_recorded_young_bytes -= used_bytes; + + // Clear the values cached in the heap region + hr->set_predicted_bytes_to_copy(0); +#endif // PREDICTIONS_VERBOSE +} + +void G1CollectorPolicy::update_incremental_cset_info(HeapRegion* hr, size_t new_rs_length) { + // Update the collection set information that is dependent on the new RS length + assert(hr->is_young(), "Precondition"); + + remove_from_incremental_cset_info(hr); + add_to_incremental_cset_info(hr, new_rs_length); +} + +void G1CollectorPolicy::add_region_to_incremental_cset_common(HeapRegion* hr) { + assert( hr->is_young(), "invariant"); + assert( hr->young_index_in_cset() == -1, "invariant" ); + assert(_inc_cset_build_state == Active, "Precondition"); + + // We need to clear and set the cached recorded/cached collection set + // information in the heap region here (before the region gets added + // to the collection set). An individual heap region's cached values + // are calculated, aggregated with the policy collection set info, + // and cached in the heap region here (initially) and (subsequently) + // by the Young List sampling code. + + size_t rs_length = hr->rem_set()->occupied(); + add_to_incremental_cset_info(hr, rs_length); + + HeapWord* hr_end = hr->end(); + _inc_cset_max_finger = MAX2(_inc_cset_max_finger, hr_end); + + assert(!hr->in_collection_set(), "invariant"); + hr->set_in_collection_set(true); + assert( hr->next_in_collection_set() == NULL, "invariant"); + + _inc_cset_size++; + _g1->register_region_with_in_cset_fast_test(hr); + + hr->set_young_index_in_cset((int) _inc_cset_young_index); + ++_inc_cset_young_index; +} + +// Add the region at the RHS of the incremental cset +void G1CollectorPolicy::add_region_to_incremental_cset_rhs(HeapRegion* hr) { + // We should only ever be appending survivors at the end of a pause + assert( hr->is_survivor(), "Logic"); + + // Do the 'common' stuff + add_region_to_incremental_cset_common(hr); + + // Now add the region at the right hand side + if (_inc_cset_tail == NULL) { + assert(_inc_cset_head == NULL, "invariant"); + _inc_cset_head = hr; + } else { + _inc_cset_tail->set_next_in_collection_set(hr); + } + _inc_cset_tail = hr; + + if (G1PrintHeapRegions) { + gclog_or_tty->print_cr(" added region to incremental cset (RHS) " + "%d:["PTR_FORMAT", "PTR_FORMAT"], " + "top "PTR_FORMAT", young %s", + hr->hrs_index(), hr->bottom(), hr->end(), + hr->top(), (hr->is_young()) ? "YES" : "NO"); + } +} + +// Add the region to the LHS of the incremental cset +void G1CollectorPolicy::add_region_to_incremental_cset_lhs(HeapRegion* hr) { + // Survivors should be added to the RHS at the end of a pause + assert(!hr->is_survivor(), "Logic"); + + // Do the 'common' stuff + add_region_to_incremental_cset_common(hr); + + // Add the region at the left hand side + hr->set_next_in_collection_set(_inc_cset_head); + if (_inc_cset_head == NULL) { + assert(_inc_cset_tail == NULL, "Invariant"); + _inc_cset_tail = hr; + } + _inc_cset_head = hr; + + if (G1PrintHeapRegions) { + gclog_or_tty->print_cr(" added region to incremental cset (LHS) " + "%d:["PTR_FORMAT", "PTR_FORMAT"], " + "top "PTR_FORMAT", young %s", + hr->hrs_index(), hr->bottom(), hr->end(), + hr->top(), (hr->is_young()) ? "YES" : "NO"); + } +} + +#ifndef PRODUCT +void G1CollectorPolicy::print_collection_set(HeapRegion* list_head, outputStream* st) { + assert(list_head == inc_cset_head() || list_head == collection_set(), "must be"); + + st->print_cr("\nCollection_set:"); + HeapRegion* csr = list_head; + while (csr != NULL) { + HeapRegion* next = csr->next_in_collection_set(); + assert(csr->in_collection_set(), "bad CS"); + st->print_cr(" [%08x-%08x], t: %08x, P: %08x, N: %08x, C: %08x, " + "age: %4d, y: %d, surv: %d", + csr->bottom(), csr->end(), + csr->top(), + csr->prev_top_at_mark_start(), + csr->next_top_at_mark_start(), + csr->top_at_conc_mark_count(), + csr->age_in_surv_rate_group_cond(), + csr->is_young(), + csr->is_survivor()); + csr = next; + } +} +#endif // !PRODUCT + +bool +G1CollectorPolicy_BestRegionsFirst::choose_collection_set() { + // Set this here - in case we're not doing young collections. + double non_young_start_time_sec = os::elapsedTime(); + + // The result that this routine will return. This will be set to + // false if: + // * we're doing a young or partially young collection and we + // have added the youg regions to collection set, or + // * we add old regions to the collection set. + bool abandon_collection = true; + start_recording_regions(); guarantee(_target_pause_time_ms > -1.0 @@ -3027,47 +2864,79 @@ if (G1PolicyVerbose > 0) { gclog_or_tty->print_cr("Adding %d young regions to the CSet", - _g1->young_list_length()); + _g1->young_list()->length()); } + _young_cset_length = 0; _last_young_gc_full = full_young_gcs() ? true : false; + if (_last_young_gc_full) ++_full_young_pause_num; else ++_partial_young_pause_num; - hr = _g1->pop_region_from_young_list(); + + // The young list is laid with the survivor regions from the previous + // pause are appended to the RHS of the young list, i.e. + // [Newly Young Regions ++ Survivors from last pause]. + + hr = _g1->young_list()->first_survivor_region(); while (hr != NULL) { - - assert( hr->young_index_in_cset() == -1, "invariant" ); - assert( hr->age_in_surv_rate_group() != -1, "invariant" ); - hr->set_young_index_in_cset((int) _young_cset_length); - - ++_young_cset_length; - double predicted_time_ms = predict_region_elapsed_time_ms(hr, true); - time_remaining_ms -= predicted_time_ms; - predicted_pause_time_ms += predicted_time_ms; - assert(!hr->in_collection_set(), "invariant"); - add_to_collection_set(hr); - record_cset_region(hr, true); - max_live_bytes -= MIN2(hr->max_live_bytes(), max_live_bytes); - if (G1PolicyVerbose > 0) { - gclog_or_tty->print_cr(" Added [" PTR_FORMAT ", " PTR_FORMAT") to CS.", - hr->bottom(), hr->end()); - gclog_or_tty->print_cr(" (" SIZE_FORMAT " KB left in heap.)", - max_live_bytes/K); - } - hr = _g1->pop_region_from_young_list(); + assert(hr->is_survivor(), "badly formed young list"); + hr->set_young(); + hr = hr->get_next_young_region(); } - record_scan_only_regions(_g1->young_list_scan_only_length()); + // Clear the fields that point to the survivor list - they are + // all young now. + _g1->young_list()->clear_survivors(); + + if (_g1->mark_in_progress()) + _g1->concurrent_mark()->register_collection_set_finger(_inc_cset_max_finger); + + _young_cset_length = _inc_cset_young_index; + _collection_set = _inc_cset_head; + _collection_set_size = _inc_cset_size; + _collection_set_bytes_used_before = _inc_cset_bytes_used_before; + + // For young regions in the collection set, we assume the worst + // case of complete survival + max_live_bytes -= _inc_cset_size * HeapRegion::GrainBytes; + + time_remaining_ms -= _inc_cset_predicted_elapsed_time_ms; + predicted_pause_time_ms += _inc_cset_predicted_elapsed_time_ms; + + // The number of recorded young regions is the incremental + // collection set's current size + set_recorded_young_regions(_inc_cset_size); + set_recorded_rs_lengths(_inc_cset_recorded_rs_lengths); + set_recorded_young_bytes(_inc_cset_recorded_young_bytes); +#if PREDICTIONS_VERBOSE + set_predicted_bytes_to_copy(_inc_cset_predicted_bytes_to_copy); +#endif // PREDICTIONS_VERBOSE + + if (G1PolicyVerbose > 0) { + gclog_or_tty->print_cr(" Added " PTR_FORMAT " Young Regions to CS.", + _inc_cset_size); + gclog_or_tty->print_cr(" (" SIZE_FORMAT " KB left in heap.)", + max_live_bytes/K); + } + + assert(_inc_cset_size == _g1->young_list()->length(), "Invariant"); + if (_inc_cset_size > 0) { + assert(_collection_set != NULL, "Invariant"); + abandon_collection = false; + } double young_end_time_sec = os::elapsedTime(); _recorded_young_cset_choice_time_ms = (young_end_time_sec - young_start_time_sec) * 1000.0; - non_young_start_time_sec = os::elapsedTime(); - - if (_young_cset_length > 0 && _last_young_gc_full) { + // We are doing young collections so reset this. + non_young_start_time_sec = young_end_time_sec; + + // Note we can use either _collection_set_size or + // _young_cset_length here + if (_collection_set_size > 0 && _last_young_gc_full) { // don't bother adding more regions... goto choose_collection_set_end; } @@ -3077,6 +2946,11 @@ bool should_continue = true; NumberSeq seq; double avg_prediction = 100000000000000000.0; // something very large + + // Save the current size of the collection set to detect + // if we actually added any old regions. + size_t n_young_regions = _collection_set_size; + do { hr = _collectionSetChooser->getNextMarkedRegion(time_remaining_ms, avg_prediction); @@ -3085,7 +2959,7 @@ time_remaining_ms -= predicted_time_ms; predicted_pause_time_ms += predicted_time_ms; add_to_collection_set(hr); - record_cset_region(hr, false); + record_non_young_cset_region(hr); max_live_bytes -= MIN2(hr->max_live_bytes(), max_live_bytes); if (G1PolicyVerbose > 0) { gclog_or_tty->print_cr(" (" SIZE_FORMAT " KB left in heap.)", @@ -3103,9 +2977,17 @@ if (!adaptive_young_list_length() && _collection_set_size < _young_list_fixed_length) _should_revert_to_full_young_gcs = true; + + if (_collection_set_size > n_young_regions) { + // We actually added old regions to the collection set + // so we are not abandoning this collection. + abandon_collection = false; + } } choose_collection_set_end: + stop_incremental_cset_building(); + count_CS_bytes_used(); end_recording_regions(); @@ -3113,6 +2995,8 @@ double non_young_end_time_sec = os::elapsedTime(); _recorded_non_young_cset_choice_time_ms = (non_young_end_time_sec - non_young_start_time_sec) * 1000.0; + + return abandon_collection; } void G1CollectorPolicy_BestRegionsFirst::record_full_collection_end() { diff -r 6ecb6e6de3d6 -r 1316cec51b4d src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp --- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp Mon Apr 19 05:40:21 2010 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp Thu Apr 22 10:02:38 2010 -0700 @@ -1,5 +1,5 @@ /* - * Copyright 2001-2009 Sun Microsystems, Inc. All Rights Reserved. + * Copyright 2001-2010 Sun Microsystems, Inc. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -61,7 +61,6 @@ define_num_seq(parallel) // parallel only define_num_seq(ext_root_scan) define_num_seq(mark_stack_scan) - define_num_seq(scan_only) define_num_seq(update_rs) define_num_seq(scan_rs) define_num_seq(scan_new_refs) // Only for temp use; added to @@ -174,8 +173,6 @@ double* _par_last_ext_root_scan_times_ms; double* _par_last_mark_stack_scan_times_ms; - double* _par_last_scan_only_times_ms; - double* _par_last_scan_only_regions_scanned; double* _par_last_update_rs_start_times_ms; double* _par_last_update_rs_times_ms; double* _par_last_update_rs_processed_buffers; @@ -196,7 +193,6 @@ bool _adaptive_young_list_length; size_t _young_list_min_length; size_t _young_list_target_length; - size_t _young_list_so_prefix_length; size_t _young_list_fixed_length; size_t _young_cset_length; @@ -234,7 +230,6 @@ TruncatedSeq* _pending_card_diff_seq; TruncatedSeq* _rs_length_diff_seq; TruncatedSeq* _cost_per_card_ms_seq; - TruncatedSeq* _cost_per_scan_only_region_ms_seq; TruncatedSeq* _fully_young_cards_per_entry_ratio_seq; TruncatedSeq* _partially_young_cards_per_entry_ratio_seq; TruncatedSeq* _cost_per_entry_ms_seq; @@ -249,19 +244,16 @@ TruncatedSeq* _rs_lengths_seq; TruncatedSeq* _cost_per_byte_ms_during_cm_seq; - TruncatedSeq* _cost_per_scan_only_region_ms_during_cm_seq; TruncatedSeq* _young_gc_eff_seq; TruncatedSeq* _max_conc_overhead_seq; size_t _recorded_young_regions; - size_t _recorded_scan_only_regions; size_t _recorded_non_young_regions; size_t _recorded_region_num; size_t _free_regions_at_end_of_collection; - size_t _scan_only_regions_at_end_of_collection; size_t _recorded_rs_lengths; size_t _max_rs_lengths; @@ -277,7 +269,6 @@ double _predicted_survival_ratio; double _predicted_rs_update_time_ms; double _predicted_rs_scan_time_ms; - double _predicted_scan_only_scan_time_ms; double _predicted_object_copy_time_ms; double _predicted_constant_other_time_ms; double _predicted_young_other_time_ms; @@ -344,8 +335,6 @@ bool verify_young_ages(); #endif // PRODUCT - void tag_scan_only(size_t short_lived_scan_only_length); - double get_new_prediction(TruncatedSeq* seq) { return MAX2(seq->davg() + sigma() * seq->dsd(), seq->davg() * confidence_factor(seq->num())); @@ -431,23 +420,6 @@ get_new_prediction(_partially_young_cost_per_entry_ms_seq); } - double predict_scan_only_time_ms_during_cm(size_t scan_only_region_num) { - if (_cost_per_scan_only_region_ms_during_cm_seq->num() < 3) - return 1.5 * (double) scan_only_region_num * - get_new_prediction(_cost_per_scan_only_region_ms_seq); - else - return (double) scan_only_region_num * - get_new_prediction(_cost_per_scan_only_region_ms_during_cm_seq); - } - - double predict_scan_only_time_ms(size_t scan_only_region_num) { - if (_in_marking_window_im) - return predict_scan_only_time_ms_during_cm(scan_only_region_num); - else - return (double) scan_only_region_num * - get_new_prediction(_cost_per_scan_only_region_ms_seq); - } - double predict_object_copy_time_ms_during_cm(size_t bytes_to_copy) { if (_cost_per_byte_ms_during_cm_seq->num() < 3) return 1.1 * (double) bytes_to_copy * @@ -490,24 +462,21 @@ size_t predict_bytes_to_copy(HeapRegion* hr); double predict_region_elapsed_time_ms(HeapRegion* hr, bool young); - // for use by: calculate_optimal_so_length(length) - void predict_gc_eff(size_t young_region_num, - size_t so_length, - double base_time_ms, - double *gc_eff, - double *pause_time_ms); - - // for use by: calculate_young_list_target_config(rs_length) - bool predict_gc_eff(size_t young_region_num, - size_t so_length, - double base_time_with_so_ms, - size_t init_free_regions, - double target_pause_time_ms, - double* gc_eff); + // for use by: calculate_young_list_target_length(rs_length) + bool predict_will_fit(size_t young_region_num, + double base_time_ms, + size_t init_free_regions, + double target_pause_time_ms); void start_recording_regions(); - void record_cset_region(HeapRegion* hr, bool young); - void record_scan_only_regions(size_t scan_only_length); + void record_cset_region_info(HeapRegion* hr, bool young); + void record_non_young_cset_region(HeapRegion* hr); + + void set_recorded_young_regions(size_t n_regions); + void set_recorded_young_bytes(size_t bytes); + void set_recorded_rs_lengths(size_t rs_lengths); + void set_predicted_bytes_to_copy(size_t bytes); + void end_recording_regions(); void record_vtime_diff_ms(double vtime_diff_ms) { @@ -638,11 +607,74 @@ void update_recent_gc_times(double end_time_sec, double elapsed_ms); // The head of the list (via "next_in_collection_set()") representing the - // current collection set. + // current collection set. Set from the incrementally built collection + // set at the start of the pause. HeapRegion* _collection_set; + + // The number of regions in the collection set. Set from the incrementally + // built collection set at the start of an evacuation pause. size_t _collection_set_size; + + // The number of bytes in the collection set before the pause. Set from + // the incrementally built collection set at the start of an evacuation + // pause. size_t _collection_set_bytes_used_before; + // The associated information that is maintained while the incremental + // collection set is being built with young regions. Used to populate + // the recorded info for the evacuation pause. + + enum CSetBuildType { + Active, // We are actively building the collection set + Inactive // We are not actively building the collection set + }; + + CSetBuildType _inc_cset_build_state; + + // The head of the incrementally built collection set. + HeapRegion* _inc_cset_head; + + // The tail of the incrementally built collection set. + HeapRegion* _inc_cset_tail; + + // The number of regions in the incrementally built collection set. + // Used to set _collection_set_size at the start of an evacuation + // pause. + size_t _inc_cset_size; + + // Used as the index in the surving young words structure + // which tracks the amount of space, for each young region, + // that survives the pause. + size_t _inc_cset_young_index; + + // The number of bytes in the incrementally built collection set. + // Used to set _collection_set_bytes_used_before at the start of + // an evacuation pause. + size_t _inc_cset_bytes_used_before; + + // Used to record the highest end of heap region in collection set + HeapWord* _inc_cset_max_finger; + + // The number of recorded used bytes in the young regions + // of the collection set. This is the sum of the used() bytes + // of retired young regions in the collection set. + size_t _inc_cset_recorded_young_bytes; + + // The RSet lengths recorded for regions in the collection set + // (updated by the periodic sampling of the regions in the + // young list/collection set). + size_t _inc_cset_recorded_rs_lengths; + + // The predicted elapsed time it will take to collect the regions + // in the collection set (updated by the periodic sampling of the + // regions in the young list/collection set). + double _inc_cset_predicted_elapsed_time_ms; + + // The predicted bytes to copy for the regions in the collection + // set (updated by the periodic sampling of the regions in the + // young list/collection set). + size_t _inc_cset_predicted_bytes_to_copy; + // Info about marking. int _n_marks; // Sticky at 2, so we know when we've done at least 2. @@ -761,9 +793,8 @@ double _mark_closure_time_ms; void calculate_young_list_min_length(); - void calculate_young_list_target_config(); - void calculate_young_list_target_config(size_t rs_lengths); - size_t calculate_optimal_so_length(size_t young_list_length); + void calculate_young_list_target_length(); + void calculate_young_list_target_length(size_t rs_lengths); public: @@ -868,11 +899,6 @@ _par_last_mark_stack_scan_times_ms[worker_i] = ms; } - void record_scan_only_time(int worker_i, double ms, int n) { - _par_last_scan_only_times_ms[worker_i] = ms; - _par_last_scan_only_regions_scanned[worker_i] = (double) n; - } - void record_satb_drain_time(double ms) { _cur_satb_drain_time_ms = ms; _satb_drain_time_set = true; @@ -987,20 +1013,67 @@ // Choose a new collection set. Marks the chosen regions as being // "in_collection_set", and links them together. The head and number of // the collection set are available via access methods. - virtual void choose_collection_set() = 0; - - void clear_collection_set() { _collection_set = NULL; } + virtual bool choose_collection_set() = 0; // The head of the list (via "next_in_collection_set()") representing the // current collection set. HeapRegion* collection_set() { return _collection_set; } + void clear_collection_set() { _collection_set = NULL; } + // The number of elements in the current collection set. size_t collection_set_size() { return _collection_set_size; } // Add "hr" to the CS. void add_to_collection_set(HeapRegion* hr); + // Incremental CSet Support + + // The head of the incrementally built collection set. + HeapRegion* inc_cset_head() { return _inc_cset_head; } + + // The tail of the incrementally built collection set. + HeapRegion* inc_set_tail() { return _inc_cset_tail; } + + // The number of elements in the incrementally built collection set. + size_t inc_cset_size() { return _inc_cset_size; } + + // Initialize incremental collection set info. + void start_incremental_cset_building(); + + void clear_incremental_cset() { + _inc_cset_head = NULL; + _inc_cset_tail = NULL; + } + + // Stop adding regions to the incremental collection set + void stop_incremental_cset_building() { _inc_cset_build_state = Inactive; } + + // Add/remove information about hr to the aggregated information + // for the incrementally built collection set. + void add_to_incremental_cset_info(HeapRegion* hr, size_t rs_length); + void remove_from_incremental_cset_info(HeapRegion* hr); + + // Update information about hr in the aggregated information for + // the incrementally built collection set. + void update_incremental_cset_info(HeapRegion* hr, size_t new_rs_length); + +private: + // Update the incremental cset information when adding a region + // (should not be called directly). + void add_region_to_incremental_cset_common(HeapRegion* hr); + +public: + // Add hr to the LHS of the incremental collection set. + void add_region_to_incremental_cset_lhs(HeapRegion* hr); + + // Add hr to the RHS of the incremental collection set. + void add_region_to_incremental_cset_rhs(HeapRegion* hr); + +#ifndef PRODUCT + void print_collection_set(HeapRegion* list_head, outputStream* st); +#endif // !PRODUCT + bool initiate_conc_mark_if_possible() { return _initiate_conc_mark_if_possible; } void set_initiate_conc_mark_if_possible() { _initiate_conc_mark_if_possible = true; } void clear_initiate_conc_mark_if_possible() { _initiate_conc_mark_if_possible = false; } @@ -1191,7 +1264,7 @@ // If the estimated is less then desirable, resize if possible. void expand_if_possible(size_t numRegions); - virtual void choose_collection_set(); + virtual bool choose_collection_set(); virtual void record_collection_pause_start(double start_time_sec, size_t start_used); virtual void record_concurrent_mark_cleanup_end(size_t freed_bytes, diff -r 6ecb6e6de3d6 -r 1316cec51b4d src/share/vm/gc_implementation/g1/g1_globals.hpp --- a/src/share/vm/gc_implementation/g1/g1_globals.hpp Mon Apr 19 05:40:21 2010 -0700 +++ b/src/share/vm/gc_implementation/g1/g1_globals.hpp Thu Apr 22 10:02:38 2010 -0700 @@ -1,5 +1,5 @@ /* - * Copyright 2001-2009 Sun Microsystems, Inc. All Rights Reserved. + * Copyright 2001-2010 Sun Microsystems, Inc. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -226,10 +226,6 @@ "the number of regions for which we'll print a surv rate " \ "summary.") \ \ - develop(bool, G1UseScanOnlyPrefix, false, \ - "It determines whether the system will calculate an optimum " \ - "scan-only set.") \ - \ product(intx, G1ReservePercent, 10, \ "It determines the minimum reserve we should have in the heap " \ "to minimize the probability of promotion failure.") \ diff -r 6ecb6e6de3d6 -r 1316cec51b4d src/share/vm/gc_implementation/g1/heapRegion.cpp --- a/src/share/vm/gc_implementation/g1/heapRegion.cpp Mon Apr 19 05:40:21 2010 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegion.cpp Thu Apr 22 10:02:38 2010 -0700 @@ -1,5 +1,5 @@ /* - * Copyright 2001-2009 Sun Microsystems, Inc. All Rights Reserved. + * Copyright 2001-2010 Sun Microsystems, Inc. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -450,7 +450,9 @@ _young_type(NotYoung), _next_young_region(NULL), _next_dirty_cards_region(NULL), _young_index_in_cset(-1), _surv_rate_group(NULL), _age_index(-1), - _rem_set(NULL), _zfs(NotZeroFilled) + _rem_set(NULL), _zfs(NotZeroFilled), + _recorded_rs_length(0), _predicted_elapsed_time_ms(0), + _predicted_bytes_to_copy(0) { _orig_end = mr.end(); // Note that initialize() will set the start of the unmarked area of the @@ -733,7 +735,7 @@ else st->print(" "); if (is_young()) - st->print(is_scan_only() ? " SO" : (is_survivor() ? " SU" : " Y ")); + st->print(is_survivor() ? " SU" : " Y "); else st->print(" "); if (is_empty()) diff -r 6ecb6e6de3d6 -r 1316cec51b4d src/share/vm/gc_implementation/g1/heapRegion.hpp --- a/src/share/vm/gc_implementation/g1/heapRegion.hpp Mon Apr 19 05:40:21 2010 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegion.hpp Thu Apr 22 10:02:38 2010 -0700 @@ -1,5 +1,5 @@ /* - * Copyright 2001-2009 Sun Microsystems, Inc. All Rights Reserved. + * Copyright 2001-2010 Sun Microsystems, Inc. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -247,7 +247,6 @@ enum YoungType { NotYoung, // a region is not young - ScanOnly, // a region is young and scan-only Young, // a region is young Survivor // a region is young and it contains // survivor @@ -292,6 +291,20 @@ _young_type = new_type; } + // Cached attributes used in the collection set policy information + + // The RSet length that was added to the total value + // for the collection set. + size_t _recorded_rs_length; + + // The predicted elapsed time that was added to total value + // for the collection set. + double _predicted_elapsed_time_ms; + + // The predicted number of bytes to copy that was added to + // the total value for the collection set. + size_t _predicted_bytes_to_copy; + public: // If "is_zeroed" is "true", the region "mr" can be assumed to contain zeros. HeapRegion(G1BlockOffsetSharedArray* sharedOffsetArray, @@ -614,7 +627,6 @@ // bool is_young() const { return _young_type != NotYoung; } - bool is_scan_only() const { return _young_type == ScanOnly; } bool is_survivor() const { return _young_type == Survivor; } int young_index_in_cset() const { return _young_index_in_cset; } @@ -629,12 +641,6 @@ return _surv_rate_group->age_in_group(_age_index); } - void recalculate_age_in_surv_rate_group() { - assert( _surv_rate_group != NULL, "pre-condition" ); - assert( _age_index > -1, "pre-condition" ); - _age_index = _surv_rate_group->recalculate_age_index(_age_index); - } - void record_surv_words_in_group(size_t words_survived) { assert( _surv_rate_group != NULL, "pre-condition" ); assert( _age_index > -1, "pre-condition" ); @@ -676,8 +682,6 @@ void set_young() { set_young_type(Young); } - void set_scan_only() { set_young_type(ScanOnly); } - void set_survivor() { set_young_type(Survivor); } void set_not_young() { set_young_type(NotYoung); } @@ -775,6 +779,22 @@ _zero_filler = NULL; } + size_t recorded_rs_length() const { return _recorded_rs_length; } + double predicted_elapsed_time_ms() const { return _predicted_elapsed_time_ms; } + size_t predicted_bytes_to_copy() const { return _predicted_bytes_to_copy; } + + void set_recorded_rs_length(size_t rs_length) { + _recorded_rs_length = rs_length; + } + + void set_predicted_elapsed_time_ms(double ms) { + _predicted_elapsed_time_ms = ms; + } + + void set_predicted_bytes_to_copy(size_t bytes) { + _predicted_bytes_to_copy = bytes; + } + #define HeapRegion_OOP_SINCE_SAVE_MARKS_DECL(OopClosureType, nv_suffix) \ virtual void oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl); SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES(HeapRegion_OOP_SINCE_SAVE_MARKS_DECL) diff -r 6ecb6e6de3d6 -r 1316cec51b4d src/share/vm/gc_implementation/g1/survRateGroup.cpp --- a/src/share/vm/gc_implementation/g1/survRateGroup.cpp Mon Apr 19 05:40:21 2010 -0700 +++ b/src/share/vm/gc_implementation/g1/survRateGroup.cpp Thu Apr 22 10:02:38 2010 -0700 @@ -1,5 +1,5 @@ /* - * Copyright 2001-2009 Sun Microsystems, Inc. All Rights Reserved. + * Copyright 2001-2010 Sun Microsystems, Inc. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -55,7 +55,6 @@ void SurvRateGroup::reset() { _all_regions_allocated = 0; - _scan_only_prefix = 0; _setup_seq_num = 0; _stats_arrays_length = 0; _accum_surv_rate = 0.0; @@ -74,7 +73,7 @@ void SurvRateGroup::start_adding_regions() { _setup_seq_num = _stats_arrays_length; - _region_num = _scan_only_prefix; + _region_num = 0; _accum_surv_rate = 0.0; #if 0 @@ -164,12 +163,6 @@ } void -SurvRateGroup::record_scan_only_prefix(size_t scan_only_prefix) { - guarantee( scan_only_prefix <= _region_num, "pre-condition" ); - _scan_only_prefix = scan_only_prefix; -} - -void SurvRateGroup::record_surviving_words(int age_in_group, size_t surv_words) { guarantee( 0 <= age_in_group && (size_t) age_in_group < _region_num, "pre-condition" ); @@ -218,13 +211,12 @@ #ifndef PRODUCT void SurvRateGroup::print() { - gclog_or_tty->print_cr("Surv Rate Group: %s (%d entries, %d scan-only)", - _name, _region_num, _scan_only_prefix); + gclog_or_tty->print_cr("Surv Rate Group: %s (%d entries)", + _name, _region_num); for (size_t i = 0; i < _region_num; ++i) { - gclog_or_tty->print_cr(" age %4d surv rate %6.2lf %% pred %6.2lf %%%s", + gclog_or_tty->print_cr(" age %4d surv rate %6.2lf %% pred %6.2lf %%", i, _surv_rate[i] * 100.0, - _g1p->get_new_prediction(_surv_rate_pred[i]) * 100.0, - (i < _scan_only_prefix) ? " S-O" : " "); + _g1p->get_new_prediction(_surv_rate_pred[i]) * 100.0); } } diff -r 6ecb6e6de3d6 -r 1316cec51b4d src/share/vm/gc_implementation/g1/survRateGroup.hpp --- a/src/share/vm/gc_implementation/g1/survRateGroup.hpp Mon Apr 19 05:40:21 2010 -0700 +++ b/src/share/vm/gc_implementation/g1/survRateGroup.hpp Thu Apr 22 10:02:38 2010 -0700 @@ -1,5 +1,5 @@ /* - * Copyright 2001-2009 Sun Microsystems, Inc. All Rights Reserved. + * Copyright 2001-2010 Sun Microsystems, Inc. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -41,7 +41,6 @@ int _all_regions_allocated; size_t _region_num; - size_t _scan_only_prefix; size_t _setup_seq_num; public: @@ -51,13 +50,11 @@ void reset(); void start_adding_regions(); void stop_adding_regions(); - void record_scan_only_prefix(size_t scan_only_prefix); void record_surviving_words(int age_in_group, size_t surv_words); void all_surviving_words_recorded(bool propagate); const char* name() { return _name; } size_t region_num() { return _region_num; } - size_t scan_only_length() { return _scan_only_prefix; } double accum_surv_rate_pred(int age) { assert(age >= 0, "must be"); if ((size_t)age < _stats_arrays_length) @@ -82,17 +79,12 @@ int next_age_index(); int age_in_group(int age_index) { - int ret = (int) (_all_regions_allocated - age_index); + int ret = (int) (_all_regions_allocated - age_index); assert( ret >= 0, "invariant" ); return ret; } - int recalculate_age_index(int age_index) { - int new_age_index = (int) _scan_only_prefix - age_in_group(age_index); - guarantee( new_age_index >= 0, "invariant" ); - return new_age_index; - } void finished_recalculating_age_indexes() { - _all_regions_allocated = (int) _scan_only_prefix; + _all_regions_allocated = 0; } #ifndef PRODUCT diff -r 6ecb6e6de3d6 -r 1316cec51b4d src/share/vm/services/g1MemoryPool.cpp --- a/src/share/vm/services/g1MemoryPool.cpp Mon Apr 19 05:40:21 2010 -0700 +++ b/src/share/vm/services/g1MemoryPool.cpp Thu Apr 22 10:02:38 2010 -0700 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007 Sun Microsystems, Inc. All Rights Reserved. + * Copyright 2007-2010 Sun Microsystems, Inc. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -45,7 +45,7 @@ // See the comment at the top of g1MemoryPool.hpp size_t G1MemoryPoolSuper::eden_space_used(G1CollectedHeap* g1h) { - size_t young_list_length = g1h->young_list_length(); + size_t young_list_length = g1h->young_list()->length(); size_t eden_used = young_list_length * HeapRegion::GrainBytes; size_t survivor_used = survivor_space_used(g1h); eden_used = subtract_up_to_zero(eden_used, survivor_used);