Mercurial > hg > graal-compiler
diff src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp @ 342:37f87013dfd8
6711316: Open source the Garbage-First garbage collector
Summary: First mercurial integration of the code for the Garbage-First garbage collector.
Reviewed-by: apetrusenko, iveresov, jmasa, sgoldman, tonyp, ysr
author | ysr |
---|---|
date | Thu, 05 Jun 2008 15:57:56 -0700 |
parents | |
children | e0c09f7ec5c4 |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Thu Jun 05 15:57:56 2008 -0700 @@ -0,0 +1,5355 @@ +/* + * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + * + */ + +#include "incls/_precompiled.incl" +#include "incls/_g1CollectedHeap.cpp.incl" + +// turn it on so that the contents of the young list (scan-only / +// to-be-collected) are printed at "strategic" points before / during +// / after the collection --- this is useful for debugging +#define SCAN_ONLY_VERBOSE 0 +// CURRENT STATUS +// This file is under construction. Search for "FIXME". + +// INVARIANTS/NOTES +// +// All allocation activity covered by the G1CollectedHeap interface is +// serialized by acquiring the HeapLock. This happens in +// mem_allocate_work, which all such allocation functions call. +// (Note that this does not apply to TLAB allocation, which is not part +// of this interface: it is done by clients of this interface.) + +// Local to this file. + +// Finds the first HeapRegion. +// No longer used, but might be handy someday. + +class FindFirstRegionClosure: public HeapRegionClosure { + HeapRegion* _a_region; +public: + FindFirstRegionClosure() : _a_region(NULL) {} + bool doHeapRegion(HeapRegion* r) { + _a_region = r; + return true; + } + HeapRegion* result() { return _a_region; } +}; + + +class RefineCardTableEntryClosure: public CardTableEntryClosure { + SuspendibleThreadSet* _sts; + G1RemSet* _g1rs; + ConcurrentG1Refine* _cg1r; + bool _concurrent; +public: + RefineCardTableEntryClosure(SuspendibleThreadSet* sts, + G1RemSet* g1rs, + ConcurrentG1Refine* cg1r) : + _sts(sts), _g1rs(g1rs), _cg1r(cg1r), _concurrent(true) + {} + bool do_card_ptr(jbyte* card_ptr, int worker_i) { + _g1rs->concurrentRefineOneCard(card_ptr, worker_i); + if (_concurrent && _sts->should_yield()) { + // Caller will actually yield. + return false; + } + // Otherwise, we finished successfully; return true. + return true; + } + void set_concurrent(bool b) { _concurrent = b; } +}; + + +class ClearLoggedCardTableEntryClosure: public CardTableEntryClosure { + int _calls; + G1CollectedHeap* _g1h; + CardTableModRefBS* _ctbs; + int _histo[256]; +public: + ClearLoggedCardTableEntryClosure() : + _calls(0) + { + _g1h = G1CollectedHeap::heap(); + _ctbs = (CardTableModRefBS*)_g1h->barrier_set(); + for (int i = 0; i < 256; i++) _histo[i] = 0; + } + bool do_card_ptr(jbyte* card_ptr, int worker_i) { + if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) { + _calls++; + unsigned char* ujb = (unsigned char*)card_ptr; + int ind = (int)(*ujb); + _histo[ind]++; + *card_ptr = -1; + } + return true; + } + int calls() { return _calls; } + void print_histo() { + gclog_or_tty->print_cr("Card table value histogram:"); + for (int i = 0; i < 256; i++) { + if (_histo[i] != 0) { + gclog_or_tty->print_cr(" %d: %d", i, _histo[i]); + } + } + } +}; + +class RedirtyLoggedCardTableEntryClosure: public CardTableEntryClosure { + int _calls; + G1CollectedHeap* _g1h; + CardTableModRefBS* _ctbs; +public: + RedirtyLoggedCardTableEntryClosure() : + _calls(0) + { + _g1h = G1CollectedHeap::heap(); + _ctbs = (CardTableModRefBS*)_g1h->barrier_set(); + } + bool do_card_ptr(jbyte* card_ptr, int worker_i) { + if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) { + _calls++; + *card_ptr = 0; + } + return true; + } + int calls() { return _calls; } +}; + +YoungList::YoungList(G1CollectedHeap* g1h) + : _g1h(g1h), _head(NULL), + _scan_only_head(NULL), _scan_only_tail(NULL), _curr_scan_only(NULL), + _length(0), _scan_only_length(0), + _last_sampled_rs_lengths(0), + _survivor_head(NULL), _survivors_tail(NULL), _survivor_length(0) +{ + guarantee( check_list_empty(false), "just making sure..." ); +} + +void YoungList::push_region(HeapRegion *hr) { + assert(!hr->is_young(), "should not already be young"); + assert(hr->get_next_young_region() == NULL, "cause it should!"); + + hr->set_next_young_region(_head); + _head = hr; + + hr->set_young(); + double yg_surv_rate = _g1h->g1_policy()->predict_yg_surv_rate((int)_length); + ++_length; +} + +void YoungList::add_survivor_region(HeapRegion* hr) { + assert(!hr->is_survivor(), "should not already be for survived"); + assert(hr->get_next_young_region() == NULL, "cause it should!"); + + hr->set_next_young_region(_survivor_head); + if (_survivor_head == NULL) { + _survivors_tail = hr; + } + _survivor_head = hr; + + hr->set_survivor(); + ++_survivor_length; +} + +HeapRegion* YoungList::pop_region() { + while (_head != NULL) { + assert( length() > 0, "list should not be empty" ); + HeapRegion* ret = _head; + _head = ret->get_next_young_region(); + ret->set_next_young_region(NULL); + --_length; + assert(ret->is_young(), "region should be very young"); + + // Replace 'Survivor' region type with 'Young'. So the region will + // be treated as a young region and will not be 'confused' with + // newly created survivor regions. + if (ret->is_survivor()) { + ret->set_young(); + } + + if (!ret->is_scan_only()) { + return ret; + } + + // scan-only, we'll add it to the scan-only list + if (_scan_only_tail == NULL) { + guarantee( _scan_only_head == NULL, "invariant" ); + + _scan_only_head = ret; + _curr_scan_only = ret; + } else { + guarantee( _scan_only_head != NULL, "invariant" ); + _scan_only_tail->set_next_young_region(ret); + } + guarantee( ret->get_next_young_region() == NULL, "invariant" ); + _scan_only_tail = ret; + + // no need to be tagged as scan-only any more + ret->set_young(); + + ++_scan_only_length; + } + assert( length() == 0, "list should be empty" ); + return NULL; +} + +void YoungList::empty_list(HeapRegion* list) { + while (list != NULL) { + HeapRegion* next = list->get_next_young_region(); + list->set_next_young_region(NULL); + list->uninstall_surv_rate_group(); + list->set_not_young(); + list = next; + } +} + +void YoungList::empty_list() { + assert(check_list_well_formed(), "young list should be well formed"); + + empty_list(_head); + _head = NULL; + _length = 0; + + empty_list(_scan_only_head); + _scan_only_head = NULL; + _scan_only_tail = NULL; + _scan_only_length = 0; + _curr_scan_only = NULL; + + empty_list(_survivor_head); + _survivor_head = NULL; + _survivors_tail = NULL; + _survivor_length = 0; + + _last_sampled_rs_lengths = 0; + + assert(check_list_empty(false), "just making sure..."); +} + +bool YoungList::check_list_well_formed() { + bool ret = true; + + size_t length = 0; + HeapRegion* curr = _head; + HeapRegion* last = NULL; + while (curr != NULL) { + if (!curr->is_young() || curr->is_scan_only()) { + gclog_or_tty->print_cr("### YOUNG REGION "PTR_FORMAT"-"PTR_FORMAT" " + "incorrectly tagged (%d, %d)", + curr->bottom(), curr->end(), + curr->is_young(), curr->is_scan_only()); + ret = false; + } + ++length; + last = curr; + curr = curr->get_next_young_region(); + } + ret = ret && (length == _length); + + if (!ret) { + gclog_or_tty->print_cr("### YOUNG LIST seems not well formed!"); + gclog_or_tty->print_cr("### list has %d entries, _length is %d", + length, _length); + } + + bool scan_only_ret = true; + length = 0; + curr = _scan_only_head; + last = NULL; + while (curr != NULL) { + if (!curr->is_young() || curr->is_scan_only()) { + gclog_or_tty->print_cr("### SCAN-ONLY REGION "PTR_FORMAT"-"PTR_FORMAT" " + "incorrectly tagged (%d, %d)", + curr->bottom(), curr->end(), + curr->is_young(), curr->is_scan_only()); + scan_only_ret = false; + } + ++length; + last = curr; + curr = curr->get_next_young_region(); + } + scan_only_ret = scan_only_ret && (length == _scan_only_length); + + if ( (last != _scan_only_tail) || + (_scan_only_head == NULL && _scan_only_tail != NULL) || + (_scan_only_head != NULL && _scan_only_tail == NULL) ) { + gclog_or_tty->print_cr("## _scan_only_tail is set incorrectly"); + scan_only_ret = false; + } + + if (_curr_scan_only != NULL && _curr_scan_only != _scan_only_head) { + gclog_or_tty->print_cr("### _curr_scan_only is set incorrectly"); + scan_only_ret = false; + } + + if (!scan_only_ret) { + gclog_or_tty->print_cr("### SCAN-ONLY LIST seems not well formed!"); + gclog_or_tty->print_cr("### list has %d entries, _scan_only_length is %d", + length, _scan_only_length); + } + + return ret && scan_only_ret; +} + +bool YoungList::check_list_empty(bool ignore_scan_only_list, + bool check_sample) { + bool ret = true; + + if (_length != 0) { + gclog_or_tty->print_cr("### YOUNG LIST should have 0 length, not %d", + _length); + ret = false; + } + if (check_sample && _last_sampled_rs_lengths != 0) { + gclog_or_tty->print_cr("### YOUNG LIST has non-zero last sampled RS lengths"); + ret = false; + } + if (_head != NULL) { + gclog_or_tty->print_cr("### YOUNG LIST does not have a NULL head"); + ret = false; + } + if (!ret) { + gclog_or_tty->print_cr("### YOUNG LIST does not seem empty"); + } + + if (ignore_scan_only_list) + return ret; + + bool scan_only_ret = true; + if (_scan_only_length != 0) { + gclog_or_tty->print_cr("### SCAN-ONLY LIST should have 0 length, not %d", + _scan_only_length); + scan_only_ret = false; + } + if (_scan_only_head != NULL) { + gclog_or_tty->print_cr("### SCAN-ONLY LIST does not have a NULL head"); + scan_only_ret = false; + } + if (_scan_only_tail != NULL) { + gclog_or_tty->print_cr("### SCAN-ONLY LIST does not have a NULL tail"); + scan_only_ret = false; + } + if (!scan_only_ret) { + gclog_or_tty->print_cr("### SCAN-ONLY LIST does not seem empty"); + } + + return ret && scan_only_ret; +} + +void +YoungList::rs_length_sampling_init() { + _sampled_rs_lengths = 0; + _curr = _head; +} + +bool +YoungList::rs_length_sampling_more() { + return _curr != NULL; +} + +void +YoungList::rs_length_sampling_next() { + assert( _curr != NULL, "invariant" ); + _sampled_rs_lengths += _curr->rem_set()->occupied(); + _curr = _curr->get_next_young_region(); + if (_curr == NULL) { + _last_sampled_rs_lengths = _sampled_rs_lengths; + // gclog_or_tty->print_cr("last sampled RS lengths = %d", _last_sampled_rs_lengths); + } +} + +void +YoungList::reset_auxilary_lists() { + // We could have just "moved" the scan-only list to the young list. + // However, the scan-only list is ordered according to the region + // age in descending order, so, by moving one entry at a time, we + // ensure that it is recreated in ascending order. + + guarantee( is_empty(), "young list should be empty" ); + assert(check_list_well_formed(), "young list should be well formed"); + + // Add survivor regions to SurvRateGroup. + _g1h->g1_policy()->note_start_adding_survivor_regions(); + for (HeapRegion* curr = _survivor_head; + curr != NULL; + curr = curr->get_next_young_region()) { + _g1h->g1_policy()->set_region_survivors(curr); + } + _g1h->g1_policy()->note_stop_adding_survivor_regions(); + + if (_survivor_head != NULL) { + _head = _survivor_head; + _length = _survivor_length + _scan_only_length; + _survivors_tail->set_next_young_region(_scan_only_head); + } else { + _head = _scan_only_head; + _length = _scan_only_length; + } + + for (HeapRegion* curr = _scan_only_head; + curr != NULL; + curr = curr->get_next_young_region()) { + curr->recalculate_age_in_surv_rate_group(); + } + _scan_only_head = NULL; + _scan_only_tail = NULL; + _scan_only_length = 0; + _curr_scan_only = NULL; + + _survivor_head = NULL; + _survivors_tail = NULL; + _survivor_length = 0; + _g1h->g1_policy()->finished_recalculating_age_indexes(); + + assert(check_list_well_formed(), "young list should be well formed"); +} + +void YoungList::print() { + HeapRegion* lists[] = {_head, _scan_only_head, _survivor_head}; + const char* names[] = {"YOUNG", "SCAN-ONLY", "SURVIVOR"}; + + for (unsigned int list = 0; list < ARRAY_SIZE(lists); ++list) { + gclog_or_tty->print_cr("%s LIST CONTENTS", names[list]); + HeapRegion *curr = lists[list]; + if (curr == NULL) + gclog_or_tty->print_cr(" empty"); + while (curr != NULL) { + gclog_or_tty->print_cr(" [%08x-%08x], t: %08x, P: %08x, N: %08x, C: %08x, " + "age: %4d, y: %d, s-o: %d, surv: %d", + curr->bottom(), curr->end(), + curr->top(), + curr->prev_top_at_mark_start(), + curr->next_top_at_mark_start(), + curr->top_at_conc_mark_count(), + curr->age_in_surv_rate_group_cond(), + curr->is_young(), + curr->is_scan_only(), + curr->is_survivor()); + curr = curr->get_next_young_region(); + } + } + + gclog_or_tty->print_cr(""); +} + +void G1CollectedHeap::stop_conc_gc_threads() { + _cg1r->cg1rThread()->stop(); + _czft->stop(); + _cmThread->stop(); +} + + +void G1CollectedHeap::check_ct_logs_at_safepoint() { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + CardTableModRefBS* ct_bs = (CardTableModRefBS*)barrier_set(); + + // Count the dirty cards at the start. + CountNonCleanMemRegionClosure count1(this); + ct_bs->mod_card_iterate(&count1); + int orig_count = count1.n(); + + // First clear the logged cards. + ClearLoggedCardTableEntryClosure clear; + dcqs.set_closure(&clear); + dcqs.apply_closure_to_all_completed_buffers(); + dcqs.iterate_closure_all_threads(false); + clear.print_histo(); + + // Now ensure that there's no dirty cards. + CountNonCleanMemRegionClosure count2(this); + ct_bs->mod_card_iterate(&count2); + if (count2.n() != 0) { + gclog_or_tty->print_cr("Card table has %d entries; %d originally", + count2.n(), orig_count); + } + guarantee(count2.n() == 0, "Card table should be clean."); + + RedirtyLoggedCardTableEntryClosure redirty; + JavaThread::dirty_card_queue_set().set_closure(&redirty); + dcqs.apply_closure_to_all_completed_buffers(); + dcqs.iterate_closure_all_threads(false); + gclog_or_tty->print_cr("Log entries = %d, dirty cards = %d.", + clear.calls(), orig_count); + guarantee(redirty.calls() == clear.calls(), + "Or else mechanism is broken."); + + CountNonCleanMemRegionClosure count3(this); + ct_bs->mod_card_iterate(&count3); + if (count3.n() != orig_count) { + gclog_or_tty->print_cr("Should have restored them all: orig = %d, final = %d.", + orig_count, count3.n()); + guarantee(count3.n() >= orig_count, "Should have restored them all."); + } + + JavaThread::dirty_card_queue_set().set_closure(_refine_cte_cl); +} + +// Private class members. + +G1CollectedHeap* G1CollectedHeap::_g1h; + +// Private methods. + +// Finds a HeapRegion that can be used to allocate a given size of block. + + +HeapRegion* G1CollectedHeap::newAllocRegion_work(size_t word_size, + bool do_expand, + bool zero_filled) { + ConcurrentZFThread::note_region_alloc(); + HeapRegion* res = alloc_free_region_from_lists(zero_filled); + if (res == NULL && do_expand) { + expand(word_size * HeapWordSize); + res = alloc_free_region_from_lists(zero_filled); + assert(res == NULL || + (!res->isHumongous() && + (!zero_filled || + res->zero_fill_state() == HeapRegion::Allocated)), + "Alloc Regions must be zero filled (and non-H)"); + } + if (res != NULL && res->is_empty()) _free_regions--; + assert(res == NULL || + (!res->isHumongous() && + (!zero_filled || + res->zero_fill_state() == HeapRegion::Allocated)), + "Non-young alloc Regions must be zero filled (and non-H)"); + + if (G1TraceRegions) { + if (res != NULL) { + gclog_or_tty->print_cr("new alloc region %d:["PTR_FORMAT", "PTR_FORMAT"], " + "top "PTR_FORMAT, + res->hrs_index(), res->bottom(), res->end(), res->top()); + } + } + + return res; +} + +HeapRegion* G1CollectedHeap::newAllocRegionWithExpansion(int purpose, + size_t word_size, + bool zero_filled) { + HeapRegion* alloc_region = NULL; + if (_gc_alloc_region_counts[purpose] < g1_policy()->max_regions(purpose)) { + alloc_region = newAllocRegion_work(word_size, true, zero_filled); + if (purpose == GCAllocForSurvived && alloc_region != NULL) { + _young_list->add_survivor_region(alloc_region); + } + ++_gc_alloc_region_counts[purpose]; + } else { + g1_policy()->note_alloc_region_limit_reached(purpose); + } + return alloc_region; +} + +// If could fit into free regions w/o expansion, try. +// Otherwise, if can expand, do so. +// Otherwise, if using ex regions might help, try with ex given back. +HeapWord* G1CollectedHeap::humongousObjAllocate(size_t word_size) { + assert(regions_accounted_for(), "Region leakage!"); + + // We can't allocate H regions while cleanupComplete is running, since + // some of the regions we find to be empty might not yet be added to the + // unclean list. (If we're already at a safepoint, this call is + // unnecessary, not to mention wrong.) + if (!SafepointSynchronize::is_at_safepoint()) + wait_for_cleanup_complete(); + + size_t num_regions = + round_to(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords; + + // Special case if < one region??? + + // Remember the ft size. + size_t x_size = expansion_regions(); + + HeapWord* res = NULL; + bool eliminated_allocated_from_lists = false; + + // Can the allocation potentially fit in the free regions? + if (free_regions() >= num_regions) { + res = _hrs->obj_allocate(word_size); + } + if (res == NULL) { + // Try expansion. + size_t fs = _hrs->free_suffix(); + if (fs + x_size >= num_regions) { + expand((num_regions - fs) * HeapRegion::GrainBytes); + res = _hrs->obj_allocate(word_size); + assert(res != NULL, "This should have worked."); + } else { + // Expansion won't help. Are there enough free regions if we get rid + // of reservations? + size_t avail = free_regions(); + if (avail >= num_regions) { + res = _hrs->obj_allocate(word_size); + if (res != NULL) { + remove_allocated_regions_from_lists(); + eliminated_allocated_from_lists = true; + } + } + } + } + if (res != NULL) { + // Increment by the number of regions allocated. + // FIXME: Assumes regions all of size GrainBytes. +#ifndef PRODUCT + mr_bs()->verify_clean_region(MemRegion(res, res + num_regions * + HeapRegion::GrainWords)); +#endif + if (!eliminated_allocated_from_lists) + remove_allocated_regions_from_lists(); + _summary_bytes_used += word_size * HeapWordSize; + _free_regions -= num_regions; + _num_humongous_regions += (int) num_regions; + } + assert(regions_accounted_for(), "Region Leakage"); + return res; +} + +HeapWord* +G1CollectedHeap::attempt_allocation_slow(size_t word_size, + bool permit_collection_pause) { + HeapWord* res = NULL; + HeapRegion* allocated_young_region = NULL; + + assert( SafepointSynchronize::is_at_safepoint() || + Heap_lock->owned_by_self(), "pre condition of the call" ); + + if (isHumongous(word_size)) { + // Allocation of a humongous object can, in a sense, complete a + // partial region, if the previous alloc was also humongous, and + // caused the test below to succeed. + if (permit_collection_pause) + do_collection_pause_if_appropriate(word_size); + res = humongousObjAllocate(word_size); + assert(_cur_alloc_region == NULL + || !_cur_alloc_region->isHumongous(), + "Prevent a regression of this bug."); + + } else { + // If we do a collection pause, this will be reset to a non-NULL + // value. If we don't, nulling here ensures that we allocate a new + // region below. + if (_cur_alloc_region != NULL) { + // We're finished with the _cur_alloc_region. + _summary_bytes_used += _cur_alloc_region->used(); + _cur_alloc_region = NULL; + } + assert(_cur_alloc_region == NULL, "Invariant."); + // Completion of a heap region is perhaps a good point at which to do + // a collection pause. + if (permit_collection_pause) + do_collection_pause_if_appropriate(word_size); + // Make sure we have an allocation region available. + if (_cur_alloc_region == NULL) { + if (!SafepointSynchronize::is_at_safepoint()) + wait_for_cleanup_complete(); + bool next_is_young = should_set_young_locked(); + // If the next region is not young, make sure it's zero-filled. + _cur_alloc_region = newAllocRegion(word_size, !next_is_young); + if (_cur_alloc_region != NULL) { + _summary_bytes_used -= _cur_alloc_region->used(); + if (next_is_young) { + set_region_short_lived_locked(_cur_alloc_region); + allocated_young_region = _cur_alloc_region; + } + } + } + assert(_cur_alloc_region == NULL || !_cur_alloc_region->isHumongous(), + "Prevent a regression of this bug."); + + // Now retry the allocation. + if (_cur_alloc_region != NULL) { + res = _cur_alloc_region->allocate(word_size); + } + } + + // NOTE: fails frequently in PRT + assert(regions_accounted_for(), "Region leakage!"); + + if (res != NULL) { + if (!SafepointSynchronize::is_at_safepoint()) { + assert( permit_collection_pause, "invariant" ); + assert( Heap_lock->owned_by_self(), "invariant" ); + Heap_lock->unlock(); + } + + if (allocated_young_region != NULL) { + HeapRegion* hr = allocated_young_region; + HeapWord* bottom = hr->bottom(); + HeapWord* end = hr->end(); + MemRegion mr(bottom, end); + ((CardTableModRefBS*)_g1h->barrier_set())->dirty(mr); + } + } + + assert( SafepointSynchronize::is_at_safepoint() || + (res == NULL && Heap_lock->owned_by_self()) || + (res != NULL && !Heap_lock->owned_by_self()), + "post condition of the call" ); + + return res; +} + +HeapWord* +G1CollectedHeap::mem_allocate(size_t word_size, + bool is_noref, + bool is_tlab, + bool* gc_overhead_limit_was_exceeded) { + debug_only(check_for_valid_allocation_state()); + assert(no_gc_in_progress(), "Allocation during gc not allowed"); + HeapWord* result = NULL; + + // Loop until the allocation is satisified, + // or unsatisfied after GC. + for (int try_count = 1; /* return or throw */; try_count += 1) { + int gc_count_before; + { + Heap_lock->lock(); + result = attempt_allocation(word_size); + if (result != NULL) { + // attempt_allocation should have unlocked the heap lock + assert(is_in(result), "result not in heap"); + return result; + } + // Read the gc count while the heap lock is held. + gc_count_before = SharedHeap::heap()->total_collections(); + Heap_lock->unlock(); + } + + // Create the garbage collection operation... + VM_G1CollectForAllocation op(word_size, + gc_count_before); + + // ...and get the VM thread to execute it. + VMThread::execute(&op); + if (op.prologue_succeeded()) { + result = op.result(); + assert(result == NULL || is_in(result), "result not in heap"); + return result; + } + + // Give a warning if we seem to be looping forever. + if ((QueuedAllocationWarningCount > 0) && + (try_count % QueuedAllocationWarningCount == 0)) { + warning("G1CollectedHeap::mem_allocate_work retries %d times", + try_count); + } + } +} + +void G1CollectedHeap::abandon_cur_alloc_region() { + if (_cur_alloc_region != NULL) { + // We're finished with the _cur_alloc_region. + if (_cur_alloc_region->is_empty()) { + _free_regions++; + free_region(_cur_alloc_region); + } else { + _summary_bytes_used += _cur_alloc_region->used(); + } + _cur_alloc_region = NULL; + } +} + +class PostMCRemSetClearClosure: public HeapRegionClosure { + ModRefBarrierSet* _mr_bs; +public: + PostMCRemSetClearClosure(ModRefBarrierSet* mr_bs) : _mr_bs(mr_bs) {} + bool doHeapRegion(HeapRegion* r) { + r->reset_gc_time_stamp(); + if (r->continuesHumongous()) + return false; + HeapRegionRemSet* hrrs = r->rem_set(); + if (hrrs != NULL) hrrs->clear(); + // You might think here that we could clear just the cards + // corresponding to the used region. But no: if we leave a dirty card + // in a region we might allocate into, then it would prevent that card + // from being enqueued, and cause it to be missed. + // Re: the performance cost: we shouldn't be doing full GC anyway! + _mr_bs->clear(MemRegion(r->bottom(), r->end())); + return false; + } +}; + + +class PostMCRemSetInvalidateClosure: public HeapRegionClosure { + ModRefBarrierSet* _mr_bs; +public: + PostMCRemSetInvalidateClosure(ModRefBarrierSet* mr_bs) : _mr_bs(mr_bs) {} + bool doHeapRegion(HeapRegion* r) { + if (r->continuesHumongous()) return false; + if (r->used_region().word_size() != 0) { + _mr_bs->invalidate(r->used_region(), true /*whole heap*/); + } + return false; + } +}; + +void G1CollectedHeap::do_collection(bool full, bool clear_all_soft_refs, + size_t word_size) { + ResourceMark rm; + + if (full && DisableExplicitGC) { + gclog_or_tty->print("\n\n\nDisabling Explicit GC\n\n\n"); + return; + } + + assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); + assert(Thread::current() == VMThread::vm_thread(), "should be in vm thread"); + + if (GC_locker::is_active()) { + return; // GC is disabled (e.g. JNI GetXXXCritical operation) + } + + { + IsGCActiveMark x; + + // Timing + gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps); + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); + TraceTime t(full ? "Full GC (System.gc())" : "Full GC", PrintGC, true, gclog_or_tty); + + double start = os::elapsedTime(); + GCOverheadReporter::recordSTWStart(start); + g1_policy()->record_full_collection_start(); + + gc_prologue(true); + increment_total_collections(); + + size_t g1h_prev_used = used(); + assert(used() == recalculate_used(), "Should be equal"); + + if (VerifyBeforeGC && total_collections() >= VerifyGCStartAt) { + HandleMark hm; // Discard invalid handles created during verification + prepare_for_verify(); + gclog_or_tty->print(" VerifyBeforeGC:"); + Universe::verify(true); + } + assert(regions_accounted_for(), "Region leakage!"); + + COMPILER2_PRESENT(DerivedPointerTable::clear()); + + // We want to discover references, but not process them yet. + // This mode is disabled in + // instanceRefKlass::process_discovered_references if the + // generation does some collection work, or + // instanceRefKlass::enqueue_discovered_references if the + // generation returns without doing any work. + ref_processor()->disable_discovery(); + ref_processor()->abandon_partial_discovery(); + ref_processor()->verify_no_references_recorded(); + + // Abandon current iterations of concurrent marking and concurrent + // refinement, if any are in progress. + concurrent_mark()->abort(); + + // Make sure we'll choose a new allocation region afterwards. + abandon_cur_alloc_region(); + assert(_cur_alloc_region == NULL, "Invariant."); + g1_rem_set()->as_HRInto_G1RemSet()->cleanupHRRS(); + tear_down_region_lists(); + set_used_regions_to_need_zero_fill(); + if (g1_policy()->in_young_gc_mode()) { + empty_young_list(); + g1_policy()->set_full_young_gcs(true); + } + + // Temporarily make reference _discovery_ single threaded (non-MT). + ReferenceProcessorMTMutator rp_disc_ser(ref_processor(), false); + + // Temporarily make refs discovery atomic + ReferenceProcessorAtomicMutator rp_disc_atomic(ref_processor(), true); + + // Temporarily clear _is_alive_non_header + ReferenceProcessorIsAliveMutator rp_is_alive_null(ref_processor(), NULL); + + ref_processor()->enable_discovery(); + + // Do collection work + { + HandleMark hm; // Discard invalid handles created during gc + G1MarkSweep::invoke_at_safepoint(ref_processor(), clear_all_soft_refs); + } + // Because freeing humongous regions may have added some unclean + // regions, it is necessary to tear down again before rebuilding. + tear_down_region_lists(); + rebuild_region_lists(); + + _summary_bytes_used = recalculate_used(); + + ref_processor()->enqueue_discovered_references(); + + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); + + if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) { + HandleMark hm; // Discard invalid handles created during verification + gclog_or_tty->print(" VerifyAfterGC:"); + Universe::verify(false); + } + NOT_PRODUCT(ref_processor()->verify_no_references_recorded()); + + reset_gc_time_stamp(); + // Since everything potentially moved, we will clear all remembered + // sets, and clear all cards. Later we will also cards in the used + // portion of the heap after the resizing (which could be a shrinking.) + // We will also reset the GC time stamps of the regions. + PostMCRemSetClearClosure rs_clear(mr_bs()); + heap_region_iterate(&rs_clear); + + // Resize the heap if necessary. + resize_if_necessary_after_full_collection(full ? 0 : word_size); + + // Since everything potentially moved, we will clear all remembered + // sets, but also dirty all cards corresponding to used regions. + PostMCRemSetInvalidateClosure rs_invalidate(mr_bs()); + heap_region_iterate(&rs_invalidate); + if (_cg1r->use_cache()) { + _cg1r->clear_and_record_card_counts(); + _cg1r->clear_hot_cache(); + } + + if (PrintGC) { + print_size_transition(gclog_or_tty, g1h_prev_used, used(), capacity()); + } + + if (true) { // FIXME + // Ask the permanent generation to adjust size for full collections + perm()->compute_new_size(); + } + + double end = os::elapsedTime(); + GCOverheadReporter::recordSTWEnd(end); + g1_policy()->record_full_collection_end(); + + gc_epilogue(true); + + // Abandon concurrent refinement. This must happen last: in the + // dirty-card logging system, some cards may be dirty by weak-ref + // processing, and may be enqueued. But the whole card table is + // dirtied, so this should abandon those logs, and set "do_traversal" + // to true. + concurrent_g1_refine()->set_pya_restart(); + + assert(regions_accounted_for(), "Region leakage!"); + } + + if (g1_policy()->in_young_gc_mode()) { + _young_list->reset_sampled_info(); + assert( check_young_list_empty(false, false), + "young list should be empty at this point"); + } +} + +void G1CollectedHeap::do_full_collection(bool clear_all_soft_refs) { + do_collection(true, clear_all_soft_refs, 0); +} + +// This code is mostly copied from TenuredGeneration. +void +G1CollectedHeap:: +resize_if_necessary_after_full_collection(size_t word_size) { + assert(MinHeapFreeRatio <= MaxHeapFreeRatio, "sanity check"); + + // Include the current allocation, if any, and bytes that will be + // pre-allocated to support collections, as "used". + const size_t used_after_gc = used(); + const size_t capacity_after_gc = capacity(); + const size_t free_after_gc = capacity_after_gc - used_after_gc; + + // We don't have floating point command-line arguments + const double minimum_free_percentage = (double) MinHeapFreeRatio / 100; + const double maximum_used_percentage = 1.0 - minimum_free_percentage; + const double maximum_free_percentage = (double) MaxHeapFreeRatio / 100; + const double minimum_used_percentage = 1.0 - maximum_free_percentage; + + size_t minimum_desired_capacity = (size_t) (used_after_gc / maximum_used_percentage); + size_t maximum_desired_capacity = (size_t) (used_after_gc / minimum_used_percentage); + + // Don't shrink less than the initial size. + minimum_desired_capacity = + MAX2(minimum_desired_capacity, + collector_policy()->initial_heap_byte_size()); + maximum_desired_capacity = + MAX2(maximum_desired_capacity, + collector_policy()->initial_heap_byte_size()); + + // We are failing here because minimum_desired_capacity is + assert(used_after_gc <= minimum_desired_capacity, "sanity check"); + assert(minimum_desired_capacity <= maximum_desired_capacity, "sanity check"); + + if (PrintGC && Verbose) { + const double free_percentage = ((double)free_after_gc) / capacity(); + gclog_or_tty->print_cr("Computing new size after full GC "); + gclog_or_tty->print_cr(" " + " minimum_free_percentage: %6.2f", + minimum_free_percentage); + gclog_or_tty->print_cr(" " + " maximum_free_percentage: %6.2f", + maximum_free_percentage); + gclog_or_tty->print_cr(" " + " capacity: %6.1fK" + " minimum_desired_capacity: %6.1fK" + " maximum_desired_capacity: %6.1fK", + capacity() / (double) K, + minimum_desired_capacity / (double) K, + maximum_desired_capacity / (double) K); + gclog_or_tty->print_cr(" " + " free_after_gc : %6.1fK" + " used_after_gc : %6.1fK", + free_after_gc / (double) K, + used_after_gc / (double) K); + gclog_or_tty->print_cr(" " + " free_percentage: %6.2f", + free_percentage); + } + if (capacity() < minimum_desired_capacity) { + // Don't expand unless it's significant + size_t expand_bytes = minimum_desired_capacity - capacity_after_gc; + expand(expand_bytes); + if (PrintGC && Verbose) { + gclog_or_tty->print_cr(" expanding:" + " minimum_desired_capacity: %6.1fK" + " expand_bytes: %6.1fK", + minimum_desired_capacity / (double) K, + expand_bytes / (double) K); + } + + // No expansion, now see if we want to shrink + } else if (capacity() > maximum_desired_capacity) { + // Capacity too large, compute shrinking size + size_t shrink_bytes = capacity_after_gc - maximum_desired_capacity; + shrink(shrink_bytes); + if (PrintGC && Verbose) { + gclog_or_tty->print_cr(" " + " shrinking:" + " initSize: %.1fK" + " maximum_desired_capacity: %.1fK", + collector_policy()->initial_heap_byte_size() / (double) K, + maximum_desired_capacity / (double) K); + gclog_or_tty->print_cr(" " + " shrink_bytes: %.1fK", + shrink_bytes / (double) K); + } + } +} + + +HeapWord* +G1CollectedHeap::satisfy_failed_allocation(size_t word_size) { + HeapWord* result = NULL; + + // In a G1 heap, we're supposed to keep allocation from failing by + // incremental pauses. Therefore, at least for now, we'll favor + // expansion over collection. (This might change in the future if we can + // do something smarter than full collection to satisfy a failed alloc.) + + result = expand_and_allocate(word_size); + if (result != NULL) { + assert(is_in(result), "result not in heap"); + return result; + } + + // OK, I guess we have to try collection. + + do_collection(false, false, word_size); + + result = attempt_allocation(word_size, /*permit_collection_pause*/false); + + if (result != NULL) { + assert(is_in(result), "result not in heap"); + return result; + } + + // Try collecting soft references. + do_collection(false, true, word_size); + result = attempt_allocation(word_size, /*permit_collection_pause*/false); + if (result != NULL) { + assert(is_in(result), "result not in heap"); + return result; + } + + // What else? We might try synchronous finalization later. If the total + // space available is large enough for the allocation, then a more + // complete compaction phase than we've tried so far might be + // appropriate. + return NULL; +} + +// Attempting to expand the heap sufficiently +// to support an allocation of the given "word_size". If +// successful, perform the allocation and return the address of the +// allocated block, or else "NULL". + +HeapWord* G1CollectedHeap::expand_and_allocate(size_t word_size) { + size_t expand_bytes = word_size * HeapWordSize; + if (expand_bytes < MinHeapDeltaBytes) { + expand_bytes = MinHeapDeltaBytes; + } + expand(expand_bytes); + assert(regions_accounted_for(), "Region leakage!"); + HeapWord* result = attempt_allocation(word_size, false /* permit_collection_pause */); + return result; +} + +size_t G1CollectedHeap::free_region_if_totally_empty(HeapRegion* hr) { + size_t pre_used = 0; + size_t cleared_h_regions = 0; + size_t freed_regions = 0; + UncleanRegionList local_list; + free_region_if_totally_empty_work(hr, pre_used, cleared_h_regions, + freed_regions, &local_list); + + finish_free_region_work(pre_used, cleared_h_regions, freed_regions, + &local_list); + return pre_used; +} + +void +G1CollectedHeap::free_region_if_totally_empty_work(HeapRegion* hr, + size_t& pre_used, + size_t& cleared_h, + size_t& freed_regions, + UncleanRegionList* list, + bool par) { + assert(!hr->continuesHumongous(), "should have filtered these out"); + size_t res = 0; + if (!hr->popular() && hr->used() > 0 && hr->garbage_bytes() == hr->used()) { + if (!hr->is_young()) { + if (G1PolicyVerbose > 0) + gclog_or_tty->print_cr("Freeing empty region "PTR_FORMAT "(" SIZE_FORMAT " bytes)" + " during cleanup", hr, hr->used()); + free_region_work(hr, pre_used, cleared_h, freed_regions, list, par); + } + } +} + +// FIXME: both this and shrink could probably be more efficient by +// doing one "VirtualSpace::expand_by" call rather than several. +void G1CollectedHeap::expand(size_t expand_bytes) { + size_t old_mem_size = _g1_storage.committed_size(); + // We expand by a minimum of 1K. + expand_bytes = MAX2(expand_bytes, (size_t)K); + size_t aligned_expand_bytes = + ReservedSpace::page_align_size_up(expand_bytes); + aligned_expand_bytes = align_size_up(aligned_expand_bytes, + HeapRegion::GrainBytes); + expand_bytes = aligned_expand_bytes; + while (expand_bytes > 0) { + HeapWord* base = (HeapWord*)_g1_storage.high(); + // Commit more storage. + bool successful = _g1_storage.expand_by(HeapRegion::GrainBytes); + if (!successful) { + expand_bytes = 0; + } else { + expand_bytes -= HeapRegion::GrainBytes; + // Expand the committed region. + HeapWord* high = (HeapWord*) _g1_storage.high(); + _g1_committed.set_end(high); + // Create a new HeapRegion. + MemRegion mr(base, high); + bool is_zeroed = !_g1_max_committed.contains(base); + HeapRegion* hr = new HeapRegion(_bot_shared, mr, is_zeroed); + + // Now update max_committed if necessary. + _g1_max_committed.set_end(MAX2(_g1_max_committed.end(), high)); + + // Add it to the HeapRegionSeq. + _hrs->insert(hr); + // Set the zero-fill state, according to whether it's already + // zeroed. + { + MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); + if (is_zeroed) { + hr->set_zero_fill_complete(); + put_free_region_on_list_locked(hr); + } else { + hr->set_zero_fill_needed(); + put_region_on_unclean_list_locked(hr); + } + } + _free_regions++; + // And we used up an expansion region to create it. + _expansion_regions--; + // Tell the cardtable about it. + Universe::heap()->barrier_set()->resize_covered_region(_g1_committed); + // And the offset table as well. + _bot_shared->resize(_g1_committed.word_size()); + } + } + if (Verbose && PrintGC) { + size_t new_mem_size = _g1_storage.committed_size(); + gclog_or_tty->print_cr("Expanding garbage-first heap from %ldK by %ldK to %ldK", + old_mem_size/K, aligned_expand_bytes/K, + new_mem_size/K); + } +} + +void G1CollectedHeap::shrink_helper(size_t shrink_bytes) +{ + size_t old_mem_size = _g1_storage.committed_size(); + size_t aligned_shrink_bytes = + ReservedSpace::page_align_size_down(shrink_bytes); + aligned_shrink_bytes = align_size_down(aligned_shrink_bytes, + HeapRegion::GrainBytes); + size_t num_regions_deleted = 0; + MemRegion mr = _hrs->shrink_by(aligned_shrink_bytes, num_regions_deleted); + + assert(mr.end() == (HeapWord*)_g1_storage.high(), "Bad shrink!"); + if (mr.byte_size() > 0) + _g1_storage.shrink_by(mr.byte_size()); + assert(mr.start() == (HeapWord*)_g1_storage.high(), "Bad shrink!"); + + _g1_committed.set_end(mr.start()); + _free_regions -= num_regions_deleted; + _expansion_regions += num_regions_deleted; + + // Tell the cardtable about it. + Universe::heap()->barrier_set()->resize_covered_region(_g1_committed); + + // And the offset table as well. + _bot_shared->resize(_g1_committed.word_size()); + + HeapRegionRemSet::shrink_heap(n_regions()); + + if (Verbose && PrintGC) { + size_t new_mem_size = _g1_storage.committed_size(); + gclog_or_tty->print_cr("Shrinking garbage-first heap from %ldK by %ldK to %ldK", + old_mem_size/K, aligned_shrink_bytes/K, + new_mem_size/K); + } +} + +void G1CollectedHeap::shrink(size_t shrink_bytes) { + release_gc_alloc_regions(); + tear_down_region_lists(); // We will rebuild them in a moment. + shrink_helper(shrink_bytes); + rebuild_region_lists(); +} + +// Public methods. + +#ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away +#pragma warning( disable:4355 ) // 'this' : used in base member initializer list +#endif // _MSC_VER + + +G1CollectedHeap::G1CollectedHeap(G1CollectorPolicy* policy_) : + SharedHeap(policy_), + _g1_policy(policy_), + _ref_processor(NULL), + _process_strong_tasks(new SubTasksDone(G1H_PS_NumElements)), + _bot_shared(NULL), + _par_alloc_during_gc_lock(Mutex::leaf, "par alloc during GC lock"), + _objs_with_preserved_marks(NULL), _preserved_marks_of_objs(NULL), + _evac_failure_scan_stack(NULL) , + _mark_in_progress(false), + _cg1r(NULL), _czft(NULL), _summary_bytes_used(0), + _cur_alloc_region(NULL), + _refine_cte_cl(NULL), + _free_region_list(NULL), _free_region_list_size(0), + _free_regions(0), + _popular_object_boundary(NULL), + _cur_pop_hr_index(0), + _popular_regions_to_be_evacuated(NULL), + _pop_obj_rc_at_copy(), + _full_collection(false), + _unclean_region_list(), + _unclean_regions_coming(false), + _young_list(new YoungList(this)), + _gc_time_stamp(0), + _surviving_young_words(NULL) +{ + _g1h = this; // To catch bugs. + if (_process_strong_tasks == NULL || !_process_strong_tasks->valid()) { + vm_exit_during_initialization("Failed necessary allocation."); + } + int n_queues = MAX2((int)ParallelGCThreads, 1); + _task_queues = new RefToScanQueueSet(n_queues); + + int n_rem_sets = HeapRegionRemSet::num_par_rem_sets(); + assert(n_rem_sets > 0, "Invariant."); + + HeapRegionRemSetIterator** iter_arr = + NEW_C_HEAP_ARRAY(HeapRegionRemSetIterator*, n_queues); + for (int i = 0; i < n_queues; i++) { + iter_arr[i] = new HeapRegionRemSetIterator(); + } + _rem_set_iterator = iter_arr; + + for (int i = 0; i < n_queues; i++) { + RefToScanQueue* q = new RefToScanQueue(); + q->initialize(); + _task_queues->register_queue(i, q); + } + + for (int ap = 0; ap < GCAllocPurposeCount; ++ap) { + _gc_alloc_regions[ap] = NULL; + _gc_alloc_region_counts[ap] = 0; + } + guarantee(_task_queues != NULL, "task_queues allocation failure."); +} + +jint G1CollectedHeap::initialize() { + os::enable_vtime(); + + // Necessary to satisfy locking discipline assertions. + + MutexLocker x(Heap_lock); + + // While there are no constraints in the GC code that HeapWordSize + // be any particular value, there are multiple other areas in the + // system which believe this to be true (e.g. oop->object_size in some + // cases incorrectly returns the size in wordSize units rather than + // HeapWordSize). + guarantee(HeapWordSize == wordSize, "HeapWordSize must equal wordSize"); + + size_t init_byte_size = collector_policy()->initial_heap_byte_size(); + size_t max_byte_size = collector_policy()->max_heap_byte_size(); + + // Ensure that the sizes are properly aligned. + Universe::check_alignment(init_byte_size, HeapRegion::GrainBytes, "g1 heap"); + Universe::check_alignment(max_byte_size, HeapRegion::GrainBytes, "g1 heap"); + + // We allocate this in any case, but only do no work if the command line + // param is off. + _cg1r = new ConcurrentG1Refine(); + + // Reserve the maximum. + PermanentGenerationSpec* pgs = collector_policy()->permanent_generation(); + // Includes the perm-gen. + ReservedSpace heap_rs(max_byte_size + pgs->max_size(), + HeapRegion::GrainBytes, + false /*ism*/); + + if (!heap_rs.is_reserved()) { + vm_exit_during_initialization("Could not reserve enough space for object heap"); + return JNI_ENOMEM; + } + + // It is important to do this in a way such that concurrent readers can't + // temporarily think somethings in the heap. (I've actually seen this + // happen in asserts: DLD.) + _reserved.set_word_size(0); + _reserved.set_start((HeapWord*)heap_rs.base()); + _reserved.set_end((HeapWord*)(heap_rs.base() + heap_rs.size())); + + _expansion_regions = max_byte_size/HeapRegion::GrainBytes; + + _num_humongous_regions = 0; + + // Create the gen rem set (and barrier set) for the entire reserved region. + _rem_set = collector_policy()->create_rem_set(_reserved, 2); + set_barrier_set(rem_set()->bs()); + if (barrier_set()->is_a(BarrierSet::ModRef)) { + _mr_bs = (ModRefBarrierSet*)_barrier_set; + } else { + vm_exit_during_initialization("G1 requires a mod ref bs."); + return JNI_ENOMEM; + } + + // Also create a G1 rem set. + if (G1UseHRIntoRS) { + if (mr_bs()->is_a(BarrierSet::CardTableModRef)) { + _g1_rem_set = new HRInto_G1RemSet(this, (CardTableModRefBS*)mr_bs()); + } else { + vm_exit_during_initialization("G1 requires a cardtable mod ref bs."); + return JNI_ENOMEM; + } + } else { + _g1_rem_set = new StupidG1RemSet(this); + } + + // Carve out the G1 part of the heap. + + ReservedSpace g1_rs = heap_rs.first_part(max_byte_size); + _g1_reserved = MemRegion((HeapWord*)g1_rs.base(), + g1_rs.size()/HeapWordSize); + ReservedSpace perm_gen_rs = heap_rs.last_part(max_byte_size); + + _perm_gen = pgs->init(perm_gen_rs, pgs->init_size(), rem_set()); + + _g1_storage.initialize(g1_rs, 0); + _g1_committed = MemRegion((HeapWord*)_g1_storage.low(), (size_t) 0); + _g1_max_committed = _g1_committed; + _hrs = new HeapRegionSeq(); + guarantee(_hrs != NULL, "Couldn't allocate HeapRegionSeq"); + guarantee(_cur_alloc_region == NULL, "from constructor"); + + _bot_shared = new G1BlockOffsetSharedArray(_reserved, + heap_word_size(init_byte_size)); + + _g1h = this; + + // Create the ConcurrentMark data structure and thread. + // (Must do this late, so that "max_regions" is defined.) + _cm = new ConcurrentMark(heap_rs, (int) max_regions()); + _cmThread = _cm->cmThread(); + + // ...and the concurrent zero-fill thread, if necessary. + if (G1ConcZeroFill) { + _czft = new ConcurrentZFThread(); + } + + + + // Allocate the popular regions; take them off free lists. + size_t pop_byte_size = G1NumPopularRegions * HeapRegion::GrainBytes; + expand(pop_byte_size); + _popular_object_boundary = + _g1_reserved.start() + (G1NumPopularRegions * HeapRegion::GrainWords); + for (int i = 0; i < G1NumPopularRegions; i++) { + HeapRegion* hr = newAllocRegion(HeapRegion::GrainWords); + // assert(hr != NULL && hr->bottom() < _popular_object_boundary, + // "Should be enough, and all should be below boundary."); + hr->set_popular(true); + } + assert(_cur_pop_hr_index == 0, "Start allocating at the first region."); + + // Initialize the from_card cache structure of HeapRegionRemSet. + HeapRegionRemSet::init_heap(max_regions()); + + // Now expand into the rest of the initial heap size. + expand(init_byte_size - pop_byte_size); + + // Perform any initialization actions delegated to the policy. + g1_policy()->init(); + + g1_policy()->note_start_of_mark_thread(); + + _refine_cte_cl = + new RefineCardTableEntryClosure(ConcurrentG1RefineThread::sts(), + g1_rem_set(), + concurrent_g1_refine()); + JavaThread::dirty_card_queue_set().set_closure(_refine_cte_cl); + + JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon, + SATB_Q_FL_lock, + 0, + Shared_SATB_Q_lock); + if (G1RSBarrierUseQueue) { + JavaThread::dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon, + DirtyCardQ_FL_lock, + G1DirtyCardQueueMax, + Shared_DirtyCardQ_lock); + } + // In case we're keeping closure specialization stats, initialize those + // counts and that mechanism. + SpecializationStats::clear(); + + _gc_alloc_region_list = NULL; + + // Do later initialization work for concurrent refinement. + _cg1r->init(); + + const char* group_names[] = { "CR", "ZF", "CM", "CL" }; + GCOverheadReporter::initGCOverheadReporter(4, group_names); + + return JNI_OK; +} + +void G1CollectedHeap::ref_processing_init() { + SharedHeap::ref_processing_init(); + MemRegion mr = reserved_region(); + _ref_processor = ReferenceProcessor::create_ref_processor( + mr, // span + false, // Reference discovery is not atomic + // (though it shouldn't matter here.) + true, // mt_discovery + NULL, // is alive closure: need to fill this in for efficiency + ParallelGCThreads, + ParallelRefProcEnabled, + true); // Setting next fields of discovered + // lists requires a barrier. +} + +size_t G1CollectedHeap::capacity() const { + return _g1_committed.byte_size(); +} + +void G1CollectedHeap::iterate_dirty_card_closure(bool concurrent, + int worker_i) { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + int n_completed_buffers = 0; + while (dcqs.apply_closure_to_completed_buffer(worker_i, 0, true)) { + n_completed_buffers++; + } + g1_policy()->record_update_rs_processed_buffers(worker_i, + (double) n_completed_buffers); + dcqs.clear_n_completed_buffers(); + // Finish up the queue... + if (worker_i == 0) concurrent_g1_refine()->clean_up_cache(worker_i, + g1_rem_set()); + assert(!dcqs.completed_buffers_exist_dirty(), "Completed buffers exist!"); +} + + +// Computes the sum of the storage used by the various regions. + +size_t G1CollectedHeap::used() const { + assert(Heap_lock->owner() != NULL, + "Should be owned on this thread's behalf."); + size_t result = _summary_bytes_used; + if (_cur_alloc_region != NULL) + result += _cur_alloc_region->used(); + return result; +} + +class SumUsedClosure: public HeapRegionClosure { + size_t _used; +public: + SumUsedClosure() : _used(0) {} + bool doHeapRegion(HeapRegion* r) { + if (!r->continuesHumongous()) { + _used += r->used(); + } + return false; + } + size_t result() { return _used; } +}; + +size_t G1CollectedHeap::recalculate_used() const { + SumUsedClosure blk; + _hrs->iterate(&blk); + return blk.result(); +} + +#ifndef PRODUCT +class SumUsedRegionsClosure: public HeapRegionClosure { + size_t _num; +public: + // _num is set to 1 to account for the popular region + SumUsedRegionsClosure() : _num(G1NumPopularRegions) {} + bool doHeapRegion(HeapRegion* r) { + if (r->continuesHumongous() || r->used() > 0 || r->is_gc_alloc_region()) { + _num += 1; + } + return false; + } + size_t result() { return _num; } +}; + +size_t G1CollectedHeap::recalculate_used_regions() const { + SumUsedRegionsClosure blk; + _hrs->iterate(&blk); + return blk.result(); +} +#endif // PRODUCT + +size_t G1CollectedHeap::unsafe_max_alloc() { + if (_free_regions > 0) return HeapRegion::GrainBytes; + // otherwise, is there space in the current allocation region? + + // We need to store the current allocation region in a local variable + // here. The problem is that this method doesn't take any locks and + // there may be other threads which overwrite the current allocation + // region field. attempt_allocation(), for example, sets it to NULL + // and this can happen *after* the NULL check here but before the call + // to free(), resulting in a SIGSEGV. Note that this doesn't appear + // to be a problem in the optimized build, since the two loads of the + // current allocation region field are optimized away. + HeapRegion* car = _cur_alloc_region; + + // FIXME: should iterate over all regions? + if (car == NULL) { + return 0; + } + return car->free(); +} + +void G1CollectedHeap::collect(GCCause::Cause cause) { + // The caller doesn't have the Heap_lock + assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock"); + MutexLocker ml(Heap_lock); + collect_locked(cause); +} + +void G1CollectedHeap::collect_as_vm_thread(GCCause::Cause cause) { + assert(Thread::current()->is_VM_thread(), "Precondition#1"); + assert(Heap_lock->is_locked(), "Precondition#2"); + GCCauseSetter gcs(this, cause); + switch (cause) { + case GCCause::_heap_inspection: + case GCCause::_heap_dump: { + HandleMark hm; + do_full_collection(false); // don't clear all soft refs + break; + } + default: // XXX FIX ME + ShouldNotReachHere(); // Unexpected use of this function + } +} + + +void G1CollectedHeap::collect_locked(GCCause::Cause cause) { + // Don't want to do a GC until cleanup is completed. + wait_for_cleanup_complete(); + + // Read the GC count while holding the Heap_lock + int gc_count_before = SharedHeap::heap()->total_collections(); + { + MutexUnlocker mu(Heap_lock); // give up heap lock, execute gets it back + VM_G1CollectFull op(gc_count_before, cause); + VMThread::execute(&op); + } +} + +bool G1CollectedHeap::is_in(const void* p) const { + if (_g1_committed.contains(p)) { + HeapRegion* hr = _hrs->addr_to_region(p); + return hr->is_in(p); + } else { + return _perm_gen->as_gen()->is_in(p); + } +} + +// Iteration functions. + +// Iterates an OopClosure over all ref-containing fields of objects +// within a HeapRegion. + +class IterateOopClosureRegionClosure: public HeapRegionClosure { + MemRegion _mr; + OopClosure* _cl; +public: + IterateOopClosureRegionClosure(MemRegion mr, OopClosure* cl) + : _mr(mr), _cl(cl) {} + bool doHeapRegion(HeapRegion* r) { + if (! r->continuesHumongous()) { + r->oop_iterate(_cl); + } + return false; + } +}; + +void G1CollectedHeap::oop_iterate(OopClosure* cl) { + IterateOopClosureRegionClosure blk(_g1_committed, cl); + _hrs->iterate(&blk); +} + +void G1CollectedHeap::oop_iterate(MemRegion mr, OopClosure* cl) { + IterateOopClosureRegionClosure blk(mr, cl); + _hrs->iterate(&blk); +} + +// Iterates an ObjectClosure over all objects within a HeapRegion. + +class IterateObjectClosureRegionClosure: public HeapRegionClosure { + ObjectClosure* _cl; +public: + IterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {} + bool doHeapRegion(HeapRegion* r) { + if (! r->continuesHumongous()) { + r->object_iterate(_cl); + } + return false; + } +}; + +void G1CollectedHeap::object_iterate(ObjectClosure* cl) { + IterateObjectClosureRegionClosure blk(cl); + _hrs->iterate(&blk); +} + +void G1CollectedHeap::object_iterate_since_last_GC(ObjectClosure* cl) { + // FIXME: is this right? + guarantee(false, "object_iterate_since_last_GC not supported by G1 heap"); +} + +// Calls a SpaceClosure on a HeapRegion. + +class SpaceClosureRegionClosure: public HeapRegionClosure { + SpaceClosure* _cl; +public: + SpaceClosureRegionClosure(SpaceClosure* cl) : _cl(cl) {} + bool doHeapRegion(HeapRegion* r) { + _cl->do_space(r); + return false; + } +}; + +void G1CollectedHeap::space_iterate(SpaceClosure* cl) { + SpaceClosureRegionClosure blk(cl); + _hrs->iterate(&blk); +} + +void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) { + _hrs->iterate(cl); +} + +void G1CollectedHeap::heap_region_iterate_from(HeapRegion* r, + HeapRegionClosure* cl) { + _hrs->iterate_from(r, cl); +} + +void +G1CollectedHeap::heap_region_iterate_from(int idx, HeapRegionClosure* cl) { + _hrs->iterate_from(idx, cl); +} + +HeapRegion* G1CollectedHeap::region_at(size_t idx) { return _hrs->at(idx); } + +const int OverpartitionFactor = 4; +void +G1CollectedHeap::heap_region_par_iterate_chunked(HeapRegionClosure* cl, + int worker, + jint claim_value) { + // We break up the heap regions into blocks of size ParallelGCThreads (to + // decrease iteration costs). + const size_t nregions = n_regions(); + const size_t n_thrds = (ParallelGCThreads > 0 ? ParallelGCThreads : 1); + const size_t partitions = n_thrds * OverpartitionFactor; + const size_t BlkSize = MAX2(nregions/partitions, (size_t)1); + const size_t n_blocks = (nregions + BlkSize - 1)/BlkSize; + assert(ParallelGCThreads > 0 || worker == 0, "Precondition"); + const int init_idx = (int) (n_blocks/n_thrds * worker); + for (size_t blk = 0; blk < n_blocks; blk++) { + size_t idx = init_idx + blk; + if (idx >= n_blocks) idx = idx - n_blocks; + size_t reg_idx = idx * BlkSize; + assert(reg_idx < nregions, "Because we rounded blk up."); + HeapRegion* r = region_at(reg_idx); + if (r->claimHeapRegion(claim_value)) { + for (size_t j = 0; j < BlkSize; j++) { + size_t reg_idx2 = reg_idx + j; + if (reg_idx2 == nregions) break; + HeapRegion* r2 = region_at(reg_idx2); + if (j > 0) r2->set_claim_value(claim_value); + bool res = cl->doHeapRegion(r2); + guarantee(!res, "Should not abort."); + } + } + } +} + +void G1CollectedHeap::collection_set_iterate(HeapRegionClosure* cl) { + HeapRegion* r = g1_policy()->collection_set(); + while (r != NULL) { + HeapRegion* next = r->next_in_collection_set(); + if (cl->doHeapRegion(r)) { + cl->incomplete(); + return; + } + r = next; + } +} + +void G1CollectedHeap::collection_set_iterate_from(HeapRegion* r, + HeapRegionClosure *cl) { + assert(r->in_collection_set(), + "Start region must be a member of the collection set."); + HeapRegion* cur = r; + while (cur != NULL) { + HeapRegion* next = cur->next_in_collection_set(); + if (cl->doHeapRegion(cur) && false) { + cl->incomplete(); + return; + } + cur = next; + } + cur = g1_policy()->collection_set(); + while (cur != r) { + HeapRegion* next = cur->next_in_collection_set(); + if (cl->doHeapRegion(cur) && false) { + cl->incomplete(); + return; + } + cur = next; + } +} + +CompactibleSpace* G1CollectedHeap::first_compactible_space() { + return _hrs->length() > 0 ? _hrs->at(0) : NULL; +} + + +Space* G1CollectedHeap::space_containing(const void* addr) const { + Space* res = heap_region_containing(addr); + if (res == NULL) + res = perm_gen()->space_containing(addr); + return res; +} + +HeapWord* G1CollectedHeap::block_start(const void* addr) const { + Space* sp = space_containing(addr); + if (sp != NULL) { + return sp->block_start(addr); + } + return NULL; +} + +size_t G1CollectedHeap::block_size(const HeapWord* addr) const { + Space* sp = space_containing(addr); + assert(sp != NULL, "block_size of address outside of heap"); + return sp->block_size(addr); +} + +bool G1CollectedHeap::block_is_obj(const HeapWord* addr) const { + Space* sp = space_containing(addr); + return sp->block_is_obj(addr); +} + +bool G1CollectedHeap::supports_tlab_allocation() const { + return true; +} + +size_t G1CollectedHeap::tlab_capacity(Thread* ignored) const { + return HeapRegion::GrainBytes; +} + +size_t G1CollectedHeap::unsafe_max_tlab_alloc(Thread* ignored) const { + // Return the remaining space in the cur alloc region, but not less than + // the min TLAB size. + // Also, no more than half the region size, since we can't allow tlabs to + // grow big enough to accomodate humongous objects. + + // We need to story it locally, since it might change between when we + // test for NULL and when we use it later. + ContiguousSpace* cur_alloc_space = _cur_alloc_region; + if (cur_alloc_space == NULL) { + return HeapRegion::GrainBytes/2; + } else { + return MAX2(MIN2(cur_alloc_space->free(), + (size_t)(HeapRegion::GrainBytes/2)), + (size_t)MinTLABSize); + } +} + +HeapWord* G1CollectedHeap::allocate_new_tlab(size_t size) { + bool dummy; + return G1CollectedHeap::mem_allocate(size, false, true, &dummy); +} + +bool G1CollectedHeap::allocs_are_zero_filled() { + return false; +} + +size_t G1CollectedHeap::large_typearray_limit() { + // FIXME + return HeapRegion::GrainBytes/HeapWordSize; +} + +size_t G1CollectedHeap::max_capacity() const { + return _g1_committed.byte_size(); +} + +jlong G1CollectedHeap::millis_since_last_gc() { + // assert(false, "NYI"); + return 0; +} + + +void G1CollectedHeap::prepare_for_verify() { + if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) { + ensure_parsability(false); + } + g1_rem_set()->prepare_for_verify(); +} + +class VerifyLivenessOopClosure: public OopClosure { + G1CollectedHeap* g1h; +public: + VerifyLivenessOopClosure(G1CollectedHeap* _g1h) { + g1h = _g1h; + } + void do_oop(narrowOop *p) { + guarantee(false, "NYI"); + } + void do_oop(oop *p) { + oop obj = *p; + assert(obj == NULL || !g1h->is_obj_dead(obj), + "Dead object referenced by a not dead object"); + } +}; + +class VerifyObjsInRegionClosure: public ObjectClosure { + G1CollectedHeap* _g1h; + size_t _live_bytes; + HeapRegion *_hr; +public: + VerifyObjsInRegionClosure(HeapRegion *hr) : _live_bytes(0), _hr(hr) { + _g1h = G1CollectedHeap::heap(); + } + void do_object(oop o) { + VerifyLivenessOopClosure isLive(_g1h); + assert(o != NULL, "Huh?"); + if (!_g1h->is_obj_dead(o)) { + o->oop_iterate(&isLive); + if (!_hr->obj_allocated_since_prev_marking(o)) + _live_bytes += (o->size() * HeapWordSize); + } + } + size_t live_bytes() { return _live_bytes; } +}; + +class PrintObjsInRegionClosure : public ObjectClosure { + HeapRegion *_hr; + G1CollectedHeap *_g1; +public: + PrintObjsInRegionClosure(HeapRegion *hr) : _hr(hr) { + _g1 = G1CollectedHeap::heap(); + }; + + void do_object(oop o) { + if (o != NULL) { + HeapWord *start = (HeapWord *) o; + size_t word_sz = o->size(); + gclog_or_tty->print("\nPrinting obj "PTR_FORMAT" of size " SIZE_FORMAT + " isMarkedPrev %d isMarkedNext %d isAllocSince %d\n", + (void*) o, word_sz, + _g1->isMarkedPrev(o), + _g1->isMarkedNext(o), + _hr->obj_allocated_since_prev_marking(o)); + HeapWord *end = start + word_sz; + HeapWord *cur; + int *val; + for (cur = start; cur < end; cur++) { + val = (int *) cur; + gclog_or_tty->print("\t "PTR_FORMAT":"PTR_FORMAT"\n", val, *val); + } + } + } +}; + +class VerifyRegionClosure: public HeapRegionClosure { +public: + bool _allow_dirty; + VerifyRegionClosure(bool allow_dirty) + : _allow_dirty(allow_dirty) {} + bool doHeapRegion(HeapRegion* r) { + guarantee(r->claim_value() == 0, "Should be unclaimed at verify points."); + if (r->isHumongous()) { + if (r->startsHumongous()) { + // Verify the single H object. + oop(r->bottom())->verify(); + size_t word_sz = oop(r->bottom())->size(); + guarantee(r->top() == r->bottom() + word_sz, + "Only one object in a humongous region"); + } + } else { + VerifyObjsInRegionClosure not_dead_yet_cl(r); + r->verify(_allow_dirty); + r->object_iterate(¬_dead_yet_cl); + guarantee(r->max_live_bytes() >= not_dead_yet_cl.live_bytes(), + "More live objects than counted in last complete marking."); + } + return false; + } +}; + +class VerifyRootsClosure: public OopsInGenClosure { +private: + G1CollectedHeap* _g1h; + bool _failures; + +public: + VerifyRootsClosure() : + _g1h(G1CollectedHeap::heap()), _failures(false) { } + + bool failures() { return _failures; } + + void do_oop(narrowOop* p) { + guarantee(false, "NYI"); + } + + void do_oop(oop* p) { + oop obj = *p; + if (obj != NULL) { + if (_g1h->is_obj_dead(obj)) { + gclog_or_tty->print_cr("Root location "PTR_FORMAT" " + "points to dead obj "PTR_FORMAT, p, (void*) obj); + obj->print_on(gclog_or_tty); + _failures = true; + } + } + } +}; + +void G1CollectedHeap::verify(bool allow_dirty, bool silent) { + if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) { + if (!silent) { gclog_or_tty->print("roots "); } + VerifyRootsClosure rootsCl; + process_strong_roots(false, + SharedHeap::SO_AllClasses, + &rootsCl, + &rootsCl); + rem_set()->invalidate(perm_gen()->used_region(), false); + if (!silent) { gclog_or_tty->print("heapRegions "); } + VerifyRegionClosure blk(allow_dirty); + _hrs->iterate(&blk); + if (!silent) gclog_or_tty->print("remset "); + rem_set()->verify(); + guarantee(!rootsCl.failures(), "should not have had failures"); + } else { + if (!silent) gclog_or_tty->print("(SKIPPING roots, heapRegions, remset) "); + } +} + +class PrintRegionClosure: public HeapRegionClosure { + outputStream* _st; +public: + PrintRegionClosure(outputStream* st) : _st(st) {} + bool doHeapRegion(HeapRegion* r) { + r->print_on(_st); + return false; + } +}; + +void G1CollectedHeap::print() const { print_on(gclog_or_tty); } + +void G1CollectedHeap::print_on(outputStream* st) const { + PrintRegionClosure blk(st); + _hrs->iterate(&blk); +} + +void G1CollectedHeap::print_gc_threads_on(outputStream* st) const { + if (ParallelGCThreads > 0) { + workers()->print_worker_threads(); + } + st->print("\"G1 concurrent mark GC Thread\" "); + _cmThread->print(); + st->cr(); + st->print("\"G1 concurrent refinement GC Thread\" "); + _cg1r->cg1rThread()->print_on(st); + st->cr(); + st->print("\"G1 zero-fill GC Thread\" "); + _czft->print_on(st); + st->cr(); +} + +void G1CollectedHeap::gc_threads_do(ThreadClosure* tc) const { + if (ParallelGCThreads > 0) { + workers()->threads_do(tc); + } + tc->do_thread(_cmThread); + tc->do_thread(_cg1r->cg1rThread()); + tc->do_thread(_czft); +} + +void G1CollectedHeap::print_tracing_info() const { + concurrent_g1_refine()->print_final_card_counts(); + + // We'll overload this to mean "trace GC pause statistics." + if (TraceGen0Time || TraceGen1Time) { + // The "G1CollectorPolicy" is keeping track of these stats, so delegate + // to that. + g1_policy()->print_tracing_info(); + } + if (SummarizeG1RSStats) { + g1_rem_set()->print_summary_info(); + } + if (SummarizeG1ConcMark) { + concurrent_mark()->print_summary_info(); + } + if (SummarizeG1ZFStats) { + ConcurrentZFThread::print_summary_info(); + } + if (G1SummarizePopularity) { + print_popularity_summary_info(); + } + g1_policy()->print_yg_surv_rate_info(); + + GCOverheadReporter::printGCOverhead(); + + SpecializationStats::print(); +} + + +int G1CollectedHeap::addr_to_arena_id(void* addr) const { + HeapRegion* hr = heap_region_containing(addr); + if (hr == NULL) { + return 0; + } else { + return 1; + } +} + +G1CollectedHeap* G1CollectedHeap::heap() { + assert(_sh->kind() == CollectedHeap::G1CollectedHeap, + "not a garbage-first heap"); + return _g1h; +} + +void G1CollectedHeap::gc_prologue(bool full /* Ignored */) { + if (PrintHeapAtGC){ + gclog_or_tty->print_cr(" {Heap before GC collections=%d:", total_collections()); + Universe::print(); + } + assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer"); + // Call allocation profiler + AllocationProfiler::iterate_since_last_gc(); + // Fill TLAB's and such + ensure_parsability(true); +} + +void G1CollectedHeap::gc_epilogue(bool full /* Ignored */) { + // FIXME: what is this about? + // I'm ignoring the "fill_newgen()" call if "alloc_event_enabled" + // is set. + COMPILER2_PRESENT(assert(DerivedPointerTable::is_empty(), + "derived pointer present")); + + if (PrintHeapAtGC){ + gclog_or_tty->print_cr(" Heap after GC collections=%d:", total_collections()); + Universe::print(); + gclog_or_tty->print("} "); + } +} + +void G1CollectedHeap::do_collection_pause() { + // Read the GC count while holding the Heap_lock + // we need to do this _before_ wait_for_cleanup_complete(), to + // ensure that we do not give up the heap lock and potentially + // pick up the wrong count + int gc_count_before = SharedHeap::heap()->total_collections(); + + // Don't want to do a GC pause while cleanup is being completed! + wait_for_cleanup_complete(); + + g1_policy()->record_stop_world_start(); + { + MutexUnlocker mu(Heap_lock); // give up heap lock, execute gets it back + VM_G1IncCollectionPause op(gc_count_before); + VMThread::execute(&op); + } +} + +void +G1CollectedHeap::doConcurrentMark() { + if (G1ConcMark) { + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); + if (!_cmThread->in_progress()) { + _cmThread->set_started(); + CGC_lock->notify(); + } + } +} + +class VerifyMarkedObjsClosure: public ObjectClosure { + G1CollectedHeap* _g1h; + public: + VerifyMarkedObjsClosure(G1CollectedHeap* g1h) : _g1h(g1h) {} + void do_object(oop obj) { + assert(obj->mark()->is_marked() ? !_g1h->is_obj_dead(obj) : true, + "markandsweep mark should agree with concurrent deadness"); + } +}; + +void +G1CollectedHeap::checkConcurrentMark() { + VerifyMarkedObjsClosure verifycl(this); + doConcurrentMark(); + // MutexLockerEx x(getMarkBitMapLock(), + // Mutex::_no_safepoint_check_flag); + object_iterate(&verifycl); +} + +void G1CollectedHeap::do_sync_mark() { + _cm->checkpointRootsInitial(); + _cm->markFromRoots(); + _cm->checkpointRootsFinal(false); +} + +// <NEW PREDICTION> + +double G1CollectedHeap::predict_region_elapsed_time_ms(HeapRegion *hr, + bool young) { + return _g1_policy->predict_region_elapsed_time_ms(hr, young); +} + +void G1CollectedHeap::check_if_region_is_too_expensive(double + predicted_time_ms) { + _g1_policy->check_if_region_is_too_expensive(predicted_time_ms); +} + +size_t G1CollectedHeap::pending_card_num() { + size_t extra_cards = 0; + JavaThread *curr = Threads::first(); + while (curr != NULL) { + DirtyCardQueue& dcq = curr->dirty_card_queue(); + extra_cards += dcq.size(); + curr = curr->next(); + } + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + size_t buffer_size = dcqs.buffer_size(); + size_t buffer_num = dcqs.completed_buffers_num(); + return buffer_size * buffer_num + extra_cards; +} + +size_t G1CollectedHeap::max_pending_card_num() { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + size_t buffer_size = dcqs.buffer_size(); + size_t buffer_num = dcqs.completed_buffers_num(); + int thread_num = Threads::number_of_threads(); + return (buffer_num + thread_num) * buffer_size; +} + +size_t G1CollectedHeap::cards_scanned() { + HRInto_G1RemSet* g1_rset = (HRInto_G1RemSet*) g1_rem_set(); + return g1_rset->cardsScanned(); +} + +void +G1CollectedHeap::setup_surviving_young_words() { + guarantee( _surviving_young_words == NULL, "pre-condition" ); + size_t array_length = g1_policy()->young_cset_length(); + _surviving_young_words = NEW_C_HEAP_ARRAY(size_t, array_length); + if (_surviving_young_words == NULL) { + vm_exit_out_of_memory(sizeof(size_t) * array_length, + "Not enough space for young surv words summary."); + } + memset(_surviving_young_words, 0, array_length * sizeof(size_t)); + for (size_t i = 0; i < array_length; ++i) { + guarantee( _surviving_young_words[i] == 0, "invariant" ); + } +} + +void +G1CollectedHeap::update_surviving_young_words(size_t* surv_young_words) { + MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); + size_t array_length = g1_policy()->young_cset_length(); + for (size_t i = 0; i < array_length; ++i) + _surviving_young_words[i] += surv_young_words[i]; +} + +void +G1CollectedHeap::cleanup_surviving_young_words() { + guarantee( _surviving_young_words != NULL, "pre-condition" ); + FREE_C_HEAP_ARRAY(size_t, _surviving_young_words); + _surviving_young_words = NULL; +} + +// </NEW PREDICTION> + +void +G1CollectedHeap::do_collection_pause_at_safepoint(HeapRegion* popular_region) { + char verbose_str[128]; + sprintf(verbose_str, "GC pause "); + if (popular_region != NULL) + strcat(verbose_str, "(popular)"); + else if (g1_policy()->in_young_gc_mode()) { + if (g1_policy()->full_young_gcs()) + strcat(verbose_str, "(young)"); + else + strcat(verbose_str, "(partial)"); + } + bool reset_should_initiate_conc_mark = false; + if (popular_region != NULL && g1_policy()->should_initiate_conc_mark()) { + // we currently do not allow an initial mark phase to be piggy-backed + // on a popular pause + reset_should_initiate_conc_mark = true; + g1_policy()->unset_should_initiate_conc_mark(); + } + if (g1_policy()->should_initiate_conc_mark()) + strcat(verbose_str, " (initial-mark)"); + + GCCauseSetter x(this, (popular_region == NULL ? + GCCause::_g1_inc_collection_pause : + GCCause::_g1_pop_region_collection_pause)); + + // if PrintGCDetails is on, we'll print long statistics information + // in the collector policy code, so let's not print this as the output + // is messy if we do. + gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps); + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); + TraceTime t(verbose_str, PrintGC && !PrintGCDetails, true, gclog_or_tty); + + ResourceMark rm; + assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); + assert(Thread::current() == VMThread::vm_thread(), "should be in vm thread"); + guarantee(!is_gc_active(), "collection is not reentrant"); + assert(regions_accounted_for(), "Region leakage!"); + ++_gc_time_stamp; + + if (g1_policy()->in_young_gc_mode()) { + assert(check_young_list_well_formed(), + "young list should be well formed"); + } + + if (GC_locker::is_active()) { + return; // GC is disabled (e.g. JNI GetXXXCritical operation) + } + + bool abandoned = false; + { // Call to jvmpi::post_class_unload_events must occur outside of active GC + IsGCActiveMark x; + + gc_prologue(false); + increment_total_collections(); + +#if G1_REM_SET_LOGGING + gclog_or_tty->print_cr("\nJust chose CS, heap:"); + print(); +#endif + + if (VerifyBeforeGC && total_collections() >= VerifyGCStartAt) { + HandleMark hm; // Discard invalid handles created during verification + prepare_for_verify(); + gclog_or_tty->print(" VerifyBeforeGC:"); + Universe::verify(false); + } + + COMPILER2_PRESENT(DerivedPointerTable::clear()); + + // We want to turn off ref discovere, if necessary, and turn it back on + // on again later if we do. + bool was_enabled = ref_processor()->discovery_enabled(); + if (was_enabled) ref_processor()->disable_discovery(); + + // Forget the current alloc region (we might even choose it to be part + // of the collection set!). + abandon_cur_alloc_region(); + + // The elapsed time induced by the start time below deliberately elides + // the possible verification above. + double start_time_sec = os::elapsedTime(); + GCOverheadReporter::recordSTWStart(start_time_sec); + size_t start_used_bytes = used(); + if (!G1ConcMark) { + do_sync_mark(); + } + + g1_policy()->record_collection_pause_start(start_time_sec, + start_used_bytes); + +#if SCAN_ONLY_VERBOSE + _young_list->print(); +#endif // SCAN_ONLY_VERBOSE + + if (g1_policy()->should_initiate_conc_mark()) { + concurrent_mark()->checkpointRootsInitialPre(); + } + save_marks(); + + // We must do this before any possible evacuation that should propogate + // marks, including evacuation of popular objects in a popular pause. + if (mark_in_progress()) { + double start_time_sec = os::elapsedTime(); + + _cm->drainAllSATBBuffers(); + double finish_mark_ms = (os::elapsedTime() - start_time_sec) * 1000.0; + g1_policy()->record_satb_drain_time(finish_mark_ms); + + } + // Record the number of elements currently on the mark stack, so we + // only iterate over these. (Since evacuation may add to the mark + // stack, doing more exposes race conditions.) If no mark is in + // progress, this will be zero. + _cm->set_oops_do_bound(); + + assert(regions_accounted_for(), "Region leakage."); + + bool abandoned = false; + + if (mark_in_progress()) + concurrent_mark()->newCSet(); + + // Now choose the CS. + if (popular_region == NULL) { + g1_policy()->choose_collection_set(); + } else { + // We may be evacuating a single region (for popularity). + g1_policy()->record_popular_pause_preamble_start(); + popularity_pause_preamble(popular_region); + g1_policy()->record_popular_pause_preamble_end(); + abandoned = (g1_policy()->collection_set() == NULL); + // Now we allow more regions to be added (we have to collect + // all popular regions). + if (!abandoned) { + g1_policy()->choose_collection_set(popular_region); + } + } + // We may abandon a pause if we find no region that will fit in the MMU + // pause. + abandoned = (g1_policy()->collection_set() == NULL); + + // Nothing to do if we were unable to choose a collection set. + if (!abandoned) { +#if G1_REM_SET_LOGGING + gclog_or_tty->print_cr("\nAfter pause, heap:"); + print(); +#endif + + setup_surviving_young_words(); + + // Set up the gc allocation regions. + get_gc_alloc_regions(); + + // Actually do the work... + evacuate_collection_set(); + free_collection_set(g1_policy()->collection_set()); + g1_policy()->clear_collection_set(); + + if (popular_region != NULL) { + // We have to wait until now, because we don't want the region to + // be rescheduled for pop-evac during RS update. + popular_region->set_popular_pending(false); + } + + release_gc_alloc_regions(); + + cleanup_surviving_young_words(); + + if (g1_policy()->in_young_gc_mode()) { + _young_list->reset_sampled_info(); + assert(check_young_list_empty(true), + "young list should be empty"); + +#if SCAN_ONLY_VERBOSE + _young_list->print(); +#endif // SCAN_ONLY_VERBOSE + + _young_list->reset_auxilary_lists(); + } + } else { + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); + } + + if (evacuation_failed()) { + _summary_bytes_used = recalculate_used(); + } else { + // The "used" of the the collection set have already been subtracted + // when they were freed. Add in the bytes evacuated. + _summary_bytes_used += g1_policy()->bytes_in_to_space(); + } + + if (g1_policy()->in_young_gc_mode() && + g1_policy()->should_initiate_conc_mark()) { + concurrent_mark()->checkpointRootsInitialPost(); + set_marking_started(); + doConcurrentMark(); + } + +#if SCAN_ONLY_VERBOSE + _young_list->print(); +#endif // SCAN_ONLY_VERBOSE + + double end_time_sec = os::elapsedTime(); + g1_policy()->record_pause_time((end_time_sec - start_time_sec)*1000.0); + GCOverheadReporter::recordSTWEnd(end_time_sec); + g1_policy()->record_collection_pause_end(popular_region != NULL, + abandoned); + + assert(regions_accounted_for(), "Region leakage."); + + if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) { + HandleMark hm; // Discard invalid handles created during verification + gclog_or_tty->print(" VerifyAfterGC:"); + Universe::verify(false); + } + + if (was_enabled) ref_processor()->enable_discovery(); + + { + size_t expand_bytes = g1_policy()->expansion_amount(); + if (expand_bytes > 0) { + size_t bytes_before = capacity(); + expand(expand_bytes); + } + } + + if (mark_in_progress()) + concurrent_mark()->update_g1_committed(); + + gc_epilogue(false); + } + + assert(verify_region_lists(), "Bad region lists."); + + if (reset_should_initiate_conc_mark) + g1_policy()->set_should_initiate_conc_mark(); + + if (ExitAfterGCNum > 0 && total_collections() == ExitAfterGCNum) { + gclog_or_tty->print_cr("Stopping after GC #%d", ExitAfterGCNum); + print_tracing_info(); + vm_exit(-1); + } +} + +void G1CollectedHeap::set_gc_alloc_region(int purpose, HeapRegion* r) { + assert(purpose >= 0 && purpose < GCAllocPurposeCount, "invalid purpose"); + HeapWord* original_top = NULL; + if (r != NULL) + original_top = r->top(); + + // We will want to record the used space in r as being there before gc. + // One we install it as a GC alloc region it's eligible for allocation. + // So record it now and use it later. + size_t r_used = 0; + if (r != NULL) { + r_used = r->used(); + + if (ParallelGCThreads > 0) { + // need to take the lock to guard against two threads calling + // get_gc_alloc_region concurrently (very unlikely but...) + MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); + r->save_marks(); + } + } + HeapRegion* old_alloc_region = _gc_alloc_regions[purpose]; + _gc_alloc_regions[purpose] = r; + if (old_alloc_region != NULL) { + // Replace aliases too. + for (int ap = 0; ap < GCAllocPurposeCount; ++ap) { + if (_gc_alloc_regions[ap] == old_alloc_region) { + _gc_alloc_regions[ap] = r; + } + } + } + if (r != NULL) { + push_gc_alloc_region(r); + if (mark_in_progress() && original_top != r->next_top_at_mark_start()) { + // We are using a region as a GC alloc region after it has been used + // as a mutator allocation region during the current marking cycle. + // The mutator-allocated objects are currently implicitly marked, but + // when we move hr->next_top_at_mark_start() forward at the the end + // of the GC pause, they won't be. We therefore mark all objects in + // the "gap". We do this object-by-object, since marking densely + // does not currently work right with marking bitmap iteration. This + // means we rely on TLAB filling at the start of pauses, and no + // "resuscitation" of filled TLAB's. If we want to do this, we need + // to fix the marking bitmap iteration. + HeapWord* curhw = r->next_top_at_mark_start(); + HeapWord* t = original_top; + + while (curhw < t) { + oop cur = (oop)curhw; + // We'll assume parallel for generality. This is rare code. + concurrent_mark()->markAndGrayObjectIfNecessary(cur); // can't we just mark them? + curhw = curhw + cur->size(); + } + assert(curhw == t, "Should have parsed correctly."); + } + if (G1PolicyVerbose > 1) { + gclog_or_tty->print("New alloc region ["PTR_FORMAT", "PTR_FORMAT", " PTR_FORMAT") " + "for survivors:", r->bottom(), original_top, r->end()); + r->print(); + } + g1_policy()->record_before_bytes(r_used); + } +} + +void G1CollectedHeap::push_gc_alloc_region(HeapRegion* hr) { + assert(Thread::current()->is_VM_thread() || + par_alloc_during_gc_lock()->owned_by_self(), "Precondition"); + assert(!hr->is_gc_alloc_region() && !hr->in_collection_set(), + "Precondition."); + hr->set_is_gc_alloc_region(true); + hr->set_next_gc_alloc_region(_gc_alloc_region_list); + _gc_alloc_region_list = hr; +} + +#ifdef G1_DEBUG +class FindGCAllocRegion: public HeapRegionClosure { +public: + bool doHeapRegion(HeapRegion* r) { + if (r->is_gc_alloc_region()) { + gclog_or_tty->print_cr("Region %d ["PTR_FORMAT"...] is still a gc_alloc_region.", + r->hrs_index(), r->bottom()); + } + return false; + } +}; +#endif // G1_DEBUG + +void G1CollectedHeap::forget_alloc_region_list() { + assert(Thread::current()->is_VM_thread(), "Precondition"); + while (_gc_alloc_region_list != NULL) { + HeapRegion* r = _gc_alloc_region_list; + assert(r->is_gc_alloc_region(), "Invariant."); + _gc_alloc_region_list = r->next_gc_alloc_region(); + r->set_next_gc_alloc_region(NULL); + r->set_is_gc_alloc_region(false); + if (r->is_empty()) { + ++_free_regions; + } + } +#ifdef G1_DEBUG + FindGCAllocRegion fa; + heap_region_iterate(&fa); +#endif // G1_DEBUG +} + + +bool G1CollectedHeap::check_gc_alloc_regions() { + // TODO: allocation regions check + return true; +} + +void G1CollectedHeap::get_gc_alloc_regions() { + for (int ap = 0; ap < GCAllocPurposeCount; ++ap) { + // Create new GC alloc regions. + HeapRegion* alloc_region = _gc_alloc_regions[ap]; + // Clear this alloc region, so that in case it turns out to be + // unacceptable, we end up with no allocation region, rather than a bad + // one. + _gc_alloc_regions[ap] = NULL; + if (alloc_region == NULL || alloc_region->in_collection_set()) { + // Can't re-use old one. Allocate a new one. + alloc_region = newAllocRegionWithExpansion(ap, 0); + } + if (alloc_region != NULL) { + set_gc_alloc_region(ap, alloc_region); + } + } + // Set alternative regions for allocation purposes that have reached + // thier limit. + for (int ap = 0; ap < GCAllocPurposeCount; ++ap) { + GCAllocPurpose alt_purpose = g1_policy()->alternative_purpose(ap); + if (_gc_alloc_regions[ap] == NULL && alt_purpose != ap) { + _gc_alloc_regions[ap] = _gc_alloc_regions[alt_purpose]; + } + } + assert(check_gc_alloc_regions(), "alloc regions messed up"); +} + +void G1CollectedHeap::release_gc_alloc_regions() { + // We keep a separate list of all regions that have been alloc regions in + // the current collection pause. Forget that now. + forget_alloc_region_list(); + + // The current alloc regions contain objs that have survived + // collection. Make them no longer GC alloc regions. + for (int ap = 0; ap < GCAllocPurposeCount; ++ap) { + HeapRegion* r = _gc_alloc_regions[ap]; + if (r != NULL && r->is_empty()) { + { + MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); + r->set_zero_fill_complete(); + put_free_region_on_list_locked(r); + } + } + // set_gc_alloc_region will also NULLify all aliases to the region + set_gc_alloc_region(ap, NULL); + _gc_alloc_region_counts[ap] = 0; + } +} + +void G1CollectedHeap::init_for_evac_failure(OopsInHeapRegionClosure* cl) { + _drain_in_progress = false; + set_evac_failure_closure(cl); + _evac_failure_scan_stack = new (ResourceObj::C_HEAP) GrowableArray<oop>(40, true); +} + +void G1CollectedHeap::finalize_for_evac_failure() { + assert(_evac_failure_scan_stack != NULL && + _evac_failure_scan_stack->length() == 0, + "Postcondition"); + assert(!_drain_in_progress, "Postcondition"); + // Don't have to delete, since the scan stack is a resource object. + _evac_failure_scan_stack = NULL; +} + + + +// *** Sequential G1 Evacuation + +HeapWord* G1CollectedHeap::allocate_during_gc(GCAllocPurpose purpose, size_t word_size) { + HeapRegion* alloc_region = _gc_alloc_regions[purpose]; + // let the caller handle alloc failure + if (alloc_region == NULL) return NULL; + assert(isHumongous(word_size) || !alloc_region->isHumongous(), + "Either the object is humongous or the region isn't"); + HeapWord* block = alloc_region->allocate(word_size); + if (block == NULL) { + block = allocate_during_gc_slow(purpose, alloc_region, false, word_size); + } + return block; +} + +class G1IsAliveClosure: public BoolObjectClosure { + G1CollectedHeap* _g1; +public: + G1IsAliveClosure(G1CollectedHeap* g1) : _g1(g1) {} + void do_object(oop p) { assert(false, "Do not call."); } + bool do_object_b(oop p) { + // It is reachable if it is outside the collection set, or is inside + // and forwarded. + +#ifdef G1_DEBUG + gclog_or_tty->print_cr("is alive "PTR_FORMAT" in CS %d forwarded %d overall %d", + (void*) p, _g1->obj_in_cs(p), p->is_forwarded(), + !_g1->obj_in_cs(p) || p->is_forwarded()); +#endif // G1_DEBUG + + return !_g1->obj_in_cs(p) || p->is_forwarded(); + } +}; + +class G1KeepAliveClosure: public OopClosure { + G1CollectedHeap* _g1; +public: + G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {} + void do_oop(narrowOop* p) { + guarantee(false, "NYI"); + } + void do_oop(oop* p) { + oop obj = *p; +#ifdef G1_DEBUG + if (PrintGC && Verbose) { + gclog_or_tty->print_cr("keep alive *"PTR_FORMAT" = "PTR_FORMAT" "PTR_FORMAT, + p, (void*) obj, (void*) *p); + } +#endif // G1_DEBUG + + if (_g1->obj_in_cs(obj)) { + assert( obj->is_forwarded(), "invariant" ); + *p = obj->forwardee(); + +#ifdef G1_DEBUG + gclog_or_tty->print_cr(" in CSet: moved "PTR_FORMAT" -> "PTR_FORMAT, + (void*) obj, (void*) *p); +#endif // G1_DEBUG + } + } +}; + +class RecreateRSetEntriesClosure: public OopClosure { +private: + G1CollectedHeap* _g1; + G1RemSet* _g1_rem_set; + HeapRegion* _from; +public: + RecreateRSetEntriesClosure(G1CollectedHeap* g1, HeapRegion* from) : + _g1(g1), _g1_rem_set(g1->g1_rem_set()), _from(from) + {} + + void do_oop(narrowOop* p) { + guarantee(false, "NYI"); + } + void do_oop(oop* p) { + assert(_from->is_in_reserved(p), "paranoia"); + if (*p != NULL) { + _g1_rem_set->write_ref(_from, p); + } + } +}; + +class RemoveSelfPointerClosure: public ObjectClosure { +private: + G1CollectedHeap* _g1; + ConcurrentMark* _cm; + HeapRegion* _hr; + HeapWord* _last_self_forwarded_end; + size_t _prev_marked_bytes; + size_t _next_marked_bytes; +public: + RemoveSelfPointerClosure(G1CollectedHeap* g1, HeapRegion* hr) : + _g1(g1), _cm(_g1->concurrent_mark()), _hr(hr), + _last_self_forwarded_end(_hr->bottom()), + _prev_marked_bytes(0), _next_marked_bytes(0) + {} + + size_t prev_marked_bytes() { return _prev_marked_bytes; } + size_t next_marked_bytes() { return _next_marked_bytes; } + + void fill_remainder() { + HeapWord* limit = _hr->top(); + MemRegion mr(_last_self_forwarded_end, limit); + if (!mr.is_empty()) { + SharedHeap::fill_region_with_object(mr); + _cm->clearRangeBothMaps(mr); + _hr->declare_filled_region_to_BOT(mr); + } + } + + void do_object(oop obj) { + if (obj->is_forwarded()) { + if (obj->forwardee() == obj) { + assert(!_g1->is_obj_dead(obj), "We should not be preserving dead objs."); + _cm->markPrev(obj); + assert(_cm->isPrevMarked(obj), "Should be marked!"); + _prev_marked_bytes += (obj->size() * HeapWordSize); + if (_g1->mark_in_progress() && !_g1->is_obj_ill(obj)) { + _cm->markAndGrayObjectIfNecessary(obj); + } + HeapWord* obj_start = (HeapWord*)obj; + if (obj_start > _last_self_forwarded_end) { + MemRegion mr(_last_self_forwarded_end, obj_start); + SharedHeap::fill_region_with_object(mr); + assert(_cm->isPrevMarked(obj), "Should be marked!"); + _cm->clearRangeBothMaps(mr); + assert(_cm->isPrevMarked(obj), "Should be marked!"); + _hr->declare_filled_region_to_BOT(mr); + } + _last_self_forwarded_end = obj_start + obj->size(); + obj->set_mark(markOopDesc::prototype()); + + // While we were processing RSet buffers during the + // collection, we actually didn't scan any cards on the + // collection set, since we didn't want to update remebered + // sets with entries that point into the collection set, given + // that live objects fromthe collection set are about to move + // and such entries will be stale very soon. This change also + // dealt with a reliability issue which involved scanning a + // card in the collection set and coming across an array that + // was being chunked and looking malformed. The problem is + // that, if evacuation fails, we might have remembered set + // entries missing given that we skipped cards on the + // collection set. So, we'll recreate such entries now. + RecreateRSetEntriesClosure cl(_g1, _hr); + obj->oop_iterate(&cl); + + assert(_cm->isPrevMarked(obj), "Should be marked!"); + } + } + } +}; + +void G1CollectedHeap::remove_self_forwarding_pointers() { + HeapRegion* cur = g1_policy()->collection_set(); + + while (cur != NULL) { + assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!"); + + if (cur->evacuation_failed()) { + RemoveSelfPointerClosure rspc(_g1h, cur); + assert(cur->in_collection_set(), "bad CS"); + cur->object_iterate(&rspc); + rspc.fill_remainder(); + + // A number of manipulations to make the TAMS be the current top, + // and the marked bytes be the ones observed in the iteration. + if (_g1h->concurrent_mark()->at_least_one_mark_complete()) { + // The comments below are the postconditions achieved by the + // calls. Note especially the last such condition, which says that + // the count of marked bytes has been properly restored. + cur->note_start_of_marking(false); + // _next_top_at_mark_start == top, _next_marked_bytes == 0 + cur->add_to_marked_bytes(rspc.prev_marked_bytes()); + // _next_marked_bytes == prev_marked_bytes. + cur->note_end_of_marking(); + // _prev_top_at_mark_start == top(), + // _prev_marked_bytes == prev_marked_bytes + } + // If there is no mark in progress, we modified the _next variables + // above needlessly, but harmlessly. + if (_g1h->mark_in_progress()) { + cur->note_start_of_marking(false); + // _next_top_at_mark_start == top, _next_marked_bytes == 0 + // _next_marked_bytes == next_marked_bytes. + } + + // Now make sure the region has the right index in the sorted array. + g1_policy()->note_change_in_marked_bytes(cur); + } + cur = cur->next_in_collection_set(); + } + assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!"); + + // Now restore saved marks, if any. + if (_objs_with_preserved_marks != NULL) { + assert(_preserved_marks_of_objs != NULL, "Both or none."); + assert(_objs_with_preserved_marks->length() == + _preserved_marks_of_objs->length(), "Both or none."); + guarantee(_objs_with_preserved_marks->length() == + _preserved_marks_of_objs->length(), "Both or none."); + for (int i = 0; i < _objs_with_preserved_marks->length(); i++) { + oop obj = _objs_with_preserved_marks->at(i); + markOop m = _preserved_marks_of_objs->at(i); + obj->set_mark(m); + } + // Delete the preserved marks growable arrays (allocated on the C heap). + delete _objs_with_preserved_marks; + delete _preserved_marks_of_objs; + _objs_with_preserved_marks = NULL; + _preserved_marks_of_objs = NULL; + } +} + +void G1CollectedHeap::push_on_evac_failure_scan_stack(oop obj) { + _evac_failure_scan_stack->push(obj); +} + +void G1CollectedHeap::drain_evac_failure_scan_stack() { + assert(_evac_failure_scan_stack != NULL, "precondition"); + + while (_evac_failure_scan_stack->length() > 0) { + oop obj = _evac_failure_scan_stack->pop(); + _evac_failure_closure->set_region(heap_region_containing(obj)); + obj->oop_iterate_backwards(_evac_failure_closure); + } +} + +void G1CollectedHeap::handle_evacuation_failure(oop old) { + markOop m = old->mark(); + // forward to self + assert(!old->is_forwarded(), "precondition"); + + old->forward_to(old); + handle_evacuation_failure_common(old, m); +} + +oop +G1CollectedHeap::handle_evacuation_failure_par(OopsInHeapRegionClosure* cl, + oop old) { + markOop m = old->mark(); + oop forward_ptr = old->forward_to_atomic(old); + if (forward_ptr == NULL) { + // Forward-to-self succeeded. + if (_evac_failure_closure != cl) { + MutexLockerEx x(EvacFailureStack_lock, Mutex::_no_safepoint_check_flag); + assert(!_drain_in_progress, + "Should only be true while someone holds the lock."); + // Set the global evac-failure closure to the current thread's. + assert(_evac_failure_closure == NULL, "Or locking has failed."); + set_evac_failure_closure(cl); + // Now do the common part. + handle_evacuation_failure_common(old, m); + // Reset to NULL. + set_evac_failure_closure(NULL); + } else { + // The lock is already held, and this is recursive. + assert(_drain_in_progress, "This should only be the recursive case."); + handle_evacuation_failure_common(old, m); + } + return old; + } else { + // Someone else had a place to copy it. + return forward_ptr; + } +} + +void G1CollectedHeap::handle_evacuation_failure_common(oop old, markOop m) { + set_evacuation_failed(true); + + preserve_mark_if_necessary(old, m); + + HeapRegion* r = heap_region_containing(old); + if (!r->evacuation_failed()) { + r->set_evacuation_failed(true); + if (G1TraceRegions) { + gclog_or_tty->print("evacuation failed in heap region "PTR_FORMAT" " + "["PTR_FORMAT","PTR_FORMAT")\n", + r, r->bottom(), r->end()); + } + } + + push_on_evac_failure_scan_stack(old); + + if (!_drain_in_progress) { + // prevent recursion in copy_to_survivor_space() + _drain_in_progress = true; + drain_evac_failure_scan_stack(); + _drain_in_progress = false; + } +} + +void G1CollectedHeap::preserve_mark_if_necessary(oop obj, markOop m) { + if (m != markOopDesc::prototype()) { + if (_objs_with_preserved_marks == NULL) { + assert(_preserved_marks_of_objs == NULL, "Both or none."); + _objs_with_preserved_marks = + new (ResourceObj::C_HEAP) GrowableArray<oop>(40, true); + _preserved_marks_of_objs = + new (ResourceObj::C_HEAP) GrowableArray<markOop>(40, true); + } + _objs_with_preserved_marks->push(obj); + _preserved_marks_of_objs->push(m); + } +} + +// *** Parallel G1 Evacuation + +HeapWord* G1CollectedHeap::par_allocate_during_gc(GCAllocPurpose purpose, + size_t word_size) { + HeapRegion* alloc_region = _gc_alloc_regions[purpose]; + // let the caller handle alloc failure + if (alloc_region == NULL) return NULL; + + HeapWord* block = alloc_region->par_allocate(word_size); + if (block == NULL) { + MutexLockerEx x(par_alloc_during_gc_lock(), + Mutex::_no_safepoint_check_flag); + block = allocate_during_gc_slow(purpose, alloc_region, true, word_size); + } + return block; +} + +HeapWord* +G1CollectedHeap::allocate_during_gc_slow(GCAllocPurpose purpose, + HeapRegion* alloc_region, + bool par, + size_t word_size) { + HeapWord* block = NULL; + // In the parallel case, a previous thread to obtain the lock may have + // already assigned a new gc_alloc_region. + if (alloc_region != _gc_alloc_regions[purpose]) { + assert(par, "But should only happen in parallel case."); + alloc_region = _gc_alloc_regions[purpose]; + if (alloc_region == NULL) return NULL; + block = alloc_region->par_allocate(word_size); + if (block != NULL) return block; + // Otherwise, continue; this new region is empty, too. + } + assert(alloc_region != NULL, "We better have an allocation region"); + // Another thread might have obtained alloc_region for the given + // purpose, and might be attempting to allocate in it, and might + // succeed. Therefore, we can't do the "finalization" stuff on the + // region below until we're sure the last allocation has happened. + // We ensure this by allocating the remaining space with a garbage + // object. + if (par) par_allocate_remaining_space(alloc_region); + // Now we can do the post-GC stuff on the region. + alloc_region->note_end_of_copying(); + g1_policy()->record_after_bytes(alloc_region->used()); + + if (_gc_alloc_region_counts[purpose] >= g1_policy()->max_regions(purpose)) { + // Cannot allocate more regions for the given purpose. + GCAllocPurpose alt_purpose = g1_policy()->alternative_purpose(purpose); + // Is there an alternative? + if (purpose != alt_purpose) { + HeapRegion* alt_region = _gc_alloc_regions[alt_purpose]; + // Has not the alternative region been aliased? + if (alloc_region != alt_region) { + // Try to allocate in the alternative region. + if (par) { + block = alt_region->par_allocate(word_size); + } else { + block = alt_region->allocate(word_size); + } + // Make an alias. + _gc_alloc_regions[purpose] = _gc_alloc_regions[alt_purpose]; + } + if (block != NULL) { + return block; + } + // Both the allocation region and the alternative one are full + // and aliased, replace them with a new allocation region. + purpose = alt_purpose; + } else { + set_gc_alloc_region(purpose, NULL); + return NULL; + } + } + + // Now allocate a new region for allocation. + alloc_region = newAllocRegionWithExpansion(purpose, word_size, false /*zero_filled*/); + + // let the caller handle alloc failure + if (alloc_region != NULL) { + + assert(check_gc_alloc_regions(), "alloc regions messed up"); + assert(alloc_region->saved_mark_at_top(), + "Mark should have been saved already."); + // We used to assert that the region was zero-filled here, but no + // longer. + + // This must be done last: once it's installed, other regions may + // allocate in it (without holding the lock.) + set_gc_alloc_region(purpose, alloc_region); + + if (par) { + block = alloc_region->par_allocate(word_size); + } else { + block = alloc_region->allocate(word_size); + } + // Caller handles alloc failure. + } else { + // This sets other apis using the same old alloc region to NULL, also. + set_gc_alloc_region(purpose, NULL); + } + return block; // May be NULL. +} + +void G1CollectedHeap::par_allocate_remaining_space(HeapRegion* r) { + HeapWord* block = NULL; + size_t free_words; + do { + free_words = r->free()/HeapWordSize; + // If there's too little space, no one can allocate, so we're done. + if (free_words < (size_t)oopDesc::header_size()) return; + // Otherwise, try to claim it. + block = r->par_allocate(free_words); + } while (block == NULL); + SharedHeap::fill_region_with_object(MemRegion(block, free_words)); +} + +#define use_local_bitmaps 1 +#define verify_local_bitmaps 0 + +#ifndef PRODUCT + +class GCLabBitMap; +class GCLabBitMapClosure: public BitMapClosure { +private: + ConcurrentMark* _cm; + GCLabBitMap* _bitmap; + +public: + GCLabBitMapClosure(ConcurrentMark* cm, + GCLabBitMap* bitmap) { + _cm = cm; + _bitmap = bitmap; + } + + virtual bool do_bit(size_t offset); +}; + +#endif // PRODUCT + +#define oop_buffer_length 256 + +class GCLabBitMap: public BitMap { +private: + ConcurrentMark* _cm; + + int _shifter; + size_t _bitmap_word_covers_words; + + // beginning of the heap + HeapWord* _heap_start; + + // this is the actual start of the GCLab + HeapWord* _real_start_word; + + // this is the actual end of the GCLab + HeapWord* _real_end_word; + + // this is the first word, possibly located before the actual start + // of the GCLab, that corresponds to the first bit of the bitmap + HeapWord* _start_word; + + // size of a GCLab in words + size_t _gclab_word_size; + + static int shifter() { + return MinObjAlignment - 1; + } + + // how many heap words does a single bitmap word corresponds to? + static size_t bitmap_word_covers_words() { + return BitsPerWord << shifter(); + } + + static size_t gclab_word_size() { + return ParallelGCG1AllocBufferSize / HeapWordSize; + } + + static size_t bitmap_size_in_bits() { + size_t bits_in_bitmap = gclab_word_size() >> shifter(); + // We are going to ensure that the beginning of a word in this + // bitmap also corresponds to the beginning of a word in the + // global marking bitmap. To handle the case where a GCLab + // starts from the middle of the bitmap, we need to add enough + // space (i.e. up to a bitmap word) to ensure that we have + // enough bits in the bitmap. + return bits_in_bitmap + BitsPerWord - 1; + } +public: + GCLabBitMap(HeapWord* heap_start) + : BitMap(bitmap_size_in_bits()), + _cm(G1CollectedHeap::heap()->concurrent_mark()), + _shifter(shifter()), + _bitmap_word_covers_words(bitmap_word_covers_words()), + _heap_start(heap_start), + _gclab_word_size(gclab_word_size()), + _real_start_word(NULL), + _real_end_word(NULL), + _start_word(NULL) + { + guarantee( size_in_words() >= bitmap_size_in_words(), + "just making sure"); + } + + inline unsigned heapWordToOffset(HeapWord* addr) { + unsigned offset = (unsigned) pointer_delta(addr, _start_word) >> _shifter; + assert(offset < size(), "offset should be within bounds"); + return offset; + } + + inline HeapWord* offsetToHeapWord(size_t offset) { + HeapWord* addr = _start_word + (offset << _shifter); + assert(_real_start_word <= addr && addr < _real_end_word, "invariant"); + return addr; + } + + bool fields_well_formed() { + bool ret1 = (_real_start_word == NULL) && + (_real_end_word == NULL) && + (_start_word == NULL); + if (ret1) + return true; + + bool ret2 = _real_start_word >= _start_word && + _start_word < _real_end_word && + (_real_start_word + _gclab_word_size) == _real_end_word && + (_start_word + _gclab_word_size + _bitmap_word_covers_words) + > _real_end_word; + return ret2; + } + + inline bool mark(HeapWord* addr) { + guarantee(use_local_bitmaps, "invariant"); + assert(fields_well_formed(), "invariant"); + + if (addr >= _real_start_word && addr < _real_end_word) { + assert(!isMarked(addr), "should not have already been marked"); + + // first mark it on the bitmap + at_put(heapWordToOffset(addr), true); + + return true; + } else { + return false; + } + } + + inline bool isMarked(HeapWord* addr) { + guarantee(use_local_bitmaps, "invariant"); + assert(fields_well_formed(), "invariant"); + + return at(heapWordToOffset(addr)); + } + + void set_buffer(HeapWord* start) { + guarantee(use_local_bitmaps, "invariant"); + clear(); + + assert(start != NULL, "invariant"); + _real_start_word = start; + _real_end_word = start + _gclab_word_size; + + size_t diff = + pointer_delta(start, _heap_start) % _bitmap_word_covers_words; + _start_word = start - diff; + + assert(fields_well_formed(), "invariant"); + } + +#ifndef PRODUCT + void verify() { + // verify that the marks have been propagated + GCLabBitMapClosure cl(_cm, this); + iterate(&cl); + } +#endif // PRODUCT + + void retire() { + guarantee(use_local_bitmaps, "invariant"); + assert(fields_well_formed(), "invariant"); + + if (_start_word != NULL) { + CMBitMap* mark_bitmap = _cm->nextMarkBitMap(); + + // this means that the bitmap was set up for the GCLab + assert(_real_start_word != NULL && _real_end_word != NULL, "invariant"); + + mark_bitmap->mostly_disjoint_range_union(this, + 0, // always start from the start of the bitmap + _start_word, + size_in_words()); + _cm->grayRegionIfNecessary(MemRegion(_real_start_word, _real_end_word)); + +#ifndef PRODUCT + if (use_local_bitmaps && verify_local_bitmaps) + verify(); +#endif // PRODUCT + } else { + assert(_real_start_word == NULL && _real_end_word == NULL, "invariant"); + } + } + + static size_t bitmap_size_in_words() { + return (bitmap_size_in_bits() + BitsPerWord - 1) / BitsPerWord; + } +}; + +#ifndef PRODUCT + +bool GCLabBitMapClosure::do_bit(size_t offset) { + HeapWord* addr = _bitmap->offsetToHeapWord(offset); + guarantee(_cm->isMarked(oop(addr)), "it should be!"); + return true; +} + +#endif // PRODUCT + +class G1ParGCAllocBuffer: public ParGCAllocBuffer { +private: + bool _retired; + bool _during_marking; + GCLabBitMap _bitmap; + +public: + G1ParGCAllocBuffer() : + ParGCAllocBuffer(ParallelGCG1AllocBufferSize / HeapWordSize), + _during_marking(G1CollectedHeap::heap()->mark_in_progress()), + _bitmap(G1CollectedHeap::heap()->reserved_region().start()), + _retired(false) + { } + + inline bool mark(HeapWord* addr) { + guarantee(use_local_bitmaps, "invariant"); + assert(_during_marking, "invariant"); + return _bitmap.mark(addr); + } + + inline void set_buf(HeapWord* buf) { + if (use_local_bitmaps && _during_marking) + _bitmap.set_buffer(buf); + ParGCAllocBuffer::set_buf(buf); + _retired = false; + } + + inline void retire(bool end_of_gc, bool retain) { + if (_retired) + return; + if (use_local_bitmaps && _during_marking) { + _bitmap.retire(); + } + ParGCAllocBuffer::retire(end_of_gc, retain); + _retired = true; + } +}; + + +class G1ParScanThreadState : public StackObj { +protected: + G1CollectedHeap* _g1h; + RefToScanQueue* _refs; + + typedef GrowableArray<oop*> OverflowQueue; + OverflowQueue* _overflowed_refs; + + G1ParGCAllocBuffer _alloc_buffers[GCAllocPurposeCount]; + + size_t _alloc_buffer_waste; + size_t _undo_waste; + + OopsInHeapRegionClosure* _evac_failure_cl; + G1ParScanHeapEvacClosure* _evac_cl; + G1ParScanPartialArrayClosure* _partial_scan_cl; + + int _hash_seed; + int _queue_num; + + int _term_attempts; +#if G1_DETAILED_STATS + int _pushes, _pops, _steals, _steal_attempts; + int _overflow_pushes; +#endif + + double _start; + double _start_strong_roots; + double _strong_roots_time; + double _start_term; + double _term_time; + + // Map from young-age-index (0 == not young, 1 is youngest) to + // surviving words. base is what we get back from the malloc call + size_t* _surviving_young_words_base; + // this points into the array, as we use the first few entries for padding + size_t* _surviving_young_words; + +#define PADDING_ELEM_NUM (64 / sizeof(size_t)) + + void add_to_alloc_buffer_waste(size_t waste) { _alloc_buffer_waste += waste; } + + void add_to_undo_waste(size_t waste) { _undo_waste += waste; } + +public: + G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num) + : _g1h(g1h), + _refs(g1h->task_queue(queue_num)), + _hash_seed(17), _queue_num(queue_num), + _term_attempts(0), +#if G1_DETAILED_STATS + _pushes(0), _pops(0), _steals(0), + _steal_attempts(0), _overflow_pushes(0), +#endif + _strong_roots_time(0), _term_time(0), + _alloc_buffer_waste(0), _undo_waste(0) + { + // we allocate G1YoungSurvRateNumRegions plus one entries, since + // we "sacrifice" entry 0 to keep track of surviving bytes for + // non-young regions (where the age is -1) + // We also add a few elements at the beginning and at the end in + // an attempt to eliminate cache contention + size_t real_length = 1 + _g1h->g1_policy()->young_cset_length(); + size_t array_length = PADDING_ELEM_NUM + + real_length + + PADDING_ELEM_NUM; + _surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length); + if (_surviving_young_words_base == NULL) + vm_exit_out_of_memory(array_length * sizeof(size_t), + "Not enough space for young surv histo."); + _surviving_young_words = _surviving_young_words_base + PADDING_ELEM_NUM; + memset(_surviving_young_words, 0, real_length * sizeof(size_t)); + + _overflowed_refs = new OverflowQueue(10); + + _start = os::elapsedTime(); + } + + ~G1ParScanThreadState() { + FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base); + } + + RefToScanQueue* refs() { return _refs; } + OverflowQueue* overflowed_refs() { return _overflowed_refs; } + + inline G1ParGCAllocBuffer* alloc_buffer(GCAllocPurpose purpose) { + return &_alloc_buffers[purpose]; + } + + size_t alloc_buffer_waste() { return _alloc_buffer_waste; } + size_t undo_waste() { return _undo_waste; } + + void push_on_queue(oop* ref) { + if (!refs()->push(ref)) { + overflowed_refs()->push(ref); + IF_G1_DETAILED_STATS(note_overflow_push()); + } else { + IF_G1_DETAILED_STATS(note_push()); + } + } + + void pop_from_queue(oop*& ref) { + if (!refs()->pop_local(ref)) { + ref = NULL; + } else { + IF_G1_DETAILED_STATS(note_pop()); + } + } + + void pop_from_overflow_queue(oop*& ref) { + ref = overflowed_refs()->pop(); + } + + int refs_to_scan() { return refs()->size(); } + int overflowed_refs_to_scan() { return overflowed_refs()->length(); } + + HeapWord* allocate_slow(GCAllocPurpose purpose, size_t word_sz) { + + HeapWord* obj = NULL; + if (word_sz * 100 < + (size_t)(ParallelGCG1AllocBufferSize / HeapWordSize) * + ParallelGCBufferWastePct) { + G1ParGCAllocBuffer* alloc_buf = alloc_buffer(purpose); + add_to_alloc_buffer_waste(alloc_buf->words_remaining()); + alloc_buf->retire(false, false); + + HeapWord* buf = + _g1h->par_allocate_during_gc(purpose, ParallelGCG1AllocBufferSize / HeapWordSize); + if (buf == NULL) return NULL; // Let caller handle allocation failure. + // Otherwise. + alloc_buf->set_buf(buf); + + obj = alloc_buf->allocate(word_sz); + assert(obj != NULL, "buffer was definitely big enough..."); + } + else { + obj = _g1h->par_allocate_during_gc(purpose, word_sz); + } + return obj; + } + + HeapWord* allocate(GCAllocPurpose purpose, size_t word_sz) { + HeapWord* obj = alloc_buffer(purpose)->allocate(word_sz); + if (obj != NULL) return obj; + return allocate_slow(purpose, word_sz); + } + + void undo_allocation(GCAllocPurpose purpose, HeapWord* obj, size_t word_sz) { + if (alloc_buffer(purpose)->contains(obj)) { + guarantee(alloc_buffer(purpose)->contains(obj + word_sz - 1), + "should contain whole object"); + alloc_buffer(purpose)->undo_allocation(obj, word_sz); + } + else { + SharedHeap::fill_region_with_object(MemRegion(obj, word_sz)); + add_to_undo_waste(word_sz); + } + } + + void set_evac_failure_closure(OopsInHeapRegionClosure* evac_failure_cl) { + _evac_failure_cl = evac_failure_cl; + } + OopsInHeapRegionClosure* evac_failure_closure() { + return _evac_failure_cl; + } + + void set_evac_closure(G1ParScanHeapEvacClosure* evac_cl) { + _evac_cl = evac_cl; + } + + void set_partial_scan_closure(G1ParScanPartialArrayClosure* partial_scan_cl) { + _partial_scan_cl = partial_scan_cl; + } + + int* hash_seed() { return &_hash_seed; } + int queue_num() { return _queue_num; } + + int term_attempts() { return _term_attempts; } + void note_term_attempt() { _term_attempts++; } + +#if G1_DETAILED_STATS + int pushes() { return _pushes; } + int pops() { return _pops; } + int steals() { return _steals; } + int steal_attempts() { return _steal_attempts; } + int overflow_pushes() { return _overflow_pushes; } + + void note_push() { _pushes++; } + void note_pop() { _pops++; } + void note_steal() { _steals++; } + void note_steal_attempt() { _steal_attempts++; } + void note_overflow_push() { _overflow_pushes++; } +#endif + + void start_strong_roots() { + _start_strong_roots = os::elapsedTime(); + } + void end_strong_roots() { + _strong_roots_time += (os::elapsedTime() - _start_strong_roots); + } + double strong_roots_time() { return _strong_roots_time; } + + void start_term_time() { + note_term_attempt(); + _start_term = os::elapsedTime(); + } + void end_term_time() { + _term_time += (os::elapsedTime() - _start_term); + } + double term_time() { return _term_time; } + + double elapsed() { + return os::elapsedTime() - _start; + } + + size_t* surviving_young_words() { + // We add on to hide entry 0 which accumulates surviving words for + // age -1 regions (i.e. non-young ones) + return _surviving_young_words; + } + + void retire_alloc_buffers() { + for (int ap = 0; ap < GCAllocPurposeCount; ++ap) { + size_t waste = _alloc_buffers[ap].words_remaining(); + add_to_alloc_buffer_waste(waste); + _alloc_buffers[ap].retire(true, false); + } + } + + void trim_queue() { + while (refs_to_scan() > 0 || overflowed_refs_to_scan() > 0) { + oop *ref_to_scan = NULL; + if (overflowed_refs_to_scan() == 0) { + pop_from_queue(ref_to_scan); + } else { + pop_from_overflow_queue(ref_to_scan); + } + if (ref_to_scan != NULL) { + if ((intptr_t)ref_to_scan & G1_PARTIAL_ARRAY_MASK) { + _partial_scan_cl->do_oop_nv(ref_to_scan); + } else { + // Note: we can use "raw" versions of "region_containing" because + // "obj_to_scan" is definitely in the heap, and is not in a + // humongous region. + HeapRegion* r = _g1h->heap_region_containing_raw(ref_to_scan); + _evac_cl->set_region(r); + _evac_cl->do_oop_nv(ref_to_scan); + } + } + } + } +}; + + +G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) : + _g1(g1), _g1_rem(_g1->g1_rem_set()), _cm(_g1->concurrent_mark()), + _par_scan_state(par_scan_state) { } + +// This closure is applied to the fields of the objects that have just been copied. +// Should probably be made inline and moved in g1OopClosures.inline.hpp. +void G1ParScanClosure::do_oop_nv(oop* p) { + oop obj = *p; + if (obj != NULL) { + if (_g1->obj_in_cs(obj)) { + if (obj->is_forwarded()) { + *p = obj->forwardee(); + } else { + _par_scan_state->push_on_queue(p); + return; + } + } + _g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num()); + } +} + +void G1ParCopyHelper::mark_forwardee(oop* p) { + // This is called _after_ do_oop_work has been called, hence after + // the object has been relocated to its new location and *p points + // to its new location. + + oop thisOop = *p; + if (thisOop != NULL) { + assert((_g1->evacuation_failed()) || (!_g1->obj_in_cs(thisOop)), + "shouldn't still be in the CSet if evacuation didn't fail."); + HeapWord* addr = (HeapWord*)thisOop; + if (_g1->is_in_g1_reserved(addr)) + _cm->grayRoot(oop(addr)); + } +} + +oop G1ParCopyHelper::copy_to_survivor_space(oop old) { + size_t word_sz = old->size(); + HeapRegion* from_region = _g1->heap_region_containing_raw(old); + // +1 to make the -1 indexes valid... + int young_index = from_region->young_index_in_cset()+1; + assert( (from_region->is_young() && young_index > 0) || + (!from_region->is_young() && young_index == 0), "invariant" ); + G1CollectorPolicy* g1p = _g1->g1_policy(); + markOop m = old->mark(); + GCAllocPurpose alloc_purpose = g1p->evacuation_destination(from_region, m->age(), + word_sz); + HeapWord* obj_ptr = _par_scan_state->allocate(alloc_purpose, word_sz); + oop obj = oop(obj_ptr); + + if (obj_ptr == NULL) { + // This will either forward-to-self, or detect that someone else has + // installed a forwarding pointer. + OopsInHeapRegionClosure* cl = _par_scan_state->evac_failure_closure(); + return _g1->handle_evacuation_failure_par(cl, old); + } + + oop forward_ptr = old->forward_to_atomic(obj); + if (forward_ptr == NULL) { + Copy::aligned_disjoint_words((HeapWord*) old, obj_ptr, word_sz); + obj->set_mark(m); + if (g1p->track_object_age(alloc_purpose)) { + obj->incr_age(); + } + // preserve "next" mark bit + if (_g1->mark_in_progress() && !_g1->is_obj_ill(old)) { + if (!use_local_bitmaps || + !_par_scan_state->alloc_buffer(alloc_purpose)->mark(obj_ptr)) { + // if we couldn't mark it on the local bitmap (this happens when + // the object was not allocated in the GCLab), we have to bite + // the bullet and do the standard parallel mark + _cm->markAndGrayObjectIfNecessary(obj); + } +#if 1 + if (_g1->isMarkedNext(old)) { + _cm->nextMarkBitMap()->parClear((HeapWord*)old); + } +#endif + } + + size_t* surv_young_words = _par_scan_state->surviving_young_words(); + surv_young_words[young_index] += word_sz; + + if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) { + arrayOop(old)->set_length(0); + _par_scan_state->push_on_queue((oop*) ((intptr_t)old | G1_PARTIAL_ARRAY_MASK)); + } else { + _scanner->set_region(_g1->heap_region_containing(obj)); + obj->oop_iterate_backwards(_scanner); + } + } else { + _par_scan_state->undo_allocation(alloc_purpose, obj_ptr, word_sz); + obj = forward_ptr; + } + return obj; +} + +template<bool do_gen_barrier, G1Barrier barrier, bool do_mark_forwardee> +void G1ParCopyClosure<do_gen_barrier, barrier, do_mark_forwardee>::do_oop_work(oop* p) { + oop obj = *p; + assert(barrier != G1BarrierRS || obj != NULL, + "Precondition: G1BarrierRS implies obj is nonNull"); + + if (obj != NULL) { + if (_g1->obj_in_cs(obj)) { +#if G1_REM_SET_LOGGING + gclog_or_tty->print_cr("Loc "PTR_FORMAT" contains pointer "PTR_FORMAT" into CS.", + p, (void*) obj); +#endif + if (obj->is_forwarded()) { + *p = obj->forwardee(); + } else { + *p = copy_to_survivor_space(obj); + } + // When scanning the RS, we only care about objs in CS. + if (barrier == G1BarrierRS) { + _g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num()); + } + } + // When scanning moved objs, must look at all oops. + if (barrier == G1BarrierEvac) { + _g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num()); + } + + if (do_gen_barrier) { + par_do_barrier(p); + } + } +} + +template void G1ParCopyClosure<false, G1BarrierEvac, false>::do_oop_work(oop* p); + +template <class T> void G1ParScanPartialArrayClosure::process_array_chunk( + oop obj, int start, int end) { + // process our set of indices (include header in first chunk) + assert(start < end, "invariant"); + T* const base = (T*)objArrayOop(obj)->base(); + T* const start_addr = base + start; + T* const end_addr = base + end; + MemRegion mr((HeapWord*)start_addr, (HeapWord*)end_addr); + _scanner.set_region(_g1->heap_region_containing(obj)); + obj->oop_iterate(&_scanner, mr); +} + +void G1ParScanPartialArrayClosure::do_oop_nv(oop* p) { + assert(!UseCompressedOops, "Needs to be fixed to work with compressed oops"); + oop old = oop((intptr_t)p & ~G1_PARTIAL_ARRAY_MASK); + assert(old->is_objArray(), "must be obj array"); + assert(old->is_forwarded(), "must be forwarded"); + assert(Universe::heap()->is_in_reserved(old), "must be in heap."); + + objArrayOop obj = objArrayOop(old->forwardee()); + assert((void*)old != (void*)old->forwardee(), "self forwarding here?"); + // Process ParGCArrayScanChunk elements now + // and push the remainder back onto queue + int start = arrayOop(old)->length(); + int end = obj->length(); + int remainder = end - start; + assert(start <= end, "just checking"); + if (remainder > 2 * ParGCArrayScanChunk) { + // Test above combines last partial chunk with a full chunk + end = start + ParGCArrayScanChunk; + arrayOop(old)->set_length(end); + // Push remainder. + _par_scan_state->push_on_queue((oop*) ((intptr_t) old | G1_PARTIAL_ARRAY_MASK)); + } else { + // Restore length so that the heap remains parsable in + // case of evacuation failure. + arrayOop(old)->set_length(end); + } + + // process our set of indices (include header in first chunk) + process_array_chunk<oop>(obj, start, end); + oop* start_addr = start == 0 ? (oop*)obj : obj->obj_at_addr<oop>(start); + oop* end_addr = (oop*)(obj->base()) + end; // obj_at_addr(end) asserts end < length + MemRegion mr((HeapWord*)start_addr, (HeapWord*)end_addr); + _scanner.set_region(_g1->heap_region_containing(obj)); + obj->oop_iterate(&_scanner, mr); +} + +int G1ScanAndBalanceClosure::_nq = 0; + +class G1ParEvacuateFollowersClosure : public VoidClosure { +protected: + G1CollectedHeap* _g1h; + G1ParScanThreadState* _par_scan_state; + RefToScanQueueSet* _queues; + ParallelTaskTerminator* _terminator; + + G1ParScanThreadState* par_scan_state() { return _par_scan_state; } + RefToScanQueueSet* queues() { return _queues; } + ParallelTaskTerminator* terminator() { return _terminator; } + +public: + G1ParEvacuateFollowersClosure(G1CollectedHeap* g1h, + G1ParScanThreadState* par_scan_state, + RefToScanQueueSet* queues, + ParallelTaskTerminator* terminator) + : _g1h(g1h), _par_scan_state(par_scan_state), + _queues(queues), _terminator(terminator) {} + + void do_void() { + G1ParScanThreadState* pss = par_scan_state(); + while (true) { + oop* ref_to_scan; + pss->trim_queue(); + IF_G1_DETAILED_STATS(pss->note_steal_attempt()); + if (queues()->steal(pss->queue_num(), + pss->hash_seed(), + ref_to_scan)) { + IF_G1_DETAILED_STATS(pss->note_steal()); + pss->push_on_queue(ref_to_scan); + continue; + } + pss->start_term_time(); + if (terminator()->offer_termination()) break; + pss->end_term_time(); + } + pss->end_term_time(); + pss->retire_alloc_buffers(); + } +}; + +class G1ParTask : public AbstractGangTask { +protected: + G1CollectedHeap* _g1h; + RefToScanQueueSet *_queues; + ParallelTaskTerminator _terminator; + + Mutex _stats_lock; + Mutex* stats_lock() { return &_stats_lock; } + + size_t getNCards() { + return (_g1h->capacity() + G1BlockOffsetSharedArray::N_bytes - 1) + / G1BlockOffsetSharedArray::N_bytes; + } + +public: + G1ParTask(G1CollectedHeap* g1h, int workers, RefToScanQueueSet *task_queues) + : AbstractGangTask("G1 collection"), + _g1h(g1h), + _queues(task_queues), + _terminator(workers, _queues), + _stats_lock(Mutex::leaf, "parallel G1 stats lock", true) + {} + + RefToScanQueueSet* queues() { return _queues; } + + RefToScanQueue *work_queue(int i) { + return queues()->queue(i); + } + + void work(int i) { + ResourceMark rm; + HandleMark hm; + + G1ParScanThreadState pss(_g1h, i); + G1ParScanHeapEvacClosure scan_evac_cl(_g1h, &pss); + G1ParScanHeapEvacClosure evac_failure_cl(_g1h, &pss); + G1ParScanPartialArrayClosure partial_scan_cl(_g1h, &pss); + + pss.set_evac_closure(&scan_evac_cl); + pss.set_evac_failure_closure(&evac_failure_cl); + pss.set_partial_scan_closure(&partial_scan_cl); + + G1ParScanExtRootClosure only_scan_root_cl(_g1h, &pss); + G1ParScanPermClosure only_scan_perm_cl(_g1h, &pss); + G1ParScanHeapRSClosure only_scan_heap_rs_cl(_g1h, &pss); + G1ParScanAndMarkExtRootClosure scan_mark_root_cl(_g1h, &pss); + G1ParScanAndMarkPermClosure scan_mark_perm_cl(_g1h, &pss); + G1ParScanAndMarkHeapRSClosure scan_mark_heap_rs_cl(_g1h, &pss); + + OopsInHeapRegionClosure *scan_root_cl; + OopsInHeapRegionClosure *scan_perm_cl; + OopsInHeapRegionClosure *scan_so_cl; + + if (_g1h->g1_policy()->should_initiate_conc_mark()) { + scan_root_cl = &scan_mark_root_cl; + scan_perm_cl = &scan_mark_perm_cl; + scan_so_cl = &scan_mark_heap_rs_cl; + } else { + scan_root_cl = &only_scan_root_cl; + scan_perm_cl = &only_scan_perm_cl; + scan_so_cl = &only_scan_heap_rs_cl; + } + + pss.start_strong_roots(); + _g1h->g1_process_strong_roots(/* not collecting perm */ false, + SharedHeap::SO_AllClasses, + scan_root_cl, + &only_scan_heap_rs_cl, + scan_so_cl, + scan_perm_cl, + i); + pss.end_strong_roots(); + { + double start = os::elapsedTime(); + G1ParEvacuateFollowersClosure evac(_g1h, &pss, _queues, &_terminator); + evac.do_void(); + double elapsed_ms = (os::elapsedTime()-start)*1000.0; + double term_ms = pss.term_time()*1000.0; + _g1h->g1_policy()->record_obj_copy_time(i, elapsed_ms-term_ms); + _g1h->g1_policy()->record_termination_time(i, term_ms); + } + _g1h->update_surviving_young_words(pss.surviving_young_words()+1); + + // Clean up any par-expanded rem sets. + HeapRegionRemSet::par_cleanup(); + + MutexLocker x(stats_lock()); + if (ParallelGCVerbose) { + gclog_or_tty->print("Thread %d complete:\n", i); +#if G1_DETAILED_STATS + gclog_or_tty->print(" Pushes: %7d Pops: %7d Overflows: %7d Steals %7d (in %d attempts)\n", + pss.pushes(), + pss.pops(), + pss.overflow_pushes(), + pss.steals(), + pss.steal_attempts()); +#endif + double elapsed = pss.elapsed(); + double strong_roots = pss.strong_roots_time(); + double term = pss.term_time(); + gclog_or_tty->print(" Elapsed: %7.2f ms.\n" + " Strong roots: %7.2f ms (%6.2f%%)\n" + " Termination: %7.2f ms (%6.2f%%) (in %d entries)\n", + elapsed * 1000.0, + strong_roots * 1000.0, (strong_roots*100.0/elapsed), + term * 1000.0, (term*100.0/elapsed), + pss.term_attempts()); + size_t total_waste = pss.alloc_buffer_waste() + pss.undo_waste(); + gclog_or_tty->print(" Waste: %8dK\n" + " Alloc Buffer: %8dK\n" + " Undo: %8dK\n", + (total_waste * HeapWordSize) / K, + (pss.alloc_buffer_waste() * HeapWordSize) / K, + (pss.undo_waste() * HeapWordSize) / K); + } + + assert(pss.refs_to_scan() == 0, "Task queue should be empty"); + assert(pss.overflowed_refs_to_scan() == 0, "Overflow queue should be empty"); + } +}; + +// *** Common G1 Evacuation Stuff + +class G1CountClosure: public OopsInHeapRegionClosure { +public: + int n; + G1CountClosure() : n(0) {} + void do_oop(narrowOop* p) { + guarantee(false, "NYI"); + } + void do_oop(oop* p) { + oop obj = *p; + assert(obj != NULL && G1CollectedHeap::heap()->obj_in_cs(obj), + "Rem set closure called on non-rem-set pointer."); + n++; + } +}; + +static G1CountClosure count_closure; + +void +G1CollectedHeap:: +g1_process_strong_roots(bool collecting_perm_gen, + SharedHeap::ScanningOption so, + OopClosure* scan_non_heap_roots, + OopsInHeapRegionClosure* scan_rs, + OopsInHeapRegionClosure* scan_so, + OopsInGenClosure* scan_perm, + int worker_i) { + // First scan the strong roots, including the perm gen. + double ext_roots_start = os::elapsedTime(); + double closure_app_time_sec = 0.0; + + BufferingOopClosure buf_scan_non_heap_roots(scan_non_heap_roots); + BufferingOopsInGenClosure buf_scan_perm(scan_perm); + buf_scan_perm.set_generation(perm_gen()); + + process_strong_roots(collecting_perm_gen, so, + &buf_scan_non_heap_roots, + &buf_scan_perm); + // Finish up any enqueued closure apps. + buf_scan_non_heap_roots.done(); + buf_scan_perm.done(); + double ext_roots_end = os::elapsedTime(); + g1_policy()->reset_obj_copy_time(worker_i); + double obj_copy_time_sec = + buf_scan_non_heap_roots.closure_app_seconds() + + buf_scan_perm.closure_app_seconds(); + g1_policy()->record_obj_copy_time(worker_i, obj_copy_time_sec * 1000.0); + double ext_root_time_ms = + ((ext_roots_end - ext_roots_start) - obj_copy_time_sec) * 1000.0; + g1_policy()->record_ext_root_scan_time(worker_i, ext_root_time_ms); + + // Scan strong roots in mark stack. + if (!_process_strong_tasks->is_task_claimed(G1H_PS_mark_stack_oops_do)) { + concurrent_mark()->oops_do(scan_non_heap_roots); + } + double mark_stack_scan_ms = (os::elapsedTime() - ext_roots_end) * 1000.0; + g1_policy()->record_mark_stack_scan_time(worker_i, mark_stack_scan_ms); + + // XXX What should this be doing in the parallel case? + g1_policy()->record_collection_pause_end_CH_strong_roots(); + if (G1VerifyRemSet) { + // :::: FIXME :::: + // The stupid remembered set doesn't know how to filter out dead + // objects, which the smart one does, and so when it is created + // and then compared the number of entries in each differs and + // the verification code fails. + guarantee(false, "verification code is broken, see note"); + + // Let's make sure that the current rem set agrees with the stupidest + // one possible! + bool refs_enabled = ref_processor()->discovery_enabled(); + if (refs_enabled) ref_processor()->disable_discovery(); + StupidG1RemSet stupid(this); + count_closure.n = 0; + stupid.oops_into_collection_set_do(&count_closure, worker_i); + int stupid_n = count_closure.n; + count_closure.n = 0; + g1_rem_set()->oops_into_collection_set_do(&count_closure, worker_i); + guarantee(count_closure.n == stupid_n, "Old and new rem sets differ."); + gclog_or_tty->print_cr("\nFound %d pointers in heap RS.", count_closure.n); + if (refs_enabled) ref_processor()->enable_discovery(); + } + if (scan_so != NULL) { + scan_scan_only_set(scan_so, worker_i); + } + // Now scan the complement of the collection set. + if (scan_rs != NULL) { + g1_rem_set()->oops_into_collection_set_do(scan_rs, worker_i); + } + // Finish with the ref_processor roots. + if (!_process_strong_tasks->is_task_claimed(G1H_PS_refProcessor_oops_do)) { + ref_processor()->oops_do(scan_non_heap_roots); + } + g1_policy()->record_collection_pause_end_G1_strong_roots(); + _process_strong_tasks->all_tasks_completed(); +} + +void +G1CollectedHeap::scan_scan_only_region(HeapRegion* r, + OopsInHeapRegionClosure* oc, + int worker_i) { + HeapWord* startAddr = r->bottom(); + HeapWord* endAddr = r->used_region().end(); + + oc->set_region(r); + + HeapWord* p = r->bottom(); + HeapWord* t = r->top(); + guarantee( p == r->next_top_at_mark_start(), "invariant" ); + while (p < t) { + oop obj = oop(p); + p += obj->oop_iterate(oc); + } +} + +void +G1CollectedHeap::scan_scan_only_set(OopsInHeapRegionClosure* oc, + int worker_i) { + double start = os::elapsedTime(); + + BufferingOopsInHeapRegionClosure boc(oc); + + FilterInHeapRegionAndIntoCSClosure scan_only(this, &boc); + FilterAndMarkInHeapRegionAndIntoCSClosure scan_and_mark(this, &boc, concurrent_mark()); + + OopsInHeapRegionClosure *foc; + if (g1_policy()->should_initiate_conc_mark()) + foc = &scan_and_mark; + else + foc = &scan_only; + + HeapRegion* hr; + int n = 0; + while ((hr = _young_list->par_get_next_scan_only_region()) != NULL) { + scan_scan_only_region(hr, foc, worker_i); + ++n; + } + boc.done(); + + double closure_app_s = boc.closure_app_seconds(); + g1_policy()->record_obj_copy_time(worker_i, closure_app_s * 1000.0); + double ms = (os::elapsedTime() - start - closure_app_s)*1000.0; + g1_policy()->record_scan_only_time(worker_i, ms, n); +} + +void +G1CollectedHeap::g1_process_weak_roots(OopClosure* root_closure, + OopClosure* non_root_closure) { + SharedHeap::process_weak_roots(root_closure, non_root_closure); +} + + +class SaveMarksClosure: public HeapRegionClosure { +public: + bool doHeapRegion(HeapRegion* r) { + r->save_marks(); + return false; + } +}; + +void G1CollectedHeap::save_marks() { + if (ParallelGCThreads == 0) { + SaveMarksClosure sm; + heap_region_iterate(&sm); + } + // We do this even in the parallel case + perm_gen()->save_marks(); +} + +void G1CollectedHeap::evacuate_collection_set() { + set_evacuation_failed(false); + + g1_rem_set()->prepare_for_oops_into_collection_set_do(); + concurrent_g1_refine()->set_use_cache(false); + int n_workers = (ParallelGCThreads > 0 ? workers()->total_workers() : 1); + + set_par_threads(n_workers); + G1ParTask g1_par_task(this, n_workers, _task_queues); + + init_for_evac_failure(NULL); + + change_strong_roots_parity(); // In preparation for parallel strong roots. + rem_set()->prepare_for_younger_refs_iterate(true); + double start_par = os::elapsedTime(); + + if (ParallelGCThreads > 0) { + // The individual threads will set their evac-failure closures. + workers()->run_task(&g1_par_task); + } else { + g1_par_task.work(0); + } + + double par_time = (os::elapsedTime() - start_par) * 1000.0; + g1_policy()->record_par_time(par_time); + set_par_threads(0); + // Is this the right thing to do here? We don't save marks + // on individual heap regions when we allocate from + // them in parallel, so this seems like the correct place for this. + all_alloc_regions_note_end_of_copying(); + { + G1IsAliveClosure is_alive(this); + G1KeepAliveClosure keep_alive(this); + JNIHandles::weak_oops_do(&is_alive, &keep_alive); + } + + g1_rem_set()->cleanup_after_oops_into_collection_set_do(); + concurrent_g1_refine()->set_use_cache(true); + + finalize_for_evac_failure(); + + // Must do this before removing self-forwarding pointers, which clears + // the per-region evac-failure flags. + concurrent_mark()->complete_marking_in_collection_set(); + + if (evacuation_failed()) { + remove_self_forwarding_pointers(); + + if (PrintGCDetails) { + gclog_or_tty->print(" (evacuation failed)"); + } else if (PrintGC) { + gclog_or_tty->print("--"); + } + } + + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); +} + +void G1CollectedHeap::free_region(HeapRegion* hr) { + size_t pre_used = 0; + size_t cleared_h_regions = 0; + size_t freed_regions = 0; + UncleanRegionList local_list; + + HeapWord* start = hr->bottom(); + HeapWord* end = hr->prev_top_at_mark_start(); + size_t used_bytes = hr->used(); + size_t live_bytes = hr->max_live_bytes(); + if (used_bytes > 0) { + guarantee( live_bytes <= used_bytes, "invariant" ); + } else { + guarantee( live_bytes == 0, "invariant" ); + } + + size_t garbage_bytes = used_bytes - live_bytes; + if (garbage_bytes > 0) + g1_policy()->decrease_known_garbage_bytes(garbage_bytes); + + free_region_work(hr, pre_used, cleared_h_regions, freed_regions, + &local_list); + finish_free_region_work(pre_used, cleared_h_regions, freed_regions, + &local_list); +} + +void +G1CollectedHeap::free_region_work(HeapRegion* hr, + size_t& pre_used, + size_t& cleared_h_regions, + size_t& freed_regions, + UncleanRegionList* list, + bool par) { + assert(!hr->popular(), "should not free popular regions"); + pre_used += hr->used(); + if (hr->isHumongous()) { + assert(hr->startsHumongous(), + "Only the start of a humongous region should be freed."); + int ind = _hrs->find(hr); + assert(ind != -1, "Should have an index."); + // Clear the start region. + hr->hr_clear(par, true /*clear_space*/); + list->insert_before_head(hr); + cleared_h_regions++; + freed_regions++; + // Clear any continued regions. + ind++; + while ((size_t)ind < n_regions()) { + HeapRegion* hrc = _hrs->at(ind); + if (!hrc->continuesHumongous()) break; + // Otherwise, does continue the H region. + assert(hrc->humongous_start_region() == hr, "Huh?"); + hrc->hr_clear(par, true /*clear_space*/); + cleared_h_regions++; + freed_regions++; + list->insert_before_head(hrc); + ind++; + } + } else { + hr->hr_clear(par, true /*clear_space*/); + list->insert_before_head(hr); + freed_regions++; + // If we're using clear2, this should not be enabled. + // assert(!hr->in_cohort(), "Can't be both free and in a cohort."); + } +} + +void G1CollectedHeap::finish_free_region_work(size_t pre_used, + size_t cleared_h_regions, + size_t freed_regions, + UncleanRegionList* list) { + if (list != NULL && list->sz() > 0) { + prepend_region_list_on_unclean_list(list); + } + // Acquire a lock, if we're parallel, to update possibly-shared + // variables. + Mutex* lock = (n_par_threads() > 0) ? ParGCRareEvent_lock : NULL; + { + MutexLockerEx x(lock, Mutex::_no_safepoint_check_flag); + _summary_bytes_used -= pre_used; + _num_humongous_regions -= (int) cleared_h_regions; + _free_regions += freed_regions; + } +} + + +void G1CollectedHeap::dirtyCardsForYoungRegions(CardTableModRefBS* ct_bs, HeapRegion* list) { + while (list != NULL) { + guarantee( list->is_young(), "invariant" ); + + HeapWord* bottom = list->bottom(); + HeapWord* end = list->end(); + MemRegion mr(bottom, end); + ct_bs->dirty(mr); + + list = list->get_next_young_region(); + } +} + +void G1CollectedHeap::cleanUpCardTable() { + CardTableModRefBS* ct_bs = (CardTableModRefBS*) (barrier_set()); + double start = os::elapsedTime(); + + ct_bs->clear(_g1_committed); + + // now, redirty the cards of the scan-only and survivor regions + // (it seemed faster to do it this way, instead of iterating over + // all regions and then clearing / dirtying as approprite) + dirtyCardsForYoungRegions(ct_bs, _young_list->first_scan_only_region()); + dirtyCardsForYoungRegions(ct_bs, _young_list->first_survivor_region()); + + double elapsed = os::elapsedTime() - start; + g1_policy()->record_clear_ct_time( elapsed * 1000.0); +} + + +void G1CollectedHeap::do_collection_pause_if_appropriate(size_t word_size) { + // First do any popular regions. + HeapRegion* hr; + while ((hr = popular_region_to_evac()) != NULL) { + evac_popular_region(hr); + } + // Now do heuristic pauses. + if (g1_policy()->should_do_collection_pause(word_size)) { + do_collection_pause(); + } +} + +void G1CollectedHeap::free_collection_set(HeapRegion* cs_head) { + double young_time_ms = 0.0; + double non_young_time_ms = 0.0; + + G1CollectorPolicy* policy = g1_policy(); + + double start_sec = os::elapsedTime(); + bool non_young = true; + + HeapRegion* cur = cs_head; + int age_bound = -1; + size_t rs_lengths = 0; + + while (cur != NULL) { + if (non_young) { + if (cur->is_young()) { + double end_sec = os::elapsedTime(); + double elapsed_ms = (end_sec - start_sec) * 1000.0; + non_young_time_ms += elapsed_ms; + + start_sec = os::elapsedTime(); + non_young = false; + } + } else { + if (!cur->is_on_free_list()) { + double end_sec = os::elapsedTime(); + double elapsed_ms = (end_sec - start_sec) * 1000.0; + young_time_ms += elapsed_ms; + + start_sec = os::elapsedTime(); + non_young = true; + } + } + + rs_lengths += cur->rem_set()->occupied(); + + HeapRegion* next = cur->next_in_collection_set(); + assert(cur->in_collection_set(), "bad CS"); + cur->set_next_in_collection_set(NULL); + cur->set_in_collection_set(false); + + if (cur->is_young()) { + int index = cur->young_index_in_cset(); + guarantee( index != -1, "invariant" ); + guarantee( (size_t)index < policy->young_cset_length(), "invariant" ); + size_t words_survived = _surviving_young_words[index]; + cur->record_surv_words_in_group(words_survived); + } else { + int index = cur->young_index_in_cset(); + guarantee( index == -1, "invariant" ); + } + + assert( (cur->is_young() && cur->young_index_in_cset() > -1) || + (!cur->is_young() && cur->young_index_in_cset() == -1), + "invariant" ); + + if (!cur->evacuation_failed()) { + // And the region is empty. + assert(!cur->is_empty(), + "Should not have empty regions in a CS."); + free_region(cur); + } else { + guarantee( !cur->is_scan_only(), "should not be scan only" ); + cur->uninstall_surv_rate_group(); + if (cur->is_young()) + cur->set_young_index_in_cset(-1); + cur->set_not_young(); + cur->set_evacuation_failed(false); + } + cur = next; + } + + policy->record_max_rs_lengths(rs_lengths); + policy->cset_regions_freed(); + + double end_sec = os::elapsedTime(); + double elapsed_ms = (end_sec - start_sec) * 1000.0; + if (non_young) + non_young_time_ms += elapsed_ms; + else + young_time_ms += elapsed_ms; + + policy->record_young_free_cset_time_ms(young_time_ms); + policy->record_non_young_free_cset_time_ms(non_young_time_ms); +} + +HeapRegion* +G1CollectedHeap::alloc_region_from_unclean_list_locked(bool zero_filled) { + assert(ZF_mon->owned_by_self(), "Precondition"); + HeapRegion* res = pop_unclean_region_list_locked(); + if (res != NULL) { + assert(!res->continuesHumongous() && + res->zero_fill_state() != HeapRegion::Allocated, + "Only free regions on unclean list."); + if (zero_filled) { + res->ensure_zero_filled_locked(); + res->set_zero_fill_allocated(); + } + } + return res; +} + +HeapRegion* G1CollectedHeap::alloc_region_from_unclean_list(bool zero_filled) { + MutexLockerEx zx(ZF_mon, Mutex::_no_safepoint_check_flag); + return alloc_region_from_unclean_list_locked(zero_filled); +} + +void G1CollectedHeap::put_region_on_unclean_list(HeapRegion* r) { + MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); + put_region_on_unclean_list_locked(r); + if (should_zf()) ZF_mon->notify_all(); // Wake up ZF thread. +} + +void G1CollectedHeap::set_unclean_regions_coming(bool b) { + MutexLockerEx x(Cleanup_mon); + set_unclean_regions_coming_locked(b); +} + +void G1CollectedHeap::set_unclean_regions_coming_locked(bool b) { + assert(Cleanup_mon->owned_by_self(), "Precondition"); + _unclean_regions_coming = b; + // Wake up mutator threads that might be waiting for completeCleanup to + // finish. + if (!b) Cleanup_mon->notify_all(); +} + +void G1CollectedHeap::wait_for_cleanup_complete() { + MutexLockerEx x(Cleanup_mon); + wait_for_cleanup_complete_locked(); +} + +void G1CollectedHeap::wait_for_cleanup_complete_locked() { + assert(Cleanup_mon->owned_by_self(), "precondition"); + while (_unclean_regions_coming) { + Cleanup_mon->wait(); + } +} + +void +G1CollectedHeap::put_region_on_unclean_list_locked(HeapRegion* r) { + assert(ZF_mon->owned_by_self(), "precondition."); + _unclean_region_list.insert_before_head(r); +} + +void +G1CollectedHeap::prepend_region_list_on_unclean_list(UncleanRegionList* list) { + MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); + prepend_region_list_on_unclean_list_locked(list); + if (should_zf()) ZF_mon->notify_all(); // Wake up ZF thread. +} + +void +G1CollectedHeap:: +prepend_region_list_on_unclean_list_locked(UncleanRegionList* list) { + assert(ZF_mon->owned_by_self(), "precondition."); + _unclean_region_list.prepend_list(list); +} + +HeapRegion* G1CollectedHeap::pop_unclean_region_list_locked() { + assert(ZF_mon->owned_by_self(), "precondition."); + HeapRegion* res = _unclean_region_list.pop(); + if (res != NULL) { + // Inform ZF thread that there's a new unclean head. + if (_unclean_region_list.hd() != NULL && should_zf()) + ZF_mon->notify_all(); + } + return res; +} + +HeapRegion* G1CollectedHeap::peek_unclean_region_list_locked() { + assert(ZF_mon->owned_by_self(), "precondition."); + return _unclean_region_list.hd(); +} + + +bool G1CollectedHeap::move_cleaned_region_to_free_list_locked() { + assert(ZF_mon->owned_by_self(), "Precondition"); + HeapRegion* r = peek_unclean_region_list_locked(); + if (r != NULL && r->zero_fill_state() == HeapRegion::ZeroFilled) { + // Result of below must be equal to "r", since we hold the lock. + (void)pop_unclean_region_list_locked(); + put_free_region_on_list_locked(r); + return true; + } else { + return false; + } +} + +bool G1CollectedHeap::move_cleaned_region_to_free_list() { + MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); + return move_cleaned_region_to_free_list_locked(); +} + + +void G1CollectedHeap::put_free_region_on_list_locked(HeapRegion* r) { + assert(ZF_mon->owned_by_self(), "precondition."); + assert(_free_region_list_size == free_region_list_length(), "Inv"); + assert(r->zero_fill_state() == HeapRegion::ZeroFilled, + "Regions on free list must be zero filled"); + assert(!r->isHumongous(), "Must not be humongous."); + assert(r->is_empty(), "Better be empty"); + assert(!r->is_on_free_list(), + "Better not already be on free list"); + assert(!r->is_on_unclean_list(), + "Better not already be on unclean list"); + r->set_on_free_list(true); + r->set_next_on_free_list(_free_region_list); + _free_region_list = r; + _free_region_list_size++; + assert(_free_region_list_size == free_region_list_length(), "Inv"); +} + +void G1CollectedHeap::put_free_region_on_list(HeapRegion* r) { + MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); + put_free_region_on_list_locked(r); +} + +HeapRegion* G1CollectedHeap::pop_free_region_list_locked() { + assert(ZF_mon->owned_by_self(), "precondition."); + assert(_free_region_list_size == free_region_list_length(), "Inv"); + HeapRegion* res = _free_region_list; + if (res != NULL) { + _free_region_list = res->next_from_free_list(); + _free_region_list_size--; + res->set_on_free_list(false); + res->set_next_on_free_list(NULL); + assert(_free_region_list_size == free_region_list_length(), "Inv"); + } + return res; +} + + +HeapRegion* G1CollectedHeap::alloc_free_region_from_lists(bool zero_filled) { + // By self, or on behalf of self. + assert(Heap_lock->is_locked(), "Precondition"); + HeapRegion* res = NULL; + bool first = true; + while (res == NULL) { + if (zero_filled || !first) { + MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); + res = pop_free_region_list_locked(); + if (res != NULL) { + assert(!res->zero_fill_is_allocated(), + "No allocated regions on free list."); + res->set_zero_fill_allocated(); + } else if (!first) { + break; // We tried both, time to return NULL. + } + } + + if (res == NULL) { + res = alloc_region_from_unclean_list(zero_filled); + } + assert(res == NULL || + !zero_filled || + res->zero_fill_is_allocated(), + "We must have allocated the region we're returning"); + first = false; + } + return res; +} + +void G1CollectedHeap::remove_allocated_regions_from_lists() { + MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); + { + HeapRegion* prev = NULL; + HeapRegion* cur = _unclean_region_list.hd(); + while (cur != NULL) { + HeapRegion* next = cur->next_from_unclean_list(); + if (cur->zero_fill_is_allocated()) { + // Remove from the list. + if (prev == NULL) { + (void)_unclean_region_list.pop(); + } else { + _unclean_region_list.delete_after(prev); + } + cur->set_on_unclean_list(false); + cur->set_next_on_unclean_list(NULL); + } else { + prev = cur; + } + cur = next; + } + assert(_unclean_region_list.sz() == unclean_region_list_length(), + "Inv"); + } + + { + HeapRegion* prev = NULL; + HeapRegion* cur = _free_region_list; + while (cur != NULL) { + HeapRegion* next = cur->next_from_free_list(); + if (cur->zero_fill_is_allocated()) { + // Remove from the list. + if (prev == NULL) { + _free_region_list = cur->next_from_free_list(); + } else { + prev->set_next_on_free_list(cur->next_from_free_list()); + } + cur->set_on_free_list(false); + cur->set_next_on_free_list(NULL); + _free_region_list_size--; + } else { + prev = cur; + } + cur = next; + } + assert(_free_region_list_size == free_region_list_length(), "Inv"); + } +} + +bool G1CollectedHeap::verify_region_lists() { + MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); + return verify_region_lists_locked(); +} + +bool G1CollectedHeap::verify_region_lists_locked() { + HeapRegion* unclean = _unclean_region_list.hd(); + while (unclean != NULL) { + guarantee(unclean->is_on_unclean_list(), "Well, it is!"); + guarantee(!unclean->is_on_free_list(), "Well, it shouldn't be!"); + guarantee(unclean->zero_fill_state() != HeapRegion::Allocated, + "Everything else is possible."); + unclean = unclean->next_from_unclean_list(); + } + guarantee(_unclean_region_list.sz() == unclean_region_list_length(), "Inv"); + + HeapRegion* free_r = _free_region_list; + while (free_r != NULL) { + assert(free_r->is_on_free_list(), "Well, it is!"); + assert(!free_r->is_on_unclean_list(), "Well, it shouldn't be!"); + switch (free_r->zero_fill_state()) { + case HeapRegion::NotZeroFilled: + case HeapRegion::ZeroFilling: + guarantee(false, "Should not be on free list."); + break; + default: + // Everything else is possible. + break; + } + free_r = free_r->next_from_free_list(); + } + guarantee(_free_region_list_size == free_region_list_length(), "Inv"); + // If we didn't do an assertion... + return true; +} + +size_t G1CollectedHeap::free_region_list_length() { + assert(ZF_mon->owned_by_self(), "precondition."); + size_t len = 0; + HeapRegion* cur = _free_region_list; + while (cur != NULL) { + len++; + cur = cur->next_from_free_list(); + } + return len; +} + +size_t G1CollectedHeap::unclean_region_list_length() { + assert(ZF_mon->owned_by_self(), "precondition."); + return _unclean_region_list.length(); +} + +size_t G1CollectedHeap::n_regions() { + return _hrs->length(); +} + +size_t G1CollectedHeap::max_regions() { + return + (size_t)align_size_up(g1_reserved_obj_bytes(), HeapRegion::GrainBytes) / + HeapRegion::GrainBytes; +} + +size_t G1CollectedHeap::free_regions() { + /* Possibly-expensive assert. + assert(_free_regions == count_free_regions(), + "_free_regions is off."); + */ + return _free_regions; +} + +bool G1CollectedHeap::should_zf() { + return _free_region_list_size < (size_t) G1ConcZFMaxRegions; +} + +class RegionCounter: public HeapRegionClosure { + size_t _n; +public: + RegionCounter() : _n(0) {} + bool doHeapRegion(HeapRegion* r) { + if (r->is_empty() && !r->popular()) { + assert(!r->isHumongous(), "H regions should not be empty."); + _n++; + } + return false; + } + int res() { return (int) _n; } +}; + +size_t G1CollectedHeap::count_free_regions() { + RegionCounter rc; + heap_region_iterate(&rc); + size_t n = rc.res(); + if (_cur_alloc_region != NULL && _cur_alloc_region->is_empty()) + n--; + return n; +} + +size_t G1CollectedHeap::count_free_regions_list() { + size_t n = 0; + size_t o = 0; + ZF_mon->lock_without_safepoint_check(); + HeapRegion* cur = _free_region_list; + while (cur != NULL) { + cur = cur->next_from_free_list(); + n++; + } + size_t m = unclean_region_list_length(); + ZF_mon->unlock(); + return n + m; +} + +bool G1CollectedHeap::should_set_young_locked() { + assert(heap_lock_held_for_gc(), + "the heap lock should already be held by or for this thread"); + return (g1_policy()->in_young_gc_mode() && + g1_policy()->should_add_next_region_to_young_list()); +} + +void G1CollectedHeap::set_region_short_lived_locked(HeapRegion* hr) { + assert(heap_lock_held_for_gc(), + "the heap lock should already be held by or for this thread"); + _young_list->push_region(hr); + g1_policy()->set_region_short_lived(hr); +} + +class NoYoungRegionsClosure: public HeapRegionClosure { +private: + bool _success; +public: + NoYoungRegionsClosure() : _success(true) { } + bool doHeapRegion(HeapRegion* r) { + if (r->is_young()) { + gclog_or_tty->print_cr("Region ["PTR_FORMAT", "PTR_FORMAT") tagged as young", + r->bottom(), r->end()); + _success = false; + } + return false; + } + bool success() { return _success; } +}; + +bool G1CollectedHeap::check_young_list_empty(bool ignore_scan_only_list, + bool check_sample) { + bool ret = true; + + ret = _young_list->check_list_empty(ignore_scan_only_list, check_sample); + if (!ignore_scan_only_list) { + NoYoungRegionsClosure closure; + heap_region_iterate(&closure); + ret = ret && closure.success(); + } + + return ret; +} + +void G1CollectedHeap::empty_young_list() { + assert(heap_lock_held_for_gc(), + "the heap lock should already be held by or for this thread"); + assert(g1_policy()->in_young_gc_mode(), "should be in young GC mode"); + + _young_list->empty_list(); +} + +bool G1CollectedHeap::all_alloc_regions_no_allocs_since_save_marks() { + bool no_allocs = true; + for (int ap = 0; ap < GCAllocPurposeCount && no_allocs; ++ap) { + HeapRegion* r = _gc_alloc_regions[ap]; + no_allocs = r == NULL || r->saved_mark_at_top(); + } + return no_allocs; +} + +void G1CollectedHeap::all_alloc_regions_note_end_of_copying() { + for (int ap = 0; ap < GCAllocPurposeCount; ++ap) { + HeapRegion* r = _gc_alloc_regions[ap]; + if (r != NULL) { + // Check for aliases. + bool has_processed_alias = false; + for (int i = 0; i < ap; ++i) { + if (_gc_alloc_regions[i] == r) { + has_processed_alias = true; + break; + } + } + if (!has_processed_alias) { + r->note_end_of_copying(); + g1_policy()->record_after_bytes(r->used()); + } + } + } +} + + +// Done at the start of full GC. +void G1CollectedHeap::tear_down_region_lists() { + MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); + while (pop_unclean_region_list_locked() != NULL) ; + assert(_unclean_region_list.hd() == NULL && _unclean_region_list.sz() == 0, + "Postconditions of loop.") + while (pop_free_region_list_locked() != NULL) ; + assert(_free_region_list == NULL, "Postcondition of loop."); + if (_free_region_list_size != 0) { + gclog_or_tty->print_cr("Size is %d.", _free_region_list_size); + print(); + } + assert(_free_region_list_size == 0, "Postconditions of loop."); +} + + +class RegionResetter: public HeapRegionClosure { + G1CollectedHeap* _g1; + int _n; +public: + RegionResetter() : _g1(G1CollectedHeap::heap()), _n(0) {} + bool doHeapRegion(HeapRegion* r) { + if (r->continuesHumongous()) return false; + if (r->top() > r->bottom()) { + if (r->top() < r->end()) { + Copy::fill_to_words(r->top(), + pointer_delta(r->end(), r->top())); + } + r->set_zero_fill_allocated(); + } else { + assert(r->is_empty(), "tautology"); + if (r->popular()) { + if (r->zero_fill_state() != HeapRegion::Allocated) { + r->ensure_zero_filled_locked(); + r->set_zero_fill_allocated(); + } + } else { + _n++; + switch (r->zero_fill_state()) { + case HeapRegion::NotZeroFilled: + case HeapRegion::ZeroFilling: + _g1->put_region_on_unclean_list_locked(r); + break; + case HeapRegion::Allocated: + r->set_zero_fill_complete(); + // no break; go on to put on free list. + case HeapRegion::ZeroFilled: + _g1->put_free_region_on_list_locked(r); + break; + } + } + } + return false; + } + + int getFreeRegionCount() {return _n;} +}; + +// Done at the end of full GC. +void G1CollectedHeap::rebuild_region_lists() { + MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); + // This needs to go at the end of the full GC. + RegionResetter rs; + heap_region_iterate(&rs); + _free_regions = rs.getFreeRegionCount(); + // Tell the ZF thread it may have work to do. + if (should_zf()) ZF_mon->notify_all(); +} + +class UsedRegionsNeedZeroFillSetter: public HeapRegionClosure { + G1CollectedHeap* _g1; + int _n; +public: + UsedRegionsNeedZeroFillSetter() : _g1(G1CollectedHeap::heap()), _n(0) {} + bool doHeapRegion(HeapRegion* r) { + if (r->continuesHumongous()) return false; + if (r->top() > r->bottom()) { + // There are assertions in "set_zero_fill_needed()" below that + // require top() == bottom(), so this is technically illegal. + // We'll skirt the law here, by making that true temporarily. + DEBUG_ONLY(HeapWord* save_top = r->top(); + r->set_top(r->bottom())); + r->set_zero_fill_needed(); + DEBUG_ONLY(r->set_top(save_top)); + } + return false; + } +}; + +// Done at the start of full GC. +void G1CollectedHeap::set_used_regions_to_need_zero_fill() { + MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag); + // This needs to go at the end of the full GC. + UsedRegionsNeedZeroFillSetter rs; + heap_region_iterate(&rs); +} + +class CountObjClosure: public ObjectClosure { + size_t _n; +public: + CountObjClosure() : _n(0) {} + void do_object(oop obj) { _n++; } + size_t n() { return _n; } +}; + +size_t G1CollectedHeap::pop_object_used_objs() { + size_t sum_objs = 0; + for (int i = 0; i < G1NumPopularRegions; i++) { + CountObjClosure cl; + _hrs->at(i)->object_iterate(&cl); + sum_objs += cl.n(); + } + return sum_objs; +} + +size_t G1CollectedHeap::pop_object_used_bytes() { + size_t sum_bytes = 0; + for (int i = 0; i < G1NumPopularRegions; i++) { + sum_bytes += _hrs->at(i)->used(); + } + return sum_bytes; +} + + +static int nq = 0; + +HeapWord* G1CollectedHeap::allocate_popular_object(size_t word_size) { + while (_cur_pop_hr_index < G1NumPopularRegions) { + HeapRegion* cur_pop_region = _hrs->at(_cur_pop_hr_index); + HeapWord* res = cur_pop_region->allocate(word_size); + if (res != NULL) { + // We account for popular objs directly in the used summary: + _summary_bytes_used += (word_size * HeapWordSize); + return res; + } + // Otherwise, try the next region (first making sure that we remember + // the last "top" value as the "next_top_at_mark_start", so that + // objects made popular during markings aren't automatically considered + // live). + cur_pop_region->note_end_of_copying(); + // Otherwise, try the next region. + _cur_pop_hr_index++; + } + // XXX: For now !!! + vm_exit_out_of_memory(word_size, + "Not enough pop obj space (To Be Fixed)"); + return NULL; +} + +class HeapRegionList: public CHeapObj { + public: + HeapRegion* hr; + HeapRegionList* next; +}; + +void G1CollectedHeap::schedule_popular_region_evac(HeapRegion* r) { + // This might happen during parallel GC, so protect by this lock. + MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); + // We don't schedule regions whose evacuations are already pending, or + // are already being evacuated. + if (!r->popular_pending() && !r->in_collection_set()) { + r->set_popular_pending(true); + if (G1TracePopularity) { + gclog_or_tty->print_cr("Scheduling region "PTR_FORMAT" " + "["PTR_FORMAT", "PTR_FORMAT") for pop-object evacuation.", + r, r->bottom(), r->end()); + } + HeapRegionList* hrl = new HeapRegionList; + hrl->hr = r; + hrl->next = _popular_regions_to_be_evacuated; + _popular_regions_to_be_evacuated = hrl; + } +} + +HeapRegion* G1CollectedHeap::popular_region_to_evac() { + MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); + HeapRegion* res = NULL; + while (_popular_regions_to_be_evacuated != NULL && res == NULL) { + HeapRegionList* hrl = _popular_regions_to_be_evacuated; + _popular_regions_to_be_evacuated = hrl->next; + res = hrl->hr; + // The G1RSPopLimit may have increased, so recheck here... + if (res->rem_set()->occupied() < (size_t) G1RSPopLimit) { + // Hah: don't need to schedule. + if (G1TracePopularity) { + gclog_or_tty->print_cr("Unscheduling region "PTR_FORMAT" " + "["PTR_FORMAT", "PTR_FORMAT") " + "for pop-object evacuation (size %d < limit %d)", + res, res->bottom(), res->end(), + res->rem_set()->occupied(), G1RSPopLimit); + } + res->set_popular_pending(false); + res = NULL; + } + // We do not reset res->popular() here; if we did so, it would allow + // the region to be "rescheduled" for popularity evacuation. Instead, + // this is done in the collection pause, with the world stopped. + // So the invariant is that the regions in the list have the popularity + // boolean set, but having the boolean set does not imply membership + // on the list (though there can at most one such pop-pending region + // not on the list at any time). + delete hrl; + } + return res; +} + +void G1CollectedHeap::evac_popular_region(HeapRegion* hr) { + while (true) { + // Don't want to do a GC pause while cleanup is being completed! + wait_for_cleanup_complete(); + + // Read the GC count while holding the Heap_lock + int gc_count_before = SharedHeap::heap()->total_collections(); + g1_policy()->record_stop_world_start(); + + { + MutexUnlocker mu(Heap_lock); // give up heap lock, execute gets it back + VM_G1PopRegionCollectionPause op(gc_count_before, hr); + VMThread::execute(&op); + + // If the prolog succeeded, we didn't do a GC for this. + if (op.prologue_succeeded()) break; + } + // Otherwise we didn't. We should recheck the size, though, since + // the limit may have increased... + if (hr->rem_set()->occupied() < (size_t) G1RSPopLimit) { + hr->set_popular_pending(false); + break; + } + } +} + +void G1CollectedHeap::atomic_inc_obj_rc(oop obj) { + Atomic::inc(obj_rc_addr(obj)); +} + +class CountRCClosure: public OopsInHeapRegionClosure { + G1CollectedHeap* _g1h; + bool _parallel; +public: + CountRCClosure(G1CollectedHeap* g1h) : + _g1h(g1h), _parallel(ParallelGCThreads > 0) + {} + void do_oop(narrowOop* p) { + guarantee(false, "NYI"); + } + void do_oop(oop* p) { + oop obj = *p; + assert(obj != NULL, "Precondition."); + if (_parallel) { + // We go sticky at the limit to avoid excess contention. + // If we want to track the actual RC's further, we'll need to keep a + // per-thread hash table or something for the popular objects. + if (_g1h->obj_rc(obj) < G1ObjPopLimit) { + _g1h->atomic_inc_obj_rc(obj); + } + } else { + _g1h->inc_obj_rc(obj); + } + } +}; + +class EvacPopObjClosure: public ObjectClosure { + G1CollectedHeap* _g1h; + size_t _pop_objs; + size_t _max_rc; +public: + EvacPopObjClosure(G1CollectedHeap* g1h) : + _g1h(g1h), _pop_objs(0), _max_rc(0) {} + + void do_object(oop obj) { + size_t rc = _g1h->obj_rc(obj); + _max_rc = MAX2(rc, _max_rc); + if (rc >= (size_t) G1ObjPopLimit) { + _g1h->_pop_obj_rc_at_copy.add((double)rc); + size_t word_sz = obj->size(); + HeapWord* new_pop_loc = _g1h->allocate_popular_object(word_sz); + oop new_pop_obj = (oop)new_pop_loc; + Copy::aligned_disjoint_words((HeapWord*)obj, new_pop_loc, word_sz); + obj->forward_to(new_pop_obj); + G1ScanAndBalanceClosure scan_and_balance(_g1h); + new_pop_obj->oop_iterate_backwards(&scan_and_balance); + // preserve "next" mark bit if marking is in progress. + if (_g1h->mark_in_progress() && !_g1h->is_obj_ill(obj)) { + _g1h->concurrent_mark()->markAndGrayObjectIfNecessary(new_pop_obj); + } + + if (G1TracePopularity) { + gclog_or_tty->print_cr("Found obj " PTR_FORMAT " of word size " SIZE_FORMAT + " pop (%d), move to " PTR_FORMAT, + (void*) obj, word_sz, + _g1h->obj_rc(obj), (void*) new_pop_obj); + } + _pop_objs++; + } + } + size_t pop_objs() { return _pop_objs; } + size_t max_rc() { return _max_rc; } +}; + +class G1ParCountRCTask : public AbstractGangTask { + G1CollectedHeap* _g1h; + BitMap _bm; + + size_t getNCards() { + return (_g1h->capacity() + G1BlockOffsetSharedArray::N_bytes - 1) + / G1BlockOffsetSharedArray::N_bytes; + } + CountRCClosure _count_rc_closure; +public: + G1ParCountRCTask(G1CollectedHeap* g1h) : + AbstractGangTask("G1 Par RC Count task"), + _g1h(g1h), _bm(getNCards()), _count_rc_closure(g1h) + {} + + void work(int i) { + ResourceMark rm; + HandleMark hm; + _g1h->g1_rem_set()->oops_into_collection_set_do(&_count_rc_closure, i); + } +}; + +void G1CollectedHeap::popularity_pause_preamble(HeapRegion* popular_region) { + // We're evacuating a single region (for popularity). + if (G1TracePopularity) { + gclog_or_tty->print_cr("Doing pop region pause for ["PTR_FORMAT", "PTR_FORMAT")", + popular_region->bottom(), popular_region->end()); + } + g1_policy()->set_single_region_collection_set(popular_region); + size_t max_rc; + if (!compute_reference_counts_and_evac_popular(popular_region, + &max_rc)) { + // We didn't evacuate any popular objects. + // We increase the RS popularity limit, to prevent this from + // happening in the future. + if (G1RSPopLimit < (1 << 30)) { + G1RSPopLimit *= 2; + } + // For now, interesting enough for a message: +#if 1 + gclog_or_tty->print_cr("In pop region pause for ["PTR_FORMAT", "PTR_FORMAT"), " + "failed to find a pop object (max = %d).", + popular_region->bottom(), popular_region->end(), + max_rc); + gclog_or_tty->print_cr("Increased G1RSPopLimit to %d.", G1RSPopLimit); +#endif // 0 + // Also, we reset the collection set to NULL, to make the rest of + // the collection do nothing. + assert(popular_region->next_in_collection_set() == NULL, + "should be single-region."); + popular_region->set_in_collection_set(false); + popular_region->set_popular_pending(false); + g1_policy()->clear_collection_set(); + } +} + +bool G1CollectedHeap:: +compute_reference_counts_and_evac_popular(HeapRegion* popular_region, + size_t* max_rc) { + HeapWord* rc_region_bot; + HeapWord* rc_region_end; + + // Set up the reference count region. + HeapRegion* rc_region = newAllocRegion(HeapRegion::GrainWords); + if (rc_region != NULL) { + rc_region_bot = rc_region->bottom(); + rc_region_end = rc_region->end(); + } else { + rc_region_bot = NEW_C_HEAP_ARRAY(HeapWord, HeapRegion::GrainWords); + if (rc_region_bot == NULL) { + vm_exit_out_of_memory(HeapRegion::GrainWords, + "No space for RC region."); + } + rc_region_end = rc_region_bot + HeapRegion::GrainWords; + } + + if (G1TracePopularity) + gclog_or_tty->print_cr("RC region is ["PTR_FORMAT", "PTR_FORMAT")", + rc_region_bot, rc_region_end); + if (rc_region_bot > popular_region->bottom()) { + _rc_region_above = true; + _rc_region_diff = + pointer_delta(rc_region_bot, popular_region->bottom(), 1); + } else { + assert(rc_region_bot < popular_region->bottom(), "Can't be equal."); + _rc_region_above = false; + _rc_region_diff = + pointer_delta(popular_region->bottom(), rc_region_bot, 1); + } + g1_policy()->record_pop_compute_rc_start(); + // Count external references. + g1_rem_set()->prepare_for_oops_into_collection_set_do(); + if (ParallelGCThreads > 0) { + + set_par_threads(workers()->total_workers()); + G1ParCountRCTask par_count_rc_task(this); + workers()->run_task(&par_count_rc_task); + set_par_threads(0); + + } else { + CountRCClosure count_rc_closure(this); + g1_rem_set()->oops_into_collection_set_do(&count_rc_closure, 0); + } + g1_rem_set()->cleanup_after_oops_into_collection_set_do(); + g1_policy()->record_pop_compute_rc_end(); + + // Now evacuate popular objects. + g1_policy()->record_pop_evac_start(); + EvacPopObjClosure evac_pop_obj_cl(this); + popular_region->object_iterate(&evac_pop_obj_cl); + *max_rc = evac_pop_obj_cl.max_rc(); + + // Make sure the last "top" value of the current popular region is copied + // as the "next_top_at_mark_start", so that objects made popular during + // markings aren't automatically considered live. + HeapRegion* cur_pop_region = _hrs->at(_cur_pop_hr_index); + cur_pop_region->note_end_of_copying(); + + if (rc_region != NULL) { + free_region(rc_region); + } else { + FREE_C_HEAP_ARRAY(HeapWord, rc_region_bot); + } + g1_policy()->record_pop_evac_end(); + + return evac_pop_obj_cl.pop_objs() > 0; +} + +class CountPopObjInfoClosure: public HeapRegionClosure { + size_t _objs; + size_t _bytes; + + class CountObjClosure: public ObjectClosure { + int _n; + public: + CountObjClosure() : _n(0) {} + void do_object(oop obj) { _n++; } + size_t n() { return _n; } + }; + +public: + CountPopObjInfoClosure() : _objs(0), _bytes(0) {} + bool doHeapRegion(HeapRegion* r) { + _bytes += r->used(); + CountObjClosure blk; + r->object_iterate(&blk); + _objs += blk.n(); + return false; + } + size_t objs() { return _objs; } + size_t bytes() { return _bytes; } +}; + + +void G1CollectedHeap::print_popularity_summary_info() const { + CountPopObjInfoClosure blk; + for (int i = 0; i <= _cur_pop_hr_index; i++) { + blk.doHeapRegion(_hrs->at(i)); + } + gclog_or_tty->print_cr("\nPopular objects: %d objs, %d bytes.", + blk.objs(), blk.bytes()); + gclog_or_tty->print_cr(" RC at copy = [avg = %5.2f, max = %5.2f, sd = %5.2f].", + _pop_obj_rc_at_copy.avg(), + _pop_obj_rc_at_copy.maximum(), + _pop_obj_rc_at_copy.sd()); +} + +void G1CollectedHeap::set_refine_cte_cl_concurrency(bool concurrent) { + _refine_cte_cl->set_concurrent(concurrent); +} + +#ifndef PRODUCT + +class PrintHeapRegionClosure: public HeapRegionClosure { +public: + bool doHeapRegion(HeapRegion *r) { + gclog_or_tty->print("Region: "PTR_FORMAT":", r); + if (r != NULL) { + if (r->is_on_free_list()) + gclog_or_tty->print("Free "); + if (r->is_young()) + gclog_or_tty->print("Young "); + if (r->isHumongous()) + gclog_or_tty->print("Is Humongous "); + r->print(); + } + return false; + } +}; + +class SortHeapRegionClosure : public HeapRegionClosure { + size_t young_regions,free_regions, unclean_regions; + size_t hum_regions, count; + size_t unaccounted, cur_unclean, cur_alloc; + size_t total_free; + HeapRegion* cur; +public: + SortHeapRegionClosure(HeapRegion *_cur) : cur(_cur), young_regions(0), + free_regions(0), unclean_regions(0), + hum_regions(0), + count(0), unaccounted(0), + cur_alloc(0), total_free(0) + {} + bool doHeapRegion(HeapRegion *r) { + count++; + if (r->is_on_free_list()) free_regions++; + else if (r->is_on_unclean_list()) unclean_regions++; + else if (r->isHumongous()) hum_regions++; + else if (r->is_young()) young_regions++; + else if (r == cur) cur_alloc++; + else unaccounted++; + return false; + } + void print() { + total_free = free_regions + unclean_regions; + gclog_or_tty->print("%d regions\n", count); + gclog_or_tty->print("%d free: free_list = %d unclean = %d\n", + total_free, free_regions, unclean_regions); + gclog_or_tty->print("%d humongous %d young\n", + hum_regions, young_regions); + gclog_or_tty->print("%d cur_alloc\n", cur_alloc); + gclog_or_tty->print("UHOH unaccounted = %d\n", unaccounted); + } +}; + +void G1CollectedHeap::print_region_counts() { + SortHeapRegionClosure sc(_cur_alloc_region); + PrintHeapRegionClosure cl; + heap_region_iterate(&cl); + heap_region_iterate(&sc); + sc.print(); + print_region_accounting_info(); +}; + +bool G1CollectedHeap::regions_accounted_for() { + // TODO: regions accounting for young/survivor/tenured + return true; +} + +bool G1CollectedHeap::print_region_accounting_info() { + gclog_or_tty->print_cr("P regions: %d.", G1NumPopularRegions); + gclog_or_tty->print_cr("Free regions: %d (count: %d count list %d) (clean: %d unclean: %d).", + free_regions(), + count_free_regions(), count_free_regions_list(), + _free_region_list_size, _unclean_region_list.sz()); + gclog_or_tty->print_cr("cur_alloc: %d.", + (_cur_alloc_region == NULL ? 0 : 1)); + gclog_or_tty->print_cr("H regions: %d.", _num_humongous_regions); + + // TODO: check regions accounting for young/survivor/tenured + return true; +} + +bool G1CollectedHeap::is_in_closed_subset(const void* p) const { + HeapRegion* hr = heap_region_containing(p); + if (hr == NULL) { + return is_in_permanent(p); + } else { + return hr->is_in(p); + } +} +#endif // PRODUCT + +void G1CollectedHeap::g1_unimplemented() { + // Unimplemented(); +} + + +// Local Variables: *** +// c-indentation-style: gnu *** +// End: ***