truffle: src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp comparison

comparison src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp @ 845:df6caf649ff7

6700789: G1: Enable use of compressed oops with G1 heaps Summary: Modifications to G1 so as to allow the use of compressed oops. Reviewed-by: apetrusenko, coleenp, jmasa, kvn, never, phh, tonyp

author	ysr
date	Tue, 14 Jul 2009 15:40:39 -0700
parents	0316eac49d5a
children	42d84bbbecf4

comparison

equal deleted inserted replaced

-:bb18957ad21e
+:df6caf649ff7
 // Computes the sum of the storage used by the various regions.
 size_t G1CollectedHeap::used() const {
-assert(Heap_lock->owner() != NULL,
+// Temporarily, until 6859911 is fixed. XXX
-"Should be owned on this thread's behalf.");
+// assert(Heap_lock->owner() != NULL,
+//        "Should be owned on this thread's behalf.");
 size_t result = _summary_bytes_used;
-if (_cur_alloc_region != NULL)
+// Read only once in case it is set to NULL concurrently
-result += _cur_alloc_region->used();
+HeapRegion* hr = _cur_alloc_region;
+if (hr != NULL)
+result += hr->used();
 return result;
 }
 class SumUsedClosure: public HeapRegionClosure {
 size_t _used;
 G1CollectedHeap* g1h;
 public:
 VerifyLivenessOopClosure(G1CollectedHeap* _g1h) {
 g1h = _g1h;
 }
-void do_oop(narrowOop *p) {
+void do_oop(narrowOop *p) { do_oop_work(p); }
-guarantee(false, "NYI");
+void do_oop(      oop *p) { do_oop_work(p); }
-}
-void do_oop(oop *p) {
+template <class T> void do_oop_work(T *p) {
-oop obj = *p;
+oop obj = oopDesc::load_decode_heap_oop(p);
-assert(obj == NULL || !g1h->is_obj_dead(obj),
+guarantee(obj == NULL || !g1h->is_obj_dead(obj),
 "Dead object referenced by a not dead object");
 }
 };
 class VerifyObjsInRegionClosure: public ObjectClosure {
 private:
 bool _use_prev_marking;
 public:
 // use_prev_marking == true  -> use "prev" marking information,
 // use_prev_marking == false -> use "next" marking information
 VerifyRegionClosure(bool allow_dirty, bool par, bool use_prev_marking)
-: _allow_dirty(allow_dirty), _par(par),
+: _allow_dirty(allow_dirty),
+_par(par),
 _use_prev_marking(use_prev_marking) {}
 bool doHeapRegion(HeapRegion* r) {
 guarantee(_par || r->claim_value() == HeapRegion::InitialClaimValue,
 "Should be unclaimed at verify points.");
 if (!r->continuesHumongous()) {
 VerifyObjsInRegionClosure not_dead_yet_cl(r, _use_prev_marking);
 bool             _use_prev_marking;
 public:
 // use_prev_marking == true  -> use "prev" marking information,
 // use_prev_marking == false -> use "next" marking information
 VerifyRootsClosure(bool use_prev_marking) :
-_g1h(G1CollectedHeap::heap()), _failures(false),
+_g1h(G1CollectedHeap::heap()),
+_failures(false),
 _use_prev_marking(use_prev_marking) { }
 bool failures() { return _failures; }
-void do_oop(narrowOop* p) {
+template <class T> void do_oop_nv(T* p) {
-guarantee(false, "NYI");
+T heap_oop = oopDesc::load_heap_oop(p);
-}
+if (!oopDesc::is_null(heap_oop)) {
+oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
-void do_oop(oop* p) {
-oop obj = *p;
-if (obj != NULL) {
 if (_g1h->is_obj_dead_cond(obj, _use_prev_marking)) {
 gclog_or_tty->print_cr("Root location "PTR_FORMAT" "
 "points to dead obj "PTR_FORMAT, p, (void*) obj);
 obj->print_on(gclog_or_tty);
 _failures = true;
 }
 }
 }
+void do_oop(oop* p)       { do_oop_nv(p); }
+void do_oop(narrowOop* p) { do_oop_nv(p); }
 };
 // This is the task used for parallel heap verification.
 class G1ParVerifyTask: public AbstractGangTask {
 // use_prev_marking == true  -> use "prev" marking information,
 // use_prev_marking == false -> use "next" marking information
 G1ParVerifyTask(G1CollectedHeap* g1h, bool allow_dirty,
 bool use_prev_marking) :
 AbstractGangTask("Parallel verify task"),
-_g1h(g1h), _allow_dirty(allow_dirty),
+_g1h(g1h),
+_allow_dirty(allow_dirty),
 _use_prev_marking(use_prev_marking) { }
 void work(int worker_i) {
 HandleMark hm;
 VerifyRegionClosure blk(_allow_dirty, true, _use_prev_marking);
 }
 }
 void
 G1CollectedHeap::doConcurrentMark() {
-if (G1ConcMark) {
+MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
-MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
+if (!_cmThread->in_progress()) {
-if (!_cmThread->in_progress()) {
+_cmThread->set_started();
-_cmThread->set_started();
+CGC_lock->notify();
-CGC_lock->notify();
-}
 }
 }
 class VerifyMarkedObjsClosure: public ObjectClosure {
 G1CollectedHeap* _g1h;
 if (_surviving_young_words == NULL) {
 vm_exit_out_of_memory(sizeof(size_t) * array_length,
 "Not enough space for young surv words summary.");
 }
 memset(_surviving_young_words, 0, array_length * sizeof(size_t));
+#ifdef ASSERT
 for (size_t i = 0;  i < array_length; ++i) {
-guarantee( _surviving_young_words[i] == 0, "invariant" );
+assert( _surviving_young_words[i] == 0, "memset above" );
 }
+#endif // !ASSERT
 }
 void
 G1CollectedHeap::update_surviving_young_words(size_t* surv_young_words) {
 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
 }
 COMPILER2_PRESENT(DerivedPointerTable::clear());
 // We want to turn off ref discovery, if necessary, and turn it back on
-// on again later if we do.
+// on again later if we do. XXX Dubious: why is discovery disabled?
 bool was_enabled = ref_processor()->discovery_enabled();
 if (was_enabled) ref_processor()->disable_discovery();
 // Forget the current alloc region (we might even choose it to be part
 // of the collection set!).
 // The elapsed time induced by the start time below deliberately elides
 // the possible verification above.
 double start_time_sec = os::elapsedTime();
 GCOverheadReporter::recordSTWStart(start_time_sec);
 size_t start_used_bytes = used();
-if (!G1ConcMark) {
-do_sync_mark();
-}
 g1_policy()->record_collection_pause_start(start_time_sec,
 start_used_bytes);
 guarantee(_in_cset_fast_test == NULL, "invariant");
 if (g1_policy()->in_young_gc_mode() &&
 g1_policy()->should_initiate_conc_mark()) {
 concurrent_mark()->checkpointRootsInitialPost();
 set_marking_started();
+// CAUTION: after the doConcurrentMark() call below,
+// the concurrent marking thread(s) could be running
+// concurrently with us. Make sure that anything after
+// this point does not assume that we are the only GC thread
+// running. Note: of course, the actual marking work will
+// not start until the safepoint itself is released in
+// ConcurrentGCThread::safepoint_desynchronize().
 doConcurrentMark();
 }
 #if SCAN_ONLY_VERBOSE
 _young_list->print();
 class G1KeepAliveClosure: public OopClosure {
 G1CollectedHeap* _g1;
 public:
 G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
-void do_oop(narrowOop* p) {
+void do_oop(narrowOop* p) { guarantee(false, "Not needed"); }
-guarantee(false, "NYI");
+void do_oop(      oop* p) {
-}
-void do_oop(oop* p) {
 oop obj = *p;
 #ifdef G1_DEBUG
 if (PrintGC && Verbose) {
 gclog_or_tty->print_cr("keep alive *"PTR_FORMAT" = "PTR_FORMAT" "PTR_FORMAT,
 p, (void*) obj, (void*) *p);
 #endif // G1_DEBUG
 if (_g1->obj_in_cs(obj)) {
 assert( obj->is_forwarded(), "invariant" );
 *p = obj->forwardee();
 #ifdef G1_DEBUG
 gclog_or_tty->print_cr("     in CSet: moved "PTR_FORMAT" -> "PTR_FORMAT,
 (void*) obj, (void*) *p);
 #endif // G1_DEBUG
 }
 G1RemSet* _g1_rem_set;
 public:
 UpdateRSetImmediate(G1CollectedHeap* g1) :
 _g1(g1), _g1_rem_set(g1->g1_rem_set()) {}
-void do_oop(narrowOop* p) {
+virtual void do_oop(narrowOop* p) { do_oop_work(p); }
-guarantee(false, "NYI");
+virtual void do_oop(      oop* p) { do_oop_work(p); }
-}
+template <class T> void do_oop_work(T* p) {
-void do_oop(oop* p) {
 assert(_from->is_in_reserved(p), "paranoia");
-if (*p != NULL && !_from->is_survivor()) {
+T heap_oop = oopDesc::load_heap_oop(p);
+if (!oopDesc::is_null(heap_oop) && !_from->is_survivor()) {
 _g1_rem_set->par_write_ref(_from, p, 0);
 }
 }
 };
 public:
 UpdateRSetDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) :
 _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) {}
-void do_oop(narrowOop* p) {
+virtual void do_oop(narrowOop* p) { do_oop_work(p); }
-guarantee(false, "NYI");
+virtual void do_oop(      oop* p) { do_oop_work(p); }
-}
+template <class T> void do_oop_work(T* p) {
-void do_oop(oop* p) {
 assert(_from->is_in_reserved(p), "paranoia");
-if (!_from->is_in_reserved(*p) && !_from->is_survivor()) {
+if (!_from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) &&
+!_from->is_survivor()) {
 size_t card_index = _ct_bs->index_for(p);
 if (_ct_bs->mark_card_deferred(card_index)) {
 _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index));
 }
 }
 block = r->par_allocate(free_words);
 } while (block == NULL);
 fill_with_object(block, free_words);
 }
-#define use_local_bitmaps         1
-#define verify_local_bitmaps      0
 #ifndef PRODUCT
-class GCLabBitMap;
-class GCLabBitMapClosure: public BitMapClosure {
-private:
-ConcurrentMark* _cm;
-GCLabBitMap*    _bitmap;
-public:
-GCLabBitMapClosure(ConcurrentMark* cm,
-GCLabBitMap* bitmap) {
-_cm     = cm;
-_bitmap = bitmap;
-}
-virtual bool do_bit(size_t offset);
-};
-#endif // PRODUCT
-#define oop_buffer_length 256
-class GCLabBitMap: public BitMap {
-private:
-ConcurrentMark* _cm;
-int       _shifter;
-size_t    _bitmap_word_covers_words;
-// beginning of the heap
-HeapWord* _heap_start;
-// this is the actual start of the GCLab
-HeapWord* _real_start_word;
-// this is the actual end of the GCLab
-HeapWord* _real_end_word;
-// this is the first word, possibly located before the actual start
-// of the GCLab, that corresponds to the first bit of the bitmap
-HeapWord* _start_word;
-// size of a GCLab in words
-size_t _gclab_word_size;
-static int shifter() {
-return MinObjAlignment - 1;
-}
-// how many heap words does a single bitmap word corresponds to?
-static size_t bitmap_word_covers_words() {
-return BitsPerWord << shifter();
-}
-static size_t gclab_word_size() {
-return G1ParallelGCAllocBufferSize / HeapWordSize;
-}
-static size_t bitmap_size_in_bits() {
-size_t bits_in_bitmap = gclab_word_size() >> shifter();
-// We are going to ensure that the beginning of a word in this
-// bitmap also corresponds to the beginning of a word in the
-// global marking bitmap. To handle the case where a GCLab
-// starts from the middle of the bitmap, we need to add enough
-// space (i.e. up to a bitmap word) to ensure that we have
-// enough bits in the bitmap.
-return bits_in_bitmap + BitsPerWord - 1;
-}
-public:
-GCLabBitMap(HeapWord* heap_start)
-: BitMap(bitmap_size_in_bits()),
-_cm(G1CollectedHeap::heap()->concurrent_mark()),
-_shifter(shifter()),
-_bitmap_word_covers_words(bitmap_word_covers_words()),
-_heap_start(heap_start),
-_gclab_word_size(gclab_word_size()),
-_real_start_word(NULL),
-_real_end_word(NULL),
-_start_word(NULL)
-{
-guarantee( size_in_words() >= bitmap_size_in_words(),
-"just making sure");
-}
-inline unsigned heapWordToOffset(HeapWord* addr) {
-unsigned offset = (unsigned) pointer_delta(addr, _start_word) >> _shifter;
-assert(offset < size(), "offset should be within bounds");
-return offset;
-}
-inline HeapWord* offsetToHeapWord(size_t offset) {
-HeapWord* addr =  _start_word + (offset << _shifter);
-assert(_real_start_word <= addr && addr < _real_end_word, "invariant");
-return addr;
-}
-bool fields_well_formed() {
-bool ret1 = (_real_start_word == NULL) &&
-(_real_end_word == NULL) &&
-(_start_word == NULL);
-if (ret1)
-return true;
-bool ret2 = _real_start_word >= _start_word &&
-_start_word < _real_end_word &&
-(_real_start_word + _gclab_word_size) == _real_end_word &&
-(_start_word + _gclab_word_size + _bitmap_word_covers_words)
-> _real_end_word;
-return ret2;
-}
-inline bool mark(HeapWord* addr) {
-guarantee(use_local_bitmaps, "invariant");
-assert(fields_well_formed(), "invariant");
-if (addr >= _real_start_word && addr < _real_end_word) {
-assert(!isMarked(addr), "should not have already been marked");
-// first mark it on the bitmap
-at_put(heapWordToOffset(addr), true);
-return true;
-} else {
-return false;
-}
-}
-inline bool isMarked(HeapWord* addr) {
-guarantee(use_local_bitmaps, "invariant");
-assert(fields_well_formed(), "invariant");
-return at(heapWordToOffset(addr));
-}
-void set_buffer(HeapWord* start) {
-guarantee(use_local_bitmaps, "invariant");
-clear();
-assert(start != NULL, "invariant");
-_real_start_word = start;
-_real_end_word   = start + _gclab_word_size;
-size_t diff =
-pointer_delta(start, _heap_start) % _bitmap_word_covers_words;
-_start_word = start - diff;
-assert(fields_well_formed(), "invariant");
-}
-#ifndef PRODUCT
-void verify() {
-// verify that the marks have been propagated
-GCLabBitMapClosure cl(_cm, this);
-iterate(&cl);
-}
-#endif // PRODUCT
-void retire() {
-guarantee(use_local_bitmaps, "invariant");
-assert(fields_well_formed(), "invariant");
-if (_start_word != NULL) {
-CMBitMap*       mark_bitmap = _cm->nextMarkBitMap();
-// this means that the bitmap was set up for the GCLab
-assert(_real_start_word != NULL && _real_end_word != NULL, "invariant");
-mark_bitmap->mostly_disjoint_range_union(this,
-0, // always start from the start of the bitmap
-_start_word,
-size_in_words());
-_cm->grayRegionIfNecessary(MemRegion(_real_start_word, _real_end_word));
-#ifndef PRODUCT
-if (use_local_bitmaps && verify_local_bitmaps)
-verify();
-#endif // PRODUCT
-} else {
-assert(_real_start_word == NULL && _real_end_word == NULL, "invariant");
-}
-}
-static size_t bitmap_size_in_words() {
-return (bitmap_size_in_bits() + BitsPerWord - 1) / BitsPerWord;
-}
-};
-#ifndef PRODUCT
 bool GCLabBitMapClosure::do_bit(size_t offset) {
 HeapWord* addr = _bitmap->offsetToHeapWord(offset);
 guarantee(_cm->isMarked(oop(addr)), "it should be!");
 return true;
 }
 #endif // PRODUCT
-class G1ParGCAllocBuffer: public ParGCAllocBuffer {
+G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num)
-private:
+: _g1h(g1h),
-bool        _retired;
+_refs(g1h->task_queue(queue_num)),
-bool        _during_marking;
+_dcq(&g1h->dirty_card_queue_set()),
-GCLabBitMap _bitmap;
+_ct_bs((CardTableModRefBS*)_g1h->barrier_set()),
+_g1_rem(g1h->g1_rem_set()),
-public:
+_hash_seed(17), _queue_num(queue_num),
-G1ParGCAllocBuffer() :
+_term_attempts(0),
-ParGCAllocBuffer(G1ParallelGCAllocBufferSize / HeapWordSize),
+_age_table(false),
-_during_marking(G1CollectedHeap::heap()->mark_in_progress()),
-_bitmap(G1CollectedHeap::heap()->reserved_region().start()),
-_retired(false)
-{ }
-inline bool mark(HeapWord* addr) {
-guarantee(use_local_bitmaps, "invariant");
-assert(_during_marking, "invariant");
-return _bitmap.mark(addr);
-}
-inline void set_buf(HeapWord* buf) {
-if (use_local_bitmaps && _during_marking)
-_bitmap.set_buffer(buf);
-ParGCAllocBuffer::set_buf(buf);
-_retired = false;
-}
-inline void retire(bool end_of_gc, bool retain) {
-if (_retired)
-return;
-if (use_local_bitmaps && _during_marking) {
-_bitmap.retire();
-}
-ParGCAllocBuffer::retire(end_of_gc, retain);
-_retired = true;
-}
-};
-class G1ParScanThreadState : public StackObj {
-protected:
-G1CollectedHeap* _g1h;
-RefToScanQueue*  _refs;
-DirtyCardQueue   _dcq;
-CardTableModRefBS* _ct_bs;
-G1RemSet* _g1_rem;
-typedef GrowableArray<oop*> OverflowQueue;
-OverflowQueue* _overflowed_refs;
-G1ParGCAllocBuffer _alloc_buffers[GCAllocPurposeCount];
-ageTable           _age_table;
-size_t           _alloc_buffer_waste;
-size_t           _undo_waste;
-OopsInHeapRegionClosure*      _evac_failure_cl;
-G1ParScanHeapEvacClosure*     _evac_cl;
-G1ParScanPartialArrayClosure* _partial_scan_cl;
-int _hash_seed;
-int _queue_num;
-int _term_attempts;
 #if G1_DETAILED_STATS
-int _pushes, _pops, _steals, _steal_attempts;
+_pushes(0), _pops(0), _steals(0),
-int _overflow_pushes;
+_steal_attempts(0),  _overflow_pushes(0),
 #endif
+_strong_roots_time(0), _term_time(0),
-double _start;
+_alloc_buffer_waste(0), _undo_waste(0)
-double _start_strong_roots;
+{
-double _strong_roots_time;
+// we allocate G1YoungSurvRateNumRegions plus one entries, since
-double _start_term;
+// we "sacrifice" entry 0 to keep track of surviving bytes for
-double _term_time;
+// non-young regions (where the age is -1)
+// We also add a few elements at the beginning and at the end in
-// Map from young-age-index (0 == not young, 1 is youngest) to
+// an attempt to eliminate cache contention
-// surviving words. base is what we get back from the malloc call
+size_t real_length = 1 + _g1h->g1_policy()->young_cset_length();
-size_t* _surviving_young_words_base;
+size_t array_length = PADDING_ELEM_NUM +
-// this points into the array, as we use the first few entries for padding
+real_length +
-size_t* _surviving_young_words;
+PADDING_ELEM_NUM;
+_surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length);
-#define PADDING_ELEM_NUM (64 / sizeof(size_t))
+if (_surviving_young_words_base == NULL)
+vm_exit_out_of_memory(array_length * sizeof(size_t),
-void   add_to_alloc_buffer_waste(size_t waste) { _alloc_buffer_waste += waste; }
+"Not enough space for young surv histo.");
+_surviving_young_words = _surviving_young_words_base + PADDING_ELEM_NUM;
-void   add_to_undo_waste(size_t waste)         { _undo_waste += waste; }
+memset(_surviving_young_words, 0, real_length * sizeof(size_t));
-DirtyCardQueue& dirty_card_queue()             { return _dcq;  }
+_overflowed_refs = new OverflowQueue(10);
-CardTableModRefBS* ctbs()                      { return _ct_bs; }
+_start = os::elapsedTime();
-void immediate_rs_update(HeapRegion* from, oop* p, int tid) {
+}
-if (!from->is_survivor()) {
-_g1_rem->par_write_ref(from, p, tid);
-}
-}
-void deferred_rs_update(HeapRegion* from, oop* p, int tid) {
-// If the new value of the field points to the same region or
-// is the to-space, we don't need to include it in the Rset updates.
-if (!from->is_in_reserved(*p) && !from->is_survivor()) {
-size_t card_index = ctbs()->index_for(p);
-// If the card hasn't been added to the buffer, do it.
-if (ctbs()->mark_card_deferred(card_index)) {
-dirty_card_queue().enqueue((jbyte*)ctbs()->byte_for_index(card_index));
-}
-}
-}
-public:
-G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num)
-: _g1h(g1h),
-_refs(g1h->task_queue(queue_num)),
-_dcq(&g1h->dirty_card_queue_set()),
-_ct_bs((CardTableModRefBS*)_g1h->barrier_set()),
-_g1_rem(g1h->g1_rem_set()),
-_hash_seed(17), _queue_num(queue_num),
-_term_attempts(0),
-_age_table(false),
-#if G1_DETAILED_STATS
-_pushes(0), _pops(0), _steals(0),
-_steal_attempts(0),  _overflow_pushes(0),
-#endif
-_strong_roots_time(0), _term_time(0),
-_alloc_buffer_waste(0), _undo_waste(0)
-{
-// we allocate G1YoungSurvRateNumRegions plus one entries, since
-// we "sacrifice" entry 0 to keep track of surviving bytes for
-// non-young regions (where the age is -1)
-// We also add a few elements at the beginning and at the end in
-// an attempt to eliminate cache contention
-size_t real_length = 1 + _g1h->g1_policy()->young_cset_length();
-size_t array_length = PADDING_ELEM_NUM +
-real_length +
-PADDING_ELEM_NUM;
-_surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length);
-if (_surviving_young_words_base == NULL)
-vm_exit_out_of_memory(array_length * sizeof(size_t),
-"Not enough space for young surv histo.");
-_surviving_young_words = _surviving_young_words_base + PADDING_ELEM_NUM;
-memset(_surviving_young_words, 0, real_length * sizeof(size_t));
-_overflowed_refs = new OverflowQueue(10);
-_start = os::elapsedTime();
-}
-~G1ParScanThreadState() {
-FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base);
-}
-RefToScanQueue*   refs()            { return _refs;             }
-OverflowQueue*    overflowed_refs() { return _overflowed_refs;  }
-ageTable*         age_table()       { return &_age_table;       }
-G1ParGCAllocBuffer* alloc_buffer(GCAllocPurpose purpose) {
-return &_alloc_buffers[purpose];
-}
-size_t alloc_buffer_waste()                    { return _alloc_buffer_waste; }
-size_t undo_waste()                            { return _undo_waste; }
-void push_on_queue(oop* ref) {
-assert(ref != NULL, "invariant");
-assert(has_partial_array_mask(ref) || _g1h->obj_in_cs(*ref), "invariant");
-if (!refs()->push(ref)) {
-overflowed_refs()->push(ref);
-IF_G1_DETAILED_STATS(note_overflow_push());
-} else {
-IF_G1_DETAILED_STATS(note_push());
-}
-}
-void pop_from_queue(oop*& ref) {
-if (!refs()->pop_local(ref)) {
-ref = NULL;
-} else {
-assert(ref != NULL, "invariant");
-assert(has_partial_array_mask(ref) || _g1h->obj_in_cs(*ref),
-"invariant");
-IF_G1_DETAILED_STATS(note_pop());
-}
-}
-void pop_from_overflow_queue(oop*& ref) {
-ref = overflowed_refs()->pop();
-}
-int refs_to_scan()                             { return refs()->size();                 }
-int overflowed_refs_to_scan()                  { return overflowed_refs()->length();    }
-void update_rs(HeapRegion* from, oop* p, int tid) {
-if (G1DeferredRSUpdate) {
-deferred_rs_update(from, p, tid);
-} else {
-immediate_rs_update(from, p, tid);
-}
-}
-HeapWord* allocate_slow(GCAllocPurpose purpose, size_t word_sz) {
-HeapWord* obj = NULL;
-if (word_sz * 100 <
-(size_t)(G1ParallelGCAllocBufferSize / HeapWordSize) *
-ParallelGCBufferWastePct) {
-G1ParGCAllocBuffer* alloc_buf = alloc_buffer(purpose);
-add_to_alloc_buffer_waste(alloc_buf->words_remaining());
-alloc_buf->retire(false, false);
-HeapWord* buf =
-_g1h->par_allocate_during_gc(purpose, G1ParallelGCAllocBufferSize / HeapWordSize);
-if (buf == NULL) return NULL; // Let caller handle allocation failure.
-// Otherwise.
-alloc_buf->set_buf(buf);
-obj = alloc_buf->allocate(word_sz);
-assert(obj != NULL, "buffer was definitely big enough...");
-} else {
-obj = _g1h->par_allocate_during_gc(purpose, word_sz);
-}
-return obj;
-}
-HeapWord* allocate(GCAllocPurpose purpose, size_t word_sz) {
-HeapWord* obj = alloc_buffer(purpose)->allocate(word_sz);
-if (obj != NULL) return obj;
-return allocate_slow(purpose, word_sz);
-}
-void undo_allocation(GCAllocPurpose purpose, HeapWord* obj, size_t word_sz) {
-if (alloc_buffer(purpose)->contains(obj)) {
-guarantee(alloc_buffer(purpose)->contains(obj + word_sz - 1),
-"should contain whole object");
-alloc_buffer(purpose)->undo_allocation(obj, word_sz);
-} else {
-CollectedHeap::fill_with_object(obj, word_sz);
-add_to_undo_waste(word_sz);
-}
-}
-void set_evac_failure_closure(OopsInHeapRegionClosure* evac_failure_cl) {
-_evac_failure_cl = evac_failure_cl;
-}
-OopsInHeapRegionClosure* evac_failure_closure() {
-return _evac_failure_cl;
-}
-void set_evac_closure(G1ParScanHeapEvacClosure* evac_cl) {
-_evac_cl = evac_cl;
-}
-void set_partial_scan_closure(G1ParScanPartialArrayClosure* partial_scan_cl) {
-_partial_scan_cl = partial_scan_cl;
-}
-int* hash_seed() { return &_hash_seed; }
-int  queue_num() { return _queue_num; }
-int term_attempts()   { return _term_attempts; }
-void note_term_attempt()  { _term_attempts++; }
-#if G1_DETAILED_STATS
-int pushes()          { return _pushes; }
-int pops()            { return _pops; }
-int steals()          { return _steals; }
-int steal_attempts()  { return _steal_attempts; }
-int overflow_pushes() { return _overflow_pushes; }
-void note_push()          { _pushes++; }
-void note_pop()           { _pops++; }
-void note_steal()         { _steals++; }
-void note_steal_attempt() { _steal_attempts++; }
-void note_overflow_push() { _overflow_pushes++; }
-#endif
-void start_strong_roots() {
-_start_strong_roots = os::elapsedTime();
-}
-void end_strong_roots() {
-_strong_roots_time += (os::elapsedTime() - _start_strong_roots);
-}
-double strong_roots_time() { return _strong_roots_time; }
-void start_term_time() {
-note_term_attempt();
-_start_term = os::elapsedTime();
-}
-void end_term_time() {
-_term_time += (os::elapsedTime() - _start_term);
-}
-double term_time() { return _term_time; }
-double elapsed() {
-return os::elapsedTime() - _start;
-}
-size_t* surviving_young_words() {
-// We add on to hide entry 0 which accumulates surviving words for
-// age -1 regions (i.e. non-young ones)
-return _surviving_young_words;
-}
-void retire_alloc_buffers() {
-for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
-size_t waste = _alloc_buffers[ap].words_remaining();
-add_to_alloc_buffer_waste(waste);
-_alloc_buffers[ap].retire(true, false);
-}
-}
-private:
-void deal_with_reference(oop* ref_to_scan) {
-if (has_partial_array_mask(ref_to_scan)) {
-_partial_scan_cl->do_oop_nv(ref_to_scan);
-} else {
-// Note: we can use "raw" versions of "region_containing" because
-// "obj_to_scan" is definitely in the heap, and is not in a
-// humongous region.
-HeapRegion* r = _g1h->heap_region_containing_raw(ref_to_scan);
-_evac_cl->set_region(r);
-_evac_cl->do_oop_nv(ref_to_scan);
-}
-}
-public:
-void trim_queue() {
-// I've replicated the loop twice, first to drain the overflow
-// queue, second to drain the task queue. This is better than
-// having a single loop, which checks both conditions and, inside
-// it, either pops the overflow queue or the task queue, as each
-// loop is tighter. Also, the decision to drain the overflow queue
-// first is not arbitrary, as the overflow queue is not visible
-// to the other workers, whereas the task queue is. So, we want to
-// drain the "invisible" entries first, while allowing the other
-// workers to potentially steal the "visible" entries.
-while (refs_to_scan() > 0 || overflowed_refs_to_scan() > 0) {
-while (overflowed_refs_to_scan() > 0) {
-oop *ref_to_scan = NULL;
-pop_from_overflow_queue(ref_to_scan);
-assert(ref_to_scan != NULL, "invariant");
-// We shouldn't have pushed it on the queue if it was not
-// pointing into the CSet.
-assert(ref_to_scan != NULL, "sanity");
-assert(has_partial_array_mask(ref_to_scan) ||
-_g1h->obj_in_cs(*ref_to_scan), "sanity");
-deal_with_reference(ref_to_scan);
-}
-while (refs_to_scan() > 0) {
-oop *ref_to_scan = NULL;
-pop_from_queue(ref_to_scan);
-if (ref_to_scan != NULL) {
-// We shouldn't have pushed it on the queue if it was not
-// pointing into the CSet.
-assert(has_partial_array_mask(ref_to_scan) ||
-_g1h->obj_in_cs(*ref_to_scan), "sanity");
-deal_with_reference(ref_to_scan);
-}
-}
-}
-}
-};
 G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
 _g1(g1), _g1_rem(_g1->g1_rem_set()), _cm(_g1->concurrent_mark()),
 _par_scan_state(par_scan_state) { }
-// This closure is applied to the fields of the objects that have just been copied.
+template <class T> void G1ParCopyHelper::mark_forwardee(T* p) {
-// Should probably be made inline and moved in g1OopClosures.inline.hpp.
-void G1ParScanClosure::do_oop_nv(oop* p) {
-oop obj = *p;
-if (obj != NULL) {
-if (_g1->in_cset_fast_test(obj)) {
-// We're not going to even bother checking whether the object is
-// already forwarded or not, as this usually causes an immediate
-// stall. We'll try to prefetch the object (for write, given that
-// we might need to install the forwarding reference) and we'll
-// get back to it when pop it from the queue
-Prefetch::write(obj->mark_addr(), 0);
-Prefetch::read(obj->mark_addr(), (HeapWordSize*2));
-// slightly paranoid test; I'm trying to catch potential
-// problems before we go into push_on_queue to know where the
-// problem is coming from
-assert(obj == *p, "the value of *p should not have changed");
-_par_scan_state->push_on_queue(p);
-} else {
-_par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
-}
-}
-}
-void G1ParCopyHelper::mark_forwardee(oop* p) {
 // This is called _after_ do_oop_work has been called, hence after
 // the object has been relocated to its new location and *p points
 // to its new location.
-oop thisOop = *p;
+T heap_oop = oopDesc::load_heap_oop(p);
-if (thisOop != NULL) {
+if (!oopDesc::is_null(heap_oop)) {
-assert((_g1->evacuation_failed()) || (!_g1->obj_in_cs(thisOop)),
+oop obj = oopDesc::decode_heap_oop(heap_oop);
+assert((_g1->evacuation_failed()) || (!_g1->obj_in_cs(obj)),
 "shouldn't still be in the CSet if evacuation didn't fail.");
-HeapWord* addr = (HeapWord*)thisOop;
+HeapWord* addr = (HeapWord*)obj;
 if (_g1->is_in_g1_reserved(addr))
 _cm->grayRoot(oop(addr));
 }
 }
 size_t* surv_young_words = _par_scan_state->surviving_young_words();
 surv_young_words[young_index] += word_sz;
 if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) {
 arrayOop(old)->set_length(0);
-_par_scan_state->push_on_queue(set_partial_array_mask(old));
+oop* old_p = set_partial_array_mask(old);
+_par_scan_state->push_on_queue(old_p);
 } else {
 // No point in using the slower heap_region_containing() method,
 // given that we know obj is in the heap.
 _scanner->set_region(_g1->heap_region_containing_raw(obj));
 obj->oop_iterate_backwards(_scanner);
 obj = forward_ptr;
 }
 return obj;
 }
-template<bool do_gen_barrier, G1Barrier barrier,
+template <bool do_gen_barrier, G1Barrier barrier, bool do_mark_forwardee, bool skip_cset_test>
-bool do_mark_forwardee, bool skip_cset_test>
+template <class T>
-void G1ParCopyClosure<do_gen_barrier, barrier,
+void G1ParCopyClosure <do_gen_barrier, barrier, do_mark_forwardee, skip_cset_test>
-do_mark_forwardee, skip_cset_test>::do_oop_work(oop* p) {
+::do_oop_work(T* p) {
-oop obj = *p;
+oop obj = oopDesc::load_decode_heap_oop(p);
 assert(barrier != G1BarrierRS || obj != NULL,
 "Precondition: G1BarrierRS implies obj is nonNull");
 // The only time we skip the cset test is when we're scanning
 // references popped from the queue. And we only push on the queue
 #if G1_REM_SET_LOGGING
 gclog_or_tty->print_cr("Loc "PTR_FORMAT" contains pointer "PTR_FORMAT" "
 "into CS.", p, (void*) obj);
 #endif
 if (obj->is_forwarded()) {
-*p = obj->forwardee();
+oopDesc::encode_store_heap_oop(p, obj->forwardee());
 } else {
-*p = copy_to_survivor_space(obj);
+oop copy_oop = copy_to_survivor_space(obj);
+oopDesc::encode_store_heap_oop(p, copy_oop);
 }
 // When scanning the RS, we only care about objs in CS.
 if (barrier == G1BarrierRS) {
 _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
 }
 par_do_barrier(p);
 }
 }
 template void G1ParCopyClosure<false, G1BarrierEvac, false, true>::do_oop_work(oop* p);
+template void G1ParCopyClosure<false, G1BarrierEvac, false, true>::do_oop_work(narrowOop* p);
-template<class T> void G1ParScanPartialArrayClosure::process_array_chunk(
-oop obj, int start, int end) {
+template <class T> void G1ParScanPartialArrayClosure::do_oop_nv(T* p) {
-// process our set of indices (include header in first chunk)
-assert(start < end, "invariant");
-T* const base      = (T*)objArrayOop(obj)->base();
-T* const start_addr = (start == 0) ? (T*) obj : base + start;
-T* const end_addr   = base + end;
-MemRegion mr((HeapWord*)start_addr, (HeapWord*)end_addr);
-_scanner.set_region(_g1->heap_region_containing(obj));
-obj->oop_iterate(&_scanner, mr);
-}
-void G1ParScanPartialArrayClosure::do_oop_nv(oop* p) {
-assert(!UseCompressedOops, "Needs to be fixed to work with compressed oops");
 assert(has_partial_array_mask(p), "invariant");
 oop old = clear_partial_array_mask(p);
 assert(old->is_objArray(), "must be obj array");
 assert(old->is_forwarded(), "must be forwarded");
 assert(Universe::heap()->is_in_reserved(old), "must be in heap.");
 if (remainder > 2 * ParGCArrayScanChunk) {
 // Test above combines last partial chunk with a full chunk
 end = start + ParGCArrayScanChunk;
 arrayOop(old)->set_length(end);
 // Push remainder.
-_par_scan_state->push_on_queue(set_partial_array_mask(old));
+oop* old_p = set_partial_array_mask(old);
+assert(arrayOop(old)->length() < obj->length(), "Empty push?");
+_par_scan_state->push_on_queue(old_p);
 } else {
 // Restore length so that the heap remains parsable in
 // case of evacuation failure.
 arrayOop(old)->set_length(end);
 }
+_scanner.set_region(_g1->heap_region_containing_raw(obj));
 // process our set of indices (include header in first chunk)
-process_array_chunk<oop>(obj, start, end);
+obj->oop_iterate_range(&_scanner, start, end);
 }
-int G1ScanAndBalanceClosure::_nq = 0;
 class G1ParEvacuateFollowersClosure : public VoidClosure {
 protected:
 G1CollectedHeap*              _g1h;
 G1ParScanThreadState*         _par_scan_state;
 _queues(queues), _terminator(terminator) {}
 void do_void() {
 G1ParScanThreadState* pss = par_scan_state();
 while (true) {
-oop* ref_to_scan;
 pss->trim_queue();
 IF_G1_DETAILED_STATS(pss->note_steal_attempt());
-if (queues()->steal(pss->queue_num(),
-pss->hash_seed(),
+StarTask stolen_task;
-ref_to_scan)) {
+if (queues()->steal(pss->queue_num(), pss->hash_seed(), stolen_task)) {
 IF_G1_DETAILED_STATS(pss->note_steal());
 // slightly paranoid tests; I'm trying to catch potential
 // problems before we go into push_on_queue to know where the
 // problem is coming from
-assert(ref_to_scan != NULL, "invariant");
+assert((oop*)stolen_task != NULL, "Error");
-assert(has_partial_array_mask(ref_to_scan) ||
+if (stolen_task.is_narrow()) {
-_g1h->obj_in_cs(*ref_to_scan), "invariant");
+assert(UseCompressedOops, "Error");
-pss->push_on_queue(ref_to_scan);
+narrowOop* p = (narrowOop*) stolen_task;
+assert(has_partial_array_mask(p) ||
+_g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "Error");
+pss->push_on_queue(p);
+} else {
+oop* p = (oop*) stolen_task;
+assert(has_partial_array_mask(p) || _g1h->obj_in_cs(*p), "Error");
+pss->push_on_queue(p);
+}
 continue;
 }
 pss->start_term_time();
 if (terminator()->offer_termination()) break;
 pss->end_term_time();
 class G1ParTask : public AbstractGangTask {
 protected:
 G1CollectedHeap*       _g1h;
 RefToScanQueueSet      *_queues;
 ParallelTaskTerminator _terminator;
+int _n_workers;
 Mutex _stats_lock;
 Mutex* stats_lock() { return &_stats_lock; }
 size_t getNCards() {
 G1ParTask(G1CollectedHeap* g1h, int workers, RefToScanQueueSet *task_queues)
 : AbstractGangTask("G1 collection"),
 _g1h(g1h),
 _queues(task_queues),
 _terminator(workers, _queues),
-_stats_lock(Mutex::leaf, "parallel G1 stats lock", true)
+_stats_lock(Mutex::leaf, "parallel G1 stats lock", true),
+_n_workers(workers)
 {}
 RefToScanQueueSet* queues() { return _queues; }
 RefToScanQueue *work_queue(int i) {
 return queues()->queue(i);
 }
 void work(int i) {
+if (i >= _n_workers) return;  // no work needed this round
 ResourceMark rm;
 HandleMark   hm;
 G1ParScanThreadState            pss(_g1h, i);
 G1ParScanHeapEvacClosure        scan_evac_cl(_g1h, &pss);
 assert(pss.overflowed_refs_to_scan() == 0, "Overflow queue should be empty");
 }
 };
 // *** Common G1 Evacuation Stuff
-class G1CountClosure: public OopsInHeapRegionClosure {
-public:
-int n;
-G1CountClosure() : n(0) {}
-void do_oop(narrowOop* p) {
-guarantee(false, "NYI");
-}
-void do_oop(oop* p) {
-oop obj = *p;
-assert(obj != NULL && G1CollectedHeap::heap()->obj_in_cs(obj),
-"Rem set closure called on non-rem-set pointer.");
-n++;
-}
-};
-static G1CountClosure count_closure;
 void
 G1CollectedHeap::
 g1_process_strong_roots(bool collecting_perm_gen,
 SharedHeap::ScanningOption so,
 #endif // PRODUCT
 void G1CollectedHeap::g1_unimplemented() {
 // Unimplemented();
 }
-// Local Variables: ***
-// c-indentation-style: gnu ***
-// End: ***

Mercurial > hg > truffle

comparison src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp @ 845:df6caf649ff7