Mercurial > hg > truffle
comparison src/share/vm/gc_implementation/parallelScavenge/psParallelCompact.cpp @ 482:7c2386d67889
6765745: par compact - allow young gen spaces to be split
Reviewed-by: jmasa
author | jcoomes |
---|---|
date | Thu, 11 Dec 2008 12:05:14 -0800 |
parents | 7d7a7c599c17 |
children | 0f773163217d |
comparison
equal
deleted
inserted
replaced
481:7d7a7c599c17 | 482:7c2386d67889 |
---|---|
86 GrowableArray<HeapWord*>* PSParallelCompact::_last_gc_live_oops = NULL; | 86 GrowableArray<HeapWord*>* PSParallelCompact::_last_gc_live_oops = NULL; |
87 GrowableArray<HeapWord*>* PSParallelCompact::_last_gc_live_oops_moved_to = NULL; | 87 GrowableArray<HeapWord*>* PSParallelCompact::_last_gc_live_oops_moved_to = NULL; |
88 GrowableArray<size_t> * PSParallelCompact::_last_gc_live_oops_size = NULL; | 88 GrowableArray<size_t> * PSParallelCompact::_last_gc_live_oops_size = NULL; |
89 #endif | 89 #endif |
90 | 90 |
91 void SplitInfo::record(size_t src_region_idx, size_t partial_obj_size, | |
92 HeapWord* destination) | |
93 { | |
94 assert(src_region_idx != 0, "invalid src_region_idx"); | |
95 assert(partial_obj_size != 0, "invalid partial_obj_size argument"); | |
96 assert(destination != NULL, "invalid destination argument"); | |
97 | |
98 _src_region_idx = src_region_idx; | |
99 _partial_obj_size = partial_obj_size; | |
100 _destination = destination; | |
101 | |
102 // These fields may not be updated below, so make sure they're clear. | |
103 assert(_dest_region_addr == NULL, "should have been cleared"); | |
104 assert(_first_src_addr == NULL, "should have been cleared"); | |
105 | |
106 // Determine the number of destination regions for the partial object. | |
107 HeapWord* const last_word = destination + partial_obj_size - 1; | |
108 const ParallelCompactData& sd = PSParallelCompact::summary_data(); | |
109 HeapWord* const beg_region_addr = sd.region_align_down(destination); | |
110 HeapWord* const end_region_addr = sd.region_align_down(last_word); | |
111 | |
112 if (beg_region_addr == end_region_addr) { | |
113 // One destination region. | |
114 _destination_count = 1; | |
115 if (end_region_addr == destination) { | |
116 // The destination falls on a region boundary, thus the first word of the | |
117 // partial object will be the first word copied to the destination region. | |
118 _dest_region_addr = end_region_addr; | |
119 _first_src_addr = sd.region_to_addr(src_region_idx); | |
120 } | |
121 } else { | |
122 // Two destination regions. When copied, the partial object will cross a | |
123 // destination region boundary, so a word somewhere within the partial | |
124 // object will be the first word copied to the second destination region. | |
125 _destination_count = 2; | |
126 _dest_region_addr = end_region_addr; | |
127 const size_t ofs = pointer_delta(end_region_addr, destination); | |
128 assert(ofs < _partial_obj_size, "sanity"); | |
129 _first_src_addr = sd.region_to_addr(src_region_idx) + ofs; | |
130 } | |
131 } | |
132 | |
133 void SplitInfo::clear() | |
134 { | |
135 _src_region_idx = 0; | |
136 _partial_obj_size = 0; | |
137 _destination = NULL; | |
138 _destination_count = 0; | |
139 _dest_region_addr = NULL; | |
140 _first_src_addr = NULL; | |
141 assert(!is_valid(), "sanity"); | |
142 } | |
143 | |
144 #ifdef ASSERT | |
145 void SplitInfo::verify_clear() | |
146 { | |
147 assert(_src_region_idx == 0, "not clear"); | |
148 assert(_partial_obj_size == 0, "not clear"); | |
149 assert(_destination == NULL, "not clear"); | |
150 assert(_destination_count == 0, "not clear"); | |
151 assert(_dest_region_addr == NULL, "not clear"); | |
152 assert(_first_src_addr == NULL, "not clear"); | |
153 } | |
154 #endif // #ifdef ASSERT | |
155 | |
156 | |
91 #ifndef PRODUCT | 157 #ifndef PRODUCT |
92 const char* PSParallelCompact::space_names[] = { | 158 const char* PSParallelCompact::space_names[] = { |
93 "perm", "old ", "eden", "from", "to " | 159 "perm", "old ", "eden", "from", "to " |
94 }; | 160 }; |
95 | 161 |
414 ++cur_region; | 480 ++cur_region; |
415 addr += RegionSize; | 481 addr += RegionSize; |
416 } | 482 } |
417 } | 483 } |
418 | 484 |
419 bool ParallelCompactData::summarize(HeapWord* target_beg, HeapWord* target_end, | 485 // Find the point at which a space can be split and, if necessary, record the |
486 // split point. | |
487 // | |
488 // If the current src region (which overflowed the destination space) doesn't | |
489 // have a partial object, the split point is at the beginning of the current src | |
490 // region (an "easy" split, no extra bookkeeping required). | |
491 // | |
492 // If the current src region has a partial object, the split point is in the | |
493 // region where that partial object starts (call it the split_region). If | |
494 // split_region has a partial object, then the split point is just after that | |
495 // partial object (a "hard" split where we have to record the split data and | |
496 // zero the partial_obj_size field). With a "hard" split, we know that the | |
497 // partial_obj ends within split_region because the partial object that caused | |
498 // the overflow starts in split_region. If split_region doesn't have a partial | |
499 // obj, then the split is at the beginning of split_region (another "easy" | |
500 // split). | |
501 HeapWord* | |
502 ParallelCompactData::summarize_split_space(size_t src_region, | |
503 SplitInfo& split_info, | |
504 HeapWord* destination, | |
505 HeapWord* target_end, | |
506 HeapWord** target_next) | |
507 { | |
508 assert(destination <= target_end, "sanity"); | |
509 assert(destination + _region_data[src_region].data_size() > target_end, | |
510 "region should not fit into target space"); | |
511 | |
512 size_t split_region = src_region; | |
513 HeapWord* split_destination = destination; | |
514 size_t partial_obj_size = _region_data[src_region].partial_obj_size(); | |
515 | |
516 if (destination + partial_obj_size > target_end) { | |
517 // The split point is just after the partial object (if any) in the | |
518 // src_region that contains the start of the object that overflowed the | |
519 // destination space. | |
520 // | |
521 // Find the start of the "overflow" object and set split_region to the | |
522 // region containing it. | |
523 HeapWord* const overflow_obj = _region_data[src_region].partial_obj_addr(); | |
524 split_region = addr_to_region_idx(overflow_obj); | |
525 | |
526 // Clear the source_region field of all destination regions whose first word | |
527 // came from data after the split point (a non-null source_region field | |
528 // implies a region must be filled). | |
529 // | |
530 // An alternative to the simple loop below: clear during post_compact(), | |
531 // which uses memcpy instead of individual stores, and is easy to | |
532 // parallelize. (The downside is that it clears the entire RegionData | |
533 // object as opposed to just one field.) | |
534 // | |
535 // post_compact() would have to clear the summary data up to the highest | |
536 // address that was written during the summary phase, which would be | |
537 // | |
538 // max(top, max(new_top, clear_top)) | |
539 // | |
540 // where clear_top is a new field in SpaceInfo. Would have to set clear_top | |
541 // to destination + partial_obj_size, where both have the values passed to | |
542 // this routine. | |
543 const RegionData* const sr = region(split_region); | |
544 const size_t beg_idx = | |
545 addr_to_region_idx(region_align_up(sr->destination() + | |
546 sr->partial_obj_size())); | |
547 const size_t end_idx = | |
548 addr_to_region_idx(region_align_up(destination + partial_obj_size)); | |
549 | |
550 if (TraceParallelOldGCSummaryPhase) { | |
551 gclog_or_tty->print_cr("split: clearing source_region field in [" | |
552 SIZE_FORMAT ", " SIZE_FORMAT ")", | |
553 beg_idx, end_idx); | |
554 } | |
555 for (size_t idx = beg_idx; idx < end_idx; ++idx) { | |
556 _region_data[idx].set_source_region(0); | |
557 } | |
558 | |
559 // Set split_destination and partial_obj_size to reflect the split region. | |
560 split_destination = sr->destination(); | |
561 partial_obj_size = sr->partial_obj_size(); | |
562 } | |
563 | |
564 // The split is recorded only if a partial object extends onto the region. | |
565 if (partial_obj_size != 0) { | |
566 _region_data[split_region].set_partial_obj_size(0); | |
567 split_info.record(split_region, partial_obj_size, split_destination); | |
568 } | |
569 | |
570 // Setup the continuation addresses. | |
571 *target_next = split_destination + partial_obj_size; | |
572 HeapWord* const source_next = region_to_addr(split_region) + partial_obj_size; | |
573 | |
574 if (TraceParallelOldGCSummaryPhase) { | |
575 const char * split_type = partial_obj_size == 0 ? "easy" : "hard"; | |
576 gclog_or_tty->print_cr("%s split: src=" PTR_FORMAT " src_c=" SIZE_FORMAT | |
577 " pos=" SIZE_FORMAT, | |
578 split_type, source_next, split_region, | |
579 partial_obj_size); | |
580 gclog_or_tty->print_cr("%s split: dst=" PTR_FORMAT " dst_c=" SIZE_FORMAT | |
581 " tn=" PTR_FORMAT, | |
582 split_type, split_destination, | |
583 addr_to_region_idx(split_destination), | |
584 *target_next); | |
585 | |
586 if (partial_obj_size != 0) { | |
587 HeapWord* const po_beg = split_info.destination(); | |
588 HeapWord* const po_end = po_beg + split_info.partial_obj_size(); | |
589 gclog_or_tty->print_cr("%s split: " | |
590 "po_beg=" PTR_FORMAT " " SIZE_FORMAT " " | |
591 "po_end=" PTR_FORMAT " " SIZE_FORMAT, | |
592 split_type, | |
593 po_beg, addr_to_region_idx(po_beg), | |
594 po_end, addr_to_region_idx(po_end)); | |
595 } | |
596 } | |
597 | |
598 return source_next; | |
599 } | |
600 | |
601 bool ParallelCompactData::summarize(SplitInfo& split_info, | |
420 HeapWord* source_beg, HeapWord* source_end, | 602 HeapWord* source_beg, HeapWord* source_end, |
421 HeapWord** target_next, | 603 HeapWord** source_next, |
422 HeapWord** source_next) { | 604 HeapWord* target_beg, HeapWord* target_end, |
423 // This is too strict. | 605 HeapWord** target_next) |
424 // assert(region_offset(source_beg) == 0, "not RegionSize aligned"); | 606 { |
425 | |
426 if (TraceParallelOldGCSummaryPhase) { | 607 if (TraceParallelOldGCSummaryPhase) { |
427 tty->print_cr("tb=" PTR_FORMAT " te=" PTR_FORMAT " " | 608 HeapWord* const source_next_val = source_next == NULL ? NULL : *source_next; |
428 "sb=" PTR_FORMAT " se=" PTR_FORMAT " " | 609 tty->print_cr("sb=" PTR_FORMAT " se=" PTR_FORMAT " sn=" PTR_FORMAT |
429 "tn=" PTR_FORMAT " sn=" PTR_FORMAT, | 610 "tb=" PTR_FORMAT " te=" PTR_FORMAT " tn=" PTR_FORMAT, |
430 target_beg, target_end, | 611 source_beg, source_end, source_next_val, |
431 source_beg, source_end, | 612 target_beg, target_end, *target_next); |
432 target_next != 0 ? *target_next : (HeapWord*) 0, | |
433 source_next != 0 ? *source_next : (HeapWord*) 0); | |
434 } | 613 } |
435 | 614 |
436 size_t cur_region = addr_to_region_idx(source_beg); | 615 size_t cur_region = addr_to_region_idx(source_beg); |
437 const size_t end_region = addr_to_region_idx(region_align_up(source_end)); | 616 const size_t end_region = addr_to_region_idx(region_align_up(source_end)); |
438 | 617 |
439 HeapWord *dest_addr = target_beg; | 618 HeapWord *dest_addr = target_beg; |
440 while (cur_region < end_region) { | 619 while (cur_region < end_region) { |
620 // The destination must be set even if the region has no data. | |
621 _region_data[cur_region].set_destination(dest_addr); | |
622 | |
441 size_t words = _region_data[cur_region].data_size(); | 623 size_t words = _region_data[cur_region].data_size(); |
442 | |
443 #if 1 | |
444 assert(pointer_delta(target_end, dest_addr) >= words, | |
445 "source region does not fit into target region"); | |
446 #else | |
447 // XXX - need some work on the corner cases here. If the region does not | |
448 // fit, then must either make sure any partial_obj from the region fits, or | |
449 // "undo" the initial part of the partial_obj that is in the previous | |
450 // region. | |
451 if (dest_addr + words >= target_end) { | |
452 // Let the caller know where to continue. | |
453 *target_next = dest_addr; | |
454 *source_next = region_to_addr(cur_region); | |
455 return false; | |
456 } | |
457 #endif // #if 1 | |
458 | |
459 _region_data[cur_region].set_destination(dest_addr); | |
460 | |
461 // Set the destination_count for cur_region, and if necessary, update | |
462 // source_region for a destination region. The source_region field is | |
463 // updated if cur_region is the first (left-most) region to be copied to a | |
464 // destination region. | |
465 // | |
466 // The destination_count calculation is a bit subtle. A region that has | |
467 // data that compacts into itself does not count itself as a destination. | |
468 // This maintains the invariant that a zero count means the region is | |
469 // available and can be claimed and then filled. | |
470 if (words > 0) { | 624 if (words > 0) { |
625 // If cur_region does not fit entirely into the target space, find a point | |
626 // at which the source space can be 'split' so that part is copied to the | |
627 // target space and the rest is copied elsewhere. | |
628 if (dest_addr + words > target_end) { | |
629 assert(source_next != NULL, "source_next is NULL when splitting"); | |
630 *source_next = summarize_split_space(cur_region, split_info, dest_addr, | |
631 target_end, target_next); | |
632 return false; | |
633 } | |
634 | |
635 // Compute the destination_count for cur_region, and if necessary, update | |
636 // source_region for a destination region. The source_region field is | |
637 // updated if cur_region is the first (left-most) region to be copied to a | |
638 // destination region. | |
639 // | |
640 // The destination_count calculation is a bit subtle. A region that has | |
641 // data that compacts into itself does not count itself as a destination. | |
642 // This maintains the invariant that a zero count means the region is | |
643 // available and can be claimed and then filled. | |
644 uint destination_count = 0; | |
645 if (split_info.is_split(cur_region)) { | |
646 // The current region has been split: the partial object will be copied | |
647 // to one destination space and the remaining data will be copied to | |
648 // another destination space. Adjust the initial destination_count and, | |
649 // if necessary, set the source_region field if the partial object will | |
650 // cross a destination region boundary. | |
651 destination_count = split_info.destination_count(); | |
652 if (destination_count == 2) { | |
653 size_t dest_idx = addr_to_region_idx(split_info.dest_region_addr()); | |
654 _region_data[dest_idx].set_source_region(cur_region); | |
655 } | |
656 } | |
657 | |
471 HeapWord* const last_addr = dest_addr + words - 1; | 658 HeapWord* const last_addr = dest_addr + words - 1; |
472 const size_t dest_region_1 = addr_to_region_idx(dest_addr); | 659 const size_t dest_region_1 = addr_to_region_idx(dest_addr); |
473 const size_t dest_region_2 = addr_to_region_idx(last_addr); | 660 const size_t dest_region_2 = addr_to_region_idx(last_addr); |
474 #if 0 | 661 |
475 // Initially assume that the destination regions will be the same and | 662 // Initially assume that the destination regions will be the same and |
476 // adjust the value below if necessary. Under this assumption, if | 663 // adjust the value below if necessary. Under this assumption, if |
477 // cur_region == dest_region_2, then cur_region will be compacted | 664 // cur_region == dest_region_2, then cur_region will be compacted |
478 // completely into itself. | 665 // completely into itself. |
479 uint destination_count = cur_region == dest_region_2 ? 0 : 1; | 666 destination_count += cur_region == dest_region_2 ? 0 : 1; |
480 if (dest_region_1 != dest_region_2) { | 667 if (dest_region_1 != dest_region_2) { |
481 // Destination regions differ; adjust destination_count. | 668 // Destination regions differ; adjust destination_count. |
482 destination_count += 1; | 669 destination_count += 1; |
483 // Data from cur_region will be copied to the start of dest_region_2. | 670 // Data from cur_region will be copied to the start of dest_region_2. |
484 _region_data[dest_region_2].set_source_region(cur_region); | 671 _region_data[dest_region_2].set_source_region(cur_region); |
485 } else if (region_offset(dest_addr) == 0) { | 672 } else if (region_offset(dest_addr) == 0) { |
486 // Data from cur_region will be copied to the start of the destination | 673 // Data from cur_region will be copied to the start of the destination |
487 // region. | 674 // region. |
488 _region_data[dest_region_1].set_source_region(cur_region); | 675 _region_data[dest_region_1].set_source_region(cur_region); |
489 } | 676 } |
490 #else | |
491 // Initially assume that the destination regions will be different and | |
492 // adjust the value below if necessary. Under this assumption, if | |
493 // cur_region == dest_region2, then cur_region will be compacted partially | |
494 // into dest_region_1 and partially into itself. | |
495 uint destination_count = cur_region == dest_region_2 ? 1 : 2; | |
496 if (dest_region_1 != dest_region_2) { | |
497 // Data from cur_region will be copied to the start of dest_region_2. | |
498 _region_data[dest_region_2].set_source_region(cur_region); | |
499 } else { | |
500 // Destination regions are the same; adjust destination_count. | |
501 destination_count -= 1; | |
502 if (region_offset(dest_addr) == 0) { | |
503 // Data from cur_region will be copied to the start of the destination | |
504 // region. | |
505 _region_data[dest_region_1].set_source_region(cur_region); | |
506 } | |
507 } | |
508 #endif // #if 0 | |
509 | 677 |
510 _region_data[cur_region].set_destination_count(destination_count); | 678 _region_data[cur_region].set_destination_count(destination_count); |
511 _region_data[cur_region].set_data_location(region_to_addr(cur_region)); | 679 _region_data[cur_region].set_data_location(region_to_addr(cur_region)); |
512 dest_addr += words; | 680 dest_addr += words; |
513 } | 681 } |
747 | 915 |
748 const size_t beg_region = _summary_data.addr_to_region_idx(bot); | 916 const size_t beg_region = _summary_data.addr_to_region_idx(bot); |
749 const size_t end_region = | 917 const size_t end_region = |
750 _summary_data.addr_to_region_idx(_summary_data.region_align_up(max_top)); | 918 _summary_data.addr_to_region_idx(_summary_data.region_align_up(max_top)); |
751 _summary_data.clear_range(beg_region, end_region); | 919 _summary_data.clear_range(beg_region, end_region); |
920 | |
921 // Clear the data used to 'split' regions. | |
922 SplitInfo& split_info = _space_info[id].split_info(); | |
923 if (split_info.is_valid()) { | |
924 split_info.clear(); | |
925 } | |
926 DEBUG_ONLY(split_info.verify_clear();) | |
752 } | 927 } |
753 | 928 |
754 void PSParallelCompact::pre_compact(PreGCValues* pre_gc_values) | 929 void PSParallelCompact::pre_compact(PreGCValues* pre_gc_values) |
755 { | 930 { |
756 // Update the from & to space pointers in space_info, since they are swapped | 931 // Update the from & to space pointers in space_info, since they are swapped |
805 | 980 |
806 void PSParallelCompact::post_compact() | 981 void PSParallelCompact::post_compact() |
807 { | 982 { |
808 TraceTime tm("post compact", print_phases(), true, gclog_or_tty); | 983 TraceTime tm("post compact", print_phases(), true, gclog_or_tty); |
809 | 984 |
810 // Clear the marking bitmap and summary data and update top() in each space. | |
811 for (unsigned int id = perm_space_id; id < last_space_id; ++id) { | 985 for (unsigned int id = perm_space_id; id < last_space_id; ++id) { |
986 // Clear the marking bitmap, summary data and split info. | |
812 clear_data_covering_space(SpaceId(id)); | 987 clear_data_covering_space(SpaceId(id)); |
813 _space_info[id].space()->set_top(_space_info[id].new_top()); | 988 // Update top(). Must be done after clearing the bitmap and summary data. |
989 _space_info[id].publish_new_top(); | |
814 } | 990 } |
815 | 991 |
816 MutableSpace* const eden_space = _space_info[eden_space_id].space(); | 992 MutableSpace* const eden_space = _space_info[eden_space_id].space(); |
817 MutableSpace* const from_space = _space_info[from_space_id].space(); | 993 MutableSpace* const from_space = _space_info[from_space_id].space(); |
818 MutableSpace* const to_space = _space_info[to_space_id].space(); | 994 MutableSpace* const to_space = _space_info[to_space_id].space(); |
1241 | 1417 |
1242 void PSParallelCompact::summarize_spaces_quick() | 1418 void PSParallelCompact::summarize_spaces_quick() |
1243 { | 1419 { |
1244 for (unsigned int i = 0; i < last_space_id; ++i) { | 1420 for (unsigned int i = 0; i < last_space_id; ++i) { |
1245 const MutableSpace* space = _space_info[i].space(); | 1421 const MutableSpace* space = _space_info[i].space(); |
1246 bool result = _summary_data.summarize(space->bottom(), space->end(), | 1422 HeapWord** nta = _space_info[i].new_top_addr(); |
1247 space->bottom(), space->top(), | 1423 bool result = _summary_data.summarize(_space_info[i].split_info(), |
1248 _space_info[i].new_top_addr()); | 1424 space->bottom(), space->top(), NULL, |
1249 assert(result, "should never fail"); | 1425 space->bottom(), space->end(), nta); |
1426 assert(result, "space must fit into itself"); | |
1250 _space_info[i].set_dense_prefix(space->bottom()); | 1427 _space_info[i].set_dense_prefix(space->bottom()); |
1251 } | 1428 } |
1252 } | 1429 } |
1253 | 1430 |
1254 void PSParallelCompact::fill_dense_prefix_end(SpaceId id) | 1431 void PSParallelCompact::fill_dense_prefix_end(SpaceId id) |
1306 obj_beg = dense_prefix_end - 3; | 1483 obj_beg = dense_prefix_end - 3; |
1307 obj_len = 3; | 1484 obj_len = 3; |
1308 } | 1485 } |
1309 #endif // #ifdef _LP64 | 1486 #endif // #ifdef _LP64 |
1310 | 1487 |
1311 gc_heap()->fill_with_object(obj_beg, obj_len); | 1488 CollectedHeap::fill_with_object(obj_beg, obj_len); |
1312 _mark_bitmap.mark_obj(obj_beg, obj_len); | 1489 _mark_bitmap.mark_obj(obj_beg, obj_len); |
1313 _summary_data.add_obj(obj_beg, obj_len); | 1490 _summary_data.add_obj(obj_beg, obj_len); |
1314 assert(start_array(id) != NULL, "sanity"); | 1491 assert(start_array(id) != NULL, "sanity"); |
1315 start_array(id)->allocate_block(obj_beg); | 1492 start_array(id)->allocate_block(obj_beg); |
1493 } | |
1494 } | |
1495 | |
1496 void | |
1497 PSParallelCompact::clear_source_region(HeapWord* beg_addr, HeapWord* end_addr) | |
1498 { | |
1499 RegionData* const beg_ptr = _summary_data.addr_to_region_ptr(beg_addr); | |
1500 HeapWord* const end_aligned_up = _summary_data.region_align_up(end_addr); | |
1501 RegionData* const end_ptr = _summary_data.addr_to_region_ptr(end_aligned_up); | |
1502 for (RegionData* cur = beg_ptr; cur < end_ptr; ++cur) { | |
1503 cur->set_source_region(0); | |
1316 } | 1504 } |
1317 } | 1505 } |
1318 | 1506 |
1319 void | 1507 void |
1320 PSParallelCompact::summarize_space(SpaceId id, bool maximum_compaction) | 1508 PSParallelCompact::summarize_space(SpaceId id, bool maximum_compaction) |
1335 HeapWord* addr = compute_dense_prefix_via_density(id, maximum_compaction); | 1523 HeapWord* addr = compute_dense_prefix_via_density(id, maximum_compaction); |
1336 print_dense_prefix_stats("density", id, maximum_compaction, addr); | 1524 print_dense_prefix_stats("density", id, maximum_compaction, addr); |
1337 } | 1525 } |
1338 #endif // #ifndef PRODUCT | 1526 #endif // #ifndef PRODUCT |
1339 | 1527 |
1340 // If dead space crosses the dense prefix boundary, it is (at least | 1528 // Recompute the summary data, taking into account the dense prefix. If every |
1341 // partially) filled with a dummy object, marked live and added to the | 1529 // last byte will be reclaimed, then the existing summary data which compacts |
1342 // summary data. This simplifies the copy/update phase and must be done | 1530 // everything can be left in place. |
1343 // before the final locations of objects are determined, to prevent leaving | |
1344 // a fragment of dead space that is too small to fill with an object. | |
1345 if (!maximum_compaction && dense_prefix_end != space->bottom()) { | 1531 if (!maximum_compaction && dense_prefix_end != space->bottom()) { |
1532 // If dead space crosses the dense prefix boundary, it is (at least | |
1533 // partially) filled with a dummy object, marked live and added to the | |
1534 // summary data. This simplifies the copy/update phase and must be done | |
1535 // before the final locations of objects are determined, to prevent leaving | |
1536 // a fragment of dead space that is too small to fill with an object. | |
1346 fill_dense_prefix_end(id); | 1537 fill_dense_prefix_end(id); |
1347 } | 1538 |
1348 | 1539 // Compute the destination of each Region, and thus each object. |
1349 // Compute the destination of each Region, and thus each object. | 1540 _summary_data.summarize_dense_prefix(space->bottom(), dense_prefix_end); |
1350 _summary_data.summarize_dense_prefix(space->bottom(), dense_prefix_end); | 1541 _summary_data.summarize(_space_info[id].split_info(), |
1351 _summary_data.summarize(dense_prefix_end, space->end(), | 1542 dense_prefix_end, space->top(), NULL, |
1352 dense_prefix_end, space->top(), | 1543 dense_prefix_end, space->end(), |
1353 _space_info[id].new_top_addr()); | 1544 _space_info[id].new_top_addr()); |
1545 } | |
1354 } | 1546 } |
1355 | 1547 |
1356 if (TraceParallelOldGCSummaryPhase) { | 1548 if (TraceParallelOldGCSummaryPhase) { |
1357 const size_t region_size = ParallelCompactData::RegionSize; | 1549 const size_t region_size = ParallelCompactData::RegionSize; |
1358 HeapWord* const dense_prefix_end = _space_info[id].dense_prefix(); | 1550 HeapWord* const dense_prefix_end = _space_info[id].dense_prefix(); |
1368 dp_region, dp_words / region_size, | 1560 dp_region, dp_words / region_size, |
1369 cr_words / region_size, new_top); | 1561 cr_words / region_size, new_top); |
1370 } | 1562 } |
1371 } | 1563 } |
1372 | 1564 |
1565 #ifndef PRODUCT | |
1566 void PSParallelCompact::summary_phase_msg(SpaceId dst_space_id, | |
1567 HeapWord* dst_beg, HeapWord* dst_end, | |
1568 SpaceId src_space_id, | |
1569 HeapWord* src_beg, HeapWord* src_end) | |
1570 { | |
1571 if (TraceParallelOldGCSummaryPhase) { | |
1572 tty->print_cr("summarizing %d [%s] into %d [%s]: " | |
1573 "src=" PTR_FORMAT "-" PTR_FORMAT " " | |
1574 SIZE_FORMAT "-" SIZE_FORMAT " " | |
1575 "dst=" PTR_FORMAT "-" PTR_FORMAT " " | |
1576 SIZE_FORMAT "-" SIZE_FORMAT, | |
1577 src_space_id, space_names[src_space_id], | |
1578 dst_space_id, space_names[dst_space_id], | |
1579 src_beg, src_end, | |
1580 _summary_data.addr_to_region_idx(src_beg), | |
1581 _summary_data.addr_to_region_idx(src_end), | |
1582 dst_beg, dst_end, | |
1583 _summary_data.addr_to_region_idx(dst_beg), | |
1584 _summary_data.addr_to_region_idx(dst_end)); | |
1585 } | |
1586 } | |
1587 #endif // #ifndef PRODUCT | |
1588 | |
1373 void PSParallelCompact::summary_phase(ParCompactionManager* cm, | 1589 void PSParallelCompact::summary_phase(ParCompactionManager* cm, |
1374 bool maximum_compaction) | 1590 bool maximum_compaction) |
1375 { | 1591 { |
1376 EventMark m("2 summarize"); | 1592 EventMark m("2 summarize"); |
1377 TraceTime tm("summary phase", print_phases(), true, gclog_or_tty); | 1593 TraceTime tm("summary phase", print_phases(), true, gclog_or_tty); |
1400 } | 1616 } |
1401 } | 1617 } |
1402 | 1618 |
1403 // The amount of live data that will end up in old space (assuming it fits). | 1619 // The amount of live data that will end up in old space (assuming it fits). |
1404 size_t old_space_total_live = 0; | 1620 size_t old_space_total_live = 0; |
1405 unsigned int id; | 1621 assert(perm_space_id < old_space_id, "should not count perm data here"); |
1406 for (id = old_space_id; id < last_space_id; ++id) { | 1622 for (unsigned int id = old_space_id; id < last_space_id; ++id) { |
1407 old_space_total_live += pointer_delta(_space_info[id].new_top(), | 1623 old_space_total_live += pointer_delta(_space_info[id].new_top(), |
1408 _space_info[id].space()->bottom()); | 1624 _space_info[id].space()->bottom()); |
1409 } | 1625 } |
1410 | 1626 |
1411 const MutableSpace* old_space = _space_info[old_space_id].space(); | 1627 MutableSpace* const old_space = _space_info[old_space_id].space(); |
1412 if (old_space_total_live > old_space->capacity_in_words()) { | 1628 if (old_space_total_live > old_space->capacity_in_words()) { |
1413 // XXX - should also try to expand | 1629 // XXX - should also try to expand |
1414 maximum_compaction = true; | 1630 maximum_compaction = true; |
1415 } else if (!UseParallelOldGCDensePrefix) { | |
1416 maximum_compaction = true; | |
1417 } | 1631 } |
1418 | 1632 |
1419 // Permanent and Old generations. | 1633 // Permanent and Old generations. |
1420 summarize_space(perm_space_id, maximum_compaction); | 1634 summarize_space(perm_space_id, maximum_compaction); |
1421 summarize_space(old_space_id, maximum_compaction); | 1635 summarize_space(old_space_id, maximum_compaction); |
1422 | 1636 |
1423 // Summarize the remaining spaces (those in the young gen) into old space. If | 1637 // Summarize the remaining spaces in the young gen. The initial target space |
1424 // the live data from a space doesn't fit, the existing summarization is left | 1638 // is the old gen. If a space does not fit entirely into the target, then the |
1425 // intact, so the data is compacted down within the space itself. | 1639 // remainder is compacted into the space itself and that space becomes the new |
1426 HeapWord** new_top_addr = _space_info[old_space_id].new_top_addr(); | 1640 // target. |
1427 HeapWord* const target_space_end = old_space->end(); | 1641 SpaceId dst_space_id = old_space_id; |
1428 for (id = eden_space_id; id < last_space_id; ++id) { | 1642 HeapWord* dst_space_end = old_space->end(); |
1643 HeapWord** new_top_addr = _space_info[dst_space_id].new_top_addr(); | |
1644 for (unsigned int id = eden_space_id; id < last_space_id; ++id) { | |
1429 const MutableSpace* space = _space_info[id].space(); | 1645 const MutableSpace* space = _space_info[id].space(); |
1430 const size_t live = pointer_delta(_space_info[id].new_top(), | 1646 const size_t live = pointer_delta(_space_info[id].new_top(), |
1431 space->bottom()); | 1647 space->bottom()); |
1432 const size_t available = pointer_delta(target_space_end, *new_top_addr); | 1648 const size_t available = pointer_delta(dst_space_end, *new_top_addr); |
1649 | |
1650 NOT_PRODUCT(summary_phase_msg(dst_space_id, *new_top_addr, dst_space_end, | |
1651 SpaceId(id), space->bottom(), space->top());) | |
1433 if (live > 0 && live <= available) { | 1652 if (live > 0 && live <= available) { |
1434 // All the live data will fit. | 1653 // All the live data will fit. |
1435 if (TraceParallelOldGCSummaryPhase) { | 1654 bool done = _summary_data.summarize(_space_info[id].split_info(), |
1436 tty->print_cr("summarizing %d into old_space @ " PTR_FORMAT, | 1655 space->bottom(), space->top(), |
1437 id, *new_top_addr); | 1656 NULL, |
1438 } | 1657 *new_top_addr, dst_space_end, |
1439 _summary_data.summarize(*new_top_addr, target_space_end, | 1658 new_top_addr); |
1440 space->bottom(), space->top(), | 1659 assert(done, "space must fit into old gen"); |
1441 new_top_addr); | 1660 |
1442 | 1661 // XXX - this is necessary because decrement_destination_counts() tests |
1662 // source_region() to determine if a region will be filled. Probably | |
1663 // better to pass src_space->new_top() into decrement_destination_counts | |
1664 // and test that instead. | |
1665 // | |
1443 // Clear the source_region field for each region in the space. | 1666 // Clear the source_region field for each region in the space. |
1444 HeapWord* const new_top = _space_info[id].new_top(); | 1667 clear_source_region(space->bottom(), _space_info[id].new_top()); |
1445 HeapWord* const clear_end = _summary_data.region_align_up(new_top); | |
1446 RegionData* beg_region = | |
1447 _summary_data.addr_to_region_ptr(space->bottom()); | |
1448 RegionData* end_region = _summary_data.addr_to_region_ptr(clear_end); | |
1449 while (beg_region < end_region) { | |
1450 beg_region->set_source_region(0); | |
1451 ++beg_region; | |
1452 } | |
1453 | 1668 |
1454 // Reset the new_top value for the space. | 1669 // Reset the new_top value for the space. |
1455 _space_info[id].set_new_top(space->bottom()); | 1670 _space_info[id].set_new_top(space->bottom()); |
1671 } else if (live > 0) { | |
1672 // Attempt to fit part of the source space into the target space. | |
1673 HeapWord* next_src_addr = NULL; | |
1674 bool done = _summary_data.summarize(_space_info[id].split_info(), | |
1675 space->bottom(), space->top(), | |
1676 &next_src_addr, | |
1677 *new_top_addr, dst_space_end, | |
1678 new_top_addr); | |
1679 assert(!done, "space should not fit into old gen"); | |
1680 assert(next_src_addr != NULL, "sanity"); | |
1681 | |
1682 // The source space becomes the new target, so the remainder is compacted | |
1683 // within the space itself. | |
1684 dst_space_id = SpaceId(id); | |
1685 dst_space_end = space->end(); | |
1686 new_top_addr = _space_info[id].new_top_addr(); | |
1687 HeapWord* const clear_end = _space_info[id].new_top(); | |
1688 NOT_PRODUCT(summary_phase_msg(dst_space_id, | |
1689 space->bottom(), dst_space_end, | |
1690 SpaceId(id), next_src_addr, space->top());) | |
1691 done = _summary_data.summarize(_space_info[id].split_info(), | |
1692 next_src_addr, space->top(), | |
1693 NULL, | |
1694 space->bottom(), dst_space_end, | |
1695 new_top_addr); | |
1696 assert(done, "space must fit when compacted into itself"); | |
1697 assert(*new_top_addr <= space->top(), "usage should not grow"); | |
1698 | |
1699 // XXX - this should go away. See comments above. | |
1700 // | |
1701 // Clear the source_region field in regions at the end of the space that | |
1702 // will not be filled. | |
1703 HeapWord* const clear_beg = _summary_data.region_align_up(*new_top_addr); | |
1704 clear_source_region(clear_beg, clear_end); | |
1456 } | 1705 } |
1457 } | 1706 } |
1458 | 1707 |
1459 if (TraceParallelOldGCSummaryPhase) { | 1708 if (TraceParallelOldGCSummaryPhase) { |
1460 tty->print_cr("summary_phase: after final summarization"); | 1709 tty->print_cr("summary_phase: after final summarization"); |
2049 | 2298 |
2050 // Iterate over all the spaces adding tasks for updating | 2299 // Iterate over all the spaces adding tasks for updating |
2051 // regions in the dense prefix. Assume that 1 gc thread | 2300 // regions in the dense prefix. Assume that 1 gc thread |
2052 // will work on opening the gaps and the remaining gc threads | 2301 // will work on opening the gaps and the remaining gc threads |
2053 // will work on the dense prefix. | 2302 // will work on the dense prefix. |
2054 SpaceId space_id = old_space_id; | 2303 unsigned int space_id; |
2055 while (space_id != last_space_id) { | 2304 for (space_id = old_space_id; space_id < last_space_id; ++ space_id) { |
2056 HeapWord* const dense_prefix_end = _space_info[space_id].dense_prefix(); | 2305 HeapWord* const dense_prefix_end = _space_info[space_id].dense_prefix(); |
2057 const MutableSpace* const space = _space_info[space_id].space(); | 2306 const MutableSpace* const space = _space_info[space_id].space(); |
2058 | 2307 |
2059 if (dense_prefix_end == space->bottom()) { | 2308 if (dense_prefix_end == space->bottom()) { |
2060 // There is no dense prefix for this space. | 2309 // There is no dense prefix for this space. |
2061 space_id = next_compaction_space_id(space_id); | |
2062 continue; | 2310 continue; |
2063 } | 2311 } |
2064 | 2312 |
2065 // The dense prefix is before this region. | 2313 // The dense prefix is before this region. |
2066 size_t region_index_end_dense_prefix = | 2314 size_t region_index_end_dense_prefix = |
2106 break; | 2354 break; |
2107 } | 2355 } |
2108 // region_index_end is not processed | 2356 // region_index_end is not processed |
2109 size_t region_index_end = MIN2(region_index_start + regions_per_thread, | 2357 size_t region_index_end = MIN2(region_index_start + regions_per_thread, |
2110 region_index_end_dense_prefix); | 2358 region_index_end_dense_prefix); |
2111 q->enqueue(new UpdateDensePrefixTask( | 2359 q->enqueue(new UpdateDensePrefixTask(SpaceId(space_id), |
2112 space_id, | 2360 region_index_start, |
2113 region_index_start, | 2361 region_index_end)); |
2114 region_index_end)); | |
2115 region_index_start = region_index_end; | 2362 region_index_start = region_index_end; |
2116 } | 2363 } |
2117 } | 2364 } |
2118 // This gets any part of the dense prefix that did not | 2365 // This gets any part of the dense prefix that did not |
2119 // fit evenly. | 2366 // fit evenly. |
2120 if (region_index_start < region_index_end_dense_prefix) { | 2367 if (region_index_start < region_index_end_dense_prefix) { |
2121 q->enqueue(new UpdateDensePrefixTask( | 2368 q->enqueue(new UpdateDensePrefixTask(SpaceId(space_id), |
2122 space_id, | 2369 region_index_start, |
2123 region_index_start, | 2370 region_index_end_dense_prefix)); |
2124 region_index_end_dense_prefix)); | 2371 } |
2125 } | 2372 } |
2126 space_id = next_compaction_space_id(space_id); | |
2127 } // End tasks for dense prefix | |
2128 } | 2373 } |
2129 | 2374 |
2130 void PSParallelCompact::enqueue_region_stealing_tasks( | 2375 void PSParallelCompact::enqueue_region_stealing_tasks( |
2131 GCTaskQueue* q, | 2376 GCTaskQueue* q, |
2132 ParallelTaskTerminator* terminator_ptr, | 2377 ParallelTaskTerminator* terminator_ptr, |
2568 cur_beg = m->find_obj_beg(cur_beg, search_end); | 2813 cur_beg = m->find_obj_beg(cur_beg, search_end); |
2569 assert(cur_beg < m->addr_to_bit(end), "not enough live words to skip"); | 2814 assert(cur_beg < m->addr_to_bit(end), "not enough live words to skip"); |
2570 return m->bit_to_addr(cur_beg); | 2815 return m->bit_to_addr(cur_beg); |
2571 } | 2816 } |
2572 | 2817 |
2573 HeapWord* | 2818 HeapWord* PSParallelCompact::first_src_addr(HeapWord* const dest_addr, |
2574 PSParallelCompact::first_src_addr(HeapWord* const dest_addr, | 2819 SpaceId src_space_id, |
2575 size_t src_region_idx) | 2820 size_t src_region_idx) |
2576 { | 2821 { |
2822 assert(summary_data().is_region_aligned(dest_addr), "not aligned"); | |
2823 | |
2824 const SplitInfo& split_info = _space_info[src_space_id].split_info(); | |
2825 if (split_info.dest_region_addr() == dest_addr) { | |
2826 // The partial object ending at the split point contains the first word to | |
2827 // be copied to dest_addr. | |
2828 return split_info.first_src_addr(); | |
2829 } | |
2830 | |
2831 const ParallelCompactData& sd = summary_data(); | |
2577 ParMarkBitMap* const bitmap = mark_bitmap(); | 2832 ParMarkBitMap* const bitmap = mark_bitmap(); |
2578 const ParallelCompactData& sd = summary_data(); | |
2579 const size_t RegionSize = ParallelCompactData::RegionSize; | 2833 const size_t RegionSize = ParallelCompactData::RegionSize; |
2580 | 2834 |
2581 assert(sd.is_region_aligned(dest_addr), "not aligned"); | 2835 assert(sd.is_region_aligned(dest_addr), "not aligned"); |
2582 | |
2583 const RegionData* const src_region_ptr = sd.region(src_region_idx); | 2836 const RegionData* const src_region_ptr = sd.region(src_region_idx); |
2584 const size_t partial_obj_size = src_region_ptr->partial_obj_size(); | 2837 const size_t partial_obj_size = src_region_ptr->partial_obj_size(); |
2585 HeapWord* const src_region_destination = src_region_ptr->destination(); | 2838 HeapWord* const src_region_destination = src_region_ptr->destination(); |
2586 | 2839 |
2587 assert(dest_addr >= src_region_destination, "wrong src region"); | 2840 assert(dest_addr >= src_region_destination, "wrong src region"); |
2738 size_t src_region_idx = region_ptr->source_region(); | 2991 size_t src_region_idx = region_ptr->source_region(); |
2739 SpaceId src_space_id = space_id(sd.region_to_addr(src_region_idx)); | 2992 SpaceId src_space_id = space_id(sd.region_to_addr(src_region_idx)); |
2740 HeapWord* src_space_top = _space_info[src_space_id].space()->top(); | 2993 HeapWord* src_space_top = _space_info[src_space_id].space()->top(); |
2741 | 2994 |
2742 MoveAndUpdateClosure closure(bitmap, cm, start_array, dest_addr, words); | 2995 MoveAndUpdateClosure closure(bitmap, cm, start_array, dest_addr, words); |
2743 closure.set_source(first_src_addr(dest_addr, src_region_idx)); | 2996 closure.set_source(first_src_addr(dest_addr, src_space_id, src_region_idx)); |
2744 | 2997 |
2745 // Adjust src_region_idx to prepare for decrementing destination counts (the | 2998 // Adjust src_region_idx to prepare for decrementing destination counts (the |
2746 // destination count is not decremented when a region is copied to itself). | 2999 // destination count is not decremented when a region is copied to itself). |
2747 if (src_region_idx == region_idx) { | 3000 if (src_region_idx == region_idx) { |
2748 src_region_idx += 1; | 3001 src_region_idx += 1; |
3009 void PSParallelCompact::compact_prologue() { | 3262 void PSParallelCompact::compact_prologue() { |
3010 _updated_int_array_klass_obj = (klassOop) | 3263 _updated_int_array_klass_obj = (klassOop) |
3011 summary_data().calc_new_pointer(Universe::intArrayKlassObj()); | 3264 summary_data().calc_new_pointer(Universe::intArrayKlassObj()); |
3012 } | 3265 } |
3013 | 3266 |
3014 // The initial implementation of this method created a field | |
3015 // _next_compaction_space_id in SpaceInfo and initialized | |
3016 // that field in SpaceInfo::initialize_space_info(). That | |
3017 // required that _next_compaction_space_id be declared a | |
3018 // SpaceId in SpaceInfo and that would have required that | |
3019 // either SpaceId be declared in a separate class or that | |
3020 // it be declared in SpaceInfo. It didn't seem consistent | |
3021 // to declare it in SpaceInfo (didn't really fit logically). | |
3022 // Alternatively, defining a separate class to define SpaceId | |
3023 // seem excessive. This implementation is simple and localizes | |
3024 // the knowledge. | |
3025 | |
3026 PSParallelCompact::SpaceId | |
3027 PSParallelCompact::next_compaction_space_id(SpaceId id) { | |
3028 assert(id < last_space_id, "id out of range"); | |
3029 switch (id) { | |
3030 case perm_space_id : | |
3031 return last_space_id; | |
3032 case old_space_id : | |
3033 return eden_space_id; | |
3034 case eden_space_id : | |
3035 return from_space_id; | |
3036 case from_space_id : | |
3037 return to_space_id; | |
3038 case to_space_id : | |
3039 return last_space_id; | |
3040 default: | |
3041 assert(false, "Bad space id"); | |
3042 return last_space_id; | |
3043 } | |
3044 } |