Mercurial > hg > graal-jvmci-8
comparison src/share/vm/memory/referenceProcessor.cpp @ 17976:8e20ef014b08
8043239: G1: Missing post barrier in processing of j.l.ref.Reference objects
Summary: Removed all write barriers during reference processing and added explicit write barriers when iterating through the discovered list.
Reviewed-by: pliden, jmasa, tschatzl
author | brutisso |
---|---|
date | Wed, 11 Jun 2014 10:46:47 +0200 |
parents | 78bbf4d43a14 |
children | 52b4284cb496 0982ec23da03 01dcaba9b3f3 |
comparison
equal
deleted
inserted
replaced
17975:bd4d69d9cb7d | 17976:8e20ef014b08 |
---|---|
94 bool mt_processing, | 94 bool mt_processing, |
95 uint mt_processing_degree, | 95 uint mt_processing_degree, |
96 bool mt_discovery, | 96 bool mt_discovery, |
97 uint mt_discovery_degree, | 97 uint mt_discovery_degree, |
98 bool atomic_discovery, | 98 bool atomic_discovery, |
99 BoolObjectClosure* is_alive_non_header, | 99 BoolObjectClosure* is_alive_non_header) : |
100 bool discovered_list_needs_post_barrier) : | |
101 _discovering_refs(false), | 100 _discovering_refs(false), |
102 _enqueuing_is_done(false), | 101 _enqueuing_is_done(false), |
103 _is_alive_non_header(is_alive_non_header), | 102 _is_alive_non_header(is_alive_non_header), |
104 _discovered_list_needs_post_barrier(discovered_list_needs_post_barrier), | |
105 _processing_is_mt(mt_processing), | 103 _processing_is_mt(mt_processing), |
106 _next_id(0) | 104 _next_id(0) |
107 { | 105 { |
108 _span = span; | 106 _span = span; |
109 _discovery_is_atomic = atomic_discovery; | 107 _discovery_is_atomic = atomic_discovery; |
338 HeapWord* pending_list_addr) { | 336 HeapWord* pending_list_addr) { |
339 // Given a list of refs linked through the "discovered" field | 337 // Given a list of refs linked through the "discovered" field |
340 // (java.lang.ref.Reference.discovered), self-loop their "next" field | 338 // (java.lang.ref.Reference.discovered), self-loop their "next" field |
341 // thus distinguishing them from active References, then | 339 // thus distinguishing them from active References, then |
342 // prepend them to the pending list. | 340 // prepend them to the pending list. |
341 // | |
342 // The Java threads will see the Reference objects linked together through | |
343 // the discovered field. Instead of trying to do the write barrier updates | |
344 // in all places in the reference processor where we manipulate the discovered | |
345 // field we make sure to do the barrier here where we anyway iterate through | |
346 // all linked Reference objects. Note that it is important to not dirty any | |
347 // cards during reference processing since this will cause card table | |
348 // verification to fail for G1. | |
349 // | |
343 // BKWRD COMPATIBILITY NOTE: For older JDKs (prior to the fix for 4956777), | 350 // BKWRD COMPATIBILITY NOTE: For older JDKs (prior to the fix for 4956777), |
344 // the "next" field is used to chain the pending list, not the discovered | 351 // the "next" field is used to chain the pending list, not the discovered |
345 // field. | 352 // field. |
346 | |
347 if (TraceReferenceGC && PrintGCDetails) { | 353 if (TraceReferenceGC && PrintGCDetails) { |
348 gclog_or_tty->print_cr("ReferenceProcessor::enqueue_discovered_reflist list " | 354 gclog_or_tty->print_cr("ReferenceProcessor::enqueue_discovered_reflist list " |
349 INTPTR_FORMAT, (address)refs_list.head()); | 355 INTPTR_FORMAT, (address)refs_list.head()); |
350 } | 356 } |
351 | 357 |
352 oop obj = NULL; | 358 oop obj = NULL; |
353 oop next_d = refs_list.head(); | 359 oop next_d = refs_list.head(); |
354 if (pending_list_uses_discovered_field()) { // New behaviour | 360 if (pending_list_uses_discovered_field()) { // New behavior |
355 // Walk down the list, self-looping the next field | 361 // Walk down the list, self-looping the next field |
356 // so that the References are not considered active. | 362 // so that the References are not considered active. |
357 while (obj != next_d) { | 363 while (obj != next_d) { |
358 obj = next_d; | 364 obj = next_d; |
359 assert(obj->is_instanceRef(), "should be reference object"); | 365 assert(obj->is_instanceRef(), "should be reference object"); |
363 (void *)obj, (void *)next_d); | 369 (void *)obj, (void *)next_d); |
364 } | 370 } |
365 assert(java_lang_ref_Reference::next(obj) == NULL, | 371 assert(java_lang_ref_Reference::next(obj) == NULL, |
366 "Reference not active; should not be discovered"); | 372 "Reference not active; should not be discovered"); |
367 // Self-loop next, so as to make Ref not active. | 373 // Self-loop next, so as to make Ref not active. |
368 // Post-barrier not needed when looping to self. | |
369 java_lang_ref_Reference::set_next_raw(obj, obj); | 374 java_lang_ref_Reference::set_next_raw(obj, obj); |
370 if (next_d == obj) { // obj is last | 375 if (next_d != obj) { |
371 // Swap refs_list into pendling_list_addr and | 376 oopDesc::bs()->write_ref_field(java_lang_ref_Reference::discovered_addr(obj), next_d); |
377 } else { | |
378 // This is the last object. | |
379 // Swap refs_list into pending_list_addr and | |
372 // set obj's discovered to what we read from pending_list_addr. | 380 // set obj's discovered to what we read from pending_list_addr. |
373 oop old = oopDesc::atomic_exchange_oop(refs_list.head(), pending_list_addr); | 381 oop old = oopDesc::atomic_exchange_oop(refs_list.head(), pending_list_addr); |
374 // Need post-barrier on pending_list_addr above; | 382 // Need post-barrier on pending_list_addr. See enqueue_discovered_ref_helper() above. |
375 // see special post-barrier code at the end of | |
376 // enqueue_discovered_reflists() further below. | |
377 java_lang_ref_Reference::set_discovered_raw(obj, old); // old may be NULL | 383 java_lang_ref_Reference::set_discovered_raw(obj, old); // old may be NULL |
378 oopDesc::bs()->write_ref_field(java_lang_ref_Reference::discovered_addr(obj), old); | 384 oopDesc::bs()->write_ref_field(java_lang_ref_Reference::discovered_addr(obj), old); |
379 } | 385 } |
380 } | 386 } |
381 } else { // Old behaviour | 387 } else { // Old behaviour |
494 } | 500 } |
495 // Remove Reference object from discovered list. Note that G1 does not need a | 501 // Remove Reference object from discovered list. Note that G1 does not need a |
496 // pre-barrier here because we know the Reference has already been found/marked, | 502 // pre-barrier here because we know the Reference has already been found/marked, |
497 // that's how it ended up in the discovered list in the first place. | 503 // that's how it ended up in the discovered list in the first place. |
498 oop_store_raw(_prev_next, new_next); | 504 oop_store_raw(_prev_next, new_next); |
499 if (_discovered_list_needs_post_barrier && _prev_next != _refs_list.adr_head()) { | |
500 // Needs post-barrier and this is not the list head (which is not on the heap) | |
501 oopDesc::bs()->write_ref_field(_prev_next, new_next); | |
502 } | |
503 NOT_PRODUCT(_removed++); | 505 NOT_PRODUCT(_removed++); |
504 _refs_list.dec_length(1); | 506 _refs_list.dec_length(1); |
505 } | 507 } |
506 | 508 |
507 // Make the Reference object active again. | 509 // Make the Reference object active again. |
508 void DiscoveredListIterator::make_active() { | 510 void DiscoveredListIterator::make_active() { |
509 // For G1 we don't want to use set_next - it | 511 // The pre barrier for G1 is probably just needed for the old |
510 // will dirty the card for the next field of | 512 // reference processing behavior. Should we guard this with |
511 // the reference object and will fail | 513 // ReferenceProcessor::pending_list_uses_discovered_field() ? |
512 // CT verification. | |
513 if (UseG1GC) { | 514 if (UseG1GC) { |
514 HeapWord* next_addr = java_lang_ref_Reference::next_addr(_ref); | 515 HeapWord* next_addr = java_lang_ref_Reference::next_addr(_ref); |
515 if (UseCompressedOops) { | 516 if (UseCompressedOops) { |
516 oopDesc::bs()->write_ref_field_pre((narrowOop*)next_addr, NULL); | 517 oopDesc::bs()->write_ref_field_pre((narrowOop*)next_addr, NULL); |
517 } else { | 518 } else { |
518 oopDesc::bs()->write_ref_field_pre((oop*)next_addr, NULL); | 519 oopDesc::bs()->write_ref_field_pre((oop*)next_addr, NULL); |
519 } | 520 } |
520 java_lang_ref_Reference::set_next_raw(_ref, NULL); | 521 } |
521 } else { | 522 java_lang_ref_Reference::set_next_raw(_ref, NULL); |
522 java_lang_ref_Reference::set_next(_ref, NULL); | |
523 } | |
524 } | 523 } |
525 | 524 |
526 void DiscoveredListIterator::clear_referent() { | 525 void DiscoveredListIterator::clear_referent() { |
527 oop_store_raw(_referent_addr, NULL); | 526 oop_store_raw(_referent_addr, NULL); |
528 } | 527 } |
544 ReferencePolicy* policy, | 543 ReferencePolicy* policy, |
545 BoolObjectClosure* is_alive, | 544 BoolObjectClosure* is_alive, |
546 OopClosure* keep_alive, | 545 OopClosure* keep_alive, |
547 VoidClosure* complete_gc) { | 546 VoidClosure* complete_gc) { |
548 assert(policy != NULL, "Must have a non-NULL policy"); | 547 assert(policy != NULL, "Must have a non-NULL policy"); |
549 DiscoveredListIterator iter(refs_list, keep_alive, is_alive, _discovered_list_needs_post_barrier); | 548 DiscoveredListIterator iter(refs_list, keep_alive, is_alive); |
550 // Decide which softly reachable refs should be kept alive. | 549 // Decide which softly reachable refs should be kept alive. |
551 while (iter.has_next()) { | 550 while (iter.has_next()) { |
552 iter.load_ptrs(DEBUG_ONLY(!discovery_is_atomic() /* allow_null_referent */)); | 551 iter.load_ptrs(DEBUG_ONLY(!discovery_is_atomic() /* allow_null_referent */)); |
553 bool referent_is_dead = (iter.referent() != NULL) && !iter.is_referent_alive(); | 552 bool referent_is_dead = (iter.referent() != NULL) && !iter.is_referent_alive(); |
554 if (referent_is_dead && | 553 if (referent_is_dead && |
584 void | 583 void |
585 ReferenceProcessor::pp2_work(DiscoveredList& refs_list, | 584 ReferenceProcessor::pp2_work(DiscoveredList& refs_list, |
586 BoolObjectClosure* is_alive, | 585 BoolObjectClosure* is_alive, |
587 OopClosure* keep_alive) { | 586 OopClosure* keep_alive) { |
588 assert(discovery_is_atomic(), "Error"); | 587 assert(discovery_is_atomic(), "Error"); |
589 DiscoveredListIterator iter(refs_list, keep_alive, is_alive, _discovered_list_needs_post_barrier); | 588 DiscoveredListIterator iter(refs_list, keep_alive, is_alive); |
590 while (iter.has_next()) { | 589 while (iter.has_next()) { |
591 iter.load_ptrs(DEBUG_ONLY(false /* allow_null_referent */)); | 590 iter.load_ptrs(DEBUG_ONLY(false /* allow_null_referent */)); |
592 DEBUG_ONLY(oop next = java_lang_ref_Reference::next(iter.obj());) | 591 DEBUG_ONLY(oop next = java_lang_ref_Reference::next(iter.obj());) |
593 assert(next == NULL, "Should not discover inactive Reference"); | 592 assert(next == NULL, "Should not discover inactive Reference"); |
594 if (iter.is_referent_alive()) { | 593 if (iter.is_referent_alive()) { |
621 ReferenceProcessor::pp2_work_concurrent_discovery(DiscoveredList& refs_list, | 620 ReferenceProcessor::pp2_work_concurrent_discovery(DiscoveredList& refs_list, |
622 BoolObjectClosure* is_alive, | 621 BoolObjectClosure* is_alive, |
623 OopClosure* keep_alive, | 622 OopClosure* keep_alive, |
624 VoidClosure* complete_gc) { | 623 VoidClosure* complete_gc) { |
625 assert(!discovery_is_atomic(), "Error"); | 624 assert(!discovery_is_atomic(), "Error"); |
626 DiscoveredListIterator iter(refs_list, keep_alive, is_alive, _discovered_list_needs_post_barrier); | 625 DiscoveredListIterator iter(refs_list, keep_alive, is_alive); |
627 while (iter.has_next()) { | 626 while (iter.has_next()) { |
628 iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); | 627 iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); |
629 HeapWord* next_addr = java_lang_ref_Reference::next_addr(iter.obj()); | 628 HeapWord* next_addr = java_lang_ref_Reference::next_addr(iter.obj()); |
630 oop next = java_lang_ref_Reference::next(iter.obj()); | 629 oop next = java_lang_ref_Reference::next(iter.obj()); |
631 if ((iter.referent() == NULL || iter.is_referent_alive() || | 630 if ((iter.referent() == NULL || iter.is_referent_alive() || |
664 bool clear_referent, | 663 bool clear_referent, |
665 BoolObjectClosure* is_alive, | 664 BoolObjectClosure* is_alive, |
666 OopClosure* keep_alive, | 665 OopClosure* keep_alive, |
667 VoidClosure* complete_gc) { | 666 VoidClosure* complete_gc) { |
668 ResourceMark rm; | 667 ResourceMark rm; |
669 DiscoveredListIterator iter(refs_list, keep_alive, is_alive, _discovered_list_needs_post_barrier); | 668 DiscoveredListIterator iter(refs_list, keep_alive, is_alive); |
670 while (iter.has_next()) { | 669 while (iter.has_next()) { |
671 iter.update_discovered(); | 670 iter.update_discovered(); |
672 iter.load_ptrs(DEBUG_ONLY(false /* allow_null_referent */)); | 671 iter.load_ptrs(DEBUG_ONLY(false /* allow_null_referent */)); |
673 if (clear_referent) { | 672 if (clear_referent) { |
674 // NULL out referent pointer | 673 // NULL out referent pointer |
780 } | 779 } |
781 private: | 780 private: |
782 bool _clear_referent; | 781 bool _clear_referent; |
783 }; | 782 }; |
784 | 783 |
785 void ReferenceProcessor::set_discovered(oop ref, oop value) { | |
786 java_lang_ref_Reference::set_discovered_raw(ref, value); | |
787 if (_discovered_list_needs_post_barrier) { | |
788 oopDesc::bs()->write_ref_field(java_lang_ref_Reference::discovered_addr(ref), value); | |
789 } | |
790 } | |
791 | |
792 // Balances reference queues. | 784 // Balances reference queues. |
793 // Move entries from all queues[0, 1, ..., _max_num_q-1] to | 785 // Move entries from all queues[0, 1, ..., _max_num_q-1] to |
794 // queues[0, 1, ..., _num_q-1] because only the first _num_q | 786 // queues[0, 1, ..., _num_q-1] because only the first _num_q |
795 // corresponding to the active workers will be processed. | 787 // corresponding to the active workers will be processed. |
796 void ReferenceProcessor::balance_queues(DiscoveredList ref_lists[]) | 788 void ReferenceProcessor::balance_queues(DiscoveredList ref_lists[]) |
844 } | 836 } |
845 | 837 |
846 // Add the chain to the to list. | 838 // Add the chain to the to list. |
847 if (ref_lists[to_idx].head() == NULL) { | 839 if (ref_lists[to_idx].head() == NULL) { |
848 // to list is empty. Make a loop at the end. | 840 // to list is empty. Make a loop at the end. |
849 set_discovered(move_tail, move_tail); | 841 java_lang_ref_Reference::set_discovered_raw(move_tail, move_tail); |
850 } else { | 842 } else { |
851 set_discovered(move_tail, ref_lists[to_idx].head()); | 843 java_lang_ref_Reference::set_discovered_raw(move_tail, ref_lists[to_idx].head()); |
852 } | 844 } |
853 ref_lists[to_idx].set_head(move_head); | 845 ref_lists[to_idx].set_head(move_head); |
854 ref_lists[to_idx].inc_length(refs_to_move); | 846 ref_lists[to_idx].inc_length(refs_to_move); |
855 | 847 |
856 // Remove the chain from the from list. | 848 // Remove the chain from the from list. |
980 } | 972 } |
981 } | 973 } |
982 | 974 |
983 void ReferenceProcessor::clean_up_discovered_reflist(DiscoveredList& refs_list) { | 975 void ReferenceProcessor::clean_up_discovered_reflist(DiscoveredList& refs_list) { |
984 assert(!discovery_is_atomic(), "Else why call this method?"); | 976 assert(!discovery_is_atomic(), "Else why call this method?"); |
985 DiscoveredListIterator iter(refs_list, NULL, NULL, _discovered_list_needs_post_barrier); | 977 DiscoveredListIterator iter(refs_list, NULL, NULL); |
986 while (iter.has_next()) { | 978 while (iter.has_next()) { |
987 iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); | 979 iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); |
988 oop next = java_lang_ref_Reference::next(iter.obj()); | 980 oop next = java_lang_ref_Reference::next(iter.obj()); |
989 assert(next->is_oop_or_null(), "bad next field"); | 981 assert(next->is_oop_or_null(), "bad next field"); |
990 // If referent has been cleared or Reference is not active, | 982 // If referent has been cleared or Reference is not active, |
1069 // discovered_addr. | 1061 // discovered_addr. |
1070 oop current_head = refs_list.head(); | 1062 oop current_head = refs_list.head(); |
1071 // The last ref must have its discovered field pointing to itself. | 1063 // The last ref must have its discovered field pointing to itself. |
1072 oop next_discovered = (current_head != NULL) ? current_head : obj; | 1064 oop next_discovered = (current_head != NULL) ? current_head : obj; |
1073 | 1065 |
1074 // Note: In the case of G1, this specific pre-barrier is strictly | |
1075 // not necessary because the only case we are interested in | |
1076 // here is when *discovered_addr is NULL (see the CAS further below), | |
1077 // so this will expand to nothing. As a result, we have manually | |
1078 // elided this out for G1, but left in the test for some future | |
1079 // collector that might have need for a pre-barrier here, e.g.:- | |
1080 // oopDesc::bs()->write_ref_field_pre((oop* or narrowOop*)discovered_addr, next_discovered); | |
1081 assert(!_discovered_list_needs_post_barrier || UseG1GC, | |
1082 "Need to check non-G1 collector: " | |
1083 "may need a pre-write-barrier for CAS from NULL below"); | |
1084 oop retest = oopDesc::atomic_compare_exchange_oop(next_discovered, discovered_addr, | 1066 oop retest = oopDesc::atomic_compare_exchange_oop(next_discovered, discovered_addr, |
1085 NULL); | 1067 NULL); |
1086 if (retest == NULL) { | 1068 if (retest == NULL) { |
1087 // This thread just won the right to enqueue the object. | 1069 // This thread just won the right to enqueue the object. |
1088 // We have separate lists for enqueueing, so no synchronization | 1070 // We have separate lists for enqueueing, so no synchronization |
1089 // is necessary. | 1071 // is necessary. |
1090 refs_list.set_head(obj); | 1072 refs_list.set_head(obj); |
1091 refs_list.inc_length(1); | 1073 refs_list.inc_length(1); |
1092 if (_discovered_list_needs_post_barrier) { | |
1093 oopDesc::bs()->write_ref_field((void*)discovered_addr, next_discovered); | |
1094 } | |
1095 | 1074 |
1096 if (TraceReferenceGC) { | 1075 if (TraceReferenceGC) { |
1097 gclog_or_tty->print_cr("Discovered reference (mt) (" INTPTR_FORMAT ": %s)", | 1076 gclog_or_tty->print_cr("Discovered reference (mt) (" INTPTR_FORMAT ": %s)", |
1098 (void *)obj, obj->klass()->internal_name()); | 1077 (void *)obj, obj->klass()->internal_name()); |
1099 } | 1078 } |
1240 } | 1219 } |
1241 | 1220 |
1242 if (_discovery_is_mt) { | 1221 if (_discovery_is_mt) { |
1243 add_to_discovered_list_mt(*list, obj, discovered_addr); | 1222 add_to_discovered_list_mt(*list, obj, discovered_addr); |
1244 } else { | 1223 } else { |
1245 // If "_discovered_list_needs_post_barrier", we do write barriers when | 1224 // We do a raw store here: the field will be visited later when processing |
1246 // updating the discovered reference list. Otherwise, we do a raw store | 1225 // the discovered references. |
1247 // here: the field will be visited later when processing the discovered | |
1248 // references. | |
1249 oop current_head = list->head(); | 1226 oop current_head = list->head(); |
1250 // The last ref must have its discovered field pointing to itself. | 1227 // The last ref must have its discovered field pointing to itself. |
1251 oop next_discovered = (current_head != NULL) ? current_head : obj; | 1228 oop next_discovered = (current_head != NULL) ? current_head : obj; |
1252 | 1229 |
1253 // As in the case further above, since we are over-writing a NULL | |
1254 // pre-value, we can safely elide the pre-barrier here for the case of G1. | |
1255 // e.g.:- oopDesc::bs()->write_ref_field_pre((oop* or narrowOop*)discovered_addr, next_discovered); | |
1256 assert(discovered == NULL, "control point invariant"); | 1230 assert(discovered == NULL, "control point invariant"); |
1257 assert(!_discovered_list_needs_post_barrier || UseG1GC, | |
1258 "For non-G1 collector, may need a pre-write-barrier for CAS from NULL below"); | |
1259 oop_store_raw(discovered_addr, next_discovered); | 1231 oop_store_raw(discovered_addr, next_discovered); |
1260 if (_discovered_list_needs_post_barrier) { | |
1261 oopDesc::bs()->write_ref_field((void*)discovered_addr, next_discovered); | |
1262 } | |
1263 list->set_head(obj); | 1232 list->set_head(obj); |
1264 list->inc_length(1); | 1233 list->inc_length(1); |
1265 | 1234 |
1266 if (TraceReferenceGC) { | 1235 if (TraceReferenceGC) { |
1267 gclog_or_tty->print_cr("Discovered reference (" INTPTR_FORMAT ": %s)", | 1236 gclog_or_tty->print_cr("Discovered reference (" INTPTR_FORMAT ": %s)", |
1351 ReferenceProcessor::preclean_discovered_reflist(DiscoveredList& refs_list, | 1320 ReferenceProcessor::preclean_discovered_reflist(DiscoveredList& refs_list, |
1352 BoolObjectClosure* is_alive, | 1321 BoolObjectClosure* is_alive, |
1353 OopClosure* keep_alive, | 1322 OopClosure* keep_alive, |
1354 VoidClosure* complete_gc, | 1323 VoidClosure* complete_gc, |
1355 YieldClosure* yield) { | 1324 YieldClosure* yield) { |
1356 DiscoveredListIterator iter(refs_list, keep_alive, is_alive, _discovered_list_needs_post_barrier); | 1325 DiscoveredListIterator iter(refs_list, keep_alive, is_alive); |
1357 while (iter.has_next()) { | 1326 while (iter.has_next()) { |
1358 iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); | 1327 iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); |
1359 oop obj = iter.obj(); | 1328 oop obj = iter.obj(); |
1360 oop next = java_lang_ref_Reference::next(obj); | 1329 oop next = java_lang_ref_Reference::next(obj); |
1361 if (iter.referent() == NULL || iter.is_referent_alive() || | 1330 if (iter.referent() == NULL || iter.is_referent_alive() || |