Mercurial > hg > truffle
comparison src/share/vm/opto/loopPredicate.cpp @ 3345:bad7ecd0b6ed
5091921: Sign flip issues in loop optimizer
Summary: Fix integer overflow problem in the code generated by loop optimizer.
Reviewed-by: never
author | kvn |
---|---|
date | Wed, 04 May 2011 13:12:42 -0700 |
parents | 2e038ad0c1d0 |
children | 4e761e7e6e12 |
comparison
equal
deleted
inserted
replaced
3344:0139aac70fb5 | 3345:bad7ecd0b6ed |
---|---|
339 Node* old_entry = iff->in(0); | 339 Node* old_entry = iff->in(0); |
340 | 340 |
341 // Cut predicate from old place. | 341 // Cut predicate from old place. |
342 Node* old = predicate_proj; | 342 Node* old = predicate_proj; |
343 igvn->_worklist.push(old); | 343 igvn->_worklist.push(old); |
344 for (DUIterator_Last imin, i = old->last_outs(imin); i >= imin; ) { | 344 for (DUIterator_Last imin, i = old->last_outs(imin); i >= imin;) { |
345 Node* use = old->last_out(i); // for each use... | 345 Node* use = old->last_out(i); // for each use... |
346 igvn->hash_delete(use); | 346 igvn->hash_delete(use); |
347 igvn->_worklist.push(use); | 347 igvn->_worklist.push(use); |
348 // Update use-def info | 348 // Update use-def info |
349 uint uses_found = 0; | 349 uint uses_found = 0; |
382 return predicate_proj; | 382 return predicate_proj; |
383 } | 383 } |
384 | 384 |
385 //--------------------------clone_loop_predicates----------------------- | 385 //--------------------------clone_loop_predicates----------------------- |
386 // Interface from IGVN | 386 // Interface from IGVN |
387 Node* PhaseIterGVN::clone_loop_predicates(Node* old_entry, Node* new_entry) { | 387 Node* PhaseIterGVN::clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) { |
388 return PhaseIdealLoop::clone_loop_predicates(old_entry, new_entry, false, NULL, this); | 388 return PhaseIdealLoop::clone_loop_predicates(old_entry, new_entry, false, clone_limit_check, NULL, this); |
389 } | 389 } |
390 Node* PhaseIterGVN::move_loop_predicates(Node* old_entry, Node* new_entry) { | 390 Node* PhaseIterGVN::move_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) { |
391 return PhaseIdealLoop::clone_loop_predicates(old_entry, new_entry, true, NULL, this); | 391 return PhaseIdealLoop::clone_loop_predicates(old_entry, new_entry, true, clone_limit_check, NULL, this); |
392 } | 392 } |
393 | 393 |
394 // Interface from PhaseIdealLoop | 394 // Interface from PhaseIdealLoop |
395 Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry) { | 395 Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) { |
396 return clone_loop_predicates(old_entry, new_entry, false, this, &this->_igvn); | 396 return clone_loop_predicates(old_entry, new_entry, false, clone_limit_check, this, &this->_igvn); |
397 } | 397 } |
398 Node* PhaseIdealLoop::move_loop_predicates(Node* old_entry, Node* new_entry) { | 398 Node* PhaseIdealLoop::move_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) { |
399 return clone_loop_predicates(old_entry, new_entry, true, this, &this->_igvn); | 399 return clone_loop_predicates(old_entry, new_entry, true, clone_limit_check, this, &this->_igvn); |
400 } | 400 } |
401 | 401 |
402 // Clone loop predicates to cloned loops (peeled, unswitched, split_if). | 402 // Clone loop predicates to cloned loops (peeled, unswitched, split_if). |
403 Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry, | 403 Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry, |
404 bool move_predicates, | 404 bool move_predicates, |
405 bool clone_limit_check, | |
405 PhaseIdealLoop* loop_phase, | 406 PhaseIdealLoop* loop_phase, |
406 PhaseIterGVN* igvn) { | 407 PhaseIterGVN* igvn) { |
407 #ifdef ASSERT | 408 #ifdef ASSERT |
408 if (new_entry == NULL || !(new_entry->is_Proj() || new_entry->is_Region() || new_entry->is_SafePoint())) { | 409 if (new_entry == NULL || !(new_entry->is_Proj() || new_entry->is_Region() || new_entry->is_SafePoint())) { |
409 if (new_entry != NULL) | 410 if (new_entry != NULL) |
411 assert(false, "not IfTrue, IfFalse, Region or SafePoint"); | 412 assert(false, "not IfTrue, IfFalse, Region or SafePoint"); |
412 } | 413 } |
413 #endif | 414 #endif |
414 // Search original predicates | 415 // Search original predicates |
415 Node* entry = old_entry; | 416 Node* entry = old_entry; |
417 ProjNode* limit_check_proj = NULL; | |
418 if (LoopLimitCheck) { | |
419 limit_check_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); | |
420 if (limit_check_proj != NULL) { | |
421 entry = entry->in(0)->in(0); | |
422 } | |
423 } | |
416 if (UseLoopPredicate) { | 424 if (UseLoopPredicate) { |
417 ProjNode* predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); | 425 ProjNode* predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); |
418 if (predicate_proj != NULL) { // right pattern that can be used by loop predication | 426 if (predicate_proj != NULL) { // right pattern that can be used by loop predication |
419 assert(entry->in(0)->in(1)->in(1)->Opcode()==Op_Opaque1, "must be"); | |
420 if (move_predicates) { | 427 if (move_predicates) { |
421 new_entry = move_predicate(predicate_proj, new_entry, | 428 new_entry = move_predicate(predicate_proj, new_entry, |
422 Deoptimization::Reason_predicate, | 429 Deoptimization::Reason_predicate, |
423 loop_phase, igvn); | 430 loop_phase, igvn); |
424 assert(new_entry == predicate_proj, "old predicate fall through projection"); | 431 assert(new_entry == predicate_proj, "old predicate fall through projection"); |
433 tty->print_cr("Loop Predicate %s: ", move_predicates ? "moved" : "cloned"); | 440 tty->print_cr("Loop Predicate %s: ", move_predicates ? "moved" : "cloned"); |
434 debug_only( new_entry->in(0)->dump(); ) | 441 debug_only( new_entry->in(0)->dump(); ) |
435 } | 442 } |
436 } | 443 } |
437 } | 444 } |
445 if (limit_check_proj != NULL && clone_limit_check) { | |
446 // Clone loop limit check last to insert it before loop. | |
447 // Don't clone a limit check which was already finalized | |
448 // for this counted loop (only one limit check is needed). | |
449 if (move_predicates) { | |
450 new_entry = move_predicate(limit_check_proj, new_entry, | |
451 Deoptimization::Reason_loop_limit_check, | |
452 loop_phase, igvn); | |
453 assert(new_entry == limit_check_proj, "old limit check fall through projection"); | |
454 } else { | |
455 new_entry = clone_predicate(limit_check_proj, new_entry, | |
456 Deoptimization::Reason_loop_limit_check, | |
457 loop_phase, igvn); | |
458 assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone limit check"); | |
459 } | |
460 if (TraceLoopLimitCheck) { | |
461 tty->print_cr("Loop Limit Check %s: ", move_predicates ? "moved" : "cloned"); | |
462 debug_only( new_entry->in(0)->dump(); ) | |
463 } | |
464 } | |
438 return new_entry; | 465 return new_entry; |
439 } | 466 } |
440 | 467 |
441 //--------------------------eliminate_loop_predicates----------------------- | 468 //--------------------------eliminate_loop_predicates----------------------- |
442 void PhaseIdealLoop::eliminate_loop_predicates(Node* entry) { | 469 void PhaseIdealLoop::eliminate_loop_predicates(Node* entry) { |
470 if (LoopLimitCheck) { | |
471 Node* predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); | |
472 if (predicate != NULL) { | |
473 entry = entry->in(0)->in(0); | |
474 } | |
475 } | |
443 if (UseLoopPredicate) { | 476 if (UseLoopPredicate) { |
444 ProjNode* predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); | 477 ProjNode* predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); |
445 if (predicate_proj != NULL) { // right pattern that can be used by loop predication | 478 if (predicate_proj != NULL) { // right pattern that can be used by loop predication |
446 Node* n = entry->in(0)->in(1)->in(1); | 479 Node* n = entry->in(0)->in(1)->in(1); |
447 assert(n->Opcode()==Op_Opaque1, "must be"); | 480 assert(n->Opcode()==Op_Opaque1, "must be"); |
454 | 487 |
455 //--------------------------skip_loop_predicates------------------------------ | 488 //--------------------------skip_loop_predicates------------------------------ |
456 // Skip related predicates. | 489 // Skip related predicates. |
457 Node* PhaseIdealLoop::skip_loop_predicates(Node* entry) { | 490 Node* PhaseIdealLoop::skip_loop_predicates(Node* entry) { |
458 Node* predicate = NULL; | 491 Node* predicate = NULL; |
492 if (LoopLimitCheck) { | |
493 predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); | |
494 if (predicate != NULL) { | |
495 entry = entry->in(0)->in(0); | |
496 } | |
497 } | |
459 if (UseLoopPredicate) { | 498 if (UseLoopPredicate) { |
460 predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); | 499 predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); |
461 if (predicate != NULL) { // right pattern that can be used by loop predication | 500 if (predicate != NULL) { // right pattern that can be used by loop predication |
462 assert(entry->is_Proj() && entry->in(0)->in(1)->in(1)->Opcode()==Op_Opaque1, "must be"); | |
463 IfNode* iff = entry->in(0)->as_If(); | 501 IfNode* iff = entry->in(0)->as_If(); |
464 ProjNode* uncommon_proj = iff->proj_out(1 - entry->as_Proj()->_con); | 502 ProjNode* uncommon_proj = iff->proj_out(1 - entry->as_Proj()->_con); |
465 Node* rgn = uncommon_proj->unique_ctrl_out(); | 503 Node* rgn = uncommon_proj->unique_ctrl_out(); |
466 assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct"); | 504 assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct"); |
467 entry = entry->in(0)->in(0); | 505 entry = entry->in(0)->in(0); |
489 | 527 |
490 //--------------------------find_predicate------------------------------------ | 528 //--------------------------find_predicate------------------------------------ |
491 // Find a predicate | 529 // Find a predicate |
492 Node* PhaseIdealLoop::find_predicate(Node* entry) { | 530 Node* PhaseIdealLoop::find_predicate(Node* entry) { |
493 Node* predicate = NULL; | 531 Node* predicate = NULL; |
532 if (LoopLimitCheck) { | |
533 predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); | |
534 if (predicate != NULL) { // right pattern that can be used by loop predication | |
535 return entry; | |
536 } | |
537 } | |
494 if (UseLoopPredicate) { | 538 if (UseLoopPredicate) { |
495 predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); | 539 predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); |
496 if (predicate != NULL) { // right pattern that can be used by loop predication | 540 if (predicate != NULL) { // right pattern that can be used by loop predication |
497 assert(entry->in(0)->in(1)->in(1)->Opcode()==Op_Opaque1, "must be"); | |
498 return entry; | 541 return entry; |
499 } | 542 } |
500 } | 543 } |
501 return NULL; | 544 return NULL; |
502 } | 545 } |
656 return false; | 699 return false; |
657 } | 700 } |
658 Node* range = cmp->in(2); | 701 Node* range = cmp->in(2); |
659 if (range->Opcode() != Op_LoadRange) { | 702 if (range->Opcode() != Op_LoadRange) { |
660 const TypeInt* tint = phase->_igvn.type(range)->isa_int(); | 703 const TypeInt* tint = phase->_igvn.type(range)->isa_int(); |
661 if (!OptimizeFill || tint == NULL || tint->empty() || tint->_lo < 0) { | 704 if (tint == NULL || tint->empty() || tint->_lo < 0) { |
662 // Allow predication on positive values that aren't LoadRanges. | 705 // Allow predication on positive values that aren't LoadRanges. |
663 // This allows optimization of loops where the length of the | 706 // This allows optimization of loops where the length of the |
664 // array is a known value and doesn't need to be loaded back | 707 // array is a known value and doesn't need to be loaded back |
665 // from the array. | 708 // from the array. |
666 return false; | 709 return false; |
694 // There are two cases for max(scale*i + offset): | 737 // There are two cases for max(scale*i + offset): |
695 // (1) stride*scale > 0 | 738 // (1) stride*scale > 0 |
696 // max(scale*i + offset) = scale*(limit-stride) + offset | 739 // max(scale*i + offset) = scale*(limit-stride) + offset |
697 // (2) stride*scale < 0 | 740 // (2) stride*scale < 0 |
698 // max(scale*i + offset) = scale*init + offset | 741 // max(scale*i + offset) = scale*init + offset |
699 BoolNode* PhaseIdealLoop::rc_predicate(Node* ctrl, | 742 BoolNode* PhaseIdealLoop::rc_predicate(IdealLoopTree *loop, Node* ctrl, |
700 int scale, Node* offset, | 743 int scale, Node* offset, |
701 Node* init, Node* limit, Node* stride, | 744 Node* init, Node* limit, Node* stride, |
702 Node* range, bool upper) { | 745 Node* range, bool upper) { |
703 stringStream* predString = NULL; | 746 stringStream* predString = NULL; |
704 if (TraceLoopPredicate) { | 747 if (TraceLoopPredicate) { |
707 } | 750 } |
708 | 751 |
709 Node* max_idx_expr = init; | 752 Node* max_idx_expr = init; |
710 int stride_con = stride->get_int(); | 753 int stride_con = stride->get_int(); |
711 if ((stride_con > 0) == (scale > 0) == upper) { | 754 if ((stride_con > 0) == (scale > 0) == upper) { |
712 max_idx_expr = new (C, 3) SubINode(limit, stride); | 755 if (LoopLimitCheck) { |
713 register_new_node(max_idx_expr, ctrl); | 756 // With LoopLimitCheck limit is not exact. |
714 if (TraceLoopPredicate) predString->print("(limit - stride) "); | 757 // Calculate exact limit here. |
758 // Note, counted loop's test is '<' or '>'. | |
759 limit = exact_limit(loop); | |
760 max_idx_expr = new (C, 3) SubINode(limit, stride); | |
761 register_new_node(max_idx_expr, ctrl); | |
762 if (TraceLoopPredicate) predString->print("(limit - stride) "); | |
763 } else { | |
764 max_idx_expr = new (C, 3) SubINode(limit, stride); | |
765 register_new_node(max_idx_expr, ctrl); | |
766 if (TraceLoopPredicate) predString->print("(limit - stride) "); | |
767 } | |
715 } else { | 768 } else { |
716 if (TraceLoopPredicate) predString->print("init "); | 769 if (TraceLoopPredicate) predString->print("init "); |
717 } | 770 } |
718 | 771 |
719 if (scale != 1) { | 772 if (scale != 1) { |
750 | 803 |
751 if (!loop->_head->is_Loop()) { | 804 if (!loop->_head->is_Loop()) { |
752 // Could be a simple region when irreducible loops are present. | 805 // Could be a simple region when irreducible loops are present. |
753 return false; | 806 return false; |
754 } | 807 } |
755 | 808 LoopNode* head = loop->_head->as_Loop(); |
756 if (loop->_head->unique_ctrl_out()->Opcode() == Op_NeverBranch) { | 809 |
810 if (head->unique_ctrl_out()->Opcode() == Op_NeverBranch) { | |
757 // do nothing for infinite loops | 811 // do nothing for infinite loops |
758 return false; | 812 return false; |
759 } | 813 } |
760 | 814 |
761 CountedLoopNode *cl = NULL; | 815 CountedLoopNode *cl = NULL; |
762 if (loop->_head->is_CountedLoop()) { | 816 if (head->is_CountedLoop()) { |
763 cl = loop->_head->as_CountedLoop(); | 817 cl = head->as_CountedLoop(); |
764 // do nothing for iteration-splitted loops | 818 // do nothing for iteration-splitted loops |
765 if (!cl->is_normal_loop()) return false; | 819 if (!cl->is_normal_loop()) return false; |
766 } | 820 } |
767 | 821 |
768 LoopNode *lpn = loop->_head->as_Loop(); | 822 Node* entry = head->in(LoopNode::EntryControl); |
769 Node* entry = lpn->in(LoopNode::EntryControl); | 823 ProjNode *predicate_proj = NULL; |
770 | 824 // Loop limit check predicate should be near the loop. |
771 ProjNode *predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); | 825 if (LoopLimitCheck) { |
826 predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); | |
827 if (predicate_proj != NULL) | |
828 entry = predicate_proj->in(0)->in(0); | |
829 } | |
830 | |
831 predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); | |
772 if (!predicate_proj) { | 832 if (!predicate_proj) { |
773 #ifndef PRODUCT | 833 #ifndef PRODUCT |
774 if (TraceLoopPredicate) { | 834 if (TraceLoopPredicate) { |
775 tty->print("missing predicate:"); | 835 tty->print("missing predicate:"); |
776 loop->dump_head(); | 836 loop->dump_head(); |
777 lpn->dump(1); | 837 head->dump(1); |
778 } | 838 } |
779 #endif | 839 #endif |
780 return false; | 840 return false; |
781 } | 841 } |
782 ConNode* zero = _igvn.intcon(0); | 842 ConNode* zero = _igvn.intcon(0); |
786 Invariance invar(area, loop); | 846 Invariance invar(area, loop); |
787 | 847 |
788 // Create list of if-projs such that a newer proj dominates all older | 848 // Create list of if-projs such that a newer proj dominates all older |
789 // projs in the list, and they all dominate loop->tail() | 849 // projs in the list, and they all dominate loop->tail() |
790 Node_List if_proj_list(area); | 850 Node_List if_proj_list(area); |
791 LoopNode *head = loop->_head->as_Loop(); | |
792 Node *current_proj = loop->tail(); //start from tail | 851 Node *current_proj = loop->tail(); //start from tail |
793 while (current_proj != head) { | 852 while (current_proj != head) { |
794 if (loop == get_loop(current_proj) && // still in the loop ? | 853 if (loop == get_loop(current_proj) && // still in the loop ? |
795 current_proj->is_Proj() && // is a projection ? | 854 current_proj->is_Proj() && // is a projection ? |
796 current_proj->in(0)->Opcode() == Op_If) { // is a if projection ? | 855 current_proj->in(0)->Opcode() == Op_If) { // is a if projection ? |
860 | 919 |
861 // Range check for counted loops | 920 // Range check for counted loops |
862 const Node* cmp = bol->in(1)->as_Cmp(); | 921 const Node* cmp = bol->in(1)->as_Cmp(); |
863 Node* idx = cmp->in(1); | 922 Node* idx = cmp->in(1); |
864 assert(!invar.is_invariant(idx), "index is variant"); | 923 assert(!invar.is_invariant(idx), "index is variant"); |
865 assert(cmp->in(2)->Opcode() == Op_LoadRange || OptimizeFill, "must be"); | |
866 Node* rng = cmp->in(2); | 924 Node* rng = cmp->in(2); |
925 assert(rng->Opcode() == Op_LoadRange || _igvn.type(rng)->is_int() >= 0, "must be"); | |
867 assert(invar.is_invariant(rng), "range must be invariant"); | 926 assert(invar.is_invariant(rng), "range must be invariant"); |
868 int scale = 1; | 927 int scale = 1; |
869 Node* offset = zero; | 928 Node* offset = zero; |
870 bool ok = is_scaled_iv_plus_offset(idx, cl->phi(), &scale, &offset); | 929 bool ok = is_scaled_iv_plus_offset(idx, cl->phi(), &scale, &offset); |
871 assert(ok, "must be index expression"); | 930 assert(ok, "must be index expression"); |
890 assert(invar.is_invariant(offset), "offset must be loop invariant"); | 949 assert(invar.is_invariant(offset), "offset must be loop invariant"); |
891 offset = invar.clone(offset, ctrl); | 950 offset = invar.clone(offset, ctrl); |
892 } | 951 } |
893 | 952 |
894 // Test the lower bound | 953 // Test the lower bound |
895 Node* lower_bound_bol = rc_predicate(ctrl, scale, offset, init, limit, stride, rng, false); | 954 Node* lower_bound_bol = rc_predicate(loop, ctrl, scale, offset, init, limit, stride, rng, false); |
896 IfNode* lower_bound_iff = lower_bound_proj->in(0)->as_If(); | 955 IfNode* lower_bound_iff = lower_bound_proj->in(0)->as_If(); |
897 _igvn.hash_delete(lower_bound_iff); | 956 _igvn.hash_delete(lower_bound_iff); |
898 lower_bound_iff->set_req(1, lower_bound_bol); | 957 lower_bound_iff->set_req(1, lower_bound_bol); |
899 if (TraceLoopPredicate) tty->print_cr("lower bound check if: %d", lower_bound_iff->_idx); | 958 if (TraceLoopPredicate) tty->print_cr("lower bound check if: %d", lower_bound_iff->_idx); |
900 | 959 |
901 // Test the upper bound | 960 // Test the upper bound |
902 Node* upper_bound_bol = rc_predicate(ctrl, scale, offset, init, limit, stride, rng, true); | 961 Node* upper_bound_bol = rc_predicate(loop, ctrl, scale, offset, init, limit, stride, rng, true); |
903 IfNode* upper_bound_iff = upper_bound_proj->in(0)->as_If(); | 962 IfNode* upper_bound_iff = upper_bound_proj->in(0)->as_If(); |
904 _igvn.hash_delete(upper_bound_iff); | 963 _igvn.hash_delete(upper_bound_iff); |
905 upper_bound_iff->set_req(1, upper_bound_bol); | 964 upper_bound_iff->set_req(1, upper_bound_bol); |
906 if (TraceLoopPredicate) tty->print_cr("upper bound check if: %d", lower_bound_iff->_idx); | 965 if (TraceLoopPredicate) tty->print_cr("upper bound check if: %d", lower_bound_iff->_idx); |
907 | 966 |