comparison src/share/vm/opto/library_call.cpp @ 11080:b800986664f4

7088419: Use x86 Hardware CRC32 Instruction with java.util.zip.CRC32 Summary: add intrinsics using new instruction to interpreter, C1, C2, for suitable x86; add test Reviewed-by: kvn, twisti
author drchase
date Tue, 02 Jul 2013 20:42:12 -0400
parents f2110083203d
children acedd49a1bce
comparison
equal deleted inserted replaced
11079:738e04fb1232 11080:b800986664f4
289 bool inline_aescrypt_Block(vmIntrinsics::ID id); 289 bool inline_aescrypt_Block(vmIntrinsics::ID id);
290 bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id); 290 bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id);
291 Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting); 291 Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting);
292 Node* get_key_start_from_aescrypt_object(Node* aescrypt_object); 292 Node* get_key_start_from_aescrypt_object(Node* aescrypt_object);
293 bool inline_encodeISOArray(); 293 bool inline_encodeISOArray();
294 bool inline_updateCRC32();
295 bool inline_updateBytesCRC32();
296 bool inline_updateByteBufferCRC32();
294 }; 297 };
295 298
296 299
297 //---------------------------make_vm_intrinsic---------------------------- 300 //---------------------------make_vm_intrinsic----------------------------
298 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) { 301 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
486 if (!UseAESIntrinsics) return NULL; 489 if (!UseAESIntrinsics) return NULL;
487 // these two require the predicated logic 490 // these two require the predicated logic
488 is_predicted = true; 491 is_predicted = true;
489 break; 492 break;
490 493
494 case vmIntrinsics::_updateCRC32:
495 case vmIntrinsics::_updateBytesCRC32:
496 case vmIntrinsics::_updateByteBufferCRC32:
497 if (!UseCRC32Intrinsics) return NULL;
498 break;
499
491 default: 500 default:
492 assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility"); 501 assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility");
493 assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?"); 502 assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?");
494 break; 503 break;
495 } 504 }
805 return inline_cipherBlockChaining_AESCrypt(intrinsic_id()); 814 return inline_cipherBlockChaining_AESCrypt(intrinsic_id());
806 815
807 case vmIntrinsics::_encodeISOArray: 816 case vmIntrinsics::_encodeISOArray:
808 return inline_encodeISOArray(); 817 return inline_encodeISOArray();
809 818
819 case vmIntrinsics::_updateCRC32:
820 return inline_updateCRC32();
821 case vmIntrinsics::_updateBytesCRC32:
822 return inline_updateBytesCRC32();
823 case vmIntrinsics::_updateByteBufferCRC32:
824 return inline_updateByteBufferCRC32();
825
810 default: 826 default:
811 // If you get here, it may be that someone has added a new intrinsic 827 // If you get here, it may be that someone has added a new intrinsic
812 // to the list in vmSymbols.hpp without implementing it here. 828 // to the list in vmSymbols.hpp without implementing it here.
813 #ifndef PRODUCT 829 #ifndef PRODUCT
814 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) { 830 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
882 return NULL; 898 return NULL;
883 } 899 }
884 900
885 IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN); 901 IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN);
886 902
887 Node* if_slow = _gvn.transform( new (C) IfTrueNode(iff) ); 903 Node* if_slow = _gvn.transform(new (C) IfTrueNode(iff));
888 if (if_slow == top()) { 904 if (if_slow == top()) {
889 // The slow branch is never taken. No need to build this guard. 905 // The slow branch is never taken. No need to build this guard.
890 return NULL; 906 return NULL;
891 } 907 }
892 908
893 if (region != NULL) 909 if (region != NULL)
894 region->add_req(if_slow); 910 region->add_req(if_slow);
895 911
896 Node* if_fast = _gvn.transform( new (C) IfFalseNode(iff) ); 912 Node* if_fast = _gvn.transform(new (C) IfFalseNode(iff));
897 set_control(if_fast); 913 set_control(if_fast);
898 914
899 return if_slow; 915 return if_slow;
900 } 916 }
901 917
910 Node* *pos_index) { 926 Node* *pos_index) {
911 if (stopped()) 927 if (stopped())
912 return NULL; // already stopped 928 return NULL; // already stopped
913 if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint] 929 if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
914 return NULL; // index is already adequately typed 930 return NULL; // index is already adequately typed
915 Node* cmp_lt = _gvn.transform( new (C) CmpINode(index, intcon(0)) ); 931 Node* cmp_lt = _gvn.transform(new (C) CmpINode(index, intcon(0)));
916 Node* bol_lt = _gvn.transform( new (C) BoolNode(cmp_lt, BoolTest::lt) ); 932 Node* bol_lt = _gvn.transform(new (C) BoolNode(cmp_lt, BoolTest::lt));
917 Node* is_neg = generate_guard(bol_lt, region, PROB_MIN); 933 Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);
918 if (is_neg != NULL && pos_index != NULL) { 934 if (is_neg != NULL && pos_index != NULL) {
919 // Emulate effect of Parse::adjust_map_after_if. 935 // Emulate effect of Parse::adjust_map_after_if.
920 Node* ccast = new (C) CastIINode(index, TypeInt::POS); 936 Node* ccast = new (C) CastIINode(index, TypeInt::POS);
921 ccast->set_req(0, control()); 937 ccast->set_req(0, control());
928 Node* *pos_index) { 944 Node* *pos_index) {
929 if (stopped()) 945 if (stopped())
930 return NULL; // already stopped 946 return NULL; // already stopped
931 if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint] 947 if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
932 return NULL; // index is already adequately typed 948 return NULL; // index is already adequately typed
933 Node* cmp_le = _gvn.transform( new (C) CmpINode(index, intcon(0)) ); 949 Node* cmp_le = _gvn.transform(new (C) CmpINode(index, intcon(0)));
934 BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le); 950 BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
935 Node* bol_le = _gvn.transform( new (C) BoolNode(cmp_le, le_or_eq) ); 951 Node* bol_le = _gvn.transform(new (C) BoolNode(cmp_le, le_or_eq));
936 Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN); 952 Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN);
937 if (is_notp != NULL && pos_index != NULL) { 953 if (is_notp != NULL && pos_index != NULL) {
938 // Emulate effect of Parse::adjust_map_after_if. 954 // Emulate effect of Parse::adjust_map_after_if.
939 Node* ccast = new (C) CastIINode(index, TypeInt::POS1); 955 Node* ccast = new (C) CastIINode(index, TypeInt::POS1);
940 ccast->set_req(0, control()); 956 ccast->set_req(0, control());
966 bool zero_offset = _gvn.type(offset) == TypeInt::ZERO; 982 bool zero_offset = _gvn.type(offset) == TypeInt::ZERO;
967 if (zero_offset && subseq_length->eqv_uncast(array_length)) 983 if (zero_offset && subseq_length->eqv_uncast(array_length))
968 return NULL; // common case of whole-array copy 984 return NULL; // common case of whole-array copy
969 Node* last = subseq_length; 985 Node* last = subseq_length;
970 if (!zero_offset) // last += offset 986 if (!zero_offset) // last += offset
971 last = _gvn.transform( new (C) AddINode(last, offset)); 987 last = _gvn.transform(new (C) AddINode(last, offset));
972 Node* cmp_lt = _gvn.transform( new (C) CmpUNode(array_length, last) ); 988 Node* cmp_lt = _gvn.transform(new (C) CmpUNode(array_length, last));
973 Node* bol_lt = _gvn.transform( new (C) BoolNode(cmp_lt, BoolTest::lt) ); 989 Node* bol_lt = _gvn.transform(new (C) BoolNode(cmp_lt, BoolTest::lt));
974 Node* is_over = generate_guard(bol_lt, region, PROB_MIN); 990 Node* is_over = generate_guard(bol_lt, region, PROB_MIN);
975 return is_over; 991 return is_over;
976 } 992 }
977 993
978 994
1149 1165
1150 // Get length of argument 1166 // Get length of argument
1151 Node* argument_cnt = load_String_length(no_ctrl, argument); 1167 Node* argument_cnt = load_String_length(no_ctrl, argument);
1152 1168
1153 // Check for receiver count != argument count 1169 // Check for receiver count != argument count
1154 Node* cmp = _gvn.transform( new(C) CmpINode(receiver_cnt, argument_cnt) ); 1170 Node* cmp = _gvn.transform(new(C) CmpINode(receiver_cnt, argument_cnt));
1155 Node* bol = _gvn.transform( new(C) BoolNode(cmp, BoolTest::ne) ); 1171 Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::ne));
1156 Node* if_ne = generate_slow_guard(bol, NULL); 1172 Node* if_ne = generate_slow_guard(bol, NULL);
1157 if (if_ne != NULL) { 1173 if (if_ne != NULL) {
1158 phi->init_req(4, intcon(0)); 1174 phi->init_req(4, intcon(0));
1159 region->init_req(4, if_ne); 1175 region->init_req(4, if_ne);
1160 } 1176 }
1256 1272
1257 Node* source = load_String_value(no_ctrl, string_object); 1273 Node* source = load_String_value(no_ctrl, string_object);
1258 Node* sourceOffset = load_String_offset(no_ctrl, string_object); 1274 Node* sourceOffset = load_String_offset(no_ctrl, string_object);
1259 Node* sourceCount = load_String_length(no_ctrl, string_object); 1275 Node* sourceCount = load_String_length(no_ctrl, string_object);
1260 1276
1261 Node* target = _gvn.transform( makecon(TypeOopPtr::make_from_constant(target_array, true)) ); 1277 Node* target = _gvn.transform( makecon(TypeOopPtr::make_from_constant(target_array, true)));
1262 jint target_length = target_array->length(); 1278 jint target_length = target_array->length();
1263 const TypeAry* target_array_type = TypeAry::make(TypeInt::CHAR, TypeInt::make(0, target_length, Type::WidenMin)); 1279 const TypeAry* target_array_type = TypeAry::make(TypeInt::CHAR, TypeInt::make(0, target_length, Type::WidenMin));
1264 const TypeAryPtr* target_type = TypeAryPtr::make(TypePtr::BotPTR, target_array_type, target_array->klass(), true, Type::OffsetBot); 1280 const TypeAryPtr* target_type = TypeAryPtr::make(TypePtr::BotPTR, target_array_type, target_array->klass(), true, Type::OffsetBot);
1265 1281
1266 IdealKit kit(this, false, true); 1282 IdealKit kit(this, false, true);
1363 1379
1364 // Get length of source string 1380 // Get length of source string
1365 Node* substr_cnt = load_String_length(no_ctrl, arg); 1381 Node* substr_cnt = load_String_length(no_ctrl, arg);
1366 1382
1367 // Check for substr count > string count 1383 // Check for substr count > string count
1368 Node* cmp = _gvn.transform( new(C) CmpINode(substr_cnt, source_cnt) ); 1384 Node* cmp = _gvn.transform(new(C) CmpINode(substr_cnt, source_cnt));
1369 Node* bol = _gvn.transform( new(C) BoolNode(cmp, BoolTest::gt) ); 1385 Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::gt));
1370 Node* if_gt = generate_slow_guard(bol, NULL); 1386 Node* if_gt = generate_slow_guard(bol, NULL);
1371 if (if_gt != NULL) { 1387 if (if_gt != NULL) {
1372 result_phi->init_req(2, intcon(-1)); 1388 result_phi->init_req(2, intcon(-1));
1373 result_rgn->init_req(2, if_gt); 1389 result_rgn->init_req(2, if_gt);
1374 } 1390 }
1375 1391
1376 if (!stopped()) { 1392 if (!stopped()) {
1377 // Check for substr count == 0 1393 // Check for substr count == 0
1378 cmp = _gvn.transform( new(C) CmpINode(substr_cnt, intcon(0)) ); 1394 cmp = _gvn.transform(new(C) CmpINode(substr_cnt, intcon(0)));
1379 bol = _gvn.transform( new(C) BoolNode(cmp, BoolTest::eq) ); 1395 bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::eq));
1380 Node* if_zero = generate_slow_guard(bol, NULL); 1396 Node* if_zero = generate_slow_guard(bol, NULL);
1381 if (if_zero != NULL) { 1397 if (if_zero != NULL) {
1382 result_phi->init_req(3, intcon(0)); 1398 result_phi->init_req(3, intcon(0));
1383 result_rgn->init_req(3, if_zero); 1399 result_rgn->init_req(3, if_zero);
1384 } 1400 }
1550 // Node for PI/4 constant 1566 // Node for PI/4 constant
1551 Node *pi4 = makecon(TypeD::make(pi_4)); 1567 Node *pi4 = makecon(TypeD::make(pi_4));
1552 // Check PI/4 : abs(arg) 1568 // Check PI/4 : abs(arg)
1553 Node *cmp = _gvn.transform(new (C) CmpDNode(pi4,abs)); 1569 Node *cmp = _gvn.transform(new (C) CmpDNode(pi4,abs));
1554 // Check: If PI/4 < abs(arg) then go slow 1570 // Check: If PI/4 < abs(arg) then go slow
1555 Node *bol = _gvn.transform( new (C) BoolNode( cmp, BoolTest::lt ) ); 1571 Node *bol = _gvn.transform(new (C) BoolNode( cmp, BoolTest::lt ));
1556 // Branch either way 1572 // Branch either way
1557 IfNode *iff = create_and_xform_if(control(),bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN); 1573 IfNode *iff = create_and_xform_if(control(),bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1558 set_control(opt_iff(r,iff)); 1574 set_control(opt_iff(r,iff));
1559 1575
1560 // Set fast path result 1576 // Set fast path result
1615 } else { 1631 } else {
1616 // If this inlining ever returned NaN in the past, we compile a call 1632 // If this inlining ever returned NaN in the past, we compile a call
1617 // to the runtime to properly handle corner cases 1633 // to the runtime to properly handle corner cases
1618 1634
1619 IfNode* iff = create_and_xform_if(control(), bolisnum, PROB_STATIC_FREQUENT, COUNT_UNKNOWN); 1635 IfNode* iff = create_and_xform_if(control(), bolisnum, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1620 Node* if_slow = _gvn.transform( new (C) IfFalseNode(iff) ); 1636 Node* if_slow = _gvn.transform(new (C) IfFalseNode(iff));
1621 Node* if_fast = _gvn.transform( new (C) IfTrueNode(iff) ); 1637 Node* if_fast = _gvn.transform(new (C) IfTrueNode(iff));
1622 1638
1623 if (!if_slow->is_top()) { 1639 if (!if_slow->is_top()) {
1624 RegionNode* result_region = new (C) RegionNode(3); 1640 RegionNode* result_region = new (C) RegionNode(3);
1625 PhiNode* result_val = new (C) PhiNode(result_region, Type::DOUBLE); 1641 PhiNode* result_val = new (C) PhiNode(result_region, Type::DOUBLE);
1626 1642
1702 // Node for 0 constant 1718 // Node for 0 constant
1703 Node *zeronode = makecon(TypeD::ZERO); 1719 Node *zeronode = makecon(TypeD::ZERO);
1704 // Check x:0 1720 // Check x:0
1705 Node *cmp = _gvn.transform(new (C) CmpDNode(x, zeronode)); 1721 Node *cmp = _gvn.transform(new (C) CmpDNode(x, zeronode));
1706 // Check: If (x<=0) then go complex path 1722 // Check: If (x<=0) then go complex path
1707 Node *bol1 = _gvn.transform( new (C) BoolNode( cmp, BoolTest::le ) ); 1723 Node *bol1 = _gvn.transform(new (C) BoolNode( cmp, BoolTest::le ));
1708 // Branch either way 1724 // Branch either way
1709 IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN); 1725 IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1710 // Fast path taken; set region slot 3 1726 // Fast path taken; set region slot 3
1711 Node *fast_taken = _gvn.transform( new (C) IfFalseNode(if1) ); 1727 Node *fast_taken = _gvn.transform(new (C) IfFalseNode(if1));
1712 r->init_req(3,fast_taken); // Capture fast-control 1728 r->init_req(3,fast_taken); // Capture fast-control
1713 1729
1714 // Fast path not-taken, i.e. slow path 1730 // Fast path not-taken, i.e. slow path
1715 Node *complex_path = _gvn.transform( new (C) IfTrueNode(if1) ); 1731 Node *complex_path = _gvn.transform(new (C) IfTrueNode(if1));
1716 1732
1717 // Set fast path result 1733 // Set fast path result
1718 Node *fast_result = _gvn.transform( new (C) PowDNode(C, control(), x, y) ); 1734 Node *fast_result = _gvn.transform(new (C) PowDNode(C, control(), x, y));
1719 phi->init_req(3, fast_result); 1735 phi->init_req(3, fast_result);
1720 1736
1721 // Complex path 1737 // Complex path
1722 // Build the second if node (if y is long) 1738 // Build the second if node (if y is long)
1723 // Node for (long)y 1739 // Node for (long)y
1724 Node *longy = _gvn.transform( new (C) ConvD2LNode(y)); 1740 Node *longy = _gvn.transform(new (C) ConvD2LNode(y));
1725 // Node for (double)((long) y) 1741 // Node for (double)((long) y)
1726 Node *doublelongy= _gvn.transform( new (C) ConvL2DNode(longy)); 1742 Node *doublelongy= _gvn.transform(new (C) ConvL2DNode(longy));
1727 // Check (double)((long) y) : y 1743 // Check (double)((long) y) : y
1728 Node *cmplongy= _gvn.transform(new (C) CmpDNode(doublelongy, y)); 1744 Node *cmplongy= _gvn.transform(new (C) CmpDNode(doublelongy, y));
1729 // Check if (y isn't long) then go to slow path 1745 // Check if (y isn't long) then go to slow path
1730 1746
1731 Node *bol2 = _gvn.transform( new (C) BoolNode( cmplongy, BoolTest::ne ) ); 1747 Node *bol2 = _gvn.transform(new (C) BoolNode( cmplongy, BoolTest::ne ));
1732 // Branch either way 1748 // Branch either way
1733 IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN); 1749 IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1734 Node* ylong_path = _gvn.transform( new (C) IfFalseNode(if2)); 1750 Node* ylong_path = _gvn.transform(new (C) IfFalseNode(if2));
1735 1751
1736 Node *slow_path = _gvn.transform( new (C) IfTrueNode(if2) ); 1752 Node *slow_path = _gvn.transform(new (C) IfTrueNode(if2));
1737 1753
1738 // Calculate DPow(abs(x), y)*(1 & (long)y) 1754 // Calculate DPow(abs(x), y)*(1 & (long)y)
1739 // Node for constant 1 1755 // Node for constant 1
1740 Node *conone = longcon(1); 1756 Node *conone = longcon(1);
1741 // 1& (long)y 1757 // 1& (long)y
1742 Node *signnode= _gvn.transform( new (C) AndLNode(conone, longy) ); 1758 Node *signnode= _gvn.transform(new (C) AndLNode(conone, longy));
1743 1759
1744 // A huge number is always even. Detect a huge number by checking 1760 // A huge number is always even. Detect a huge number by checking
1745 // if y + 1 == y and set integer to be tested for parity to 0. 1761 // if y + 1 == y and set integer to be tested for parity to 0.
1746 // Required for corner case: 1762 // Required for corner case:
1747 // (long)9.223372036854776E18 = max_jlong 1763 // (long)9.223372036854776E18 = max_jlong
1748 // (double)(long)9.223372036854776E18 = 9.223372036854776E18 1764 // (double)(long)9.223372036854776E18 = 9.223372036854776E18
1749 // max_jlong is odd but 9.223372036854776E18 is even 1765 // max_jlong is odd but 9.223372036854776E18 is even
1750 Node* yplus1 = _gvn.transform( new (C) AddDNode(y, makecon(TypeD::make(1)))); 1766 Node* yplus1 = _gvn.transform(new (C) AddDNode(y, makecon(TypeD::make(1))));
1751 Node *cmpyplus1= _gvn.transform(new (C) CmpDNode(yplus1, y)); 1767 Node *cmpyplus1= _gvn.transform(new (C) CmpDNode(yplus1, y));
1752 Node *bolyplus1 = _gvn.transform( new (C) BoolNode( cmpyplus1, BoolTest::eq ) ); 1768 Node *bolyplus1 = _gvn.transform(new (C) BoolNode( cmpyplus1, BoolTest::eq ));
1753 Node* correctedsign = NULL; 1769 Node* correctedsign = NULL;
1754 if (ConditionalMoveLimit != 0) { 1770 if (ConditionalMoveLimit != 0) {
1755 correctedsign = _gvn.transform( CMoveNode::make(C, NULL, bolyplus1, signnode, longcon(0), TypeLong::LONG)); 1771 correctedsign = _gvn.transform( CMoveNode::make(C, NULL, bolyplus1, signnode, longcon(0), TypeLong::LONG));
1756 } else { 1772 } else {
1757 IfNode *ifyplus1 = create_and_xform_if(ylong_path,bolyplus1, PROB_FAIR, COUNT_UNKNOWN); 1773 IfNode *ifyplus1 = create_and_xform_if(ylong_path,bolyplus1, PROB_FAIR, COUNT_UNKNOWN);
1758 RegionNode *r = new (C) RegionNode(3); 1774 RegionNode *r = new (C) RegionNode(3);
1759 Node *phi = new (C) PhiNode(r, TypeLong::LONG); 1775 Node *phi = new (C) PhiNode(r, TypeLong::LONG);
1760 r->init_req(1, _gvn.transform( new (C) IfFalseNode(ifyplus1))); 1776 r->init_req(1, _gvn.transform(new (C) IfFalseNode(ifyplus1)));
1761 r->init_req(2, _gvn.transform( new (C) IfTrueNode(ifyplus1))); 1777 r->init_req(2, _gvn.transform(new (C) IfTrueNode(ifyplus1)));
1762 phi->init_req(1, signnode); 1778 phi->init_req(1, signnode);
1763 phi->init_req(2, longcon(0)); 1779 phi->init_req(2, longcon(0));
1764 correctedsign = _gvn.transform(phi); 1780 correctedsign = _gvn.transform(phi);
1765 ylong_path = _gvn.transform(r); 1781 ylong_path = _gvn.transform(r);
1766 record_for_igvn(r); 1782 record_for_igvn(r);
1769 // zero node 1785 // zero node
1770 Node *conzero = longcon(0); 1786 Node *conzero = longcon(0);
1771 // Check (1&(long)y)==0? 1787 // Check (1&(long)y)==0?
1772 Node *cmpeq1 = _gvn.transform(new (C) CmpLNode(correctedsign, conzero)); 1788 Node *cmpeq1 = _gvn.transform(new (C) CmpLNode(correctedsign, conzero));
1773 // Check if (1&(long)y)!=0?, if so the result is negative 1789 // Check if (1&(long)y)!=0?, if so the result is negative
1774 Node *bol3 = _gvn.transform( new (C) BoolNode( cmpeq1, BoolTest::ne ) ); 1790 Node *bol3 = _gvn.transform(new (C) BoolNode( cmpeq1, BoolTest::ne ));
1775 // abs(x) 1791 // abs(x)
1776 Node *absx=_gvn.transform( new (C) AbsDNode(x)); 1792 Node *absx=_gvn.transform(new (C) AbsDNode(x));
1777 // abs(x)^y 1793 // abs(x)^y
1778 Node *absxpowy = _gvn.transform( new (C) PowDNode(C, control(), absx, y) ); 1794 Node *absxpowy = _gvn.transform(new (C) PowDNode(C, control(), absx, y));
1779 // -abs(x)^y 1795 // -abs(x)^y
1780 Node *negabsxpowy = _gvn.transform(new (C) NegDNode (absxpowy)); 1796 Node *negabsxpowy = _gvn.transform(new (C) NegDNode (absxpowy));
1781 // (1&(long)y)==1?-DPow(abs(x), y):DPow(abs(x), y) 1797 // (1&(long)y)==1?-DPow(abs(x), y):DPow(abs(x), y)
1782 Node *signresult = NULL; 1798 Node *signresult = NULL;
1783 if (ConditionalMoveLimit != 0) { 1799 if (ConditionalMoveLimit != 0) {
1784 signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE)); 1800 signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE));
1785 } else { 1801 } else {
1786 IfNode *ifyeven = create_and_xform_if(ylong_path,bol3, PROB_FAIR, COUNT_UNKNOWN); 1802 IfNode *ifyeven = create_and_xform_if(ylong_path,bol3, PROB_FAIR, COUNT_UNKNOWN);
1787 RegionNode *r = new (C) RegionNode(3); 1803 RegionNode *r = new (C) RegionNode(3);
1788 Node *phi = new (C) PhiNode(r, Type::DOUBLE); 1804 Node *phi = new (C) PhiNode(r, Type::DOUBLE);
1789 r->init_req(1, _gvn.transform( new (C) IfFalseNode(ifyeven))); 1805 r->init_req(1, _gvn.transform(new (C) IfFalseNode(ifyeven)));
1790 r->init_req(2, _gvn.transform( new (C) IfTrueNode(ifyeven))); 1806 r->init_req(2, _gvn.transform(new (C) IfTrueNode(ifyeven)));
1791 phi->init_req(1, absxpowy); 1807 phi->init_req(1, absxpowy);
1792 phi->init_req(2, negabsxpowy); 1808 phi->init_req(2, negabsxpowy);
1793 signresult = _gvn.transform(phi); 1809 signresult = _gvn.transform(phi);
1794 ylong_path = _gvn.transform(r); 1810 ylong_path = _gvn.transform(r);
1795 record_for_igvn(r); 1811 record_for_igvn(r);
1918 // and similar uses of System.arraycopy. 1934 // and similar uses of System.arraycopy.
1919 // First, compute the normalized version of CmpI(x, y). 1935 // First, compute the normalized version of CmpI(x, y).
1920 int cmp_op = Op_CmpI; 1936 int cmp_op = Op_CmpI;
1921 Node* xkey = xvalue; 1937 Node* xkey = xvalue;
1922 Node* ykey = yvalue; 1938 Node* ykey = yvalue;
1923 Node* ideal_cmpxy = _gvn.transform( new(C) CmpINode(xkey, ykey) ); 1939 Node* ideal_cmpxy = _gvn.transform(new(C) CmpINode(xkey, ykey));
1924 if (ideal_cmpxy->is_Cmp()) { 1940 if (ideal_cmpxy->is_Cmp()) {
1925 // E.g., if we have CmpI(length - offset, count), 1941 // E.g., if we have CmpI(length - offset, count),
1926 // it might idealize to CmpI(length, count + offset) 1942 // it might idealize to CmpI(length, count + offset)
1927 cmp_op = ideal_cmpxy->Opcode(); 1943 cmp_op = ideal_cmpxy->Opcode();
1928 xkey = ideal_cmpxy->in(1); 1944 xkey = ideal_cmpxy->in(1);
2011 Node* answer_if_false = NULL; 2027 Node* answer_if_false = NULL;
2012 switch (best_btest) { 2028 switch (best_btest) {
2013 default: 2029 default:
2014 if (cmpxy == NULL) 2030 if (cmpxy == NULL)
2015 cmpxy = ideal_cmpxy; 2031 cmpxy = ideal_cmpxy;
2016 best_bol = _gvn.transform( new(C) BoolNode(cmpxy, BoolTest::lt) ); 2032 best_bol = _gvn.transform(new(C) BoolNode(cmpxy, BoolTest::lt));
2017 // and fall through: 2033 // and fall through:
2018 case BoolTest::lt: // x < y 2034 case BoolTest::lt: // x < y
2019 case BoolTest::le: // x <= y 2035 case BoolTest::le: // x <= y
2020 answer_if_true = (want_max ? yvalue : xvalue); 2036 answer_if_true = (want_max ? yvalue : xvalue);
2021 answer_if_false = (want_max ? xvalue : yvalue); 2037 answer_if_false = (want_max ? xvalue : yvalue);
2071 if (base_type == NULL) { 2087 if (base_type == NULL) {
2072 // Unknown type. 2088 // Unknown type.
2073 return Type::AnyPtr; 2089 return Type::AnyPtr;
2074 } else if (base_type == TypePtr::NULL_PTR) { 2090 } else if (base_type == TypePtr::NULL_PTR) {
2075 // Since this is a NULL+long form, we have to switch to a rawptr. 2091 // Since this is a NULL+long form, we have to switch to a rawptr.
2076 base = _gvn.transform( new (C) CastX2PNode(offset) ); 2092 base = _gvn.transform(new (C) CastX2PNode(offset));
2077 offset = MakeConX(0); 2093 offset = MakeConX(0);
2078 return Type::RawPtr; 2094 return Type::RawPtr;
2079 } else if (base_type->base() == Type::RawPtr) { 2095 } else if (base_type->base() == Type::RawPtr) {
2080 return Type::RawPtr; 2096 return Type::RawPtr;
2081 } else if (base_type->isa_oopptr()) { 2097 } else if (base_type->isa_oopptr()) {
2465 val = dstore_rounding(val); 2481 val = dstore_rounding(val);
2466 break; 2482 break;
2467 case T_ADDRESS: 2483 case T_ADDRESS:
2468 // Repackage the long as a pointer. 2484 // Repackage the long as a pointer.
2469 val = ConvL2X(val); 2485 val = ConvL2X(val);
2470 val = _gvn.transform( new (C) CastX2PNode(val) ); 2486 val = _gvn.transform(new (C) CastX2PNode(val));
2471 break; 2487 break;
2472 } 2488 }
2473 2489
2474 if (type != T_OBJECT ) { 2490 if (type != T_OBJECT ) {
2475 (void) store_to_memory(control(), adr, val, type, adr_type, is_volatile); 2491 (void) store_to_memory(control(), adr, val, type, adr_type, is_volatile);
2773 } 2789 }
2774 2790
2775 // SCMemProjNodes represent the memory state of a LoadStore. Their 2791 // SCMemProjNodes represent the memory state of a LoadStore. Their
2776 // main role is to prevent LoadStore nodes from being optimized away 2792 // main role is to prevent LoadStore nodes from being optimized away
2777 // when their results aren't used. 2793 // when their results aren't used.
2778 Node* proj = _gvn.transform( new (C) SCMemProjNode(load_store)); 2794 Node* proj = _gvn.transform(new (C) SCMemProjNode(load_store));
2779 set_memory(proj, alias_idx); 2795 set_memory(proj, alias_idx);
2780 2796
2781 // Add the trailing membar surrounding the access 2797 // Add the trailing membar surrounding the access
2782 insert_mem_bar(Op_MemBarCPUOrder); 2798 insert_mem_bar(Op_MemBarCPUOrder);
2783 insert_mem_bar(Op_MemBarAcquire); 2799 insert_mem_bar(Op_MemBarAcquire);
3008 3024
3009 // (a) Receiving thread must be the current thread. 3025 // (a) Receiving thread must be the current thread.
3010 Node* rec_thr = argument(0); 3026 Node* rec_thr = argument(0);
3011 Node* tls_ptr = NULL; 3027 Node* tls_ptr = NULL;
3012 Node* cur_thr = generate_current_thread(tls_ptr); 3028 Node* cur_thr = generate_current_thread(tls_ptr);
3013 Node* cmp_thr = _gvn.transform( new (C) CmpPNode(cur_thr, rec_thr) ); 3029 Node* cmp_thr = _gvn.transform(new (C) CmpPNode(cur_thr, rec_thr));
3014 Node* bol_thr = _gvn.transform( new (C) BoolNode(cmp_thr, BoolTest::ne) ); 3030 Node* bol_thr = _gvn.transform(new (C) BoolNode(cmp_thr, BoolTest::ne));
3015 3031
3016 generate_slow_guard(bol_thr, slow_region); 3032 generate_slow_guard(bol_thr, slow_region);
3017 3033
3018 // (b) Interrupt bit on TLS must be false. 3034 // (b) Interrupt bit on TLS must be false.
3019 Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset())); 3035 Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
3020 Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS); 3036 Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS);
3021 p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset())); 3037 p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset()));
3022 3038
3023 // Set the control input on the field _interrupted read to prevent it floating up. 3039 // Set the control input on the field _interrupted read to prevent it floating up.
3024 Node* int_bit = make_load(control(), p, TypeInt::BOOL, T_INT); 3040 Node* int_bit = make_load(control(), p, TypeInt::BOOL, T_INT);
3025 Node* cmp_bit = _gvn.transform( new (C) CmpINode(int_bit, intcon(0)) ); 3041 Node* cmp_bit = _gvn.transform(new (C) CmpINode(int_bit, intcon(0)));
3026 Node* bol_bit = _gvn.transform( new (C) BoolNode(cmp_bit, BoolTest::ne) ); 3042 Node* bol_bit = _gvn.transform(new (C) BoolNode(cmp_bit, BoolTest::ne));
3027 3043
3028 IfNode* iff_bit = create_and_map_if(control(), bol_bit, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN); 3044 IfNode* iff_bit = create_and_map_if(control(), bol_bit, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN);
3029 3045
3030 // First fast path: if (!TLS._interrupted) return false; 3046 // First fast path: if (!TLS._interrupted) return false;
3031 Node* false_bit = _gvn.transform( new (C) IfFalseNode(iff_bit) ); 3047 Node* false_bit = _gvn.transform(new (C) IfFalseNode(iff_bit));
3032 result_rgn->init_req(no_int_result_path, false_bit); 3048 result_rgn->init_req(no_int_result_path, false_bit);
3033 result_val->init_req(no_int_result_path, intcon(0)); 3049 result_val->init_req(no_int_result_path, intcon(0));
3034 3050
3035 // drop through to next case 3051 // drop through to next case
3036 set_control( _gvn.transform(new (C) IfTrueNode(iff_bit)) ); 3052 set_control( _gvn.transform(new (C) IfTrueNode(iff_bit)));
3037 3053
3038 // (c) Or, if interrupt bit is set and clear_int is false, use 2nd fast path. 3054 // (c) Or, if interrupt bit is set and clear_int is false, use 2nd fast path.
3039 Node* clr_arg = argument(1); 3055 Node* clr_arg = argument(1);
3040 Node* cmp_arg = _gvn.transform( new (C) CmpINode(clr_arg, intcon(0)) ); 3056 Node* cmp_arg = _gvn.transform(new (C) CmpINode(clr_arg, intcon(0)));
3041 Node* bol_arg = _gvn.transform( new (C) BoolNode(cmp_arg, BoolTest::ne) ); 3057 Node* bol_arg = _gvn.transform(new (C) BoolNode(cmp_arg, BoolTest::ne));
3042 IfNode* iff_arg = create_and_map_if(control(), bol_arg, PROB_FAIR, COUNT_UNKNOWN); 3058 IfNode* iff_arg = create_and_map_if(control(), bol_arg, PROB_FAIR, COUNT_UNKNOWN);
3043 3059
3044 // Second fast path: ... else if (!clear_int) return true; 3060 // Second fast path: ... else if (!clear_int) return true;
3045 Node* false_arg = _gvn.transform( new (C) IfFalseNode(iff_arg) ); 3061 Node* false_arg = _gvn.transform(new (C) IfFalseNode(iff_arg));
3046 result_rgn->init_req(no_clear_result_path, false_arg); 3062 result_rgn->init_req(no_clear_result_path, false_arg);
3047 result_val->init_req(no_clear_result_path, intcon(1)); 3063 result_val->init_req(no_clear_result_path, intcon(1));
3048 3064
3049 // drop through to next case 3065 // drop through to next case
3050 set_control( _gvn.transform(new (C) IfTrueNode(iff_arg)) ); 3066 set_control( _gvn.transform(new (C) IfTrueNode(iff_arg)));
3051 3067
3052 // (d) Otherwise, go to the slow path. 3068 // (d) Otherwise, go to the slow path.
3053 slow_region->add_req(control()); 3069 slow_region->add_req(control());
3054 set_control( _gvn.transform(slow_region) ); 3070 set_control( _gvn.transform(slow_region));
3055 3071
3056 if (stopped()) { 3072 if (stopped()) {
3057 // There is no slow path. 3073 // There is no slow path.
3058 result_rgn->init_req(slow_result_path, top()); 3074 result_rgn->init_req(slow_result_path, top());
3059 result_val->init_req(slow_result_path, top()); 3075 result_val->init_req(slow_result_path, top());
3105 int null_path, 3121 int null_path,
3106 int offset) { 3122 int offset) {
3107 if (region == NULL) never_see_null = true; 3123 if (region == NULL) never_see_null = true;
3108 Node* p = basic_plus_adr(mirror, offset); 3124 Node* p = basic_plus_adr(mirror, offset);
3109 const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL; 3125 const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL;
3110 Node* kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type) ); 3126 Node* kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type));
3111 Node* null_ctl = top(); 3127 Node* null_ctl = top();
3112 kls = null_check_oop(kls, &null_ctl, never_see_null); 3128 kls = null_check_oop(kls, &null_ctl, never_see_null);
3113 if (region != NULL) { 3129 if (region != NULL) {
3114 // Set region->in(null_path) if the mirror is a primitive (e.g, int.class). 3130 // Set region->in(null_path) if the mirror is a primitive (e.g, int.class).
3115 region->init_req(null_path, null_ctl); 3131 region->init_req(null_path, null_ctl);
3127 // Like generate_guard, adds a new path onto the region. 3143 // Like generate_guard, adds a new path onto the region.
3128 Node* modp = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset())); 3144 Node* modp = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset()));
3129 Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT); 3145 Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT);
3130 Node* mask = intcon(modifier_mask); 3146 Node* mask = intcon(modifier_mask);
3131 Node* bits = intcon(modifier_bits); 3147 Node* bits = intcon(modifier_bits);
3132 Node* mbit = _gvn.transform( new (C) AndINode(mods, mask) ); 3148 Node* mbit = _gvn.transform(new (C) AndINode(mods, mask));
3133 Node* cmp = _gvn.transform( new (C) CmpINode(mbit, bits) ); 3149 Node* cmp = _gvn.transform(new (C) CmpINode(mbit, bits));
3134 Node* bol = _gvn.transform( new (C) BoolNode(cmp, BoolTest::ne) ); 3150 Node* bol = _gvn.transform(new (C) BoolNode(cmp, BoolTest::ne));
3135 return generate_fair_guard(bol, region); 3151 return generate_fair_guard(bol, region);
3136 } 3152 }
3137 Node* LibraryCallKit::generate_interface_guard(Node* kls, RegionNode* region) { 3153 Node* LibraryCallKit::generate_interface_guard(Node* kls, RegionNode* region) {
3138 return generate_access_flags_guard(kls, JVM_ACC_INTERFACE, 0, region); 3154 return generate_access_flags_guard(kls, JVM_ACC_INTERFACE, 0, region);
3139 } 3155 }
3280 if (generate_array_guard(kls, region) != NULL) 3296 if (generate_array_guard(kls, region) != NULL)
3281 // A guard was added. If the guard is taken, it was an array. 3297 // A guard was added. If the guard is taken, it was an array.
3282 phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror()))); 3298 phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror())));
3283 // If we fall through, it's a plain class. Get its _super. 3299 // If we fall through, it's a plain class. Get its _super.
3284 p = basic_plus_adr(kls, in_bytes(Klass::super_offset())); 3300 p = basic_plus_adr(kls, in_bytes(Klass::super_offset()));
3285 kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL) ); 3301 kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL));
3286 null_ctl = top(); 3302 null_ctl = top();
3287 kls = null_check_oop(kls, &null_ctl); 3303 kls = null_check_oop(kls, &null_ctl);
3288 if (null_ctl != top()) { 3304 if (null_ctl != top()) {
3289 // If the guard is taken, Object.superClass is null (both klass and mirror). 3305 // If the guard is taken, Object.superClass is null (both klass and mirror).
3290 region->add_req(null_ctl); 3306 region->add_req(null_ctl);
3393 // we must return true when they are identical primitives. 3409 // we must return true when they are identical primitives.
3394 // It is convenient to test this after the first null klass check. 3410 // It is convenient to test this after the first null klass check.
3395 set_control(region->in(_prim_0_path)); // go back to first null check 3411 set_control(region->in(_prim_0_path)); // go back to first null check
3396 if (!stopped()) { 3412 if (!stopped()) {
3397 // Since superc is primitive, make a guard for the superc==subc case. 3413 // Since superc is primitive, make a guard for the superc==subc case.
3398 Node* cmp_eq = _gvn.transform( new (C) CmpPNode(args[0], args[1]) ); 3414 Node* cmp_eq = _gvn.transform(new (C) CmpPNode(args[0], args[1]));
3399 Node* bol_eq = _gvn.transform( new (C) BoolNode(cmp_eq, BoolTest::eq) ); 3415 Node* bol_eq = _gvn.transform(new (C) BoolNode(cmp_eq, BoolTest::eq));
3400 generate_guard(bol_eq, region, PROB_FAIR); 3416 generate_guard(bol_eq, region, PROB_FAIR);
3401 if (region->req() == PATH_LIMIT+1) { 3417 if (region->req() == PATH_LIMIT+1) {
3402 // A guard was added. If the added guard is taken, superc==subc. 3418 // A guard was added. If the added guard is taken, superc==subc.
3403 region->swap_edges(PATH_LIMIT, _prim_same_path); 3419 region->swap_edges(PATH_LIMIT, _prim_same_path);
3404 region->del_req(PATH_LIMIT); 3420 region->del_req(PATH_LIMIT);
3459 // Now test the correct condition. 3475 // Now test the correct condition.
3460 jint nval = (obj_array 3476 jint nval = (obj_array
3461 ? ((jint)Klass::_lh_array_tag_type_value 3477 ? ((jint)Klass::_lh_array_tag_type_value
3462 << Klass::_lh_array_tag_shift) 3478 << Klass::_lh_array_tag_shift)
3463 : Klass::_lh_neutral_value); 3479 : Klass::_lh_neutral_value);
3464 Node* cmp = _gvn.transform( new(C) CmpINode(layout_val, intcon(nval)) ); 3480 Node* cmp = _gvn.transform(new(C) CmpINode(layout_val, intcon(nval)));
3465 BoolTest::mask btest = BoolTest::lt; // correct for testing is_[obj]array 3481 BoolTest::mask btest = BoolTest::lt; // correct for testing is_[obj]array
3466 // invert the test if we are looking for a non-array 3482 // invert the test if we are looking for a non-array
3467 if (not_array) btest = BoolTest(btest).negate(); 3483 if (not_array) btest = BoolTest(btest).negate();
3468 Node* bol = _gvn.transform( new(C) BoolNode(cmp, btest) ); 3484 Node* bol = _gvn.transform(new(C) BoolNode(cmp, btest));
3469 return generate_fair_guard(bol, region); 3485 return generate_fair_guard(bol, region);
3470 } 3486 }
3471 3487
3472 3488
3473 //-----------------------inline_native_newArray-------------------------- 3489 //-----------------------inline_native_newArray--------------------------
3523 result_mem->init_req(_normal_path, reset_memory()); 3539 result_mem->init_req(_normal_path, reset_memory());
3524 } 3540 }
3525 3541
3526 // Return the combined state. 3542 // Return the combined state.
3527 set_i_o( _gvn.transform(result_io) ); 3543 set_i_o( _gvn.transform(result_io) );
3528 set_all_memory( _gvn.transform(result_mem) ); 3544 set_all_memory( _gvn.transform(result_mem));
3529 3545
3530 C->set_has_split_ifs(true); // Has chance for split-if optimization 3546 C->set_has_split_ifs(true); // Has chance for split-if optimization
3531 set_result(result_reg, result_val); 3547 set_result(result_reg, result_val);
3532 return true; 3548 return true;
3533 } 3549 }
3676 3692
3677 // Compare the target method with the expected method (e.g., Object.hashCode). 3693 // Compare the target method with the expected method (e.g., Object.hashCode).
3678 const TypePtr* native_call_addr = TypeMetadataPtr::make(method); 3694 const TypePtr* native_call_addr = TypeMetadataPtr::make(method);
3679 3695
3680 Node* native_call = makecon(native_call_addr); 3696 Node* native_call = makecon(native_call_addr);
3681 Node* chk_native = _gvn.transform( new(C) CmpPNode(target_call, native_call) ); 3697 Node* chk_native = _gvn.transform(new(C) CmpPNode(target_call, native_call));
3682 Node* test_native = _gvn.transform( new(C) BoolNode(chk_native, BoolTest::ne) ); 3698 Node* test_native = _gvn.transform(new(C) BoolNode(chk_native, BoolTest::ne));
3683 3699
3684 return generate_slow_guard(test_native, slow_region); 3700 return generate_slow_guard(test_native, slow_region);
3685 } 3701 }
3686 3702
3687 //-----------------------generate_method_call---------------------------- 3703 //-----------------------generate_method_call----------------------------
3798 Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes()); 3814 Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
3799 Node* header = make_load(control(), header_addr, TypeX_X, TypeX_X->basic_type()); 3815 Node* header = make_load(control(), header_addr, TypeX_X, TypeX_X->basic_type());
3800 3816
3801 // Test the header to see if it is unlocked. 3817 // Test the header to see if it is unlocked.
3802 Node *lock_mask = _gvn.MakeConX(markOopDesc::biased_lock_mask_in_place); 3818 Node *lock_mask = _gvn.MakeConX(markOopDesc::biased_lock_mask_in_place);
3803 Node *lmasked_header = _gvn.transform( new (C) AndXNode(header, lock_mask) ); 3819 Node *lmasked_header = _gvn.transform(new (C) AndXNode(header, lock_mask));
3804 Node *unlocked_val = _gvn.MakeConX(markOopDesc::unlocked_value); 3820 Node *unlocked_val = _gvn.MakeConX(markOopDesc::unlocked_value);
3805 Node *chk_unlocked = _gvn.transform( new (C) CmpXNode( lmasked_header, unlocked_val)); 3821 Node *chk_unlocked = _gvn.transform(new (C) CmpXNode( lmasked_header, unlocked_val));
3806 Node *test_unlocked = _gvn.transform( new (C) BoolNode( chk_unlocked, BoolTest::ne) ); 3822 Node *test_unlocked = _gvn.transform(new (C) BoolNode( chk_unlocked, BoolTest::ne));
3807 3823
3808 generate_slow_guard(test_unlocked, slow_region); 3824 generate_slow_guard(test_unlocked, slow_region);
3809 3825
3810 // Get the hash value and check to see that it has been properly assigned. 3826 // Get the hash value and check to see that it has been properly assigned.
3811 // We depend on hash_mask being at most 32 bits and avoid the use of 3827 // We depend on hash_mask being at most 32 bits and avoid the use of
3812 // hash_mask_in_place because it could be larger than 32 bits in a 64-bit 3828 // hash_mask_in_place because it could be larger than 32 bits in a 64-bit
3813 // vm: see markOop.hpp. 3829 // vm: see markOop.hpp.
3814 Node *hash_mask = _gvn.intcon(markOopDesc::hash_mask); 3830 Node *hash_mask = _gvn.intcon(markOopDesc::hash_mask);
3815 Node *hash_shift = _gvn.intcon(markOopDesc::hash_shift); 3831 Node *hash_shift = _gvn.intcon(markOopDesc::hash_shift);
3816 Node *hshifted_header= _gvn.transform( new (C) URShiftXNode(header, hash_shift) ); 3832 Node *hshifted_header= _gvn.transform(new (C) URShiftXNode(header, hash_shift));
3817 // This hack lets the hash bits live anywhere in the mark object now, as long 3833 // This hack lets the hash bits live anywhere in the mark object now, as long
3818 // as the shift drops the relevant bits into the low 32 bits. Note that 3834 // as the shift drops the relevant bits into the low 32 bits. Note that
3819 // Java spec says that HashCode is an int so there's no point in capturing 3835 // Java spec says that HashCode is an int so there's no point in capturing
3820 // an 'X'-sized hashcode (32 in 32-bit build or 64 in 64-bit build). 3836 // an 'X'-sized hashcode (32 in 32-bit build or 64 in 64-bit build).
3821 hshifted_header = ConvX2I(hshifted_header); 3837 hshifted_header = ConvX2I(hshifted_header);
3822 Node *hash_val = _gvn.transform( new (C) AndINode(hshifted_header, hash_mask) ); 3838 Node *hash_val = _gvn.transform(new (C) AndINode(hshifted_header, hash_mask));
3823 3839
3824 Node *no_hash_val = _gvn.intcon(markOopDesc::no_hash); 3840 Node *no_hash_val = _gvn.intcon(markOopDesc::no_hash);
3825 Node *chk_assigned = _gvn.transform( new (C) CmpINode( hash_val, no_hash_val)); 3841 Node *chk_assigned = _gvn.transform(new (C) CmpINode( hash_val, no_hash_val));
3826 Node *test_assigned = _gvn.transform( new (C) BoolNode( chk_assigned, BoolTest::eq) ); 3842 Node *test_assigned = _gvn.transform(new (C) BoolNode( chk_assigned, BoolTest::eq));
3827 3843
3828 generate_slow_guard(test_assigned, slow_region); 3844 generate_slow_guard(test_assigned, slow_region);
3829 3845
3830 Node* init_mem = reset_memory(); 3846 Node* init_mem = reset_memory();
3831 // fill in the rest of the null path: 3847 // fill in the rest of the null path:
3852 result_mem ->set_req(_slow_path, reset_memory()); 3868 result_mem ->set_req(_slow_path, reset_memory());
3853 } 3869 }
3854 3870
3855 // Return the combined state. 3871 // Return the combined state.
3856 set_i_o( _gvn.transform(result_io) ); 3872 set_i_o( _gvn.transform(result_io) );
3857 set_all_memory( _gvn.transform(result_mem) ); 3873 set_all_memory( _gvn.transform(result_mem));
3858 3874
3859 set_result(result_reg, result_val); 3875 set_result(result_reg, result_val);
3860 return true; 3876 return true;
3861 } 3877 }
3862 3878
3980 // NaN case is less traveled, which makes all the difference. 3996 // NaN case is less traveled, which makes all the difference.
3981 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN); 3997 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
3982 Node *opt_isnan = _gvn.transform(ifisnan); 3998 Node *opt_isnan = _gvn.transform(ifisnan);
3983 assert( opt_isnan->is_If(), "Expect an IfNode"); 3999 assert( opt_isnan->is_If(), "Expect an IfNode");
3984 IfNode *opt_ifisnan = (IfNode*)opt_isnan; 4000 IfNode *opt_ifisnan = (IfNode*)opt_isnan;
3985 Node *iftrue = _gvn.transform( new (C) IfTrueNode(opt_ifisnan) ); 4001 Node *iftrue = _gvn.transform(new (C) IfTrueNode(opt_ifisnan));
3986 4002
3987 set_control(iftrue); 4003 set_control(iftrue);
3988 4004
3989 static const jlong nan_bits = CONST64(0x7ff8000000000000); 4005 static const jlong nan_bits = CONST64(0x7ff8000000000000);
3990 Node *slow_result = longcon(nan_bits); // return NaN 4006 Node *slow_result = longcon(nan_bits); // return NaN
4021 // NaN case is less traveled, which makes all the difference. 4037 // NaN case is less traveled, which makes all the difference.
4022 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN); 4038 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
4023 Node *opt_isnan = _gvn.transform(ifisnan); 4039 Node *opt_isnan = _gvn.transform(ifisnan);
4024 assert( opt_isnan->is_If(), "Expect an IfNode"); 4040 assert( opt_isnan->is_If(), "Expect an IfNode");
4025 IfNode *opt_ifisnan = (IfNode*)opt_isnan; 4041 IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4026 Node *iftrue = _gvn.transform( new (C) IfTrueNode(opt_ifisnan) ); 4042 Node *iftrue = _gvn.transform(new (C) IfTrueNode(opt_ifisnan));
4027 4043
4028 set_control(iftrue); 4044 set_control(iftrue);
4029 4045
4030 static const jint nan_bits = 0x7fc00000; 4046 static const jint nan_bits = 0x7fc00000;
4031 Node *slow_result = makecon(TypeInt::make(nan_bits)); // return NaN 4047 Node *slow_result = makecon(TypeInt::make(nan_bits)); // return NaN
4150 src = basic_plus_adr(src, base_off); 4166 src = basic_plus_adr(src, base_off);
4151 dest = basic_plus_adr(dest, base_off); 4167 dest = basic_plus_adr(dest, base_off);
4152 4168
4153 // Compute the length also, if needed: 4169 // Compute the length also, if needed:
4154 Node* countx = size; 4170 Node* countx = size;
4155 countx = _gvn.transform( new (C) SubXNode(countx, MakeConX(base_off)) ); 4171 countx = _gvn.transform(new (C) SubXNode(countx, MakeConX(base_off)));
4156 countx = _gvn.transform( new (C) URShiftXNode(countx, intcon(LogBytesPerLong) )); 4172 countx = _gvn.transform(new (C) URShiftXNode(countx, intcon(LogBytesPerLong) ));
4157 4173
4158 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; 4174 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4159 bool disjoint_bases = true; 4175 bool disjoint_bases = true;
4160 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases, 4176 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
4161 src, NULL, dest, NULL, countx, 4177 src, NULL, dest, NULL, countx,
4355 result_i_o ->set_req(_slow_path, i_o()); 4371 result_i_o ->set_req(_slow_path, i_o());
4356 result_mem ->set_req(_slow_path, reset_memory()); 4372 result_mem ->set_req(_slow_path, reset_memory());
4357 } 4373 }
4358 4374
4359 // Return the combined state. 4375 // Return the combined state.
4360 set_control( _gvn.transform(result_reg) ); 4376 set_control( _gvn.transform(result_reg));
4361 set_i_o( _gvn.transform(result_i_o) ); 4377 set_i_o( _gvn.transform(result_i_o));
4362 set_all_memory( _gvn.transform(result_mem) ); 4378 set_all_memory( _gvn.transform(result_mem));
4363 } // original reexecute is set back here 4379 } // original reexecute is set back here
4364 4380
4365 set_result(_gvn.transform(result_val)); 4381 set_result(_gvn.transform(result_val));
4366 return true; 4382 return true;
4367 } 4383 }
4682 // We have to initialize the *uncopied* part of the array to zero. 4698 // We have to initialize the *uncopied* part of the array to zero.
4683 // The copy destination is the slice dest[off..off+len]. The other slices 4699 // The copy destination is the slice dest[off..off+len]. The other slices
4684 // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length]. 4700 // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
4685 Node* dest_size = alloc->in(AllocateNode::AllocSize); 4701 Node* dest_size = alloc->in(AllocateNode::AllocSize);
4686 Node* dest_length = alloc->in(AllocateNode::ALength); 4702 Node* dest_length = alloc->in(AllocateNode::ALength);
4687 Node* dest_tail = _gvn.transform( new(C) AddINode(dest_offset, 4703 Node* dest_tail = _gvn.transform(new(C) AddINode(dest_offset,
4688 copy_length) ); 4704 copy_length));
4689 4705
4690 // If there is a head section that needs zeroing, do it now. 4706 // If there is a head section that needs zeroing, do it now.
4691 if (find_int_con(dest_offset, -1) != 0) { 4707 if (find_int_con(dest_offset, -1) != 0) {
4692 generate_clear_array(adr_type, dest, basic_elem_type, 4708 generate_clear_array(adr_type, dest, basic_elem_type,
4693 intcon(0), dest_offset, 4709 intcon(0), dest_offset,
4699 // There are two wins: Avoid generating the ClearArray 4715 // There are two wins: Avoid generating the ClearArray
4700 // with its attendant messy index arithmetic, and upgrade 4716 // with its attendant messy index arithmetic, and upgrade
4701 // the copy to a more hardware-friendly word size of 64 bits. 4717 // the copy to a more hardware-friendly word size of 64 bits.
4702 Node* tail_ctl = NULL; 4718 Node* tail_ctl = NULL;
4703 if (!stopped() && !dest_tail->eqv_uncast(dest_length)) { 4719 if (!stopped() && !dest_tail->eqv_uncast(dest_length)) {
4704 Node* cmp_lt = _gvn.transform( new(C) CmpINode(dest_tail, dest_length) ); 4720 Node* cmp_lt = _gvn.transform(new(C) CmpINode(dest_tail, dest_length));
4705 Node* bol_lt = _gvn.transform( new(C) BoolNode(cmp_lt, BoolTest::lt) ); 4721 Node* bol_lt = _gvn.transform(new(C) BoolNode(cmp_lt, BoolTest::lt));
4706 tail_ctl = generate_slow_guard(bol_lt, NULL); 4722 tail_ctl = generate_slow_guard(bol_lt, NULL);
4707 assert(tail_ctl != NULL || !stopped(), "must be an outcome"); 4723 assert(tail_ctl != NULL || !stopped(), "must be an outcome");
4708 } 4724 }
4709 4725
4710 // At this point, let's assume there is no tail. 4726 // At this point, let's assume there is no tail.
4743 generate_clear_array(adr_type, dest, basic_elem_type, 4759 generate_clear_array(adr_type, dest, basic_elem_type,
4744 dest_tail, NULL, 4760 dest_tail, NULL,
4745 dest_size); 4761 dest_size);
4746 done_ctl->init_req(2, control()); 4762 done_ctl->init_req(2, control());
4747 done_mem->init_req(2, memory(adr_type)); 4763 done_mem->init_req(2, memory(adr_type));
4748 set_control( _gvn.transform(done_ctl) ); 4764 set_control( _gvn.transform(done_ctl));
4749 set_memory( _gvn.transform(done_mem), adr_type ); 4765 set_memory( _gvn.transform(done_mem), adr_type );
4750 } 4766 }
4751 } 4767 }
4752 } 4768 }
4753 4769
4830 set_control(checked_control); 4846 set_control(checked_control);
4831 if (!stopped()) { 4847 if (!stopped()) {
4832 // Clean up after the checked call. 4848 // Clean up after the checked call.
4833 // The returned value is either 0 or -1^K, 4849 // The returned value is either 0 or -1^K,
4834 // where K = number of partially transferred array elements. 4850 // where K = number of partially transferred array elements.
4835 Node* cmp = _gvn.transform( new(C) CmpINode(checked_value, intcon(0)) ); 4851 Node* cmp = _gvn.transform(new(C) CmpINode(checked_value, intcon(0)));
4836 Node* bol = _gvn.transform( new(C) BoolNode(cmp, BoolTest::eq) ); 4852 Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::eq));
4837 IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN); 4853 IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
4838 4854
4839 // If it is 0, we are done, so transfer to the end. 4855 // If it is 0, we are done, so transfer to the end.
4840 Node* checks_done = _gvn.transform( new(C) IfTrueNode(iff) ); 4856 Node* checks_done = _gvn.transform(new(C) IfTrueNode(iff));
4841 result_region->init_req(checked_path, checks_done); 4857 result_region->init_req(checked_path, checks_done);
4842 result_i_o ->init_req(checked_path, checked_i_o); 4858 result_i_o ->init_req(checked_path, checked_i_o);
4843 result_memory->init_req(checked_path, checked_mem); 4859 result_memory->init_req(checked_path, checked_mem);
4844 4860
4845 // If it is not zero, merge into the slow call. 4861 // If it is not zero, merge into the slow call.
4846 set_control( _gvn.transform( new(C) IfFalseNode(iff) )); 4862 set_control( _gvn.transform(new(C) IfFalseNode(iff) ));
4847 RegionNode* slow_reg2 = new(C) RegionNode(3); 4863 RegionNode* slow_reg2 = new(C) RegionNode(3);
4848 PhiNode* slow_i_o2 = new(C) PhiNode(slow_reg2, Type::ABIO); 4864 PhiNode* slow_i_o2 = new(C) PhiNode(slow_reg2, Type::ABIO);
4849 PhiNode* slow_mem2 = new(C) PhiNode(slow_reg2, Type::MEMORY, adr_type); 4865 PhiNode* slow_mem2 = new(C) PhiNode(slow_reg2, Type::MEMORY, adr_type);
4850 record_for_igvn(slow_reg2); 4866 record_for_igvn(slow_reg2);
4851 slow_reg2 ->init_req(1, slow_control); 4867 slow_reg2 ->init_req(1, slow_control);
4864 // This can cause double writes, but that's OK since dest is brand new. 4880 // This can cause double writes, but that's OK since dest is brand new.
4865 // So we ignore the low 31 bits of the value returned from the stub. 4881 // So we ignore the low 31 bits of the value returned from the stub.
4866 } else { 4882 } else {
4867 // We must continue the copy exactly where it failed, or else 4883 // We must continue the copy exactly where it failed, or else
4868 // another thread might see the wrong number of writes to dest. 4884 // another thread might see the wrong number of writes to dest.
4869 Node* checked_offset = _gvn.transform( new(C) XorINode(checked_value, intcon(-1)) ); 4885 Node* checked_offset = _gvn.transform(new(C) XorINode(checked_value, intcon(-1)));
4870 Node* slow_offset = new(C) PhiNode(slow_reg2, TypeInt::INT); 4886 Node* slow_offset = new(C) PhiNode(slow_reg2, TypeInt::INT);
4871 slow_offset->init_req(1, intcon(0)); 4887 slow_offset->init_req(1, intcon(0));
4872 slow_offset->init_req(2, checked_offset); 4888 slow_offset->init_req(2, checked_offset);
4873 slow_offset = _gvn.transform(slow_offset); 4889 slow_offset = _gvn.transform(slow_offset);
4874 4890
4875 // Adjust the arguments by the conditionally incoming offset. 4891 // Adjust the arguments by the conditionally incoming offset.
4876 Node* src_off_plus = _gvn.transform( new(C) AddINode(src_offset, slow_offset) ); 4892 Node* src_off_plus = _gvn.transform(new(C) AddINode(src_offset, slow_offset));
4877 Node* dest_off_plus = _gvn.transform( new(C) AddINode(dest_offset, slow_offset) ); 4893 Node* dest_off_plus = _gvn.transform(new(C) AddINode(dest_offset, slow_offset));
4878 Node* length_minus = _gvn.transform( new(C) SubINode(copy_length, slow_offset) ); 4894 Node* length_minus = _gvn.transform(new(C) SubINode(copy_length, slow_offset));
4879 4895
4880 // Tweak the node variables to adjust the code produced below: 4896 // Tweak the node variables to adjust the code produced below:
4881 src_offset = src_off_plus; 4897 src_offset = src_off_plus;
4882 dest_offset = dest_off_plus; 4898 dest_offset = dest_off_plus;
4883 copy_length = length_minus; 4899 copy_length = length_minus;
4912 if (result_region->in(i) == NULL) 4928 if (result_region->in(i) == NULL)
4913 result_region->init_req(i, top()); 4929 result_region->init_req(i, top());
4914 } 4930 }
4915 4931
4916 // Finished; return the combined state. 4932 // Finished; return the combined state.
4917 set_control( _gvn.transform(result_region) ); 4933 set_control( _gvn.transform(result_region));
4918 set_i_o( _gvn.transform(result_i_o) ); 4934 set_i_o( _gvn.transform(result_i_o) );
4919 set_memory( _gvn.transform(result_memory), adr_type ); 4935 set_memory( _gvn.transform(result_memory), adr_type );
4920 4936
4921 // The memory edges above are precise in order to model effects around 4937 // The memory edges above are precise in order to model effects around
4922 // array copies accurately to allow value numbering of field loads around 4938 // array copies accurately to allow value numbering of field loads around
5094 // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8) 5110 // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
5095 intptr_t end_base = abase + (slice_idx_con << scale); 5111 intptr_t end_base = abase + (slice_idx_con << scale);
5096 int end_round = (-1 << scale) & (BytesPerLong - 1); 5112 int end_round = (-1 << scale) & (BytesPerLong - 1);
5097 Node* end = ConvI2X(slice_len); 5113 Node* end = ConvI2X(slice_len);
5098 if (scale != 0) 5114 if (scale != 0)
5099 end = _gvn.transform( new(C) LShiftXNode(end, intcon(scale) )); 5115 end = _gvn.transform(new(C) LShiftXNode(end, intcon(scale) ));
5100 end_base += end_round; 5116 end_base += end_round;
5101 end = _gvn.transform( new(C) AddXNode(end, MakeConX(end_base)) ); 5117 end = _gvn.transform(new(C) AddXNode(end, MakeConX(end_base)));
5102 end = _gvn.transform( new(C) AndXNode(end, MakeConX(~end_round)) ); 5118 end = _gvn.transform(new(C) AndXNode(end, MakeConX(~end_round)));
5103 mem = ClearArrayNode::clear_memory(control(), mem, dest, 5119 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5104 start_con, end, &_gvn); 5120 start_con, end, &_gvn);
5105 } else if (start_con < 0 && dest_size != top()) { 5121 } else if (start_con < 0 && dest_size != top()) {
5106 // Non-constant start, pre-rounded end after the tail of the array. 5122 // Non-constant start, pre-rounded end after the tail of the array.
5107 // This is almost certainly a "round-to-end" operation. 5123 // This is almost certainly a "round-to-end" operation.
5108 Node* start = slice_idx; 5124 Node* start = slice_idx;
5109 start = ConvI2X(start); 5125 start = ConvI2X(start);
5110 if (scale != 0) 5126 if (scale != 0)
5111 start = _gvn.transform( new(C) LShiftXNode( start, intcon(scale) )); 5127 start = _gvn.transform(new(C) LShiftXNode( start, intcon(scale) ));
5112 start = _gvn.transform( new(C) AddXNode(start, MakeConX(abase)) ); 5128 start = _gvn.transform(new(C) AddXNode(start, MakeConX(abase)));
5113 if ((bump_bit | clear_low) != 0) { 5129 if ((bump_bit | clear_low) != 0) {
5114 int to_clear = (bump_bit | clear_low); 5130 int to_clear = (bump_bit | clear_low);
5115 // Align up mod 8, then store a jint zero unconditionally 5131 // Align up mod 8, then store a jint zero unconditionally
5116 // just before the mod-8 boundary. 5132 // just before the mod-8 boundary.
5117 if (((abase + bump_bit) & ~to_clear) - bump_bit 5133 if (((abase + bump_bit) & ~to_clear) - bump_bit
5118 < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) { 5134 < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
5119 bump_bit = 0; 5135 bump_bit = 0;
5120 assert((abase & to_clear) == 0, "array base must be long-aligned"); 5136 assert((abase & to_clear) == 0, "array base must be long-aligned");
5121 } else { 5137 } else {
5122 // Bump 'start' up to (or past) the next jint boundary: 5138 // Bump 'start' up to (or past) the next jint boundary:
5123 start = _gvn.transform( new(C) AddXNode(start, MakeConX(bump_bit)) ); 5139 start = _gvn.transform(new(C) AddXNode(start, MakeConX(bump_bit)));
5124 assert((abase & clear_low) == 0, "array base must be int-aligned"); 5140 assert((abase & clear_low) == 0, "array base must be int-aligned");
5125 } 5141 }
5126 // Round bumped 'start' down to jlong boundary in body of array. 5142 // Round bumped 'start' down to jlong boundary in body of array.
5127 start = _gvn.transform( new(C) AndXNode(start, MakeConX(~to_clear)) ); 5143 start = _gvn.transform(new(C) AndXNode(start, MakeConX(~to_clear)));
5128 if (bump_bit != 0) { 5144 if (bump_bit != 0) {
5129 // Store a zero to the immediately preceding jint: 5145 // Store a zero to the immediately preceding jint:
5130 Node* x1 = _gvn.transform( new(C) AddXNode(start, MakeConX(-bump_bit)) ); 5146 Node* x1 = _gvn.transform(new(C) AddXNode(start, MakeConX(-bump_bit)));
5131 Node* p1 = basic_plus_adr(dest, x1); 5147 Node* p1 = basic_plus_adr(dest, x1);
5132 mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT); 5148 mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT);
5133 mem = _gvn.transform(mem); 5149 mem = _gvn.transform(mem);
5134 } 5150 }
5135 } 5151 }
5192 5208
5193 // Do this copy by giant steps. 5209 // Do this copy by giant steps.
5194 Node* sptr = basic_plus_adr(src, src_off); 5210 Node* sptr = basic_plus_adr(src, src_off);
5195 Node* dptr = basic_plus_adr(dest, dest_off); 5211 Node* dptr = basic_plus_adr(dest, dest_off);
5196 Node* countx = dest_size; 5212 Node* countx = dest_size;
5197 countx = _gvn.transform( new (C) SubXNode(countx, MakeConX(dest_off)) ); 5213 countx = _gvn.transform(new (C) SubXNode(countx, MakeConX(dest_off)));
5198 countx = _gvn.transform( new (C) URShiftXNode(countx, intcon(LogBytesPerLong)) ); 5214 countx = _gvn.transform(new (C) URShiftXNode(countx, intcon(LogBytesPerLong)));
5199 5215
5200 bool disjoint_bases = true; // since alloc != NULL 5216 bool disjoint_bases = true; // since alloc != NULL
5201 generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases, 5217 generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
5202 sptr, NULL, dptr, NULL, countx, dest_uninitialized); 5218 sptr, NULL, dptr, NULL, countx, dest_uninitialized);
5203 5219
5358 set_memory(res_mem, mtype); 5374 set_memory(res_mem, mtype);
5359 set_result(enc); 5375 set_result(enc);
5360 return true; 5376 return true;
5361 } 5377 }
5362 5378
5379 /**
5380 * Calculate CRC32 for byte.
5381 * int java.util.zip.CRC32.update(int crc, int b)
5382 */
5383 bool LibraryCallKit::inline_updateCRC32() {
5384 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5385 assert(callee()->signature()->size() == 2, "update has 2 parameters");
5386 // no receiver since it is static method
5387 Node* crc = argument(0); // type: int
5388 Node* b = argument(1); // type: int
5389
5390 /*
5391 * int c = ~ crc;
5392 * b = timesXtoThe32[(b ^ c) & 0xFF];
5393 * b = b ^ (c >>> 8);
5394 * crc = ~b;
5395 */
5396
5397 Node* M1 = intcon(-1);
5398 crc = _gvn.transform(new (C) XorINode(crc, M1));
5399 Node* result = _gvn.transform(new (C) XorINode(crc, b));
5400 result = _gvn.transform(new (C) AndINode(result, intcon(0xFF)));
5401
5402 Node* base = makecon(TypeRawPtr::make(StubRoutines::crc_table_addr()));
5403 Node* offset = _gvn.transform(new (C) LShiftINode(result, intcon(0x2)));
5404 Node* adr = basic_plus_adr(top(), base, ConvI2X(offset));
5405 result = make_load(control(), adr, TypeInt::INT, T_INT);
5406
5407 crc = _gvn.transform(new (C) URShiftINode(crc, intcon(8)));
5408 result = _gvn.transform(new (C) XorINode(crc, result));
5409 result = _gvn.transform(new (C) XorINode(result, M1));
5410 set_result(result);
5411 return true;
5412 }
5413
5414 /**
5415 * Calculate CRC32 for byte[] array.
5416 * int java.util.zip.CRC32.updateBytes(int crc, byte[] buf, int off, int len)
5417 */
5418 bool LibraryCallKit::inline_updateBytesCRC32() {
5419 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5420 assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters");
5421 // no receiver since it is static method
5422 Node* crc = argument(0); // type: int
5423 Node* src = argument(1); // type: oop
5424 Node* offset = argument(2); // type: int
5425 Node* length = argument(3); // type: int
5426
5427 const Type* src_type = src->Value(&_gvn);
5428 const TypeAryPtr* top_src = src_type->isa_aryptr();
5429 if (top_src == NULL || top_src->klass() == NULL) {
5430 // failed array check
5431 return false;
5432 }
5433
5434 // Figure out the size and type of the elements we will be copying.
5435 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5436 if (src_elem != T_BYTE) {
5437 return false;
5438 }
5439
5440 // 'src_start' points to src array + scaled offset
5441 Node* src_start = array_element_address(src, offset, src_elem);
5442
5443 // We assume that range check is done by caller.
5444 // TODO: generate range check (offset+length < src.length) in debug VM.
5445
5446 // Call the stub.
5447 address stubAddr = StubRoutines::updateBytesCRC32();
5448 const char *stubName = "updateBytesCRC32";
5449
5450 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
5451 stubAddr, stubName, TypePtr::BOTTOM,
5452 crc, src_start, length);
5453 Node* result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
5454 set_result(result);
5455 return true;
5456 }
5457
5458 /**
5459 * Calculate CRC32 for ByteBuffer.
5460 * int java.util.zip.CRC32.updateByteBuffer(int crc, long buf, int off, int len)
5461 */
5462 bool LibraryCallKit::inline_updateByteBufferCRC32() {
5463 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5464 assert(callee()->signature()->size() == 5, "updateByteBuffer has 4 parameters and one is long");
5465 // no receiver since it is static method
5466 Node* crc = argument(0); // type: int
5467 Node* src = argument(1); // type: long
5468 Node* offset = argument(3); // type: int
5469 Node* length = argument(4); // type: int
5470
5471 src = ConvL2X(src); // adjust Java long to machine word
5472 Node* base = _gvn.transform(new (C) CastX2PNode(src));
5473 offset = ConvI2X(offset);
5474
5475 // 'src_start' points to src array + scaled offset
5476 Node* src_start = basic_plus_adr(top(), base, offset);
5477
5478 // Call the stub.
5479 address stubAddr = StubRoutines::updateBytesCRC32();
5480 const char *stubName = "updateBytesCRC32";
5481
5482 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
5483 stubAddr, stubName, TypePtr::BOTTOM,
5484 crc, src_start, length);
5485 Node* result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
5486 set_result(result);
5487 return true;
5488 }
5489
5363 //----------------------------inline_reference_get---------------------------- 5490 //----------------------------inline_reference_get----------------------------
5364 // public T java.lang.ref.Reference.get(); 5491 // public T java.lang.ref.Reference.get();
5365 bool LibraryCallKit::inline_reference_get() { 5492 bool LibraryCallKit::inline_reference_get() {
5366 const int referent_offset = java_lang_ref_Reference::referent_offset; 5493 const int referent_offset = java_lang_ref_Reference::referent_offset;
5367 guarantee(referent_offset > 0, "should have already been set"); 5494 guarantee(referent_offset > 0, "should have already been set");