Mercurial > hg > truffle
comparison src/share/vm/opto/library_call.cpp @ 2324:0ac769a57c64
6627983: G1: Bad oop deference during marking
Summary: Bulk zeroing reduction didn't work with G1, because arraycopy would call pre-barriers on uninitialized oops. The solution is to have version of arraycopy stubs that don't have pre-barriers. Also refactored arraycopy stubs generation on SPARC to be more readable and reduced the number of stubs necessary in some cases.
Reviewed-by: jrose, kvn, never
author | iveresov |
---|---|
date | Tue, 01 Mar 2011 14:56:48 -0800 |
parents | 41d4973cf100 |
children | b099aaf51bf8 9dc311b8473e |
comparison
equal
deleted
inserted
replaced
2323:bc6b27fb3568 | 2324:0ac769a57c64 |
---|---|
95 Node* generate_limit_guard(Node* offset, Node* subseq_length, | 95 Node* generate_limit_guard(Node* offset, Node* subseq_length, |
96 Node* array_length, | 96 Node* array_length, |
97 RegionNode* region); | 97 RegionNode* region); |
98 Node* generate_current_thread(Node* &tls_output); | 98 Node* generate_current_thread(Node* &tls_output); |
99 address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset, | 99 address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset, |
100 bool disjoint_bases, const char* &name); | 100 bool disjoint_bases, const char* &name, bool dest_uninitialized); |
101 Node* load_mirror_from_klass(Node* klass); | 101 Node* load_mirror_from_klass(Node* klass); |
102 Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null, | 102 Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null, |
103 int nargs, | 103 int nargs, |
104 RegionNode* region, int null_path, | 104 RegionNode* region, int null_path, |
105 int offset); | 105 int offset); |
210 bool generate_block_arraycopy(const TypePtr* adr_type, | 210 bool generate_block_arraycopy(const TypePtr* adr_type, |
211 BasicType basic_elem_type, | 211 BasicType basic_elem_type, |
212 AllocateNode* alloc, | 212 AllocateNode* alloc, |
213 Node* src, Node* src_offset, | 213 Node* src, Node* src_offset, |
214 Node* dest, Node* dest_offset, | 214 Node* dest, Node* dest_offset, |
215 Node* dest_size); | 215 Node* dest_size, bool dest_uninitialized); |
216 void generate_slow_arraycopy(const TypePtr* adr_type, | 216 void generate_slow_arraycopy(const TypePtr* adr_type, |
217 Node* src, Node* src_offset, | 217 Node* src, Node* src_offset, |
218 Node* dest, Node* dest_offset, | 218 Node* dest, Node* dest_offset, |
219 Node* copy_length); | 219 Node* copy_length, bool dest_uninitialized); |
220 Node* generate_checkcast_arraycopy(const TypePtr* adr_type, | 220 Node* generate_checkcast_arraycopy(const TypePtr* adr_type, |
221 Node* dest_elem_klass, | 221 Node* dest_elem_klass, |
222 Node* src, Node* src_offset, | 222 Node* src, Node* src_offset, |
223 Node* dest, Node* dest_offset, | 223 Node* dest, Node* dest_offset, |
224 Node* copy_length); | 224 Node* copy_length, bool dest_uninitialized); |
225 Node* generate_generic_arraycopy(const TypePtr* adr_type, | 225 Node* generate_generic_arraycopy(const TypePtr* adr_type, |
226 Node* src, Node* src_offset, | 226 Node* src, Node* src_offset, |
227 Node* dest, Node* dest_offset, | 227 Node* dest, Node* dest_offset, |
228 Node* copy_length); | 228 Node* copy_length, bool dest_uninitialized); |
229 void generate_unchecked_arraycopy(const TypePtr* adr_type, | 229 void generate_unchecked_arraycopy(const TypePtr* adr_type, |
230 BasicType basic_elem_type, | 230 BasicType basic_elem_type, |
231 bool disjoint_bases, | 231 bool disjoint_bases, |
232 Node* src, Node* src_offset, | 232 Node* src, Node* src_offset, |
233 Node* dest, Node* dest_offset, | 233 Node* dest, Node* dest_offset, |
234 Node* copy_length); | 234 Node* copy_length, bool dest_uninitialized); |
235 bool inline_unsafe_CAS(BasicType type); | 235 bool inline_unsafe_CAS(BasicType type); |
236 bool inline_unsafe_ordered_store(BasicType type); | 236 bool inline_unsafe_ordered_store(BasicType type); |
237 bool inline_fp_conversions(vmIntrinsics::ID id); | 237 bool inline_fp_conversions(vmIntrinsics::ID id); |
238 bool inline_numberOfLeadingZeros(vmIntrinsics::ID id); | 238 bool inline_numberOfLeadingZeros(vmIntrinsics::ID id); |
239 bool inline_numberOfTrailingZeros(vmIntrinsics::ID id); | 239 bool inline_numberOfTrailingZeros(vmIntrinsics::ID id); |
4090 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) )); | 4090 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) )); |
4091 | 4091 |
4092 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; | 4092 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; |
4093 bool disjoint_bases = true; | 4093 bool disjoint_bases = true; |
4094 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases, | 4094 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases, |
4095 src, NULL, dest, NULL, countx); | 4095 src, NULL, dest, NULL, countx, |
4096 /*dest_uninitialized*/true); | |
4096 | 4097 |
4097 // If necessary, emit some card marks afterwards. (Non-arrays only.) | 4098 // If necessary, emit some card marks afterwards. (Non-arrays only.) |
4098 if (card_mark) { | 4099 if (card_mark) { |
4099 assert(!is_array, ""); | 4100 assert(!is_array, ""); |
4100 // Put in store barrier for any and all oops we are sticking | 4101 // Put in store barrier for any and all oops we are sticking |
4304 }; | 4305 }; |
4305 | 4306 |
4306 // Note: The condition "disjoint" applies also for overlapping copies | 4307 // Note: The condition "disjoint" applies also for overlapping copies |
4307 // where an descending copy is permitted (i.e., dest_offset <= src_offset). | 4308 // where an descending copy is permitted (i.e., dest_offset <= src_offset). |
4308 static address | 4309 static address |
4309 select_arraycopy_function(BasicType t, bool aligned, bool disjoint, const char* &name) { | 4310 select_arraycopy_function(BasicType t, bool aligned, bool disjoint, const char* &name, bool dest_uninitialized) { |
4310 int selector = | 4311 int selector = |
4311 (aligned ? COPYFUNC_ALIGNED : COPYFUNC_UNALIGNED) + | 4312 (aligned ? COPYFUNC_ALIGNED : COPYFUNC_UNALIGNED) + |
4312 (disjoint ? COPYFUNC_DISJOINT : COPYFUNC_CONJOINT); | 4313 (disjoint ? COPYFUNC_DISJOINT : COPYFUNC_CONJOINT); |
4313 | 4314 |
4314 #define RETURN_STUB(xxx_arraycopy) { \ | 4315 #define RETURN_STUB(xxx_arraycopy) { \ |
4315 name = #xxx_arraycopy; \ | 4316 name = #xxx_arraycopy; \ |
4316 return StubRoutines::xxx_arraycopy(); } | 4317 return StubRoutines::xxx_arraycopy(); } |
4318 | |
4319 #define RETURN_STUB_PARM(xxx_arraycopy, parm) { \ | |
4320 name = #xxx_arraycopy; \ | |
4321 return StubRoutines::xxx_arraycopy(parm); } | |
4317 | 4322 |
4318 switch (t) { | 4323 switch (t) { |
4319 case T_BYTE: | 4324 case T_BYTE: |
4320 case T_BOOLEAN: | 4325 case T_BOOLEAN: |
4321 switch (selector) { | 4326 switch (selector) { |
4349 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jlong_disjoint_arraycopy); | 4354 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jlong_disjoint_arraycopy); |
4350 } | 4355 } |
4351 case T_ARRAY: | 4356 case T_ARRAY: |
4352 case T_OBJECT: | 4357 case T_OBJECT: |
4353 switch (selector) { | 4358 switch (selector) { |
4354 case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(oop_arraycopy); | 4359 case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB_PARM(oop_arraycopy, dest_uninitialized); |
4355 case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_oop_arraycopy); | 4360 case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB_PARM(arrayof_oop_arraycopy, dest_uninitialized); |
4356 case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(oop_disjoint_arraycopy); | 4361 case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB_PARM(oop_disjoint_arraycopy, dest_uninitialized); |
4357 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_oop_disjoint_arraycopy); | 4362 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB_PARM(arrayof_oop_disjoint_arraycopy, dest_uninitialized); |
4358 } | 4363 } |
4359 default: | 4364 default: |
4360 ShouldNotReachHere(); | 4365 ShouldNotReachHere(); |
4361 return NULL; | 4366 return NULL; |
4362 } | 4367 } |
4363 | 4368 |
4364 #undef RETURN_STUB | 4369 #undef RETURN_STUB |
4370 #undef RETURN_STUB_PARM | |
4365 } | 4371 } |
4366 | 4372 |
4367 //------------------------------basictype2arraycopy---------------------------- | 4373 //------------------------------basictype2arraycopy---------------------------- |
4368 address LibraryCallKit::basictype2arraycopy(BasicType t, | 4374 address LibraryCallKit::basictype2arraycopy(BasicType t, |
4369 Node* src_offset, | 4375 Node* src_offset, |
4370 Node* dest_offset, | 4376 Node* dest_offset, |
4371 bool disjoint_bases, | 4377 bool disjoint_bases, |
4372 const char* &name) { | 4378 const char* &name, |
4379 bool dest_uninitialized) { | |
4373 const TypeInt* src_offset_inttype = gvn().find_int_type(src_offset);; | 4380 const TypeInt* src_offset_inttype = gvn().find_int_type(src_offset);; |
4374 const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);; | 4381 const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);; |
4375 | 4382 |
4376 bool aligned = false; | 4383 bool aligned = false; |
4377 bool disjoint = disjoint_bases; | 4384 bool disjoint = disjoint_bases; |
4393 } else if (src_offset == dest_offset && src_offset != NULL) { | 4400 } else if (src_offset == dest_offset && src_offset != NULL) { |
4394 // This can occur if the offsets are identical non-constants. | 4401 // This can occur if the offsets are identical non-constants. |
4395 disjoint = true; | 4402 disjoint = true; |
4396 } | 4403 } |
4397 | 4404 |
4398 return select_arraycopy_function(t, aligned, disjoint, name); | 4405 return select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized); |
4399 } | 4406 } |
4400 | 4407 |
4401 | 4408 |
4402 //------------------------------inline_arraycopy----------------------- | 4409 //------------------------------inline_arraycopy----------------------- |
4403 bool LibraryCallKit::inline_arraycopy() { | 4410 bool LibraryCallKit::inline_arraycopy() { |
4449 | 4456 |
4450 if (src_elem != dest_elem || dest_elem == T_VOID) { | 4457 if (src_elem != dest_elem || dest_elem == T_VOID) { |
4451 // The component types are not the same or are not recognized. Punt. | 4458 // The component types are not the same or are not recognized. Punt. |
4452 // (But, avoid the native method wrapper to JVM_ArrayCopy.) | 4459 // (But, avoid the native method wrapper to JVM_ArrayCopy.) |
4453 generate_slow_arraycopy(TypePtr::BOTTOM, | 4460 generate_slow_arraycopy(TypePtr::BOTTOM, |
4454 src, src_offset, dest, dest_offset, length); | 4461 src, src_offset, dest, dest_offset, length, |
4462 /*dest_uninitialized*/false); | |
4455 return true; | 4463 return true; |
4456 } | 4464 } |
4457 | 4465 |
4458 //--------------------------------------------------------------------------- | 4466 //--------------------------------------------------------------------------- |
4459 // We will make a fast path for this call to arraycopy. | 4467 // We will make a fast path for this call to arraycopy. |
4562 record_for_igvn(slow_region); | 4570 record_for_igvn(slow_region); |
4563 } | 4571 } |
4564 | 4572 |
4565 Node* original_dest = dest; | 4573 Node* original_dest = dest; |
4566 AllocateArrayNode* alloc = NULL; // used for zeroing, if needed | 4574 AllocateArrayNode* alloc = NULL; // used for zeroing, if needed |
4567 bool must_clear_dest = false; | 4575 bool dest_uninitialized = false; |
4568 | 4576 |
4569 // See if this is the initialization of a newly-allocated array. | 4577 // See if this is the initialization of a newly-allocated array. |
4570 // If so, we will take responsibility here for initializing it to zero. | 4578 // If so, we will take responsibility here for initializing it to zero. |
4571 // (Note: Because tightly_coupled_allocation performs checks on the | 4579 // (Note: Because tightly_coupled_allocation performs checks on the |
4572 // out-edges of the dest, we need to avoid making derived pointers | 4580 // out-edges of the dest, we need to avoid making derived pointers |
4585 assert(dest->is_CheckCastPP(), "sanity"); | 4593 assert(dest->is_CheckCastPP(), "sanity"); |
4586 assert(dest->in(0)->in(0) == init, "dest pinned"); | 4594 assert(dest->in(0)->in(0) == init, "dest pinned"); |
4587 adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory | 4595 adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory |
4588 // From this point on, every exit path is responsible for | 4596 // From this point on, every exit path is responsible for |
4589 // initializing any non-copied parts of the object to zero. | 4597 // initializing any non-copied parts of the object to zero. |
4590 must_clear_dest = true; | 4598 // Also, if this flag is set we make sure that arraycopy interacts properly |
4599 // with G1, eliding pre-barriers. See CR 6627983. | |
4600 dest_uninitialized = true; | |
4591 } else { | 4601 } else { |
4592 // No zeroing elimination here. | 4602 // No zeroing elimination here. |
4593 alloc = NULL; | 4603 alloc = NULL; |
4594 //original_dest = dest; | 4604 //original_dest = dest; |
4595 //must_clear_dest = false; | 4605 //dest_uninitialized = false; |
4596 } | 4606 } |
4597 | 4607 |
4598 // Results are placed here: | 4608 // Results are placed here: |
4599 enum { fast_path = 1, // normal void-returning assembly stub | 4609 enum { fast_path = 1, // normal void-returning assembly stub |
4600 checked_path = 2, // special assembly stub with cleanup | 4610 checked_path = 2, // special assembly stub with cleanup |
4622 Node* checked_mem = NULL; | 4632 Node* checked_mem = NULL; |
4623 Node* checked_i_o = NULL; | 4633 Node* checked_i_o = NULL; |
4624 Node* checked_value = NULL; | 4634 Node* checked_value = NULL; |
4625 | 4635 |
4626 if (basic_elem_type == T_CONFLICT) { | 4636 if (basic_elem_type == T_CONFLICT) { |
4627 assert(!must_clear_dest, ""); | 4637 assert(!dest_uninitialized, ""); |
4628 Node* cv = generate_generic_arraycopy(adr_type, | 4638 Node* cv = generate_generic_arraycopy(adr_type, |
4629 src, src_offset, dest, dest_offset, | 4639 src, src_offset, dest, dest_offset, |
4630 copy_length); | 4640 copy_length, dest_uninitialized); |
4631 if (cv == NULL) cv = intcon(-1); // failure (no stub available) | 4641 if (cv == NULL) cv = intcon(-1); // failure (no stub available) |
4632 checked_control = control(); | 4642 checked_control = control(); |
4633 checked_i_o = i_o(); | 4643 checked_i_o = i_o(); |
4634 checked_mem = memory(adr_type); | 4644 checked_mem = memory(adr_type); |
4635 checked_value = cv; | 4645 checked_value = cv; |
4645 if (!length_never_negative) { | 4655 if (!length_never_negative) { |
4646 generate_negative_guard(copy_length, slow_region); | 4656 generate_negative_guard(copy_length, slow_region); |
4647 } | 4657 } |
4648 | 4658 |
4649 // copy_length is 0. | 4659 // copy_length is 0. |
4650 if (!stopped() && must_clear_dest) { | 4660 if (!stopped() && dest_uninitialized) { |
4651 Node* dest_length = alloc->in(AllocateNode::ALength); | 4661 Node* dest_length = alloc->in(AllocateNode::ALength); |
4652 if (_gvn.eqv_uncast(copy_length, dest_length) | 4662 if (_gvn.eqv_uncast(copy_length, dest_length) |
4653 || _gvn.find_int_con(dest_length, 1) <= 0) { | 4663 || _gvn.find_int_con(dest_length, 1) <= 0) { |
4654 // There is no zeroing to do. No need for a secondary raw memory barrier. | 4664 // There is no zeroing to do. No need for a secondary raw memory barrier. |
4655 } else { | 4665 } else { |
4671 result_region->init_req(zero_path, control()); | 4681 result_region->init_req(zero_path, control()); |
4672 result_i_o ->init_req(zero_path, i_o()); | 4682 result_i_o ->init_req(zero_path, i_o()); |
4673 result_memory->init_req(zero_path, memory(adr_type)); | 4683 result_memory->init_req(zero_path, memory(adr_type)); |
4674 } | 4684 } |
4675 | 4685 |
4676 if (!stopped() && must_clear_dest) { | 4686 if (!stopped() && dest_uninitialized) { |
4677 // We have to initialize the *uncopied* part of the array to zero. | 4687 // We have to initialize the *uncopied* part of the array to zero. |
4678 // The copy destination is the slice dest[off..off+len]. The other slices | 4688 // The copy destination is the slice dest[off..off+len]. The other slices |
4679 // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length]. | 4689 // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length]. |
4680 Node* dest_size = alloc->in(AllocateNode::AllocSize); | 4690 Node* dest_size = alloc->in(AllocateNode::AllocSize); |
4681 Node* dest_length = alloc->in(AllocateNode::ALength); | 4691 Node* dest_length = alloc->in(AllocateNode::ALength); |
4707 // There is no tail. Try an upgrade to a 64-bit copy. | 4717 // There is no tail. Try an upgrade to a 64-bit copy. |
4708 bool didit = false; | 4718 bool didit = false; |
4709 { PreserveJVMState pjvms(this); | 4719 { PreserveJVMState pjvms(this); |
4710 didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc, | 4720 didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc, |
4711 src, src_offset, dest, dest_offset, | 4721 src, src_offset, dest, dest_offset, |
4712 dest_size); | 4722 dest_size, dest_uninitialized); |
4713 if (didit) { | 4723 if (didit) { |
4714 // Present the results of the block-copying fast call. | 4724 // Present the results of the block-copying fast call. |
4715 result_region->init_req(bcopy_path, control()); | 4725 result_region->init_req(bcopy_path, control()); |
4716 result_i_o ->init_req(bcopy_path, i_o()); | 4726 result_i_o ->init_req(bcopy_path, i_o()); |
4717 result_memory->init_req(bcopy_path, memory(adr_type)); | 4727 result_memory->init_req(bcopy_path, memory(adr_type)); |
4783 Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM); | 4793 Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM); |
4784 Node* dest_elem_klass = _gvn.transform(n1); | 4794 Node* dest_elem_klass = _gvn.transform(n1); |
4785 Node* cv = generate_checkcast_arraycopy(adr_type, | 4795 Node* cv = generate_checkcast_arraycopy(adr_type, |
4786 dest_elem_klass, | 4796 dest_elem_klass, |
4787 src, src_offset, dest, dest_offset, | 4797 src, src_offset, dest, dest_offset, |
4788 ConvI2X(copy_length)); | 4798 ConvI2X(copy_length), dest_uninitialized); |
4789 if (cv == NULL) cv = intcon(-1); // failure (no stub available) | 4799 if (cv == NULL) cv = intcon(-1); // failure (no stub available) |
4790 checked_control = control(); | 4800 checked_control = control(); |
4791 checked_i_o = i_o(); | 4801 checked_i_o = i_o(); |
4792 checked_mem = memory(adr_type); | 4802 checked_mem = memory(adr_type); |
4793 checked_value = cv; | 4803 checked_value = cv; |
4806 if (!stopped()) { | 4816 if (!stopped()) { |
4807 // Generate the fast path, if possible. | 4817 // Generate the fast path, if possible. |
4808 PreserveJVMState pjvms(this); | 4818 PreserveJVMState pjvms(this); |
4809 generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases, | 4819 generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases, |
4810 src, src_offset, dest, dest_offset, | 4820 src, src_offset, dest, dest_offset, |
4811 ConvI2X(copy_length)); | 4821 ConvI2X(copy_length), dest_uninitialized); |
4812 | 4822 |
4813 // Present the results of the fast call. | 4823 // Present the results of the fast call. |
4814 result_region->init_req(fast_path, control()); | 4824 result_region->init_req(fast_path, control()); |
4815 result_i_o ->init_req(fast_path, i_o()); | 4825 result_i_o ->init_req(fast_path, i_o()); |
4816 result_memory->init_req(fast_path, memory(adr_type)); | 4826 result_memory->init_req(fast_path, memory(adr_type)); |
4885 PreserveJVMState pjvms(this); // replace_in_map may trash the map | 4895 PreserveJVMState pjvms(this); // replace_in_map may trash the map |
4886 | 4896 |
4887 set_memory(slow_mem, adr_type); | 4897 set_memory(slow_mem, adr_type); |
4888 set_i_o(slow_i_o); | 4898 set_i_o(slow_i_o); |
4889 | 4899 |
4890 if (must_clear_dest) { | 4900 if (dest_uninitialized) { |
4891 generate_clear_array(adr_type, dest, basic_elem_type, | 4901 generate_clear_array(adr_type, dest, basic_elem_type, |
4892 intcon(0), NULL, | 4902 intcon(0), NULL, |
4893 alloc->in(AllocateNode::AllocSize)); | 4903 alloc->in(AllocateNode::AllocSize)); |
4894 } | 4904 } |
4895 | 4905 |
4896 generate_slow_arraycopy(adr_type, | 4906 generate_slow_arraycopy(adr_type, |
4897 src, src_offset, dest, dest_offset, | 4907 src, src_offset, dest, dest_offset, |
4898 copy_length); | 4908 copy_length, /*dest_uninitialized*/false); |
4899 | 4909 |
4900 result_region->init_req(slow_call_path, control()); | 4910 result_region->init_req(slow_call_path, control()); |
4901 result_i_o ->init_req(slow_call_path, i_o()); | 4911 result_i_o ->init_req(slow_call_path, i_o()); |
4902 result_memory->init_req(slow_call_path, memory(adr_type)); | 4912 result_memory->init_req(slow_call_path, memory(adr_type)); |
4903 } | 4913 } |
5137 LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type, | 5147 LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type, |
5138 BasicType basic_elem_type, | 5148 BasicType basic_elem_type, |
5139 AllocateNode* alloc, | 5149 AllocateNode* alloc, |
5140 Node* src, Node* src_offset, | 5150 Node* src, Node* src_offset, |
5141 Node* dest, Node* dest_offset, | 5151 Node* dest, Node* dest_offset, |
5142 Node* dest_size) { | 5152 Node* dest_size, bool dest_uninitialized) { |
5143 // See if there is an advantage from block transfer. | 5153 // See if there is an advantage from block transfer. |
5144 int scale = exact_log2(type2aelembytes(basic_elem_type)); | 5154 int scale = exact_log2(type2aelembytes(basic_elem_type)); |
5145 if (scale >= LogBytesPerLong) | 5155 if (scale >= LogBytesPerLong) |
5146 return false; // it is already a block transfer | 5156 return false; // it is already a block transfer |
5147 | 5157 |
5182 countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(dest_off)) ); | 5192 countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(dest_off)) ); |
5183 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong)) ); | 5193 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong)) ); |
5184 | 5194 |
5185 bool disjoint_bases = true; // since alloc != NULL | 5195 bool disjoint_bases = true; // since alloc != NULL |
5186 generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases, | 5196 generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases, |
5187 sptr, NULL, dptr, NULL, countx); | 5197 sptr, NULL, dptr, NULL, countx, dest_uninitialized); |
5188 | 5198 |
5189 return true; | 5199 return true; |
5190 } | 5200 } |
5191 | 5201 |
5192 | 5202 |
5195 // but without the native wrapper overhead. | 5205 // but without the native wrapper overhead. |
5196 void | 5206 void |
5197 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type, | 5207 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type, |
5198 Node* src, Node* src_offset, | 5208 Node* src, Node* src_offset, |
5199 Node* dest, Node* dest_offset, | 5209 Node* dest, Node* dest_offset, |
5200 Node* copy_length) { | 5210 Node* copy_length, bool dest_uninitialized) { |
5211 assert(!dest_uninitialized, "Invariant"); | |
5201 Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON, | 5212 Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON, |
5202 OptoRuntime::slow_arraycopy_Type(), | 5213 OptoRuntime::slow_arraycopy_Type(), |
5203 OptoRuntime::slow_arraycopy_Java(), | 5214 OptoRuntime::slow_arraycopy_Java(), |
5204 "slow_arraycopy", adr_type, | 5215 "slow_arraycopy", adr_type, |
5205 src, src_offset, dest, dest_offset, | 5216 src, src_offset, dest, dest_offset, |
5213 Node* | 5224 Node* |
5214 LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type, | 5225 LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type, |
5215 Node* dest_elem_klass, | 5226 Node* dest_elem_klass, |
5216 Node* src, Node* src_offset, | 5227 Node* src, Node* src_offset, |
5217 Node* dest, Node* dest_offset, | 5228 Node* dest, Node* dest_offset, |
5218 Node* copy_length) { | 5229 Node* copy_length, bool dest_uninitialized) { |
5219 if (stopped()) return NULL; | 5230 if (stopped()) return NULL; |
5220 | 5231 |
5221 address copyfunc_addr = StubRoutines::checkcast_arraycopy(); | 5232 address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized); |
5222 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. | 5233 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. |
5223 return NULL; | 5234 return NULL; |
5224 } | 5235 } |
5225 | 5236 |
5226 // Pick out the parameters required to perform a store-check | 5237 // Pick out the parameters required to perform a store-check |
5254 // Helper function; generates code for cases requiring runtime checks. | 5265 // Helper function; generates code for cases requiring runtime checks. |
5255 Node* | 5266 Node* |
5256 LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type, | 5267 LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type, |
5257 Node* src, Node* src_offset, | 5268 Node* src, Node* src_offset, |
5258 Node* dest, Node* dest_offset, | 5269 Node* dest, Node* dest_offset, |
5259 Node* copy_length) { | 5270 Node* copy_length, bool dest_uninitialized) { |
5271 assert(!dest_uninitialized, "Invariant"); | |
5260 if (stopped()) return NULL; | 5272 if (stopped()) return NULL; |
5261 | |
5262 address copyfunc_addr = StubRoutines::generic_arraycopy(); | 5273 address copyfunc_addr = StubRoutines::generic_arraycopy(); |
5263 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. | 5274 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. |
5264 return NULL; | 5275 return NULL; |
5265 } | 5276 } |
5266 | 5277 |
5277 LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type, | 5288 LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type, |
5278 BasicType basic_elem_type, | 5289 BasicType basic_elem_type, |
5279 bool disjoint_bases, | 5290 bool disjoint_bases, |
5280 Node* src, Node* src_offset, | 5291 Node* src, Node* src_offset, |
5281 Node* dest, Node* dest_offset, | 5292 Node* dest, Node* dest_offset, |
5282 Node* copy_length) { | 5293 Node* copy_length, bool dest_uninitialized) { |
5283 if (stopped()) return; // nothing to do | 5294 if (stopped()) return; // nothing to do |
5284 | 5295 |
5285 Node* src_start = src; | 5296 Node* src_start = src; |
5286 Node* dest_start = dest; | 5297 Node* dest_start = dest; |
5287 if (src_offset != NULL || dest_offset != NULL) { | 5298 if (src_offset != NULL || dest_offset != NULL) { |
5292 | 5303 |
5293 // Figure out which arraycopy runtime method to call. | 5304 // Figure out which arraycopy runtime method to call. |
5294 const char* copyfunc_name = "arraycopy"; | 5305 const char* copyfunc_name = "arraycopy"; |
5295 address copyfunc_addr = | 5306 address copyfunc_addr = |
5296 basictype2arraycopy(basic_elem_type, src_offset, dest_offset, | 5307 basictype2arraycopy(basic_elem_type, src_offset, dest_offset, |
5297 disjoint_bases, copyfunc_name); | 5308 disjoint_bases, copyfunc_name, dest_uninitialized); |
5298 | 5309 |
5299 // Call it. Note that the count_ix value is not scaled to a byte-size. | 5310 // Call it. Note that the count_ix value is not scaled to a byte-size. |
5300 make_runtime_call(RC_LEAF|RC_NO_FP, | 5311 make_runtime_call(RC_LEAF|RC_NO_FP, |
5301 OptoRuntime::fast_arraycopy_Type(), | 5312 OptoRuntime::fast_arraycopy_Type(), |
5302 copyfunc_addr, copyfunc_name, adr_type, | 5313 copyfunc_addr, copyfunc_name, adr_type, |