Mercurial > hg > truffle
comparison src/share/vm/opto/library_call.cpp @ 839:bb18957ad21e
Merge
author | ysr |
---|---|
date | Fri, 10 Jul 2009 16:01:20 -0700 |
parents | 4325cdaa78ad |
children | fc4be448891f |
comparison
equal
deleted
inserted
replaced
838:0316eac49d5a | 839:bb18957ad21e |
---|---|
163 | 163 |
164 bool inline_native_newArray(); | 164 bool inline_native_newArray(); |
165 bool inline_native_getLength(); | 165 bool inline_native_getLength(); |
166 bool inline_array_copyOf(bool is_copyOfRange); | 166 bool inline_array_copyOf(bool is_copyOfRange); |
167 bool inline_array_equals(); | 167 bool inline_array_equals(); |
168 void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark); | |
168 bool inline_native_clone(bool is_virtual); | 169 bool inline_native_clone(bool is_virtual); |
169 bool inline_native_Reflection_getCallerClass(); | 170 bool inline_native_Reflection_getCallerClass(); |
170 bool inline_native_AtomicLong_get(); | 171 bool inline_native_AtomicLong_get(); |
171 bool inline_native_AtomicLong_attemptUpdate(); | 172 bool inline_native_AtomicLong_attemptUpdate(); |
172 bool is_method_invoke_or_aux_frame(JVMState* jvms); | 173 bool is_method_invoke_or_aux_frame(JVMState* jvms); |
179 void generate_arraycopy(const TypePtr* adr_type, | 180 void generate_arraycopy(const TypePtr* adr_type, |
180 BasicType basic_elem_type, | 181 BasicType basic_elem_type, |
181 Node* src, Node* src_offset, | 182 Node* src, Node* src_offset, |
182 Node* dest, Node* dest_offset, | 183 Node* dest, Node* dest_offset, |
183 Node* copy_length, | 184 Node* copy_length, |
184 int nargs, // arguments on stack for debug info | |
185 bool disjoint_bases = false, | 185 bool disjoint_bases = false, |
186 bool length_never_negative = false, | 186 bool length_never_negative = false, |
187 RegionNode* slow_region = NULL); | 187 RegionNode* slow_region = NULL); |
188 AllocateArrayNode* tightly_coupled_allocation(Node* ptr, | 188 AllocateArrayNode* tightly_coupled_allocation(Node* ptr, |
189 RegionNode* slow_region); | 189 RegionNode* slow_region); |
200 Node* dest, Node* dest_offset, | 200 Node* dest, Node* dest_offset, |
201 Node* dest_size); | 201 Node* dest_size); |
202 void generate_slow_arraycopy(const TypePtr* adr_type, | 202 void generate_slow_arraycopy(const TypePtr* adr_type, |
203 Node* src, Node* src_offset, | 203 Node* src, Node* src_offset, |
204 Node* dest, Node* dest_offset, | 204 Node* dest, Node* dest_offset, |
205 Node* copy_length, | 205 Node* copy_length); |
206 int nargs); | |
207 Node* generate_checkcast_arraycopy(const TypePtr* adr_type, | 206 Node* generate_checkcast_arraycopy(const TypePtr* adr_type, |
208 Node* dest_elem_klass, | 207 Node* dest_elem_klass, |
209 Node* src, Node* src_offset, | 208 Node* src, Node* src_offset, |
210 Node* dest, Node* dest_offset, | 209 Node* dest, Node* dest_offset, |
211 Node* copy_length, int nargs); | 210 Node* copy_length); |
212 Node* generate_generic_arraycopy(const TypePtr* adr_type, | 211 Node* generate_generic_arraycopy(const TypePtr* adr_type, |
213 Node* src, Node* src_offset, | 212 Node* src, Node* src_offset, |
214 Node* dest, Node* dest_offset, | 213 Node* dest, Node* dest_offset, |
215 Node* copy_length, int nargs); | 214 Node* copy_length); |
216 void generate_unchecked_arraycopy(const TypePtr* adr_type, | 215 void generate_unchecked_arraycopy(const TypePtr* adr_type, |
217 BasicType basic_elem_type, | 216 BasicType basic_elem_type, |
218 bool disjoint_bases, | 217 bool disjoint_bases, |
219 Node* src, Node* src_offset, | 218 Node* src, Node* src_offset, |
220 Node* dest, Node* dest_offset, | 219 Node* dest, Node* dest_offset, |
3227 // How many elements will we copy from the original? | 3226 // How many elements will we copy from the original? |
3228 // The answer is MinI(orig_length - start, length). | 3227 // The answer is MinI(orig_length - start, length). |
3229 Node* orig_tail = _gvn.transform( new(C, 3) SubINode(orig_length, start) ); | 3228 Node* orig_tail = _gvn.transform( new(C, 3) SubINode(orig_length, start) ); |
3230 Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length); | 3229 Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length); |
3231 | 3230 |
3232 Node* newcopy = new_array(klass_node, length, nargs); | 3231 const bool raw_mem_only = true; |
3232 Node* newcopy = new_array(klass_node, length, nargs, raw_mem_only); | |
3233 | 3233 |
3234 // Generate a direct call to the right arraycopy function(s). | 3234 // Generate a direct call to the right arraycopy function(s). |
3235 // We know the copy is disjoint but we might not know if the | 3235 // We know the copy is disjoint but we might not know if the |
3236 // oop stores need checking. | 3236 // oop stores need checking. |
3237 // Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class). | 3237 // Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class). |
3238 // This will fail a store-check if x contains any non-nulls. | 3238 // This will fail a store-check if x contains any non-nulls. |
3239 bool disjoint_bases = true; | 3239 bool disjoint_bases = true; |
3240 bool length_never_negative = true; | 3240 bool length_never_negative = true; |
3241 generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT, | 3241 generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT, |
3242 original, start, newcopy, intcon(0), moved, | 3242 original, start, newcopy, intcon(0), moved, |
3243 nargs, disjoint_bases, length_never_negative); | 3243 disjoint_bases, length_never_negative); |
3244 | 3244 |
3245 push(newcopy); | 3245 push(newcopy); |
3246 } | 3246 } |
3247 | 3247 |
3248 C->set_has_split_ifs(true); // Has chance for split-if optimization | 3248 C->set_has_split_ifs(true); // Has chance for split-if optimization |
3880 insert_mem_bar(Op_MemBarCPUOrder); | 3880 insert_mem_bar(Op_MemBarCPUOrder); |
3881 | 3881 |
3882 return true; | 3882 return true; |
3883 } | 3883 } |
3884 | 3884 |
3885 //------------------------clone_coping----------------------------------- | |
3886 // Helper function for inline_native_clone. | |
3887 void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) { | |
3888 assert(obj_size != NULL, ""); | |
3889 Node* raw_obj = alloc_obj->in(1); | |
3890 assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); | |
3891 | |
3892 if (ReduceBulkZeroing) { | |
3893 // We will be completely responsible for initializing this object - | |
3894 // mark Initialize node as complete. | |
3895 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); | |
3896 // The object was just allocated - there should be no any stores! | |
3897 guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), ""); | |
3898 } | |
3899 | |
3900 // Cast to Object for arraycopy. | |
3901 // We can't use the original CheckCastPP since it should be moved | |
3902 // after the arraycopy to prevent stores flowing above it. | |
3903 Node* new_obj = new(C, 2) CheckCastPPNode(alloc_obj->in(0), raw_obj, | |
3904 TypeInstPtr::NOTNULL); | |
3905 new_obj = _gvn.transform(new_obj); | |
3906 // Substitute in the locally valid dest_oop. | |
3907 replace_in_map(alloc_obj, new_obj); | |
3908 | |
3909 // Copy the fastest available way. | |
3910 // TODO: generate fields copies for small objects instead. | |
3911 Node* src = obj; | |
3912 Node* dest = new_obj; | |
3913 Node* size = _gvn.transform(obj_size); | |
3914 | |
3915 // Exclude the header but include array length to copy by 8 bytes words. | |
3916 // Can't use base_offset_in_bytes(bt) since basic type is unknown. | |
3917 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() : | |
3918 instanceOopDesc::base_offset_in_bytes(); | |
3919 // base_off: | |
3920 // 8 - 32-bit VM | |
3921 // 12 - 64-bit VM, compressed oops | |
3922 // 16 - 64-bit VM, normal oops | |
3923 if (base_off % BytesPerLong != 0) { | |
3924 assert(UseCompressedOops, ""); | |
3925 if (is_array) { | |
3926 // Exclude length to copy by 8 bytes words. | |
3927 base_off += sizeof(int); | |
3928 } else { | |
3929 // Include klass to copy by 8 bytes words. | |
3930 base_off = instanceOopDesc::klass_offset_in_bytes(); | |
3931 } | |
3932 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment"); | |
3933 } | |
3934 src = basic_plus_adr(src, base_off); | |
3935 dest = basic_plus_adr(dest, base_off); | |
3936 | |
3937 // Compute the length also, if needed: | |
3938 Node* countx = size; | |
3939 countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(base_off)) ); | |
3940 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) )); | |
3941 | |
3942 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; | |
3943 bool disjoint_bases = true; | |
3944 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases, | |
3945 src, NULL, dest, NULL, countx); | |
3946 | |
3947 // If necessary, emit some card marks afterwards. (Non-arrays only.) | |
3948 if (card_mark) { | |
3949 assert(!is_array, ""); | |
3950 // Put in store barrier for any and all oops we are sticking | |
3951 // into this object. (We could avoid this if we could prove | |
3952 // that the object type contains no oop fields at all.) | |
3953 Node* no_particular_value = NULL; | |
3954 Node* no_particular_field = NULL; | |
3955 int raw_adr_idx = Compile::AliasIdxRaw; | |
3956 post_barrier(control(), | |
3957 memory(raw_adr_type), | |
3958 new_obj, | |
3959 no_particular_field, | |
3960 raw_adr_idx, | |
3961 no_particular_value, | |
3962 T_OBJECT, | |
3963 false); | |
3964 } | |
3965 | |
3966 // Move the original CheckCastPP after arraycopy. | |
3967 _gvn.hash_delete(alloc_obj); | |
3968 alloc_obj->set_req(0, control()); | |
3969 // Replace raw memory edge with new CheckCastPP to have a live oop | |
3970 // at safepoints instead of raw value. | |
3971 assert(new_obj->is_CheckCastPP() && new_obj->in(1) == alloc_obj->in(1), "sanity"); | |
3972 alloc_obj->set_req(1, new_obj); // cast to the original type | |
3973 _gvn.hash_find_insert(alloc_obj); // put back into GVN table | |
3974 // Restore in the locally valid dest_oop. | |
3975 replace_in_map(new_obj, alloc_obj); | |
3976 } | |
3885 | 3977 |
3886 //------------------------inline_native_clone---------------------------- | 3978 //------------------------inline_native_clone---------------------------- |
3887 // Here are the simple edge cases: | 3979 // Here are the simple edge cases: |
3888 // null receiver => normal trap | 3980 // null receiver => normal trap |
3889 // virtual and clone was overridden => slow path to out-of-line clone | 3981 // virtual and clone was overridden => slow path to out-of-line clone |
3914 insert_mem_bar(Op_MemBarCPUOrder); | 4006 insert_mem_bar(Op_MemBarCPUOrder); |
3915 | 4007 |
3916 // paths into result_reg: | 4008 // paths into result_reg: |
3917 enum { | 4009 enum { |
3918 _slow_path = 1, // out-of-line call to clone method (virtual or not) | 4010 _slow_path = 1, // out-of-line call to clone method (virtual or not) |
3919 _objArray_path, // plain allocation, plus arrayof_oop_arraycopy | 4011 _objArray_path, // plain array allocation, plus arrayof_oop_arraycopy |
3920 _fast_path, // plain allocation, plus a CopyArray operation | 4012 _array_path, // plain array allocation, plus arrayof_long_arraycopy |
4013 _instance_path, // plain instance allocation, plus arrayof_long_arraycopy | |
3921 PATH_LIMIT | 4014 PATH_LIMIT |
3922 }; | 4015 }; |
3923 RegionNode* result_reg = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT); | 4016 RegionNode* result_reg = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT); |
3924 PhiNode* result_val = new(C, PATH_LIMIT) PhiNode(result_reg, | 4017 PhiNode* result_val = new(C, PATH_LIMIT) PhiNode(result_reg, |
3925 TypeInstPtr::NOTNULL); | 4018 TypeInstPtr::NOTNULL); |
3930 | 4023 |
3931 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; | 4024 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; |
3932 int raw_adr_idx = Compile::AliasIdxRaw; | 4025 int raw_adr_idx = Compile::AliasIdxRaw; |
3933 const bool raw_mem_only = true; | 4026 const bool raw_mem_only = true; |
3934 | 4027 |
3935 // paths into alloc_reg (on the fast path, just before the CopyArray): | |
3936 enum { _typeArray_alloc = 1, _instance_alloc, ALLOC_LIMIT }; | |
3937 RegionNode* alloc_reg = new(C, ALLOC_LIMIT) RegionNode(ALLOC_LIMIT); | |
3938 PhiNode* alloc_val = new(C, ALLOC_LIMIT) PhiNode(alloc_reg, raw_adr_type); | |
3939 PhiNode* alloc_siz = new(C, ALLOC_LIMIT) PhiNode(alloc_reg, TypeX_X); | |
3940 PhiNode* alloc_i_o = new(C, ALLOC_LIMIT) PhiNode(alloc_reg, Type::ABIO); | |
3941 PhiNode* alloc_mem = new(C, ALLOC_LIMIT) PhiNode(alloc_reg, Type::MEMORY, | |
3942 raw_adr_type); | |
3943 record_for_igvn(alloc_reg); | |
3944 | |
3945 bool card_mark = false; // (see below) | |
3946 | |
3947 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL); | 4028 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL); |
3948 if (array_ctl != NULL) { | 4029 if (array_ctl != NULL) { |
3949 // It's an array. | 4030 // It's an array. |
3950 PreserveJVMState pjvms(this); | 4031 PreserveJVMState pjvms(this); |
3951 set_control(array_ctl); | 4032 set_control(array_ctl); |
3952 Node* obj_length = load_array_length(obj); | 4033 Node* obj_length = load_array_length(obj); |
3953 Node* obj_size = NULL; | 4034 Node* obj_size = NULL; |
3954 Node* alloc_obj = new_array(obj_klass, obj_length, nargs, | 4035 Node* alloc_obj = new_array(obj_klass, obj_length, nargs, |
3955 raw_mem_only, &obj_size); | 4036 raw_mem_only, &obj_size); |
3956 assert(obj_size != NULL, ""); | |
3957 Node* raw_obj = alloc_obj->in(1); | |
3958 assert(raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); | |
3959 if (ReduceBulkZeroing) { | |
3960 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); | |
3961 if (alloc != NULL) { | |
3962 // We will be completely responsible for initializing this object. | |
3963 alloc->maybe_set_complete(&_gvn); | |
3964 } | |
3965 } | |
3966 | 4037 |
3967 if (!use_ReduceInitialCardMarks()) { | 4038 if (!use_ReduceInitialCardMarks()) { |
3968 // If it is an oop array, it requires very special treatment, | 4039 // If it is an oop array, it requires very special treatment, |
3969 // because card marking is required on each card of the array. | 4040 // because card marking is required on each card of the array. |
3970 Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL); | 4041 Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL); |
3974 // Generate a direct call to the right arraycopy function(s). | 4045 // Generate a direct call to the right arraycopy function(s). |
3975 bool disjoint_bases = true; | 4046 bool disjoint_bases = true; |
3976 bool length_never_negative = true; | 4047 bool length_never_negative = true; |
3977 generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT, | 4048 generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT, |
3978 obj, intcon(0), alloc_obj, intcon(0), | 4049 obj, intcon(0), alloc_obj, intcon(0), |
3979 obj_length, nargs, | 4050 obj_length, |
3980 disjoint_bases, length_never_negative); | 4051 disjoint_bases, length_never_negative); |
3981 result_reg->init_req(_objArray_path, control()); | 4052 result_reg->init_req(_objArray_path, control()); |
3982 result_val->init_req(_objArray_path, alloc_obj); | 4053 result_val->init_req(_objArray_path, alloc_obj); |
3983 result_i_o ->set_req(_objArray_path, i_o()); | 4054 result_i_o ->set_req(_objArray_path, i_o()); |
3984 result_mem ->set_req(_objArray_path, reset_memory()); | 4055 result_mem ->set_req(_objArray_path, reset_memory()); |
3989 // causes the non-eden paths to simulate a fresh allocation, | 4060 // causes the non-eden paths to simulate a fresh allocation, |
3990 // insofar that no further card marks are required to initialize | 4061 // insofar that no further card marks are required to initialize |
3991 // the object. | 4062 // the object. |
3992 | 4063 |
3993 // Otherwise, there are no card marks to worry about. | 4064 // Otherwise, there are no card marks to worry about. |
3994 alloc_val->init_req(_typeArray_alloc, raw_obj); | 4065 |
3995 alloc_siz->init_req(_typeArray_alloc, obj_size); | 4066 if (!stopped()) { |
3996 alloc_reg->init_req(_typeArray_alloc, control()); | 4067 copy_to_clone(obj, alloc_obj, obj_size, true, false); |
3997 alloc_i_o->init_req(_typeArray_alloc, i_o()); | 4068 |
3998 alloc_mem->init_req(_typeArray_alloc, memory(raw_adr_type)); | 4069 // Present the results of the copy. |
3999 } | 4070 result_reg->init_req(_array_path, control()); |
4000 | 4071 result_val->init_req(_array_path, alloc_obj); |
4001 // We only go to the fast case code if we pass a number of guards. | 4072 result_i_o ->set_req(_array_path, i_o()); |
4073 result_mem ->set_req(_array_path, reset_memory()); | |
4074 } | |
4075 } | |
4076 | |
4077 // We only go to the instance fast case code if we pass a number of guards. | |
4002 // The paths which do not pass are accumulated in the slow_region. | 4078 // The paths which do not pass are accumulated in the slow_region. |
4003 RegionNode* slow_region = new (C, 1) RegionNode(1); | 4079 RegionNode* slow_region = new (C, 1) RegionNode(1); |
4004 record_for_igvn(slow_region); | 4080 record_for_igvn(slow_region); |
4005 if (!stopped()) { | 4081 if (!stopped()) { |
4006 // It's an instance. Make the slow-path tests. | 4082 // It's an instance (we did array above). Make the slow-path tests. |
4007 // If this is a virtual call, we generate a funny guard. We grab | 4083 // If this is a virtual call, we generate a funny guard. We grab |
4008 // the vtable entry corresponding to clone() from the target object. | 4084 // the vtable entry corresponding to clone() from the target object. |
4009 // If the target method which we are calling happens to be the | 4085 // If the target method which we are calling happens to be the |
4010 // Object clone() method, we pass the guard. We do not need this | 4086 // Object clone() method, we pass the guard. We do not need this |
4011 // guard for non-virtual calls; the caller is known to be the native | 4087 // guard for non-virtual calls; the caller is known to be the native |
4028 if (!stopped()) { | 4104 if (!stopped()) { |
4029 // It's an instance, and it passed the slow-path tests. | 4105 // It's an instance, and it passed the slow-path tests. |
4030 PreserveJVMState pjvms(this); | 4106 PreserveJVMState pjvms(this); |
4031 Node* obj_size = NULL; | 4107 Node* obj_size = NULL; |
4032 Node* alloc_obj = new_instance(obj_klass, NULL, raw_mem_only, &obj_size); | 4108 Node* alloc_obj = new_instance(obj_klass, NULL, raw_mem_only, &obj_size); |
4033 assert(obj_size != NULL, ""); | 4109 |
4034 Node* raw_obj = alloc_obj->in(1); | 4110 copy_to_clone(obj, alloc_obj, obj_size, false, !use_ReduceInitialCardMarks()); |
4035 assert(raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); | 4111 |
4036 if (ReduceBulkZeroing) { | 4112 // Present the results of the slow call. |
4037 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); | 4113 result_reg->init_req(_instance_path, control()); |
4038 if (alloc != NULL && !alloc->maybe_set_complete(&_gvn)) | 4114 result_val->init_req(_instance_path, alloc_obj); |
4039 alloc = NULL; | 4115 result_i_o ->set_req(_instance_path, i_o()); |
4040 } | 4116 result_mem ->set_req(_instance_path, reset_memory()); |
4041 if (!use_ReduceInitialCardMarks()) { | |
4042 // Put in store barrier for any and all oops we are sticking | |
4043 // into this object. (We could avoid this if we could prove | |
4044 // that the object type contains no oop fields at all.) | |
4045 card_mark = true; | |
4046 } | |
4047 alloc_val->init_req(_instance_alloc, raw_obj); | |
4048 alloc_siz->init_req(_instance_alloc, obj_size); | |
4049 alloc_reg->init_req(_instance_alloc, control()); | |
4050 alloc_i_o->init_req(_instance_alloc, i_o()); | |
4051 alloc_mem->init_req(_instance_alloc, memory(raw_adr_type)); | |
4052 } | 4117 } |
4053 | 4118 |
4054 // Generate code for the slow case. We make a call to clone(). | 4119 // Generate code for the slow case. We make a call to clone(). |
4055 set_control(_gvn.transform(slow_region)); | 4120 set_control(_gvn.transform(slow_region)); |
4056 if (!stopped()) { | 4121 if (!stopped()) { |
4062 result_val->init_req(_slow_path, slow_result); | 4127 result_val->init_req(_slow_path, slow_result); |
4063 result_i_o ->set_req(_slow_path, i_o()); | 4128 result_i_o ->set_req(_slow_path, i_o()); |
4064 result_mem ->set_req(_slow_path, reset_memory()); | 4129 result_mem ->set_req(_slow_path, reset_memory()); |
4065 } | 4130 } |
4066 | 4131 |
4067 // The object is allocated, as an array and/or an instance. Now copy it. | |
4068 set_control( _gvn.transform(alloc_reg) ); | |
4069 set_i_o( _gvn.transform(alloc_i_o) ); | |
4070 set_memory( _gvn.transform(alloc_mem), raw_adr_type ); | |
4071 Node* raw_obj = _gvn.transform(alloc_val); | |
4072 | |
4073 if (!stopped()) { | |
4074 // Copy the fastest available way. | |
4075 // (No need for PreserveJVMState, since we're using it all up now.) | |
4076 // TODO: generate fields/elements copies for small objects instead. | |
4077 Node* src = obj; | |
4078 Node* dest = raw_obj; | |
4079 Node* size = _gvn.transform(alloc_siz); | |
4080 | |
4081 // Exclude the header. | |
4082 int base_off = instanceOopDesc::base_offset_in_bytes(); | |
4083 if (UseCompressedOops) { | |
4084 assert(base_off % BytesPerLong != 0, "base with compressed oops"); | |
4085 // With compressed oops base_offset_in_bytes is 12 which creates | |
4086 // the gap since countx is rounded by 8 bytes below. | |
4087 // Copy klass and the gap. | |
4088 base_off = instanceOopDesc::klass_offset_in_bytes(); | |
4089 } | |
4090 src = basic_plus_adr(src, base_off); | |
4091 dest = basic_plus_adr(dest, base_off); | |
4092 | |
4093 // Compute the length also, if needed: | |
4094 Node* countx = size; | |
4095 countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(base_off)) ); | |
4096 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) )); | |
4097 | |
4098 // Select an appropriate instruction to initialize the range. | |
4099 // The CopyArray instruction (if supported) can be optimized | |
4100 // into a discrete set of scalar loads and stores. | |
4101 bool disjoint_bases = true; | |
4102 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases, | |
4103 src, NULL, dest, NULL, countx); | |
4104 | |
4105 // Now that the object is properly initialized, type it as an oop. | |
4106 // Use a secondary InitializeNode memory barrier. | |
4107 InitializeNode* init = insert_mem_bar_volatile(Op_Initialize, raw_adr_idx, | |
4108 raw_obj)->as_Initialize(); | |
4109 init->set_complete(&_gvn); // (there is no corresponding AllocateNode) | |
4110 Node* new_obj = new(C, 2) CheckCastPPNode(control(), raw_obj, | |
4111 TypeInstPtr::NOTNULL); | |
4112 new_obj = _gvn.transform(new_obj); | |
4113 | |
4114 // If necessary, emit some card marks afterwards. (Non-arrays only.) | |
4115 if (card_mark) { | |
4116 Node* no_particular_value = NULL; | |
4117 Node* no_particular_field = NULL; | |
4118 post_barrier(control(), | |
4119 memory(raw_adr_type), | |
4120 new_obj, | |
4121 no_particular_field, | |
4122 raw_adr_idx, | |
4123 no_particular_value, | |
4124 T_OBJECT, | |
4125 false); | |
4126 } | |
4127 // Present the results of the slow call. | |
4128 result_reg->init_req(_fast_path, control()); | |
4129 result_val->init_req(_fast_path, new_obj); | |
4130 result_i_o ->set_req(_fast_path, i_o()); | |
4131 result_mem ->set_req(_fast_path, reset_memory()); | |
4132 } | |
4133 | |
4134 // Return the combined state. | 4132 // Return the combined state. |
4135 set_control( _gvn.transform(result_reg) ); | 4133 set_control( _gvn.transform(result_reg) ); |
4136 set_i_o( _gvn.transform(result_i_o) ); | 4134 set_i_o( _gvn.transform(result_i_o) ); |
4137 set_all_memory( _gvn.transform(result_mem) ); | 4135 set_all_memory( _gvn.transform(result_mem) ); |
4138 | 4136 |
4139 // Cast the result to a sharper type, since we know what clone does. | 4137 push(_gvn.transform(result_val)); |
4140 Node* new_obj = _gvn.transform(result_val); | |
4141 Node* cast = new (C, 2) CheckCastPPNode(control(), new_obj, toop); | |
4142 push(_gvn.transform(cast)); | |
4143 | 4138 |
4144 return true; | 4139 return true; |
4145 } | 4140 } |
4146 | 4141 |
4147 | 4142 |
4276 // Do not let writes into the source float below the arraycopy. | 4271 // Do not let writes into the source float below the arraycopy. |
4277 insert_mem_bar(Op_MemBarCPUOrder); | 4272 insert_mem_bar(Op_MemBarCPUOrder); |
4278 | 4273 |
4279 // Call StubRoutines::generic_arraycopy stub. | 4274 // Call StubRoutines::generic_arraycopy stub. |
4280 generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT, | 4275 generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT, |
4281 src, src_offset, dest, dest_offset, length, | 4276 src, src_offset, dest, dest_offset, length); |
4282 nargs); | |
4283 | 4277 |
4284 // Do not let reads from the destination float above the arraycopy. | 4278 // Do not let reads from the destination float above the arraycopy. |
4285 // Since we cannot type the arrays, we don't know which slices | 4279 // Since we cannot type the arrays, we don't know which slices |
4286 // might be affected. We could restrict this barrier only to those | 4280 // might be affected. We could restrict this barrier only to those |
4287 // memory slices which pertain to array elements--but don't bother. | 4281 // memory slices which pertain to array elements--but don't bother. |
4300 | 4294 |
4301 if (src_elem != dest_elem || dest_elem == T_VOID) { | 4295 if (src_elem != dest_elem || dest_elem == T_VOID) { |
4302 // The component types are not the same or are not recognized. Punt. | 4296 // The component types are not the same or are not recognized. Punt. |
4303 // (But, avoid the native method wrapper to JVM_ArrayCopy.) | 4297 // (But, avoid the native method wrapper to JVM_ArrayCopy.) |
4304 generate_slow_arraycopy(TypePtr::BOTTOM, | 4298 generate_slow_arraycopy(TypePtr::BOTTOM, |
4305 src, src_offset, dest, dest_offset, length, | 4299 src, src_offset, dest, dest_offset, length); |
4306 nargs); | |
4307 return true; | 4300 return true; |
4308 } | 4301 } |
4309 | 4302 |
4310 //--------------------------------------------------------------------------- | 4303 //--------------------------------------------------------------------------- |
4311 // We will make a fast path for this call to arraycopy. | 4304 // We will make a fast path for this call to arraycopy. |
4358 | 4351 |
4359 // This is where the memory effects are placed: | 4352 // This is where the memory effects are placed: |
4360 const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem); | 4353 const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem); |
4361 generate_arraycopy(adr_type, dest_elem, | 4354 generate_arraycopy(adr_type, dest_elem, |
4362 src, src_offset, dest, dest_offset, length, | 4355 src, src_offset, dest, dest_offset, length, |
4363 nargs, false, false, slow_region); | 4356 false, false, slow_region); |
4364 | 4357 |
4365 return true; | 4358 return true; |
4366 } | 4359 } |
4367 | 4360 |
4368 //-----------------------------generate_arraycopy---------------------- | 4361 //-----------------------------generate_arraycopy---------------------- |
4403 LibraryCallKit::generate_arraycopy(const TypePtr* adr_type, | 4396 LibraryCallKit::generate_arraycopy(const TypePtr* adr_type, |
4404 BasicType basic_elem_type, | 4397 BasicType basic_elem_type, |
4405 Node* src, Node* src_offset, | 4398 Node* src, Node* src_offset, |
4406 Node* dest, Node* dest_offset, | 4399 Node* dest, Node* dest_offset, |
4407 Node* copy_length, | 4400 Node* copy_length, |
4408 int nargs, | |
4409 bool disjoint_bases, | 4401 bool disjoint_bases, |
4410 bool length_never_negative, | 4402 bool length_never_negative, |
4411 RegionNode* slow_region) { | 4403 RegionNode* slow_region) { |
4412 | 4404 |
4413 if (slow_region == NULL) { | 4405 if (slow_region == NULL) { |
4415 record_for_igvn(slow_region); | 4407 record_for_igvn(slow_region); |
4416 } | 4408 } |
4417 | 4409 |
4418 Node* original_dest = dest; | 4410 Node* original_dest = dest; |
4419 AllocateArrayNode* alloc = NULL; // used for zeroing, if needed | 4411 AllocateArrayNode* alloc = NULL; // used for zeroing, if needed |
4420 Node* raw_dest = NULL; // used before zeroing, if needed | |
4421 bool must_clear_dest = false; | 4412 bool must_clear_dest = false; |
4422 | 4413 |
4423 // See if this is the initialization of a newly-allocated array. | 4414 // See if this is the initialization of a newly-allocated array. |
4424 // If so, we will take responsibility here for initializing it to zero. | 4415 // If so, we will take responsibility here for initializing it to zero. |
4425 // (Note: Because tightly_coupled_allocation performs checks on the | 4416 // (Note: Because tightly_coupled_allocation performs checks on the |
4434 && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0 | 4425 && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0 |
4435 && alloc->maybe_set_complete(&_gvn)) { | 4426 && alloc->maybe_set_complete(&_gvn)) { |
4436 // "You break it, you buy it." | 4427 // "You break it, you buy it." |
4437 InitializeNode* init = alloc->initialization(); | 4428 InitializeNode* init = alloc->initialization(); |
4438 assert(init->is_complete(), "we just did this"); | 4429 assert(init->is_complete(), "we just did this"); |
4439 assert(dest->Opcode() == Op_CheckCastPP, "sanity"); | 4430 assert(dest->is_CheckCastPP(), "sanity"); |
4440 assert(dest->in(0)->in(0) == init, "dest pinned"); | 4431 assert(dest->in(0)->in(0) == init, "dest pinned"); |
4441 raw_dest = dest->in(1); // grab the raw pointer! | 4432 |
4442 original_dest = dest; | 4433 // Cast to Object for arraycopy. |
4443 dest = raw_dest; | 4434 // We can't use the original CheckCastPP since it should be moved |
4435 // after the arraycopy to prevent stores flowing above it. | |
4436 Node* new_obj = new(C, 2) CheckCastPPNode(dest->in(0), dest->in(1), | |
4437 TypeInstPtr::NOTNULL); | |
4438 dest = _gvn.transform(new_obj); | |
4439 // Substitute in the locally valid dest_oop. | |
4440 replace_in_map(original_dest, dest); | |
4444 adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory | 4441 adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory |
4445 // Decouple the original InitializeNode, turning it into a simple membar. | |
4446 // We will build a new one at the end of this routine. | |
4447 init->set_req(InitializeNode::RawAddress, top()); | |
4448 // From this point on, every exit path is responsible for | 4442 // From this point on, every exit path is responsible for |
4449 // initializing any non-copied parts of the object to zero. | 4443 // initializing any non-copied parts of the object to zero. |
4450 must_clear_dest = true; | 4444 must_clear_dest = true; |
4451 } else { | 4445 } else { |
4452 // No zeroing elimination here. | 4446 // No zeroing elimination here. |
4485 | 4479 |
4486 if (basic_elem_type == T_CONFLICT) { | 4480 if (basic_elem_type == T_CONFLICT) { |
4487 assert(!must_clear_dest, ""); | 4481 assert(!must_clear_dest, ""); |
4488 Node* cv = generate_generic_arraycopy(adr_type, | 4482 Node* cv = generate_generic_arraycopy(adr_type, |
4489 src, src_offset, dest, dest_offset, | 4483 src, src_offset, dest, dest_offset, |
4490 copy_length, nargs); | 4484 copy_length); |
4491 if (cv == NULL) cv = intcon(-1); // failure (no stub available) | 4485 if (cv == NULL) cv = intcon(-1); // failure (no stub available) |
4492 checked_control = control(); | 4486 checked_control = control(); |
4493 checked_i_o = i_o(); | 4487 checked_i_o = i_o(); |
4494 checked_mem = memory(adr_type); | 4488 checked_mem = memory(adr_type); |
4495 checked_value = cv; | 4489 checked_value = cv; |
4504 // (6) length must not be negative. | 4498 // (6) length must not be negative. |
4505 if (!length_never_negative) { | 4499 if (!length_never_negative) { |
4506 generate_negative_guard(copy_length, slow_region); | 4500 generate_negative_guard(copy_length, slow_region); |
4507 } | 4501 } |
4508 | 4502 |
4503 // copy_length is 0. | |
4509 if (!stopped() && must_clear_dest) { | 4504 if (!stopped() && must_clear_dest) { |
4510 Node* dest_length = alloc->in(AllocateNode::ALength); | 4505 Node* dest_length = alloc->in(AllocateNode::ALength); |
4511 if (_gvn.eqv_uncast(copy_length, dest_length) | 4506 if (_gvn.eqv_uncast(copy_length, dest_length) |
4512 || _gvn.find_int_con(dest_length, 1) <= 0) { | 4507 || _gvn.find_int_con(dest_length, 1) <= 0) { |
4513 // There is no zeroing to do. | 4508 // There is no zeroing to do. No need for a secondary raw memory barrier. |
4514 } else { | 4509 } else { |
4515 // Clear the whole thing since there are no source elements to copy. | 4510 // Clear the whole thing since there are no source elements to copy. |
4516 generate_clear_array(adr_type, dest, basic_elem_type, | 4511 generate_clear_array(adr_type, dest, basic_elem_type, |
4517 intcon(0), NULL, | 4512 intcon(0), NULL, |
4518 alloc->in(AllocateNode::AllocSize)); | 4513 alloc->in(AllocateNode::AllocSize)); |
4514 // Use a secondary InitializeNode as raw memory barrier. | |
4515 // Currently it is needed only on this path since other | |
4516 // paths have stub or runtime calls as raw memory barriers. | |
4517 InitializeNode* init = insert_mem_bar_volatile(Op_Initialize, | |
4518 Compile::AliasIdxRaw, | |
4519 top())->as_Initialize(); | |
4520 init->set_complete(&_gvn); // (there is no corresponding AllocateNode) | |
4519 } | 4521 } |
4520 } | 4522 } |
4521 | 4523 |
4522 // Present the results of the fast call. | 4524 // Present the results of the fast call. |
4523 result_region->init_req(zero_path, control()); | 4525 result_region->init_req(zero_path, control()); |
4635 Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM); | 4637 Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM); |
4636 Node* dest_elem_klass = _gvn.transform(n1); | 4638 Node* dest_elem_klass = _gvn.transform(n1); |
4637 Node* cv = generate_checkcast_arraycopy(adr_type, | 4639 Node* cv = generate_checkcast_arraycopy(adr_type, |
4638 dest_elem_klass, | 4640 dest_elem_klass, |
4639 src, src_offset, dest, dest_offset, | 4641 src, src_offset, dest, dest_offset, |
4640 copy_length, | 4642 copy_length); |
4641 nargs); | |
4642 if (cv == NULL) cv = intcon(-1); // failure (no stub available) | 4643 if (cv == NULL) cv = intcon(-1); // failure (no stub available) |
4643 checked_control = control(); | 4644 checked_control = control(); |
4644 checked_i_o = i_o(); | 4645 checked_i_o = i_o(); |
4645 checked_mem = memory(adr_type); | 4646 checked_mem = memory(adr_type); |
4646 checked_value = cv; | 4647 checked_value = cv; |
4698 record_for_igvn(slow_reg2); | 4699 record_for_igvn(slow_reg2); |
4699 slow_reg2 ->init_req(1, slow_control); | 4700 slow_reg2 ->init_req(1, slow_control); |
4700 slow_i_o2 ->init_req(1, slow_i_o); | 4701 slow_i_o2 ->init_req(1, slow_i_o); |
4701 slow_mem2 ->init_req(1, slow_mem); | 4702 slow_mem2 ->init_req(1, slow_mem); |
4702 slow_reg2 ->init_req(2, control()); | 4703 slow_reg2 ->init_req(2, control()); |
4703 slow_i_o2 ->init_req(2, i_o()); | 4704 slow_i_o2 ->init_req(2, checked_i_o); |
4704 slow_mem2 ->init_req(2, memory(adr_type)); | 4705 slow_mem2 ->init_req(2, checked_mem); |
4705 | 4706 |
4706 slow_control = _gvn.transform(slow_reg2); | 4707 slow_control = _gvn.transform(slow_reg2); |
4707 slow_i_o = _gvn.transform(slow_i_o2); | 4708 slow_i_o = _gvn.transform(slow_i_o2); |
4708 slow_mem = _gvn.transform(slow_mem2); | 4709 slow_mem = _gvn.transform(slow_mem2); |
4709 | 4710 |
4744 generate_clear_array(adr_type, dest, basic_elem_type, | 4745 generate_clear_array(adr_type, dest, basic_elem_type, |
4745 intcon(0), NULL, | 4746 intcon(0), NULL, |
4746 alloc->in(AllocateNode::AllocSize)); | 4747 alloc->in(AllocateNode::AllocSize)); |
4747 } | 4748 } |
4748 | 4749 |
4749 if (dest != original_dest) { | |
4750 // Promote from rawptr to oop, so it looks right in the call's GC map. | |
4751 dest = _gvn.transform( new(C,2) CheckCastPPNode(control(), dest, | |
4752 TypeInstPtr::NOTNULL) ); | |
4753 | |
4754 // Edit the call's debug-info to avoid referring to original_dest. | |
4755 // (The problem with original_dest is that it isn't ready until | |
4756 // after the InitializeNode completes, but this stuff is before.) | |
4757 // Substitute in the locally valid dest_oop. | |
4758 replace_in_map(original_dest, dest); | |
4759 } | |
4760 | |
4761 generate_slow_arraycopy(adr_type, | 4750 generate_slow_arraycopy(adr_type, |
4762 src, src_offset, dest, dest_offset, | 4751 src, src_offset, dest, dest_offset, |
4763 copy_length, nargs); | 4752 copy_length); |
4764 | 4753 |
4765 result_region->init_req(slow_call_path, control()); | 4754 result_region->init_req(slow_call_path, control()); |
4766 result_i_o ->init_req(slow_call_path, i_o()); | 4755 result_i_o ->init_req(slow_call_path, i_o()); |
4767 result_memory->init_req(slow_call_path, memory(adr_type)); | 4756 result_memory->init_req(slow_call_path, memory(adr_type)); |
4768 } | 4757 } |
4778 set_i_o( _gvn.transform(result_i_o) ); | 4767 set_i_o( _gvn.transform(result_i_o) ); |
4779 set_memory( _gvn.transform(result_memory), adr_type ); | 4768 set_memory( _gvn.transform(result_memory), adr_type ); |
4780 | 4769 |
4781 if (dest != original_dest) { | 4770 if (dest != original_dest) { |
4782 // Pin the "finished" array node after the arraycopy/zeroing operations. | 4771 // Pin the "finished" array node after the arraycopy/zeroing operations. |
4783 // Use a secondary InitializeNode memory barrier. | |
4784 InitializeNode* init = insert_mem_bar_volatile(Op_Initialize, | |
4785 Compile::AliasIdxRaw, | |
4786 raw_dest)->as_Initialize(); | |
4787 init->set_complete(&_gvn); // (there is no corresponding AllocateNode) | |
4788 _gvn.hash_delete(original_dest); | 4772 _gvn.hash_delete(original_dest); |
4789 original_dest->set_req(0, control()); | 4773 original_dest->set_req(0, control()); |
4774 // Replace raw memory edge with new CheckCastPP to have a live oop | |
4775 // at safepoints instead of raw value. | |
4776 assert(dest->is_CheckCastPP() && dest->in(1) == original_dest->in(1), "sanity"); | |
4777 original_dest->set_req(1, dest); // cast to the original type | |
4790 _gvn.hash_find_insert(original_dest); // put back into GVN table | 4778 _gvn.hash_find_insert(original_dest); // put back into GVN table |
4791 } | 4779 // Restore in the locally valid dest_oop. |
4792 | 4780 replace_in_map(dest, original_dest); |
4781 } | |
4793 // The memory edges above are precise in order to model effects around | 4782 // The memory edges above are precise in order to model effects around |
4794 // array copies accurately to allow value numbering of field loads around | 4783 // array copies accurately to allow value numbering of field loads around |
4795 // arraycopy. Such field loads, both before and after, are common in Java | 4784 // arraycopy. Such field loads, both before and after, are common in Java |
4796 // collections and similar classes involving header/array data structures. | 4785 // collections and similar classes involving header/array data structures. |
4797 // | 4786 // |
5071 // but without the native wrapper overhead. | 5060 // but without the native wrapper overhead. |
5072 void | 5061 void |
5073 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type, | 5062 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type, |
5074 Node* src, Node* src_offset, | 5063 Node* src, Node* src_offset, |
5075 Node* dest, Node* dest_offset, | 5064 Node* dest, Node* dest_offset, |
5076 Node* copy_length, | 5065 Node* copy_length) { |
5077 int nargs) { | |
5078 _sp += nargs; // any deopt will start just before call to enclosing method | |
5079 Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON, | 5066 Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON, |
5080 OptoRuntime::slow_arraycopy_Type(), | 5067 OptoRuntime::slow_arraycopy_Type(), |
5081 OptoRuntime::slow_arraycopy_Java(), | 5068 OptoRuntime::slow_arraycopy_Java(), |
5082 "slow_arraycopy", adr_type, | 5069 "slow_arraycopy", adr_type, |
5083 src, src_offset, dest, dest_offset, | 5070 src, src_offset, dest, dest_offset, |
5084 copy_length); | 5071 copy_length); |
5085 _sp -= nargs; | |
5086 | 5072 |
5087 // Handle exceptions thrown by this fellow: | 5073 // Handle exceptions thrown by this fellow: |
5088 make_slow_call_ex(call, env()->Throwable_klass(), false); | 5074 make_slow_call_ex(call, env()->Throwable_klass(), false); |
5089 } | 5075 } |
5090 | 5076 |
5092 Node* | 5078 Node* |
5093 LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type, | 5079 LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type, |
5094 Node* dest_elem_klass, | 5080 Node* dest_elem_klass, |
5095 Node* src, Node* src_offset, | 5081 Node* src, Node* src_offset, |
5096 Node* dest, Node* dest_offset, | 5082 Node* dest, Node* dest_offset, |
5097 Node* copy_length, | 5083 Node* copy_length) { |
5098 int nargs) { | |
5099 if (stopped()) return NULL; | 5084 if (stopped()) return NULL; |
5100 | 5085 |
5101 address copyfunc_addr = StubRoutines::checkcast_arraycopy(); | 5086 address copyfunc_addr = StubRoutines::checkcast_arraycopy(); |
5102 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. | 5087 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. |
5103 return NULL; | 5088 return NULL; |
5134 // Helper function; generates code for cases requiring runtime checks. | 5119 // Helper function; generates code for cases requiring runtime checks. |
5135 Node* | 5120 Node* |
5136 LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type, | 5121 LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type, |
5137 Node* src, Node* src_offset, | 5122 Node* src, Node* src_offset, |
5138 Node* dest, Node* dest_offset, | 5123 Node* dest, Node* dest_offset, |
5139 Node* copy_length, | 5124 Node* copy_length) { |
5140 int nargs) { | |
5141 if (stopped()) return NULL; | 5125 if (stopped()) return NULL; |
5142 | 5126 |
5143 address copyfunc_addr = StubRoutines::generic_arraycopy(); | 5127 address copyfunc_addr = StubRoutines::generic_arraycopy(); |
5144 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. | 5128 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. |
5145 return NULL; | 5129 return NULL; |