Mercurial > hg > graal-compiler

comparison src/share/vm/opto/graphKit.cpp @ 851:fc4be448891f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
6851742: (EA) allocation elimination doesn't work with UseG1GC Summary: Fix eliminate_card_mark() to eliminate G1 pre/post barriers. Reviewed-by: never
author kvn
date Thu, 16 Jul 2009 14:10:42 -0700
parents bf3489cc0aa0
children 9987d9d5eb0e
comparison
equal deleted inserted replaced
850:fd50a67f97d1 851:fc4be448891f
1371 record_for_igvn(st); 1371 record_for_igvn(st);
1372 1372
1373 return st; 1373 return st;
1374 } 1374 }
1375 1375
1376
1376 void GraphKit::pre_barrier(Node* ctl, 1377 void GraphKit::pre_barrier(Node* ctl,
1377 Node* obj, 1378 Node* obj,
1378 Node* adr, 1379 Node* adr,
1379 uint adr_idx, 1380 uint adr_idx,
1380 Node *val, 1381 Node* val,
1381 const TypeOopPtr* val_type, 1382 const TypeOopPtr* val_type,
1382 BasicType bt) { 1383 BasicType bt) {
1383 BarrierSet* bs = Universe::heap()->barrier_set(); 1384 BarrierSet* bs = Universe::heap()->barrier_set();
1384 set_control(ctl); 1385 set_control(ctl);
1385 switch (bs->kind()) { 1386 switch (bs->kind()) {
1386 case BarrierSet::G1SATBCT: 1387 case BarrierSet::G1SATBCT:
1387 case BarrierSet::G1SATBCTLogging: 1388 case BarrierSet::G1SATBCTLogging:
1388 g1_write_barrier_pre(obj, adr, adr_idx, val, val_type, bt); 1389 g1_write_barrier_pre(obj, adr, adr_idx, val, val_type, bt);
1389 break; 1390 break;
1390 1391
1391 case BarrierSet::CardTableModRef: 1392 case BarrierSet::CardTableModRef:
1392 case BarrierSet::CardTableExtension: 1393 case BarrierSet::CardTableExtension:
1393 case BarrierSet::ModRef: 1394 case BarrierSet::ModRef:
1402 1403
1403 void GraphKit::post_barrier(Node* ctl, 1404 void GraphKit::post_barrier(Node* ctl,
1404 Node* store, 1405 Node* store,
1405 Node* obj, 1406 Node* obj,
1406 Node* adr, 1407 Node* adr,
1407 uint adr_idx, 1408 uint adr_idx,
1408 Node *val, 1409 Node* val,
1409 BasicType bt, 1410 BasicType bt,
1410 bool use_precise) { 1411 bool use_precise) {
1411 BarrierSet* bs = Universe::heap()->barrier_set(); 1412 BarrierSet* bs = Universe::heap()->barrier_set();
1412 set_control(ctl); 1413 set_control(ctl);
1413 switch (bs->kind()) { 1414 switch (bs->kind()) {
1414 case BarrierSet::G1SATBCT: 1415 case BarrierSet::G1SATBCT:
1415 case BarrierSet::G1SATBCTLogging: 1416 case BarrierSet::G1SATBCTLogging:
1416 g1_write_barrier_post(store, obj, adr, adr_idx, val, bt, use_precise); 1417 g1_write_barrier_post(store, obj, adr, adr_idx, val, bt, use_precise);
1417 break; 1418 break;
1418 1419
1419 case BarrierSet::CardTableModRef: 1420 case BarrierSet::CardTableModRef:
1420 case BarrierSet::CardTableExtension: 1421 case BarrierSet::CardTableExtension:
1421 write_barrier_post(store, obj, adr, val, use_precise); 1422 write_barrier_post(store, obj, adr, val, use_precise);
1429 ShouldNotReachHere(); 1430 ShouldNotReachHere();
1430 1431
1431 } 1432 }
1432 } 1433 }
1433 1434
1434 Node* GraphKit::store_oop_to_object(Node* ctl, 1435 Node* GraphKit::store_oop(Node* ctl,
1435 Node* obj, 1436 Node* obj,
1436 Node* adr, 1437 Node* adr,
1437 const TypePtr* adr_type, 1438 const TypePtr* adr_type,
1438 Node *val, 1439 Node* val,
1439 const TypeOopPtr* val_type, 1440 const TypeOopPtr* val_type,
1440 BasicType bt) { 1441 BasicType bt,
1442 bool use_precise) {
1443
1444 set_control(ctl);
1445 if (stopped()) return top(); // Dead path ?
1446
1447 assert(bt == T_OBJECT, "sanity");
1448 assert(val != NULL, "not dead path");
1441 uint adr_idx = C->get_alias_index(adr_type); 1449 uint adr_idx = C->get_alias_index(adr_type);
1442 Node* store; 1450 assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
1443 pre_barrier(ctl, obj, adr, adr_idx, val, val_type, bt); 1451
1444 store = store_to_memory(control(), adr, val, bt, adr_idx); 1452 pre_barrier(control(), obj, adr, adr_idx, val, val_type, bt);
1445 post_barrier(control(), store, obj, adr, adr_idx, val, bt, false); 1453 Node* store = store_to_memory(control(), adr, val, bt, adr_idx);
1454 post_barrier(control(), store, obj, adr, adr_idx, val, bt, use_precise);
1446 return store; 1455 return store;
1447 } 1456 }
1448 1457
1449 Node* GraphKit::store_oop_to_array(Node* ctl, 1458 // Could be an array or object we don't know at compile time (unsafe ref.)
1450 Node* obj,
1451 Node* adr,
1452 const TypePtr* adr_type,
1453 Node *val,
1454 const TypeOopPtr* val_type,
1455 BasicType bt) {
1456 uint adr_idx = C->get_alias_index(adr_type);
1457 Node* store;
1458 pre_barrier(ctl, obj, adr, adr_idx, val, val_type, bt);
1459 store = store_to_memory(control(), adr, val, bt, adr_idx);
1460 post_barrier(control(), store, obj, adr, adr_idx, val, bt, true);
1461 return store;
1462 }
1463
1464 Node* GraphKit::store_oop_to_unknown(Node* ctl, 1459 Node* GraphKit::store_oop_to_unknown(Node* ctl,
1465 Node* obj, 1460 Node* obj, // containing obj
1466 Node* adr, 1461 Node* adr, // actual adress to store val at
1467 const TypePtr* adr_type, 1462 const TypePtr* adr_type,
1468 Node *val, 1463 Node* val,
1469 BasicType bt) { 1464 BasicType bt) {
1470 Compile::AliasType* at = C->alias_type(adr_type); 1465 Compile::AliasType* at = C->alias_type(adr_type);
1471 const TypeOopPtr* val_type = NULL; 1466 const TypeOopPtr* val_type = NULL;
1472 if (adr_type->isa_instptr()) { 1467 if (adr_type->isa_instptr()) {
1473 if (at->field() != NULL) { 1468 if (at->field() != NULL) {
1474 // known field. This code is a copy of the do_put_xxx logic. 1469 // known field. This code is a copy of the do_put_xxx logic.
1483 val_type = adr_type->is_aryptr()->elem()->make_oopptr(); 1478 val_type = adr_type->is_aryptr()->elem()->make_oopptr();
1484 } 1479 }
1485 if (val_type == NULL) { 1480 if (val_type == NULL) {
1486 val_type = TypeInstPtr::BOTTOM; 1481 val_type = TypeInstPtr::BOTTOM;
1487 } 1482 }
1488 1483 return store_oop(ctl, obj, adr, adr_type, val, val_type, bt, true);
1489 uint adr_idx = at->index();
1490 pre_barrier(ctl, obj, adr, adr_idx, val, val_type, bt);
1491 Node* store = store_to_memory(control(), adr, val, bt, adr_idx);
1492 post_barrier(control(), store, obj, adr, adr_idx, val, bt, true);
1493 return store;
1494 } 1484 }
1495 1485
1496 1486
1497 //-------------------------array_element_address------------------------- 1487 //-------------------------array_element_address-------------------------
1498 Node* GraphKit::array_element_address(Node* ary, Node* idx, BasicType elembt, 1488 Node* GraphKit::array_element_address(Node* ary, Node* idx, BasicType elembt,
1799 // it does not require card marks. 1789 // it does not require card marks.
1800 Node* GraphKit::just_allocated_object(Node* current_control) { 1790 Node* GraphKit::just_allocated_object(Node* current_control) {
1801 if (C->recent_alloc_ctl() == current_control) 1791 if (C->recent_alloc_ctl() == current_control)
1802 return C->recent_alloc_obj(); 1792 return C->recent_alloc_obj();
1803 return NULL; 1793 return NULL;
1804 }
1805
1806
1807 //------------------------------store_barrier----------------------------------
1808 // Insert a write-barrier store. This is to let generational GC work; we have
1809 // to flag all oop-stores before the next GC point.
1810 void GraphKit::write_barrier_post(Node* oop_store, Node* obj, Node* adr,
1811 Node* val, bool use_precise) {
1812 // No store check needed if we're storing a NULL or an old object
1813 // (latter case is probably a string constant). The concurrent
1814 // mark sweep garbage collector, however, needs to have all nonNull
1815 // oop updates flagged via card-marks.
1816 if (val != NULL && val->is_Con()) {
1817 // must be either an oop or NULL
1818 const Type* t = val->bottom_type();
1819 if (t == TypePtr::NULL_PTR || t == Type::TOP)
1820 // stores of null never (?) need barriers
1821 return;
1822 ciObject* con = t->is_oopptr()->const_oop();
1823 if (con != NULL
1824 && con->is_perm()
1825 && Universe::heap()->can_elide_permanent_oop_store_barriers())
1826 // no store barrier needed, because no old-to-new ref created
1827 return;
1828 }
1829
1830 if (use_ReduceInitialCardMarks()
1831 && obj == just_allocated_object(control())) {
1832 // We can skip marks on a freshly-allocated object.
1833 // Keep this code in sync with do_eager_card_mark in runtime.cpp.
1834 // That routine eagerly marks the occasional object which is produced
1835 // by the slow path, so that we don't have to do it here.
1836 return;
1837 }
1838
1839 if (!use_precise) {
1840 // All card marks for a (non-array) instance are in one place:
1841 adr = obj;
1842 }
1843 // (Else it's an array (or unknown), and we want more precise card marks.)
1844 assert(adr != NULL, "");
1845
1846 // Get the alias_index for raw card-mark memory
1847 int adr_type = Compile::AliasIdxRaw;
1848 // Convert the pointer to an int prior to doing math on it
1849 Node* cast = _gvn.transform(new (C, 2) CastP2XNode(control(), adr));
1850 // Divide by card size
1851 assert(Universe::heap()->barrier_set()->kind() == BarrierSet::CardTableModRef,
1852 "Only one we handle so far.");
1853 CardTableModRefBS* ct =
1854 (CardTableModRefBS*)(Universe::heap()->barrier_set());
1855 Node *b = _gvn.transform(new (C, 3) URShiftXNode( cast, _gvn.intcon(CardTableModRefBS::card_shift) ));
1856 // We store into a byte array, so do not bother to left-shift by zero
1857 Node *c = byte_map_base_node();
1858 // Combine
1859 Node *sb_ctl = control();
1860 Node *sb_adr = _gvn.transform(new (C, 4) AddPNode( top()/*no base ptr*/, c, b ));
1861 Node *sb_val = _gvn.intcon(0);
1862 // Smash zero into card
1863 if( !UseConcMarkSweepGC ) {
1864 BasicType bt = T_BYTE;
1865 store_to_memory(sb_ctl, sb_adr, sb_val, bt, adr_type);
1866 } else {
1867 // Specialized path for CM store barrier
1868 cms_card_mark( sb_ctl, sb_adr, sb_val, oop_store);
1869 }
1870 }
1871
1872 // Specialized path for CMS store barrier
1873 void GraphKit::cms_card_mark(Node* ctl, Node* adr, Node* val, Node *oop_store) {
1874 BasicType bt = T_BYTE;
1875 int adr_idx = Compile::AliasIdxRaw;
1876 Node* mem = memory(adr_idx);
1877
1878 // The type input is NULL in PRODUCT builds
1879 const TypePtr* type = NULL;
1880 debug_only(type = C->get_adr_type(adr_idx));
1881
1882 // Add required edge to oop_store, optimizer does not support precedence edges.
1883 // Convert required edge to precedence edge before allocation.
1884 Node *store = _gvn.transform( new (C, 5) StoreCMNode(ctl, mem, adr, type, val, oop_store) );
1885 set_memory(store, adr_idx);
1886
1887 // For CMS, back-to-back card-marks can only remove the first one
1888 // and this requires DU info. Push on worklist for optimizer.
1889 if (mem->req() > MemNode::Address && adr == mem->in(MemNode::Address))
1890 record_for_igvn(store);
1891 } 1794 }
1892 1795
1893 1796
1894 void GraphKit::round_double_arguments(ciMethod* dest_method) { 1797 void GraphKit::round_double_arguments(ciMethod* dest_method) {
1895 // (Note: TypeFunc::make has a cache that makes this fast.) 1798 // (Note: TypeFunc::make has a cache that makes this fast.)
3213 } 3116 }
3214 } 3117 }
3215 return NULL; 3118 return NULL;
3216 } 3119 }
3217 3120
3121 //----------------------------- store barriers ----------------------------
3122 #define __ ideal.
3123
3124 void GraphKit::sync_kit(IdealKit& ideal) {
3125 // Final sync IdealKit and graphKit.
3126 __ drain_delay_transform();
3127 set_all_memory(__ merged_memory());
3128 set_control(__ ctrl());
3129 }
3130
3131 // vanilla/CMS post barrier
3132 // Insert a write-barrier store. This is to let generational GC work; we have
3133 // to flag all oop-stores before the next GC point.
3134 void GraphKit::write_barrier_post(Node* oop_store,
3135 Node* obj,
3136 Node* adr,
3137 Node* val,
3138 bool use_precise) {
3139 // No store check needed if we're storing a NULL or an old object
3140 // (latter case is probably a string constant). The concurrent
3141 // mark sweep garbage collector, however, needs to have all nonNull
3142 // oop updates flagged via card-marks.
3143 if (val != NULL && val->is_Con()) {
3144 // must be either an oop or NULL
3145 const Type* t = val->bottom_type();
3146 if (t == TypePtr::NULL_PTR || t == Type::TOP)
3147 // stores of null never (?) need barriers
3148 return;
3149 ciObject* con = t->is_oopptr()->const_oop();
3150 if (con != NULL
3151 && con->is_perm()
3152 && Universe::heap()->can_elide_permanent_oop_store_barriers())
3153 // no store barrier needed, because no old-to-new ref created
3154 return;
3155 }
3156
3157 if (!use_precise) {
3158 // All card marks for a (non-array) instance are in one place:
3159 adr = obj;
3160 }
3161 // (Else it's an array (or unknown), and we want more precise card marks.)
3162 assert(adr != NULL, "");
3163
3164 IdealKit ideal(gvn(), control(), merged_memory(), true);
3165
3166 // Convert the pointer to an int prior to doing math on it
3167 Node* cast = __ CastPX(__ ctrl(), adr);
3168
3169 // Divide by card size
3170 assert(Universe::heap()->barrier_set()->kind() == BarrierSet::CardTableModRef,
3171 "Only one we handle so far.");
3172 Node* card_offset = __ URShiftX( cast, __ ConI(CardTableModRefBS::card_shift) );
3173
3174 // Combine card table base and card offset
3175 Node* card_adr = __ AddP(__ top(), byte_map_base_node(), card_offset );
3176
3177 // Get the alias_index for raw card-mark memory
3178 int adr_type = Compile::AliasIdxRaw;
3179 // Smash zero into card
3180 Node* zero = __ ConI(0);
3181 BasicType bt = T_BYTE;
3182 if( !UseConcMarkSweepGC ) {
3183 __ store(__ ctrl(), card_adr, zero, bt, adr_type);
3184 } else {
3185 // Specialized path for CM store barrier
3186 __ storeCM(__ ctrl(), card_adr, zero, oop_store, bt, adr_type);
3187 }
3188
3189 // Final sync IdealKit and GraphKit.
3190 sync_kit(ideal);
3191 }
3192
3193 // G1 pre/post barriers
3218 void GraphKit::g1_write_barrier_pre(Node* obj, 3194 void GraphKit::g1_write_barrier_pre(Node* obj,
3219 Node* adr, 3195 Node* adr,
3220 uint alias_idx, 3196 uint alias_idx,
3221 Node* val, 3197 Node* val,
3222 const TypeOopPtr* val_type, 3198 const TypeOopPtr* val_type,
3223 BasicType bt) { 3199 BasicType bt) {
3224 IdealKit ideal(gvn(), control(), merged_memory(), true); 3200 IdealKit ideal(gvn(), control(), merged_memory(), true);
3225 #define __ ideal. 3201
3226 __ declares_done(); 3202 Node* tls = __ thread(); // ThreadLocalStorage
3227
3228 Node* thread = __ thread();
3229 3203
3230 Node* no_ctrl = NULL; 3204 Node* no_ctrl = NULL;
3231 Node* no_base = __ top(); 3205 Node* no_base = __ top();
3232 Node* zero = __ ConI(0); 3206 Node* zero = __ ConI(0);
3233 3207
3246 PtrQueue::byte_offset_of_buf()); 3220 PtrQueue::byte_offset_of_buf());
3247 // Now the actual pointers into the thread 3221 // Now the actual pointers into the thread
3248 3222
3249 // set_control( ctl); 3223 // set_control( ctl);
3250 3224
3251 Node* marking_adr = __ AddP(no_base, thread, __ ConX(marking_offset)); 3225 Node* marking_adr = __ AddP(no_base, tls, __ ConX(marking_offset));
3252 Node* buffer_adr = __ AddP(no_base, thread, __ ConX(buffer_offset)); 3226 Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
3253 Node* index_adr = __ AddP(no_base, thread, __ ConX(index_offset)); 3227 Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset));
3254 3228
3255 // Now some of the values 3229 // Now some of the values
3256 3230
3257 Node* marking = __ load(__ ctrl(), marking_adr, TypeInt::INT, active_type, Compile::AliasIdxRaw); 3231 Node* marking = __ load(__ ctrl(), marking_adr, TypeInt::INT, active_type, Compile::AliasIdxRaw);
3258 3232
3276 3250
3277 // decrement the index 3251 // decrement the index
3278 Node* next_index = __ SubI(index, __ ConI(sizeof(intptr_t))); 3252 Node* next_index = __ SubI(index, __ ConI(sizeof(intptr_t)));
3279 Node* next_indexX = next_index; 3253 Node* next_indexX = next_index;
3280 #ifdef _LP64 3254 #ifdef _LP64
3281 // We could refine the type for what it's worth 3255 // We could refine the type for what it's worth
3282 // const TypeLong* lidxtype = TypeLong::make(CONST64(0), get_size_from_queue); 3256 // const TypeLong* lidxtype = TypeLong::make(CONST64(0), get_size_from_queue);
3283 next_indexX = _gvn.transform( new (C, 2) ConvI2LNode(next_index, TypeLong::make(0, max_jlong, Type::WidenMax)) ); 3257 next_indexX = _gvn.transform( new (C, 2) ConvI2LNode(next_index, TypeLong::make(0, max_jlong, Type::WidenMax)) );
3284 #endif // _LP64 3258 #endif
3285 3259
3286 // Now get the buffer location we will log the original value into and store it 3260 // Now get the buffer location we will log the original value into and store it
3287
3288 Node *log_addr = __ AddP(no_base, buffer, next_indexX); 3261 Node *log_addr = __ AddP(no_base, buffer, next_indexX);
3289 // __ store(__ ctrl(), log_addr, orig, T_OBJECT, C->get_alias_index(TypeOopPtr::BOTTOM));
3290 __ store(__ ctrl(), log_addr, orig, T_OBJECT, Compile::AliasIdxRaw); 3262 __ store(__ ctrl(), log_addr, orig, T_OBJECT, Compile::AliasIdxRaw);
3291 3263
3292
3293 // update the index 3264 // update the index
3294 // __ store(__ ctrl(), index_adr, next_index, T_INT, Compile::AliasIdxRaw); 3265 __ store(__ ctrl(), index_adr, next_index, T_INT, Compile::AliasIdxRaw);
3295 // This is a hack to force this store to occur before the oop store that is coming up
3296 __ store(__ ctrl(), index_adr, next_index, T_INT, C->get_alias_index(TypeOopPtr::BOTTOM));
3297 3266
3298 } __ else_(); { 3267 } __ else_(); {
3299 3268
3300 // logging buffer is full, call the runtime 3269 // logging buffer is full, call the runtime
3301 const TypeFunc *tf = OptoRuntime::g1_wb_pre_Type(); 3270 const TypeFunc *tf = OptoRuntime::g1_wb_pre_Type();
3302 // __ make_leaf_call(tf, OptoRuntime::g1_wb_pre_Java(), "g1_wb_pre", orig, thread); 3271 __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), "g1_wb_pre", orig, tls);
3303 __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), "g1_wb_pre", orig, thread); 3272 } __ end_if(); // (!index)
3304 } __ end_if(); 3273 } __ end_if(); // (orig != NULL)
3305 } __ end_if(); 3274 } __ end_if(); // (!marking)
3306 } __ end_if(); 3275
3307 3276 // Final sync IdealKit and GraphKit.
3308 __ drain_delay_transform(); 3277 sync_kit(ideal);
3309 set_control( __ ctrl());
3310 set_all_memory( __ merged_memory());
3311
3312 #undef __
3313 } 3278 }
3314 3279
3315 // 3280 //
3316 // Update the card table and add card address to the queue 3281 // Update the card table and add card address to the queue
3317 // 3282 //
3318 void GraphKit::g1_mark_card(IdealKit* ideal, Node* card_adr, Node* store, Node* index, Node* index_adr, Node* buffer, const TypeFunc* tf) { 3283 void GraphKit::g1_mark_card(IdealKit& ideal,
3319 #define __ ideal-> 3284 Node* card_adr,
3285 Node* oop_store,
3286 Node* index,
3287 Node* index_adr,
3288 Node* buffer,
3289 const TypeFunc* tf) {
3290
3320 Node* zero = __ ConI(0); 3291 Node* zero = __ ConI(0);
3321 Node* no_base = __ top(); 3292 Node* no_base = __ top();
3322 BasicType card_bt = T_BYTE; 3293 BasicType card_bt = T_BYTE;
3323 // Smash zero into card. MUST BE ORDERED WRT TO STORE 3294 // Smash zero into card. MUST BE ORDERED WRT TO STORE
3324 __ storeCM(__ ctrl(), card_adr, zero, store, card_bt, Compile::AliasIdxRaw); 3295 __ storeCM(__ ctrl(), card_adr, zero, oop_store, card_bt, Compile::AliasIdxRaw);
3325 3296
3326 // Now do the queue work 3297 // Now do the queue work
3327 __ if_then(index, BoolTest::ne, zero); { 3298 __ if_then(index, BoolTest::ne, zero); {
3328 3299
3329 Node* next_index = __ SubI(index, __ ConI(sizeof(intptr_t))); 3300 Node* next_index = __ SubI(index, __ ConI(sizeof(intptr_t)));
3330 Node* next_indexX = next_index; 3301 Node* next_indexX = next_index;
3331 #ifdef _LP64 3302 #ifdef _LP64
3332 // We could refine the type for what it's worth 3303 // We could refine the type for what it's worth
3333 // const TypeLong* lidxtype = TypeLong::make(CONST64(0), get_size_from_queue); 3304 // const TypeLong* lidxtype = TypeLong::make(CONST64(0), get_size_from_queue);
3334 next_indexX = _gvn.transform( new (C, 2) ConvI2LNode(next_index, TypeLong::make(0, max_jlong, Type::WidenMax)) ); 3305 next_indexX = _gvn.transform( new (C, 2) ConvI2LNode(next_index, TypeLong::make(0, max_jlong, Type::WidenMax)) );
3339 __ store(__ ctrl(), index_adr, next_index, T_INT, Compile::AliasIdxRaw); 3310 __ store(__ ctrl(), index_adr, next_index, T_INT, Compile::AliasIdxRaw);
3340 3311
3341 } __ else_(); { 3312 } __ else_(); {
3342 __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), "g1_wb_post", card_adr, __ thread()); 3313 __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), "g1_wb_post", card_adr, __ thread());
3343 } __ end_if(); 3314 } __ end_if();
3344 #undef __ 3315
3345 } 3316 }
3346 3317
3347 void GraphKit::g1_write_barrier_post(Node* store, 3318 void GraphKit::g1_write_barrier_post(Node* oop_store,
3348 Node* obj, 3319 Node* obj,
3349 Node* adr, 3320 Node* adr,
3350 uint alias_idx, 3321 uint alias_idx,
3351 Node* val, 3322 Node* val,
3352 BasicType bt, 3323 BasicType bt,
3367 } 3338 }
3368 // (Else it's an array (or unknown), and we want more precise card marks.) 3339 // (Else it's an array (or unknown), and we want more precise card marks.)
3369 assert(adr != NULL, ""); 3340 assert(adr != NULL, "");
3370 3341
3371 IdealKit ideal(gvn(), control(), merged_memory(), true); 3342 IdealKit ideal(gvn(), control(), merged_memory(), true);
3372 #define __ ideal. 3343
3373 __ declares_done(); 3344 Node* tls = __ thread(); // ThreadLocalStorage
3374
3375 Node* thread = __ thread();
3376 3345
3377 Node* no_ctrl = NULL; 3346 Node* no_ctrl = NULL;
3378 Node* no_base = __ top(); 3347 Node* no_base = __ top();
3379 float likely = PROB_LIKELY(0.999); 3348 float likely = PROB_LIKELY(0.999);
3380 float unlikely = PROB_UNLIKELY(0.999); 3349 float unlikely = PROB_UNLIKELY(0.999);
3392 const int buffer_offset = in_bytes(JavaThread::dirty_card_queue_offset() + 3361 const int buffer_offset = in_bytes(JavaThread::dirty_card_queue_offset() +
3393 PtrQueue::byte_offset_of_buf()); 3362 PtrQueue::byte_offset_of_buf());
3394 3363
3395 // Pointers into the thread 3364 // Pointers into the thread
3396 3365
3397 Node* buffer_adr = __ AddP(no_base, thread, __ ConX(buffer_offset)); 3366 Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
3398 Node* index_adr = __ AddP(no_base, thread, __ ConX(index_offset)); 3367 Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset));
3399 3368
3400 // Now some values 3369 // Now some values
3401 3370
3402 Node* index = __ load(no_ctrl, index_adr, TypeInt::INT, T_INT, Compile::AliasIdxRaw); 3371 Node* index = __ load(no_ctrl, index_adr, TypeInt::INT, T_INT, Compile::AliasIdxRaw);
3403 Node* buffer = __ load(no_ctrl, buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw); 3372 Node* buffer = __ load(no_ctrl, buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
3404 3373
3405 3374
3406 // Convert the store obj pointer to an int prior to doing math on it 3375 // Convert the store obj pointer to an int prior to doing math on it
3407 // Use addr not obj gets accurate card marks
3408
3409 // Node* cast = __ CastPX(no_ctrl, adr /* obj */);
3410
3411 // Must use ctrl to prevent "integerized oop" existing across safepoint 3376 // Must use ctrl to prevent "integerized oop" existing across safepoint
3412 Node* cast = __ CastPX(__ ctrl(), ( use_precise ? adr : obj )); 3377 Node* cast = __ CastPX(__ ctrl(), adr);
3413 3378
3414 // Divide pointer by card size 3379 // Divide pointer by card size
3415 Node* card_offset = __ URShiftX( cast, __ ConI(CardTableModRefBS::card_shift) ); 3380 Node* card_offset = __ URShiftX( cast, __ ConI(CardTableModRefBS::card_shift) );
3416 3381
3417 // Combine card table base and card offset 3382 // Combine card table base and card offset
3418 Node *card_adr = __ AddP(no_base, byte_map_base_node(), card_offset ); 3383 Node* card_adr = __ AddP(no_base, byte_map_base_node(), card_offset );
3419 3384
3420 // If we know the value being stored does it cross regions? 3385 // If we know the value being stored does it cross regions?
3421 3386
3422 if (val != NULL) { 3387 if (val != NULL) {
3423 // Does the store cause us to cross regions? 3388 // Does the store cause us to cross regions?
3437 3402
3438 // load the original value of the card 3403 // load the original value of the card
3439 Node* card_val = __ load(__ ctrl(), card_adr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); 3404 Node* card_val = __ load(__ ctrl(), card_adr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw);
3440 3405
3441 __ if_then(card_val, BoolTest::ne, zero); { 3406 __ if_then(card_val, BoolTest::ne, zero); {
3442 g1_mark_card(&ideal, card_adr, store, index, index_adr, buffer, tf); 3407 g1_mark_card(ideal, card_adr, oop_store, index, index_adr, buffer, tf);
3443 } __ end_if(); 3408 } __ end_if();
3444 } __ end_if(); 3409 } __ end_if();
3445 } __ end_if(); 3410 } __ end_if();
3446 } else { 3411 } else {
3447 g1_mark_card(&ideal, card_adr, store, index, index_adr, buffer, tf); 3412 // Object.clone() instrinsic uses this path.
3448 } 3413 g1_mark_card(ideal, card_adr, oop_store, index, index_adr, buffer, tf);
3449 3414 }
3450 3415
3451 __ drain_delay_transform(); 3416 // Final sync IdealKit and GraphKit.
3452 set_control( __ ctrl()); 3417 sync_kit(ideal);
3453 set_all_memory( __ merged_memory()); 3418 }
3454 #undef __ 3419 #undef __
3455 3420
3456 }