Mercurial > hg > graal-compiler
annotate src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp @ 353:9bb2c10ac07b
6723570: G1: assertion failure: p == current_top or oop(p)->is_oop(),"p is not a block start" (revisited!)
Summary: Fixed the incorrect assigment to G1OffsetTableContigSpace::_gc_time_stamp. Also added a little more paranoia to operations on a global timestamp.
Reviewed-by: tonyp
author | iveresov |
---|---|
date | Thu, 10 Jul 2008 09:29:54 -0700 |
parents | d1635bf93939 |
children | d1605aabd0a1 12eea04c8b06 |
rev | line source |
---|---|
0 | 1 |
2 /* | |
3 * Copyright 2006-2007 Sun Microsystems, Inc. All Rights Reserved. | |
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |
5 * | |
6 * This code is free software; you can redistribute it and/or modify it | |
7 * under the terms of the GNU General Public License version 2 only, as | |
8 * published by the Free Software Foundation. | |
9 * | |
10 * This code is distributed in the hope that it will be useful, but WITHOUT | |
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
13 * version 2 for more details (a copy is included in the LICENSE file that | |
14 * accompanied this code). | |
15 * | |
16 * You should have received a copy of the GNU General Public License version | |
17 * 2 along with this work; if not, write to the Free Software Foundation, | |
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
19 * | |
20 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, | |
21 * CA 95054 USA or visit www.sun.com if you need additional information or | |
22 * have any questions. | |
23 * | |
24 */ | |
25 | |
26 # include "incls/_precompiled.incl" | |
27 # include "incls/_mutableNUMASpace.cpp.incl" | |
28 | |
29 | |
30 MutableNUMASpace::MutableNUMASpace() { | |
31 _lgrp_spaces = new (ResourceObj::C_HEAP) GrowableArray<LGRPSpace*>(0, true); | |
32 _page_size = os::vm_page_size(); | |
33 _adaptation_cycles = 0; | |
34 _samples_count = 0; | |
35 update_layout(true); | |
36 } | |
37 | |
38 MutableNUMASpace::~MutableNUMASpace() { | |
39 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
40 delete lgrp_spaces()->at(i); | |
41 } | |
42 delete lgrp_spaces(); | |
43 } | |
44 | |
45 void MutableNUMASpace::mangle_unused_area() { | |
46 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
47 LGRPSpace *ls = lgrp_spaces()->at(i); | |
48 MutableSpace *s = ls->space(); | |
141 | 49 if (!os::numa_has_static_binding()) { |
50 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom()); | |
51 if (top < s->end()) { | |
52 ls->add_invalid_region(MemRegion(top, s->end())); | |
53 } | |
0 | 54 } |
55 s->mangle_unused_area(); | |
56 } | |
57 } | |
58 | |
59 // There may be unallocated holes in the middle chunks | |
60 // that should be filled with dead objects to ensure parseability. | |
61 void MutableNUMASpace::ensure_parsability() { | |
62 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
63 LGRPSpace *ls = lgrp_spaces()->at(i); | |
64 MutableSpace *s = ls->space(); | |
144
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
65 if (s->top() < top()) { // For all spaces preceeding the one containing top() |
0 | 66 if (s->free_in_words() > 0) { |
67 SharedHeap::fill_region_with_object(MemRegion(s->top(), s->end())); | |
144
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
68 size_t area_touched_words = pointer_delta(s->end(), s->top()); |
0 | 69 #ifndef ASSERT |
70 if (!ZapUnusedHeapArea) { | |
71 area_touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)), | |
72 area_touched_words); | |
73 } | |
74 #endif | |
141 | 75 if (!os::numa_has_static_binding()) { |
76 MemRegion invalid; | |
77 HeapWord *crossing_start = (HeapWord*)round_to((intptr_t)s->top(), os::vm_page_size()); | |
78 HeapWord *crossing_end = (HeapWord*)round_to((intptr_t)(s->top() + area_touched_words), | |
79 os::vm_page_size()); | |
80 if (crossing_start != crossing_end) { | |
81 // If object header crossed a small page boundary we mark the area | |
82 // as invalid rounding it to a page_size(). | |
83 HeapWord *start = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom()); | |
84 HeapWord *end = MIN2((HeapWord*)round_to((intptr_t)(s->top() + area_touched_words), page_size()), | |
85 s->end()); | |
86 invalid = MemRegion(start, end); | |
87 } | |
88 | |
89 ls->add_invalid_region(invalid); | |
0 | 90 } |
91 } | |
92 } else { | |
141 | 93 if (!os::numa_has_static_binding()) { |
0 | 94 #ifdef ASSERT |
95 MemRegion invalid(s->top(), s->end()); | |
96 ls->add_invalid_region(invalid); | |
141 | 97 #else |
98 if (ZapUnusedHeapArea) { | |
99 MemRegion invalid(s->top(), s->end()); | |
100 ls->add_invalid_region(invalid); | |
144
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
101 } else { |
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
102 return; |
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
103 } |
0 | 104 #endif |
144
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
105 } else { |
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
106 return; |
141 | 107 } |
0 | 108 } |
109 } | |
110 } | |
111 | |
112 size_t MutableNUMASpace::used_in_words() const { | |
113 size_t s = 0; | |
114 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
115 s += lgrp_spaces()->at(i)->space()->used_in_words(); | |
116 } | |
117 return s; | |
118 } | |
119 | |
120 size_t MutableNUMASpace::free_in_words() const { | |
121 size_t s = 0; | |
122 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
123 s += lgrp_spaces()->at(i)->space()->free_in_words(); | |
124 } | |
125 return s; | |
126 } | |
127 | |
128 | |
129 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const { | |
130 guarantee(thr != NULL, "No thread"); | |
131 int lgrp_id = thr->lgrp_id(); | |
132 assert(lgrp_id != -1, "No lgrp_id set"); | |
133 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); | |
134 if (i == -1) { | |
135 return 0; | |
136 } | |
137 return lgrp_spaces()->at(i)->space()->capacity_in_bytes(); | |
138 } | |
139 | |
140 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const { | |
141 guarantee(thr != NULL, "No thread"); | |
142 int lgrp_id = thr->lgrp_id(); | |
143 assert(lgrp_id != -1, "No lgrp_id set"); | |
144 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); | |
145 if (i == -1) { | |
146 return 0; | |
147 } | |
148 return lgrp_spaces()->at(i)->space()->free_in_bytes(); | |
149 } | |
150 | |
151 // Check if the NUMA topology has changed. Add and remove spaces if needed. | |
152 // The update can be forced by setting the force parameter equal to true. | |
153 bool MutableNUMASpace::update_layout(bool force) { | |
154 // Check if the topology had changed. | |
155 bool changed = os::numa_topology_changed(); | |
156 if (force || changed) { | |
157 // Compute lgrp intersection. Add/remove spaces. | |
158 int lgrp_limit = (int)os::numa_get_groups_num(); | |
159 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit); | |
160 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit); | |
161 assert(lgrp_num > 0, "There should be at least one locality group"); | |
162 // Add new spaces for the new nodes | |
163 for (int i = 0; i < lgrp_num; i++) { | |
164 bool found = false; | |
165 for (int j = 0; j < lgrp_spaces()->length(); j++) { | |
166 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) { | |
167 found = true; | |
168 break; | |
169 } | |
170 } | |
171 if (!found) { | |
172 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i])); | |
173 } | |
174 } | |
175 | |
176 // Remove spaces for the removed nodes. | |
177 for (int i = 0; i < lgrp_spaces()->length();) { | |
178 bool found = false; | |
179 for (int j = 0; j < lgrp_num; j++) { | |
180 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) { | |
181 found = true; | |
182 break; | |
183 } | |
184 } | |
185 if (!found) { | |
186 delete lgrp_spaces()->at(i); | |
187 lgrp_spaces()->remove_at(i); | |
188 } else { | |
189 i++; | |
190 } | |
191 } | |
192 | |
193 FREE_C_HEAP_ARRAY(int, lgrp_ids); | |
194 | |
195 if (changed) { | |
196 for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) { | |
197 thread->set_lgrp_id(-1); | |
198 } | |
199 } | |
200 return true; | |
201 } | |
202 return false; | |
203 } | |
204 | |
205 // Bias region towards the first-touching lgrp. Set the right page sizes. | |
141 | 206 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) { |
0 | 207 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size()); |
208 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size()); | |
209 if (end > start) { | |
210 MemRegion aligned_region(start, end); | |
211 assert((intptr_t)aligned_region.start() % page_size() == 0 && | |
212 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment"); | |
213 assert(region().contains(aligned_region), "Sanity"); | |
141 | 214 // First we tell the OS which page size we want in the given range. The underlying |
215 // large page can be broken down if we require small pages. | |
0 | 216 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); |
141 | 217 // Then we uncommit the pages in the range. |
218 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size()); | |
219 // And make them local/first-touch biased. | |
220 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id); | |
0 | 221 } |
222 } | |
223 | |
224 // Free all pages in the region. | |
225 void MutableNUMASpace::free_region(MemRegion mr) { | |
226 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size()); | |
227 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size()); | |
228 if (end > start) { | |
229 MemRegion aligned_region(start, end); | |
230 assert((intptr_t)aligned_region.start() % page_size() == 0 && | |
231 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment"); | |
232 assert(region().contains(aligned_region), "Sanity"); | |
233 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size()); | |
234 } | |
235 } | |
236 | |
237 // Update space layout. Perform adaptation. | |
238 void MutableNUMASpace::update() { | |
239 if (update_layout(false)) { | |
240 // If the topology has changed, make all chunks zero-sized. | |
241 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
242 MutableSpace *s = lgrp_spaces()->at(i)->space(); | |
243 s->set_end(s->bottom()); | |
244 s->set_top(s->bottom()); | |
245 } | |
246 initialize(region(), true); | |
247 } else { | |
248 bool should_initialize = false; | |
141 | 249 if (!os::numa_has_static_binding()) { |
250 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
251 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) { | |
252 should_initialize = true; | |
253 break; | |
254 } | |
0 | 255 } |
256 } | |
257 | |
258 if (should_initialize || | |
259 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) { | |
260 initialize(region(), true); | |
261 } | |
262 } | |
263 | |
264 if (NUMAStats) { | |
265 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
266 lgrp_spaces()->at(i)->accumulate_statistics(page_size()); | |
267 } | |
268 } | |
269 | |
270 scan_pages(NUMAPageScanRate); | |
271 } | |
272 | |
273 // Scan pages. Free pages that have smaller size or wrong placement. | |
274 void MutableNUMASpace::scan_pages(size_t page_count) | |
275 { | |
276 size_t pages_per_chunk = page_count / lgrp_spaces()->length(); | |
277 if (pages_per_chunk > 0) { | |
278 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
279 LGRPSpace *ls = lgrp_spaces()->at(i); | |
280 ls->scan_pages(page_size(), pages_per_chunk); | |
281 } | |
282 } | |
283 } | |
284 | |
285 // Accumulate statistics about the allocation rate of each lgrp. | |
286 void MutableNUMASpace::accumulate_statistics() { | |
287 if (UseAdaptiveNUMAChunkSizing) { | |
288 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
289 lgrp_spaces()->at(i)->sample(); | |
290 } | |
291 increment_samples_count(); | |
292 } | |
293 | |
294 if (NUMAStats) { | |
295 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
296 lgrp_spaces()->at(i)->accumulate_statistics(page_size()); | |
297 } | |
298 } | |
299 } | |
300 | |
301 // Get the current size of a chunk. | |
302 // This function computes the size of the chunk based on the | |
303 // difference between chunk ends. This allows it to work correctly in | |
304 // case the whole space is resized and during the process of adaptive | |
305 // chunk resizing. | |
306 size_t MutableNUMASpace::current_chunk_size(int i) { | |
307 HeapWord *cur_end, *prev_end; | |
308 if (i == 0) { | |
309 prev_end = bottom(); | |
310 } else { | |
311 prev_end = lgrp_spaces()->at(i - 1)->space()->end(); | |
312 } | |
313 if (i == lgrp_spaces()->length() - 1) { | |
314 cur_end = end(); | |
315 } else { | |
316 cur_end = lgrp_spaces()->at(i)->space()->end(); | |
317 } | |
318 if (cur_end > prev_end) { | |
319 return pointer_delta(cur_end, prev_end, sizeof(char)); | |
320 } | |
321 return 0; | |
322 } | |
323 | |
324 // Return the default chunk size by equally diving the space. | |
325 // page_size() aligned. | |
326 size_t MutableNUMASpace::default_chunk_size() { | |
327 return base_space_size() / lgrp_spaces()->length() * page_size(); | |
328 } | |
329 | |
330 // Produce a new chunk size. page_size() aligned. | |
331 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) { | |
332 size_t pages_available = base_space_size(); | |
333 for (int j = 0; j < i; j++) { | |
334 pages_available -= round_down(current_chunk_size(j), page_size()) / page_size(); | |
335 } | |
336 pages_available -= lgrp_spaces()->length() - i - 1; | |
337 assert(pages_available > 0, "No pages left"); | |
338 float alloc_rate = 0; | |
339 for (int j = i; j < lgrp_spaces()->length(); j++) { | |
340 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average(); | |
341 } | |
342 size_t chunk_size = 0; | |
343 if (alloc_rate > 0) { | |
344 LGRPSpace *ls = lgrp_spaces()->at(i); | |
345 chunk_size = (size_t)(ls->alloc_rate()->average() * pages_available / alloc_rate) * page_size(); | |
346 } | |
347 chunk_size = MAX2(chunk_size, page_size()); | |
348 | |
349 if (limit > 0) { | |
350 limit = round_down(limit, page_size()); | |
351 if (chunk_size > current_chunk_size(i)) { | |
352 chunk_size = MIN2((off_t)chunk_size, (off_t)current_chunk_size(i) + (off_t)limit); | |
353 } else { | |
354 chunk_size = MAX2((off_t)chunk_size, (off_t)current_chunk_size(i) - (off_t)limit); | |
355 } | |
356 } | |
357 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range"); | |
358 return chunk_size; | |
359 } | |
360 | |
361 | |
362 // Return the bottom_region and the top_region. Align them to page_size() boundary. | |
363 // |------------------new_region---------------------------------| | |
364 // |----bottom_region--|---intersection---|------top_region------| | |
365 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection, | |
366 MemRegion* bottom_region, MemRegion *top_region) { | |
367 // Is there bottom? | |
368 if (new_region.start() < intersection.start()) { // Yes | |
369 // Try to coalesce small pages into a large one. | |
370 if (UseLargePages && page_size() >= os::large_page_size()) { | |
371 HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), os::large_page_size()); | |
372 if (new_region.contains(p) | |
373 && pointer_delta(p, new_region.start(), sizeof(char)) >= os::large_page_size()) { | |
374 if (intersection.contains(p)) { | |
375 intersection = MemRegion(p, intersection.end()); | |
376 } else { | |
377 intersection = MemRegion(p, p); | |
378 } | |
379 } | |
380 } | |
381 *bottom_region = MemRegion(new_region.start(), intersection.start()); | |
382 } else { | |
383 *bottom_region = MemRegion(); | |
384 } | |
385 | |
386 // Is there top? | |
387 if (intersection.end() < new_region.end()) { // Yes | |
388 // Try to coalesce small pages into a large one. | |
389 if (UseLargePages && page_size() >= os::large_page_size()) { | |
390 HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), os::large_page_size()); | |
391 if (new_region.contains(p) | |
392 && pointer_delta(new_region.end(), p, sizeof(char)) >= os::large_page_size()) { | |
393 if (intersection.contains(p)) { | |
394 intersection = MemRegion(intersection.start(), p); | |
395 } else { | |
396 intersection = MemRegion(p, p); | |
397 } | |
398 } | |
399 } | |
400 *top_region = MemRegion(intersection.end(), new_region.end()); | |
401 } else { | |
402 *top_region = MemRegion(); | |
403 } | |
404 } | |
405 | |
406 // Try to merge the invalid region with the bottom or top region by decreasing | |
407 // the intersection area. Return the invalid_region aligned to the page_size() | |
408 // boundary if it's inside the intersection. Return non-empty invalid_region | |
409 // if it lies inside the intersection (also page-aligned). | |
410 // |------------------new_region---------------------------------| | |
411 // |----------------|-------invalid---|--------------------------| | |
412 // |----bottom_region--|---intersection---|------top_region------| | |
413 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection, | |
414 MemRegion *invalid_region) { | |
415 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) { | |
416 *intersection = MemRegion(invalid_region->end(), intersection->end()); | |
417 *invalid_region = MemRegion(); | |
418 } else | |
419 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) { | |
420 *intersection = MemRegion(intersection->start(), invalid_region->start()); | |
421 *invalid_region = MemRegion(); | |
422 } else | |
423 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) { | |
424 *intersection = MemRegion(new_region.start(), new_region.start()); | |
425 *invalid_region = MemRegion(); | |
426 } else | |
427 if (intersection->contains(invalid_region)) { | |
428 // That's the only case we have to make an additional bias_region() call. | |
429 HeapWord* start = invalid_region->start(); | |
430 HeapWord* end = invalid_region->end(); | |
431 if (UseLargePages && page_size() >= os::large_page_size()) { | |
432 HeapWord *p = (HeapWord*)round_down((intptr_t) start, os::large_page_size()); | |
433 if (new_region.contains(p)) { | |
434 start = p; | |
435 } | |
436 p = (HeapWord*)round_to((intptr_t) end, os::large_page_size()); | |
437 if (new_region.contains(end)) { | |
438 end = p; | |
439 } | |
440 } | |
441 if (intersection->start() > start) { | |
442 *intersection = MemRegion(start, intersection->end()); | |
443 } | |
444 if (intersection->end() < end) { | |
445 *intersection = MemRegion(intersection->start(), end); | |
446 } | |
447 *invalid_region = MemRegion(start, end); | |
448 } | |
449 } | |
450 | |
451 void MutableNUMASpace::initialize(MemRegion mr, bool clear_space) { | |
452 assert(clear_space, "Reallocation will destory data!"); | |
453 assert(lgrp_spaces()->length() > 0, "There should be at least one space"); | |
454 | |
455 MemRegion old_region = region(), new_region; | |
456 set_bottom(mr.start()); | |
457 set_end(mr.end()); | |
458 MutableSpace::set_top(bottom()); | |
459 | |
460 // Compute chunk sizes | |
461 size_t prev_page_size = page_size(); | |
462 set_page_size(UseLargePages ? os::large_page_size() : os::vm_page_size()); | |
463 HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size()); | |
464 HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size()); | |
465 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size(); | |
466 | |
467 // Try small pages if the chunk size is too small | |
468 if (base_space_size_pages / lgrp_spaces()->length() == 0 | |
469 && page_size() > (size_t)os::vm_page_size()) { | |
470 set_page_size(os::vm_page_size()); | |
471 rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size()); | |
472 rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size()); | |
473 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size(); | |
474 } | |
475 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small"); | |
476 set_base_space_size(base_space_size_pages); | |
477 | |
478 // Handle space resize | |
479 MemRegion top_region, bottom_region; | |
480 if (!old_region.equals(region())) { | |
481 new_region = MemRegion(rounded_bottom, rounded_end); | |
482 MemRegion intersection = new_region.intersection(old_region); | |
483 if (intersection.start() == NULL || | |
484 intersection.end() == NULL || | |
485 prev_page_size > page_size()) { // If the page size got smaller we have to change | |
486 // the page size preference for the whole space. | |
487 intersection = MemRegion(new_region.start(), new_region.start()); | |
488 } | |
489 select_tails(new_region, intersection, &bottom_region, &top_region); | |
141 | 490 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id()); |
491 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id()); | |
0 | 492 } |
493 | |
494 // Check if the space layout has changed significantly? | |
495 // This happens when the space has been resized so that either head or tail | |
496 // chunk became less than a page. | |
497 bool layout_valid = UseAdaptiveNUMAChunkSizing && | |
498 current_chunk_size(0) > page_size() && | |
499 current_chunk_size(lgrp_spaces()->length() - 1) > page_size(); | |
500 | |
501 | |
502 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
503 LGRPSpace *ls = lgrp_spaces()->at(i); | |
504 MutableSpace *s = ls->space(); | |
505 old_region = s->region(); | |
506 | |
507 size_t chunk_byte_size = 0, old_chunk_byte_size = 0; | |
508 if (i < lgrp_spaces()->length() - 1) { | |
509 if (!UseAdaptiveNUMAChunkSizing || | |
510 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) || | |
511 samples_count() < AdaptiveSizePolicyReadyThreshold) { | |
512 // No adaptation. Divide the space equally. | |
513 chunk_byte_size = default_chunk_size(); | |
514 } else | |
515 if (!layout_valid || NUMASpaceResizeRate == 0) { | |
516 // Fast adaptation. If no space resize rate is set, resize | |
517 // the chunks instantly. | |
518 chunk_byte_size = adaptive_chunk_size(i, 0); | |
519 } else { | |
520 // Slow adaptation. Resize the chunks moving no more than | |
521 // NUMASpaceResizeRate bytes per collection. | |
522 size_t limit = NUMASpaceResizeRate / | |
523 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2); | |
524 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size())); | |
525 } | |
526 | |
527 assert(chunk_byte_size >= page_size(), "Chunk size too small"); | |
528 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check"); | |
529 } | |
530 | |
531 if (i == 0) { // Bottom chunk | |
532 if (i != lgrp_spaces()->length() - 1) { | |
533 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize)); | |
534 } else { | |
535 new_region = MemRegion(bottom(), end()); | |
536 } | |
537 } else | |
538 if (i < lgrp_spaces()->length() - 1) { // Middle chunks | |
539 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space(); | |
540 new_region = MemRegion(ps->end(), | |
541 ps->end() + (chunk_byte_size >> LogHeapWordSize)); | |
542 } else { // Top chunk | |
543 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space(); | |
544 new_region = MemRegion(ps->end(), end()); | |
545 } | |
546 guarantee(region().contains(new_region), "Region invariant"); | |
547 | |
548 | |
549 // The general case: | |
550 // |---------------------|--invalid---|--------------------------| | |
551 // |------------------new_region---------------------------------| | |
552 // |----bottom_region--|---intersection---|------top_region------| | |
553 // |----old_region----| | |
554 // The intersection part has all pages in place we don't need to migrate them. | |
555 // Pages for the top and bottom part should be freed and then reallocated. | |
556 | |
557 MemRegion intersection = old_region.intersection(new_region); | |
558 | |
559 if (intersection.start() == NULL || intersection.end() == NULL) { | |
560 intersection = MemRegion(new_region.start(), new_region.start()); | |
561 } | |
562 | |
141 | 563 if (!os::numa_has_static_binding()) { |
564 MemRegion invalid_region = ls->invalid_region().intersection(new_region); | |
565 // Invalid region is a range of memory that could've possibly | |
566 // been allocated on the other node. That's relevant only on Solaris where | |
567 // there is no static memory binding. | |
568 if (!invalid_region.is_empty()) { | |
569 merge_regions(new_region, &intersection, &invalid_region); | |
570 free_region(invalid_region); | |
571 ls->set_invalid_region(MemRegion()); | |
572 } | |
0 | 573 } |
141 | 574 |
0 | 575 select_tails(new_region, intersection, &bottom_region, &top_region); |
141 | 576 |
577 if (!os::numa_has_static_binding()) { | |
578 // If that's a system with the first-touch policy then it's enough | |
579 // to free the pages. | |
580 free_region(bottom_region); | |
581 free_region(top_region); | |
582 } else { | |
583 // In a system with static binding we have to change the bias whenever | |
584 // we reshape the heap. | |
585 bias_region(bottom_region, ls->lgrp_id()); | |
586 bias_region(top_region, ls->lgrp_id()); | |
587 } | |
0 | 588 |
589 // If we clear the region, we would mangle it in debug. That would cause page | |
590 // allocation in a different place. Hence setting the top directly. | |
591 s->initialize(new_region, false); | |
592 s->set_top(s->bottom()); | |
593 | |
594 set_adaptation_cycles(samples_count()); | |
595 } | |
596 } | |
597 | |
598 // Set the top of the whole space. | |
599 // Mark the the holes in chunks below the top() as invalid. | |
600 void MutableNUMASpace::set_top(HeapWord* value) { | |
601 bool found_top = false; | |
190
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
602 for (int i = 0; i < lgrp_spaces()->length();) { |
0 | 603 LGRPSpace *ls = lgrp_spaces()->at(i); |
604 MutableSpace *s = ls->space(); | |
605 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom()); | |
606 | |
607 if (s->contains(value)) { | |
190
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
608 // Check if setting the chunk's top to a given value would create a hole less than |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
609 // a minimal object; assuming that's not the last chunk in which case we don't care. |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
610 if (i < lgrp_spaces()->length() - 1) { |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
611 size_t remainder = pointer_delta(s->end(), value); |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
612 const size_t minimal_object_size = oopDesc::header_size(); |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
613 if (remainder < minimal_object_size && remainder > 0) { |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
614 // Add a filler object of a minimal size, it will cross the chunk boundary. |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
615 SharedHeap::fill_region_with_object(MemRegion(value, minimal_object_size)); |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
616 value += minimal_object_size; |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
617 assert(!s->contains(value), "Should be in the next chunk"); |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
618 // Restart the loop from the same chunk, since the value has moved |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
619 // to the next one. |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
620 continue; |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
621 } |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
622 } |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
623 |
141 | 624 if (!os::numa_has_static_binding() && top < value && top < s->end()) { |
0 | 625 ls->add_invalid_region(MemRegion(top, value)); |
626 } | |
627 s->set_top(value); | |
628 found_top = true; | |
629 } else { | |
630 if (found_top) { | |
631 s->set_top(s->bottom()); | |
632 } else { | |
141 | 633 if (!os::numa_has_static_binding() && top < s->end()) { |
634 ls->add_invalid_region(MemRegion(top, s->end())); | |
635 } | |
636 s->set_top(s->end()); | |
0 | 637 } |
638 } | |
190
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
639 i++; |
0 | 640 } |
641 MutableSpace::set_top(value); | |
642 } | |
643 | |
644 void MutableNUMASpace::clear() { | |
645 MutableSpace::set_top(bottom()); | |
646 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
647 lgrp_spaces()->at(i)->space()->clear(); | |
648 } | |
649 } | |
650 | |
141 | 651 /* |
652 Linux supports static memory binding, therefore the most part of the | |
653 logic dealing with the possible invalid page allocation is effectively | |
654 disabled. Besides there is no notion of the home node in Linux. A | |
655 thread is allowed to migrate freely. Although the scheduler is rather | |
656 reluctant to move threads between the nodes. We check for the current | |
657 node every allocation. And with a high probability a thread stays on | |
658 the same node for some time allowing local access to recently allocated | |
659 objects. | |
660 */ | |
661 | |
0 | 662 HeapWord* MutableNUMASpace::allocate(size_t size) { |
141 | 663 Thread* thr = Thread::current(); |
664 int lgrp_id = thr->lgrp_id(); | |
665 if (lgrp_id == -1 || !os::numa_has_group_homing()) { | |
0 | 666 lgrp_id = os::numa_get_group_id(); |
141 | 667 thr->set_lgrp_id(lgrp_id); |
0 | 668 } |
669 | |
670 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); | |
671 | |
672 // It is possible that a new CPU has been hotplugged and | |
673 // we haven't reshaped the space accordingly. | |
674 if (i == -1) { | |
675 i = os::random() % lgrp_spaces()->length(); | |
676 } | |
677 | |
678 MutableSpace *s = lgrp_spaces()->at(i)->space(); | |
679 HeapWord *p = s->allocate(size); | |
680 | |
144
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
681 if (p != NULL) { |
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
682 size_t remainder = s->free_in_words(); |
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
683 if (remainder < (size_t)oopDesc::header_size() && remainder > 0) { |
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
684 s->set_top(s->top() - size); |
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
685 p = NULL; |
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
686 } |
0 | 687 } |
688 if (p != NULL) { | |
689 if (top() < s->top()) { // Keep _top updated. | |
690 MutableSpace::set_top(s->top()); | |
691 } | |
692 } | |
141 | 693 // Make the page allocation happen here if there is no static binding.. |
694 if (p != NULL && !os::numa_has_static_binding()) { | |
0 | 695 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) { |
696 *(int*)i = 0; | |
697 } | |
698 } | |
699 return p; | |
700 } | |
701 | |
702 // This version is lock-free. | |
703 HeapWord* MutableNUMASpace::cas_allocate(size_t size) { | |
141 | 704 Thread* thr = Thread::current(); |
705 int lgrp_id = thr->lgrp_id(); | |
706 if (lgrp_id == -1 || !os::numa_has_group_homing()) { | |
0 | 707 lgrp_id = os::numa_get_group_id(); |
141 | 708 thr->set_lgrp_id(lgrp_id); |
0 | 709 } |
710 | |
711 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); | |
712 // It is possible that a new CPU has been hotplugged and | |
713 // we haven't reshaped the space accordingly. | |
714 if (i == -1) { | |
715 i = os::random() % lgrp_spaces()->length(); | |
716 } | |
717 MutableSpace *s = lgrp_spaces()->at(i)->space(); | |
718 HeapWord *p = s->cas_allocate(size); | |
144
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
719 if (p != NULL) { |
190
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
720 size_t remainder = pointer_delta(s->end(), p + size); |
144
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
721 if (remainder < (size_t)oopDesc::header_size() && remainder > 0) { |
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
722 if (s->cas_deallocate(p, size)) { |
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
723 // We were the last to allocate and created a fragment less than |
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
724 // a minimal object. |
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
725 p = NULL; |
190
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
726 } else { |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
727 guarantee(false, "Deallocation should always succeed"); |
144
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
728 } |
0 | 729 } |
730 } | |
731 if (p != NULL) { | |
732 HeapWord* cur_top, *cur_chunk_top = p + size; | |
733 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated. | |
734 if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) { | |
735 break; | |
736 } | |
737 } | |
738 } | |
739 | |
141 | 740 // Make the page allocation happen here if there is no static binding. |
741 if (p != NULL && !os::numa_has_static_binding() ) { | |
0 | 742 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) { |
743 *(int*)i = 0; | |
744 } | |
745 } | |
746 return p; | |
747 } | |
748 | |
749 void MutableNUMASpace::print_short_on(outputStream* st) const { | |
750 MutableSpace::print_short_on(st); | |
751 st->print(" ("); | |
752 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
753 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id()); | |
754 lgrp_spaces()->at(i)->space()->print_short_on(st); | |
755 if (i < lgrp_spaces()->length() - 1) { | |
756 st->print(", "); | |
757 } | |
758 } | |
759 st->print(")"); | |
760 } | |
761 | |
762 void MutableNUMASpace::print_on(outputStream* st) const { | |
763 MutableSpace::print_on(st); | |
764 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
765 LGRPSpace *ls = lgrp_spaces()->at(i); | |
766 st->print(" lgrp %d", ls->lgrp_id()); | |
767 ls->space()->print_on(st); | |
768 if (NUMAStats) { | |
144
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
769 for (int i = 0; i < lgrp_spaces()->length(); i++) { |
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
770 lgrp_spaces()->at(i)->accumulate_statistics(page_size()); |
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
771 } |
0 | 772 st->print(" local/remote/unbiased/uncommitted: %dK/%dK/%dK/%dK, large/small pages: %d/%d\n", |
773 ls->space_stats()->_local_space / K, | |
774 ls->space_stats()->_remote_space / K, | |
775 ls->space_stats()->_unbiased_space / K, | |
776 ls->space_stats()->_uncommited_space / K, | |
777 ls->space_stats()->_large_pages, | |
778 ls->space_stats()->_small_pages); | |
779 } | |
780 } | |
781 } | |
782 | |
190
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
783 void MutableNUMASpace::verify(bool allow_dirty) { |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
784 // This can be called after setting an arbitary value to the space's top, |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
785 // so an object can cross the chunk boundary. We ensure the parsablity |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
786 // of the space and just walk the objects in linear fashion. |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
787 ensure_parsability(); |
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
788 MutableSpace::verify(allow_dirty); |
0 | 789 } |
790 | |
791 // Scan pages and gather stats about page placement and size. | |
792 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) { | |
793 clear_space_stats(); | |
794 char *start = (char*)round_to((intptr_t) space()->bottom(), page_size); | |
795 char* end = (char*)round_down((intptr_t) space()->end(), page_size); | |
796 if (start < end) { | |
797 for (char *p = start; p < end;) { | |
798 os::page_info info; | |
799 if (os::get_page_info(p, &info)) { | |
800 if (info.size > 0) { | |
801 if (info.size > (size_t)os::vm_page_size()) { | |
802 space_stats()->_large_pages++; | |
803 } else { | |
804 space_stats()->_small_pages++; | |
805 } | |
806 if (info.lgrp_id == lgrp_id()) { | |
807 space_stats()->_local_space += info.size; | |
808 } else { | |
809 space_stats()->_remote_space += info.size; | |
810 } | |
811 p += info.size; | |
812 } else { | |
813 p += os::vm_page_size(); | |
814 space_stats()->_uncommited_space += os::vm_page_size(); | |
815 } | |
816 } else { | |
817 return; | |
818 } | |
819 } | |
820 } | |
821 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) + | |
822 pointer_delta(space()->end(), end, sizeof(char)); | |
823 | |
824 } | |
825 | |
826 // Scan page_count pages and verify if they have the right size and right placement. | |
827 // If invalid pages are found they are freed in hope that subsequent reallocation | |
828 // will be more successful. | |
829 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count) | |
830 { | |
831 char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size); | |
832 char* range_end = (char*)round_down((intptr_t) space()->end(), page_size); | |
833 | |
834 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) { | |
835 set_last_page_scanned(range_start); | |
836 } | |
837 | |
838 char *scan_start = last_page_scanned(); | |
839 char* scan_end = MIN2(scan_start + page_size * page_count, range_end); | |
840 | |
841 os::page_info page_expected, page_found; | |
842 page_expected.size = page_size; | |
843 page_expected.lgrp_id = lgrp_id(); | |
844 | |
845 char *s = scan_start; | |
846 while (s < scan_end) { | |
847 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found); | |
848 if (e == NULL) { | |
849 break; | |
850 } | |
851 if (e != scan_end) { | |
852 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id()) | |
853 && page_expected.size != 0) { | |
854 os::free_memory(s, pointer_delta(e, s, sizeof(char))); | |
855 } | |
856 page_expected = page_found; | |
857 } | |
858 s = e; | |
859 } | |
860 | |
861 set_last_page_scanned(scan_end); | |
862 } |