Mercurial > hg > truffle
annotate src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp @ 5965:cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
Summary: Make sure that MutableNUMASpace::ensure_parsability() only calls CollectedHeap::fill_with_object() with valid sizes and make sure CollectedHeap::filler_array_max_size() returns a value that can be converted to an int without overflow
Reviewed-by: azeemj, jmasa, iveresov
| author | brutisso |
|---|---|
| date | Fri, 23 Mar 2012 15:28:24 +0100 |
| parents | 20bfb6d15a94 |
| children | b632e80fc9dc |
| rev | line source |
|---|---|
| 0 | 1 |
| 2 /* | |
|
5965
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
3 * Copyright (c) 2006, 2012, Oracle and/or its affiliates. All rights reserved. |
| 0 | 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 5 * | |
| 6 * This code is free software; you can redistribute it and/or modify it | |
| 7 * under the terms of the GNU General Public License version 2 only, as | |
| 8 * published by the Free Software Foundation. | |
| 9 * | |
| 10 * This code is distributed in the hope that it will be useful, but WITHOUT | |
| 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| 13 * version 2 for more details (a copy is included in the LICENSE file that | |
| 14 * accompanied this code). | |
| 15 * | |
| 16 * You should have received a copy of the GNU General Public License version | |
| 17 * 2 along with this work; if not, write to the Free Software Foundation, | |
| 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| 19 * | |
|
1552
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
628
diff
changeset
|
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
628
diff
changeset
|
21 * or visit www.oracle.com if you need additional information or have any |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
628
diff
changeset
|
22 * questions. |
| 0 | 23 * |
| 24 */ | |
| 25 | |
| 1972 | 26 #include "precompiled.hpp" |
| 27 #include "gc_implementation/shared/mutableNUMASpace.hpp" | |
| 28 #include "gc_implementation/shared/spaceDecorator.hpp" | |
| 29 #include "memory/sharedHeap.hpp" | |
| 30 #include "oops/oop.inline.hpp" | |
| 31 #ifdef TARGET_OS_FAMILY_linux | |
| 32 # include "thread_linux.inline.hpp" | |
| 33 #endif | |
| 34 #ifdef TARGET_OS_FAMILY_solaris | |
| 35 # include "thread_solaris.inline.hpp" | |
| 36 #endif | |
| 37 #ifdef TARGET_OS_FAMILY_windows | |
| 38 # include "thread_windows.inline.hpp" | |
| 39 #endif | |
| 3960 | 40 #ifdef TARGET_OS_FAMILY_bsd |
| 41 # include "thread_bsd.inline.hpp" | |
| 42 #endif | |
| 0 | 43 |
| 44 | |
|
535
4e400c36026f
6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents:
481
diff
changeset
|
45 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) { |
| 0 | 46 _lgrp_spaces = new (ResourceObj::C_HEAP) GrowableArray<LGRPSpace*>(0, true); |
| 47 _page_size = os::vm_page_size(); | |
| 48 _adaptation_cycles = 0; | |
| 49 _samples_count = 0; | |
| 50 update_layout(true); | |
| 51 } | |
| 52 | |
| 53 MutableNUMASpace::~MutableNUMASpace() { | |
| 54 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
| 55 delete lgrp_spaces()->at(i); | |
| 56 } | |
| 57 delete lgrp_spaces(); | |
| 58 } | |
| 59 | |
|
263
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
60 #ifndef PRODUCT |
| 0 | 61 void MutableNUMASpace::mangle_unused_area() { |
|
263
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
62 // This method should do nothing. |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
63 // It can be called on a numa space during a full compaction. |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
64 } |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
65 void MutableNUMASpace::mangle_unused_area_complete() { |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
66 // This method should do nothing. |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
67 // It can be called on a numa space during a full compaction. |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
68 } |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
69 void MutableNUMASpace::mangle_region(MemRegion mr) { |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
70 // This method should do nothing because numa spaces are not mangled. |
| 0 | 71 } |
|
263
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
72 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) { |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
73 assert(false, "Do not mangle MutableNUMASpace's"); |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
74 } |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
75 void MutableNUMASpace::set_top_for_allocations() { |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
76 // This method should do nothing. |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
77 } |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
78 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) { |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
79 // This method should do nothing. |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
80 } |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
81 void MutableNUMASpace::check_mangled_unused_area_complete() { |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
82 // This method should do nothing. |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
83 } |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
84 #endif // NOT_PRODUCT |
| 0 | 85 |
| 86 // There may be unallocated holes in the middle chunks | |
| 87 // that should be filled with dead objects to ensure parseability. | |
| 88 void MutableNUMASpace::ensure_parsability() { | |
| 89 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
| 90 LGRPSpace *ls = lgrp_spaces()->at(i); | |
| 91 MutableSpace *s = ls->space(); | |
| 605 | 92 if (s->top() < top()) { // For all spaces preceding the one containing top() |
| 0 | 93 if (s->free_in_words() > 0) { |
|
5965
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
94 intptr_t cur_top = (intptr_t)s->top(); |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
95 size_t words_left_to_fill = pointer_delta(s->end(), s->top());; |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
96 while (words_left_to_fill > 0) { |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
97 size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size()); |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
98 assert(words_to_fill >= CollectedHeap::min_fill_size(), |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
99 err_msg("Remaining size ("SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")", |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
100 words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size())); |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
101 CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill); |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
102 if (!os::numa_has_static_binding()) { |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
103 size_t touched_words = words_to_fill; |
| 0 | 104 #ifndef ASSERT |
|
5965
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
105 if (!ZapUnusedHeapArea) { |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
106 touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)), |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
107 touched_words); |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
108 } |
| 0 | 109 #endif |
|
5965
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
110 MemRegion invalid; |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
111 HeapWord *crossing_start = (HeapWord*)round_to(cur_top, os::vm_page_size()); |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
112 HeapWord *crossing_end = (HeapWord*)round_to(cur_top + touched_words, os::vm_page_size()); |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
113 if (crossing_start != crossing_end) { |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
114 // If object header crossed a small page boundary we mark the area |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
115 // as invalid rounding it to a page_size(). |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
116 HeapWord *start = MAX2((HeapWord*)round_down(cur_top, page_size()), s->bottom()); |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
117 HeapWord *end = MIN2((HeapWord*)round_to(cur_top + touched_words, page_size()), s->end()); |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
118 invalid = MemRegion(start, end); |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
119 } |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
120 |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
121 ls->add_invalid_region(invalid); |
| 141 | 122 } |
|
5965
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
123 cur_top = cur_top + (words_to_fill * HeapWordSize); |
|
cc74fa5a91a9
7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97
brutisso
parents:
4734
diff
changeset
|
124 words_left_to_fill -= words_to_fill; |
| 0 | 125 } |
| 126 } | |
| 127 } else { | |
| 141 | 128 if (!os::numa_has_static_binding()) { |
| 0 | 129 #ifdef ASSERT |
| 130 MemRegion invalid(s->top(), s->end()); | |
| 131 ls->add_invalid_region(invalid); | |
| 141 | 132 #else |
| 133 if (ZapUnusedHeapArea) { | |
| 134 MemRegion invalid(s->top(), s->end()); | |
| 135 ls->add_invalid_region(invalid); | |
|
144
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
136 } else { |
|
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
137 return; |
|
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
138 } |
| 0 | 139 #endif |
|
144
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
140 } else { |
|
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
141 return; |
| 141 | 142 } |
| 0 | 143 } |
| 144 } | |
| 145 } | |
| 146 | |
| 147 size_t MutableNUMASpace::used_in_words() const { | |
| 148 size_t s = 0; | |
| 149 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
| 150 s += lgrp_spaces()->at(i)->space()->used_in_words(); | |
| 151 } | |
| 152 return s; | |
| 153 } | |
| 154 | |
| 155 size_t MutableNUMASpace::free_in_words() const { | |
| 156 size_t s = 0; | |
| 157 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
| 158 s += lgrp_spaces()->at(i)->space()->free_in_words(); | |
| 159 } | |
| 160 return s; | |
| 161 } | |
| 162 | |
| 163 | |
| 164 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const { | |
| 165 guarantee(thr != NULL, "No thread"); | |
| 166 int lgrp_id = thr->lgrp_id(); | |
|
268
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
167 if (lgrp_id == -1) { |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
168 // This case can occur after the topology of the system has |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
169 // changed. Thread can change their location, the new home |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
170 // group will be determined during the first allocation |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
171 // attempt. For now we can safely assume that all spaces |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
172 // have equal size because the whole space will be reinitialized. |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
173 if (lgrp_spaces()->length() > 0) { |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
174 return capacity_in_bytes() / lgrp_spaces()->length(); |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
175 } else { |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
176 assert(false, "There should be at least one locality group"); |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
177 return 0; |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
178 } |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
179 } |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
180 // That's the normal case, where we know the locality group of the thread. |
| 0 | 181 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); |
| 182 if (i == -1) { | |
| 183 return 0; | |
| 184 } | |
| 185 return lgrp_spaces()->at(i)->space()->capacity_in_bytes(); | |
| 186 } | |
| 187 | |
| 188 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const { | |
|
268
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
189 // Please see the comments for tlab_capacity(). |
| 0 | 190 guarantee(thr != NULL, "No thread"); |
| 191 int lgrp_id = thr->lgrp_id(); | |
|
268
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
192 if (lgrp_id == -1) { |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
193 if (lgrp_spaces()->length() > 0) { |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
194 return free_in_bytes() / lgrp_spaces()->length(); |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
195 } else { |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
196 assert(false, "There should be at least one locality group"); |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
197 return 0; |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
198 } |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
199 } |
| 0 | 200 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); |
| 201 if (i == -1) { | |
| 202 return 0; | |
| 203 } | |
| 204 return lgrp_spaces()->at(i)->space()->free_in_bytes(); | |
| 205 } | |
| 206 | |
|
373
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
207 |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
208 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const { |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
209 guarantee(thr != NULL, "No thread"); |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
210 int lgrp_id = thr->lgrp_id(); |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
211 if (lgrp_id == -1) { |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
212 if (lgrp_spaces()->length() > 0) { |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
213 return capacity_in_words() / lgrp_spaces()->length(); |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
214 } else { |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
215 assert(false, "There should be at least one locality group"); |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
216 return 0; |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
217 } |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
218 } |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
219 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
220 if (i == -1) { |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
221 return 0; |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
222 } |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
223 return lgrp_spaces()->at(i)->space()->capacity_in_words(); |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
224 } |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
225 |
| 0 | 226 // Check if the NUMA topology has changed. Add and remove spaces if needed. |
| 227 // The update can be forced by setting the force parameter equal to true. | |
| 228 bool MutableNUMASpace::update_layout(bool force) { | |
| 229 // Check if the topology had changed. | |
| 230 bool changed = os::numa_topology_changed(); | |
| 231 if (force || changed) { | |
| 232 // Compute lgrp intersection. Add/remove spaces. | |
| 233 int lgrp_limit = (int)os::numa_get_groups_num(); | |
| 234 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit); | |
| 235 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit); | |
| 236 assert(lgrp_num > 0, "There should be at least one locality group"); | |
| 237 // Add new spaces for the new nodes | |
| 238 for (int i = 0; i < lgrp_num; i++) { | |
| 239 bool found = false; | |
| 240 for (int j = 0; j < lgrp_spaces()->length(); j++) { | |
| 241 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) { | |
| 242 found = true; | |
| 243 break; | |
| 244 } | |
| 245 } | |
| 246 if (!found) { | |
|
535
4e400c36026f
6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents:
481
diff
changeset
|
247 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment())); |
| 0 | 248 } |
| 249 } | |
| 250 | |
| 251 // Remove spaces for the removed nodes. | |
| 252 for (int i = 0; i < lgrp_spaces()->length();) { | |
| 253 bool found = false; | |
| 254 for (int j = 0; j < lgrp_num; j++) { | |
| 255 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) { | |
| 256 found = true; | |
| 257 break; | |
| 258 } | |
| 259 } | |
| 260 if (!found) { | |
| 261 delete lgrp_spaces()->at(i); | |
| 262 lgrp_spaces()->remove_at(i); | |
| 263 } else { | |
| 264 i++; | |
| 265 } | |
| 266 } | |
| 267 | |
| 268 FREE_C_HEAP_ARRAY(int, lgrp_ids); | |
| 269 | |
| 270 if (changed) { | |
| 271 for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) { | |
| 272 thread->set_lgrp_id(-1); | |
| 273 } | |
| 274 } | |
| 275 return true; | |
| 276 } | |
| 277 return false; | |
| 278 } | |
| 279 | |
| 280 // Bias region towards the first-touching lgrp. Set the right page sizes. | |
| 141 | 281 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) { |
| 0 | 282 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size()); |
| 283 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size()); | |
| 284 if (end > start) { | |
| 285 MemRegion aligned_region(start, end); | |
| 286 assert((intptr_t)aligned_region.start() % page_size() == 0 && | |
| 287 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment"); | |
| 288 assert(region().contains(aligned_region), "Sanity"); | |
| 141 | 289 // First we tell the OS which page size we want in the given range. The underlying |
| 290 // large page can be broken down if we require small pages. | |
| 0 | 291 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); |
| 141 | 292 // Then we uncommit the pages in the range. |
|
4734
20bfb6d15a94
7124829: NUMA: memory leak on Linux with large pages
iveresov
parents:
3960
diff
changeset
|
293 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); |
| 141 | 294 // And make them local/first-touch biased. |
| 295 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id); | |
| 0 | 296 } |
| 297 } | |
| 298 | |
| 299 // Free all pages in the region. | |
| 300 void MutableNUMASpace::free_region(MemRegion mr) { | |
| 301 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size()); | |
| 302 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size()); | |
| 303 if (end > start) { | |
| 304 MemRegion aligned_region(start, end); | |
| 305 assert((intptr_t)aligned_region.start() % page_size() == 0 && | |
| 306 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment"); | |
| 307 assert(region().contains(aligned_region), "Sanity"); | |
|
4734
20bfb6d15a94
7124829: NUMA: memory leak on Linux with large pages
iveresov
parents:
3960
diff
changeset
|
308 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); |
| 0 | 309 } |
| 310 } | |
| 311 | |
| 312 // Update space layout. Perform adaptation. | |
| 313 void MutableNUMASpace::update() { | |
| 314 if (update_layout(false)) { | |
| 315 // If the topology has changed, make all chunks zero-sized. | |
|
268
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
316 // And clear the alloc-rate statistics. |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
317 // In future we may want to handle this more gracefully in order |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
318 // to avoid the reallocation of the pages as much as possible. |
| 0 | 319 for (int i = 0; i < lgrp_spaces()->length(); i++) { |
|
268
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
320 LGRPSpace *ls = lgrp_spaces()->at(i); |
|
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
321 MutableSpace *s = ls->space(); |
| 0 | 322 s->set_end(s->bottom()); |
| 323 s->set_top(s->bottom()); | |
|
268
d6340ab4105b
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
iveresov
parents:
263
diff
changeset
|
324 ls->clear_alloc_rate(); |
| 0 | 325 } |
|
263
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
326 // A NUMA space is never mangled |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
327 initialize(region(), |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
328 SpaceDecorator::Clear, |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
329 SpaceDecorator::DontMangle); |
| 0 | 330 } else { |
| 331 bool should_initialize = false; | |
| 141 | 332 if (!os::numa_has_static_binding()) { |
| 333 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
| 334 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) { | |
| 335 should_initialize = true; | |
| 336 break; | |
| 337 } | |
| 0 | 338 } |
| 339 } | |
| 340 | |
| 341 if (should_initialize || | |
| 342 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) { | |
|
263
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
343 // A NUMA space is never mangled |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
344 initialize(region(), |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
345 SpaceDecorator::Clear, |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
346 SpaceDecorator::DontMangle); |
| 0 | 347 } |
| 348 } | |
| 349 | |
| 350 if (NUMAStats) { | |
| 351 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
| 352 lgrp_spaces()->at(i)->accumulate_statistics(page_size()); | |
| 353 } | |
| 354 } | |
| 355 | |
| 356 scan_pages(NUMAPageScanRate); | |
| 357 } | |
| 358 | |
| 359 // Scan pages. Free pages that have smaller size or wrong placement. | |
| 360 void MutableNUMASpace::scan_pages(size_t page_count) | |
| 361 { | |
| 362 size_t pages_per_chunk = page_count / lgrp_spaces()->length(); | |
| 363 if (pages_per_chunk > 0) { | |
| 364 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
| 365 LGRPSpace *ls = lgrp_spaces()->at(i); | |
| 366 ls->scan_pages(page_size(), pages_per_chunk); | |
| 367 } | |
| 368 } | |
| 369 } | |
| 370 | |
| 371 // Accumulate statistics about the allocation rate of each lgrp. | |
| 372 void MutableNUMASpace::accumulate_statistics() { | |
| 373 if (UseAdaptiveNUMAChunkSizing) { | |
| 374 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
| 375 lgrp_spaces()->at(i)->sample(); | |
| 376 } | |
| 377 increment_samples_count(); | |
| 378 } | |
| 379 | |
| 380 if (NUMAStats) { | |
| 381 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
| 382 lgrp_spaces()->at(i)->accumulate_statistics(page_size()); | |
| 383 } | |
| 384 } | |
| 385 } | |
| 386 | |
| 387 // Get the current size of a chunk. | |
| 388 // This function computes the size of the chunk based on the | |
| 389 // difference between chunk ends. This allows it to work correctly in | |
| 390 // case the whole space is resized and during the process of adaptive | |
| 391 // chunk resizing. | |
| 392 size_t MutableNUMASpace::current_chunk_size(int i) { | |
| 393 HeapWord *cur_end, *prev_end; | |
| 394 if (i == 0) { | |
| 395 prev_end = bottom(); | |
| 396 } else { | |
| 397 prev_end = lgrp_spaces()->at(i - 1)->space()->end(); | |
| 398 } | |
| 399 if (i == lgrp_spaces()->length() - 1) { | |
| 400 cur_end = end(); | |
| 401 } else { | |
| 402 cur_end = lgrp_spaces()->at(i)->space()->end(); | |
| 403 } | |
| 404 if (cur_end > prev_end) { | |
| 405 return pointer_delta(cur_end, prev_end, sizeof(char)); | |
| 406 } | |
| 407 return 0; | |
| 408 } | |
| 409 | |
| 410 // Return the default chunk size by equally diving the space. | |
| 411 // page_size() aligned. | |
| 412 size_t MutableNUMASpace::default_chunk_size() { | |
| 413 return base_space_size() / lgrp_spaces()->length() * page_size(); | |
| 414 } | |
| 415 | |
| 416 // Produce a new chunk size. page_size() aligned. | |
|
391
ab4a7734b9c4
6753547: NUMA allocator: Invalid chunk size computation during adaptive resizing
iveresov
parents:
373
diff
changeset
|
417 // This function is expected to be called on sequence of i's from 0 to |
|
ab4a7734b9c4
6753547: NUMA allocator: Invalid chunk size computation during adaptive resizing
iveresov
parents:
373
diff
changeset
|
418 // lgrp_spaces()->length(). |
| 0 | 419 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) { |
| 420 size_t pages_available = base_space_size(); | |
| 421 for (int j = 0; j < i; j++) { | |
| 422 pages_available -= round_down(current_chunk_size(j), page_size()) / page_size(); | |
| 423 } | |
| 424 pages_available -= lgrp_spaces()->length() - i - 1; | |
| 425 assert(pages_available > 0, "No pages left"); | |
| 426 float alloc_rate = 0; | |
| 427 for (int j = i; j < lgrp_spaces()->length(); j++) { | |
| 428 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average(); | |
| 429 } | |
| 430 size_t chunk_size = 0; | |
| 431 if (alloc_rate > 0) { | |
| 432 LGRPSpace *ls = lgrp_spaces()->at(i); | |
|
391
ab4a7734b9c4
6753547: NUMA allocator: Invalid chunk size computation during adaptive resizing
iveresov
parents:
373
diff
changeset
|
433 chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size(); |
| 0 | 434 } |
| 435 chunk_size = MAX2(chunk_size, page_size()); | |
| 436 | |
| 437 if (limit > 0) { | |
| 438 limit = round_down(limit, page_size()); | |
| 439 if (chunk_size > current_chunk_size(i)) { | |
|
462
85f1b9537f70
6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents:
391
diff
changeset
|
440 size_t upper_bound = pages_available * page_size(); |
|
85f1b9537f70
6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents:
391
diff
changeset
|
441 if (upper_bound > limit && |
|
85f1b9537f70
6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents:
391
diff
changeset
|
442 current_chunk_size(i) < upper_bound - limit) { |
|
85f1b9537f70
6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents:
391
diff
changeset
|
443 // The resulting upper bound should not exceed the available |
|
85f1b9537f70
6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents:
391
diff
changeset
|
444 // amount of memory (pages_available * page_size()). |
|
85f1b9537f70
6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents:
391
diff
changeset
|
445 upper_bound = current_chunk_size(i) + limit; |
|
85f1b9537f70
6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents:
391
diff
changeset
|
446 } |
|
85f1b9537f70
6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents:
391
diff
changeset
|
447 chunk_size = MIN2(chunk_size, upper_bound); |
| 0 | 448 } else { |
|
462
85f1b9537f70
6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents:
391
diff
changeset
|
449 size_t lower_bound = page_size(); |
|
85f1b9537f70
6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents:
391
diff
changeset
|
450 if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow. |
|
85f1b9537f70
6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents:
391
diff
changeset
|
451 lower_bound = current_chunk_size(i) - limit; |
|
85f1b9537f70
6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents:
391
diff
changeset
|
452 } |
|
85f1b9537f70
6779436: NUMA allocator: libnuma expects certain size of the buffer in numa_node_to_cpus()
iveresov
parents:
391
diff
changeset
|
453 chunk_size = MAX2(chunk_size, lower_bound); |
| 0 | 454 } |
| 455 } | |
| 456 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range"); | |
| 457 return chunk_size; | |
| 458 } | |
| 459 | |
| 460 | |
| 461 // Return the bottom_region and the top_region. Align them to page_size() boundary. | |
| 462 // |------------------new_region---------------------------------| | |
| 463 // |----bottom_region--|---intersection---|------top_region------| | |
| 464 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection, | |
| 465 MemRegion* bottom_region, MemRegion *top_region) { | |
| 466 // Is there bottom? | |
| 467 if (new_region.start() < intersection.start()) { // Yes | |
| 468 // Try to coalesce small pages into a large one. | |
|
535
4e400c36026f
6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents:
481
diff
changeset
|
469 if (UseLargePages && page_size() >= alignment()) { |
|
4e400c36026f
6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents:
481
diff
changeset
|
470 HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment()); |
| 0 | 471 if (new_region.contains(p) |
|
535
4e400c36026f
6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents:
481
diff
changeset
|
472 && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) { |
| 0 | 473 if (intersection.contains(p)) { |
| 474 intersection = MemRegion(p, intersection.end()); | |
| 475 } else { | |
| 476 intersection = MemRegion(p, p); | |
| 477 } | |
| 478 } | |
| 479 } | |
| 480 *bottom_region = MemRegion(new_region.start(), intersection.start()); | |
| 481 } else { | |
| 482 *bottom_region = MemRegion(); | |
| 483 } | |
| 484 | |
| 485 // Is there top? | |
| 486 if (intersection.end() < new_region.end()) { // Yes | |
| 487 // Try to coalesce small pages into a large one. | |
|
535
4e400c36026f
6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents:
481
diff
changeset
|
488 if (UseLargePages && page_size() >= alignment()) { |
|
4e400c36026f
6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents:
481
diff
changeset
|
489 HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment()); |
| 0 | 490 if (new_region.contains(p) |
|
535
4e400c36026f
6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents:
481
diff
changeset
|
491 && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) { |
| 0 | 492 if (intersection.contains(p)) { |
| 493 intersection = MemRegion(intersection.start(), p); | |
| 494 } else { | |
| 495 intersection = MemRegion(p, p); | |
| 496 } | |
| 497 } | |
| 498 } | |
| 499 *top_region = MemRegion(intersection.end(), new_region.end()); | |
| 500 } else { | |
| 501 *top_region = MemRegion(); | |
| 502 } | |
| 503 } | |
| 504 | |
| 505 // Try to merge the invalid region with the bottom or top region by decreasing | |
| 506 // the intersection area. Return the invalid_region aligned to the page_size() | |
| 507 // boundary if it's inside the intersection. Return non-empty invalid_region | |
| 508 // if it lies inside the intersection (also page-aligned). | |
| 509 // |------------------new_region---------------------------------| | |
| 510 // |----------------|-------invalid---|--------------------------| | |
| 511 // |----bottom_region--|---intersection---|------top_region------| | |
| 512 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection, | |
| 513 MemRegion *invalid_region) { | |
| 514 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) { | |
| 515 *intersection = MemRegion(invalid_region->end(), intersection->end()); | |
| 516 *invalid_region = MemRegion(); | |
| 517 } else | |
| 518 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) { | |
| 519 *intersection = MemRegion(intersection->start(), invalid_region->start()); | |
| 520 *invalid_region = MemRegion(); | |
| 521 } else | |
| 522 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) { | |
| 523 *intersection = MemRegion(new_region.start(), new_region.start()); | |
| 524 *invalid_region = MemRegion(); | |
| 525 } else | |
| 526 if (intersection->contains(invalid_region)) { | |
| 527 // That's the only case we have to make an additional bias_region() call. | |
| 528 HeapWord* start = invalid_region->start(); | |
| 529 HeapWord* end = invalid_region->end(); | |
|
535
4e400c36026f
6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents:
481
diff
changeset
|
530 if (UseLargePages && page_size() >= alignment()) { |
|
4e400c36026f
6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents:
481
diff
changeset
|
531 HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment()); |
| 0 | 532 if (new_region.contains(p)) { |
| 533 start = p; | |
| 534 } | |
|
535
4e400c36026f
6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents:
481
diff
changeset
|
535 p = (HeapWord*)round_to((intptr_t) end, alignment()); |
| 0 | 536 if (new_region.contains(end)) { |
| 537 end = p; | |
| 538 } | |
| 539 } | |
| 540 if (intersection->start() > start) { | |
| 541 *intersection = MemRegion(start, intersection->end()); | |
| 542 } | |
| 543 if (intersection->end() < end) { | |
| 544 *intersection = MemRegion(intersection->start(), end); | |
| 545 } | |
| 546 *invalid_region = MemRegion(start, end); | |
| 547 } | |
| 548 } | |
| 549 | |
|
263
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
550 void MutableNUMASpace::initialize(MemRegion mr, |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
551 bool clear_space, |
|
535
4e400c36026f
6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents:
481
diff
changeset
|
552 bool mangle_space, |
|
4e400c36026f
6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents:
481
diff
changeset
|
553 bool setup_pages) { |
| 0 | 554 assert(clear_space, "Reallocation will destory data!"); |
| 555 assert(lgrp_spaces()->length() > 0, "There should be at least one space"); | |
| 556 | |
| 557 MemRegion old_region = region(), new_region; | |
| 558 set_bottom(mr.start()); | |
| 559 set_end(mr.end()); | |
|
263
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
560 // Must always clear the space |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
561 clear(SpaceDecorator::DontMangle); |
| 0 | 562 |
| 563 // Compute chunk sizes | |
| 564 size_t prev_page_size = page_size(); | |
|
535
4e400c36026f
6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents:
481
diff
changeset
|
565 set_page_size(UseLargePages ? alignment() : os::vm_page_size()); |
| 0 | 566 HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size()); |
| 567 HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size()); | |
| 568 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size(); | |
| 569 | |
| 570 // Try small pages if the chunk size is too small | |
| 571 if (base_space_size_pages / lgrp_spaces()->length() == 0 | |
| 572 && page_size() > (size_t)os::vm_page_size()) { | |
| 573 set_page_size(os::vm_page_size()); | |
| 574 rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size()); | |
| 575 rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size()); | |
| 576 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size(); | |
| 577 } | |
| 578 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small"); | |
| 579 set_base_space_size(base_space_size_pages); | |
| 580 | |
| 581 // Handle space resize | |
| 582 MemRegion top_region, bottom_region; | |
| 583 if (!old_region.equals(region())) { | |
| 584 new_region = MemRegion(rounded_bottom, rounded_end); | |
| 585 MemRegion intersection = new_region.intersection(old_region); | |
| 586 if (intersection.start() == NULL || | |
| 587 intersection.end() == NULL || | |
| 588 prev_page_size > page_size()) { // If the page size got smaller we have to change | |
| 589 // the page size preference for the whole space. | |
| 590 intersection = MemRegion(new_region.start(), new_region.start()); | |
| 591 } | |
| 592 select_tails(new_region, intersection, &bottom_region, &top_region); | |
| 141 | 593 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id()); |
| 594 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id()); | |
| 0 | 595 } |
| 596 | |
| 597 // Check if the space layout has changed significantly? | |
| 598 // This happens when the space has been resized so that either head or tail | |
| 599 // chunk became less than a page. | |
| 600 bool layout_valid = UseAdaptiveNUMAChunkSizing && | |
| 601 current_chunk_size(0) > page_size() && | |
| 602 current_chunk_size(lgrp_spaces()->length() - 1) > page_size(); | |
| 603 | |
| 604 | |
| 605 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
| 606 LGRPSpace *ls = lgrp_spaces()->at(i); | |
| 607 MutableSpace *s = ls->space(); | |
| 608 old_region = s->region(); | |
| 609 | |
| 610 size_t chunk_byte_size = 0, old_chunk_byte_size = 0; | |
| 611 if (i < lgrp_spaces()->length() - 1) { | |
| 612 if (!UseAdaptiveNUMAChunkSizing || | |
| 613 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) || | |
| 614 samples_count() < AdaptiveSizePolicyReadyThreshold) { | |
| 615 // No adaptation. Divide the space equally. | |
| 616 chunk_byte_size = default_chunk_size(); | |
| 617 } else | |
| 618 if (!layout_valid || NUMASpaceResizeRate == 0) { | |
| 619 // Fast adaptation. If no space resize rate is set, resize | |
| 620 // the chunks instantly. | |
| 621 chunk_byte_size = adaptive_chunk_size(i, 0); | |
| 622 } else { | |
| 623 // Slow adaptation. Resize the chunks moving no more than | |
| 624 // NUMASpaceResizeRate bytes per collection. | |
| 625 size_t limit = NUMASpaceResizeRate / | |
| 626 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2); | |
| 627 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size())); | |
| 628 } | |
| 629 | |
| 630 assert(chunk_byte_size >= page_size(), "Chunk size too small"); | |
| 631 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check"); | |
| 632 } | |
| 633 | |
| 634 if (i == 0) { // Bottom chunk | |
| 635 if (i != lgrp_spaces()->length() - 1) { | |
| 636 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize)); | |
| 637 } else { | |
| 638 new_region = MemRegion(bottom(), end()); | |
| 639 } | |
| 640 } else | |
| 641 if (i < lgrp_spaces()->length() - 1) { // Middle chunks | |
| 642 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space(); | |
| 643 new_region = MemRegion(ps->end(), | |
| 644 ps->end() + (chunk_byte_size >> LogHeapWordSize)); | |
| 645 } else { // Top chunk | |
| 646 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space(); | |
| 647 new_region = MemRegion(ps->end(), end()); | |
| 648 } | |
| 649 guarantee(region().contains(new_region), "Region invariant"); | |
| 650 | |
| 651 | |
| 652 // The general case: | |
| 653 // |---------------------|--invalid---|--------------------------| | |
| 654 // |------------------new_region---------------------------------| | |
| 655 // |----bottom_region--|---intersection---|------top_region------| | |
| 656 // |----old_region----| | |
| 657 // The intersection part has all pages in place we don't need to migrate them. | |
| 658 // Pages for the top and bottom part should be freed and then reallocated. | |
| 659 | |
| 660 MemRegion intersection = old_region.intersection(new_region); | |
| 661 | |
| 662 if (intersection.start() == NULL || intersection.end() == NULL) { | |
| 663 intersection = MemRegion(new_region.start(), new_region.start()); | |
| 664 } | |
| 665 | |
| 141 | 666 if (!os::numa_has_static_binding()) { |
| 667 MemRegion invalid_region = ls->invalid_region().intersection(new_region); | |
| 668 // Invalid region is a range of memory that could've possibly | |
| 669 // been allocated on the other node. That's relevant only on Solaris where | |
| 670 // there is no static memory binding. | |
| 671 if (!invalid_region.is_empty()) { | |
| 672 merge_regions(new_region, &intersection, &invalid_region); | |
| 673 free_region(invalid_region); | |
| 674 ls->set_invalid_region(MemRegion()); | |
| 675 } | |
| 0 | 676 } |
| 141 | 677 |
| 0 | 678 select_tails(new_region, intersection, &bottom_region, &top_region); |
| 141 | 679 |
| 680 if (!os::numa_has_static_binding()) { | |
| 681 // If that's a system with the first-touch policy then it's enough | |
| 682 // to free the pages. | |
| 683 free_region(bottom_region); | |
| 684 free_region(top_region); | |
| 685 } else { | |
| 686 // In a system with static binding we have to change the bias whenever | |
| 687 // we reshape the heap. | |
| 688 bias_region(bottom_region, ls->lgrp_id()); | |
| 689 bias_region(top_region, ls->lgrp_id()); | |
| 690 } | |
| 0 | 691 |
|
263
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
692 // Clear space (set top = bottom) but never mangle. |
|
535
4e400c36026f
6783381: NUMA allocator: don't pretouch eden space with UseNUMA
iveresov
parents:
481
diff
changeset
|
693 s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages); |
| 0 | 694 |
| 695 set_adaptation_cycles(samples_count()); | |
| 696 } | |
| 697 } | |
| 698 | |
| 699 // Set the top of the whole space. | |
| 700 // Mark the the holes in chunks below the top() as invalid. | |
| 701 void MutableNUMASpace::set_top(HeapWord* value) { | |
| 702 bool found_top = false; | |
|
190
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
703 for (int i = 0; i < lgrp_spaces()->length();) { |
| 0 | 704 LGRPSpace *ls = lgrp_spaces()->at(i); |
| 705 MutableSpace *s = ls->space(); | |
| 706 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom()); | |
| 707 | |
| 708 if (s->contains(value)) { | |
|
190
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
709 // Check if setting the chunk's top to a given value would create a hole less than |
|
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
710 // a minimal object; assuming that's not the last chunk in which case we don't care. |
|
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
711 if (i < lgrp_spaces()->length() - 1) { |
|
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
712 size_t remainder = pointer_delta(s->end(), value); |
|
481
7d7a7c599c17
6578152: fill_region_with_object has usability and safety issues
jcoomes
parents:
462
diff
changeset
|
713 const size_t min_fill_size = CollectedHeap::min_fill_size(); |
|
7d7a7c599c17
6578152: fill_region_with_object has usability and safety issues
jcoomes
parents:
462
diff
changeset
|
714 if (remainder < min_fill_size && remainder > 0) { |
|
7d7a7c599c17
6578152: fill_region_with_object has usability and safety issues
jcoomes
parents:
462
diff
changeset
|
715 // Add a minimum size filler object; it will cross the chunk boundary. |
|
7d7a7c599c17
6578152: fill_region_with_object has usability and safety issues
jcoomes
parents:
462
diff
changeset
|
716 CollectedHeap::fill_with_object(value, min_fill_size); |
|
7d7a7c599c17
6578152: fill_region_with_object has usability and safety issues
jcoomes
parents:
462
diff
changeset
|
717 value += min_fill_size; |
|
190
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
718 assert(!s->contains(value), "Should be in the next chunk"); |
|
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
719 // Restart the loop from the same chunk, since the value has moved |
|
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
720 // to the next one. |
|
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
721 continue; |
|
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
722 } |
|
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
723 } |
|
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
724 |
| 141 | 725 if (!os::numa_has_static_binding() && top < value && top < s->end()) { |
| 0 | 726 ls->add_invalid_region(MemRegion(top, value)); |
| 727 } | |
| 728 s->set_top(value); | |
| 729 found_top = true; | |
| 730 } else { | |
| 731 if (found_top) { | |
| 732 s->set_top(s->bottom()); | |
| 733 } else { | |
| 141 | 734 if (!os::numa_has_static_binding() && top < s->end()) { |
| 735 ls->add_invalid_region(MemRegion(top, s->end())); | |
| 736 } | |
| 737 s->set_top(s->end()); | |
| 0 | 738 } |
| 739 } | |
|
190
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
740 i++; |
| 0 | 741 } |
| 742 MutableSpace::set_top(value); | |
| 743 } | |
| 744 | |
|
263
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
745 void MutableNUMASpace::clear(bool mangle_space) { |
| 0 | 746 MutableSpace::set_top(bottom()); |
| 747 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
|
263
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
748 // Never mangle NUMA spaces because the mangling will |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
749 // bind the memory to a possibly unwanted lgroup. |
|
12eea04c8b06
6672698: mangle_unused_area() should not remangle the entire heap at each collection.
jmasa
parents:
190
diff
changeset
|
750 lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle); |
| 0 | 751 } |
| 752 } | |
| 753 | |
| 141 | 754 /* |
| 755 Linux supports static memory binding, therefore the most part of the | |
| 756 logic dealing with the possible invalid page allocation is effectively | |
| 757 disabled. Besides there is no notion of the home node in Linux. A | |
| 758 thread is allowed to migrate freely. Although the scheduler is rather | |
| 759 reluctant to move threads between the nodes. We check for the current | |
| 760 node every allocation. And with a high probability a thread stays on | |
| 761 the same node for some time allowing local access to recently allocated | |
| 762 objects. | |
| 763 */ | |
| 764 | |
| 0 | 765 HeapWord* MutableNUMASpace::allocate(size_t size) { |
| 141 | 766 Thread* thr = Thread::current(); |
| 767 int lgrp_id = thr->lgrp_id(); | |
| 768 if (lgrp_id == -1 || !os::numa_has_group_homing()) { | |
| 0 | 769 lgrp_id = os::numa_get_group_id(); |
| 141 | 770 thr->set_lgrp_id(lgrp_id); |
| 0 | 771 } |
| 772 | |
| 773 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); | |
| 774 | |
| 775 // It is possible that a new CPU has been hotplugged and | |
| 776 // we haven't reshaped the space accordingly. | |
| 777 if (i == -1) { | |
| 778 i = os::random() % lgrp_spaces()->length(); | |
| 779 } | |
| 780 | |
|
373
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
781 LGRPSpace* ls = lgrp_spaces()->at(i); |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
782 MutableSpace *s = ls->space(); |
| 0 | 783 HeapWord *p = s->allocate(size); |
| 784 | |
|
144
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
785 if (p != NULL) { |
|
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
786 size_t remainder = s->free_in_words(); |
|
1571
2d127394260e
6916623: Align object to 16 bytes to use Compressed Oops with java heap up to 64Gb
kvn
parents:
628
diff
changeset
|
787 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) { |
|
144
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
788 s->set_top(s->top() - size); |
|
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
789 p = NULL; |
|
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
790 } |
| 0 | 791 } |
| 792 if (p != NULL) { | |
| 793 if (top() < s->top()) { // Keep _top updated. | |
| 794 MutableSpace::set_top(s->top()); | |
| 795 } | |
| 796 } | |
| 141 | 797 // Make the page allocation happen here if there is no static binding.. |
| 798 if (p != NULL && !os::numa_has_static_binding()) { | |
| 0 | 799 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) { |
| 800 *(int*)i = 0; | |
| 801 } | |
| 802 } | |
|
373
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
803 if (p == NULL) { |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
804 ls->set_allocation_failed(); |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
805 } |
| 0 | 806 return p; |
| 807 } | |
| 808 | |
| 809 // This version is lock-free. | |
| 810 HeapWord* MutableNUMASpace::cas_allocate(size_t size) { | |
| 141 | 811 Thread* thr = Thread::current(); |
| 812 int lgrp_id = thr->lgrp_id(); | |
| 813 if (lgrp_id == -1 || !os::numa_has_group_homing()) { | |
| 0 | 814 lgrp_id = os::numa_get_group_id(); |
| 141 | 815 thr->set_lgrp_id(lgrp_id); |
| 0 | 816 } |
| 817 | |
| 818 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); | |
| 819 // It is possible that a new CPU has been hotplugged and | |
| 820 // we haven't reshaped the space accordingly. | |
| 821 if (i == -1) { | |
| 822 i = os::random() % lgrp_spaces()->length(); | |
| 823 } | |
|
373
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
824 LGRPSpace *ls = lgrp_spaces()->at(i); |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
825 MutableSpace *s = ls->space(); |
| 0 | 826 HeapWord *p = s->cas_allocate(size); |
|
144
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
827 if (p != NULL) { |
|
190
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
828 size_t remainder = pointer_delta(s->end(), p + size); |
|
1571
2d127394260e
6916623: Align object to 16 bytes to use Compressed Oops with java heap up to 64Gb
kvn
parents:
628
diff
changeset
|
829 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) { |
|
144
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
830 if (s->cas_deallocate(p, size)) { |
|
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
831 // We were the last to allocate and created a fragment less than |
|
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
832 // a minimal object. |
|
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
833 p = NULL; |
|
190
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
834 } else { |
|
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
835 guarantee(false, "Deallocation should always succeed"); |
|
144
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
836 } |
| 0 | 837 } |
| 838 } | |
| 839 if (p != NULL) { | |
| 840 HeapWord* cur_top, *cur_chunk_top = p + size; | |
| 841 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated. | |
| 842 if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) { | |
| 843 break; | |
| 844 } | |
| 845 } | |
| 846 } | |
| 847 | |
| 141 | 848 // Make the page allocation happen here if there is no static binding. |
| 849 if (p != NULL && !os::numa_has_static_binding() ) { | |
| 0 | 850 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) { |
| 851 *(int*)i = 0; | |
| 852 } | |
| 853 } | |
|
373
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
854 if (p == NULL) { |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
855 ls->set_allocation_failed(); |
|
06df86c2ec37
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
iveresov
parents:
269
diff
changeset
|
856 } |
| 0 | 857 return p; |
| 858 } | |
| 859 | |
| 860 void MutableNUMASpace::print_short_on(outputStream* st) const { | |
| 861 MutableSpace::print_short_on(st); | |
| 862 st->print(" ("); | |
| 863 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
| 864 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id()); | |
| 865 lgrp_spaces()->at(i)->space()->print_short_on(st); | |
| 866 if (i < lgrp_spaces()->length() - 1) { | |
| 867 st->print(", "); | |
| 868 } | |
| 869 } | |
| 870 st->print(")"); | |
| 871 } | |
| 872 | |
| 873 void MutableNUMASpace::print_on(outputStream* st) const { | |
| 874 MutableSpace::print_on(st); | |
| 875 for (int i = 0; i < lgrp_spaces()->length(); i++) { | |
| 876 LGRPSpace *ls = lgrp_spaces()->at(i); | |
| 877 st->print(" lgrp %d", ls->lgrp_id()); | |
| 878 ls->space()->print_on(st); | |
| 879 if (NUMAStats) { | |
|
144
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
880 for (int i = 0; i < lgrp_spaces()->length(); i++) { |
|
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
881 lgrp_spaces()->at(i)->accumulate_statistics(page_size()); |
|
e3729351c946
6697534: Premature GC and invalid lgrp selection with NUMA-aware allocator.
iveresov
parents:
141
diff
changeset
|
882 } |
| 0 | 883 st->print(" local/remote/unbiased/uncommitted: %dK/%dK/%dK/%dK, large/small pages: %d/%d\n", |
| 884 ls->space_stats()->_local_space / K, | |
| 885 ls->space_stats()->_remote_space / K, | |
| 886 ls->space_stats()->_unbiased_space / K, | |
| 887 ls->space_stats()->_uncommited_space / K, | |
| 888 ls->space_stats()->_large_pages, | |
| 889 ls->space_stats()->_small_pages); | |
| 890 } | |
| 891 } | |
| 892 } | |
| 893 | |
|
190
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
894 void MutableNUMASpace::verify(bool allow_dirty) { |
|
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
895 // This can be called after setting an arbitary value to the space's top, |
|
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
896 // so an object can cross the chunk boundary. We ensure the parsablity |
|
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
897 // of the space and just walk the objects in linear fashion. |
|
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
898 ensure_parsability(); |
|
d1635bf93939
6711930: NUMA allocator: ParOld can create a hole less than minimal object size in the lgrp chunk
iveresov
parents:
144
diff
changeset
|
899 MutableSpace::verify(allow_dirty); |
| 0 | 900 } |
| 901 | |
| 902 // Scan pages and gather stats about page placement and size. | |
| 903 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) { | |
| 904 clear_space_stats(); | |
| 905 char *start = (char*)round_to((intptr_t) space()->bottom(), page_size); | |
| 906 char* end = (char*)round_down((intptr_t) space()->end(), page_size); | |
| 907 if (start < end) { | |
| 908 for (char *p = start; p < end;) { | |
| 909 os::page_info info; | |
| 910 if (os::get_page_info(p, &info)) { | |
| 911 if (info.size > 0) { | |
| 912 if (info.size > (size_t)os::vm_page_size()) { | |
| 913 space_stats()->_large_pages++; | |
| 914 } else { | |
| 915 space_stats()->_small_pages++; | |
| 916 } | |
| 917 if (info.lgrp_id == lgrp_id()) { | |
| 918 space_stats()->_local_space += info.size; | |
| 919 } else { | |
| 920 space_stats()->_remote_space += info.size; | |
| 921 } | |
| 922 p += info.size; | |
| 923 } else { | |
| 924 p += os::vm_page_size(); | |
| 925 space_stats()->_uncommited_space += os::vm_page_size(); | |
| 926 } | |
| 927 } else { | |
| 928 return; | |
| 929 } | |
| 930 } | |
| 931 } | |
| 932 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) + | |
| 933 pointer_delta(space()->end(), end, sizeof(char)); | |
| 934 | |
| 935 } | |
| 936 | |
| 937 // Scan page_count pages and verify if they have the right size and right placement. | |
| 938 // If invalid pages are found they are freed in hope that subsequent reallocation | |
| 939 // will be more successful. | |
| 940 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count) | |
| 941 { | |
| 942 char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size); | |
| 943 char* range_end = (char*)round_down((intptr_t) space()->end(), page_size); | |
| 944 | |
| 945 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) { | |
| 946 set_last_page_scanned(range_start); | |
| 947 } | |
| 948 | |
| 949 char *scan_start = last_page_scanned(); | |
| 950 char* scan_end = MIN2(scan_start + page_size * page_count, range_end); | |
| 951 | |
| 952 os::page_info page_expected, page_found; | |
| 953 page_expected.size = page_size; | |
| 954 page_expected.lgrp_id = lgrp_id(); | |
| 955 | |
| 956 char *s = scan_start; | |
| 957 while (s < scan_end) { | |
| 958 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found); | |
| 959 if (e == NULL) { | |
| 960 break; | |
| 961 } | |
| 962 if (e != scan_end) { | |
| 963 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id()) | |
| 964 && page_expected.size != 0) { | |
|
4734
20bfb6d15a94
7124829: NUMA: memory leak on Linux with large pages
iveresov
parents:
3960
diff
changeset
|
965 os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size); |
| 0 | 966 } |
| 967 page_expected = page_found; | |
| 968 } | |
| 969 s = e; | |
| 970 } | |
| 971 | |
| 972 set_last_page_scanned(scan_end); | |
| 973 } |