001/*
002 * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved.
003 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
004 *
005 * This code is free software; you can redistribute it and/or modify it
006 * under the terms of the GNU General Public License version 2 only, as
007 * published by the Free Software Foundation.
008 *
009 * This code is distributed in the hope that it will be useful, but WITHOUT
010 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
011 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
012 * version 2 for more details (a copy is included in the LICENSE file that
013 * accompanied this code).
014 *
015 * You should have received a copy of the GNU General Public License version
016 * 2 along with this work; if not, write to the Free Software Foundation,
017 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
018 *
019 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
020 * or visit www.oracle.com if you need additional information or have any
021 * questions.
022 */
023package com.oracle.graal.hotspot.stubs;
024
025import jdk.internal.jvmci.code.*;
026import jdk.internal.jvmci.hotspot.*;
027import jdk.internal.jvmci.meta.*;
028import static com.oracle.graal.hotspot.nodes.DirectCompareAndSwapNode.*;
029import static com.oracle.graal.hotspot.replacements.HotSpotReplacementsUtil.*;
030import static com.oracle.graal.hotspot.stubs.StubUtil.*;
031import static com.oracle.graal.nodes.extended.BranchProbabilityNode.*;
032
033import com.oracle.graal.api.replacements.*;
034import com.oracle.graal.compiler.common.spi.*;
035import com.oracle.graal.graph.Node.ConstantNodeParameter;
036import com.oracle.graal.graph.Node.NodeIntrinsic;
037import com.oracle.graal.hotspot.*;
038import com.oracle.graal.hotspot.meta.*;
039import com.oracle.graal.hotspot.nodes.*;
040import com.oracle.graal.hotspot.replacements.*;
041import com.oracle.graal.hotspot.word.*;
042import com.oracle.graal.nodes.*;
043import com.oracle.graal.nodes.memory.address.*;
044import com.oracle.graal.replacements.*;
045import com.oracle.graal.replacements.Snippet.ConstantParameter;
046import com.oracle.graal.word.*;
047
048/**
049 * Stub implementing the fast path for TLAB refill during instance class allocation. This stub is
050 * called from the {@linkplain NewObjectSnippets inline} allocation code when TLAB allocation fails.
051 * If this stub fails to refill the TLAB or allocate the object, it calls out to the HotSpot C++
052 * runtime for to complete the allocation.
053 */
054public class NewInstanceStub extends SnippetStub {
055
056    public NewInstanceStub(HotSpotProviders providers, HotSpotForeignCallLinkage linkage) {
057        super("newInstance", providers, linkage);
058    }
059
060    @Override
061    protected Object[] makeConstArgs() {
062        HotSpotResolvedObjectType intArrayType = (HotSpotResolvedObjectType) providers.getMetaAccess().lookupJavaType(int[].class);
063        int count = method.getSignature().getParameterCount(false);
064        Object[] args = new Object[count];
065        assert checkConstArg(1, "intArrayHub");
066        assert checkConstArg(2, "threadRegister");
067        args[1] = ConstantNode.forConstant(providers.getStampProvider().createHubStamp(true), intArrayType.klass(), null);
068        args[2] = providers.getRegisters().getThreadRegister();
069        return args;
070    }
071
072    private static Word allocate(Word thread, int size) {
073        Word top = readTlabTop(thread);
074        Word end = readTlabEnd(thread);
075        Word newTop = top.add(size);
076        /*
077         * this check might lead to problems if the TLAB is within 16GB of the address space end
078         * (checked in c++ code)
079         */
080        if (probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
081            writeTlabTop(thread, newTop);
082            return top;
083        }
084        return Word.zero();
085    }
086
087    @Fold
088    private static boolean logging() {
089        return Boolean.getBoolean("graal.logNewInstanceStub");
090    }
091
092    /**
093     * Re-attempts allocation after an initial TLAB allocation failed or was skipped (e.g., due to
094     * -XX:-UseTLAB).
095     *
096     * @param hub the hub of the object to be allocated
097     * @param intArrayHub the hub for {@code int[].class}
098     */
099    @Snippet
100    private static Object newInstance(KlassPointer hub, @ConstantParameter KlassPointer intArrayHub, @ConstantParameter Register threadRegister) {
101        /*
102         * The type is known to be an instance so Klass::_layout_helper is the instance size as a
103         * raw number
104         */
105        int sizeInBytes = loadKlassLayoutHelperIntrinsic(hub);
106        Word thread = registerAsWord(threadRegister);
107        if (!forceSlowPath() && inlineContiguousAllocationSupported()) {
108            if (isInstanceKlassFullyInitialized(hub)) {
109                Word memory = refillAllocate(thread, intArrayHub, sizeInBytes, logging());
110                if (memory.notEqual(0)) {
111                    Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(), PROTOTYPE_MARK_WORD_LOCATION);
112                    NewObjectSnippets.formatObjectForStub(hub, sizeInBytes, memory, prototypeMarkWord);
113                    return verifyObject(memory.toObject());
114                }
115            }
116        }
117
118        if (logging()) {
119            printf("newInstance: calling new_instance_c\n");
120        }
121
122        newInstanceC(NEW_INSTANCE_C, thread, hub);
123        handlePendingException(thread, true);
124        return verifyObject(getAndClearObjectResult(thread));
125    }
126
127    /**
128     * Attempts to refill the current thread's TLAB and retries the allocation.
129     *
130     * @param intArrayHub the hub for {@code int[].class}
131     * @param sizeInBytes the size of the allocation
132     * @param log specifies if logging is enabled
133     *
134     * @return the newly allocated, uninitialized chunk of memory, or {@link Word#zero()} if the
135     *         operation was unsuccessful
136     */
137    static Word refillAllocate(Word thread, KlassPointer intArrayHub, int sizeInBytes, boolean log) {
138        // If G1 is enabled, the "eden" allocation space is not the same always
139        // and therefore we have to go to slowpath to allocate a new TLAB.
140        if (useG1GC()) {
141            return Word.zero();
142        }
143        if (!useTLAB()) {
144            return edenAllocate(Word.unsigned(sizeInBytes), log);
145        }
146        Word intArrayMarkWord = Word.unsigned(tlabIntArrayMarkWord());
147        int alignmentReserveInBytes = tlabAlignmentReserveInHeapWords() * wordSize();
148
149        Word top = readTlabTop(thread);
150        Word end = readTlabEnd(thread);
151
152        // calculate amount of free space
153        long tlabFreeSpaceInBytes = end.subtract(top).rawValue();
154
155        if (log) {
156            printf("refillTLAB: thread=%p\n", thread.rawValue());
157            printf("refillTLAB: top=%p\n", top.rawValue());
158            printf("refillTLAB: end=%p\n", end.rawValue());
159            printf("refillTLAB: tlabFreeSpaceInBytes=%ld\n", tlabFreeSpaceInBytes);
160        }
161
162        long tlabFreeSpaceInWords = tlabFreeSpaceInBytes >>> log2WordSize();
163
164        // Retain TLAB and allocate object in shared space if
165        // the amount free in the TLAB is too large to discard.
166        Word refillWasteLimit = thread.readWord(tlabRefillWasteLimitOffset(), TLAB_REFILL_WASTE_LIMIT_LOCATION);
167        if (tlabFreeSpaceInWords <= refillWasteLimit.rawValue()) {
168            if (tlabStats()) {
169                // increment number of refills
170                thread.writeInt(tlabNumberOfRefillsOffset(), thread.readInt(tlabNumberOfRefillsOffset(), TLAB_NOF_REFILLS_LOCATION) + 1, TLAB_NOF_REFILLS_LOCATION);
171                if (log) {
172                    printf("thread: %p -- number_of_refills %d\n", thread.rawValue(), thread.readInt(tlabNumberOfRefillsOffset(), TLAB_NOF_REFILLS_LOCATION));
173                }
174                // accumulate wastage
175                int wastage = thread.readInt(tlabFastRefillWasteOffset(), TLAB_FAST_REFILL_WASTE_LOCATION) + (int) tlabFreeSpaceInWords;
176                if (log) {
177                    printf("thread: %p -- accumulated wastage %d\n", thread.rawValue(), wastage);
178                }
179                thread.writeInt(tlabFastRefillWasteOffset(), wastage, TLAB_FAST_REFILL_WASTE_LOCATION);
180            }
181
182            // if TLAB is currently allocated (top or end != null) then
183            // fill [top, end + alignment_reserve) with array object
184            if (top.notEqual(0)) {
185                int headerSize = arrayBaseOffset(Kind.Int);
186                // just like the HotSpot assembler stubs, assumes that tlabFreeSpaceInInts fits in
187                // an int
188                int tlabFreeSpaceInInts = (int) tlabFreeSpaceInBytes >>> 2;
189                int length = ((alignmentReserveInBytes - headerSize) >>> 2) + tlabFreeSpaceInInts;
190                NewObjectSnippets.formatArray(intArrayHub, 0, length, headerSize, top, intArrayMarkWord, false, false, false);
191
192                long allocated = thread.readLong(threadAllocatedBytesOffset(), TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
193                allocated = allocated + top.subtract(readTlabStart(thread)).rawValue();
194                thread.writeLong(threadAllocatedBytesOffset(), allocated, TLAB_THREAD_ALLOCATED_BYTES_LOCATION);
195            }
196
197            // refill the TLAB with an eden allocation
198            Word tlabRefillSizeInWords = thread.readWord(threadTlabSizeOffset(), TLAB_SIZE_LOCATION);
199            Word tlabRefillSizeInBytes = tlabRefillSizeInWords.multiply(wordSize());
200            // allocate new TLAB, address returned in top
201            top = edenAllocate(tlabRefillSizeInBytes, log);
202            if (top.notEqual(0)) {
203                end = top.add(tlabRefillSizeInBytes.subtract(alignmentReserveInBytes));
204                initializeTlab(thread, top, end);
205
206                return NewInstanceStub.allocate(thread, sizeInBytes);
207            } else {
208                return Word.zero();
209            }
210        } else {
211            // Retain TLAB
212            Word newRefillWasteLimit = refillWasteLimit.add(tlabRefillWasteIncrement());
213            thread.writeWord(tlabRefillWasteLimitOffset(), newRefillWasteLimit, TLAB_REFILL_WASTE_LIMIT_LOCATION);
214            if (log) {
215                printf("refillTLAB: retaining TLAB - newRefillWasteLimit=%p\n", newRefillWasteLimit.rawValue());
216            }
217
218            if (tlabStats()) {
219                thread.writeInt(tlabSlowAllocationsOffset(), thread.readInt(tlabSlowAllocationsOffset(), TLAB_SLOW_ALLOCATIONS_LOCATION) + 1, TLAB_SLOW_ALLOCATIONS_LOCATION);
220            }
221
222            return edenAllocate(Word.unsigned(sizeInBytes), log);
223        }
224    }
225
226    /**
227     * Attempts to allocate a chunk of memory from Eden space.
228     *
229     * @param sizeInBytes the size of the chunk to allocate
230     * @param log specifies if logging is enabled
231     * @return the allocated chunk or {@link Word#zero()} if allocation fails
232     */
233    public static Word edenAllocate(Word sizeInBytes, boolean log) {
234        Word heapTopAddress = Word.unsigned(heapTopAddress());
235        Word heapEndAddress = Word.unsigned(heapEndAddress());
236
237        while (true) {
238            Word heapTop = heapTopAddress.readWord(0, HEAP_TOP_LOCATION);
239            Word newHeapTop = heapTop.add(sizeInBytes);
240            if (newHeapTop.belowOrEqual(heapTop)) {
241                return Word.zero();
242            }
243
244            Word heapEnd = heapEndAddress.readWord(0, HEAP_END_LOCATION);
245            if (newHeapTop.aboveThan(heapEnd)) {
246                return Word.zero();
247            }
248
249            if (compareAndSwap(RawAddressNode.address(heapTopAddress), heapTop, newHeapTop, HEAP_TOP_LOCATION).equal(heapTop)) {
250                return heapTop;
251            }
252        }
253    }
254
255    @Fold
256    private static boolean forceSlowPath() {
257        return Boolean.getBoolean("graal.newInstanceStub.forceSlowPath");
258    }
259
260    public static final ForeignCallDescriptor NEW_INSTANCE_C = newDescriptor(NewInstanceStub.class, "newInstanceC", void.class, Word.class, KlassPointer.class);
261
262    @NodeIntrinsic(StubForeignCallNode.class)
263    public static native void newInstanceC(@ConstantNodeParameter ForeignCallDescriptor newInstanceC, Word thread, KlassPointer hub);
264}