Source code

001/*
002 * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved.
003 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
004 *
005 * This code is free software; you can redistribute it and/or modify it
006 * under the terms of the GNU General Public License version 2 only, as
007 * published by the Free Software Foundation.
008 *
009 * This code is distributed in the hope that it will be useful, but WITHOUT
010 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
011 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
012 * version 2 for more details (a copy is included in the LICENSE file that
013 * accompanied this code).
014 *
015 * You should have received a copy of the GNU General Public License version
016 * 2 along with this work; if not, write to the Free Software Foundation,
017 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
018 *
019 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
020 * or visit www.oracle.com if you need additional information or have any
021 * questions.
022 */
023package com.oracle.graal.hotspot.replacements;
024
025import static com.oracle.graal.compiler.common.GraalOptions.*;
026import static com.oracle.graal.hotspot.replacements.HotSpotReplacementsUtil.*;
027import static com.oracle.graal.nodes.extended.BranchProbabilityNode.*;
028import static com.oracle.graal.replacements.SnippetTemplate.*;
029import static jdk.internal.jvmci.code.MemoryBarriers.*;
030import jdk.internal.jvmci.code.*;
031import jdk.internal.jvmci.hotspot.HotSpotVMConfig.*;
032import jdk.internal.jvmci.meta.*;
033
034import com.oracle.graal.compiler.common.*;
035import com.oracle.graal.compiler.common.spi.*;
036import com.oracle.graal.graph.Node.ConstantNodeParameter;
037import com.oracle.graal.graph.Node.NodeIntrinsic;
038import com.oracle.graal.hotspot.meta.*;
039import com.oracle.graal.hotspot.nodes.*;
040import com.oracle.graal.hotspot.nodes.type.*;
041import com.oracle.graal.nodes.*;
042import com.oracle.graal.nodes.extended.*;
043import com.oracle.graal.nodes.memory.HeapAccess.BarrierType;
044import com.oracle.graal.nodes.memory.address.*;
045import com.oracle.graal.nodes.memory.address.AddressNode.Address;
046import com.oracle.graal.nodes.spi.*;
047import com.oracle.graal.replacements.*;
048import com.oracle.graal.replacements.Snippet.ConstantParameter;
049import com.oracle.graal.replacements.SnippetTemplate.AbstractTemplates;
050import com.oracle.graal.replacements.SnippetTemplate.Arguments;
051import com.oracle.graal.replacements.SnippetTemplate.SnippetInfo;
052import com.oracle.graal.replacements.nodes.*;
053import com.oracle.graal.word.*;
054
055public class WriteBarrierSnippets implements Snippets {
056
057    private static final SnippetCounter.Group countersWriteBarriers = SnippetCounters.getValue() ? new SnippetCounter.Group("WriteBarriers") : null;
058    private static final SnippetCounter serialWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "serialWriteBarrier", "Number of Serial Write Barriers");
059    private static final SnippetCounter g1AttemptedPreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1AttemptedPreWriteBarrier", "Number of G1 attempted Pre Write Barriers");
060    private static final SnippetCounter g1EffectivePreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectivePreWriteBarrier", "Number of G1 effective Pre Write Barriers");
061    private static final SnippetCounter g1ExecutedPreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1ExecutedPreWriteBarrier", "Number of G1 executed Pre Write Barriers");
062    private static final SnippetCounter g1AttemptedPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1AttemptedPostWriteBarrier", "Number of attempted G1 Post Write Barriers");
063    private static final SnippetCounter g1EffectiveAfterXORPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectiveAfterXORPostWriteBarrier",
064                    "Number of effective G1 Post Write Barriers (after passing the XOR test)");
065    private static final SnippetCounter g1EffectiveAfterNullPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectiveAfterNullPostWriteBarrier",
066                    "Number of effective G1 Post Write Barriers (after passing the NULL test)");
067    private static final SnippetCounter g1ExecutedPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1ExecutedPostWriteBarrier", "Number of executed G1 Post Write Barriers");
068
069    public static final LocationIdentity GC_CARD_LOCATION = NamedLocationIdentity.mutable("GC-Card");
070    public static final LocationIdentity GC_LOG_LOCATION = NamedLocationIdentity.mutable("GC-Log");
071    public static final LocationIdentity GC_INDEX_LOCATION = NamedLocationIdentity.mutable("GC-Index");
072
073    private static void serialWriteBarrier(Pointer ptr) {
074        serialWriteBarrierCounter.inc();
075        int cardTableShift = (isImmutableCode() && generatePIC()) ? CardTableShiftNode.cardTableShift() : cardTableShift();
076        long cardTableAddress = (isImmutableCode() && generatePIC()) ? CardTableAddressNode.cardTableAddress() : cardTableStart();
077        Word base = Word.fromWordBase(ptr.unsignedShiftRight(cardTableShift));
078        long startAddress = cardTableAddress;
079        int displacement = 0;
080        if (((int) startAddress) == startAddress) {
081            displacement = (int) startAddress;
082        } else {
083            base = base.add(Word.unsigned(cardTableAddress));
084        }
085        base.writeByte(displacement, (byte) 0, GC_CARD_LOCATION);
086    }
087
088    @Snippet
089    public static void serialImpreciseWriteBarrier(Object object) {
090        serialWriteBarrier(Word.fromObject(object));
091    }
092
093    @Snippet
094    public static void serialPreciseWriteBarrier(Address address) {
095        serialWriteBarrier(Word.fromAddress(address));
096    }
097
098    @Snippet
099    public static void serialArrayRangeWriteBarrier(Object object, int startIndex, int length) {
100        if (length == 0) {
101            return;
102        }
103        Object dest = FixedValueAnchorNode.getObject(object);
104        int cardShift = cardTableShift();
105        long cardStart = cardTableStart();
106        final int scale = arrayIndexScale(Kind.Object);
107        int header = arrayBaseOffset(Kind.Object);
108        long dstAddr = GetObjectAddressNode.get(dest);
109        long start = (dstAddr + header + (long) startIndex * scale) >>> cardShift;
110        long end = (dstAddr + header + ((long) startIndex + length - 1) * scale) >>> cardShift;
111        long count = end - start + 1;
112        while (count-- > 0) {
113            DirectStoreNode.storeBoolean((start + cardStart) + count, false, Kind.Boolean);
114        }
115    }
116
117    @Snippet
118    public static void g1PreWriteBarrier(Address address, Object object, Object expectedObject, @ConstantParameter boolean doLoad, @ConstantParameter boolean nullCheck,
119                    @ConstantParameter Register threadRegister, @ConstantParameter boolean trace) {
120        if (nullCheck) {
121            NullCheckNode.nullCheck(address);
122        }
123        Word thread = registerAsWord(threadRegister);
124        verifyOop(object);
125        Object fixedExpectedObject = FixedValueAnchorNode.getObject(expectedObject);
126        Word field = Word.fromWordBase(Word.fromAddress(address));
127        Word previousOop = Word.fromWordBase(Word.fromObject(fixedExpectedObject));
128        byte markingValue = thread.readByte(g1SATBQueueMarkingOffset());
129        Word bufferAddress = thread.readWord(g1SATBQueueBufferOffset());
130        Word indexAddress = thread.add(g1SATBQueueIndexOffset());
131        Word indexValue = indexAddress.readWord(0);
132        int gcCycle = 0;
133        if (trace) {
134            gcCycle = (int) Word.unsigned(HotSpotReplacementsUtil.gcTotalCollectionsAddress()).readLong(0);
135            log(trace, "[%d] G1-Pre Thread %p Object %p\n", gcCycle, thread.rawValue(), Word.fromObject(object).rawValue());
136            log(trace, "[%d] G1-Pre Thread %p Expected Object %p\n", gcCycle, thread.rawValue(), Word.fromObject(fixedExpectedObject).rawValue());
137            log(trace, "[%d] G1-Pre Thread %p Field %p\n", gcCycle, thread.rawValue(), field.rawValue());
138            log(trace, "[%d] G1-Pre Thread %p Marking %d\n", gcCycle, thread.rawValue(), markingValue);
139            log(trace, "[%d] G1-Pre Thread %p DoLoad %d\n", gcCycle, thread.rawValue(), doLoad ? 1L : 0L);
140        }
141        g1AttemptedPreWriteBarrierCounter.inc();
142        // If the concurrent marker is enabled, the barrier is issued.
143        if (probability(NOT_FREQUENT_PROBABILITY, markingValue != (byte) 0)) {
144            // If the previous value has to be loaded (before the write), the load is issued.
145            // The load is always issued except the cases of CAS and referent field.
146            if (probability(LIKELY_PROBABILITY, doLoad)) {
147                previousOop = Word.fromWordBase(Word.fromObject(field.readObject(0, BarrierType.NONE)));
148                if (trace) {
149                    log(trace, "[%d] G1-Pre Thread %p Previous Object %p\n ", gcCycle, thread.rawValue(), previousOop.rawValue());
150                    verifyOop(previousOop.toObject());
151                }
152            }
153            g1EffectivePreWriteBarrierCounter.inc();
154            // If the previous value is null the barrier should not be issued.
155            if (probability(FREQUENT_PROBABILITY, previousOop.notEqual(0))) {
156                g1ExecutedPreWriteBarrierCounter.inc();
157                // If the thread-local SATB buffer is full issue a native call which will
158                // initialize a new one and add the entry.
159                if (probability(FREQUENT_PROBABILITY, indexValue.notEqual(0))) {
160                    Word nextIndex = indexValue.subtract(wordSize());
161                    Word logAddress = bufferAddress.add(nextIndex);
162                    // Log the object to be marked as well as update the SATB's buffer next index.
163                    logAddress.writeWord(0, previousOop, GC_LOG_LOCATION);
164                    indexAddress.writeWord(0, nextIndex, GC_INDEX_LOCATION);
165                } else {
166                    g1PreBarrierStub(G1WBPRECALL, previousOop.toObject());
167                }
168            }
169        }
170    }
171
172    @Snippet
173    public static void g1PostWriteBarrier(Address address, Object object, Object value, @ConstantParameter boolean usePrecise, @ConstantParameter Register threadRegister,
174                    @ConstantParameter boolean trace) {
175        Word thread = registerAsWord(threadRegister);
176        Object fixedValue = FixedValueAnchorNode.getObject(value);
177        verifyOop(object);
178        verifyOop(fixedValue);
179        validateObject(object, fixedValue);
180        Word oop;
181        if (usePrecise) {
182            oop = Word.fromWordBase(Word.fromAddress(address));
183        } else {
184            oop = Word.fromWordBase(Word.fromObject(object));
185        }
186        int gcCycle = 0;
187        if (trace) {
188            gcCycle = (int) Word.unsigned(HotSpotReplacementsUtil.gcTotalCollectionsAddress()).readLong(0);
189            log(trace, "[%d] G1-Post Thread: %p Object: %p\n", gcCycle, thread.rawValue(), Word.fromObject(object).rawValue());
190            log(trace, "[%d] G1-Post Thread: %p Field: %p\n", gcCycle, thread.rawValue(), oop.rawValue());
191        }
192        Word writtenValue = Word.fromWordBase(Word.fromObject(fixedValue));
193        Word bufferAddress = thread.readWord(g1CardQueueBufferOffset());
194        Word indexAddress = thread.add(g1CardQueueIndexOffset());
195        Word indexValue = thread.readWord(g1CardQueueIndexOffset());
196        // The result of the xor reveals whether the installed pointer crosses heap regions.
197        // In case it does the write barrier has to be issued.
198        Word xorResult = (oop.xor(writtenValue)).unsignedShiftRight(logOfHeapRegionGrainBytes());
199
200        // Calculate the address of the card to be enqueued to the
201        // thread local card queue.
202        Word cardBase = oop.unsignedShiftRight(cardTableShift());
203        long startAddress = cardTableStart();
204        int displacement = 0;
205        if (((int) startAddress) == startAddress) {
206            displacement = (int) startAddress;
207        } else {
208            cardBase = cardBase.add(Word.unsigned(cardTableStart()));
209        }
210        Word cardAddress = cardBase.add(displacement);
211
212        g1AttemptedPostWriteBarrierCounter.inc();
213        if (probability(FREQUENT_PROBABILITY, xorResult.notEqual(0))) {
214            g1EffectiveAfterXORPostWriteBarrierCounter.inc();
215
216            // If the written value is not null continue with the barrier addition.
217            if (probability(FREQUENT_PROBABILITY, writtenValue.notEqual(0))) {
218                byte cardByte = cardAddress.readByte(0, GC_CARD_LOCATION);
219                g1EffectiveAfterNullPostWriteBarrierCounter.inc();
220
221                // If the card is already dirty, (hence already enqueued) skip the insertion.
222                if (probability(NOT_FREQUENT_PROBABILITY, cardByte != g1YoungCardValue())) {
223                    MembarNode.memoryBarrier(STORE_LOAD, GC_CARD_LOCATION);
224                    byte cardByteReload = cardAddress.readByte(0, GC_CARD_LOCATION);
225                    if (probability(NOT_FREQUENT_PROBABILITY, cardByteReload != dirtyCardValue())) {
226                        log(trace, "[%d] G1-Post Thread: %p Card: %p \n", gcCycle, thread.rawValue(), Word.unsigned(cardByte).rawValue());
227                        cardAddress.writeByte(0, (byte) 0, GC_CARD_LOCATION);
228                        g1ExecutedPostWriteBarrierCounter.inc();
229
230                        // If the thread local card queue is full, issue a native call which will
231                        // initialize a new one and add the card entry.
232                        if (probability(FREQUENT_PROBABILITY, indexValue.notEqual(0))) {
233                            Word nextIndex = indexValue.subtract(wordSize());
234                            Word logAddress = bufferAddress.add(nextIndex);
235                            // Log the object to be scanned as well as update
236                            // the card queue's next index.
237                            logAddress.writeWord(0, cardAddress, GC_LOG_LOCATION);
238                            indexAddress.writeWord(0, nextIndex, GC_INDEX_LOCATION);
239                        } else {
240                            g1PostBarrierStub(G1WBPOSTCALL, cardAddress);
241                        }
242                    }
243                }
244            }
245        }
246    }
247
248    @Snippet
249    public static void g1ArrayRangePreWriteBarrier(Object object, int startIndex, int length, @ConstantParameter Register threadRegister) {
250        Word thread = registerAsWord(threadRegister);
251        byte markingValue = thread.readByte(g1SATBQueueMarkingOffset());
252        // If the concurrent marker is not enabled or the vector length is zero, return.
253        if (markingValue == (byte) 0 || length == 0) {
254            return;
255        }
256        Object dest = FixedValueAnchorNode.getObject(object);
257        Word bufferAddress = thread.readWord(g1SATBQueueBufferOffset());
258        Word indexAddress = thread.add(g1SATBQueueIndexOffset());
259        long dstAddr = GetObjectAddressNode.get(dest);
260        long indexValue = indexAddress.readWord(0).rawValue();
261        final int scale = arrayIndexScale(Kind.Object);
262        int header = arrayBaseOffset(Kind.Object);
263
264        for (int i = startIndex; i < length; i++) {
265            long address = dstAddr + header + (i * scale);
266            Pointer oop = Word.fromObject(Word.unsigned(address).readObject(0, BarrierType.NONE));
267            verifyOop(oop.toObject());
268            if (oop.notEqual(0)) {
269                if (indexValue != 0) {
270                    indexValue = indexValue - wordSize();
271                    Word logAddress = bufferAddress.add(Word.unsigned(indexValue));
272                    // Log the object to be marked as well as update the SATB's buffer next index.
273                    logAddress.writeWord(0, oop, GC_LOG_LOCATION);
274                    indexAddress.writeWord(0, Word.unsigned(indexValue), GC_INDEX_LOCATION);
275                } else {
276                    g1PreBarrierStub(G1WBPRECALL, oop.toObject());
277                }
278            }
279        }
280    }
281
282    @Snippet
283    public static void g1ArrayRangePostWriteBarrier(Object object, int startIndex, int length, @ConstantParameter Register threadRegister) {
284        if (length == 0) {
285            return;
286        }
287        Object dest = FixedValueAnchorNode.getObject(object);
288        Word thread = registerAsWord(threadRegister);
289        Word bufferAddress = thread.readWord(g1CardQueueBufferOffset());
290        Word indexAddress = thread.add(g1CardQueueIndexOffset());
291        long indexValue = thread.readWord(g1CardQueueIndexOffset()).rawValue();
292
293        int cardShift = cardTableShift();
294        long cardStart = cardTableStart();
295        final int scale = arrayIndexScale(Kind.Object);
296        int header = arrayBaseOffset(Kind.Object);
297        long dstAddr = GetObjectAddressNode.get(dest);
298        long start = (dstAddr + header + (long) startIndex * scale) >>> cardShift;
299        long end = (dstAddr + header + ((long) startIndex + length - 1) * scale) >>> cardShift;
300        long count = end - start + 1;
301
302        while (count-- > 0) {
303            Word cardAddress = Word.unsigned((start + cardStart) + count);
304            byte cardByte = cardAddress.readByte(0, GC_CARD_LOCATION);
305            // If the card is already dirty, (hence already enqueued) skip the insertion.
306            if (probability(NOT_FREQUENT_PROBABILITY, cardByte != g1YoungCardValue())) {
307                MembarNode.memoryBarrier(STORE_LOAD, GC_CARD_LOCATION);
308                byte cardByteReload = cardAddress.readByte(0, GC_CARD_LOCATION);
309                if (probability(NOT_FREQUENT_PROBABILITY, cardByteReload != dirtyCardValue())) {
310                    cardAddress.writeByte(0, (byte) 0, GC_CARD_LOCATION);
311                    // If the thread local card queue is full, issue a native call which will
312                    // initialize a new one and add the card entry.
313                    if (indexValue != 0) {
314                        indexValue = indexValue - wordSize();
315                        Word logAddress = bufferAddress.add(Word.unsigned(indexValue));
316                        // Log the object to be scanned as well as update
317                        // the card queue's next index.
318                        logAddress.writeWord(0, cardAddress, GC_LOG_LOCATION);
319                        indexAddress.writeWord(0, Word.unsigned(indexValue), GC_INDEX_LOCATION);
320                    } else {
321                        g1PostBarrierStub(G1WBPOSTCALL, cardAddress);
322                    }
323                }
324            }
325        }
326    }
327
328    public static final ForeignCallDescriptor G1WBPRECALL = new ForeignCallDescriptor("write_barrier_pre", void.class, Object.class);
329
330    @NodeIntrinsic(ForeignCallNode.class)
331    private static native void g1PreBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object);
332
333    public static final ForeignCallDescriptor G1WBPOSTCALL = new ForeignCallDescriptor("write_barrier_post", void.class, Word.class);
334
335    @NodeIntrinsic(ForeignCallNode.class)
336    public static native void g1PostBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word card);
337
338    public static class Templates extends AbstractTemplates {
339
340        private final SnippetInfo serialImpreciseWriteBarrier = snippet(WriteBarrierSnippets.class, "serialImpreciseWriteBarrier", GC_CARD_LOCATION);
341        private final SnippetInfo serialPreciseWriteBarrier = snippet(WriteBarrierSnippets.class, "serialPreciseWriteBarrier", GC_CARD_LOCATION);
342        private final SnippetInfo serialArrayRangeWriteBarrier = snippet(WriteBarrierSnippets.class, "serialArrayRangeWriteBarrier");
343        private final SnippetInfo g1PreWriteBarrier = snippet(WriteBarrierSnippets.class, "g1PreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
344        private final SnippetInfo g1ReferentReadBarrier = snippet(WriteBarrierSnippets.class, "g1PreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
345        private final SnippetInfo g1PostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1PostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);
346        private final SnippetInfo g1ArrayRangePreWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
347        private final SnippetInfo g1ArrayRangePostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);
348
349        private final CompressEncoding oopEncoding;
350
351        public Templates(HotSpotProviders providers, TargetDescription target, CompressEncoding oopEncoding) {
352            super(providers, providers.getSnippetReflection(), target);
353            this.oopEncoding = oopEncoding;
354        }
355
356        public void lower(SerialWriteBarrier writeBarrier, LoweringTool tool) {
357            Arguments args;
358            if (writeBarrier.usePrecise()) {
359                args = new Arguments(serialPreciseWriteBarrier, writeBarrier.graph().getGuardsStage(), tool.getLoweringStage());
360                args.add("address", writeBarrier.getAddress());
361            } else {
362                args = new Arguments(serialImpreciseWriteBarrier, writeBarrier.graph().getGuardsStage(), tool.getLoweringStage());
363                OffsetAddressNode address = (OffsetAddressNode) writeBarrier.getAddress();
364                args.add("object", address.getBase());
365            }
366            template(args).instantiate(providers.getMetaAccess(), writeBarrier, DEFAULT_REPLACER, args);
367        }
368
369        public void lower(SerialArrayRangeWriteBarrier arrayRangeWriteBarrier, LoweringTool tool) {
370            Arguments args = new Arguments(serialArrayRangeWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());
371            args.add("object", arrayRangeWriteBarrier.getObject());
372            args.add("startIndex", arrayRangeWriteBarrier.getStartIndex());
373            args.add("length", arrayRangeWriteBarrier.getLength());
374            template(args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);
375        }
376
377        public void lower(G1PreWriteBarrier writeBarrierPre, HotSpotRegistersProvider registers, LoweringTool tool) {
378            Arguments args = new Arguments(g1PreWriteBarrier, writeBarrierPre.graph().getGuardsStage(), tool.getLoweringStage());
379            AddressNode address = writeBarrierPre.getAddress();
380            args.add("address", address);
381            if (address instanceof OffsetAddressNode) {
382                args.add("object", ((OffsetAddressNode) address).getBase());
383            } else {
384                args.add("object", null);
385            }
386
387            ValueNode expected = writeBarrierPre.getExpectedObject();
388            if (expected != null && expected.stamp() instanceof NarrowOopStamp) {
389                assert oopEncoding != null;
390                expected = CompressionNode.uncompress(expected, oopEncoding);
391            }
392            args.add("expectedObject", expected);
393
394            args.addConst("doLoad", writeBarrierPre.doLoad());
395            args.addConst("nullCheck", writeBarrierPre.getNullCheck());
396            args.addConst("threadRegister", registers.getThreadRegister());
397            args.addConst("trace", traceBarrier());
398            template(args).instantiate(providers.getMetaAccess(), writeBarrierPre, DEFAULT_REPLACER, args);
399        }
400
401        public void lower(G1ReferentFieldReadBarrier readBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {
402            Arguments args = new Arguments(g1ReferentReadBarrier, readBarrier.graph().getGuardsStage(), tool.getLoweringStage());
403            AddressNode address = readBarrier.getAddress();
404            args.add("address", address);
405            if (address instanceof OffsetAddressNode) {
406                args.add("object", ((OffsetAddressNode) address).getBase());
407            } else {
408                args.add("object", null);
409            }
410
411            ValueNode expected = readBarrier.getExpectedObject();
412            if (expected != null && expected.stamp() instanceof NarrowOopStamp) {
413                assert oopEncoding != null;
414                expected = CompressionNode.uncompress(expected, oopEncoding);
415            }
416
417            args.add("expectedObject", expected);
418            args.addConst("doLoad", readBarrier.doLoad());
419            args.addConst("nullCheck", false);
420            args.addConst("threadRegister", registers.getThreadRegister());
421            args.addConst("trace", traceBarrier());
422            template(args).instantiate(providers.getMetaAccess(), readBarrier, DEFAULT_REPLACER, args);
423        }
424
425        public void lower(G1PostWriteBarrier writeBarrierPost, HotSpotRegistersProvider registers, LoweringTool tool) {
426            StructuredGraph graph = writeBarrierPost.graph();
427            if (writeBarrierPost.alwaysNull()) {
428                graph.removeFixed(writeBarrierPost);
429                return;
430            }
431            Arguments args = new Arguments(g1PostWriteBarrier, graph.getGuardsStage(), tool.getLoweringStage());
432            AddressNode address = writeBarrierPost.getAddress();
433            args.add("address", address);
434            if (address instanceof OffsetAddressNode) {
435                args.add("object", ((OffsetAddressNode) address).getBase());
436            } else {
437                assert writeBarrierPost.usePrecise() : "found imprecise barrier that's not an object access " + writeBarrierPost;
438                args.add("object", null);
439            }
440
441            ValueNode value = writeBarrierPost.getValue();
442            if (value.stamp() instanceof NarrowOopStamp) {
443                assert oopEncoding != null;
444                value = CompressionNode.uncompress(value, oopEncoding);
445            }
446            args.add("value", value);
447
448            args.addConst("usePrecise", writeBarrierPost.usePrecise());
449            args.addConst("threadRegister", registers.getThreadRegister());
450            args.addConst("trace", traceBarrier());
451            template(args).instantiate(providers.getMetaAccess(), writeBarrierPost, DEFAULT_REPLACER, args);
452        }
453
454        public void lower(G1ArrayRangePreWriteBarrier arrayRangeWriteBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {
455            Arguments args = new Arguments(g1ArrayRangePreWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());
456            args.add("object", arrayRangeWriteBarrier.getObject());
457            args.add("startIndex", arrayRangeWriteBarrier.getStartIndex());
458            args.add("length", arrayRangeWriteBarrier.getLength());
459            args.addConst("threadRegister", registers.getThreadRegister());
460            template(args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);
461        }
462
463        public void lower(G1ArrayRangePostWriteBarrier arrayRangeWriteBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {
464            Arguments args = new Arguments(g1ArrayRangePostWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());
465            args.add("object", arrayRangeWriteBarrier.getObject());
466            args.add("startIndex", arrayRangeWriteBarrier.getStartIndex());
467            args.add("length", arrayRangeWriteBarrier.getLength());
468            args.addConst("threadRegister", registers.getThreadRegister());
469            template(args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);
470        }
471    }
472
473    /**
474     * Log method of debugging purposes.
475     */
476    public static void log(boolean enabled, String format, long value) {
477        if (enabled) {
478            Log.printf(format, value);
479        }
480    }
481
482    public static void log(boolean enabled, String format, long value1, long value2) {
483        if (enabled) {
484            Log.printf(format, value1, value2);
485        }
486    }
487
488    public static void log(boolean enabled, String format, long value1, long value2, long value3) {
489        if (enabled) {
490            Log.printf(format, value1, value2, value3);
491        }
492    }
493
494    public static boolean traceBarrier() {
495        return GraalOptions.GCDebugStartCycle.getValue() > 0 && ((int) Word.unsigned(HotSpotReplacementsUtil.gcTotalCollectionsAddress()).readLong(0) > GraalOptions.GCDebugStartCycle.getValue());
496    }
497
498    /**
499     * Validation helper method which performs sanity checks on write operations. The addresses of
500     * both the object and the value being written are checked in order to determine if they reside
501     * in a valid heap region. If an object is stale, an invalid access is performed in order to
502     * prematurely crash the VM and debug the stack trace of the faulty method.
503     */
504    public static void validateObject(Object parent, Object child) {
505        if (verifyOops() && child != null && !validateOop(VALIDATE_OBJECT, parent, child)) {
506            log(true, "Verification ERROR, Parent: %p Child: %p\n", Word.fromObject(parent).rawValue(), Word.fromObject(child).rawValue());
507            DirectObjectStoreNode.storeObject(null, 0, 0, null, LocationIdentity.any(), Kind.Object);
508        }
509    }
510
511    public static final ForeignCallDescriptor VALIDATE_OBJECT = new ForeignCallDescriptor("validate_object", boolean.class, Word.class, Word.class);
512
513    @NodeIntrinsic(ForeignCallNode.class)
514    private static native boolean validateOop(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object parent, Object object);
515
516}