Source code

001/*
002 * Copyright (c) 2012, 2014, Oracle and/or its affiliates. All rights reserved.
003 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
004 *
005 * This code is free software; you can redistribute it and/or modify it
006 * under the terms of the GNU General Public License version 2 only, as
007 * published by the Free Software Foundation.
008 *
009 * This code is distributed in the hope that it will be useful, but WITHOUT
010 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
011 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
012 * version 2 for more details (a copy is included in the LICENSE file that
013 * accompanied this code).
014 *
015 * You should have received a copy of the GNU General Public License version
016 * 2 along with this work; if not, write to the Free Software Foundation,
017 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
018 *
019 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
020 * or visit www.oracle.com if you need additional information or have any
021 * questions.
022 */
023package com.oracle.graal.phases.common.inlining;
024
025import static com.oracle.graal.compiler.common.GraalOptions.*;
026import static jdk.internal.jvmci.meta.DeoptimizationAction.*;
027import static jdk.internal.jvmci.meta.DeoptimizationReason.*;
028
029import java.lang.reflect.*;
030import java.util.*;
031
032import jdk.internal.jvmci.code.*;
033import jdk.internal.jvmci.common.*;
034import com.oracle.graal.debug.*;
035import com.oracle.graal.debug.Debug.Scope;
036
037import jdk.internal.jvmci.meta.*;
038
039import com.oracle.graal.api.replacements.*;
040import com.oracle.graal.compiler.common.type.*;
041import com.oracle.graal.graph.*;
042import com.oracle.graal.graph.Graph.DuplicationReplacement;
043import com.oracle.graal.nodeinfo.*;
044import com.oracle.graal.nodes.*;
045import com.oracle.graal.nodes.CallTargetNode.InvokeKind;
046import com.oracle.graal.nodes.calc.*;
047import com.oracle.graal.nodes.extended.*;
048import com.oracle.graal.nodes.java.*;
049import com.oracle.graal.nodes.spi.*;
050import com.oracle.graal.nodes.type.*;
051import com.oracle.graal.nodes.util.*;
052import com.oracle.graal.phases.common.inlining.info.*;
053
054public class InliningUtil {
055
056    private static final String inliningDecisionsScopeString = "InliningDecisions";
057
058    /**
059     * Print a HotSpot-style inlining message to the console.
060     */
061    private static void printInlining(final InlineInfo info, final int inliningDepth, final boolean success, final String msg, final Object... args) {
062        printInlining(info.methodAt(0), info.invoke(), inliningDepth, success, msg, args);
063    }
064
065    /**
066     * Print a HotSpot-style inlining message to the console.
067     */
068    private static void printInlining(final ResolvedJavaMethod method, final Invoke invoke, final int inliningDepth, final boolean success, final String msg, final Object... args) {
069        if (HotSpotPrintInlining.getValue()) {
070            // 1234567
071            TTY.print("        ");     // print timestamp
072            // 1234
073            TTY.print("     ");        // print compilation number
074            // % s ! b n
075            TTY.print("%c%c%c%c%c ", ' ', method.isSynchronized() ? 's' : ' ', ' ', ' ', method.isNative() ? 'n' : ' ');
076            TTY.print("     ");        // more indent
077            TTY.print("    ");         // initial inlining indent
078            for (int i = 0; i < inliningDepth; i++) {
079                TTY.print("  ");
080            }
081            TTY.println(String.format("@ %d  %s   %s%s", invoke.bci(), methodName(method, null), success ? "" : "not inlining ", String.format(msg, args)));
082        }
083    }
084
085    public static void logInlinedMethod(InlineInfo info, int inliningDepth, boolean allowLogging, String msg, Object... args) {
086        logInliningDecision(info, inliningDepth, allowLogging, true, msg, args);
087    }
088
089    public static void logNotInlinedMethod(InlineInfo info, int inliningDepth, String msg, Object... args) {
090        logInliningDecision(info, inliningDepth, true, false, msg, args);
091    }
092
093    public static void logInliningDecision(InlineInfo info, int inliningDepth, boolean allowLogging, boolean success, String msg, final Object... args) {
094        if (allowLogging) {
095            printInlining(info, inliningDepth, success, msg, args);
096            if (shouldLogInliningDecision()) {
097                logInliningDecision(methodName(info), success, msg, args);
098            }
099        }
100    }
101
102    public static void logInliningDecision(final String msg, final Object... args) {
103        try (Scope s = Debug.scope(inliningDecisionsScopeString)) {
104            // Can't use log here since we are varargs
105            if (Debug.isLogEnabled()) {
106                Debug.logv(msg, args);
107            }
108        }
109    }
110
111    public static void logNotInlinedMethod(Invoke invoke, String msg) {
112        if (shouldLogInliningDecision()) {
113            String methodString = invoke.toString();
114            if (invoke.callTarget() == null) {
115                methodString += " callTarget=null";
116            } else {
117                String targetName = invoke.callTarget().targetName();
118                if (!methodString.endsWith(targetName)) {
119                    methodString += " " + targetName;
120                }
121            }
122            logInliningDecision(methodString, false, msg, new Object[0]);
123        }
124    }
125
126    public static void logNotInlined(Invoke invoke, int inliningDepth, ResolvedJavaMethod method, String msg) {
127        logNotInlinedInvoke(invoke, inliningDepth, method, msg, new Object[0]);
128    }
129
130    public static void logNotInlinedInvoke(Invoke invoke, int inliningDepth, ResolvedJavaMethod method, String msg, Object... args) {
131        printInlining(method, invoke, inliningDepth, false, msg, args);
132        if (shouldLogInliningDecision()) {
133            String methodString = methodName(method, invoke);
134            logInliningDecision(methodString, false, msg, args);
135        }
136    }
137
138    private static void logInliningDecision(final String methodString, final boolean success, final String msg, final Object... args) {
139        String inliningMsg = "inlining " + methodString + ": " + msg;
140        if (!success) {
141            inliningMsg = "not " + inliningMsg;
142        }
143        logInliningDecision(inliningMsg, args);
144    }
145
146    public static boolean shouldLogInliningDecision() {
147        try (Scope s = Debug.scope(inliningDecisionsScopeString)) {
148            return Debug.isLogEnabled();
149        }
150    }
151
152    private static String methodName(ResolvedJavaMethod method, Invoke invoke) {
153        if (invoke != null && invoke.stateAfter() != null) {
154            return methodName(invoke.stateAfter(), invoke.bci()) + ": " + method.format("%H.%n(%p):%r") + " (" + method.getCodeSize() + " bytes)";
155        } else {
156            return method.format("%H.%n(%p):%r") + " (" + method.getCodeSize() + " bytes)";
157        }
158    }
159
160    private static String methodName(InlineInfo info) {
161        if (info == null) {
162            return "null";
163        } else if (info.invoke() != null && info.invoke().stateAfter() != null) {
164            return methodName(info.invoke().stateAfter(), info.invoke().bci()) + ": " + info.toString();
165        } else {
166            return info.toString();
167        }
168    }
169
170    private static String methodName(FrameState frameState, int bci) {
171        StringBuilder sb = new StringBuilder();
172        if (frameState.outerFrameState() != null) {
173            sb.append(methodName(frameState.outerFrameState(), frameState.outerFrameState().bci));
174            sb.append("->");
175        }
176        sb.append(frameState.method().format("%h.%n"));
177        sb.append("@").append(bci);
178        return sb.toString();
179    }
180
181    public static void replaceInvokeCallTarget(Invoke invoke, StructuredGraph graph, InvokeKind invokeKind, ResolvedJavaMethod targetMethod) {
182        MethodCallTargetNode oldCallTarget = (MethodCallTargetNode) invoke.callTarget();
183        MethodCallTargetNode newCallTarget = graph.add(new MethodCallTargetNode(invokeKind, targetMethod, oldCallTarget.arguments().toArray(new ValueNode[0]), oldCallTarget.returnType(),
184                        oldCallTarget.getProfile()));
185        invoke.asNode().replaceFirstInput(oldCallTarget, newCallTarget);
186    }
187
188    public static GuardedValueNode createAnchoredReceiver(StructuredGraph graph, GuardingNode anchor, ResolvedJavaType commonType, ValueNode receiver, boolean exact) {
189        return createAnchoredReceiver(graph, anchor, receiver, exact ? StampFactory.exactNonNull(commonType) : StampFactory.declaredTrustedNonNull(commonType));
190    }
191
192    private static GuardedValueNode createAnchoredReceiver(StructuredGraph graph, GuardingNode anchor, ValueNode receiver, Stamp stamp) {
193        // to avoid that floating reads on receiver fields float above the type check
194        return graph.unique(new GuardedValueNode(receiver, anchor, stamp));
195    }
196
197    /**
198     * @return null iff the check succeeds, otherwise a (non-null) descriptive message.
199     */
200    public static String checkInvokeConditions(Invoke invoke) {
201        if (invoke.predecessor() == null || !invoke.asNode().isAlive()) {
202            return "the invoke is dead code";
203        }
204        if (!(invoke.callTarget() instanceof MethodCallTargetNode)) {
205            return "the invoke has already been lowered, or has been created as a low-level node";
206        }
207        MethodCallTargetNode callTarget = (MethodCallTargetNode) invoke.callTarget();
208        if (callTarget.targetMethod() == null) {
209            return "target method is null";
210        }
211        assert invoke.stateAfter() != null : invoke;
212        if (!invoke.useForInlining()) {
213            return "the invoke is marked to be not used for inlining";
214        }
215        ValueNode receiver = callTarget.receiver();
216        if (receiver != null && receiver.isConstant() && receiver.isNullConstant()) {
217            return "receiver is null";
218        }
219        return null;
220    }
221
222    /**
223     * Performs an actual inlining, thereby replacing the given invoke with the given inlineGraph.
224     *
225     * @param invoke the invoke that will be replaced
226     * @param inlineGraph the graph that the invoke will be replaced with
227     * @param receiverNullCheck true if a null check needs to be generated for non-static inlinings,
228     *            false if no such check is required
229     * @param canonicalizedNodes if non-null then append to this list any nodes which should be
230     *            canonicalized after inlining
231     */
232    public static Map<Node, Node> inline(Invoke invoke, StructuredGraph inlineGraph, boolean receiverNullCheck, List<Node> canonicalizedNodes) {
233        if (Fingerprint.ENABLED) {
234            Fingerprint.submit("inlining %s into %s: %s", formatGraph(inlineGraph), formatGraph(invoke.asNode().graph()), inlineGraph.getNodes().snapshot());
235        }
236        final NodeInputList<ValueNode> parameters = invoke.callTarget().arguments();
237        FixedNode invokeNode = invoke.asNode();
238        StructuredGraph graph = invokeNode.graph();
239        assert inlineGraph.getGuardsStage().ordinal() >= graph.getGuardsStage().ordinal();
240        assert !invokeNode.graph().isAfterFloatingReadPhase() : "inline isn't handled correctly after floating reads phase";
241
242        if (receiverNullCheck && !((MethodCallTargetNode) invoke.callTarget()).isStatic()) {
243            nonNullReceiver(invoke);
244        }
245
246        ArrayList<Node> nodes = new ArrayList<>(inlineGraph.getNodes().count());
247        ArrayList<ReturnNode> returnNodes = new ArrayList<>(4);
248        UnwindNode unwindNode = null;
249        final StartNode entryPointNode = inlineGraph.start();
250        FixedNode firstCFGNode = entryPointNode.next();
251        if (firstCFGNode == null) {
252            throw new IllegalStateException("Inlined graph is in invalid state: " + inlineGraph);
253        }
254        for (Node node : inlineGraph.getNodes()) {
255            if (node == entryPointNode || (node == entryPointNode.stateAfter() && node.usages().count() == 1) || node instanceof ParameterNode) {
256                // Do nothing.
257            } else {
258                nodes.add(node);
259                if (node instanceof ReturnNode) {
260                    returnNodes.add((ReturnNode) node);
261                } else if (node instanceof UnwindNode) {
262                    assert unwindNode == null;
263                    unwindNode = (UnwindNode) node;
264                }
265            }
266        }
267
268        final AbstractBeginNode prevBegin = AbstractBeginNode.prevBegin(invokeNode);
269        DuplicationReplacement localReplacement = new DuplicationReplacement() {
270
271            public Node replacement(Node node) {
272                if (node instanceof ParameterNode) {
273                    return parameters.get(((ParameterNode) node).index());
274                } else if (node == entryPointNode) {
275                    return prevBegin;
276                }
277                return node;
278            }
279        };
280
281        assert invokeNode.successors().first() != null : invoke;
282        assert invokeNode.predecessor() != null;
283
284        Map<Node, Node> duplicates = graph.addDuplicates(nodes, inlineGraph, inlineGraph.getNodeCount(), localReplacement);
285
286        FrameState stateAfter = invoke.stateAfter();
287        assert stateAfter == null || stateAfter.isAlive();
288
289        FrameState stateAtExceptionEdge = null;
290        if (invoke instanceof InvokeWithExceptionNode) {
291            InvokeWithExceptionNode invokeWithException = ((InvokeWithExceptionNode) invoke);
292            if (unwindNode != null) {
293                ExceptionObjectNode obj = (ExceptionObjectNode) invokeWithException.exceptionEdge();
294                stateAtExceptionEdge = obj.stateAfter();
295            }
296        }
297
298        processSimpleInfopoints(invoke, inlineGraph, duplicates);
299        if (stateAfter != null) {
300            processFrameStates(invoke, inlineGraph, duplicates, stateAtExceptionEdge, returnNodes.size() > 1);
301            int callerLockDepth = stateAfter.nestedLockDepth();
302            if (callerLockDepth != 0) {
303                for (MonitorIdNode original : inlineGraph.getNodes(MonitorIdNode.TYPE)) {
304                    MonitorIdNode monitor = (MonitorIdNode) duplicates.get(original);
305                    processMonitorId(invoke.stateAfter(), monitor);
306                }
307            }
308        } else {
309            assert checkContainsOnlyInvalidOrAfterFrameState(duplicates);
310        }
311
312        firstCFGNode = (FixedNode) duplicates.get(firstCFGNode);
313        for (int i = 0; i < returnNodes.size(); i++) {
314            returnNodes.set(i, (ReturnNode) duplicates.get(returnNodes.get(i)));
315        }
316        if (unwindNode != null) {
317            unwindNode = (UnwindNode) duplicates.get(unwindNode);
318        }
319
320        finishInlining(invoke, graph, firstCFGNode, returnNodes, unwindNode, inlineGraph.getAssumptions(), inlineGraph, canonicalizedNodes);
321
322        GraphUtil.killCFG(invokeNode);
323
324        return duplicates;
325    }
326
327    public static ValueNode finishInlining(Invoke invoke, StructuredGraph graph, FixedNode firstNode, List<ReturnNode> returnNodes, UnwindNode unwindNode, Assumptions inlinedAssumptions,
328                    StructuredGraph inlineGraph, List<Node> canonicalizedNodes) {
329        FixedNode invokeNode = invoke.asNode();
330        FrameState stateAfter = invoke.stateAfter();
331        assert stateAfter == null || stateAfter.isAlive();
332
333        invokeNode.replaceAtPredecessor(firstNode);
334
335        if (invoke instanceof InvokeWithExceptionNode) {
336            InvokeWithExceptionNode invokeWithException = ((InvokeWithExceptionNode) invoke);
337            if (unwindNode != null && unwindNode.isAlive()) {
338                assert unwindNode.predecessor() != null;
339                assert invokeWithException.exceptionEdge().successors().count() == 1;
340                ExceptionObjectNode obj = (ExceptionObjectNode) invokeWithException.exceptionEdge();
341                obj.replaceAtUsages(unwindNode.exception());
342                Node n = obj.next();
343                obj.setNext(null);
344                unwindNode.replaceAndDelete(n);
345
346                obj.replaceAtPredecessor(null);
347                obj.safeDelete();
348            } else {
349                invokeWithException.killExceptionEdge();
350            }
351
352            // get rid of memory kill
353            AbstractBeginNode begin = invokeWithException.next();
354            if (begin instanceof KillingBeginNode) {
355                AbstractBeginNode newBegin = new BeginNode();
356                graph.addAfterFixed(begin, graph.add(newBegin));
357                begin.replaceAtUsages(newBegin);
358                graph.removeFixed(begin);
359            }
360        } else {
361            if (unwindNode != null && unwindNode.isAlive()) {
362                DeoptimizeNode deoptimizeNode = graph.add(new DeoptimizeNode(DeoptimizationAction.InvalidateRecompile, DeoptimizationReason.NotCompiledExceptionHandler));
363                unwindNode.replaceAndDelete(deoptimizeNode);
364            }
365        }
366
367        ValueNode returnValue;
368        if (!returnNodes.isEmpty()) {
369            FixedNode n = invoke.next();
370            invoke.setNext(null);
371            if (returnNodes.size() == 1) {
372                ReturnNode returnNode = returnNodes.get(0);
373                returnValue = returnNode.result();
374                invokeNode.replaceAtUsages(returnValue);
375                returnNode.replaceAndDelete(n);
376            } else {
377                AbstractMergeNode merge = graph.add(new MergeNode());
378                merge.setStateAfter(stateAfter);
379                returnValue = mergeReturns(merge, returnNodes, canonicalizedNodes);
380                invokeNode.replaceAtUsages(returnValue);
381                merge.setNext(n);
382            }
383        } else {
384            returnValue = null;
385            invokeNode.replaceAtUsages(null);
386            GraphUtil.killCFG(invoke.next());
387        }
388
389        // Copy assumptions from inlinee to caller
390        Assumptions assumptions = graph.getAssumptions();
391        if (assumptions != null) {
392            if (inlinedAssumptions != null) {
393                assumptions.record(inlinedAssumptions);
394            }
395        } else {
396            assert inlinedAssumptions == null : "cannot inline graph which makes assumptions into a graph that doesn't";
397        }
398
399        // Copy inlined methods from inlinee to caller
400        graph.updateInlinedMethods(inlineGraph);
401        if (inlineGraph.hasUnsafeAccess()) {
402            graph.markUnsafeAccess();
403        }
404        assert inlineGraph.getSpeculationLog() == null : "Only the root graph should have a speculation log";
405
406        return returnValue;
407    }
408
409    private static String formatGraph(StructuredGraph graph) {
410        if (graph.method() == null) {
411            return graph.name;
412        }
413        return graph.method().format("%H.%n(%p)");
414    }
415
416    private static void processSimpleInfopoints(Invoke invoke, StructuredGraph inlineGraph, Map<Node, Node> duplicates) {
417        if (inlineGraph.getNodes(SimpleInfopointNode.TYPE).isEmpty()) {
418            return;
419        }
420        BytecodePosition pos = null;
421        for (SimpleInfopointNode original : inlineGraph.getNodes(SimpleInfopointNode.TYPE)) {
422            SimpleInfopointNode duplicate = (SimpleInfopointNode) duplicates.get(original);
423            pos = processSimpleInfopoint(invoke, duplicate, pos);
424        }
425    }
426
427    public static BytecodePosition processSimpleInfopoint(Invoke invoke, SimpleInfopointNode infopointNode, BytecodePosition incomingPos) {
428        BytecodePosition pos = processBytecodePosition(invoke, incomingPos);
429        infopointNode.addCaller(pos);
430        assert infopointNode.verify();
431        return pos;
432    }
433
434    public static BytecodePosition processBytecodePosition(Invoke invoke, BytecodePosition incomingPos) {
435        assert invoke.stateAfter() != null;
436        assert incomingPos == null || incomingPos.equals(InliningUtil.processBytecodePosition(invoke, null)) : incomingPos + " " + InliningUtil.processBytecodePosition(invoke, null);
437        return incomingPos != null ? incomingPos : new BytecodePosition(FrameState.toBytecodePosition(invoke.stateAfter().outerFrameState()), invoke.stateAfter().method(), invoke.bci());
438    }
439
440    public static void processMonitorId(FrameState stateAfter, MonitorIdNode monitorIdNode) {
441        if (stateAfter != null) {
442            int callerLockDepth = stateAfter.nestedLockDepth();
443            monitorIdNode.setLockDepth(monitorIdNode.getLockDepth() + callerLockDepth);
444        }
445    }
446
447    protected static void processFrameStates(Invoke invoke, StructuredGraph inlineGraph, Map<Node, Node> duplicates, FrameState stateAtExceptionEdge, boolean alwaysDuplicateStateAfter) {
448        FrameState stateAtReturn = invoke.stateAfter();
449        FrameState outerFrameState = null;
450        Kind invokeReturnKind = invoke.asNode().getKind();
451        for (FrameState original : inlineGraph.getNodes(FrameState.TYPE)) {
452            FrameState frameState = (FrameState) duplicates.get(original);
453            if (frameState != null && frameState.isAlive()) {
454                if (outerFrameState == null) {
455                    outerFrameState = stateAtReturn.duplicateModifiedDuringCall(invoke.bci(), invokeReturnKind);
456                }
457                processFrameState(frameState, invoke, inlineGraph.method(), stateAtExceptionEdge, outerFrameState, alwaysDuplicateStateAfter);
458            }
459        }
460    }
461
462    public static FrameState processFrameState(FrameState frameState, Invoke invoke, ResolvedJavaMethod inlinedMethod, FrameState stateAtExceptionEdge, FrameState outerFrameState,
463                    boolean alwaysDuplicateStateAfter) {
464
465        FrameState stateAtReturn = invoke.stateAfter();
466        Kind invokeReturnKind = invoke.asNode().getKind();
467
468        if (frameState.bci == BytecodeFrame.AFTER_BCI) {
469            FrameState stateAfterReturn = stateAtReturn;
470            if (frameState.method() == null) {
471                // This is a frame state for a side effect within an intrinsic
472                // that was parsed for post-parse intrinsification
473                for (Node usage : frameState.usages()) {
474                    if (usage instanceof ForeignCallNode) {
475                        // A foreign call inside an intrinsic needs to have
476                        // the BCI of the invoke being intrinsified
477                        ForeignCallNode foreign = (ForeignCallNode) usage;
478                        foreign.setBci(invoke.bci());
479                    }
480                }
481            }
482
483            /*
484             * pop return kind from invoke's stateAfter and replace with this frameState's return
485             * value (top of stack)
486             */
487            if (frameState.stackSize() > 0 && (alwaysDuplicateStateAfter || stateAfterReturn.stackAt(0) != frameState.stackAt(0))) {
488                stateAfterReturn = stateAtReturn.duplicateModified(invokeReturnKind, invokeReturnKind, frameState.stackAt(0));
489            }
490
491            frameState.replaceAndDelete(stateAfterReturn);
492            return stateAfterReturn;
493        } else if (stateAtExceptionEdge != null && isStateAfterException(frameState)) {
494            /*
495             * pop exception object from invoke's stateAfter and replace with this frameState's
496             * exception object (top of stack)
497             */
498            FrameState stateAfterException = stateAtExceptionEdge;
499            if (frameState.stackSize() > 0 && stateAtExceptionEdge.stackAt(0) != frameState.stackAt(0)) {
500                stateAfterException = stateAtExceptionEdge.duplicateModified(Kind.Object, Kind.Object, frameState.stackAt(0));
501            }
502            frameState.replaceAndDelete(stateAfterException);
503            return stateAfterException;
504        } else if (frameState.bci == BytecodeFrame.UNWIND_BCI || frameState.bci == BytecodeFrame.AFTER_EXCEPTION_BCI) {
505            return handleMissingAfterExceptionFrameState(frameState);
506        } else if (frameState.bci == BytecodeFrame.BEFORE_BCI) {
507            // This is an intrinsic. Deoptimizing within an intrinsic
508            // must re-execute the intrinsified invocation
509            assert frameState.outerFrameState() == null;
510            NodeInputList<ValueNode> invokeArgsList = invoke.callTarget().arguments();
511            ValueNode[] invokeArgs = invokeArgsList.isEmpty() ? NO_ARGS : invokeArgsList.toArray(new ValueNode[invokeArgsList.size()]);
512            ResolvedJavaMethod targetMethod = invoke.callTarget().targetMethod();
513            FrameState stateBeforeCall = stateAtReturn.duplicateModifiedBeforeCall(invoke.bci(), invokeReturnKind, targetMethod.getSignature().toParameterKinds(!targetMethod.isStatic()), invokeArgs);
514            frameState.replaceAndDelete(stateBeforeCall);
515            return stateBeforeCall;
516        } else {
517            // only handle the outermost frame states
518            if (frameState.outerFrameState() == null) {
519                assert checkInlineeFrameState(invoke, inlinedMethod, frameState);
520                frameState.setOuterFrameState(outerFrameState);
521            }
522            return frameState;
523        }
524    }
525
526    static boolean checkInlineeFrameState(Invoke invoke, ResolvedJavaMethod inlinedMethod, FrameState frameState) {
527        assert frameState.bci != BytecodeFrame.AFTER_EXCEPTION_BCI : frameState;
528        assert frameState.bci != BytecodeFrame.BEFORE_BCI : frameState;
529        assert frameState.bci != BytecodeFrame.UNKNOWN_BCI : frameState;
530        assert frameState.bci != BytecodeFrame.UNWIND_BCI : frameState;
531        if (frameState.bci != BytecodeFrame.INVALID_FRAMESTATE_BCI) {
532            if (frameState.method().equals(inlinedMethod)) {
533                // Normal inlining expects all outermost inlinee frame states to
534                // denote the inlinee method
535            } else if (frameState.method().equals(invoke.callTarget().targetMethod())) {
536                // This occurs when an intrinsic calls back to the original
537                // method to handle a slow path. During parsing of such a
538                // partial intrinsic, these calls are given frame states
539                // that exclude the outer frame state denoting a position
540                // in the intrinsic code.
541                assert inlinedMethod.getAnnotation(MethodSubstitution.class) != null : "expected an intrinsic when inlinee frame state matches method of call target but does not match the method of the inlinee graph: " +
542                                frameState;
543            } else if (frameState.method().getName().equals(inlinedMethod.getName())) {
544                // This can happen for method substitutions.
545            } else {
546                throw new AssertionError(String.format("inlinedMethod=%s frameState.method=%s frameState=%s invoke.method=%s", inlinedMethod, frameState.method(), frameState,
547                                invoke.callTarget().targetMethod()));
548            }
549        }
550        return true;
551    }
552
553    private static final ValueNode[] NO_ARGS = {};
554
555    private static boolean isStateAfterException(FrameState frameState) {
556        return frameState.bci == BytecodeFrame.AFTER_EXCEPTION_BCI || (frameState.bci == BytecodeFrame.UNWIND_BCI && !frameState.method().isSynchronized());
557    }
558
559    protected static FrameState handleMissingAfterExceptionFrameState(FrameState nonReplaceableFrameState) {
560        Graph graph = nonReplaceableFrameState.graph();
561        NodeWorkList workList = graph.createNodeWorkList();
562        workList.add(nonReplaceableFrameState);
563        for (Node node : workList) {
564            FrameState fs = (FrameState) node;
565            for (Node usage : fs.usages().snapshot()) {
566                if (!usage.isAlive()) {
567                    continue;
568                }
569                if (usage instanceof FrameState) {
570                    workList.add(usage);
571                } else {
572                    StateSplit stateSplit = (StateSplit) usage;
573                    FixedNode fixedStateSplit = stateSplit.asNode();
574                    if (fixedStateSplit instanceof AbstractMergeNode) {
575                        AbstractMergeNode merge = (AbstractMergeNode) fixedStateSplit;
576                        while (merge.isAlive()) {
577                            AbstractEndNode end = merge.forwardEnds().first();
578                            DeoptimizeNode deoptimizeNode = graph.add(new DeoptimizeNode(DeoptimizationAction.InvalidateRecompile, DeoptimizationReason.NotCompiledExceptionHandler));
579                            end.replaceAtPredecessor(deoptimizeNode);
580                            GraphUtil.killCFG(end);
581                        }
582                    } else {
583                        FixedNode deoptimizeNode = graph.add(new DeoptimizeNode(DeoptimizationAction.InvalidateRecompile, DeoptimizationReason.NotCompiledExceptionHandler));
584                        if (fixedStateSplit instanceof AbstractBeginNode) {
585                            deoptimizeNode = BeginNode.begin(deoptimizeNode);
586                        }
587                        fixedStateSplit.replaceAtPredecessor(deoptimizeNode);
588                        GraphUtil.killCFG(fixedStateSplit);
589                    }
590                }
591            }
592        }
593        return null;
594    }
595
596    public static ValueNode mergeReturns(AbstractMergeNode merge, List<? extends ReturnNode> returnNodes, List<Node> canonicalizedNodes) {
597        ValueNode singleReturnValue = null;
598        PhiNode returnValuePhi = null;
599        for (ReturnNode returnNode : returnNodes) {
600            ValueNode result = returnNode.result();
601            if (result != null) {
602                if (returnValuePhi == null && (singleReturnValue == null || singleReturnValue == result)) {
603                    /* Only one return value, so no need yet for a phi node. */
604                    singleReturnValue = result;
605
606                } else if (returnValuePhi == null) {
607                    /* Found a second return value, so create phi node. */
608                    returnValuePhi = merge.graph().addWithoutUnique(new ValuePhiNode(result.stamp().unrestricted(), merge));
609                    if (canonicalizedNodes != null) {
610                        canonicalizedNodes.add(returnValuePhi);
611                    }
612                    for (int i = 0; i < merge.forwardEndCount(); i++) {
613                        returnValuePhi.addInput(singleReturnValue);
614                    }
615                    returnValuePhi.addInput(result);
616
617                } else {
618                    /* Multiple return values, just add to existing phi node. */
619                    returnValuePhi.addInput(result);
620                }
621            }
622
623            // create and wire up a new EndNode
624            EndNode endNode = merge.graph().add(new EndNode());
625            merge.addForwardEnd(endNode);
626            returnNode.replaceAndDelete(endNode);
627        }
628
629        if (returnValuePhi != null) {
630            assert returnValuePhi.verify();
631            returnValuePhi.inferStamp();
632            return returnValuePhi;
633        } else {
634            return singleReturnValue;
635        }
636    }
637
638    private static boolean checkContainsOnlyInvalidOrAfterFrameState(Map<Node, Node> duplicates) {
639        for (Node node : duplicates.values()) {
640            if (node instanceof FrameState) {
641                FrameState frameState = (FrameState) node;
642                assert frameState.bci == BytecodeFrame.AFTER_BCI || frameState.bci == BytecodeFrame.INVALID_FRAMESTATE_BCI : node.toString(Verbosity.Debugger);
643            }
644        }
645        return true;
646    }
647
648    /**
649     * Gets the receiver for an invoke, adding a guard if necessary to ensure it is non-null, and
650     * ensuring that the resulting type is compatible with the method being invoked.
651     */
652    public static ValueNode nonNullReceiver(Invoke invoke) {
653        MethodCallTargetNode callTarget = (MethodCallTargetNode) invoke.callTarget();
654        assert !callTarget.isStatic() : callTarget.targetMethod();
655        StructuredGraph graph = callTarget.graph();
656        ValueNode firstParam = callTarget.arguments().get(0);
657        if (firstParam.getKind() == Kind.Object) {
658            Stamp paramStamp = firstParam.stamp();
659            Stamp stamp = paramStamp.join(StampFactory.declaredNonNull(callTarget.targetMethod().getDeclaringClass()));
660            if (!StampTool.isPointerNonNull(firstParam)) {
661                IsNullNode condition = graph.unique(new IsNullNode(firstParam));
662                FixedGuardNode fixedGuard = graph.add(new FixedGuardNode(condition, NullCheckException, InvalidateReprofile, true));
663                PiNode nonNullReceiver = graph.unique(new PiNode(firstParam, stamp, fixedGuard));
664                graph.addBeforeFixed(invoke.asNode(), fixedGuard);
665                callTarget.replaceFirstInput(firstParam, nonNullReceiver);
666                return nonNullReceiver;
667            }
668            if (!stamp.equals(paramStamp)) {
669                PiNode cast = graph.unique(new PiNode(firstParam, stamp));
670                callTarget.replaceFirstInput(firstParam, cast);
671                return cast;
672            }
673        }
674        return firstParam;
675    }
676
677    public static boolean canIntrinsify(Replacements replacements, ResolvedJavaMethod target, int invokeBci) {
678        return replacements.hasSubstitution(target, false, invokeBci);
679    }
680
681    public static StructuredGraph getIntrinsicGraph(Replacements replacements, ResolvedJavaMethod target, int invokeBci) {
682        return replacements.getSubstitution(target, invokeBci);
683    }
684
685    public static FixedWithNextNode inlineMacroNode(Invoke invoke, ResolvedJavaMethod concrete, Class<? extends FixedWithNextNode> macroNodeClass) throws JVMCIError {
686        StructuredGraph graph = invoke.asNode().graph();
687        if (!concrete.equals(((MethodCallTargetNode) invoke.callTarget()).targetMethod())) {
688            assert ((MethodCallTargetNode) invoke.callTarget()).invokeKind().hasReceiver();
689            InliningUtil.replaceInvokeCallTarget(invoke, graph, InvokeKind.Special, concrete);
690        }
691
692        FixedWithNextNode macroNode = createMacroNodeInstance(macroNodeClass, invoke);
693
694        CallTargetNode callTarget = invoke.callTarget();
695        if (invoke instanceof InvokeNode) {
696            graph.replaceFixedWithFixed((InvokeNode) invoke, graph.add(macroNode));
697        } else {
698            InvokeWithExceptionNode invokeWithException = (InvokeWithExceptionNode) invoke;
699            invokeWithException.killExceptionEdge();
700            graph.replaceSplitWithFixed(invokeWithException, graph.add(macroNode), invokeWithException.next());
701        }
702        GraphUtil.killWithUnusedFloatingInputs(callTarget);
703        return macroNode;
704    }
705
706    private static FixedWithNextNode createMacroNodeInstance(Class<? extends FixedWithNextNode> macroNodeClass, Invoke invoke) throws JVMCIError {
707        try {
708            Constructor<?> cons = macroNodeClass.getDeclaredConstructor(Invoke.class);
709            return (FixedWithNextNode) cons.newInstance(invoke);
710        } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) {
711            throw new GraalGraphJVMCIError(e).addContext(invoke.asNode()).addContext("macroSubstitution", macroNodeClass);
712        }
713    }
714}