|
7070
|
1 /*
|
|
|
2 * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
|
|
|
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
|
|
|
4 *
|
|
|
5 * This code is free software; you can redistribute it and/or modify it
|
|
|
6 * under the terms of the GNU General Public License version 2 only, as
|
|
|
7 * published by the Free Software Foundation.
|
|
|
8 *
|
|
|
9 * This code is distributed in the hope that it will be useful, but WITHOUT
|
|
|
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
|
12 * version 2 for more details (a copy is included in the LICENSE file that
|
|
|
13 * accompanied this code).
|
|
|
14 *
|
|
|
15 * You should have received a copy of the GNU General Public License version
|
|
|
16 * 2 along with this work; if not, write to the Free Software Foundation,
|
|
|
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
|
18 *
|
|
|
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
|
|
|
20 * or visit www.oracle.com if you need additional information or have any
|
|
|
21 * questions.
|
|
|
22 */
|
|
|
23 package com.oracle.graal.phases.common;
|
|
|
24
|
|
|
25 import java.lang.reflect.*;
|
|
|
26 import java.util.*;
|
|
|
27 import java.util.concurrent.*;
|
|
|
28
|
|
|
29 import com.oracle.graal.api.code.*;
|
|
|
30 import com.oracle.graal.api.meta.*;
|
|
|
31 import com.oracle.graal.api.meta.JavaTypeProfile.ProfiledType;
|
|
|
32 import com.oracle.graal.api.meta.ResolvedJavaType.Representation;
|
|
|
33 import com.oracle.graal.debug.*;
|
|
|
34 import com.oracle.graal.graph.*;
|
|
|
35 import com.oracle.graal.nodes.*;
|
|
|
36 import com.oracle.graal.nodes.calc.*;
|
|
|
37 import com.oracle.graal.nodes.extended.*;
|
|
|
38 import com.oracle.graal.nodes.java.*;
|
|
|
39 import com.oracle.graal.nodes.java.MethodCallTargetNode.InvokeKind;
|
|
|
40 import com.oracle.graal.nodes.spi.*;
|
|
|
41 import com.oracle.graal.nodes.type.*;
|
|
|
42 import com.oracle.graal.nodes.util.*;
|
|
|
43 import com.oracle.graal.phases.*;
|
|
|
44
|
|
|
45 public class InliningUtil {
|
|
|
46 private static final DebugMetric metricInliningTailDuplication = Debug.metric("InliningTailDuplication");
|
|
|
47 private static final String inliningDecisionsScopeString = "InliningDecisions";
|
|
|
48
|
|
|
49 public interface InliningCallback {
|
|
|
50 StructuredGraph buildGraph(final ResolvedJavaMethod method);
|
|
|
51 }
|
|
|
52
|
|
|
53 public interface InliningPolicy {
|
|
|
54 void initialize(StructuredGraph graph);
|
|
|
55 boolean continueInlining(StructuredGraph graph);
|
|
|
56 InlineInfo next();
|
|
|
57 void scanInvokes(Iterable<? extends Node> newNodes);
|
|
|
58 double inliningWeight(ResolvedJavaMethod caller, ResolvedJavaMethod method, Invoke invoke);
|
|
|
59 boolean isWorthInlining(InlineInfo info);
|
|
|
60 }
|
|
|
61
|
|
|
62 public interface WeightComputationPolicy {
|
|
|
63 double computeWeight(ResolvedJavaMethod caller, ResolvedJavaMethod method, Invoke invoke, boolean preferredInvoke);
|
|
|
64 }
|
|
|
65
|
|
|
66 public static void logNotInlinedMethod(InlineInfo info, String msg, Object... args) {
|
|
|
67 logInliningDecision(info, false, msg, args);
|
|
|
68 }
|
|
|
69
|
|
|
70 public static void logInliningDecision(InlineInfo info, boolean success, String msg, final Object... args) {
|
|
|
71 if (shouldLogInliningDecision()) {
|
|
|
72 logInliningDecision(methodName(info), success, msg, args);
|
|
|
73 }
|
|
|
74 }
|
|
|
75
|
|
|
76 public static void logInliningDecision(final String msg, final Object... args) {
|
|
|
77 Debug.scope(inliningDecisionsScopeString, new Runnable() {
|
|
|
78 public void run() {
|
|
|
79 Debug.log(msg, args);
|
|
|
80 }
|
|
|
81 });
|
|
|
82 }
|
|
|
83
|
|
|
84 private static boolean logNotInlinedMethodAndReturnFalse(Invoke invoke, String msg) {
|
|
|
85 if (shouldLogInliningDecision()) {
|
|
|
86 String methodString = invoke.callTarget() == null ? "callTarget=null" : invoke.callTarget().targetName();
|
|
|
87 logInliningDecision(methodString, false, msg, new Object[0]);
|
|
|
88 }
|
|
|
89 return false;
|
|
|
90 }
|
|
|
91
|
|
|
92 private static InlineInfo logNotInlinedMethodAndReturnNull(Invoke invoke, ResolvedJavaMethod method, String msg) {
|
|
|
93 if (shouldLogInliningDecision()) {
|
|
|
94 String methodString = methodName(method, invoke);
|
|
|
95 logInliningDecision(methodString, false, msg, new Object[0]);
|
|
|
96 }
|
|
|
97 return null;
|
|
|
98 }
|
|
|
99
|
|
|
100 private static boolean logNotInlinedMethodAndReturnFalse(Invoke invoke, ResolvedJavaMethod method, String msg) {
|
|
|
101 if (shouldLogInliningDecision()) {
|
|
|
102 String methodString = methodName(method, invoke);
|
|
|
103 logInliningDecision(methodString, false, msg, new Object[0]);
|
|
|
104 }
|
|
|
105 return false;
|
|
|
106 }
|
|
|
107
|
|
|
108 private static void logInliningDecision(final String methodString, final boolean success, final String msg, final Object... args) {
|
|
|
109 String inliningMsg = "inlining " + methodString + ": " + msg;
|
|
|
110 if (!success) {
|
|
|
111 inliningMsg = "not " + inliningMsg;
|
|
|
112 }
|
|
|
113 logInliningDecision(inliningMsg, args);
|
|
|
114 }
|
|
|
115
|
|
|
116 private static boolean shouldLogInliningDecision() {
|
|
|
117 return Debug.scope(inliningDecisionsScopeString, new Callable<Boolean>() {
|
|
|
118 public Boolean call() {
|
|
|
119 return Debug.isLogEnabled();
|
|
|
120 }
|
|
|
121 });
|
|
|
122 }
|
|
|
123
|
|
|
124 private static String methodName(ResolvedJavaMethod method, Invoke invoke) {
|
|
|
125 if (invoke != null && invoke.stateAfter() != null) {
|
|
|
126 return methodName(invoke.stateAfter(), invoke.bci()) + ": " + MetaUtil.format("%H.%n(%p):%r", method) + " (" + method.getCodeSize() + " bytes)";
|
|
|
127 } else {
|
|
|
128 return MetaUtil.format("%H.%n(%p):%r", method) + " (" + method.getCodeSize() + " bytes)";
|
|
|
129 }
|
|
|
130 }
|
|
|
131
|
|
|
132 private static String methodName(InlineInfo info) {
|
|
|
133 if (info == null) {
|
|
|
134 return "null";
|
|
|
135 } else if (info.invoke() != null && info.invoke().stateAfter() != null) {
|
|
|
136 return methodName(info.invoke().stateAfter(), info.invoke().bci()) + ": " + info.toString();
|
|
|
137 } else {
|
|
|
138 return info.toString();
|
|
|
139 }
|
|
|
140 }
|
|
|
141
|
|
|
142 private static String methodName(FrameState frameState, int bci) {
|
|
|
143 StringBuilder sb = new StringBuilder();
|
|
|
144 if (frameState.outerFrameState() != null) {
|
|
|
145 sb.append(methodName(frameState.outerFrameState(), frameState.outerFrameState().bci));
|
|
|
146 sb.append("->");
|
|
|
147 }
|
|
|
148 sb.append(MetaUtil.format("%h.%n", frameState.method()));
|
|
|
149 sb.append("@").append(bci);
|
|
|
150 return sb.toString();
|
|
|
151 }
|
|
|
152
|
|
|
153 /**
|
|
|
154 * Represents an opportunity for inlining at the given invoke, with the given weight and level.
|
|
|
155 * The weight is the amortized weight of the additional code - so smaller is better.
|
|
|
156 * The level is the number of nested inlinings that lead to this invoke.
|
|
|
157 */
|
|
|
158 public interface InlineInfo extends Comparable<InlineInfo> {
|
|
|
159 Invoke invoke();
|
|
|
160 double weight();
|
|
|
161 int level();
|
|
|
162 int compiledCodeSize();
|
|
|
163 int compareTo(InlineInfo o);
|
|
|
164
|
|
|
165 /**
|
|
|
166 * Performs the inlining described by this object and returns the node that represents the return value of the
|
|
|
167 * inlined method (or null for void methods and methods that have no non-exceptional exit).
|
|
|
168 */
|
|
|
169 void inline(StructuredGraph graph, GraalCodeCacheProvider runtime, InliningCallback callback, Assumptions assumptions);
|
|
|
170 }
|
|
|
171
|
|
|
172 public abstract static class AbstractInlineInfo implements InlineInfo {
|
|
|
173 protected final Invoke invoke;
|
|
|
174 protected final double weight;
|
|
|
175
|
|
|
176 public AbstractInlineInfo(Invoke invoke, double weight) {
|
|
|
177 this.invoke = invoke;
|
|
|
178 this.weight = weight;
|
|
|
179 }
|
|
|
180
|
|
|
181 @Override
|
|
|
182 public int compareTo(InlineInfo o) {
|
|
|
183 return (weight < o.weight()) ? -1 : (weight > o.weight()) ? 1 : 0;
|
|
|
184 }
|
|
|
185
|
|
|
186 public Invoke invoke() {
|
|
|
187 return invoke;
|
|
|
188 }
|
|
|
189
|
|
|
190 public double weight() {
|
|
|
191 return weight;
|
|
|
192 }
|
|
|
193
|
|
|
194 public int level() {
|
|
|
195 return computeInliningLevel(invoke);
|
|
|
196 }
|
|
|
197
|
|
|
198 protected static StructuredGraph getGraph(final Invoke invoke, final ResolvedJavaMethod concrete, final GraalCodeCacheProvider runtime, final InliningCallback callback) {
|
|
|
199 return Debug.scope("GetInliningGraph", concrete, new Callable<StructuredGraph>() {
|
|
|
200 @Override
|
|
|
201 public StructuredGraph call() throws Exception {
|
|
|
202 StructuredGraph result = getIntrinsicGraph(invoke, concrete, runtime);
|
|
|
203 if (result == null) {
|
|
|
204 assert !Modifier.isNative(concrete.getModifiers());
|
|
|
205 result = callback.buildGraph(concrete);
|
|
|
206 }
|
|
|
207 return result;
|
|
|
208 }
|
|
|
209 });
|
|
|
210 }
|
|
|
211 }
|
|
|
212
|
|
|
213 /**
|
|
|
214 * Represents an inlining opportunity where the compiler can statically determine a monomorphic target method and
|
|
|
215 * therefore is able to determine the called method exactly.
|
|
|
216 */
|
|
|
217 private static class ExactInlineInfo extends AbstractInlineInfo {
|
|
|
218 public final ResolvedJavaMethod concrete;
|
|
|
219
|
|
|
220 public ExactInlineInfo(Invoke invoke, double weight, ResolvedJavaMethod concrete) {
|
|
|
221 super(invoke, weight);
|
|
|
222 this.concrete = concrete;
|
|
|
223 }
|
|
|
224
|
|
|
225 @Override
|
|
|
226 public void inline(StructuredGraph compilerGraph, GraalCodeCacheProvider runtime, InliningCallback callback, Assumptions assumptions) {
|
|
|
227 StructuredGraph graph = getGraph(invoke, concrete, runtime, callback);
|
|
|
228 assumptions.recordMethodContents(concrete);
|
|
|
229 InliningUtil.inline(invoke, graph, true);
|
|
|
230 }
|
|
|
231
|
|
|
232 @Override
|
|
|
233 public int compiledCodeSize() {
|
|
|
234 return concrete.getCompiledCodeSize();
|
|
|
235 }
|
|
|
236
|
|
|
237 @Override
|
|
|
238 public String toString() {
|
|
|
239 return "exact " + MetaUtil.format("%H.%n(%p):%r", concrete);
|
|
|
240 }
|
|
|
241 }
|
|
|
242
|
|
|
243 /**
|
|
|
244 * Represents an inlining opportunity for which profiling information suggests a monomorphic receiver, but for which
|
|
|
245 * the receiver type cannot be proven. A type check guard will be generated if this inlining is performed.
|
|
|
246 */
|
|
|
247 private static class TypeGuardInlineInfo extends AbstractInlineInfo {
|
|
|
248 public final ResolvedJavaMethod concrete;
|
|
|
249 public final ResolvedJavaType type;
|
|
|
250
|
|
|
251 public TypeGuardInlineInfo(Invoke invoke, double weight, ResolvedJavaMethod concrete, ResolvedJavaType type) {
|
|
|
252 super(invoke, weight);
|
|
|
253 this.concrete = concrete;
|
|
|
254 this.type = type;
|
|
|
255 }
|
|
|
256
|
|
|
257 @Override
|
|
|
258 public int compiledCodeSize() {
|
|
|
259 return concrete.getCompiledCodeSize();
|
|
|
260 }
|
|
|
261
|
|
|
262 @Override
|
|
|
263 public void inline(StructuredGraph graph, GraalCodeCacheProvider runtime, InliningCallback callback, Assumptions assumptions) {
|
|
|
264 // receiver null check must be before the type check
|
|
|
265 InliningUtil.receiverNullCheck(invoke);
|
|
|
266 ValueNode receiver = invoke.methodCallTarget().receiver();
|
|
|
267 ConstantNode typeHub = ConstantNode.forConstant(type.getEncoding(Representation.ObjectHub), runtime, graph);
|
|
|
268 LoadHubNode receiverHub = graph.add(new LoadHubNode(receiver, typeHub.kind()));
|
|
|
269 CompareNode typeCheck = CompareNode.createCompareNode(Condition.EQ, receiverHub, typeHub);
|
|
|
270 FixedGuardNode guard = graph.add(new FixedGuardNode(typeCheck, DeoptimizationReason.TypeCheckedInliningViolated, DeoptimizationAction.InvalidateReprofile, invoke.leafGraphId()));
|
|
|
271 ValueAnchorNode anchor = graph.add(new ValueAnchorNode());
|
|
|
272 assert invoke.predecessor() != null;
|
|
|
273
|
|
|
274 ValueNode anchoredReceiver = createAnchoredReceiver(graph, anchor, type, receiver, true);
|
|
|
275 invoke.callTarget().replaceFirstInput(receiver, anchoredReceiver);
|
|
|
276
|
|
|
277 graph.addBeforeFixed(invoke.node(), receiverHub);
|
|
|
278 graph.addBeforeFixed(invoke.node(), guard);
|
|
|
279 graph.addBeforeFixed(invoke.node(), anchor);
|
|
|
280
|
|
|
281 StructuredGraph calleeGraph = getGraph(invoke, concrete, runtime, callback);
|
|
|
282 assumptions.recordMethodContents(concrete);
|
|
|
283 InliningUtil.inline(invoke, calleeGraph, false);
|
|
|
284 }
|
|
|
285
|
|
|
286 @Override
|
|
|
287 public String toString() {
|
|
|
288 return "type-checked " + MetaUtil.format("%H.%n(%p):%r", concrete);
|
|
|
289 }
|
|
|
290 }
|
|
|
291
|
|
|
292 /**
|
|
|
293 * Polymorphic inlining of m methods with n type checks (n >= m) in case that the profiling information suggests a reasonable
|
|
|
294 * amounts of different receiver types and different methods. If an unknown type is encountered a deoptimization is triggered.
|
|
|
295 */
|
|
|
296 private static class MultiTypeGuardInlineInfo extends AbstractInlineInfo {
|
|
|
297 public final List<ResolvedJavaMethod> concretes;
|
|
|
298 public final ProfiledType[] ptypes;
|
|
|
299 public final int[] typesToConcretes;
|
|
|
300 public final double notRecordedTypeProbability;
|
|
|
301
|
|
|
302 public MultiTypeGuardInlineInfo(Invoke invoke, double weight, List<ResolvedJavaMethod> concretes, ProfiledType[] ptypes,
|
|
|
303 int[] typesToConcretes, double notRecordedTypeProbability) {
|
|
|
304 super(invoke, weight);
|
|
|
305 assert concretes.size() > 0 && concretes.size() <= ptypes.length : "must have at least one method but no more than types methods";
|
|
|
306 assert ptypes.length == typesToConcretes.length : "array lengths must match";
|
|
|
307
|
|
|
308 this.concretes = concretes;
|
|
|
309 this.ptypes = ptypes;
|
|
|
310 this.typesToConcretes = typesToConcretes;
|
|
|
311 this.notRecordedTypeProbability = notRecordedTypeProbability;
|
|
|
312 }
|
|
|
313
|
|
|
314 @Override
|
|
|
315 public int compiledCodeSize() {
|
|
|
316 int result = 0;
|
|
|
317 for (ResolvedJavaMethod m: concretes) {
|
|
|
318 result += m.getCompiledCodeSize();
|
|
|
319 }
|
|
|
320 return result;
|
|
|
321 }
|
|
|
322
|
|
|
323 @Override
|
|
|
324 public void inline(StructuredGraph graph, GraalCodeCacheProvider runtime, InliningCallback callback, Assumptions assumptions) {
|
|
|
325 int numberOfMethods = concretes.size();
|
|
|
326 boolean hasReturnValue = invoke.node().kind() != Kind.Void;
|
|
|
327
|
|
|
328 // receiver null check must be the first node
|
|
|
329 InliningUtil.receiverNullCheck(invoke);
|
|
|
330 if (numberOfMethods > 1 || shouldFallbackToInvoke()) {
|
|
|
331 inlineMultipleMethods(graph, runtime, callback, assumptions, numberOfMethods, hasReturnValue);
|
|
|
332 } else {
|
|
|
333 inlineSingleMethod(graph, runtime, callback, assumptions);
|
|
|
334 }
|
|
|
335 }
|
|
|
336
|
|
|
337 private boolean shouldFallbackToInvoke() {
|
|
|
338 return notRecordedTypeProbability > 0;
|
|
|
339 }
|
|
|
340
|
|
|
341 private void inlineMultipleMethods(StructuredGraph graph, GraalCodeCacheProvider runtime, InliningCallback callback, Assumptions assumptions, int numberOfMethods, boolean hasReturnValue) {
|
|
|
342 FixedNode continuation = invoke.next();
|
|
|
343
|
|
|
344 ValueNode originalReceiver = invoke.methodCallTarget().receiver();
|
|
|
345 // setup merge and phi nodes for results and exceptions
|
|
|
346 MergeNode returnMerge = graph.add(new MergeNode());
|
|
|
347 returnMerge.setProbability(invoke.probability());
|
|
|
348 returnMerge.setStateAfter(invoke.stateAfter().duplicate(invoke.stateAfter().bci));
|
|
|
349
|
|
|
350 PhiNode returnValuePhi = null;
|
|
|
351 if (hasReturnValue) {
|
|
|
352 returnValuePhi = graph.unique(new PhiNode(invoke.node().kind(), returnMerge));
|
|
|
353 }
|
|
|
354
|
|
|
355 MergeNode exceptionMerge = null;
|
|
|
356 PhiNode exceptionObjectPhi = null;
|
|
|
357 if (invoke instanceof InvokeWithExceptionNode) {
|
|
|
358 InvokeWithExceptionNode invokeWithException = (InvokeWithExceptionNode) invoke;
|
|
|
359 DispatchBeginNode exceptionEdge = invokeWithException.exceptionEdge();
|
|
|
360 ExceptionObjectNode exceptionObject = (ExceptionObjectNode) exceptionEdge.next();
|
|
|
361
|
|
|
362 exceptionMerge = graph.add(new MergeNode());
|
|
|
363 exceptionMerge.setProbability(exceptionEdge.probability());
|
|
|
364
|
|
|
365 FixedNode exceptionSux = exceptionObject.next();
|
|
|
366 graph.addBeforeFixed(exceptionSux, exceptionMerge);
|
|
|
367 exceptionObjectPhi = graph.unique(new PhiNode(Kind.Object, exceptionMerge));
|
|
|
368 exceptionMerge.setStateAfter(exceptionEdge.stateAfter().duplicateModified(invoke.stateAfter().bci, true, Kind.Void, exceptionObjectPhi));
|
|
|
369 }
|
|
|
370
|
|
|
371 // create one separate block for each invoked method
|
|
|
372 BeginNode[] successors = new BeginNode[numberOfMethods + 1];
|
|
|
373 for (int i = 0; i < numberOfMethods; i++) {
|
|
|
374 double probability = 0;
|
|
|
375 for (int j = 0; j < typesToConcretes.length; j++) {
|
|
|
376 if (typesToConcretes[j] == i) {
|
|
|
377 probability += ptypes[j].getProbability();
|
|
|
378 }
|
|
|
379 }
|
|
|
380
|
|
|
381 successors[i] = createInvocationBlock(graph, invoke, returnMerge, returnValuePhi, exceptionMerge, exceptionObjectPhi, invoke.probability() * probability, true);
|
|
|
382 }
|
|
|
383
|
|
|
384 // create the successor for an unknown type
|
|
|
385 FixedNode unknownTypeSux;
|
|
|
386 if (shouldFallbackToInvoke()) {
|
|
|
387 unknownTypeSux = createInvocationBlock(graph, invoke, returnMerge, returnValuePhi, exceptionMerge, exceptionObjectPhi, notRecordedTypeProbability, false);
|
|
|
388 } else {
|
|
|
389 unknownTypeSux = graph.add(new DeoptimizeNode(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.TypeCheckedInliningViolated, invoke.leafGraphId()));
|
|
|
390 }
|
|
|
391 successors[successors.length - 1] = BeginNode.begin(unknownTypeSux);
|
|
|
392
|
|
|
393 // replace the invoke exception edge
|
|
|
394 if (invoke instanceof InvokeWithExceptionNode) {
|
|
|
395 InvokeWithExceptionNode invokeWithExceptionNode = (InvokeWithExceptionNode) invoke;
|
|
|
396 BeginNode exceptionEdge = invokeWithExceptionNode.exceptionEdge();
|
|
|
397 ExceptionObjectNode exceptionObject = (ExceptionObjectNode) exceptionEdge.next();
|
|
|
398 exceptionObject.replaceAtUsages(exceptionObjectPhi);
|
|
|
399 exceptionObject.setNext(null);
|
|
|
400 GraphUtil.killCFG(invokeWithExceptionNode.exceptionEdge());
|
|
|
401 }
|
|
|
402
|
|
|
403 // get all graphs and record assumptions
|
|
|
404 assert invoke.node().isAlive();
|
|
|
405 StructuredGraph[] calleeGraphs = new StructuredGraph[numberOfMethods];
|
|
|
406 for (int i = 0; i < numberOfMethods; i++) {
|
|
|
407 ResolvedJavaMethod concrete = concretes.get(i);
|
|
|
408 calleeGraphs[i] = getGraph(invoke, concrete, runtime, callback);
|
|
|
409 assumptions.recordMethodContents(concrete);
|
|
|
410 }
|
|
|
411
|
|
|
412 // replace the invoke with a switch on the type of the actual receiver
|
|
|
413 Kind hubKind = invoke.methodCallTarget().targetMethod().getDeclaringClass().getEncoding(Representation.ObjectHub).getKind();
|
|
|
414 LoadHubNode receiverHub = graph.add(new LoadHubNode(invoke.methodCallTarget().receiver(), hubKind));
|
|
|
415 graph.addBeforeFixed(invoke.node(), receiverHub);
|
|
|
416 FixedNode dispatchOnType = createDispatchOnType(graph, receiverHub, successors);
|
|
|
417
|
|
|
418 assert invoke.next() == continuation;
|
|
|
419 invoke.setNext(null);
|
|
|
420 returnMerge.setNext(continuation);
|
|
|
421 invoke.node().replaceAtUsages(returnValuePhi);
|
|
|
422 invoke.node().replaceAndDelete(dispatchOnType);
|
|
|
423
|
|
|
424 ArrayList<PiNode> replacements = new ArrayList<>();
|
|
|
425
|
|
|
426 // do the actual inlining for every invoke
|
|
|
427 for (int i = 0; i < numberOfMethods; i++) {
|
|
|
428 BeginNode node = successors[i];
|
|
|
429 Invoke invokeForInlining = (Invoke) node.next();
|
|
|
430
|
|
|
431 ResolvedJavaType commonType = getLeastCommonType(i);
|
|
|
432 ValueNode receiver = invokeForInlining.methodCallTarget().receiver();
|
|
|
433 boolean exact = getTypeCount(i) == 1;
|
|
|
434 PiNode anchoredReceiver = createAnchoredReceiver(graph, node, commonType, receiver, exact);
|
|
|
435 invokeForInlining.callTarget().replaceFirstInput(receiver, anchoredReceiver);
|
|
|
436
|
|
|
437 StructuredGraph calleeGraph = calleeGraphs[i];
|
|
|
438 InliningUtil.inline(invokeForInlining, calleeGraph, false);
|
|
|
439 replacements.add(anchoredReceiver);
|
|
|
440 }
|
|
|
441 if (shouldFallbackToInvoke()) {
|
|
|
442 replacements.add(null);
|
|
|
443 }
|
|
|
444 if (GraalOptions.OptTailDuplication) {
|
|
|
445 /*
|
|
|
446 * We might want to perform tail duplication at the merge after a type switch, if there are invokes that would
|
|
|
447 * benefit from the improvement in type information.
|
|
|
448 */
|
|
|
449 FixedNode current = returnMerge;
|
|
|
450 int opportunities = 0;
|
|
|
451 do {
|
|
|
452 if (current instanceof InvokeNode && ((InvokeNode) current).methodCallTarget().receiver() == originalReceiver) {
|
|
|
453 opportunities++;
|
|
|
454 } else if (current.inputs().contains(originalReceiver)) {
|
|
|
455 opportunities++;
|
|
|
456 }
|
|
|
457 current = ((FixedWithNextNode) current).next();
|
|
|
458 } while (current instanceof FixedWithNextNode);
|
|
|
459 if (opportunities > 0) {
|
|
|
460 metricInliningTailDuplication.increment();
|
|
|
461 Debug.log("MultiTypeGuardInlineInfo starting tail duplication (%d opportunities)", opportunities);
|
|
|
462 TailDuplicationPhase.tailDuplicate(returnMerge, TailDuplicationPhase.TRUE_DECISION, replacements);
|
|
|
463 }
|
|
|
464 }
|
|
|
465 }
|
|
|
466
|
|
|
467 private int getTypeCount(int concreteMethodIndex) {
|
|
|
468 int count = 0;
|
|
|
469 for (int i = 0; i < typesToConcretes.length; i++) {
|
|
|
470 if (typesToConcretes[i] == concreteMethodIndex) {
|
|
|
471 count++;
|
|
|
472 }
|
|
|
473 }
|
|
|
474 return count;
|
|
|
475 }
|
|
|
476
|
|
|
477 private ResolvedJavaType getLeastCommonType(int concreteMethodIndex) {
|
|
|
478 ResolvedJavaType commonType = null;
|
|
|
479 for (int i = 0; i < typesToConcretes.length; i++) {
|
|
|
480 if (typesToConcretes[i] == concreteMethodIndex) {
|
|
|
481 if (commonType == null) {
|
|
|
482 commonType = ptypes[i].getType();
|
|
|
483 } else {
|
|
|
484 commonType = commonType.findLeastCommonAncestor(ptypes[i].getType());
|
|
|
485 }
|
|
|
486 }
|
|
|
487 }
|
|
|
488 assert commonType != null;
|
|
|
489 return commonType;
|
|
|
490 }
|
|
|
491
|
|
|
492 private void inlineSingleMethod(StructuredGraph graph, GraalCodeCacheProvider runtime, InliningCallback callback, Assumptions assumptions) {
|
|
|
493 assert concretes.size() == 1 && ptypes.length > 1 && !shouldFallbackToInvoke() && notRecordedTypeProbability == 0;
|
|
|
494
|
|
|
495 BeginNode calleeEntryNode = graph.add(new BeginNode());
|
|
|
496 calleeEntryNode.setProbability(invoke.probability());
|
|
|
497 Kind hubKind = invoke.methodCallTarget().targetMethod().getDeclaringClass().getEncoding(Representation.ObjectHub).getKind();
|
|
|
498 LoadHubNode receiverHub = graph.add(new LoadHubNode(invoke.methodCallTarget().receiver(), hubKind));
|
|
|
499 graph.addBeforeFixed(invoke.node(), receiverHub);
|
|
|
500
|
|
|
501 BeginNode unknownTypeSux = BeginNode.begin(graph.add(new DeoptimizeNode(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.TypeCheckedInliningViolated, invoke.leafGraphId())));
|
|
|
502 BeginNode[] successors = new BeginNode[] {calleeEntryNode, unknownTypeSux};
|
|
|
503 FixedNode dispatchOnType = createDispatchOnType(graph, receiverHub, successors);
|
|
|
504
|
|
|
505 FixedWithNextNode pred = (FixedWithNextNode) invoke.node().predecessor();
|
|
|
506 pred.setNext(dispatchOnType);
|
|
|
507 calleeEntryNode.setNext(invoke.node());
|
|
|
508
|
|
|
509 ResolvedJavaMethod concrete = concretes.get(0);
|
|
|
510 StructuredGraph calleeGraph = getGraph(invoke, concrete, runtime, callback);
|
|
|
511 assumptions.recordMethodContents(concrete);
|
|
|
512 InliningUtil.inline(invoke, calleeGraph, false);
|
|
|
513 }
|
|
|
514
|
|
|
515 private FixedNode createDispatchOnType(StructuredGraph graph, LoadHubNode hub, BeginNode[] successors) {
|
|
|
516 assert ptypes.length > 1;
|
|
|
517
|
|
|
518 ResolvedJavaType[] keys = new ResolvedJavaType[ptypes.length];
|
|
|
519 double[] keyProbabilities = new double[ptypes.length + 1];
|
|
|
520 int[] keySuccessors = new int[ptypes.length + 1];
|
|
|
521 for (int i = 0; i < ptypes.length; i++) {
|
|
|
522 keys[i] = ptypes[i].getType();
|
|
|
523 keyProbabilities[i] = ptypes[i].getProbability();
|
|
|
524 keySuccessors[i] = typesToConcretes[i];
|
|
|
525 assert keySuccessors[i] < successors.length - 1 : "last successor is the unknownTypeSux";
|
|
|
526 }
|
|
|
527 keyProbabilities[keyProbabilities.length - 1] = notRecordedTypeProbability;
|
|
|
528 keySuccessors[keySuccessors.length - 1] = successors.length - 1;
|
|
|
529
|
|
|
530 double[] successorProbabilities = SwitchNode.successorProbabilites(successors.length, keySuccessors, keyProbabilities);
|
|
|
531 TypeSwitchNode typeSwitch = graph.add(new TypeSwitchNode(hub, successors, successorProbabilities, keys, keyProbabilities, keySuccessors));
|
|
|
532
|
|
|
533 return typeSwitch;
|
|
|
534 }
|
|
|
535
|
|
|
536 private static BeginNode createInvocationBlock(StructuredGraph graph, Invoke invoke, MergeNode returnMerge, PhiNode returnValuePhi,
|
|
|
537 MergeNode exceptionMerge, PhiNode exceptionObjectPhi, double probability, boolean useForInlining) {
|
|
|
538 Invoke duplicatedInvoke = duplicateInvokeForInlining(graph, invoke, exceptionMerge, exceptionObjectPhi, useForInlining, probability);
|
|
|
539 BeginNode calleeEntryNode = graph.add(new BeginNode());
|
|
|
540 calleeEntryNode.setNext(duplicatedInvoke.node());
|
|
|
541 calleeEntryNode.setProbability(probability);
|
|
|
542
|
|
|
543 EndNode endNode = graph.add(new EndNode());
|
|
|
544 endNode.setProbability(probability);
|
|
|
545
|
|
|
546 duplicatedInvoke.setNext(endNode);
|
|
|
547 returnMerge.addForwardEnd(endNode);
|
|
|
548
|
|
|
549 if (returnValuePhi != null) {
|
|
|
550 returnValuePhi.addInput(duplicatedInvoke.node());
|
|
|
551 }
|
|
|
552 return calleeEntryNode;
|
|
|
553 }
|
|
|
554
|
|
|
555 private static Invoke duplicateInvokeForInlining(StructuredGraph graph, Invoke invoke, MergeNode exceptionMerge, PhiNode exceptionObjectPhi, boolean useForInlining, double probability) {
|
|
|
556 Invoke result = (Invoke) invoke.node().copyWithInputs();
|
|
|
557 Node callTarget = result.callTarget().copyWithInputs();
|
|
|
558 result.node().replaceFirstInput(result.callTarget(), callTarget);
|
|
|
559 result.setUseForInlining(useForInlining);
|
|
|
560 result.setProbability(probability);
|
|
|
561
|
|
|
562 Kind kind = invoke.node().kind();
|
|
|
563 if (!kind.isVoid()) {
|
|
|
564 FrameState stateAfter = invoke.stateAfter();
|
|
|
565 stateAfter = stateAfter.duplicate(stateAfter.bci);
|
|
|
566 stateAfter.replaceFirstInput(invoke.node(), result.node());
|
|
|
567 result.setStateAfter(stateAfter);
|
|
|
568 }
|
|
|
569
|
|
|
570 if (invoke instanceof InvokeWithExceptionNode) {
|
|
|
571 assert exceptionMerge != null && exceptionObjectPhi != null;
|
|
|
572
|
|
|
573 InvokeWithExceptionNode invokeWithException = (InvokeWithExceptionNode) invoke;
|
|
|
574 BeginNode exceptionEdge = invokeWithException.exceptionEdge();
|
|
|
575 ExceptionObjectNode exceptionObject = (ExceptionObjectNode) exceptionEdge.next();
|
|
|
576 FrameState stateAfterException = exceptionObject.stateAfter();
|
|
|
577
|
|
|
578 BeginNode newExceptionEdge = (BeginNode) exceptionEdge.copyWithInputs();
|
|
|
579 ExceptionObjectNode newExceptionObject = (ExceptionObjectNode) exceptionObject.copyWithInputs();
|
|
|
580 // set new state (pop old exception object, push new one)
|
|
|
581 newExceptionObject.setStateAfter(stateAfterException.duplicateModified(stateAfterException.bci, stateAfterException.rethrowException(), Kind.Object, newExceptionObject));
|
|
|
582 newExceptionEdge.setNext(newExceptionObject);
|
|
|
583
|
|
|
584 EndNode endNode = graph.add(new EndNode());
|
|
|
585 newExceptionObject.setNext(endNode);
|
|
|
586 exceptionMerge.addForwardEnd(endNode);
|
|
|
587 exceptionObjectPhi.addInput(newExceptionObject);
|
|
|
588
|
|
|
589 ((InvokeWithExceptionNode) result).setExceptionEdge(newExceptionEdge);
|
|
|
590 }
|
|
|
591 return result;
|
|
|
592 }
|
|
|
593
|
|
|
594 @Override
|
|
|
595 public String toString() {
|
|
|
596 StringBuilder builder = new StringBuilder(shouldFallbackToInvoke() ? "megamorphic" : "polymorphic");
|
|
|
597 builder.append(String.format(", %d methods with %d type checks:", concretes.size(), ptypes.length));
|
|
|
598 for (int i = 0; i < concretes.size(); i++) {
|
|
|
599 builder.append(MetaUtil.format(" %H.%n(%p):%r", concretes.get(i)));
|
|
|
600 }
|
|
|
601 return builder.toString();
|
|
|
602 }
|
|
|
603 }
|
|
|
604
|
|
|
605
|
|
|
606 /**
|
|
|
607 * Represents an inlining opportunity where the current class hierarchy leads to a monomorphic target method,
|
|
|
608 * but for which an assumption has to be registered because of non-final classes.
|
|
|
609 */
|
|
|
610 private static class AssumptionInlineInfo extends ExactInlineInfo {
|
|
|
611 public final ResolvedJavaType context;
|
|
|
612
|
|
|
613 public AssumptionInlineInfo(Invoke invoke, double weight, ResolvedJavaType context, ResolvedJavaMethod concrete) {
|
|
|
614 super(invoke, weight, concrete);
|
|
|
615 this.context = context;
|
|
|
616 }
|
|
|
617
|
|
|
618 @Override
|
|
|
619 public void inline(StructuredGraph graph, GraalCodeCacheProvider runtime, InliningCallback callback, Assumptions assumptions) {
|
|
|
620 if (Debug.isLogEnabled()) {
|
|
|
621 String targetName = MetaUtil.format("%H.%n(%p):%r", invoke.methodCallTarget().targetMethod());
|
|
|
622 String concreteName = MetaUtil.format("%H.%n(%p):%r", concrete);
|
|
|
623 Debug.log("recording concrete method assumption: %s on receiver type %s -> %s", targetName, context, concreteName);
|
|
|
624 }
|
|
|
625 assumptions.recordConcreteMethod(invoke.methodCallTarget().targetMethod(), context, concrete);
|
|
|
626
|
|
|
627 super.inline(graph, runtime, callback, assumptions);
|
|
|
628 }
|
|
|
629
|
|
|
630 @Override
|
|
|
631 public String toString() {
|
|
|
632 return "assumption " + MetaUtil.format("%H.%n(%p):%r", concrete);
|
|
|
633 }
|
|
|
634 }
|
|
|
635
|
|
|
636 /**
|
|
|
637 * Determines if inlining is possible at the given invoke node.
|
|
|
638 * @param invoke the invoke that should be inlined
|
|
|
639 * @param runtime a GraalRuntime instance used to determine of the invoke can be inlined and/or should be intrinsified
|
|
|
640 * @param inliningPolicy used to determine the weight of a specific inlining
|
|
|
641 * @return an instance of InlineInfo, or null if no inlining is possible at the given invoke
|
|
|
642 */
|
|
|
643 public static InlineInfo getInlineInfo(Invoke invoke, GraalCodeCacheProvider runtime, Assumptions assumptions, InliningPolicy inliningPolicy, OptimisticOptimizations optimisticOpts) {
|
|
|
644 if (!checkInvokeConditions(invoke)) {
|
|
|
645 return null;
|
|
|
646 }
|
|
|
647 ResolvedJavaMethod caller = getCaller(invoke);
|
|
|
648 MethodCallTargetNode callTarget = invoke.methodCallTarget();
|
|
|
649 ResolvedJavaMethod targetMethod = callTarget.targetMethod();
|
|
|
650
|
|
|
651 if (callTarget.invokeKind() == InvokeKind.Special || targetMethod.canBeStaticallyBound()) {
|
|
|
652 if (!checkTargetConditions(invoke, targetMethod, optimisticOpts, runtime)) {
|
|
|
653 return null;
|
|
|
654 }
|
|
|
655 double weight = inliningPolicy.inliningWeight(caller, targetMethod, invoke);
|
|
|
656 return new ExactInlineInfo(invoke, weight, targetMethod);
|
|
|
657 }
|
|
|
658 ObjectStamp receiverStamp = callTarget.receiver().objectStamp();
|
|
|
659 ResolvedJavaType receiverType = receiverStamp.type();
|
|
|
660 if (receiverStamp.isExactType()) {
|
|
|
661 assert receiverType.isSubtypeOf(targetMethod.getDeclaringClass()) : receiverType + " subtype of " + targetMethod.getDeclaringClass() + " for " + targetMethod;
|
|
|
662 ResolvedJavaMethod resolved = receiverType.resolveMethod(targetMethod);
|
|
|
663 if (!checkTargetConditions(invoke, resolved, optimisticOpts, runtime)) {
|
|
|
664 return null;
|
|
|
665 }
|
|
|
666 double weight = inliningPolicy.inliningWeight(caller, resolved, invoke);
|
|
|
667 return new ExactInlineInfo(invoke, weight, resolved);
|
|
|
668 }
|
|
|
669 ResolvedJavaType holder = targetMethod.getDeclaringClass();
|
|
|
670
|
|
|
671 if (receiverStamp.type() != null) {
|
|
|
672 // the invoke target might be more specific than the holder (happens after inlining: locals lose their declared type...)
|
|
|
673 // TODO (lstadler) fix this
|
|
|
674 if (receiverType != null && receiverType.isSubtypeOf(holder)) {
|
|
|
675 holder = receiverType;
|
|
|
676 }
|
|
|
677 }
|
|
|
678 // TODO (thomaswue) fix this
|
|
|
679 if (assumptions.useOptimisticAssumptions()) {
|
|
|
680 ResolvedJavaMethod concrete = holder.findUniqueConcreteMethod(targetMethod);
|
|
|
681 if (concrete != null) {
|
|
|
682 if (!checkTargetConditions(invoke, concrete, optimisticOpts, runtime)) {
|
|
|
683 return null;
|
|
|
684 }
|
|
|
685 double weight = inliningPolicy.inliningWeight(caller, concrete, invoke);
|
|
|
686 return new AssumptionInlineInfo(invoke, weight, holder, concrete);
|
|
|
687 }
|
|
|
688 }
|
|
|
689
|
|
|
690 // type check based inlining
|
|
|
691 return getTypeCheckedInlineInfo(invoke, inliningPolicy, caller, targetMethod, optimisticOpts, runtime);
|
|
|
692 }
|
|
|
693
|
|
|
694 private static InlineInfo getTypeCheckedInlineInfo(Invoke invoke, InliningPolicy inliningPolicy, ResolvedJavaMethod caller,
|
|
|
695 ResolvedJavaMethod targetMethod, OptimisticOptimizations optimisticOpts, GraalCodeCacheProvider runtime) {
|
|
|
696 ProfilingInfo profilingInfo = caller.getProfilingInfo();
|
|
|
697 JavaTypeProfile typeProfile = profilingInfo.getTypeProfile(invoke.bci());
|
|
|
698 if (typeProfile == null) {
|
|
|
699 return logNotInlinedMethodAndReturnNull(invoke, targetMethod, "no type profile exists");
|
|
|
700 }
|
|
|
701
|
|
|
702 ProfiledType[] ptypes = typeProfile.getTypes();
|
|
|
703 if (ptypes == null || ptypes.length <= 0) {
|
|
|
704 return logNotInlinedMethodAndReturnNull(invoke, targetMethod, "no types/probabilities were recorded");
|
|
|
705 }
|
|
|
706
|
|
|
707 double notRecordedTypeProbability = typeProfile.getNotRecordedProbability();
|
|
|
708 if (ptypes.length == 1 && notRecordedTypeProbability == 0) {
|
|
|
709 if (!optimisticOpts.inlineMonomorphicCalls()) {
|
|
|
710 return logNotInlinedMethodAndReturnNull(invoke, targetMethod, "inlining monomorphic calls is disabled");
|
|
|
711 }
|
|
|
712
|
|
|
713 ResolvedJavaType type = ptypes[0].getType();
|
|
|
714 ResolvedJavaMethod concrete = type.resolveMethod(targetMethod);
|
|
|
715 if (!checkTargetConditions(invoke, concrete, optimisticOpts, runtime)) {
|
|
|
716 return null;
|
|
|
717 }
|
|
|
718 double weight = inliningPolicy.inliningWeight(caller, concrete, invoke);
|
|
|
719 return new TypeGuardInlineInfo(invoke, weight, concrete, type);
|
|
|
720 } else {
|
|
|
721 invoke.setPolymorphic(true);
|
|
|
722
|
|
|
723
|
|
|
724 if (!optimisticOpts.inlinePolymorphicCalls() && notRecordedTypeProbability == 0) {
|
|
|
725 return logNotInlinedMethodAndReturnNull(invoke, targetMethod, "inlining polymorphic calls is disabled");
|
|
|
726 }
|
|
|
727 if (!optimisticOpts.inlineMegamorphicCalls() && notRecordedTypeProbability > 0) {
|
|
|
728 return logNotInlinedMethodAndReturnNull(invoke, targetMethod, "inlining megamorphic calls is disabled");
|
|
|
729 }
|
|
|
730
|
|
|
731 // TODO (chaeubl) inlining of multiple methods should work differently
|
|
|
732 // 1. check which methods can be inlined
|
|
|
733 // 2. for those methods, use weight and probability to compute which of them should be inlined
|
|
|
734 // 3. do the inlining
|
|
|
735 // a) all seen methods can be inlined -> do so and guard with deopt
|
|
|
736 // b) some methods can be inlined -> inline them and fall back to invocation if violated
|
|
|
737
|
|
|
738 // determine concrete methods and map type to specific method
|
|
|
739 ArrayList<ResolvedJavaMethod> concreteMethods = new ArrayList<>();
|
|
|
740 int[] typesToConcretes = new int[ptypes.length];
|
|
|
741 for (int i = 0; i < ptypes.length; i++) {
|
|
|
742 ResolvedJavaMethod concrete = ptypes[i].getType().resolveMethod(targetMethod);
|
|
|
743
|
|
|
744 int index = concreteMethods.indexOf(concrete);
|
|
|
745 if (index < 0) {
|
|
|
746 index = concreteMethods.size();
|
|
|
747 concreteMethods.add(concrete);
|
|
|
748 }
|
|
|
749 typesToConcretes[i] = index;
|
|
|
750 }
|
|
|
751
|
|
|
752 double totalWeight = 0;
|
|
|
753 for (ResolvedJavaMethod concrete: concreteMethods) {
|
|
|
754 if (!checkTargetConditions(invoke, concrete, optimisticOpts, runtime)) {
|
|
|
755 return logNotInlinedMethodAndReturnNull(invoke, targetMethod, "it is a polymorphic method call and at least one invoked method cannot be inlined");
|
|
|
756 }
|
|
|
757 totalWeight += inliningPolicy.inliningWeight(caller, concrete, invoke);
|
|
|
758 }
|
|
|
759 return new MultiTypeGuardInlineInfo(invoke, totalWeight, concreteMethods, ptypes, typesToConcretes, notRecordedTypeProbability);
|
|
|
760 }
|
|
|
761 }
|
|
|
762
|
|
|
763
|
|
|
764 private static ResolvedJavaMethod getCaller(Invoke invoke) {
|
|
|
765 return invoke.stateAfter().method();
|
|
|
766 }
|
|
|
767
|
|
|
768 private static PiNode createAnchoredReceiver(StructuredGraph graph, FixedNode anchor, ResolvedJavaType commonType, ValueNode receiver, boolean exact) {
|
|
|
769 // to avoid that floating reads on receiver fields float above the type check
|
|
|
770 return graph.unique(new PiNode(receiver, anchor, exact ? StampFactory.exactNonNull(commonType) : StampFactory.declaredNonNull(commonType)));
|
|
|
771 }
|
|
|
772
|
|
|
773 private static boolean checkInvokeConditions(Invoke invoke) {
|
|
|
774 if (!(invoke.callTarget() instanceof MethodCallTargetNode)) {
|
|
|
775 return logNotInlinedMethodAndReturnFalse(invoke, "the invoke has already been lowered, or has been created as a low-level node");
|
|
|
776 } else if (invoke.methodCallTarget().targetMethod() == null) {
|
|
|
777 return logNotInlinedMethodAndReturnFalse(invoke, "target method is null");
|
|
|
778 } else if (invoke.stateAfter() == null) {
|
|
|
779 return logNotInlinedMethodAndReturnFalse(invoke, "the invoke has no after state");
|
|
|
780 } else if (invoke.predecessor() == null || !invoke.node().isAlive()) {
|
|
|
781 return logNotInlinedMethodAndReturnFalse(invoke, "the invoke is dead code");
|
|
|
782 } else if (!invoke.useForInlining()) {
|
|
|
783 return logNotInlinedMethodAndReturnFalse(invoke, "the invoke is marked to be not used for inlining");
|
|
|
784 } else {
|
|
|
785 return true;
|
|
|
786 }
|
|
|
787 }
|
|
|
788
|
|
|
789 private static boolean checkTargetConditions(Invoke invoke, ResolvedJavaMethod method, OptimisticOptimizations optimisticOpts, GraalCodeCacheProvider runtime) {
|
|
|
790 if (method == null) {
|
|
|
791 return logNotInlinedMethodAndReturnFalse(invoke, method, "the method is not resolved");
|
|
|
792 } else if (Modifier.isNative(method.getModifiers()) && (!GraalOptions.Intrinsify || !InliningUtil.canIntrinsify(invoke, method, runtime))) {
|
|
|
793 return logNotInlinedMethodAndReturnFalse(invoke, method, "it is a non-intrinsic native method");
|
|
|
794 } else if (Modifier.isAbstract(method.getModifiers())) {
|
|
|
795 return logNotInlinedMethodAndReturnFalse(invoke, method, "it is an abstract method");
|
|
|
796 } else if (!method.getDeclaringClass().isInitialized()) {
|
|
|
797 return logNotInlinedMethodAndReturnFalse(invoke, method, "the method's class is not initialized");
|
|
|
798 } else if (!method.canBeInlined()) {
|
|
|
799 return logNotInlinedMethodAndReturnFalse(invoke, method, "it is marked non-inlinable");
|
|
|
800 } else if (computeInliningLevel(invoke) > GraalOptions.MaximumInlineLevel) {
|
|
|
801 return logNotInlinedMethodAndReturnFalse(invoke, method, "it exceeds the maximum inlining depth");
|
|
|
802 } else if (computeRecursiveInliningLevel(invoke.stateAfter(), method) > GraalOptions.MaximumRecursiveInlining) {
|
|
|
803 return logNotInlinedMethodAndReturnFalse(invoke, method, "it exceeds the maximum recursive inlining depth");
|
|
|
804 } else if (new OptimisticOptimizations(method).lessOptimisticThan(optimisticOpts)) {
|
|
|
805 return logNotInlinedMethodAndReturnFalse(invoke, method, "the callee uses less optimistic optimizations than caller");
|
|
|
806 } else {
|
|
|
807 return true;
|
|
|
808 }
|
|
|
809 }
|
|
|
810
|
|
|
811 private static int computeInliningLevel(Invoke invoke) {
|
|
|
812 int count = -1;
|
|
|
813 FrameState curState = invoke.stateAfter();
|
|
|
814 while (curState != null) {
|
|
|
815 count++;
|
|
|
816 curState = curState.outerFrameState();
|
|
|
817 }
|
|
|
818 return count;
|
|
|
819 }
|
|
|
820
|
|
|
821 private static int computeRecursiveInliningLevel(FrameState state, ResolvedJavaMethod method) {
|
|
|
822 assert state != null;
|
|
|
823
|
|
|
824 int count = 0;
|
|
|
825 FrameState curState = state;
|
|
|
826 while (curState != null) {
|
|
|
827 if (curState.method() == method) {
|
|
|
828 count++;
|
|
|
829 }
|
|
|
830 curState = curState.outerFrameState();
|
|
|
831 }
|
|
|
832 return count;
|
|
|
833 }
|
|
|
834
|
|
|
835 /**
|
|
|
836 * Performs an actual inlining, thereby replacing the given invoke with the given inlineGraph.
|
|
|
837 *
|
|
|
838 * @param invoke the invoke that will be replaced
|
|
|
839 * @param inlineGraph the graph that the invoke will be replaced with
|
|
|
840 * @param receiverNullCheck true if a null check needs to be generated for non-static inlinings, false if no such check is required
|
|
|
841 */
|
|
|
842 public static void inline(Invoke invoke, StructuredGraph inlineGraph, boolean receiverNullCheck) {
|
|
|
843 NodeInputList<ValueNode> parameters = invoke.callTarget().arguments();
|
|
|
844 StructuredGraph graph = (StructuredGraph) invoke.node().graph();
|
|
|
845
|
|
|
846 FrameState stateAfter = invoke.stateAfter();
|
|
|
847 assert stateAfter.isAlive();
|
|
|
848
|
|
|
849 IdentityHashMap<Node, Node> replacements = new IdentityHashMap<>();
|
|
|
850 ArrayList<Node> nodes = new ArrayList<>();
|
|
|
851 ReturnNode returnNode = null;
|
|
|
852 UnwindNode unwindNode = null;
|
|
|
853 StartNode entryPointNode = inlineGraph.start();
|
|
|
854 FixedNode firstCFGNode = entryPointNode.next();
|
|
|
855 for (Node node : inlineGraph.getNodes()) {
|
|
|
856 if (node == entryPointNode || node == entryPointNode.stateAfter()) {
|
|
|
857 // Do nothing.
|
|
|
858 } else if (node instanceof LocalNode) {
|
|
|
859 replacements.put(node, parameters.get(((LocalNode) node).index()));
|
|
|
860 } else {
|
|
|
861 nodes.add(node);
|
|
|
862 if (node instanceof ReturnNode) {
|
|
|
863 assert returnNode == null;
|
|
|
864 returnNode = (ReturnNode) node;
|
|
|
865 } else if (node instanceof UnwindNode) {
|
|
|
866 assert unwindNode == null;
|
|
|
867 unwindNode = (UnwindNode) node;
|
|
|
868 }
|
|
|
869 }
|
|
|
870 }
|
|
|
871 replacements.put(entryPointNode, BeginNode.prevBegin(invoke.node())); // ensure proper anchoring of things that where anchored to the StartNode
|
|
|
872
|
|
|
873 assert invoke.node().successors().first() != null : invoke;
|
|
|
874 assert invoke.node().predecessor() != null;
|
|
|
875
|
|
|
876 Map<Node, Node> duplicates = graph.addDuplicates(nodes, replacements);
|
|
|
877 FixedNode firstCFGNodeDuplicate = (FixedNode) duplicates.get(firstCFGNode);
|
|
|
878 if (receiverNullCheck) {
|
|
|
879 receiverNullCheck(invoke);
|
|
|
880 }
|
|
|
881 invoke.node().replaceAtPredecessor(firstCFGNodeDuplicate);
|
|
|
882
|
|
|
883 FrameState stateAtExceptionEdge = null;
|
|
|
884 if (invoke instanceof InvokeWithExceptionNode) {
|
|
|
885 InvokeWithExceptionNode invokeWithException = ((InvokeWithExceptionNode) invoke);
|
|
|
886 if (unwindNode != null) {
|
|
|
887 assert unwindNode.predecessor() != null;
|
|
|
888 assert invokeWithException.exceptionEdge().successors().count() == 1;
|
|
|
889 ExceptionObjectNode obj = (ExceptionObjectNode) invokeWithException.exceptionEdge().next();
|
|
|
890 stateAtExceptionEdge = obj.stateAfter();
|
|
|
891 UnwindNode unwindDuplicate = (UnwindNode) duplicates.get(unwindNode);
|
|
|
892 obj.replaceAtUsages(unwindDuplicate.exception());
|
|
|
893 unwindDuplicate.clearInputs();
|
|
|
894 Node n = obj.next();
|
|
|
895 obj.setNext(null);
|
|
|
896 unwindDuplicate.replaceAndDelete(n);
|
|
|
897 } else {
|
|
|
898 invokeWithException.killExceptionEdge();
|
|
|
899 }
|
|
|
900 } else {
|
|
|
901 if (unwindNode != null) {
|
|
|
902 UnwindNode unwindDuplicate = (UnwindNode) duplicates.get(unwindNode);
|
|
|
903 DeoptimizeNode deoptimizeNode = new DeoptimizeNode(DeoptimizationAction.InvalidateRecompile, DeoptimizationReason.NotCompiledExceptionHandler, invoke.leafGraphId());
|
|
|
904 unwindDuplicate.replaceAndDelete(graph.add(deoptimizeNode));
|
|
|
905 // move the deopt upwards if there is a monitor exit that tries to use the "after exception" frame state
|
|
|
906 // (because there is no "after exception" frame state!)
|
|
|
907 if (deoptimizeNode.predecessor() instanceof MonitorExitNode) {
|
|
|
908 MonitorExitNode monitorExit = (MonitorExitNode) deoptimizeNode.predecessor();
|
|
|
909 if (monitorExit.stateAfter() != null && monitorExit.stateAfter().bci == FrameState.AFTER_EXCEPTION_BCI) {
|
|
|
910 FrameState monitorFrameState = monitorExit.stateAfter();
|
|
|
911 graph.removeFixed(monitorExit);
|
|
|
912 monitorFrameState.safeDelete();
|
|
|
913 }
|
|
|
914 }
|
|
|
915 }
|
|
|
916 }
|
|
|
917
|
|
|
918 FrameState outerFrameState = null;
|
|
|
919 double invokeProbability = invoke.node().probability();
|
|
|
920 for (Node node : duplicates.values()) {
|
|
|
921 if (GraalOptions.ProbabilityAnalysis) {
|
|
|
922 if (node instanceof FixedNode) {
|
|
|
923 FixedNode fixed = (FixedNode) node;
|
|
|
924 double newProbability = fixed.probability() * invokeProbability;
|
|
|
925 if (GraalOptions.LimitInlinedProbability) {
|
|
|
926 newProbability = Math.min(newProbability, invokeProbability);
|
|
|
927 }
|
|
|
928 fixed.setProbability(newProbability);
|
|
|
929 }
|
|
|
930 }
|
|
|
931 if (node instanceof FrameState) {
|
|
|
932 FrameState frameState = (FrameState) node;
|
|
|
933 assert frameState.bci != FrameState.BEFORE_BCI;
|
|
|
934 if (frameState.bci == FrameState.AFTER_BCI) {
|
|
|
935 frameState.replaceAndDelete(stateAfter);
|
|
|
936 } else if (frameState.bci == FrameState.AFTER_EXCEPTION_BCI) {
|
|
|
937 if (frameState.isAlive()) {
|
|
|
938 assert stateAtExceptionEdge != null;
|
|
|
939 frameState.replaceAndDelete(stateAtExceptionEdge);
|
|
|
940 } else {
|
|
|
941 assert stateAtExceptionEdge == null;
|
|
|
942 }
|
|
|
943 } else {
|
|
|
944 // only handle the outermost frame states
|
|
|
945 if (frameState.outerFrameState() == null) {
|
|
|
946 assert frameState.method() == inlineGraph.method();
|
|
|
947 if (outerFrameState == null) {
|
|
|
948 outerFrameState = stateAfter.duplicateModified(invoke.bci(), stateAfter.rethrowException(), invoke.node().kind());
|
|
|
949 outerFrameState.setDuringCall(true);
|
|
|
950 }
|
|
|
951 frameState.setOuterFrameState(outerFrameState);
|
|
|
952 }
|
|
|
953 }
|
|
|
954 }
|
|
|
955 }
|
|
|
956
|
|
|
957 Node returnValue = null;
|
|
|
958 if (returnNode != null) {
|
|
|
959 if (returnNode.result() instanceof LocalNode) {
|
|
|
960 returnValue = replacements.get(returnNode.result());
|
|
|
961 } else {
|
|
|
962 returnValue = duplicates.get(returnNode.result());
|
|
|
963 }
|
|
|
964 invoke.node().replaceAtUsages(returnValue);
|
|
|
965 Node returnDuplicate = duplicates.get(returnNode);
|
|
|
966 returnDuplicate.clearInputs();
|
|
|
967 Node n = invoke.next();
|
|
|
968 invoke.setNext(null);
|
|
|
969 returnDuplicate.replaceAndDelete(n);
|
|
|
970 }
|
|
|
971
|
|
|
972 invoke.node().replaceAtUsages(null);
|
|
|
973 GraphUtil.killCFG(invoke.node());
|
|
|
974 }
|
|
|
975
|
|
|
976 public static void receiverNullCheck(Invoke invoke) {
|
|
|
977 MethodCallTargetNode callTarget = invoke.methodCallTarget();
|
|
|
978 StructuredGraph graph = (StructuredGraph) invoke.graph();
|
|
|
979 NodeInputList<ValueNode> parameters = callTarget.arguments();
|
|
|
980 ValueNode firstParam = parameters.size() <= 0 ? null : parameters.get(0);
|
|
|
981 if (!callTarget.isStatic() && firstParam.kind() == Kind.Object && !firstParam.objectStamp().nonNull()) {
|
|
|
982 graph.addBeforeFixed(invoke.node(), graph.add(new FixedGuardNode(graph.unique(new IsNullNode(firstParam)), DeoptimizationReason.NullCheckException, DeoptimizationAction.InvalidateReprofile, true, invoke.leafGraphId())));
|
|
|
983 }
|
|
|
984 }
|
|
|
985
|
|
|
986 public static boolean canIntrinsify(Invoke invoke, ResolvedJavaMethod target, GraalCodeCacheProvider runtime) {
|
|
|
987 return getIntrinsicGraph(invoke, target, runtime) != null;
|
|
|
988 }
|
|
|
989
|
|
|
990 private static StructuredGraph getIntrinsicGraph(Invoke invoke, ResolvedJavaMethod target, GraalCodeCacheProvider runtime) {
|
|
|
991 assert invoke.node().isAlive();
|
|
|
992
|
|
|
993 StructuredGraph intrinsicGraph = (StructuredGraph) target.getCompilerStorage().get(Graph.class);
|
|
|
994 if (intrinsicGraph == null) {
|
|
|
995 // TODO remove once all intrinsics are available via compilerStorage
|
|
|
996 intrinsicGraph = runtime.intrinsicGraph(invoke.stateAfter().method(), invoke.bci(), target, invoke.callTarget().arguments());
|
|
|
997 }
|
|
|
998 return intrinsicGraph;
|
|
|
999 }
|
|
|
1000 }
|
