Mercurial > hg > truffle
view graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/util/LoopUtil.java @ 3215:0ab38d143795
Fix for loop inversion now runs tests, fop, lusearch, eclipse, avrora and scimark
author | Gilles Duboscq <gilles.duboscq@oracle.com> |
---|---|
date | Wed, 13 Jul 2011 15:08:49 +0200 |
parents | 1e5ca59c8769 |
children | 7c4b4daac19b 2423a432fa6b |
line wrap: on
line source
/* * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. */ package com.oracle.max.graal.compiler.util; import java.util.*; import java.util.Map.Entry; import com.oracle.max.graal.compiler.*; import com.oracle.max.graal.compiler.ir.*; import com.oracle.max.graal.compiler.ir.Phi.*; import com.oracle.max.graal.compiler.observer.*; import com.oracle.max.graal.compiler.schedule.*; import com.oracle.max.graal.compiler.util.GraphUtil.ColorSplitingLambda; import com.oracle.max.graal.compiler.util.GraphUtil.ColoringLambda; import com.oracle.max.graal.compiler.value.*; import com.oracle.max.graal.graph.*; import com.sun.cri.ci.*; public class LoopUtil { public static class Loop { private final LoopBegin loopBegin; private NodeBitMap cfgNodes; private Loop parent; private NodeBitMap exits; private NodeBitMap inOrBefore; private NodeBitMap inOrAfter; private NodeBitMap nodes; public Loop(LoopBegin loopBegin, NodeBitMap nodes, NodeBitMap exits) { this.loopBegin = loopBegin; this.cfgNodes = nodes; this.exits = exits; } public LoopBegin loopBegin() { return loopBegin; } public NodeBitMap cfgNodes() { return cfgNodes; } public NodeBitMap nodes() { if (nodes == null) { nodes = loopBegin().graph().createNodeBitMap(); nodes.setUnion(inOrAfter()); nodes.setIntersect(inOrBefore()); } return nodes; } public Loop parent() { return parent; } public NodeBitMap exits() { return exits; } public void setParent(Loop parent) { this.parent = parent; } public boolean isChild(Loop loop) { return loop.parent != null && (loop.parent == this || loop.parent.isChild(this)); } public NodeBitMap inOrAfter() { if (inOrAfter == null) { inOrAfter = LoopUtil.inOrAfter(this); } return inOrAfter; } public NodeBitMap inOrBefore() { if (inOrBefore == null) { inOrBefore = LoopUtil.inOrBefore(this, inOrAfter()); } return inOrBefore; } public void invalidateCached() { inOrAfter = null; inOrBefore = null; nodes = null; } @Override public String toString() { return "Loop #" + loopBegin().id(); } } private static class PeelingResult { public final FixedNode begin; public final FixedNode end; public final NodeMap<StateSplit> exits; public final NodeBitMap unaffectedExits; public final NodeMap<Placeholder> phis; public final NodeMap<Node> phiInits; public final NodeMap<Node> dataOut; public final NodeBitMap exitFrameStates; public final NodeBitMap peeledNodes; public PeelingResult(FixedNode begin, FixedNode end, NodeMap<StateSplit> exits, NodeMap<Placeholder> phis, NodeMap<Node> phiInits, NodeMap<Node> dataOut, NodeBitMap unaffectedExits, NodeBitMap exitFrameStates, NodeBitMap peeledNodes) { this.begin = begin; this.end = end; this.exits = exits; this.phis = phis; this.phiInits = phiInits; this.dataOut = dataOut; this.unaffectedExits = unaffectedExits; this.exitFrameStates = exitFrameStates; this.peeledNodes = peeledNodes; } } public static List<Loop> computeLoops(Graph graph) { List<Loop> loops = new LinkedList<LoopUtil.Loop>(); for (LoopBegin loopBegin : graph.getNodes(LoopBegin.class)) { NodeBitMap cfgNodes = markUpCFG(loopBegin, loopBegin.loopEnd()); // computeLoopNodes(loopBegin); cfgNodes.mark(loopBegin); NodeBitMap exits = computeLoopExits(loopBegin, cfgNodes); loops.add(new Loop(loopBegin, cfgNodes, exits)); } for (Loop loop : loops) { for (Loop other : loops) { if (other != loop && other.cfgNodes().isMarked(loop.loopBegin())) { if (loop.parent() == null || loop.parent().cfgNodes().isMarked(other.loopBegin())) { loop.setParent(other); } } } } return loops; } public static NodeBitMap computeLoopExits(LoopBegin loopBegin, NodeBitMap cfgNodes) { Graph graph = loopBegin.graph(); NodeBitMap exits = graph.createNodeBitMap(); for (Node n : cfgNodes) { if (IdentifyBlocksPhase.trueSuccessorCount(n) > 1) { for (Node sux : n.cfgSuccessors()) { if (!cfgNodes.isMarked(sux) && sux instanceof FixedNode) { exits.mark(sux); } } } } return exits; } private static boolean recurse = false; public static NodeBitMap computeLoopNodesFrom(Loop loop, FixedNode from) { LoopBegin loopBegin = loop.loopBegin(); NodeBitMap loopNodes = markUpCFG(loopBegin, from); loopNodes.mark(loopBegin); NodeBitMap inOrAfter = inOrAfter(loop, loopNodes, false); NodeBitMap inOrBefore = inOrBefore(loop, inOrAfter, loopNodes, false); if (!recurse) { recurse = true; GraalCompilation compilation = GraalCompilation.compilation(); if (compilation.compiler.isObserved()) { Map<String, Object> debug = new HashMap<String, Object>(); debug.put("loopNodes", loopNodes); debug.put("inOrAfter", inOrAfter); debug.put("inOrBefore", inOrBefore); compilation.compiler.fireCompilationEvent(new CompilationEvent(compilation, "Compute loop nodes loop#" + loopBegin.id(), loopBegin.graph(), true, false, debug)); } recurse = false; } inOrAfter.setIntersect(inOrBefore); loopNodes.setUnion(inOrAfter); if (from == loopBegin.loopEnd()) { // fill the Loop cache value for loop nodes this is correct even if after/before were partial loop.nodes = loopNodes; } return loopNodes; } public static NodeBitMap markUpCFG(LoopBegin loopBegin) { return markUpCFG(loopBegin, loopBegin.loopEnd()); } public static NodeBitMap markUpCFG(LoopBegin loopBegin, FixedNode from) { NodeFlood workCFG = loopBegin.graph().createNodeFlood(); workCFG.add(from); NodeBitMap loopNodes = loopBegin.graph().createNodeBitMap(); for (Node n : workCFG) { if (n == loopBegin) { continue; } loopNodes.mark(n); if (n instanceof LoopBegin) { workCFG.add(((LoopBegin) n).loopEnd()); } for (Node pred : n.cfgPredecessors()) { workCFG.add(pred); } } return loopNodes; } public static void inverseLoop(Loop loop, If split) { assert loop.cfgNodes().isMarked(split); FixedNode noExit = split.trueSuccessor(); FixedNode exit = split.falseSuccessor(); if (loop.cfgNodes().isMarked(exit) && !loop.cfgNodes().isMarked(noExit)) { FixedNode tmp = noExit; noExit = exit; exit = tmp; } assert !loop.cfgNodes().isMarked(exit); assert loop.cfgNodes().isMarked(noExit); PeelingResult peeling = preparePeeling(loop, split); rewireInversion(peeling, loop, split); // move peeled part to the end LoopBegin loopBegin = loop.loopBegin(); LoopEnd loopEnd = loopBegin.loopEnd(); FixedNode lastNode = (FixedNode) loopEnd.singlePredecessor(); if (loopBegin.next() != lastNode) { lastNode.successors().replace(loopEnd, loopBegin.next()); loopBegin.setNext(noExit); split.successors().replace(noExit, loopEnd); } //rewire phi usage in peeled part int backIndex = loopBegin.phiPredecessorIndex(loopBegin.loopEnd()); for (Phi phi : loopBegin.phis()) { Value backValue = phi.valueAt(backIndex); if (loop.nodes().isMarked(backValue) && peeling.peeledNodes.isNotNewNotMarked(backValue)) { List<Node> usages = new ArrayList<Node>(phi.usages()); for (Node usage : usages) { if (peeling.peeledNodes.isNotNewMarked(usage)) { usage.inputs().replace(phi, backValue); } } } } loop.invalidateCached(); // update parents Loop parent = loop.parent(); while (parent != null) { parent.cfgNodes = markUpCFG(parent.loopBegin, parent.loopBegin.loopEnd()); parent.invalidateCached(); parent.exits = computeLoopExits(parent.loopBegin, parent.cfgNodes); parent = parent.parent; } GraalMetrics.LoopsInverted++; } public static void peelLoop(Loop loop) { LoopEnd loopEnd = loop.loopBegin().loopEnd(); PeelingResult peeling = preparePeeling(loop, loopEnd); GraalCompilation compilation = GraalCompilation.compilation(); if (compilation.compiler.isObserved()) { compilation.compiler.fireCompilationEvent(new CompilationEvent(compilation, "After peeling preparation", loopEnd.graph(), true, false)); } /*System.out.println("Peeling : "); System.out.println(" begin = " + peeling.begin); System.out.println(" end = " + peeling.end); System.out.println(" Phis :"); for (Entry<Node, Placeholder> entry : peeling.phis.entries()) { System.out.println(" - " + entry.getKey() + " -> " + entry.getValue()); } System.out.println(" Exits :"); for (Entry<Node, StateSplit> entry : peeling.exits.entries()) { System.out.println(" - " + entry.getKey() + " -> " + entry.getValue()); } System.out.println(" PhiInits :"); for (Entry<Node, Node> entry : peeling.phiInits.entries()) { System.out.println(" - " + entry.getKey() + " -> " + entry.getValue()); } System.out.println(" DataOut :"); for (Entry<Node, Node> entry : peeling.dataOut.entries()) { System.out.println(" - " + entry.getKey() + " -> " + entry.getValue()); }*/ rewirePeeling(peeling, loop); /*if (compilation.compiler.isObserved()) { compilation.compiler.fireCompilationEvent(new CompilationEvent(compilation, "After rewirePeeling", loopEnd.graph(), true, false)); }*/ loop.invalidateCached(); // update parents Loop parent = loop.parent(); while (parent != null) { parent.cfgNodes = markUpCFG(parent.loopBegin, parent.loopBegin.loopEnd()); parent.invalidateCached(); parent.exits = computeLoopExits(parent.loopBegin, parent.cfgNodes); parent = parent.parent; } GraalMetrics.LoopsPeeled++; } private static void rewireInversion(PeelingResult peeling, Loop loop, FixedNode from) { rewirePeeling(peeling, loop, from, true); } private static void rewirePeeling(PeelingResult peeling, Loop loop) { rewirePeeling(peeling, loop, loop.loopBegin().loopEnd(), false); } private static void rewirePeeling(PeelingResult peeling, Loop loop, FixedNode from, boolean inversion) { LoopBegin loopBegin = loop.loopBegin(); Graph graph = loopBegin.graph(); Node loopPred = loopBegin.singlePredecessor(); loopPred.successors().replace(loopBegin.forwardEdge(), peeling.begin); NodeBitMap loopNodes = loop.cfgNodes(); Node originalLast = from; if (originalLast == loopBegin.loopEnd()) { originalLast = loopBegin.loopEnd().singlePredecessor(); } int size = originalLast.successors().size(); boolean found = false; for (int i = 0; i < size; i++) { Node sux = originalLast.successors().get(i); if (sux == null) { continue; } if (loopNodes.isMarked(sux)) { assert !found; peeling.end.successors().set(i, loopBegin.forwardEdge()); found = true; } } assert found; int phiInitIndex = loopBegin.phiPredecessorIndex(loopBegin.forwardEdge()); for (Entry<Node, Placeholder> entry : peeling.phis.entries()) { Phi phi = (Phi) entry.getKey(); Placeholder p = entry.getValue(); Value init = phi.valueAt(phiInitIndex); p.replaceAndDelete(init); for (Entry<Node, Node> dataEntry : peeling.dataOut.entries()) { if (dataEntry.getValue() == p) { dataEntry.setValue(init); //System.out.println("Patch dataOut : " + dataEntry.getKey() + " -> " + dataEntry.getValue()); } } } for (Entry<Node, Node> entry : peeling.phiInits.entries()) { Phi phi = (Phi) entry.getKey(); Node newInit = entry.getValue(); phi.setValueAt(phiInitIndex, (Value) newInit); } if (from == loopBegin.loopEnd()) { for (LoopCounter counter : loopBegin.counters()) { counter.setInit(new IntegerAdd(counter.kind, counter.init(), counter.stride(), graph)); } } NodeMap<NodeMap<Value>> newExitValues = graph.createNodeMap(); List<Node> exitPoints = new LinkedList<Node>(); for (Node exit : peeling.unaffectedExits) { exitPoints.add(exit); } for (Entry<Node, StateSplit> entry : peeling.exits.entries()) { StateSplit original = (StateSplit) entry.getKey(); StateSplit newExit = entry.getValue(); EndNode oEnd = new EndNode(graph); EndNode nEnd = new EndNode(graph); Merge merge = new Merge(graph); FrameState originalState = original.stateAfter(); merge.addEnd(nEnd); merge.addEnd(oEnd); merge.setStateAfter(originalState.duplicate(originalState.bci, true)); merge.setNext(original.next()); original.setNext(oEnd); newExit.setNext(nEnd); exitPoints.add(original); exitPoints.add(newExit); } int phiBackIndex = loopBegin.phiPredecessorIndex(loopBegin.loopEnd()); for (Entry<Node, StateSplit> entry : peeling.exits.entries()) { StateSplit original = (StateSplit) entry.getKey(); EndNode oEnd = (EndNode) original.next(); Merge merge = oEnd.merge(); EndNode nEnd = merge.endAt(1 - merge.phiPredecessorIndex(oEnd)); Node newExit = nEnd.singlePredecessor(); for (Entry<Node, Node> dataEntry : peeling.dataOut.entries()) { Node originalValue = dataEntry.getKey(); Node newValue = dataEntry.getValue(); NodeMap<Value> phiMap = newExitValues.get(originalValue); if (phiMap == null) { phiMap = graph.createNodeMap(); newExitValues.set(originalValue, phiMap); } Value backValue = null; if (inversion && originalValue instanceof Phi && ((Phi) originalValue).merge() == loopBegin) { backValue = ((Phi) originalValue).valueAt(phiBackIndex); if (peeling.peeledNodes.isNotNewMarked(backValue)) { backValue = null; } } if (backValue != null) { phiMap.set(original, backValue); } else { phiMap.set(original, (Value) originalValue); } phiMap.set(newExit, (Value) newValue); } } if (inversion) { // rewire dataOut in non-peeled body NodeBitMap exitMergesPhis = graph.createNodeBitMap(); for (Entry<Node, StateSplit> entry : peeling.exits.entries()) { StateSplit newExit = entry.getValue(); Merge merge = ((EndNode) newExit.next()).merge(); exitMergesPhis.markAll(merge.phis()); } for (Entry<Node, Node> entry : peeling.dataOut.entries()) { Value originalValue = (Value) entry.getKey(); if (originalValue instanceof Phi && ((Phi) originalValue).merge() == loopBegin) { continue; } Value newValue = (Value) entry.getValue(); Phi phi = null; List<Node> usages = new ArrayList<Node>(originalValue.usages()); for (Node usage : usages) { if (exitMergesPhis.isMarked(usage) || ( loop.nodes().isNotNewMarked(usage) && peeling.peeledNodes.isNotNewNotMarked(usage) && !(usage instanceof Phi && ((Phi) usage).merge() == loopBegin)) && !(usage instanceof FrameState && ((FrameState) usage).block() == loopBegin)) { if (phi == null) { phi = new Phi(originalValue.kind, loopBegin, PhiType.Value, graph); phi.addInput(newValue); phi.addInput(originalValue); NodeMap<Value> exitMap = newExitValues.get(originalValue); for (Node exit : peeling.unaffectedExits) { exitMap.set(exit, phi); } } usage.inputs().replace(originalValue, phi); } } } } for (Entry<Node, NodeMap<Value>> entry : newExitValues.entries()) { Value original = (Value) entry.getKey(); NodeMap<Value> pointToValue = entry.getValue(); for (Node exit : exitPoints) { Node valueAtExit = pointToValue.get(exit); if (valueAtExit == null) { pointToValue.set(exit, original); } } } replaceValuesAtLoopExits(newExitValues, loop, exitPoints, peeling.exitFrameStates); } private static void replaceValuesAtLoopExits(final NodeMap<NodeMap<Value>> newExitValues, Loop loop, List<Node> exitPoints, final NodeBitMap exitFrameStates) { Graph graph = loop.loopBegin().graph(); final NodeMap<Node> colors = graph.createNodeMap(); // prepare inital colors for (Node exitPoint : exitPoints) { colors.set(exitPoint, exitPoint); } /*System.out.println("newExitValues"); for (Entry<Node, NodeMap<Value>> entry : newExitValues.entries()) { System.out.println(" - " + entry.getKey() + " :"); for (Entry<Node, Value> entry2 : entry.getValue().entries()) { System.out.println(" + " + entry2.getKey() + " -> " + entry2.getValue()); } }*/ // color GraphUtil.colorCFGDown(colors, new ColoringLambda<Node>() { @Override public Node color(Iterable<Node> incomming, Merge merge) { Node color = null; for (Node c : incomming) { if (c == null) { return null; } if (color == null) { color = c; } else if (color != c) { return merge; } } return color; } @Override public Node danglingColor(Iterable<Node> incomming, Merge merge) { Node color = null; for (Node c : incomming) { if (color == null) { color = c; } else if (color != c) { return merge; } } assert color != null; return color; } }); loop.invalidateCached(); final NodeBitMap inOrBefore = loop.inOrBefore(); GraalCompilation compilation = GraalCompilation.compilation(); if (compilation.compiler.isObserved()) { Map<String, Object> debug = new HashMap<String, Object>(); debug.put("loopExits", colors); debug.put("inOrBefore", inOrBefore); debug.put("inOrAfter", loop.inOrAfter()); debug.put("nodes", loop.nodes()); debug.put("exitFrameStates", exitFrameStates); compilation.compiler.fireCompilationEvent(new CompilationEvent(compilation, "After coloring", graph, true, false, debug)); } GraphUtil.splitFromColoring(colors, new ColorSplitingLambda<Node>(){ @Override public void fixSplit(Node oldNode, Node newNode, Node color) { assert color != null; this.fixNode(newNode, color); } private Value getValueAt(Node point, NodeMap<Value> valueMap, CiKind kind) { Value value = valueMap.get(point); if (value != null) { //System.out.println("getValueAt(" + point + ", valueMap, kind) = (cached) " + value); return value; } Merge merge = (Merge) point; ArrayList<Value> values = new ArrayList<Value>(merge.phiPredecessorCount()); Value v = null; boolean createPhi = false; for (EndNode end : merge.cfgPredecessors()) { Value valueAt = getValueAt(colors.get(end), valueMap, kind); if (v == null) { v = valueAt; } else if (v != valueAt) { createPhi = true; } values.add(valueAt); } if (createPhi) { Phi phi = new Phi(kind, merge, PhiType.Value, merge.graph()); valueMap.set(point, phi); for (EndNode end : merge.cfgPredecessors()) { phi.addInput(getValueAt(colors.get(end), valueMap, kind)); } //System.out.println("getValueAt(" + point + ", valueMap, kind) = (new-phi) " + phi); return phi; } else { assert v != null; valueMap.set(point, v); //System.out.println("getValueAt(" + point + ", valueMap, kind) = (unique) " + v); return v; } } @Override public boolean explore(Node n) { /*System.out.println("Explore " + n + "?"); System.out.println(" - exitFS : " + (!exitFrameStates.isNew(n) && exitFrameStates.isMarked(n))); System.out.println(" - !inOrBefore : " + (!inOrBefore.isNew(n) && !inOrBefore.isMarked(n))); System.out.println(" - inputs > 0 : " + (n.inputs().size() > 0)); System.out.println(" - !danglingMergeFrameState : " + (!danglingMergeFrameState(n)));*/ return exitFrameStates.isNotNewMarked(n) || (inOrBefore.isNotNewNotMarked(n) && n.inputs().size() > 0 && !afterColoringFramestate(n)); //TODO (gd) hum } public boolean afterColoringFramestate(Node n) { if (!(n instanceof FrameState)) { return false; } final FrameState frameState = (FrameState) n; Merge block = frameState.block(); if (block != null) { return colors.get(block.next()) == null; } StateSplit stateSplit = frameState.stateSplit(); if (stateSplit != null) { return colors.get(stateSplit) == null; } for (FrameState inner : frameState.innerFrameStates()) { if (!afterColoringFramestate(inner)) { return false; } } return true; } @Override public void fixNode(Node node, Node color) { //System.out.println("fixNode(" + node + ", " + color + ")"); if (color == null) { // 'white' it out : make non-explorable if (!exitFrameStates.isNew(node)) { exitFrameStates.clear(node); } inOrBefore.mark(node); } else { for (int i = 0; i < node.inputs().size(); i++) { Node input = node.inputs().get(i); if (input == null || newExitValues.isNew(input)) { continue; } NodeMap<Value> valueMap = newExitValues.get(input); if (valueMap != null) { Value replacement = getValueAt(color, valueMap, ((Value) input).kind); node.inputs().set(i, replacement); } } } } @Override public Value fixPhiInput(Value input, Node color) { if (newExitValues.isNew(input)) { return input; } NodeMap<Value> valueMap = newExitValues.get(input); if (valueMap != null) { return getValueAt(color, valueMap, input.kind); } return input; } @Override public List<Node> parentColors(Node color) { if (!(color instanceof Merge)) { return Collections.emptyList(); } Merge merge = (Merge) color; List<Node> parentColors = new ArrayList<Node>(merge.phiPredecessorCount()); for (Node pred : merge.phiPredecessors()) { parentColors.add(colors.get(pred)); } return parentColors; } @Override public Merge merge(Node color) { return (Merge) color; } }); /*if (compilation.compiler.isObserved()) { compilation.compiler.fireCompilationEvent(new CompilationEvent(compilation, "After split from colors", graph, true, false)); }*/ } private static PeelingResult preparePeeling(Loop loop, FixedNode from) { LoopBegin loopBegin = loop.loopBegin(); Graph graph = loopBegin.graph(); NodeBitMap marked = computeLoopNodesFrom(loop, from); GraalCompilation compilation = GraalCompilation.compilation(); /*if (compilation.compiler.isObserved()) { Map<String, Object> debug = new HashMap<String, Object>(); debug.put("marked", marked); compilation.compiler.fireCompilationEvent(new CompilationEvent(compilation, "After computeLoopNodesFrom", loopBegin.graph(), true, false, debug)); }*/ if (from == loopBegin.loopEnd()) { clearWithState(from, marked); } clearWithState(loopBegin, marked); Map<Node, Node> replacements = new HashMap<Node, Node>(); NodeMap<Placeholder> phis = graph.createNodeMap(); NodeMap<StateSplit> exits = graph.createNodeMap(); NodeBitMap unaffectedExits = graph.createNodeBitMap(); NodeBitMap clonedExits = graph.createNodeBitMap(); NodeBitMap exitFrameStates = graph.createNodeBitMap(); for (Node exit : loop.exits()) { if (marked.isMarked(exit.singlePredecessor())) { StateSplit pExit = findNearestMergableExitPoint((FixedNode) exit, marked); markWithState(pExit, marked); clonedExits.mark(pExit); FrameState stateAfter = pExit.stateAfter(); while (stateAfter != null) { exitFrameStates.mark(stateAfter); stateAfter = stateAfter.outerFrameState(); } } else { unaffectedExits.mark(exit); } } NodeBitMap dataOut = graph.createNodeBitMap(); for (Node n : marked) { if (!(n instanceof FrameState)) { for (Node usage : n.dataUsages()) { if ((!marked.isMarked(usage) && !((usage instanceof Phi) && ((Phi) usage).merge() != loopBegin)) || (marked.isMarked(usage) && exitFrameStates.isMarked(usage))) { dataOut.mark(n); break; } } } } for (Node n : marked) { if (n instanceof Phi && ((Phi) n).merge() == loopBegin) { Placeholder p = new Placeholder(graph); replacements.put(n, p); phis.set(n, p); marked.clear(n); } for (Node input : n.dataInputs()) { if (!marked.isMarked(input) && (!(input instanceof Phi) || ((Phi) input).merge() != loopBegin) && replacements.get(input) == null) { replacements.put(input, input); } } } //GraalCompilation compilation = GraalCompilation.compilation(); if (compilation.compiler.isObserved()) { Map<String, Object> debug = new HashMap<String, Object>(); debug.put("marked", marked); compilation.compiler.fireCompilationEvent(new CompilationEvent(compilation, "Before addDuplicate loop#" + loopBegin.id(), loopBegin.graph(), true, false, debug)); } Map<Node, Node> duplicates = graph.addDuplicate(marked, replacements); /*System.out.println("Dup mapping :"); for (Entry<Node, Node> entry : duplicates.entrySet()) { System.out.println(" - " + entry.getKey().id() + " -> " + entry.getValue().id()); }*/ NodeMap<Node> dataOutMapping = graph.createNodeMap(); for (Node n : dataOut) { Node newOut = duplicates.get(n); if (newOut == null) { newOut = replacements.get(n); } assert newOut != null; dataOutMapping.set(n, newOut); } for (Node n : clonedExits) { exits.set(n, (StateSplit) duplicates.get(n)); } NodeMap<Node> phiInits = graph.createNodeMap(); int backIndex = loopBegin.phiPredecessorIndex(loopBegin.loopEnd()); int fowardIndex = loopBegin.phiPredecessorIndex(loopBegin.forwardEdge()); for (Phi phi : loopBegin.phis()) { Value backValue = phi.valueAt(backIndex); if (marked.isMarked(backValue)) { phiInits.set(phi, duplicates.get(backValue)); } else if (from == loopBegin.loopEnd()) { if (backValue instanceof Phi && ((Phi) backValue).merge() == loopBegin) { Phi backPhi = (Phi) backValue; phiInits.set(phi, backPhi.valueAt(fowardIndex)); } else { phiInits.set(phi, backValue); } } } FixedNode newBegin = (FixedNode) duplicates.get(loopBegin.next()); FixedNode newFrom = (FixedNode) duplicates.get(from == loopBegin.loopEnd() ? from.singlePredecessor() : from); return new PeelingResult(newBegin, newFrom, exits, phis, phiInits, dataOutMapping, unaffectedExits, exitFrameStates, marked); } private static StateSplit findNearestMergableExitPoint(FixedNode exit, NodeBitMap marked) { LinkedList<FixedNode> branches = new LinkedList<FixedNode>(); branches.add(exit); while (true) { if (branches.size() == 1) { final FixedNode fixedNode = branches.get(0); if (fixedNode instanceof StateSplit && ((StateSplit) fixedNode).stateAfter() != null) { return (StateSplit) fixedNode; } } else { FixedNode current = branches.poll(); // TODO (gd) find appropriate point : will be useful if a loop exit goes "up" as a result of making a branch dead in the loop body } } } private static NodeBitMap inOrAfter(Loop loop) { return inOrAfter(loop, loop.cfgNodes()); } private static NodeBitMap inOrAfter(Loop loop, NodeBitMap cfgNodes) { return inOrAfter(loop, cfgNodes, true); } private static NodeBitMap inOrAfter(Loop loop, NodeBitMap cfgNodes, boolean full) { Graph graph = loop.loopBegin().graph(); NodeBitMap inOrAfter = graph.createNodeBitMap(); NodeFlood work = graph.createNodeFlood(); work.addAll(cfgNodes); for (Node n : work) { //inOrAfter.mark(n); markWithState(n, inOrAfter); if (full) { for (Node sux : n.successors()) { if (sux != null) { work.add(sux); } } } for (Node usage : n.usages()) { if (usage instanceof Phi) { // filter out data graph cycles Phi phi = (Phi) usage; Merge merge = phi.merge(); if (merge instanceof LoopBegin) { LoopBegin phiLoop = (LoopBegin) merge; int backIndex = phiLoop.phiPredecessorIndex(phiLoop.loopEnd()); if (phi.valueAt(backIndex) == n) { continue; } } } work.add(usage); } } return inOrAfter; } private static NodeBitMap inOrBefore(Loop loop) { return inOrBefore(loop, inOrAfter(loop)); } private static NodeBitMap inOrBefore(Loop loop, NodeBitMap inOrAfter) { return inOrBefore(loop, inOrAfter, loop.cfgNodes()); } private static NodeBitMap inOrBefore(Loop loop, NodeBitMap inOrAfter, NodeBitMap cfgNodes) { return inOrBefore(loop, inOrAfter, cfgNodes, true); } private static NodeBitMap inOrBefore(Loop loop, NodeBitMap inOrAfter, NodeBitMap cfgNodes, boolean full) { Graph graph = loop.loopBegin().graph(); NodeBitMap inOrBefore = graph.createNodeBitMap(); NodeFlood work = graph.createNodeFlood(); work.addAll(cfgNodes); for (Node n : work) { inOrBefore.mark(n); if (full) { for (Node pred : n.predecessors()) { work.add(pred); } } if (n instanceof Phi) { // filter out data graph cycles Phi phi = (Phi) n; if (phi.type() == PhiType.Value) { int backIndex = -1; Merge merge = phi.merge(); if (merge instanceof LoopBegin && cfgNodes.isNotNewNotMarked(((LoopBegin) merge).loopEnd())) { LoopBegin phiLoop = (LoopBegin) merge; backIndex = phiLoop.phiPredecessorIndex(phiLoop.loopEnd()); } for (int i = 0; i < phi.valueCount(); i++) { if (i != backIndex) { work.add(phi.valueAt(i)); } } } } else { for (Node in : n.inputs()) { if (in != null) { work.add(in); } } if (full) { for (Node sux : n.cfgSuccessors()) { // go down into branches that are not 'inOfAfter' if (sux != null && !inOrAfter.isMarked(sux)) { work.add(sux); } } if (n instanceof LoopBegin && n != loop.loopBegin()) { Loop p = loop.parent; boolean isParent = false; while (p != null) { if (p.loopBegin() == n) { isParent = true; break; } p = p.parent; } if (!isParent) { work.add(((LoopBegin) n).loopEnd()); } } } if (cfgNodes.isNotNewMarked(n)) { //add all values from the exits framestates for (Node sux : n.cfgSuccessors()) { if (loop.exits().isNotNewMarked(sux) && sux instanceof StateSplit) { FrameState stateAfter = ((StateSplit) sux).stateAfter(); while (stateAfter != null) { for (Node in : stateAfter.inputs()) { if (!(in instanceof FrameState)) { work.add(in); } } stateAfter = stateAfter.outerFrameState(); } } } } if (n instanceof Merge) { //add phis & counters for (Node usage : n.dataUsages()) { if (!(usage instanceof LoopEnd)) { work.add(usage); } } } if (n instanceof AbstractVectorNode) { for (Node usage : n.usages()) { work.add(usage); } } } } return inOrBefore; } private static void markWithState(Node n, NodeBitMap map) { map.mark(n); if (n instanceof StateSplit) { FrameState stateAfter = ((StateSplit) n).stateAfter(); while (stateAfter != null) { map.mark(stateAfter); stateAfter = stateAfter.outerFrameState(); } } } private static void clearWithState(Node n, NodeBitMap map) { map.clear(n); if (n instanceof StateSplit) { FrameState stateAfter = ((StateSplit) n).stateAfter(); while (stateAfter != null) { map.clear(stateAfter); stateAfter = stateAfter.outerFrameState(); } } } }