# HG changeset patch
# User Thomas Wuerthinger <thomas@wuerthinger.net>
# Date 1309981667 -7200
# Node ID fe8423692c050e4fbde55cc5eb3c3392dcdb1695
# Parent  bb38f184055d0905492f7b0edbbf7f9c94d19fc9# Parent  ad788d3b0dc45c2512e25d55686714ab0cd60bf1
Merge.

diff -r ad788d3b0dc4 -r fe8423692c05 graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/GraalCompilation.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/GraalCompilation.java	Wed Jul 06 17:50:32 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/GraalCompilation.java	Wed Jul 06 21:47:47 2011 +0200
@@ -35,6 +35,7 @@
 import com.oracle.max.graal.compiler.lir.*;
 import com.oracle.max.graal.compiler.observer.*;
 import com.oracle.max.graal.compiler.value.*;
+import com.oracle.max.graal.graph.*;
 import com.sun.cri.ci.*;
 import com.sun.cri.ri.*;
 
@@ -239,8 +240,10 @@
 
             lirGenerator = compiler.backend.newLIRGenerator(this);
 
-            for (LIRBlock begin : hir.linearScanOrder()) {
-                lirGenerator.doBlock(begin);
+            BitMap blockVisited = new BitMap(hir.linearScanOrder().size());
+            for (LIRBlock b : hir.linearScanOrder()) {
+                lirGenerator.doBlock(b);
+//                iterateBlocks(b, blockVisited);
             }
 
             if (GraalOptions.Time) {
@@ -255,10 +258,28 @@
         }
     }
 
+    private void iterateBlocks(LIRBlock b, BitMap blockVisited) {
+        if (blockVisited.get(b.blockID())) {
+            return;
+        }
+//        TTY.println("visit B" + b.blockID() + "(" + b.isLinearScanLoopHeader() + ")");
+//        TTY.println("predecessors: " + b.blockPredecessors());
+        blockVisited.set(b.blockID());
+        if (!b.isLinearScanLoopHeader()) {
+            for (LIRBlock pred : b.blockPredecessors()) {
+//                TTY.println("iterate " + pred + " " + blockVisited.get(pred.blockID()));
+                iterateBlocks(pred, blockVisited);
+            }
+        } else {
+            iterateBlocks(b.blockPredecessors().get(0), blockVisited);
+        }
+        lirGenerator.doBlock(b);
+    }
+
     private CiTargetMethod emitCode() {
         if (GraalOptions.GenLIR && GraalOptions.GenCode) {
             final LIRAssembler lirAssembler = compiler.backend.newLIRAssembler(this);
-            lirAssembler.emitCode(hir.linearScanOrder());
+            lirAssembler.emitCode(hir.codeEmittingOrder());
 
             // generate code for slow cases
             lirAssembler.emitLocalStubs();
diff -r ad788d3b0dc4 -r fe8423692c05 graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/GraalOptions.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/GraalOptions.java	Wed Jul 06 17:50:32 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/GraalOptions.java	Wed Jul 06 21:47:47 2011 +0200
@@ -129,6 +129,7 @@
     public static boolean UseBranchPrediction                = true;
     public static boolean UseExceptionProbability            = ____;
     public static int     MatureInvocationCount              = 100;
+    public static boolean GenSafepoints                      = true;
 
     public static boolean UseConstDirectCall                 = ____;
 
@@ -158,5 +159,6 @@
     public static boolean OptCanonicalizer                   = true;
     public static boolean OptLoops                           = ____;
     public static boolean OptOptimisticSchedule              = ____;
+    public static boolean OptReorderLoops                    = ____;
     public static boolean LoopPeeling                        = ____;
 }
diff -r ad788d3b0dc4 -r fe8423692c05 graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/alloc/ControlFlowOptimizer.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/alloc/ControlFlowOptimizer.java	Wed Jul 06 17:50:32 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/alloc/ControlFlowOptimizer.java	Wed Jul 06 21:47:47 2011 +0200
@@ -25,6 +25,7 @@
 import java.util.*;
 
 import com.oracle.max.graal.compiler.*;
+import com.oracle.max.graal.compiler.debug.*;
 import com.oracle.max.graal.compiler.graph.*;
 import com.oracle.max.graal.compiler.ir.*;
 import com.oracle.max.graal.compiler.lir.*;
@@ -42,7 +43,7 @@
      */
     public static void optimize(IR ir) {
         ControlFlowOptimizer optimizer = new ControlFlowOptimizer(ir);
-        List<LIRBlock> code = ir.linearScanOrder();
+        List<LIRBlock> code = ir.codeEmittingOrder();
         optimizer.reorderShortLoops(code);
         optimizer.deleteEmptyBlocks(code);
         optimizer.deleteUnnecessaryJumps(code);
diff -r ad788d3b0dc4 -r fe8423692c05 graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/alloc/LinearScan.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/alloc/LinearScan.java	Wed Jul 06 17:50:32 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/alloc/LinearScan.java	Wed Jul 06 21:47:47 2011 +0200
@@ -636,6 +636,7 @@
                 // Add uses of live locals from interpreter's point of view for proper debug information generation
                 LIRDebugInfo info = op.info;
                 if (info != null) {
+                    assert info.state != null;
                     info.state.forEachLiveStateValue(new ValueProcedure() {
                         public void doValue(Value value) {
                             CiValue operand = value.operand();
diff -r ad788d3b0dc4 -r fe8423692c05 graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/gen/LIRGenerator.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/gen/LIRGenerator.java	Wed Jul 06 17:50:32 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/gen/LIRGenerator.java	Wed Jul 06 21:47:47 2011 +0200
@@ -241,11 +241,35 @@
             TTY.println("BEGIN Generating LIR for block B" + block.blockID());
         }
 
-        if (block.blockPredecessors().size() > 1) {
+        if (block == ir.startBlock) {
+            XirSnippet prologue = xir.genPrologue(null, compilation.method);
+            if (prologue != null) {
+                emitXir(prologue, null, null, null, false);
+            }
+            FrameState fs = setOperandsForLocals();
+            if (GraalOptions.TraceLIRGeneratorLevel >= 2) {
+                TTY.println("STATE CHANGE (setOperandsForLocals)");
+                if (GraalOptions.TraceLIRGeneratorLevel >= 3) {
+                    TTY.println(fs.toString());
+                }
+            }
+            lastState = fs;
+        } else if (block.blockPredecessors().size() > 1) {
             if (GraalOptions.TraceLIRGeneratorLevel >= 2) {
                 TTY.println("STATE RESET");
             }
             lastState = null;
+        }  else if (block.blockPredecessors().size() == 1) {
+            LIRBlock pred = block.blockPredecessors().get(0);
+            FrameState fs = pred.lastState();
+            assert fs != null : "block B" + block.blockID() + " pred block B" + pred.blockID();
+            if (GraalOptions.TraceLIRGeneratorLevel >= 2) {
+                TTY.println("STATE CHANGE (singlePred)");
+                if (GraalOptions.TraceLIRGeneratorLevel >= 3) {
+                    TTY.println(fs.toString());
+                }
+            }
+            lastState = fs;
         }
 
         for (Node instr : block.getInstructions()) {
@@ -444,13 +468,13 @@
     public void visitExceptionObject(ExceptionObject x) {
         XirSnippet snippet = xir.genExceptionObject(site(x));
         emitXir(snippet, x, null, null, true);
-        lastState = lastState.duplicateWithException(lastState.bci, x);
-        if (GraalOptions.TraceLIRGeneratorLevel >= 2) {
-            TTY.println("STATE CHANGE (visitExceptionObject)");
-            if (GraalOptions.TraceLIRGeneratorLevel >= 3) {
-                TTY.println(lastState.toString());
-            }
-        }
+//        lastState = lastState.duplicateWithException(lastState.bci, x);
+//        if (GraalOptions.TraceLIRGeneratorLevel >= 2) {
+//            TTY.println("STATE CHANGE (visitExceptionObject)");
+//            if (GraalOptions.TraceLIRGeneratorLevel >= 3) {
+//                TTY.println(lastState.toString());
+//            }
+//        }
     }
 
     @Override
@@ -1080,30 +1104,6 @@
         block.setLir(lir);
 
         lir.branchDestination(block.label());
-        if (block == ir.startBlock) {
-            XirSnippet prologue = xir.genPrologue(null, compilation.method);
-            if (prologue != null) {
-                emitXir(prologue, null, null, null, false);
-            }
-            FrameState fs = setOperandsForLocals();
-            if (GraalOptions.TraceLIRGeneratorLevel >= 2) {
-                TTY.println("STATE CHANGE (setOperandsForLocals)");
-                if (GraalOptions.TraceLIRGeneratorLevel >= 3) {
-                    TTY.println(fs.toString());
-                }
-            }
-            lastState = fs;
-        } else if (block.blockPredecessors().size() == 1) {
-            FrameState fs = block.blockPredecessors().get(0).lastState();
-            //assert fs != null;
-            if (GraalOptions.TraceLIRGeneratorLevel >= 2) {
-                TTY.println("STATE CHANGE (singlePred)");
-                if (GraalOptions.TraceLIRGeneratorLevel >= 3) {
-                    TTY.println(fs.toString());
-                }
-            }
-            lastState = fs;
-        }
     }
 
     /**
@@ -1505,6 +1505,9 @@
     public void visitLoopEnd(LoopEnd x) {
         setNoResult(x);
         moveToPhi(x.loopBegin(), x);
+        if (GraalOptions.GenSafepoints) {
+            xir.genSafepoint(site(x));
+        }
         lir.jump(getLIRBlock(x.loopBegin()));
     }
 
diff -r ad788d3b0dc4 -r fe8423692c05 graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/graph/BlockMap.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/graph/BlockMap.java	Wed Jul 06 17:50:32 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/graph/BlockMap.java	Wed Jul 06 21:47:47 2011 +0200
@@ -523,7 +523,7 @@
             }
 
             assert block.loops == 0;
-            block.loops = 1 << nextLoop;
+            block.loops = (long) 1 << (long) nextLoop;
             nextLoop++;
         }
         assert Long.bitCount(block.loops) == 1;
diff -r ad788d3b0dc4 -r fe8423692c05 graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/graph/IR.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/graph/IR.java	Wed Jul 06 17:50:32 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/graph/IR.java	Wed Jul 06 21:47:47 2011 +0200
@@ -52,7 +52,12 @@
     /**
      * The linear-scan ordered list of blocks.
      */
-    private List<LIRBlock> orderedBlocks;
+    private List<LIRBlock> linearScanOrder;
+
+    /**
+     * The order in which the code is emitted.
+     */
+    private List<LIRBlock> codeEmittingOrder;
 
     /**
      * Creates a new IR instance for the specified compilation.
@@ -138,6 +143,14 @@
             block.setLoopDepth(b.loopDepth());
             block.setLoopIndex(b.loopIndex());
 
+            if (b.isLoopEnd()) {
+                block.setLinearScanLoopEnd();
+            }
+
+            if (b.isLoopHeader()) {
+                block.setLinearScanLoopHeader();
+            }
+
             block.setFirstInstruction(b.firstNode());
             block.setLastInstruction(b.lastNode());
             lirBlocks.add(block);
@@ -153,9 +166,8 @@
             }
         }
 
-        orderedBlocks = lirBlocks;
         valueToBlock = new HashMap<Node, LIRBlock>();
-        for (LIRBlock b : orderedBlocks) {
+        for (LIRBlock b : lirBlocks) {
             for (Node i : b.getInstructions()) {
                 valueToBlock.put(i, b);
             }
@@ -169,11 +181,12 @@
             GraalTimers.COMPUTE_LINEAR_SCAN_ORDER.start();
         }
 
-        ComputeLinearScanOrder clso = new ComputeLinearScanOrder(lirBlocks.size(), startBlock);
-        orderedBlocks = clso.linearScanOrder();
+        ComputeLinearScanOrder clso = new ComputeLinearScanOrder(lirBlocks.size(), compilation.stats.loopCount, startBlock);
+        linearScanOrder = clso.linearScanOrder();
+        codeEmittingOrder = clso.codeEmittingOrder();
 
         int z = 0;
-        for (LIRBlock b : orderedBlocks) {
+        for (LIRBlock b : linearScanOrder) {
             b.setLinearScanNumber(z++);
         }
 
@@ -190,7 +203,11 @@
      * @return the blocks in linear scan order
      */
     public List<LIRBlock> linearScanOrder() {
-        return orderedBlocks;
+        return linearScanOrder;
+    }
+
+    public List<LIRBlock> codeEmittingOrder() {
+        return codeEmittingOrder;
     }
 
     public void printGraph(String phase, Graph graph) {
diff -r ad788d3b0dc4 -r fe8423692c05 graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/ir/ComputeLinearScanOrder.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/ir/ComputeLinearScanOrder.java	Wed Jul 06 17:50:32 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/ir/ComputeLinearScanOrder.java	Wed Jul 06 21:47:47 2011 +0200
@@ -36,12 +36,14 @@
     private int numBlocks; // total number of blocks (smaller than maxBlockId)
 
     List<LIRBlock> linearScanOrder; // the resulting list of blocks in correct order
+    List<LIRBlock> codeEmittingOrder;
 
     final BitMap visitedBlocks; // used for recursive processing of blocks
     final BitMap activeBlocks; // used for recursive processing of blocks
     final BitMap dominatorBlocks; // temporary BitMap used for computation of dominator
     final int[] forwardBranches; // number of incoming forward branches for each block
     final List<LIRBlock> workList; // temporary list (used in markLoops and computeOrder)
+    final LIRBlock[] loopHeaders;
 
     // accessors for visitedBlocks and activeBlocks
     void initVisited() {
@@ -86,7 +88,8 @@
         return linearScanOrder;
     }
 
-    public ComputeLinearScanOrder(int maxBlockId, LIRBlock startBlock) {
+    public ComputeLinearScanOrder(int maxBlockId, int loopCount, LIRBlock startBlock) {
+        loopHeaders = new LIRBlock[loopCount];
 
         this.maxBlockId = maxBlockId;
         visitedBlocks = new BitMap(maxBlockId);
@@ -257,6 +260,24 @@
         // be equal.
         cur.setLinearScanNumber(linearScanOrder.size());
         linearScanOrder.add(cur);
+
+        if (!cur.isLinearScanLoopHeader() || !GraalOptions.OptReorderLoops) {
+            codeEmittingOrder.add(cur);
+
+            if (cur.isLinearScanLoopEnd() && GraalOptions.OptReorderLoops) {
+                LIRBlock loopHeader = loopHeaders[cur.loopIndex()];
+                assert loopHeader != null;
+                codeEmittingOrder.add(loopHeader);
+
+                for (LIRBlock succ : loopHeader.blockSuccessors()) {
+                    if (succ.loopDepth() == loopHeader.loopDepth()) {
+                        succ.setAlign(true);
+                    }
+                }
+            }
+        } else {
+            loopHeaders[cur.loopIndex()] = cur;
+        }
     }
 
     private void computeOrder(LIRBlock startBlock) {
@@ -267,6 +288,8 @@
         // the start block is always the first block in the linear scan order
         linearScanOrder = new ArrayList<LIRBlock>(numBlocks);
 
+        codeEmittingOrder = new ArrayList<LIRBlock>(numBlocks);
+
         // start processing with standard entry block
         assert workList.isEmpty() : "list must be empty before processing";
 
@@ -327,4 +350,8 @@
             }
         }
     }
+
+    public List<LIRBlock> codeEmittingOrder() {
+        return codeEmittingOrder;
+    }
 }
diff -r ad788d3b0dc4 -r fe8423692c05 graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/ir/CreateVectorNode.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/ir/CreateVectorNode.java	Wed Jul 06 17:50:32 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/ir/CreateVectorNode.java	Wed Jul 06 21:47:47 2011 +0200
@@ -113,6 +113,7 @@
 
         LoopEnd loopEnd = new LoopEnd(graph());
         loopEnd.setLoopBegin(loopBegin);
+        loopBegin.setStateAfter(stateAfter());
         Compare condition;
         if (reversed) {
             condition = new Compare(loopVariable, Condition.GE, Constant.forInt(0, graph()), graph());
diff -r ad788d3b0dc4 -r fe8423692c05 graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/ir/Invoke.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/ir/Invoke.java	Wed Jul 06 17:50:32 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/ir/Invoke.java	Wed Jul 06 21:47:47 2011 +0200
@@ -41,11 +41,21 @@
     private static final int SUCCESSOR_COUNT = 1;
     private static final int SUCCESSOR_EXCEPTION_EDGE = 0;
 
+    private boolean canInline = true;
+
     @Override
     protected int inputCount() {
         return super.inputCount() + argumentCount;
     }
 
+    public boolean canInline() {
+        return canInline;
+    }
+
+    public void setCanInline(boolean b) {
+        canInline = b;
+    }
+
     @Override
     protected int successorCount() {
         return super.successorCount() + SUCCESSOR_COUNT;
@@ -204,6 +214,7 @@
     @Override
     public Node copy(Graph into) {
         Invoke x = new Invoke(bci, opcode, kind, new Value[argumentCount], target, returnType, into);
+        x.setCanInline(canInline);
         return x;
     }
 }
diff -r ad788d3b0dc4 -r fe8423692c05 graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/lir/LIRAssembler.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/lir/LIRAssembler.java	Wed Jul 06 17:50:32 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/lir/LIRAssembler.java	Wed Jul 06 21:47:47 2011 +0200
@@ -30,12 +30,12 @@
 import com.oracle.max.graal.compiler.debug.*;
 import com.oracle.max.graal.compiler.gen.*;
 import com.oracle.max.graal.compiler.ir.*;
-import com.oracle.max.graal.compiler.lir.FrameMap.*;
+import com.oracle.max.graal.compiler.lir.FrameMap.StackBlock;
 import com.oracle.max.graal.compiler.util.*;
 import com.sun.cri.ci.*;
-import com.sun.cri.ci.CiTargetMethod.*;
+import com.sun.cri.ci.CiTargetMethod.Mark;
 import com.sun.cri.ri.*;
-import com.sun.cri.xir.CiXirAssembler.*;
+import com.sun.cri.xir.CiXirAssembler.XirMark;
 
 /**
  * The {@code LIRAssembler} class definition.
@@ -108,6 +108,10 @@
 
     void emitBlock(LIRBlock block) {
 
+        if (block.align()) {
+            emitAlignment();
+        }
+
         block.setBlockEntryPco(codePos());
 
         if (GraalOptions.PrintLIRWithAssembly) {
diff -r ad788d3b0dc4 -r fe8423692c05 graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/lir/LIRBlock.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/lir/LIRBlock.java	Wed Jul 06 17:50:32 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/lir/LIRBlock.java	Wed Jul 06 21:47:47 2011 +0200
@@ -45,6 +45,7 @@
     private List<LIRBlock> predecessors = new ArrayList<LIRBlock>(4);
     private List<LIRBlock> successors = new ArrayList<LIRBlock>(4);
     private LIRDebugInfo debugInfo;
+    private boolean align;
 
     /**
      * Bit map specifying which {@linkplain OperandPool operands} are live upon entry to this block.
@@ -107,6 +108,14 @@
         return firstLirInstructionID;
     }
 
+    public boolean align() {
+        return align;
+    }
+
+    public void setAlign(boolean b) {
+        align = b;
+    }
+
     public void setFirstLirInstructionId(int firstLirInstructionId) {
         this.firstLirInstructionID = firstLirInstructionId;
     }
diff -r ad788d3b0dc4 -r fe8423692c05 graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/phases/InliningPhase.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/phases/InliningPhase.java	Wed Jul 06 17:50:32 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/phases/InliningPhase.java	Wed Jul 06 21:47:47 2011 +0200
@@ -118,7 +118,7 @@
     private RiMethod inlineInvoke(Invoke invoke, int iterations, float ratio) {
         RiMethod parent = invoke.stateAfter().method();
         RiTypeProfile profile = parent.typeProfile(invoke.bci);
-        if (GraalOptions.Intrinsify && compilation.runtime.intrinsicGraph(invoke.target, invoke.arguments()) != null) {
+        if (GraalOptions.Intrinsify && compilation.runtime.intrinsicGraph(parent, invoke.bci, invoke.target, invoke.arguments()) != null) {
             // Always intrinsify.
             return invoke.target;
         }
@@ -209,6 +209,12 @@
     }
 
     private boolean checkInvokeConditions(Invoke invoke) {
+        if (!invoke.canInline()) {
+            if (GraalOptions.TraceInlining) {
+                TTY.println("not inlining %s because the invoke is manually set to be non-inlinable", methodName(invoke.target, invoke));
+            }
+            return false;
+        }
         if (invoke.stateAfter() == null) {
             if (GraalOptions.TraceInlining) {
                 TTY.println("not inlining %s because the invoke has no after state", methodName(invoke.target, invoke));
@@ -317,6 +323,7 @@
     }
 
     private void inlineMethod(Invoke invoke, RiMethod method) {
+        RiMethod parent = invoke.stateAfter().method();
         FrameState stateAfter = invoke.stateAfter();
         FixedNode exceptionEdge = invoke.exceptionEdge();
         if (exceptionEdge instanceof Placeholder) {
@@ -339,7 +346,7 @@
 
         CompilerGraph graph = null;
         if (GraalOptions.Intrinsify) {
-            graph = (CompilerGraph) compilation.runtime.intrinsicGraph(method, invoke.arguments());
+            graph = (CompilerGraph) compilation.runtime.intrinsicGraph(parent, invoke.bci, method, invoke.arguments());
         }
         if (graph != null) {
             if (GraalOptions.TraceInlining) {
@@ -405,7 +412,7 @@
 
         FrameState stateBefore = null;
         for (Node node : duplicates.values()) {
-            if (node instanceof Invoke) {
+            if (node instanceof Invoke && ((Invoke) node).canInline()) {
                 newInvokes.add((Invoke) node);
             } else if (node instanceof FrameState) {
                 FrameState frameState = (FrameState) node;
diff -r ad788d3b0dc4 -r fe8423692c05 graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/schedule/Block.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/schedule/Block.java	Wed Jul 06 17:50:32 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/schedule/Block.java	Wed Jul 06 21:47:47 2011 +0200
@@ -95,14 +95,14 @@
                     firstNode.graph().start().setNext(a);
                     this.anchor = a;
                 }
-            } else if (firstNode instanceof Merge) {
-                Merge merge = (Merge) firstNode;
-                if (merge.next() instanceof Anchor) {
-                    this.anchor = (Anchor) merge.next();
+            } else if (firstNode instanceof Merge || firstNode instanceof ExceptionObject) {
+                FixedNodeWithNext fixedNode = (FixedNodeWithNext) firstNode;
+                if (fixedNode.next() instanceof Anchor) {
+                    this.anchor = (Anchor) fixedNode.next();
                 } else {
                     Anchor a = new Anchor(firstNode.graph());
-                    a.setNext(merge.next());
-                    merge.setNext(a);
+                    a.setNext(fixedNode.next());
+                    fixedNode.setNext(a);
                     this.anchor = a;
                 }
             } else {
diff -r ad788d3b0dc4 -r fe8423692c05 graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/schedule/IdentifyBlocksPhase.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/schedule/IdentifyBlocksPhase.java	Wed Jul 06 17:50:32 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/schedule/IdentifyBlocksPhase.java	Wed Jul 06 21:47:47 2011 +0200
@@ -194,6 +194,7 @@
                 block.addSuccessor(loopBeginBlock);
                 BitMap map = new BitMap(blocks.size());
                 markBlocks(block, loopBeginBlock, map, loopCount++, block.loopDepth());
+                assert loopBeginBlock.loopDepth() == block.loopDepth() && loopBeginBlock.loopIndex() == block.loopIndex();
             }
         }
 
@@ -221,6 +222,10 @@
         for (Block pred : block.getPredecessors()) {
             markBlocks(pred, endBlock, map, loopIndex, initialDepth);
         }
+
+        if (block.isLoopHeader()) {
+            markBlocks(nodeToBlock.get(((LoopBegin) block.firstNode()).loopEnd()), endBlock, map, loopIndex, initialDepth);
+        }
     }
 
     private void computeJavaBlocks() {
diff -r ad788d3b0dc4 -r fe8423692c05 graal/com.oracle.max.graal.runtime/src/com/oracle/max/graal/runtime/HotSpotRuntime.java
--- a/graal/com.oracle.max.graal.runtime/src/com/oracle/max/graal/runtime/HotSpotRuntime.java	Wed Jul 06 17:50:32 2011 +0200
+++ b/graal/com.oracle.max.graal.runtime/src/com/oracle/max/graal/runtime/HotSpotRuntime.java	Wed Jul 06 21:47:47 2011 +0200
@@ -32,6 +32,7 @@
 import com.oracle.max.graal.compiler.value.*;
 import com.oracle.max.graal.graph.*;
 import com.oracle.max.graal.runtime.nodes.*;
+import com.sun.cri.bytecode.*;
 import com.sun.cri.ci.*;
 import com.sun.cri.ci.CiTargetMethod.Call;
 import com.sun.cri.ci.CiTargetMethod.DataPatch;
@@ -307,10 +308,11 @@
             LocationNode arrayLocation = createArrayLocation(graph, elementKind);
             arrayLocation.setIndex(storeIndexed.index());
             Value value = storeIndexed.value();
+            Value array = storeIndexed.array();
             if (elementKind == CiKind.Object && !value.isNullConstant()) {
                 // Store check!
-                if (storeIndexed.array().exactType() != null) {
-                    RiType elementType = storeIndexed.array().exactType().componentType();
+                if (array.exactType() != null) {
+                    RiType elementType = array.exactType().componentType();
                     if (elementType.superType() != null) {
                         Constant type = new Constant(elementType.getEncoding(Representation.ObjectHub), graph);
                         value = new CheckCast(type, value, graph);
@@ -318,17 +320,16 @@
                         assert elementType.name().equals("Ljava/lang/Object;") : elementType.name();
                     }
                 } else {
-                    ReadNode arrayKlass = new ReadNode(CiKind.Object, storeIndexed.array(), LocationNode.create(LocationNode.FINAL_LOCATION, CiKind.Object, config.hubOffset, graph), graph);
-                    ReadNode arrayElementKlass = new ReadNode(CiKind.Object, arrayKlass, LocationNode.create(LocationNode.FINAL_LOCATION, CiKind.Object, config.arrayClassElementOffset, graph), graph);
+                    ReadNode arrayElementKlass = readArrayElementKlass(graph, array);
                     value = new CheckCast(arrayElementKlass, value, graph);
                 }
             }
-            WriteNode memoryWrite = new WriteNode(elementKind.stackKind(), storeIndexed.array(), value, arrayLocation, graph);
+            WriteNode memoryWrite = new WriteNode(elementKind.stackKind(), array, value, arrayLocation, graph);
             memoryWrite.setGuard(boundsCheck);
             memoryWrite.setStateAfter(storeIndexed.stateAfter());
             anchor.setNext(memoryWrite);
             if (elementKind == CiKind.Object && !value.isNullConstant()) {
-                ArrayWriteBarrier writeBarrier = new ArrayWriteBarrier(storeIndexed.array(), arrayLocation, graph);
+                ArrayWriteBarrier writeBarrier = new ArrayWriteBarrier(array, arrayLocation, graph);
                 memoryWrite.setNext(writeBarrier);
                 writeBarrier.setNext(storeIndexed.next());
             } else {
@@ -339,6 +340,12 @@
         }
     }
 
+    private ReadNode readArrayElementKlass(Graph graph, Value array) {
+        ReadNode arrayKlass = readHub(graph, array);
+        ReadNode arrayElementKlass = new ReadNode(CiKind.Object, arrayKlass, LocationNode.create(LocationNode.FINAL_LOCATION, CiKind.Object, config.arrayClassElementOffset, graph), graph);
+        return arrayElementKlass;
+    }
+
     private LocationNode createArrayLocation(Graph graph, CiKind elementKind) {
         return LocationNode.create(LocationNode.getArrayLocation(elementKind), elementKind, config.getArrayOffset(elementKind), graph);
     }
@@ -348,7 +355,7 @@
     }
 
     @Override
-    public Graph intrinsicGraph(RiMethod method, List<? extends Node> parameters) {
+    public Graph intrinsicGraph(RiMethod caller, int bci, RiMethod method, List<? extends Node> parameters) {
         if (!intrinsicGraphs.containsKey(method)) {
             RiType holder = method.holder();
             String fullName = method.name() + method.signature().asString();
@@ -357,7 +364,7 @@
                 if (fullName.equals("getClass()Ljava/lang/Class;")) {
                     CompilerGraph graph = new CompilerGraph(this);
                     Local receiver = new Local(CiKind.Object, 0, graph);
-                    ReadNode klassOop = new ReadNode(CiKind.Object, receiver, LocationNode.create(LocationNode.FINAL_LOCATION, CiKind.Object, config.hubOffset, graph), graph);
+                    ReadNode klassOop = readHub(graph, receiver);
                     Return ret = new Return(new ReadNode(CiKind.Object, klassOop, LocationNode.create(LocationNode.FINAL_LOCATION, CiKind.Object, config.classMirrorOffset, graph), graph), graph);
                     graph.start().setNext(ret);
                     graph.setReturn(ret);
@@ -387,7 +394,6 @@
                     }
 
                     CiKind componentType = src.declaredType().componentType().kind();
-
                     if (componentType == CiKind.Object) {
                         return null;
                     }
@@ -435,7 +441,22 @@
                     merge1.addEnd(new EndNode(graph));
                     merge1.setStateAfter(stateBefore);
 
-                    ifNode.setFalseSuccessor(merge1.endAt(0));
+
+                    Invoke newInvoke = null;
+                    if (componentType == CiKind.Object) {
+                        Value srcClass = readHub(graph, src);
+                        Value destClass = readHub(graph, dest);
+                        If elementClassIf = new If(new Compare(srcClass, Condition.EQ, destClass, graph), graph);
+                        ifNode.setFalseSuccessor(elementClassIf);
+                        newInvoke = new Invoke(bci, Bytecodes.INVOKESTATIC, CiKind.Void, new Value[]{src, srcPos, dest, destPos, length}, method, method.signature().returnType(method.holder()), graph);
+                        newInvoke.setCanInline(false);
+                        newInvoke.setStateAfter(stateAfter);
+                        elementClassIf.setFalseSuccessor(newInvoke);
+                        elementClassIf.setTrueSuccessor(merge1.endAt(0));
+                    } else {
+                        ifNode.setFalseSuccessor(merge1.endAt(0));
+                    }
+
                     secondIf.setFalseSuccessor(merge1.endAt(1));
                     merge1.setNext(normalVector);
 
@@ -447,6 +468,11 @@
                     normalVector.setNext(merge2.endAt(0));
                     reverseVector.setNext(merge2.endAt(1));
 
+                    if (newInvoke != null) {
+                        merge2.addEnd(new EndNode(graph));
+                        newInvoke.setNext(merge2.endAt(2));
+                    }
+
                     Return ret = new Return(null, graph);
                     merge2.setNext(ret);
                     graph.setReturn(ret);
@@ -496,4 +522,8 @@
         }
         return intrinsicGraphs.get(method);
     }
+
+    private ReadNode readHub(Graph graph, Value value) {
+        return new ReadNode(CiKind.Object, value, LocationNode.create(LocationNode.FINAL_LOCATION, CiKind.Object, config.hubOffset, graph), graph);
+    }
 }
diff -r ad788d3b0dc4 -r fe8423692c05 graal/com.oracle.max.graal.runtime/src/com/oracle/max/graal/runtime/HotSpotXirGenerator.java
--- a/graal/com.oracle.max.graal.runtime/src/com/oracle/max/graal/runtime/HotSpotXirGenerator.java	Wed Jul 06 17:50:32 2011 +0200
+++ b/graal/com.oracle.max.graal.runtime/src/com/oracle/max/graal/runtime/HotSpotXirGenerator.java	Wed Jul 06 21:47:47 2011 +0200
@@ -164,8 +164,8 @@
         protected XirTemplate create(CiXirAssembler asm, long flags) {
             asm.restart(CiKind.Void);
 
-            // XirOperand temp = asm.createRegister("temp", CiKind.Word, AMD64.rax);
-            // asm.pload(CiKind.Word, temp, asm.w(config.safepointPollingAddress), true);
+            XirOperand temp = asm.createRegister("temp", CiKind.Word, AMD64.rax);
+            asm.pload(CiKind.Word, temp, asm.w(config.safepointPollingAddress), true);
 
             return asm.finishTemplate("safepoint");
         }
diff -r ad788d3b0dc4 -r fe8423692c05 runscimark.sh
--- a/runscimark.sh	Wed Jul 06 17:50:32 2011 +0200
+++ b/runscimark.sh	Wed Jul 06 21:47:47 2011 +0200
@@ -12,16 +12,7 @@
   exit 1;
 fi
 if [ -z "${SCIMARK}" ]; then
-  echo "SCIMARK is not defined. It must point to a SciMark benchmark jar."
+  echo "SCIMARK is not defined. It must point to a directory with the SciMark benchmark jar."
   exit 1;
 fi
-COUNT=$1
-shift
-if [ -z "${COUNT}" ]; then
-  COUNT=5000
-fi
-for (( i = 1; i <= ${COUNT}; i++ ))      ### Outer for loop ###
-do
-  echo "$i "
-  ${JDK7}/jre/bin/java -client -d64 -graal -esa -ea -Xms32m -Xmx100m -Xbootclasspath/a:${SCIMARK} -G:+Time $@ jnt.scimark2.commandline
-done
+${JDK7}/jre/bin/java -client -d64 -graal -Xms256m -Xmx512m -Xbootclasspath/a:${SCIMARK}/scimark2lib.jar $@ jnt.scimark2.commandline