001/*
002 * Copyright (c) 2009, 2014, Oracle and/or its affiliates. All rights reserved.
003 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
004 *
005 * This code is free software; you can redistribute it and/or modify it
006 * under the terms of the GNU General Public License version 2 only, as
007 * published by the Free Software Foundation.
008 *
009 * This code is distributed in the hope that it will be useful, but WITHOUT
010 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
011 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
012 * version 2 for more details (a copy is included in the LICENSE file that
013 * accompanied this code).
014 *
015 * You should have received a copy of the GNU General Public License version
016 * 2 along with this work; if not, write to the Free Software Foundation,
017 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
018 *
019 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
020 * or visit www.oracle.com if you need additional information or have any
021 * questions.
022 */
023package com.oracle.graal.lir.alloc.lsra;
024
025import static com.oracle.graal.lir.LIRValueUtil.*;
026import static jdk.internal.jvmci.code.CodeUtil.*;
027import static jdk.internal.jvmci.code.ValueUtil.*;
028
029import java.util.*;
030
031import jdk.internal.jvmci.code.*;
032import com.oracle.graal.debug.*;
033import jdk.internal.jvmci.meta.*;
034
035import com.oracle.graal.compiler.common.alloc.RegisterAllocationConfig.AllocatableRegisters;
036import com.oracle.graal.compiler.common.cfg.*;
037import com.oracle.graal.compiler.common.util.*;
038import com.oracle.graal.lir.*;
039import com.oracle.graal.lir.StandardOp.MoveOp;
040import com.oracle.graal.lir.alloc.lsra.Interval.RegisterBinding;
041import com.oracle.graal.lir.alloc.lsra.Interval.RegisterPriority;
042import com.oracle.graal.lir.alloc.lsra.Interval.SpillState;
043import com.oracle.graal.lir.alloc.lsra.Interval.State;
044
045/**
046 */
047class LinearScanWalker extends IntervalWalker {
048
049    protected Register[] availableRegs;
050
051    protected final int[] usePos;
052    protected final int[] blockPos;
053
054    protected List<Interval>[] spillIntervals;
055
056    private MoveResolver moveResolver; // for ordering spill moves
057
058    private int minReg;
059
060    private int maxReg;
061
062    /**
063     * Only 10% of the lists in {@link #spillIntervals} are actually used. But when they are used,
064     * they can grow quite long. The maximum length observed was 45 (all numbers taken from a
065     * bootstrap run of Graal). Therefore, we initialize {@link #spillIntervals} with this marker
066     * value, and allocate a "real" list only on demand in {@link #setUsePos}.
067     */
068    private static final List<Interval> EMPTY_LIST = new ArrayList<>(0);
069
070    // accessors mapped to same functions in class LinearScan
071    int blockCount() {
072        return allocator.blockCount();
073    }
074
075    AbstractBlockBase<?> blockAt(int idx) {
076        return allocator.blockAt(idx);
077    }
078
079    AbstractBlockBase<?> blockOfOpWithId(int opId) {
080        return allocator.blockForId(opId);
081    }
082
083    LinearScanWalker(LinearScan allocator, Interval unhandledFixedFirst, Interval unhandledAnyFirst) {
084        super(allocator, unhandledFixedFirst, unhandledAnyFirst);
085
086        moveResolver = allocator.createMoveResolver();
087        spillIntervals = Util.uncheckedCast(new List[allocator.getRegisters().length]);
088        for (int i = 0; i < allocator.getRegisters().length; i++) {
089            spillIntervals[i] = EMPTY_LIST;
090        }
091        usePos = new int[allocator.getRegisters().length];
092        blockPos = new int[allocator.getRegisters().length];
093    }
094
095    void initUseLists(boolean onlyProcessUsePos) {
096        for (Register register : availableRegs) {
097            int i = register.number;
098            usePos[i] = Integer.MAX_VALUE;
099
100            if (!onlyProcessUsePos) {
101                blockPos[i] = Integer.MAX_VALUE;
102                spillIntervals[i].clear();
103            }
104        }
105    }
106
107    int maxRegisterNumber() {
108        return maxReg;
109    }
110
111    int minRegisterNumber() {
112        return minReg;
113    }
114
115    boolean isRegisterInRange(int reg) {
116        return reg >= minRegisterNumber() && reg <= maxRegisterNumber();
117    }
118
119    void excludeFromUse(Interval i) {
120        Value location = i.location();
121        int i1 = asRegister(location).number;
122        if (isRegisterInRange(i1)) {
123            usePos[i1] = 0;
124        }
125    }
126
127    void setUsePos(Interval interval, int usePos, boolean onlyProcessUsePos) {
128        if (usePos != -1) {
129            assert usePos != 0 : "must use excludeFromUse to set usePos to 0";
130            int i = asRegister(interval.location()).number;
131            if (isRegisterInRange(i)) {
132                if (this.usePos[i] > usePos) {
133                    this.usePos[i] = usePos;
134                }
135                if (!onlyProcessUsePos) {
136                    List<Interval> list = spillIntervals[i];
137                    if (list == EMPTY_LIST) {
138                        list = new ArrayList<>(2);
139                        spillIntervals[i] = list;
140                    }
141                    list.add(interval);
142                }
143            }
144        }
145    }
146
147    void setBlockPos(Interval i, int blockPos) {
148        if (blockPos != -1) {
149            int reg = asRegister(i.location()).number;
150            if (isRegisterInRange(reg)) {
151                if (this.blockPos[reg] > blockPos) {
152                    this.blockPos[reg] = blockPos;
153                }
154                if (usePos[reg] > blockPos) {
155                    usePos[reg] = blockPos;
156                }
157            }
158        }
159    }
160
161    void freeExcludeActiveFixed() {
162        Interval interval = activeLists.get(RegisterBinding.Fixed);
163        while (interval != Interval.EndMarker) {
164            assert isRegister(interval.location()) : "active interval must have a register assigned";
165            excludeFromUse(interval);
166            interval = interval.next;
167        }
168    }
169
170    void freeExcludeActiveAny() {
171        Interval interval = activeLists.get(RegisterBinding.Any);
172        while (interval != Interval.EndMarker) {
173            assert isRegister(interval.location()) : "active interval must have a register assigned";
174            excludeFromUse(interval);
175            interval = interval.next;
176        }
177    }
178
179    void freeCollectInactiveFixed(Interval current) {
180        Interval interval = inactiveLists.get(RegisterBinding.Fixed);
181        while (interval != Interval.EndMarker) {
182            if (current.to() <= interval.currentFrom()) {
183                assert interval.currentIntersectsAt(current) == -1 : "must not intersect";
184                setUsePos(interval, interval.currentFrom(), true);
185            } else {
186                setUsePos(interval, interval.currentIntersectsAt(current), true);
187            }
188            interval = interval.next;
189        }
190    }
191
192    void freeCollectInactiveAny(Interval current) {
193        Interval interval = inactiveLists.get(RegisterBinding.Any);
194        while (interval != Interval.EndMarker) {
195            setUsePos(interval, interval.currentIntersectsAt(current), true);
196            interval = interval.next;
197        }
198    }
199
200    void freeCollectUnhandled(RegisterBinding kind, Interval current) {
201        Interval interval = unhandledLists.get(kind);
202        while (interval != Interval.EndMarker) {
203            setUsePos(interval, interval.intersectsAt(current), true);
204            if (kind == RegisterBinding.Fixed && current.to() <= interval.from()) {
205                setUsePos(interval, interval.from(), true);
206            }
207            interval = interval.next;
208        }
209    }
210
211    void spillExcludeActiveFixed() {
212        Interval interval = activeLists.get(RegisterBinding.Fixed);
213        while (interval != Interval.EndMarker) {
214            excludeFromUse(interval);
215            interval = interval.next;
216        }
217    }
218
219    void spillBlockUnhandledFixed(Interval current) {
220        Interval interval = unhandledLists.get(RegisterBinding.Fixed);
221        while (interval != Interval.EndMarker) {
222            setBlockPos(interval, interval.intersectsAt(current));
223            interval = interval.next;
224        }
225    }
226
227    void spillBlockInactiveFixed(Interval current) {
228        Interval interval = inactiveLists.get(RegisterBinding.Fixed);
229        while (interval != Interval.EndMarker) {
230            if (current.to() > interval.currentFrom()) {
231                setBlockPos(interval, interval.currentIntersectsAt(current));
232            } else {
233                assert interval.currentIntersectsAt(current) == -1 : "invalid optimization: intervals intersect";
234            }
235
236            interval = interval.next;
237        }
238    }
239
240    void spillCollectActiveAny(RegisterPriority registerPriority) {
241        Interval interval = activeLists.get(RegisterBinding.Any);
242        while (interval != Interval.EndMarker) {
243            setUsePos(interval, Math.min(interval.nextUsage(registerPriority, currentPosition), interval.to()), false);
244            interval = interval.next;
245        }
246    }
247
248    void spillCollectInactiveAny(Interval current) {
249        Interval interval = inactiveLists.get(RegisterBinding.Any);
250        while (interval != Interval.EndMarker) {
251            if (interval.currentIntersects(current)) {
252                setUsePos(interval, Math.min(interval.nextUsage(RegisterPriority.LiveAtLoopEnd, currentPosition), interval.to()), false);
253            }
254            interval = interval.next;
255        }
256    }
257
258    void insertMove(int operandId, Interval srcIt, Interval dstIt) {
259        // output all moves here. When source and target are equal, the move is
260        // optimized away later in assignRegNums
261
262        int opId = (operandId + 1) & ~1;
263        AbstractBlockBase<?> opBlock = allocator.blockForId(opId);
264        assert opId > 0 && allocator.blockForId(opId - 2) == opBlock : "cannot insert move at block boundary";
265
266        // calculate index of instruction inside instruction list of current block
267        // the minimal index (for a block with no spill moves) can be calculated because the
268        // numbering of instructions is known.
269        // When the block already contains spill moves, the index must be increased until the
270        // correct index is reached.
271        List<LIRInstruction> instructions = allocator.getLIR().getLIRforBlock(opBlock);
272        int index = (opId - instructions.get(0).id()) >> 1;
273        assert instructions.get(index).id() <= opId : "error in calculation";
274
275        while (instructions.get(index).id() != opId) {
276            index++;
277            assert 0 <= index && index < instructions.size() : "index out of bounds";
278        }
279        assert 1 <= index && index < instructions.size() : "index out of bounds";
280        assert instructions.get(index).id() == opId : "error in calculation";
281
282        // insert new instruction before instruction at position index
283        moveResolver.moveInsertPosition(instructions, index);
284        moveResolver.addMapping(srcIt, dstIt);
285    }
286
287    int findOptimalSplitPos(AbstractBlockBase<?> minBlock, AbstractBlockBase<?> maxBlock, int maxSplitPos) {
288        int fromBlockNr = minBlock.getLinearScanNumber();
289        int toBlockNr = maxBlock.getLinearScanNumber();
290
291        assert 0 <= fromBlockNr && fromBlockNr < blockCount() : "out of range";
292        assert 0 <= toBlockNr && toBlockNr < blockCount() : "out of range";
293        assert fromBlockNr < toBlockNr : "must cross block boundary";
294
295        // Try to split at end of maxBlock. If this would be after
296        // maxSplitPos, then use the begin of maxBlock
297        int optimalSplitPos = allocator.getLastLirInstructionId(maxBlock) + 2;
298        if (optimalSplitPos > maxSplitPos) {
299            optimalSplitPos = allocator.getFirstLirInstructionId(maxBlock);
300        }
301
302        int minLoopDepth = maxBlock.getLoopDepth();
303        for (int i = toBlockNr - 1; i >= fromBlockNr; i--) {
304            AbstractBlockBase<?> cur = blockAt(i);
305
306            if (cur.getLoopDepth() < minLoopDepth) {
307                // block with lower loop-depth found . split at the end of this block
308                minLoopDepth = cur.getLoopDepth();
309                optimalSplitPos = allocator.getLastLirInstructionId(cur) + 2;
310            }
311        }
312        assert optimalSplitPos > allocator.maxOpId() || allocator.isBlockBegin(optimalSplitPos) : "algorithm must move split pos to block boundary";
313
314        return optimalSplitPos;
315    }
316
317    int findOptimalSplitPos(Interval interval, int minSplitPos, int maxSplitPos, boolean doLoopOptimization) {
318        int optimalSplitPos = -1;
319        if (minSplitPos == maxSplitPos) {
320            // trivial case, no optimization of split position possible
321            if (Debug.isLogEnabled()) {
322                Debug.log("min-pos and max-pos are equal, no optimization possible");
323            }
324            optimalSplitPos = minSplitPos;
325
326        } else {
327            assert minSplitPos < maxSplitPos : "must be true then";
328            assert minSplitPos > 0 : "cannot access minSplitPos - 1 otherwise";
329
330            // reason for using minSplitPos - 1: when the minimal split pos is exactly at the
331            // beginning of a block, then minSplitPos is also a possible split position.
332            // Use the block before as minBlock, because then minBlock.lastLirInstructionId() + 2 ==
333            // minSplitPos
334            AbstractBlockBase<?> minBlock = allocator.blockForId(minSplitPos - 1);
335
336            // reason for using maxSplitPos - 1: otherwise there would be an assert on failure
337            // when an interval ends at the end of the last block of the method
338            // (in this case, maxSplitPos == allocator().maxLirOpId() + 2, and there is no
339            // block at this opId)
340            AbstractBlockBase<?> maxBlock = allocator.blockForId(maxSplitPos - 1);
341
342            assert minBlock.getLinearScanNumber() <= maxBlock.getLinearScanNumber() : "invalid order";
343            if (minBlock == maxBlock) {
344                // split position cannot be moved to block boundary : so split as late as possible
345                if (Debug.isLogEnabled()) {
346                    Debug.log("cannot move split pos to block boundary because minPos and maxPos are in same block");
347                }
348                optimalSplitPos = maxSplitPos;
349
350            } else {
351                if (interval.hasHoleBetween(maxSplitPos - 1, maxSplitPos) && !allocator.isBlockBegin(maxSplitPos)) {
352                    // Do not move split position if the interval has a hole before maxSplitPos.
353                    // Intervals resulting from Phi-Functions have more than one definition (marked
354                    // as mustHaveRegister) with a hole before each definition. When the register is
355                    // needed
356                    // for the second definition : an earlier reloading is unnecessary.
357                    if (Debug.isLogEnabled()) {
358                        Debug.log("interval has hole just before maxSplitPos, so splitting at maxSplitPos");
359                    }
360                    optimalSplitPos = maxSplitPos;
361
362                } else {
363                    // seach optimal block boundary between minSplitPos and maxSplitPos
364                    if (Debug.isLogEnabled()) {
365                        Debug.log("moving split pos to optimal block boundary between block B%d and B%d", minBlock.getId(), maxBlock.getId());
366                    }
367
368                    if (doLoopOptimization) {
369                        // Loop optimization: if a loop-end marker is found between min- and
370                        // max-position :
371                        // then split before this loop
372                        int loopEndPos = interval.nextUsageExact(RegisterPriority.LiveAtLoopEnd, allocator.getLastLirInstructionId(minBlock) + 2);
373                        if (Debug.isLogEnabled()) {
374                            Debug.log("loop optimization: loop end found at pos %d", loopEndPos);
375                        }
376
377                        assert loopEndPos > minSplitPos : "invalid order";
378                        if (loopEndPos < maxSplitPos) {
379                            // loop-end marker found between min- and max-position
380                            // if it is not the end marker for the same loop as the min-position :
381                            // then move
382                            // the max-position to this loop block.
383                            // Desired result: uses tagged as shouldHaveRegister inside a loop cause
384                            // a reloading
385                            // of the interval (normally, only mustHaveRegister causes a reloading)
386                            AbstractBlockBase<?> loopBlock = allocator.blockForId(loopEndPos);
387
388                            if (Debug.isLogEnabled()) {
389                                Debug.log("interval is used in loop that ends in block B%d, so trying to move maxBlock back from B%d to B%d", loopBlock.getId(), maxBlock.getId(), loopBlock.getId());
390                            }
391                            assert loopBlock != minBlock : "loopBlock and minBlock must be different because block boundary is needed between";
392
393                            int maxSpillPos = allocator.getLastLirInstructionId(loopBlock) + 2;
394                            optimalSplitPos = findOptimalSplitPos(minBlock, loopBlock, maxSpillPos);
395                            if (optimalSplitPos == maxSpillPos) {
396                                optimalSplitPos = -1;
397                                if (Debug.isLogEnabled()) {
398                                    Debug.log("loop optimization not necessary");
399                                }
400                            } else {
401                                if (Debug.isLogEnabled()) {
402                                    Debug.log("loop optimization successful");
403                                }
404                            }
405                        }
406                    }
407
408                    if (optimalSplitPos == -1) {
409                        // not calculated by loop optimization
410                        optimalSplitPos = findOptimalSplitPos(minBlock, maxBlock, maxSplitPos);
411                    }
412                }
413            }
414        }
415        if (Debug.isLogEnabled()) {
416            Debug.log("optimal split position: %d", optimalSplitPos);
417        }
418
419        return optimalSplitPos;
420    }
421
422    // split an interval at the optimal position between minSplitPos and
423    // maxSplitPos in two parts:
424    // 1) the left part has already a location assigned
425    // 2) the right part is sorted into to the unhandled-list
426    void splitBeforeUsage(Interval interval, int minSplitPos, int maxSplitPos) {
427
428        try (Indent indent = Debug.logAndIndent("splitting interval %s between %d and %d", interval, minSplitPos, maxSplitPos)) {
429
430            assert interval.from() < minSplitPos : "cannot split at start of interval";
431            assert currentPosition < minSplitPos : "cannot split before current position";
432            assert minSplitPos <= maxSplitPos : "invalid order";
433            assert maxSplitPos <= interval.to() : "cannot split after end of interval";
434
435            int optimalSplitPos = findOptimalSplitPos(interval, minSplitPos, maxSplitPos, true);
436
437            assert minSplitPos <= optimalSplitPos && optimalSplitPos <= maxSplitPos : "out of range";
438            assert optimalSplitPos <= interval.to() : "cannot split after end of interval";
439            assert optimalSplitPos > interval.from() : "cannot split at start of interval";
440
441            if (optimalSplitPos == interval.to() && interval.nextUsage(RegisterPriority.MustHaveRegister, minSplitPos) == Integer.MAX_VALUE) {
442                // the split position would be just before the end of the interval
443                // . no split at all necessary
444                if (Debug.isLogEnabled()) {
445                    Debug.log("no split necessary because optimal split position is at end of interval");
446                }
447                return;
448            }
449
450            // must calculate this before the actual split is performed and before split position is
451            // moved to odd opId
452            boolean moveNecessary = !allocator.isBlockBegin(optimalSplitPos) && !interval.hasHoleBetween(optimalSplitPos - 1, optimalSplitPos);
453
454            if (!allocator.isBlockBegin(optimalSplitPos)) {
455                // move position before actual instruction (odd opId)
456                optimalSplitPos = (optimalSplitPos - 1) | 1;
457            }
458
459            if (Debug.isLogEnabled()) {
460                Debug.log("splitting at position %d", optimalSplitPos);
461            }
462
463            assert allocator.isBlockBegin(optimalSplitPos) || ((optimalSplitPos & 1) == 1) : "split pos must be odd when not on block boundary";
464            assert !allocator.isBlockBegin(optimalSplitPos) || ((optimalSplitPos & 1) == 0) : "split pos must be even on block boundary";
465
466            Interval splitPart = interval.split(optimalSplitPos, allocator);
467
468            splitPart.setInsertMoveWhenActivated(moveNecessary);
469
470            assert splitPart.from() >= currentPosition : "cannot append new interval before current walk position";
471            unhandledLists.addToListSortedByStartAndUsePositions(RegisterBinding.Any, splitPart);
472
473            if (Debug.isLogEnabled()) {
474                Debug.log("left interval  %s: %s", moveNecessary ? "      " : "", interval.logString(allocator));
475                Debug.log("right interval %s: %s", moveNecessary ? "(move)" : "", splitPart.logString(allocator));
476            }
477        }
478    }
479
480    // split an interval at the optimal position between minSplitPos and
481    // maxSplitPos in two parts:
482    // 1) the left part has already a location assigned
483    // 2) the right part is always on the stack and therefore ignored in further processing
484
485    void splitForSpilling(Interval interval) {
486        // calculate allowed range of splitting position
487        int maxSplitPos = currentPosition;
488        int previousUsage = interval.previousUsage(RegisterPriority.ShouldHaveRegister, maxSplitPos);
489        if (previousUsage == currentPosition) {
490            /*
491             * If there is a usage with ShouldHaveRegister priority at the current position fall
492             * back to MustHaveRegister priority. This only happens if register priority was
493             * downgraded to MustHaveRegister in #allocLockedRegister.
494             */
495            previousUsage = interval.previousUsage(RegisterPriority.MustHaveRegister, maxSplitPos);
496        }
497        int minSplitPos = Math.max(previousUsage + 1, interval.from());
498
499        try (Indent indent = Debug.logAndIndent("splitting and spilling interval %s between %d and %d", interval, minSplitPos, maxSplitPos)) {
500
501            assert interval.state == State.Active : "why spill interval that is not active?";
502            assert interval.from() <= minSplitPos : "cannot split before start of interval";
503            assert minSplitPos <= maxSplitPos : "invalid order";
504            assert maxSplitPos < interval.to() : "cannot split at end end of interval";
505            assert currentPosition < interval.to() : "interval must not end before current position";
506
507            if (minSplitPos == interval.from()) {
508                // the whole interval is never used, so spill it entirely to memory
509
510                try (Indent indent2 = Debug.logAndIndent("spilling entire interval because split pos is at beginning of interval (use positions: %d)", interval.usePosList().size())) {
511
512                    assert interval.firstUsage(RegisterPriority.MustHaveRegister) > currentPosition : String.format("interval %s must not have use position before currentPosition %d", interval,
513                                    currentPosition);
514
515                    allocator.assignSpillSlot(interval);
516                    handleSpillSlot(interval);
517                    changeSpillState(interval, minSplitPos);
518
519                    // Also kick parent intervals out of register to memory when they have no use
520                    // position. This avoids short interval in register surrounded by intervals in
521                    // memory . avoid useless moves from memory to register and back
522                    Interval parent = interval;
523                    while (parent != null && parent.isSplitChild()) {
524                        parent = parent.getSplitChildBeforeOpId(parent.from());
525
526                        if (isRegister(parent.location())) {
527                            if (parent.firstUsage(RegisterPriority.ShouldHaveRegister) == Integer.MAX_VALUE) {
528                                // parent is never used, so kick it out of its assigned register
529                                if (Debug.isLogEnabled()) {
530                                    Debug.log("kicking out interval %d out of its register because it is never used", parent.operandNumber);
531                                }
532                                allocator.assignSpillSlot(parent);
533                                handleSpillSlot(parent);
534                            } else {
535                                // do not go further back because the register is actually used by
536                                // the interval
537                                parent = null;
538                            }
539                        }
540                    }
541                }
542
543            } else {
544                // search optimal split pos, split interval and spill only the right hand part
545                int optimalSplitPos = findOptimalSplitPos(interval, minSplitPos, maxSplitPos, false);
546
547                assert minSplitPos <= optimalSplitPos && optimalSplitPos <= maxSplitPos : "out of range";
548                assert optimalSplitPos < interval.to() : "cannot split at end of interval";
549                assert optimalSplitPos >= interval.from() : "cannot split before start of interval";
550
551                if (!allocator.isBlockBegin(optimalSplitPos)) {
552                    // move position before actual instruction (odd opId)
553                    optimalSplitPos = (optimalSplitPos - 1) | 1;
554                }
555
556                try (Indent indent2 = Debug.logAndIndent("splitting at position %d", optimalSplitPos)) {
557                    assert allocator.isBlockBegin(optimalSplitPos) || ((optimalSplitPos & 1) == 1) : "split pos must be odd when not on block boundary";
558                    assert !allocator.isBlockBegin(optimalSplitPos) || ((optimalSplitPos & 1) == 0) : "split pos must be even on block boundary";
559
560                    Interval spilledPart = interval.split(optimalSplitPos, allocator);
561                    allocator.assignSpillSlot(spilledPart);
562                    handleSpillSlot(spilledPart);
563                    changeSpillState(spilledPart, optimalSplitPos);
564
565                    if (!allocator.isBlockBegin(optimalSplitPos)) {
566                        if (Debug.isLogEnabled()) {
567                            Debug.log("inserting move from interval %d to %d", interval.operandNumber, spilledPart.operandNumber);
568                        }
569                        insertMove(optimalSplitPos, interval, spilledPart);
570                    }
571
572                    // the currentSplitChild is needed later when moves are inserted for reloading
573                    assert spilledPart.currentSplitChild() == interval : "overwriting wrong currentSplitChild";
574                    spilledPart.makeCurrentSplitChild();
575
576                    if (Debug.isLogEnabled()) {
577                        Debug.log("left interval: %s", interval.logString(allocator));
578                        Debug.log("spilled interval   : %s", spilledPart.logString(allocator));
579                    }
580                }
581            }
582        }
583    }
584
585    // called during register allocation
586    private void changeSpillState(Interval interval, int spillPos) {
587        switch (interval.spillState()) {
588            case NoSpillStore: {
589                int defLoopDepth = allocator.blockForId(interval.spillDefinitionPos()).getLoopDepth();
590                int spillLoopDepth = allocator.blockForId(spillPos).getLoopDepth();
591
592                if (defLoopDepth < spillLoopDepth) {
593                    /*
594                     * The loop depth of the spilling position is higher then the loop depth at the
595                     * definition of the interval. Move write to memory out of loop.
596                     */
597                    if (LinearScan.Options.LIROptLSRAOptimizeSpillPosition.getValue()) {
598                        // find best spill position in dominator the tree
599                        interval.setSpillState(SpillState.SpillInDominator);
600                    } else {
601                        // store at definition of the interval
602                        interval.setSpillState(SpillState.StoreAtDefinition);
603                    }
604                } else {
605                    /*
606                     * The interval is currently spilled only once, so for now there is no reason to
607                     * store the interval at the definition.
608                     */
609                    interval.setSpillState(SpillState.OneSpillStore);
610                }
611                break;
612            }
613
614            case OneSpillStore: {
615                if (LinearScan.Options.LIROptLSRAOptimizeSpillPosition.getValue()) {
616                    // the interval is spilled more then once
617                    interval.setSpillState(SpillState.SpillInDominator);
618                } else {
619                    // It is better to store it to memory at the definition.
620                    interval.setSpillState(SpillState.StoreAtDefinition);
621                }
622                break;
623            }
624
625            case SpillInDominator:
626            case StoreAtDefinition:
627            case StartInMemory:
628            case NoOptimization:
629            case NoDefinitionFound:
630                // nothing to do
631                break;
632
633            default:
634                throw new BailoutException("other states not allowed at this time");
635        }
636    }
637
638    /**
639     * This is called for every interval that is assigned to a stack slot.
640     */
641    protected void handleSpillSlot(Interval interval) {
642        assert interval.location() != null && (interval.canMaterialize() || isStackSlotValue(interval.location())) : "interval not assigned to a stack slot " + interval;
643        // Do nothing. Stack slots are not processed in this implementation.
644    }
645
646    void splitStackInterval(Interval interval) {
647        int minSplitPos = currentPosition + 1;
648        int maxSplitPos = Math.min(interval.firstUsage(RegisterPriority.ShouldHaveRegister), interval.to());
649
650        splitBeforeUsage(interval, minSplitPos, maxSplitPos);
651    }
652
653    void splitWhenPartialRegisterAvailable(Interval interval, int registerAvailableUntil) {
654        int minSplitPos = Math.max(interval.previousUsage(RegisterPriority.ShouldHaveRegister, registerAvailableUntil), interval.from() + 1);
655        splitBeforeUsage(interval, minSplitPos, registerAvailableUntil);
656    }
657
658    void splitAndSpillInterval(Interval interval) {
659        assert interval.state == State.Active || interval.state == State.Inactive : "other states not allowed";
660
661        int currentPos = currentPosition;
662        if (interval.state == State.Inactive) {
663            // the interval is currently inactive, so no spill slot is needed for now.
664            // when the split part is activated, the interval has a new chance to get a register,
665            // so in the best case no stack slot is necessary
666            assert interval.hasHoleBetween(currentPos - 1, currentPos + 1) : "interval can not be inactive otherwise";
667            splitBeforeUsage(interval, currentPos + 1, currentPos + 1);
668
669        } else {
670            // search the position where the interval must have a register and split
671            // at the optimal position before.
672            // The new created part is added to the unhandled list and will get a register
673            // when it is activated
674            int minSplitPos = currentPos + 1;
675            int maxSplitPos = Math.min(interval.nextUsage(RegisterPriority.MustHaveRegister, minSplitPos), interval.to());
676
677            splitBeforeUsage(interval, minSplitPos, maxSplitPos);
678
679            assert interval.nextUsage(RegisterPriority.MustHaveRegister, currentPos) == Integer.MAX_VALUE : "the remaining part is spilled to stack and therefore has no register";
680            splitForSpilling(interval);
681        }
682    }
683
684    boolean allocFreeRegister(Interval interval) {
685        try (Indent indent = Debug.logAndIndent("trying to find free register for %s", interval)) {
686
687            initUseLists(true);
688            freeExcludeActiveFixed();
689            freeExcludeActiveAny();
690            freeCollectInactiveFixed(interval);
691            freeCollectInactiveAny(interval);
692            // freeCollectUnhandled(fixedKind, cur);
693            assert unhandledLists.get(RegisterBinding.Fixed) == Interval.EndMarker : "must not have unhandled fixed intervals because all fixed intervals have a use at position 0";
694
695            // usePos contains the start of the next interval that has this register assigned
696            // (either as a fixed register or a normal allocated register in the past)
697            // only intervals overlapping with cur are processed, non-overlapping invervals can be
698            // ignored safely
699            if (Debug.isLogEnabled()) {
700                // Enable this logging to see all register states
701                try (Indent indent2 = Debug.logAndIndent("state of registers:")) {
702                    for (Register register : availableRegs) {
703                        int i = register.number;
704                        Debug.log("reg %d: usePos: %d", register.number, usePos[i]);
705                    }
706                }
707            }
708
709            Register hint = null;
710            Interval locationHint = interval.locationHint(true);
711            if (locationHint != null && locationHint.location() != null && isRegister(locationHint.location())) {
712                hint = asRegister(locationHint.location());
713                if (Debug.isLogEnabled()) {
714                    Debug.log("hint register %d from interval %s", hint.number, locationHint);
715                }
716            }
717            assert interval.location() == null : "register already assigned to interval";
718
719            // the register must be free at least until this position
720            int regNeededUntil = interval.from() + 1;
721            int intervalTo = interval.to();
722
723            boolean needSplit = false;
724            int splitPos = -1;
725
726            Register reg = null;
727            Register minFullReg = null;
728            Register maxPartialReg = null;
729
730            for (Register availableReg : availableRegs) {
731                int number = availableReg.number;
732                if (usePos[number] >= intervalTo) {
733                    // this register is free for the full interval
734                    if (minFullReg == null || availableReg.equals(hint) || (usePos[number] < usePos[minFullReg.number] && !minFullReg.equals(hint))) {
735                        minFullReg = availableReg;
736                    }
737                } else if (usePos[number] > regNeededUntil) {
738                    // this register is at least free until regNeededUntil
739                    if (maxPartialReg == null || availableReg.equals(hint) || (usePos[number] > usePos[maxPartialReg.number] && !maxPartialReg.equals(hint))) {
740                        maxPartialReg = availableReg;
741                    }
742                }
743            }
744
745            if (minFullReg != null) {
746                reg = minFullReg;
747            } else if (maxPartialReg != null) {
748                needSplit = true;
749                reg = maxPartialReg;
750            } else {
751                return false;
752            }
753
754            splitPos = usePos[reg.number];
755            interval.assignLocation(reg.asValue(interval.kind()));
756            if (Debug.isLogEnabled()) {
757                Debug.log("selected register %d", reg.number);
758            }
759
760            assert splitPos > 0 : "invalid splitPos";
761            if (needSplit) {
762                // register not available for full interval, so split it
763                splitWhenPartialRegisterAvailable(interval, splitPos);
764            }
765            // only return true if interval is completely assigned
766            return true;
767        }
768    }
769
770    void splitAndSpillIntersectingIntervals(Register reg) {
771        assert reg != null : "no register assigned";
772
773        for (int i = 0; i < spillIntervals[reg.number].size(); i++) {
774            Interval interval = spillIntervals[reg.number].get(i);
775            removeFromList(interval);
776            splitAndSpillInterval(interval);
777        }
778    }
779
780    // Split an Interval and spill it to memory so that cur can be placed in a register
781    void allocLockedRegister(Interval interval) {
782        try (Indent indent = Debug.logAndIndent("alloc locked register: need to split and spill to get register for %s", interval)) {
783
784            // the register must be free at least until this position
785            int firstUsage = interval.firstUsage(RegisterPriority.MustHaveRegister);
786            int firstShouldHaveUsage = interval.firstUsage(RegisterPriority.ShouldHaveRegister);
787            int regNeededUntil = Math.min(firstUsage, interval.from() + 1);
788            int intervalTo = interval.to();
789            assert regNeededUntil >= 0 && regNeededUntil < Integer.MAX_VALUE : "interval has no use";
790
791            Register reg;
792            Register ignore;
793            /*
794             * In the common case we don't spill registers that have _any_ use position that is
795             * closer than the next use of the current interval, but if we can't spill the current
796             * interval we weaken this strategy and also allow spilling of intervals that have a
797             * non-mandatory requirements (no MustHaveRegister use position).
798             */
799            for (RegisterPriority registerPriority = RegisterPriority.LiveAtLoopEnd; true; registerPriority = RegisterPriority.MustHaveRegister) {
800                // collect current usage of registers
801                initUseLists(false);
802                spillExcludeActiveFixed();
803                // spillBlockUnhandledFixed(cur);
804                assert unhandledLists.get(RegisterBinding.Fixed) == Interval.EndMarker : "must not have unhandled fixed intervals because all fixed intervals have a use at position 0";
805                spillBlockInactiveFixed(interval);
806                spillCollectActiveAny(registerPriority);
807                spillCollectInactiveAny(interval);
808                if (Debug.isLogEnabled()) {
809                    printRegisterState();
810                }
811
812                reg = null;
813                ignore = interval.location() != null && isRegister(interval.location()) ? asRegister(interval.location()) : null;
814
815                for (Register availableReg : availableRegs) {
816                    int number = availableReg.number;
817                    if (availableReg.equals(ignore)) {
818                        // this register must be ignored
819                    } else if (usePos[number] > regNeededUntil) {
820                        if (reg == null || (usePos[number] > usePos[reg.number])) {
821                            reg = availableReg;
822                        }
823                    }
824                }
825
826                int regUsePos = (reg == null ? 0 : usePos[reg.number]);
827                if (regUsePos <= firstShouldHaveUsage) {
828                    if (Debug.isLogEnabled()) {
829                        Debug.log("able to spill current interval. firstUsage(register): %d, usePos: %d", firstUsage, regUsePos);
830                    }
831
832                    if (firstUsage <= interval.from() + 1) {
833                        if (registerPriority.equals(RegisterPriority.LiveAtLoopEnd)) {
834                            /*
835                             * Tool of last resort: we can not spill the current interval so we try
836                             * to spill an active interval that has a usage but do not require a
837                             * register.
838                             */
839                            Debug.log("retry with register priority must have register");
840                            continue;
841                        }
842                        String description = "cannot spill interval (" + interval + ") that is used in first instruction (possible reason: no register found) firstUsage=" + firstUsage +
843                                        ", interval.from()=" + interval.from() + "; already used candidates: " + Arrays.toString(availableRegs);
844                        /*
845                         * assign a reasonable register and do a bailout in product mode to avoid
846                         * errors
847                         */
848                        allocator.assignSpillSlot(interval);
849                        Debug.dump(allocator.getLIR(), description);
850                        allocator.printIntervals(description);
851                        throw new OutOfRegistersException("LinearScan: no register found", description);
852                    }
853
854                    splitAndSpillInterval(interval);
855                    return;
856                }
857                break;
858            }
859
860            boolean needSplit = blockPos[reg.number] <= intervalTo;
861
862            int splitPos = blockPos[reg.number];
863
864            if (Debug.isLogEnabled()) {
865                Debug.log("decided to use register %d", reg.number);
866            }
867            assert splitPos > 0 : "invalid splitPos";
868            assert needSplit || splitPos > interval.from() : "splitting interval at from";
869
870            interval.assignLocation(reg.asValue(interval.kind()));
871            if (needSplit) {
872                // register not available for full interval : so split it
873                splitWhenPartialRegisterAvailable(interval, splitPos);
874            }
875
876            // perform splitting and spilling for all affected intervals
877            splitAndSpillIntersectingIntervals(reg);
878            return;
879        }
880    }
881
882    void printRegisterState() {
883        try (Indent indent2 = Debug.logAndIndent("state of registers:")) {
884            for (Register reg : availableRegs) {
885                int i = reg.number;
886                try (Indent indent3 = Debug.logAndIndent("reg %d: usePos: %d, blockPos: %d, intervals: ", i, usePos[i], blockPos[i])) {
887                    for (int j = 0; j < spillIntervals[i].size(); j++) {
888                        Debug.log("%s ", spillIntervals[i].get(j));
889                    }
890                }
891            }
892        }
893    }
894
895    boolean noAllocationPossible(Interval interval) {
896        if (allocator.callKillsRegisters()) {
897            // fast calculation of intervals that can never get a register because the
898            // the next instruction is a call that blocks all registers
899            // Note: this only works if a call kills all registers
900
901            // check if this interval is the result of a split operation
902            // (an interval got a register until this position)
903            int pos = interval.from();
904            if (isOdd(pos)) {
905                // the current instruction is a call that blocks all registers
906                if (pos < allocator.maxOpId() && allocator.hasCall(pos + 1) && interval.to() > pos + 1) {
907                    if (Debug.isLogEnabled()) {
908                        Debug.log("free register cannot be available because all registers blocked by following call");
909                    }
910
911                    // safety check that there is really no register available
912                    assert !allocFreeRegister(interval) : "found a register for this interval";
913                    return true;
914                }
915            }
916        }
917        return false;
918    }
919
920    void initVarsForAlloc(Interval interval) {
921        AllocatableRegisters allocatableRegisters = allocator.getRegisterAllocationConfig().getAllocatableRegisters(interval.kind().getPlatformKind());
922        availableRegs = allocatableRegisters.allocatableRegisters;
923        minReg = allocatableRegisters.minRegisterNumber;
924        maxReg = allocatableRegisters.maxRegisterNumber;
925    }
926
927    static boolean isMove(LIRInstruction op, Interval from, Interval to) {
928        if (op instanceof MoveOp) {
929            MoveOp move = (MoveOp) op;
930            if (isVariable(move.getInput()) && isVariable(move.getResult())) {
931                return move.getInput() != null && move.getInput().equals(from.operand) && move.getResult() != null && move.getResult().equals(to.operand);
932            }
933        }
934        return false;
935    }
936
937    // optimization (especially for phi functions of nested loops):
938    // assign same spill slot to non-intersecting intervals
939    void combineSpilledIntervals(Interval interval) {
940        if (interval.isSplitChild()) {
941            // optimization is only suitable for split parents
942            return;
943        }
944
945        Interval registerHint = interval.locationHint(false);
946        if (registerHint == null) {
947            // cur is not the target of a move : otherwise registerHint would be set
948            return;
949        }
950        assert registerHint.isSplitParent() : "register hint must be split parent";
951
952        if (interval.spillState() != SpillState.NoOptimization || registerHint.spillState() != SpillState.NoOptimization) {
953            // combining the stack slots for intervals where spill move optimization is applied
954            // is not benefitial and would cause problems
955            return;
956        }
957
958        int beginPos = interval.from();
959        int endPos = interval.to();
960        if (endPos > allocator.maxOpId() || isOdd(beginPos) || isOdd(endPos)) {
961            // safety check that lirOpWithId is allowed
962            return;
963        }
964
965        if (!isMove(allocator.instructionForId(beginPos), registerHint, interval) || !isMove(allocator.instructionForId(endPos), interval, registerHint)) {
966            // cur and registerHint are not connected with two moves
967            return;
968        }
969
970        Interval beginHint = registerHint.getSplitChildAtOpId(beginPos, LIRInstruction.OperandMode.USE, allocator);
971        Interval endHint = registerHint.getSplitChildAtOpId(endPos, LIRInstruction.OperandMode.DEF, allocator);
972        if (beginHint == endHint || beginHint.to() != beginPos || endHint.from() != endPos) {
973            // registerHint must be split : otherwise the re-writing of use positions does not work
974            return;
975        }
976
977        assert beginHint.location() != null : "must have register assigned";
978        assert endHint.location() == null : "must not have register assigned";
979        assert interval.firstUsage(RegisterPriority.MustHaveRegister) == beginPos : "must have use position at begin of interval because of move";
980        assert endHint.firstUsage(RegisterPriority.MustHaveRegister) == endPos : "must have use position at begin of interval because of move";
981
982        if (isRegister(beginHint.location())) {
983            // registerHint is not spilled at beginPos : so it would not be benefitial to
984            // immediately spill cur
985            return;
986        }
987        assert registerHint.spillSlot() != null : "must be set when part of interval was spilled";
988
989        // modify intervals such that cur gets the same stack slot as registerHint
990        // delete use positions to prevent the intervals to get a register at beginning
991        interval.setSpillSlot(registerHint.spillSlot());
992        interval.removeFirstUsePos();
993        endHint.removeFirstUsePos();
994    }
995
996    // allocate a physical register or memory location to an interval
997    @Override
998    protected boolean activateCurrent(Interval interval) {
999        boolean result = true;
1000
1001        try (Indent indent = Debug.logAndIndent("activating interval %s,  splitParent: %d", interval, interval.splitParent().operandNumber)) {
1002
1003            final Value operand = interval.operand;
1004            if (interval.location() != null && isStackSlotValue(interval.location())) {
1005                // activating an interval that has a stack slot assigned . split it at first use
1006                // position
1007                // used for method parameters
1008                if (Debug.isLogEnabled()) {
1009                    Debug.log("interval has spill slot assigned (method parameter) . split it before first use");
1010                }
1011                splitStackInterval(interval);
1012                result = false;
1013
1014            } else {
1015                if (interval.location() == null) {
1016                    // interval has not assigned register . normal allocation
1017                    // (this is the normal case for most intervals)
1018                    if (Debug.isLogEnabled()) {
1019                        Debug.log("normal allocation of register");
1020                    }
1021
1022                    // assign same spill slot to non-intersecting intervals
1023                    combineSpilledIntervals(interval);
1024
1025                    initVarsForAlloc(interval);
1026                    if (noAllocationPossible(interval) || !allocFreeRegister(interval)) {
1027                        // no empty register available.
1028                        // split and spill another interval so that this interval gets a register
1029                        allocLockedRegister(interval);
1030                    }
1031
1032                    // spilled intervals need not be move to active-list
1033                    if (!isRegister(interval.location())) {
1034                        result = false;
1035                    }
1036                }
1037            }
1038
1039            // load spilled values that become active from stack slot to register
1040            if (interval.insertMoveWhenActivated()) {
1041                assert interval.isSplitChild();
1042                assert interval.currentSplitChild() != null;
1043                assert !interval.currentSplitChild().operand.equals(operand) : "cannot insert move between same interval";
1044                if (Debug.isLogEnabled()) {
1045                    Debug.log("Inserting move from interval %d to %d because insertMoveWhenActivated is set", interval.currentSplitChild().operandNumber, interval.operandNumber);
1046                }
1047
1048                insertMove(interval.from(), interval.currentSplitChild(), interval);
1049            }
1050            interval.makeCurrentSplitChild();
1051
1052        }
1053
1054        return result; // true = interval is moved to active list
1055    }
1056
1057    public void finishAllocation() {
1058        // must be called when all intervals are allocated
1059        moveResolver.resolveAndAppendMoves();
1060    }
1061}