2017-08-30 06:32:07 +08:00
|
|
|
//===- InlineSpiller.cpp - Insert spills and restores inline --------------===//
|
2010-06-30 07:58:39 +08:00
|
|
|
//
|
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
//
|
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
// License. See LICENSE.TXT for details.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
|
|
|
// The inline spiller modifies the machine function directly instead of
|
|
|
|
// inserting spills and restores in VirtRegMap.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
2017-08-30 06:32:07 +08:00
|
|
|
#include "LiveRangeCalc.h"
|
2010-06-30 07:58:39 +08:00
|
|
|
#include "Spiller.h"
|
2016-05-12 06:37:43 +08:00
|
|
|
#include "SplitKit.h"
|
2017-08-30 06:32:07 +08:00
|
|
|
#include "llvm/ADT/ArrayRef.h"
|
|
|
|
#include "llvm/ADT/DenseMap.h"
|
2016-04-13 11:08:27 +08:00
|
|
|
#include "llvm/ADT/MapVector.h"
|
2017-08-30 06:32:07 +08:00
|
|
|
#include "llvm/ADT/None.h"
|
|
|
|
#include "llvm/ADT/STLExtras.h"
|
2013-05-23 23:42:57 +08:00
|
|
|
#include "llvm/ADT/SetVector.h"
|
2017-08-30 06:32:07 +08:00
|
|
|
#include "llvm/ADT/SmallPtrSet.h"
|
|
|
|
#include "llvm/ADT/SmallVector.h"
|
2011-05-06 01:22:53 +08:00
|
|
|
#include "llvm/ADT/Statistic.h"
|
2010-11-11 07:55:56 +08:00
|
|
|
#include "llvm/Analysis/AliasAnalysis.h"
|
2017-08-30 06:32:07 +08:00
|
|
|
#include "llvm/CodeGen/LiveInterval.h"
|
2017-12-13 10:51:04 +08:00
|
|
|
#include "llvm/CodeGen/LiveIntervals.h"
|
2012-04-03 06:44:18 +08:00
|
|
|
#include "llvm/CodeGen/LiveRangeEdit.h"
|
2017-12-19 07:19:44 +08:00
|
|
|
#include "llvm/CodeGen/LiveStacks.h"
|
2017-08-30 06:32:07 +08:00
|
|
|
#include "llvm/CodeGen/MachineBasicBlock.h"
|
2013-06-18 03:00:36 +08:00
|
|
|
#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
|
2014-01-07 19:48:04 +08:00
|
|
|
#include "llvm/CodeGen/MachineDominators.h"
|
2010-06-30 07:58:39 +08:00
|
|
|
#include "llvm/CodeGen/MachineFunction.h"
|
2017-08-30 06:32:07 +08:00
|
|
|
#include "llvm/CodeGen/MachineFunctionPass.h"
|
|
|
|
#include "llvm/CodeGen/MachineInstr.h"
|
2013-06-17 04:34:15 +08:00
|
|
|
#include "llvm/CodeGen/MachineInstrBuilder.h"
|
2012-12-04 00:50:05 +08:00
|
|
|
#include "llvm/CodeGen/MachineInstrBundle.h"
|
2011-03-16 05:13:25 +08:00
|
|
|
#include "llvm/CodeGen/MachineLoopInfo.h"
|
2017-08-30 06:32:07 +08:00
|
|
|
#include "llvm/CodeGen/MachineOperand.h"
|
2010-06-30 07:58:39 +08:00
|
|
|
#include "llvm/CodeGen/MachineRegisterInfo.h"
|
2017-08-30 06:32:07 +08:00
|
|
|
#include "llvm/CodeGen/SlotIndexes.h"
|
2017-11-08 09:01:31 +08:00
|
|
|
#include "llvm/CodeGen/TargetInstrInfo.h"
|
2017-11-17 09:07:10 +08:00
|
|
|
#include "llvm/CodeGen/TargetOpcodes.h"
|
|
|
|
#include "llvm/CodeGen/TargetRegisterInfo.h"
|
|
|
|
#include "llvm/CodeGen/TargetSubtargetInfo.h"
|
2012-11-29 03:13:06 +08:00
|
|
|
#include "llvm/CodeGen/VirtRegMap.h"
|
2017-08-30 06:32:07 +08:00
|
|
|
#include "llvm/Support/BlockFrequency.h"
|
|
|
|
#include "llvm/Support/BranchProbability.h"
|
2011-09-16 05:06:00 +08:00
|
|
|
#include "llvm/Support/CommandLine.h"
|
2017-08-30 06:32:07 +08:00
|
|
|
#include "llvm/Support/Compiler.h"
|
2010-06-30 07:58:39 +08:00
|
|
|
#include "llvm/Support/Debug.h"
|
2017-08-30 06:32:07 +08:00
|
|
|
#include "llvm/Support/ErrorHandling.h"
|
2010-06-30 07:58:39 +08:00
|
|
|
#include "llvm/Support/raw_ostream.h"
|
2017-08-30 06:32:07 +08:00
|
|
|
#include <cassert>
|
|
|
|
#include <iterator>
|
|
|
|
#include <tuple>
|
|
|
|
#include <utility>
|
|
|
|
#include <vector>
|
2010-06-30 07:58:39 +08:00
|
|
|
|
|
|
|
using namespace llvm;
|
|
|
|
|
2014-04-22 10:02:50 +08:00
|
|
|
#define DEBUG_TYPE "regalloc"
|
|
|
|
|
2011-05-06 01:22:53 +08:00
|
|
|
STATISTIC(NumSpilledRanges, "Number of spilled live ranges");
|
2011-09-16 01:54:28 +08:00
|
|
|
STATISTIC(NumSnippets, "Number of spilled snippets");
|
2011-05-06 01:22:53 +08:00
|
|
|
STATISTIC(NumSpills, "Number of spills inserted");
|
2011-09-16 01:54:28 +08:00
|
|
|
STATISTIC(NumSpillsRemoved, "Number of spills removed");
|
2011-05-06 01:22:53 +08:00
|
|
|
STATISTIC(NumReloads, "Number of reloads inserted");
|
2011-09-16 01:54:28 +08:00
|
|
|
STATISTIC(NumReloadsRemoved, "Number of reloads removed");
|
2011-05-06 01:22:53 +08:00
|
|
|
STATISTIC(NumFolded, "Number of folded stack accesses");
|
|
|
|
STATISTIC(NumFoldedLoads, "Number of folded loads");
|
|
|
|
STATISTIC(NumRemats, "Number of rematerialized defs for spilling");
|
|
|
|
|
2011-09-16 05:06:00 +08:00
|
|
|
static cl::opt<bool> DisableHoisting("disable-spill-hoist", cl::Hidden,
|
|
|
|
cl::desc("Disable inline spill hoisting"));
|
|
|
|
|
2010-06-30 07:58:39 +08:00
|
|
|
namespace {
|
2017-08-30 06:32:07 +08:00
|
|
|
|
2016-04-16 07:16:44 +08:00
|
|
|
class HoistSpillHelper : private LiveRangeEdit::Delegate {
|
|
|
|
MachineFunction &MF;
|
2016-04-13 11:08:27 +08:00
|
|
|
LiveIntervals &LIS;
|
|
|
|
LiveStacks &LSS;
|
|
|
|
AliasAnalysis *AA;
|
|
|
|
MachineDominatorTree &MDT;
|
|
|
|
MachineLoopInfo &Loops;
|
|
|
|
VirtRegMap &VRM;
|
|
|
|
MachineRegisterInfo &MRI;
|
|
|
|
const TargetInstrInfo &TII;
|
|
|
|
const TargetRegisterInfo &TRI;
|
|
|
|
const MachineBlockFrequencyInfo &MBFI;
|
|
|
|
|
2016-05-12 06:37:43 +08:00
|
|
|
InsertPointAnalysis IPA;
|
|
|
|
|
2017-09-14 05:41:30 +08:00
|
|
|
// Map from StackSlot to the LiveInterval of the original register.
|
|
|
|
// Note the LiveInterval of the original register may have been deleted
|
|
|
|
// after it is spilled. We keep a copy here to track the range where
|
|
|
|
// spills can be moved.
|
|
|
|
DenseMap<int, std::unique_ptr<LiveInterval>> StackSlotToOrigLI;
|
2017-08-30 06:32:07 +08:00
|
|
|
|
2016-04-13 11:08:27 +08:00
|
|
|
// Map from pair of (StackSlot and Original VNI) to a set of spills which
|
|
|
|
// have the same stackslot and have equal values defined by Original VNI.
|
|
|
|
// These spills are mergeable and are hoist candiates.
|
2017-08-30 06:32:07 +08:00
|
|
|
using MergeableSpillsMap =
|
|
|
|
MapVector<std::pair<int, VNInfo *>, SmallPtrSet<MachineInstr *, 16>>;
|
2016-04-13 11:08:27 +08:00
|
|
|
MergeableSpillsMap MergeableSpills;
|
|
|
|
|
|
|
|
/// This is the map from original register to a set containing all its
|
|
|
|
/// siblings. To hoist a spill to another BB, we need to find out a live
|
|
|
|
/// sibling there and use it as the source of the new spill.
|
|
|
|
DenseMap<unsigned, SmallSetVector<unsigned, 16>> Virt2SiblingsMap;
|
|
|
|
|
2017-09-14 05:41:30 +08:00
|
|
|
bool isSpillCandBB(LiveInterval &OrigLI, VNInfo &OrigVNI,
|
|
|
|
MachineBasicBlock &BB, unsigned &LiveReg);
|
2016-04-13 11:08:27 +08:00
|
|
|
|
|
|
|
void rmRedundantSpills(
|
|
|
|
SmallPtrSet<MachineInstr *, 16> &Spills,
|
|
|
|
SmallVectorImpl<MachineInstr *> &SpillsToRm,
|
|
|
|
DenseMap<MachineDomTreeNode *, MachineInstr *> &SpillBBToSpill);
|
|
|
|
|
|
|
|
void getVisitOrders(
|
|
|
|
MachineBasicBlock *Root, SmallPtrSet<MachineInstr *, 16> &Spills,
|
|
|
|
SmallVectorImpl<MachineDomTreeNode *> &Orders,
|
|
|
|
SmallVectorImpl<MachineInstr *> &SpillsToRm,
|
|
|
|
DenseMap<MachineDomTreeNode *, unsigned> &SpillsToKeep,
|
|
|
|
DenseMap<MachineDomTreeNode *, MachineInstr *> &SpillBBToSpill);
|
|
|
|
|
2017-09-14 05:41:30 +08:00
|
|
|
void runHoistSpills(LiveInterval &OrigLI, VNInfo &OrigVNI,
|
2016-04-13 11:08:27 +08:00
|
|
|
SmallPtrSet<MachineInstr *, 16> &Spills,
|
|
|
|
SmallVectorImpl<MachineInstr *> &SpillsToRm,
|
|
|
|
DenseMap<MachineBasicBlock *, unsigned> &SpillsToIns);
|
|
|
|
|
|
|
|
public:
|
|
|
|
HoistSpillHelper(MachineFunctionPass &pass, MachineFunction &mf,
|
|
|
|
VirtRegMap &vrm)
|
2016-04-16 07:16:44 +08:00
|
|
|
: MF(mf), LIS(pass.getAnalysis<LiveIntervals>()),
|
2016-04-13 11:08:27 +08:00
|
|
|
LSS(pass.getAnalysis<LiveStacks>()),
|
|
|
|
AA(&pass.getAnalysis<AAResultsWrapperPass>().getAAResults()),
|
|
|
|
MDT(pass.getAnalysis<MachineDominatorTree>()),
|
|
|
|
Loops(pass.getAnalysis<MachineLoopInfo>()), VRM(vrm),
|
2017-08-30 06:32:07 +08:00
|
|
|
MRI(mf.getRegInfo()), TII(*mf.getSubtarget().getInstrInfo()),
|
2016-04-13 11:08:27 +08:00
|
|
|
TRI(*mf.getSubtarget().getRegisterInfo()),
|
2016-05-12 06:37:43 +08:00
|
|
|
MBFI(pass.getAnalysis<MachineBlockFrequencyInfo>()),
|
|
|
|
IPA(LIS, mf.getNumBlockIDs()) {}
|
2016-04-13 11:08:27 +08:00
|
|
|
|
2016-07-01 07:28:15 +08:00
|
|
|
void addToMergeableSpills(MachineInstr &Spill, int StackSlot,
|
2016-04-13 11:08:27 +08:00
|
|
|
unsigned Original);
|
2016-07-01 07:28:15 +08:00
|
|
|
bool rmFromMergeableSpills(MachineInstr &Spill, int StackSlot);
|
2016-04-16 07:16:44 +08:00
|
|
|
void hoistAllSpills();
|
|
|
|
void LRE_DidCloneVirtReg(unsigned, unsigned) override;
|
2016-04-13 11:08:27 +08:00
|
|
|
};
|
|
|
|
|
2010-06-30 07:58:39 +08:00
|
|
|
class InlineSpiller : public Spiller {
|
2011-03-15 03:56:43 +08:00
|
|
|
MachineFunction &MF;
|
|
|
|
LiveIntervals &LIS;
|
|
|
|
LiveStacks &LSS;
|
|
|
|
AliasAnalysis *AA;
|
2011-03-16 05:13:25 +08:00
|
|
|
MachineDominatorTree &MDT;
|
|
|
|
MachineLoopInfo &Loops;
|
2011-03-15 03:56:43 +08:00
|
|
|
VirtRegMap &VRM;
|
|
|
|
MachineRegisterInfo &MRI;
|
|
|
|
const TargetInstrInfo &TII;
|
|
|
|
const TargetRegisterInfo &TRI;
|
2013-06-18 03:00:36 +08:00
|
|
|
const MachineBlockFrequencyInfo &MBFI;
|
2010-07-01 07:03:52 +08:00
|
|
|
|
|
|
|
// Variables that are valid during spill(), but used by multiple methods.
|
2011-03-15 03:56:43 +08:00
|
|
|
LiveRangeEdit *Edit;
|
2011-03-27 06:16:41 +08:00
|
|
|
LiveInterval *StackInt;
|
2011-03-15 03:56:43 +08:00
|
|
|
int StackSlot;
|
2011-03-16 05:13:25 +08:00
|
|
|
unsigned Original;
|
2010-06-30 07:58:39 +08:00
|
|
|
|
2011-03-15 03:56:43 +08:00
|
|
|
// All registers to spill to StackSlot, including the main register.
|
2011-03-12 12:17:20 +08:00
|
|
|
SmallVector<unsigned, 8> RegsToSpill;
|
|
|
|
|
|
|
|
// All COPY instructions to/from snippets.
|
|
|
|
// They are ignored since both operands refer to the same stack slot.
|
|
|
|
SmallPtrSet<MachineInstr*, 8> SnippetCopies;
|
|
|
|
|
2010-10-21 06:00:51 +08:00
|
|
|
// Values that failed to remat at some point.
|
2011-03-15 03:56:43 +08:00
|
|
|
SmallPtrSet<VNInfo*, 8> UsedValues;
|
2010-07-03 01:44:57 +08:00
|
|
|
|
2011-03-18 12:23:06 +08:00
|
|
|
// Dead defs generated during spilling.
|
|
|
|
SmallVector<MachineInstr*, 8> DeadDefs;
|
2011-03-16 05:13:25 +08:00
|
|
|
|
2016-04-13 11:08:27 +08:00
|
|
|
// Object records spills information and does the hoisting.
|
|
|
|
HoistSpillHelper HSpiller;
|
|
|
|
|
2017-08-30 06:32:07 +08:00
|
|
|
~InlineSpiller() override = default;
|
2010-06-30 07:58:39 +08:00
|
|
|
|
|
|
|
public:
|
2014-08-05 05:25:23 +08:00
|
|
|
InlineSpiller(MachineFunctionPass &pass, MachineFunction &mf, VirtRegMap &vrm)
|
|
|
|
: MF(mf), LIS(pass.getAnalysis<LiveIntervals>()),
|
|
|
|
LSS(pass.getAnalysis<LiveStacks>()),
|
[PM/AA] Rebuild LLVM's alias analysis infrastructure in a way compatible
with the new pass manager, and no longer relying on analysis groups.
This builds essentially a ground-up new AA infrastructure stack for
LLVM. The core ideas are the same that are used throughout the new pass
manager: type erased polymorphism and direct composition. The design is
as follows:
- FunctionAAResults is a type-erasing alias analysis results aggregation
interface to walk a single query across a range of results from
different alias analyses. Currently this is function-specific as we
always assume that aliasing queries are *within* a function.
- AAResultBase is a CRTP utility providing stub implementations of
various parts of the alias analysis result concept, notably in several
cases in terms of other more general parts of the interface. This can
be used to implement only a narrow part of the interface rather than
the entire interface. This isn't really ideal, this logic should be
hoisted into FunctionAAResults as currently it will cause
a significant amount of redundant work, but it faithfully models the
behavior of the prior infrastructure.
- All the alias analysis passes are ported to be wrapper passes for the
legacy PM and new-style analysis passes for the new PM with a shared
result object. In some cases (most notably CFL), this is an extremely
naive approach that we should revisit when we can specialize for the
new pass manager.
- BasicAA has been restructured to reflect that it is much more
fundamentally a function analysis because it uses dominator trees and
loop info that need to be constructed for each function.
All of the references to getting alias analysis results have been
updated to use the new aggregation interface. All the preservation and
other pass management code has been updated accordingly.
The way the FunctionAAResultsWrapperPass works is to detect the
available alias analyses when run, and add them to the results object.
This means that we should be able to continue to respect when various
passes are added to the pipeline, for example adding CFL or adding TBAA
passes should just cause their results to be available and to get folded
into this. The exception to this rule is BasicAA which really needs to
be a function pass due to using dominator trees and loop info. As
a consequence, the FunctionAAResultsWrapperPass directly depends on
BasicAA and always includes it in the aggregation.
This has significant implications for preserving analyses. Generally,
most passes shouldn't bother preserving FunctionAAResultsWrapperPass
because rebuilding the results just updates the set of known AA passes.
The exception to this rule are LoopPass instances which need to preserve
all the function analyses that the loop pass manager will end up
needing. This means preserving both BasicAAWrapperPass and the
aggregating FunctionAAResultsWrapperPass.
Now, when preserving an alias analysis, you do so by directly preserving
that analysis. This is only necessary for non-immutable-pass-provided
alias analyses though, and there are only three of interest: BasicAA,
GlobalsAA (formerly GlobalsModRef), and SCEVAA. Usually BasicAA is
preserved when needed because it (like DominatorTree and LoopInfo) is
marked as a CFG-only pass. I've expanded GlobalsAA into the preserved
set everywhere we previously were preserving all of AliasAnalysis, and
I've added SCEVAA in the intersection of that with where we preserve
SCEV itself.
One significant challenge to all of this is that the CGSCC passes were
actually using the alias analysis implementations by taking advantage of
a pretty amazing set of loop holes in the old pass manager's analysis
management code which allowed analysis groups to slide through in many
cases. Moving away from analysis groups makes this problem much more
obvious. To fix it, I've leveraged the flexibility the design of the new
PM components provides to just directly construct the relevant alias
analyses for the relevant functions in the IPO passes that need them.
This is a bit hacky, but should go away with the new pass manager, and
is already in many ways cleaner than the prior state.
Another significant challenge is that various facilities of the old
alias analysis infrastructure just don't fit any more. The most
significant of these is the alias analysis 'counter' pass. That pass
relied on the ability to snoop on AA queries at different points in the
analysis group chain. Instead, I'm planning to build printing
functionality directly into the aggregation layer. I've not included
that in this patch merely to keep it smaller.
Note that all of this needs a nearly complete rewrite of the AA
documentation. I'm planning to do that, but I'd like to make sure the
new design settles, and to flesh out a bit more of what it looks like in
the new pass manager first.
Differential Revision: http://reviews.llvm.org/D12080
llvm-svn: 247167
2015-09-10 01:55:00 +08:00
|
|
|
AA(&pass.getAnalysis<AAResultsWrapperPass>().getAAResults()),
|
2014-08-05 05:25:23 +08:00
|
|
|
MDT(pass.getAnalysis<MachineDominatorTree>()),
|
|
|
|
Loops(pass.getAnalysis<MachineLoopInfo>()), VRM(vrm),
|
2017-08-30 06:32:07 +08:00
|
|
|
MRI(mf.getRegInfo()), TII(*mf.getSubtarget().getInstrInfo()),
|
2014-08-05 10:39:49 +08:00
|
|
|
TRI(*mf.getSubtarget().getRegisterInfo()),
|
2016-04-13 11:08:27 +08:00
|
|
|
MBFI(pass.getAnalysis<MachineBlockFrequencyInfo>()),
|
|
|
|
HSpiller(pass, mf, vrm) {}
|
2010-06-30 07:58:39 +08:00
|
|
|
|
2014-03-07 17:26:03 +08:00
|
|
|
void spill(LiveRangeEdit &) override;
|
2016-04-13 11:08:27 +08:00
|
|
|
void postOptimization() override;
|
2010-10-15 07:49:52 +08:00
|
|
|
|
2010-07-03 01:44:57 +08:00
|
|
|
private:
|
2011-03-12 12:17:20 +08:00
|
|
|
bool isSnippet(const LiveInterval &SnipLI);
|
|
|
|
void collectRegsToSpill();
|
|
|
|
|
2016-08-12 11:55:06 +08:00
|
|
|
bool isRegToSpill(unsigned Reg) { return is_contained(RegsToSpill, Reg); }
|
2011-03-18 12:23:06 +08:00
|
|
|
|
|
|
|
bool isSibling(unsigned Reg);
|
2016-04-13 11:08:27 +08:00
|
|
|
bool hoistSpillInsideBB(LiveInterval &SpillLI, MachineInstr &CopyMI);
|
2011-03-20 13:44:55 +08:00
|
|
|
void eliminateRedundantSpills(LiveInterval &LI, VNInfo *VNI);
|
2011-03-18 12:23:06 +08:00
|
|
|
|
2011-03-29 11:12:02 +08:00
|
|
|
void markValueUsed(LiveInterval*, VNInfo*);
|
2016-02-28 04:23:14 +08:00
|
|
|
bool reMaterializeFor(LiveInterval &, MachineInstr &MI);
|
2010-07-03 01:44:57 +08:00
|
|
|
void reMaterializeAll();
|
|
|
|
|
2011-03-12 12:17:20 +08:00
|
|
|
bool coalesceStackAccess(MachineInstr *MI, unsigned Reg);
|
2017-08-30 06:32:07 +08:00
|
|
|
bool foldMemoryOperand(ArrayRef<std::pair<MachineInstr *, unsigned>>,
|
2014-04-14 08:51:57 +08:00
|
|
|
MachineInstr *LoadMI = nullptr);
|
2013-08-15 07:50:16 +08:00
|
|
|
void insertReload(unsigned VReg, SlotIndex, MachineBasicBlock::iterator MI);
|
|
|
|
void insertSpill(unsigned VReg, bool isKill, MachineBasicBlock::iterator MI);
|
2011-03-12 12:17:20 +08:00
|
|
|
|
|
|
|
void spillAroundUses(unsigned Reg);
|
2011-03-30 05:20:19 +08:00
|
|
|
void spillAll();
|
2010-06-30 07:58:39 +08:00
|
|
|
};
|
|
|
|
|
2017-08-30 06:32:07 +08:00
|
|
|
} // end anonymous namespace
|
2014-11-07 03:12:38 +08:00
|
|
|
|
2017-08-30 06:32:07 +08:00
|
|
|
Spiller::~Spiller() = default;
|
2014-11-07 03:12:38 +08:00
|
|
|
|
2017-08-30 06:32:07 +08:00
|
|
|
void Spiller::anchor() {}
|
2014-11-07 03:12:38 +08:00
|
|
|
|
2017-08-30 06:32:07 +08:00
|
|
|
Spiller *llvm::createInlineSpiller(MachineFunctionPass &pass,
|
|
|
|
MachineFunction &mf,
|
|
|
|
VirtRegMap &vrm) {
|
|
|
|
return new InlineSpiller(pass, mf, vrm);
|
2015-06-23 17:49:53 +08:00
|
|
|
}
|
2010-06-30 07:58:39 +08:00
|
|
|
|
2011-03-12 12:17:20 +08:00
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
// Snippets
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
// When spilling a virtual register, we also spill any snippets it is connected
|
|
|
|
// to. The snippets are small live ranges that only have a single real use,
|
|
|
|
// leftovers from live range splitting. Spilling them enables memory operand
|
|
|
|
// folding or tightens the live range around the single use.
|
|
|
|
//
|
|
|
|
// This minimizes register pressure and maximizes the store-to-load distance for
|
|
|
|
// spill slots which can be important in tight loops.
|
|
|
|
|
|
|
|
/// isFullCopyOf - If MI is a COPY to or from Reg, return the other register,
|
|
|
|
/// otherwise return 0.
|
2016-06-30 08:01:54 +08:00
|
|
|
static unsigned isFullCopyOf(const MachineInstr &MI, unsigned Reg) {
|
|
|
|
if (!MI.isFullCopy())
|
2011-03-12 12:17:20 +08:00
|
|
|
return 0;
|
2016-06-30 08:01:54 +08:00
|
|
|
if (MI.getOperand(0).getReg() == Reg)
|
|
|
|
return MI.getOperand(1).getReg();
|
|
|
|
if (MI.getOperand(1).getReg() == Reg)
|
|
|
|
return MI.getOperand(0).getReg();
|
2011-03-12 12:17:20 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// isSnippet - Identify if a live interval is a snippet that should be spilled.
|
|
|
|
/// It is assumed that SnipLI is a virtual register with the same original as
|
2011-03-15 03:56:43 +08:00
|
|
|
/// Edit->getReg().
|
2011-03-12 12:17:20 +08:00
|
|
|
bool InlineSpiller::isSnippet(const LiveInterval &SnipLI) {
|
2011-03-15 03:56:43 +08:00
|
|
|
unsigned Reg = Edit->getReg();
|
2011-03-12 12:17:20 +08:00
|
|
|
|
|
|
|
// A snippet is a tiny live range with only a single instruction using it
|
|
|
|
// besides copies to/from Reg or spills/fills. We accept:
|
|
|
|
//
|
|
|
|
// %snip = COPY %Reg / FILL fi#
|
|
|
|
// %snip = USE %snip
|
|
|
|
// %Reg = COPY %snip / SPILL %snip, fi#
|
|
|
|
//
|
2011-03-15 03:56:43 +08:00
|
|
|
if (SnipLI.getNumValNums() > 2 || !LIS.intervalIsInOneMBB(SnipLI))
|
2011-03-12 12:17:20 +08:00
|
|
|
return false;
|
|
|
|
|
2014-04-14 08:51:57 +08:00
|
|
|
MachineInstr *UseMI = nullptr;
|
2011-03-12 12:17:20 +08:00
|
|
|
|
|
|
|
// Check that all uses satisfy our criteria.
|
2014-03-13 14:02:25 +08:00
|
|
|
for (MachineRegisterInfo::reg_instr_nodbg_iterator
|
|
|
|
RI = MRI.reg_instr_nodbg_begin(SnipLI.reg),
|
|
|
|
E = MRI.reg_instr_nodbg_end(); RI != E; ) {
|
2016-06-30 08:01:54 +08:00
|
|
|
MachineInstr &MI = *RI++;
|
2011-03-12 12:17:20 +08:00
|
|
|
|
|
|
|
// Allow copies to/from Reg.
|
|
|
|
if (isFullCopyOf(MI, Reg))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
// Allow stack slot loads.
|
|
|
|
int FI;
|
2011-03-15 03:56:43 +08:00
|
|
|
if (SnipLI.reg == TII.isLoadFromStackSlot(MI, FI) && FI == StackSlot)
|
2011-03-12 12:17:20 +08:00
|
|
|
continue;
|
|
|
|
|
|
|
|
// Allow stack slot stores.
|
2011-03-15 03:56:43 +08:00
|
|
|
if (SnipLI.reg == TII.isStoreToStackSlot(MI, FI) && FI == StackSlot)
|
2011-03-12 12:17:20 +08:00
|
|
|
continue;
|
|
|
|
|
|
|
|
// Allow a single additional instruction.
|
2016-06-30 08:01:54 +08:00
|
|
|
if (UseMI && &MI != UseMI)
|
2011-03-12 12:17:20 +08:00
|
|
|
return false;
|
2016-06-30 08:01:54 +08:00
|
|
|
UseMI = &MI;
|
2011-03-12 12:17:20 +08:00
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// collectRegsToSpill - Collect live range snippets that only have a single
|
|
|
|
/// real use.
|
|
|
|
void InlineSpiller::collectRegsToSpill() {
|
2011-03-15 03:56:43 +08:00
|
|
|
unsigned Reg = Edit->getReg();
|
2011-03-12 12:17:20 +08:00
|
|
|
|
|
|
|
// Main register always spills.
|
|
|
|
RegsToSpill.assign(1, Reg);
|
|
|
|
SnippetCopies.clear();
|
|
|
|
|
|
|
|
// Snippets all have the same original, so there can't be any for an original
|
|
|
|
// register.
|
2011-03-16 05:13:25 +08:00
|
|
|
if (Original == Reg)
|
2011-03-12 12:17:20 +08:00
|
|
|
return;
|
|
|
|
|
2014-03-13 14:02:25 +08:00
|
|
|
for (MachineRegisterInfo::reg_instr_iterator
|
|
|
|
RI = MRI.reg_instr_begin(Reg), E = MRI.reg_instr_end(); RI != E; ) {
|
2016-06-30 08:01:54 +08:00
|
|
|
MachineInstr &MI = *RI++;
|
2011-03-12 12:17:20 +08:00
|
|
|
unsigned SnipReg = isFullCopyOf(MI, Reg);
|
2011-03-18 12:23:06 +08:00
|
|
|
if (!isSibling(SnipReg))
|
2011-03-12 12:17:20 +08:00
|
|
|
continue;
|
2011-03-15 03:56:43 +08:00
|
|
|
LiveInterval &SnipLI = LIS.getInterval(SnipReg);
|
2011-03-12 12:17:20 +08:00
|
|
|
if (!isSnippet(SnipLI))
|
|
|
|
continue;
|
2016-06-30 08:01:54 +08:00
|
|
|
SnippetCopies.insert(&MI);
|
2011-05-06 01:22:53 +08:00
|
|
|
if (isRegToSpill(SnipReg))
|
|
|
|
continue;
|
|
|
|
RegsToSpill.push_back(SnipReg);
|
2011-03-12 12:17:20 +08:00
|
|
|
DEBUG(dbgs() << "\talso spill snippet " << SnipLI << '\n');
|
2011-05-06 01:22:53 +08:00
|
|
|
++NumSnippets;
|
2011-03-12 12:17:20 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-03-18 12:23:06 +08:00
|
|
|
bool InlineSpiller::isSibling(unsigned Reg) {
|
|
|
|
return TargetRegisterInfo::isVirtualRegister(Reg) &&
|
|
|
|
VRM.getOriginal(Reg) == Original;
|
|
|
|
}
|
|
|
|
|
2016-04-13 11:08:27 +08:00
|
|
|
/// It is beneficial to spill to earlier place in the same BB in case
|
|
|
|
/// as follows:
|
|
|
|
/// There is an alternative def earlier in the same MBB.
|
|
|
|
/// Hoist the spill as far as possible in SpillMBB. This can ease
|
|
|
|
/// register pressure:
|
2011-09-10 02:11:41 +08:00
|
|
|
///
|
2016-04-13 11:08:27 +08:00
|
|
|
/// x = def
|
|
|
|
/// y = use x
|
|
|
|
/// s = copy x
|
2011-03-16 05:13:25 +08:00
|
|
|
///
|
2016-04-13 11:08:27 +08:00
|
|
|
/// Hoisting the spill of s to immediately after the def removes the
|
|
|
|
/// interference between x and y:
|
2011-03-16 05:13:25 +08:00
|
|
|
///
|
2016-04-13 11:08:27 +08:00
|
|
|
/// x = def
|
|
|
|
/// spill x
|
2017-12-07 18:40:31 +08:00
|
|
|
/// y = use killed x
|
2011-03-16 05:13:25 +08:00
|
|
|
///
|
2016-04-13 11:08:27 +08:00
|
|
|
/// This hoist only helps when the copy kills its source.
|
2011-03-29 11:12:02 +08:00
|
|
|
///
|
2016-04-13 11:08:27 +08:00
|
|
|
bool InlineSpiller::hoistSpillInsideBB(LiveInterval &SpillLI,
|
|
|
|
MachineInstr &CopyMI) {
|
2011-03-18 12:23:06 +08:00
|
|
|
SlotIndex Idx = LIS.getInstructionIndex(CopyMI);
|
2016-04-13 11:08:27 +08:00
|
|
|
#ifndef NDEBUG
|
2011-11-14 04:45:27 +08:00
|
|
|
VNInfo *VNI = SpillLI.getVNInfoAt(Idx.getRegSlot());
|
|
|
|
assert(VNI && VNI->def == Idx.getRegSlot() && "Not defined by copy");
|
2016-04-13 11:08:27 +08:00
|
|
|
#endif
|
2016-04-08 23:17:43 +08:00
|
|
|
|
2016-04-13 11:08:27 +08:00
|
|
|
unsigned SrcReg = CopyMI.getOperand(1).getReg();
|
|
|
|
LiveInterval &SrcLI = LIS.getInterval(SrcReg);
|
|
|
|
VNInfo *SrcVNI = SrcLI.getVNInfoAt(Idx);
|
|
|
|
LiveQueryResult SrcQ = SrcLI.Query(Idx);
|
|
|
|
MachineBasicBlock *DefMBB = LIS.getMBBFromIndex(SrcVNI->def);
|
|
|
|
if (DefMBB != CopyMI.getParent() || !SrcQ.isKill())
|
2016-04-08 23:17:43 +08:00
|
|
|
return false;
|
|
|
|
|
2011-03-18 12:23:06 +08:00
|
|
|
// Conservatively extend the stack slot range to the range of the original
|
|
|
|
// value. We may be able to do better with stack slot coloring by being more
|
|
|
|
// careful here.
|
2011-03-27 06:16:41 +08:00
|
|
|
assert(StackInt && "No stack slot assigned yet.");
|
2011-03-18 12:23:06 +08:00
|
|
|
LiveInterval &OrigLI = LIS.getInterval(Original);
|
|
|
|
VNInfo *OrigVNI = OrigLI.getVNInfoAt(Idx);
|
2011-03-27 06:16:41 +08:00
|
|
|
StackInt->MergeValueInAsValue(OrigLI, OrigVNI, StackInt->getValNumInfo(0));
|
2011-03-20 07:02:47 +08:00
|
|
|
DEBUG(dbgs() << "\tmerged orig valno " << OrigVNI->id << ": "
|
2011-03-27 06:16:41 +08:00
|
|
|
<< *StackInt << '\n');
|
2011-03-18 12:23:06 +08:00
|
|
|
|
2016-04-13 11:08:27 +08:00
|
|
|
// We are going to spill SrcVNI immediately after its def, so clear out
|
2011-03-18 12:23:06 +08:00
|
|
|
// any later spills of the same value.
|
2016-04-13 11:08:27 +08:00
|
|
|
eliminateRedundantSpills(SrcLI, SrcVNI);
|
2011-03-18 12:23:06 +08:00
|
|
|
|
2016-04-13 11:08:27 +08:00
|
|
|
MachineBasicBlock *MBB = LIS.getMBBFromIndex(SrcVNI->def);
|
2011-03-18 12:23:06 +08:00
|
|
|
MachineBasicBlock::iterator MII;
|
2016-04-13 11:08:27 +08:00
|
|
|
if (SrcVNI->isPHIDef())
|
2016-09-16 22:07:29 +08:00
|
|
|
MII = MBB->SkipPHIsLabelsAndDebug(MBB->begin());
|
2011-03-18 12:23:06 +08:00
|
|
|
else {
|
2016-04-13 11:08:27 +08:00
|
|
|
MachineInstr *DefMI = LIS.getInstructionFromIndex(SrcVNI->def);
|
2011-04-30 14:42:21 +08:00
|
|
|
assert(DefMI && "Defining instruction disappeared");
|
|
|
|
MII = DefMI;
|
2011-03-18 12:23:06 +08:00
|
|
|
++MII;
|
|
|
|
}
|
|
|
|
// Insert spill without kill flag immediately after def.
|
2016-04-13 11:08:27 +08:00
|
|
|
TII.storeRegToStackSlot(*MBB, MII, SrcReg, false, StackSlot,
|
|
|
|
MRI.getRegClass(SrcReg), &TRI);
|
2011-03-18 12:23:06 +08:00
|
|
|
--MII; // Point to store instruction.
|
2016-02-27 14:40:41 +08:00
|
|
|
LIS.InsertMachineInstrInMaps(*MII);
|
2016-04-13 11:08:27 +08:00
|
|
|
DEBUG(dbgs() << "\thoisted: " << SrcVNI->def << '\t' << *MII);
|
2011-05-06 01:22:53 +08:00
|
|
|
|
2016-07-01 07:28:15 +08:00
|
|
|
HSpiller.addToMergeableSpills(*MII, StackSlot, Original);
|
2011-09-16 01:54:28 +08:00
|
|
|
++NumSpills;
|
2011-03-18 12:23:06 +08:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2011-03-20 13:44:55 +08:00
|
|
|
/// eliminateRedundantSpills - SLI:VNI is known to be on the stack. Remove any
|
|
|
|
/// redundant spills of this value in SLI.reg and sibling copies.
|
|
|
|
void InlineSpiller::eliminateRedundantSpills(LiveInterval &SLI, VNInfo *VNI) {
|
2011-03-20 13:44:58 +08:00
|
|
|
assert(VNI && "Missing value");
|
2011-03-20 13:44:55 +08:00
|
|
|
SmallVector<std::pair<LiveInterval*, VNInfo*>, 8> WorkList;
|
|
|
|
WorkList.push_back(std::make_pair(&SLI, VNI));
|
2011-03-27 06:16:41 +08:00
|
|
|
assert(StackInt && "No stack slot assigned yet.");
|
2011-03-18 12:23:06 +08:00
|
|
|
|
|
|
|
do {
|
2011-03-20 13:44:55 +08:00
|
|
|
LiveInterval *LI;
|
2014-03-02 21:30:33 +08:00
|
|
|
std::tie(LI, VNI) = WorkList.pop_back_val();
|
2011-03-20 13:44:55 +08:00
|
|
|
unsigned Reg = LI->reg;
|
2011-04-30 14:42:21 +08:00
|
|
|
DEBUG(dbgs() << "Checking redundant spills for "
|
|
|
|
<< VNI->id << '@' << VNI->def << " in " << *LI << '\n');
|
2011-03-18 12:23:06 +08:00
|
|
|
|
|
|
|
// Regs to spill are taken care of.
|
|
|
|
if (isRegToSpill(Reg))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
// Add all of VNI's live range to StackInt.
|
2011-03-27 06:16:41 +08:00
|
|
|
StackInt->MergeValueInAsValue(*LI, VNI, StackInt->getValNumInfo(0));
|
|
|
|
DEBUG(dbgs() << "Merged to stack int: " << *StackInt << '\n');
|
2011-03-18 12:23:06 +08:00
|
|
|
|
|
|
|
// Find all spills and copies of VNI.
|
2014-03-13 14:02:25 +08:00
|
|
|
for (MachineRegisterInfo::use_instr_nodbg_iterator
|
|
|
|
UI = MRI.use_instr_nodbg_begin(Reg), E = MRI.use_instr_nodbg_end();
|
|
|
|
UI != E; ) {
|
2016-06-30 08:01:54 +08:00
|
|
|
MachineInstr &MI = *UI++;
|
|
|
|
if (!MI.isCopy() && !MI.mayStore())
|
2011-03-18 12:23:06 +08:00
|
|
|
continue;
|
2016-06-30 08:01:54 +08:00
|
|
|
SlotIndex Idx = LIS.getInstructionIndex(MI);
|
2011-03-20 13:44:55 +08:00
|
|
|
if (LI->getVNInfoAt(Idx) != VNI)
|
2011-03-18 12:23:06 +08:00
|
|
|
continue;
|
|
|
|
|
|
|
|
// Follow sibling copies down the dominator tree.
|
|
|
|
if (unsigned DstReg = isFullCopyOf(MI, Reg)) {
|
|
|
|
if (isSibling(DstReg)) {
|
|
|
|
LiveInterval &DstLI = LIS.getInterval(DstReg);
|
2011-11-14 04:45:27 +08:00
|
|
|
VNInfo *DstVNI = DstLI.getVNInfoAt(Idx.getRegSlot());
|
2011-03-18 12:23:06 +08:00
|
|
|
assert(DstVNI && "Missing defined value");
|
2011-11-14 04:45:27 +08:00
|
|
|
assert(DstVNI->def == Idx.getRegSlot() && "Wrong copy def slot");
|
2011-03-20 13:44:55 +08:00
|
|
|
WorkList.push_back(std::make_pair(&DstLI, DstVNI));
|
2011-03-18 12:23:06 +08:00
|
|
|
}
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Erase spills.
|
|
|
|
int FI;
|
|
|
|
if (Reg == TII.isStoreToStackSlot(MI, FI) && FI == StackSlot) {
|
2016-06-30 08:01:54 +08:00
|
|
|
DEBUG(dbgs() << "Redundant spill " << Idx << '\t' << MI);
|
2011-03-18 12:23:06 +08:00
|
|
|
// eliminateDeadDefs won't normally remove stores, so switch opcode.
|
2016-06-30 08:01:54 +08:00
|
|
|
MI.setDesc(TII.get(TargetOpcode::KILL));
|
|
|
|
DeadDefs.push_back(&MI);
|
2011-09-16 01:54:28 +08:00
|
|
|
++NumSpillsRemoved;
|
2016-07-01 07:28:15 +08:00
|
|
|
if (HSpiller.rmFromMergeableSpills(MI, StackSlot))
|
2016-04-13 11:08:27 +08:00
|
|
|
--NumSpills;
|
2011-03-18 12:23:06 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
} while (!WorkList.empty());
|
|
|
|
}
|
|
|
|
|
2011-03-29 11:12:02 +08:00
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
// Rematerialization
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
/// markValueUsed - Remember that VNI failed to rematerialize, so its defining
|
|
|
|
/// instruction cannot be eliminated. See through snippet copies
|
|
|
|
void InlineSpiller::markValueUsed(LiveInterval *LI, VNInfo *VNI) {
|
|
|
|
SmallVector<std::pair<LiveInterval*, VNInfo*>, 8> WorkList;
|
|
|
|
WorkList.push_back(std::make_pair(LI, VNI));
|
|
|
|
do {
|
2014-03-02 21:30:33 +08:00
|
|
|
std::tie(LI, VNI) = WorkList.pop_back_val();
|
2014-11-19 15:49:26 +08:00
|
|
|
if (!UsedValues.insert(VNI).second)
|
2011-03-29 11:12:02 +08:00
|
|
|
continue;
|
|
|
|
|
|
|
|
if (VNI->isPHIDef()) {
|
|
|
|
MachineBasicBlock *MBB = LIS.getMBBFromIndex(VNI->def);
|
2015-12-24 13:20:40 +08:00
|
|
|
for (MachineBasicBlock *P : MBB->predecessors()) {
|
|
|
|
VNInfo *PVNI = LI->getVNInfoBefore(LIS.getMBBEndIdx(P));
|
2011-03-29 11:12:02 +08:00
|
|
|
if (PVNI)
|
|
|
|
WorkList.push_back(std::make_pair(LI, PVNI));
|
|
|
|
}
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Follow snippet copies.
|
|
|
|
MachineInstr *MI = LIS.getInstructionFromIndex(VNI->def);
|
|
|
|
if (!SnippetCopies.count(MI))
|
|
|
|
continue;
|
|
|
|
LiveInterval &SnipLI = LIS.getInterval(MI->getOperand(1).getReg());
|
|
|
|
assert(isRegToSpill(SnipLI.reg) && "Unexpected register in copy");
|
2011-11-14 04:45:27 +08:00
|
|
|
VNInfo *SnipVNI = SnipLI.getVNInfoAt(VNI->def.getRegSlot(true));
|
2011-03-29 11:12:02 +08:00
|
|
|
assert(SnipVNI && "Snippet undefined before copy");
|
|
|
|
WorkList.push_back(std::make_pair(&SnipLI, SnipVNI));
|
|
|
|
} while (!WorkList.empty());
|
|
|
|
}
|
|
|
|
|
2011-02-23 07:01:49 +08:00
|
|
|
/// reMaterializeFor - Attempt to rematerialize before MI instead of reloading.
|
2016-02-28 04:23:14 +08:00
|
|
|
bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg, MachineInstr &MI) {
|
2014-09-01 19:04:07 +08:00
|
|
|
// Analyze instruction
|
|
|
|
SmallVector<std::pair<MachineInstr *, unsigned>, 8> Ops;
|
|
|
|
MIBundleOperands::VirtRegInfo RI =
|
2016-02-28 04:23:14 +08:00
|
|
|
MIBundleOperands(MI).analyzeVirtReg(VirtReg.reg, &Ops);
|
2014-09-01 19:04:07 +08:00
|
|
|
|
|
|
|
if (!RI.Reads)
|
|
|
|
return false;
|
|
|
|
|
2016-02-28 04:23:14 +08:00
|
|
|
SlotIndex UseIdx = LIS.getInstructionIndex(MI).getRegSlot(true);
|
2011-07-18 13:31:59 +08:00
|
|
|
VNInfo *ParentVNI = VirtReg.getVNInfoAt(UseIdx.getBaseIndex());
|
2010-10-21 06:00:51 +08:00
|
|
|
|
2011-03-29 11:12:02 +08:00
|
|
|
if (!ParentVNI) {
|
2010-07-03 01:44:57 +08:00
|
|
|
DEBUG(dbgs() << "\tadding <undef> flags: ");
|
2016-02-28 04:23:14 +08:00
|
|
|
for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
|
|
|
|
MachineOperand &MO = MI.getOperand(i);
|
2011-03-30 01:47:02 +08:00
|
|
|
if (MO.isReg() && MO.isUse() && MO.getReg() == VirtReg.reg)
|
2010-07-01 07:03:52 +08:00
|
|
|
MO.setIsUndef();
|
|
|
|
}
|
2016-02-28 04:23:14 +08:00
|
|
|
DEBUG(dbgs() << UseIdx << '\t' << MI);
|
2010-07-01 07:03:52 +08:00
|
|
|
return true;
|
|
|
|
}
|
2010-10-21 06:00:51 +08:00
|
|
|
|
2016-02-28 04:23:14 +08:00
|
|
|
if (SnippetCopies.count(&MI))
|
2011-03-12 12:17:20 +08:00
|
|
|
return false;
|
|
|
|
|
2016-04-13 11:08:27 +08:00
|
|
|
LiveInterval &OrigLI = LIS.getInterval(Original);
|
|
|
|
VNInfo *OrigVNI = OrigLI.getVNInfoAt(UseIdx);
|
2011-03-29 11:12:02 +08:00
|
|
|
LiveRangeEdit::Remat RM(ParentVNI);
|
2016-04-13 11:08:27 +08:00
|
|
|
RM.OrigMI = LIS.getInstructionFromIndex(OrigVNI->def);
|
|
|
|
|
|
|
|
if (!Edit->canRematerializeAt(RM, OrigVNI, UseIdx, false)) {
|
2011-03-29 11:12:02 +08:00
|
|
|
markValueUsed(&VirtReg, ParentVNI);
|
2016-02-28 04:23:14 +08:00
|
|
|
DEBUG(dbgs() << "\tcannot remat for " << UseIdx << '\t' << MI);
|
2010-07-01 07:03:52 +08:00
|
|
|
return false;
|
2010-07-03 01:44:57 +08:00
|
|
|
}
|
2010-07-01 07:03:52 +08:00
|
|
|
|
2011-03-30 01:47:02 +08:00
|
|
|
// If the instruction also writes VirtReg.reg, it had better not require the
|
|
|
|
// same register for uses and defs.
|
2012-03-01 09:43:25 +08:00
|
|
|
if (RI.Tied) {
|
|
|
|
markValueUsed(&VirtReg, ParentVNI);
|
2016-02-28 04:23:14 +08:00
|
|
|
DEBUG(dbgs() << "\tcannot remat tied reg: " << UseIdx << '\t' << MI);
|
2012-03-01 09:43:25 +08:00
|
|
|
return false;
|
2010-07-01 07:03:52 +08:00
|
|
|
}
|
|
|
|
|
2010-12-18 11:04:14 +08:00
|
|
|
// Before rematerializing into a register for a single instruction, try to
|
|
|
|
// fold a load into the instruction. That avoids allocating a new register.
|
2011-12-07 15:15:52 +08:00
|
|
|
if (RM.OrigMI->canFoldAsLoad() &&
|
2012-03-01 09:43:25 +08:00
|
|
|
foldMemoryOperand(Ops, RM.OrigMI)) {
|
2011-03-15 03:56:43 +08:00
|
|
|
Edit->markRematerialized(RM.ParentVNI);
|
2011-05-06 01:22:53 +08:00
|
|
|
++NumFoldedLoads;
|
2010-12-18 11:04:14 +08:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2016-08-17 01:12:50 +08:00
|
|
|
// Allocate a new register for the remat.
|
2013-08-15 07:50:16 +08:00
|
|
|
unsigned NewVReg = Edit->createFrom(Original);
|
2010-07-03 01:44:57 +08:00
|
|
|
|
|
|
|
// Finally we can rematerialize OrigMI before MI.
|
2016-02-28 04:23:14 +08:00
|
|
|
SlotIndex DefIdx =
|
|
|
|
Edit->rematerializeAt(*MI.getParent(), MI, NewVReg, RM, TRI);
|
2016-08-17 01:12:50 +08:00
|
|
|
|
|
|
|
// We take the DebugLoc from MI, since OrigMI may be attributed to a
|
2017-02-25 09:50:45 +08:00
|
|
|
// different source location.
|
2016-08-17 01:12:50 +08:00
|
|
|
auto *NewMI = LIS.getInstructionFromIndex(DefIdx);
|
|
|
|
NewMI->setDebugLoc(MI.getDebugLoc());
|
|
|
|
|
2013-08-15 07:50:16 +08:00
|
|
|
(void)DefIdx;
|
2011-02-09 03:33:55 +08:00
|
|
|
DEBUG(dbgs() << "\tremat: " << DefIdx << '\t'
|
2011-03-15 03:56:43 +08:00
|
|
|
<< *LIS.getInstructionFromIndex(DefIdx));
|
2010-07-03 01:44:57 +08:00
|
|
|
|
|
|
|
// Replace operands
|
2015-12-24 13:20:40 +08:00
|
|
|
for (const auto &OpPair : Ops) {
|
|
|
|
MachineOperand &MO = OpPair.first->getOperand(OpPair.second);
|
2011-03-30 01:47:02 +08:00
|
|
|
if (MO.isReg() && MO.isUse() && MO.getReg() == VirtReg.reg) {
|
2013-08-15 07:50:16 +08:00
|
|
|
MO.setReg(NewVReg);
|
2010-07-03 01:44:57 +08:00
|
|
|
MO.setIsKill();
|
|
|
|
}
|
|
|
|
}
|
2016-02-28 04:23:14 +08:00
|
|
|
DEBUG(dbgs() << "\t " << UseIdx << '\t' << MI << '\n');
|
2010-07-03 01:44:57 +08:00
|
|
|
|
2011-05-06 01:22:53 +08:00
|
|
|
++NumRemats;
|
2010-07-01 07:03:52 +08:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2010-10-15 07:49:52 +08:00
|
|
|
/// reMaterializeAll - Try to rematerialize as many uses as possible,
|
2010-07-03 01:44:57 +08:00
|
|
|
/// and trim the live ranges after.
|
|
|
|
void InlineSpiller::reMaterializeAll() {
|
2012-04-03 06:22:53 +08:00
|
|
|
if (!Edit->anyRematerializable(AA))
|
2010-07-03 01:44:57 +08:00
|
|
|
return;
|
|
|
|
|
2011-03-15 03:56:43 +08:00
|
|
|
UsedValues.clear();
|
2010-10-21 06:00:51 +08:00
|
|
|
|
2011-03-29 11:12:02 +08:00
|
|
|
// Try to remat before all uses of snippets.
|
2010-07-03 01:44:57 +08:00
|
|
|
bool anyRemat = false;
|
2015-12-24 13:20:40 +08:00
|
|
|
for (unsigned Reg : RegsToSpill) {
|
2011-03-29 11:12:02 +08:00
|
|
|
LiveInterval &LI = LIS.getInterval(Reg);
|
2014-09-01 19:04:07 +08:00
|
|
|
for (MachineRegisterInfo::reg_bundle_iterator
|
|
|
|
RegI = MRI.reg_bundle_begin(Reg), E = MRI.reg_bundle_end();
|
|
|
|
RegI != E; ) {
|
2016-02-28 04:23:14 +08:00
|
|
|
MachineInstr &MI = *RegI++;
|
2014-09-01 19:04:07 +08:00
|
|
|
|
|
|
|
// Debug values are not allowed to affect codegen.
|
2016-02-28 04:23:14 +08:00
|
|
|
if (MI.isDebugValue())
|
2014-09-01 19:04:07 +08:00
|
|
|
continue;
|
|
|
|
|
2011-03-29 11:12:02 +08:00
|
|
|
anyRemat |= reMaterializeFor(LI, MI);
|
2014-03-13 14:02:25 +08:00
|
|
|
}
|
2011-03-29 11:12:02 +08:00
|
|
|
}
|
2010-07-03 01:44:57 +08:00
|
|
|
if (!anyRemat)
|
|
|
|
return;
|
|
|
|
|
|
|
|
// Remove any values that were completely rematted.
|
2015-12-24 13:20:40 +08:00
|
|
|
for (unsigned Reg : RegsToSpill) {
|
2011-03-29 11:12:02 +08:00
|
|
|
LiveInterval &LI = LIS.getInterval(Reg);
|
|
|
|
for (LiveInterval::vni_iterator I = LI.vni_begin(), E = LI.vni_end();
|
|
|
|
I != E; ++I) {
|
|
|
|
VNInfo *VNI = *I;
|
2011-03-30 01:47:00 +08:00
|
|
|
if (VNI->isUnused() || VNI->isPHIDef() || UsedValues.count(VNI))
|
2011-03-29 11:12:02 +08:00
|
|
|
continue;
|
|
|
|
MachineInstr *MI = LIS.getInstructionFromIndex(VNI->def);
|
|
|
|
MI->addRegisterDead(Reg, &TRI);
|
|
|
|
if (!MI->allDefsAreDead())
|
2010-07-03 03:54:40 +08:00
|
|
|
continue;
|
2011-03-29 11:12:02 +08:00
|
|
|
DEBUG(dbgs() << "All defs dead: " << *MI);
|
|
|
|
DeadDefs.push_back(MI);
|
2010-07-03 03:54:40 +08:00
|
|
|
}
|
2010-07-03 01:44:57 +08:00
|
|
|
}
|
2011-03-30 01:47:00 +08:00
|
|
|
|
|
|
|
// Eliminate dead code after remat. Note that some snippet copies may be
|
|
|
|
// deleted here.
|
|
|
|
if (DeadDefs.empty())
|
|
|
|
return;
|
|
|
|
DEBUG(dbgs() << "Remat created " << DeadDefs.size() << " dead defs.\n");
|
2016-07-09 05:08:09 +08:00
|
|
|
Edit->eliminateDeadDefs(DeadDefs, RegsToSpill, AA);
|
2011-03-30 01:47:00 +08:00
|
|
|
|
2016-02-06 02:14:24 +08:00
|
|
|
// LiveRangeEdit::eliminateDeadDef is used to remove dead define instructions
|
|
|
|
// after rematerialization. To remove a VNI for a vreg from its LiveInterval,
|
|
|
|
// LiveIntervals::removeVRegDefAt is used. However, after non-PHI VNIs are all
|
|
|
|
// removed, PHI VNI are still left in the LiveInterval.
|
|
|
|
// So to get rid of unused reg, we need to check whether it has non-dbg
|
|
|
|
// reference instead of whether it has non-empty interval.
|
2013-05-05 19:29:14 +08:00
|
|
|
unsigned ResultPos = 0;
|
2015-12-24 13:20:40 +08:00
|
|
|
for (unsigned Reg : RegsToSpill) {
|
2016-02-06 02:14:24 +08:00
|
|
|
if (MRI.reg_nodbg_empty(Reg)) {
|
2013-05-05 19:29:14 +08:00
|
|
|
Edit->eraseVirtReg(Reg);
|
2011-03-30 01:47:00 +08:00
|
|
|
continue;
|
2013-05-05 19:29:14 +08:00
|
|
|
}
|
2017-07-22 08:24:01 +08:00
|
|
|
|
2017-07-25 02:07:55 +08:00
|
|
|
assert(LIS.hasInterval(Reg) &&
|
|
|
|
(!LIS.getInterval(Reg).empty() || !MRI.reg_nodbg_empty(Reg)) &&
|
|
|
|
"Empty and not used live-range?!");
|
|
|
|
|
2013-05-05 19:29:14 +08:00
|
|
|
RegsToSpill[ResultPos++] = Reg;
|
2011-03-30 01:47:00 +08:00
|
|
|
}
|
2013-05-05 19:29:14 +08:00
|
|
|
RegsToSpill.erase(RegsToSpill.begin() + ResultPos, RegsToSpill.end());
|
2011-03-30 01:47:00 +08:00
|
|
|
DEBUG(dbgs() << RegsToSpill.size() << " registers to spill after remat.\n");
|
2010-07-03 01:44:57 +08:00
|
|
|
}
|
|
|
|
|
2011-03-30 05:20:19 +08:00
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
// Spilling
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
2011-03-15 03:56:43 +08:00
|
|
|
/// If MI is a load or store of StackSlot, it can be removed.
|
2011-03-12 12:17:20 +08:00
|
|
|
bool InlineSpiller::coalesceStackAccess(MachineInstr *MI, unsigned Reg) {
|
2010-08-05 06:35:11 +08:00
|
|
|
int FI = 0;
|
2016-06-30 08:01:54 +08:00
|
|
|
unsigned InstrReg = TII.isLoadFromStackSlot(*MI, FI);
|
2011-09-16 01:54:28 +08:00
|
|
|
bool IsLoad = InstrReg;
|
|
|
|
if (!IsLoad)
|
2016-06-30 08:01:54 +08:00
|
|
|
InstrReg = TII.isStoreToStackSlot(*MI, FI);
|
2010-08-05 06:35:11 +08:00
|
|
|
|
|
|
|
// We have a stack access. Is it the right register and slot?
|
2011-03-15 03:56:43 +08:00
|
|
|
if (InstrReg != Reg || FI != StackSlot)
|
2010-08-05 06:35:11 +08:00
|
|
|
return false;
|
|
|
|
|
2016-04-13 11:08:27 +08:00
|
|
|
if (!IsLoad)
|
2016-07-01 07:28:15 +08:00
|
|
|
HSpiller.rmFromMergeableSpills(*MI, StackSlot);
|
2016-04-13 11:08:27 +08:00
|
|
|
|
2010-08-05 06:35:11 +08:00
|
|
|
DEBUG(dbgs() << "Coalescing stack access: " << *MI);
|
2016-02-27 14:40:41 +08:00
|
|
|
LIS.RemoveMachineInstrFromMaps(*MI);
|
2010-08-05 06:35:11 +08:00
|
|
|
MI->eraseFromParent();
|
2011-09-16 01:54:28 +08:00
|
|
|
|
|
|
|
if (IsLoad) {
|
|
|
|
++NumReloadsRemoved;
|
|
|
|
--NumReloads;
|
|
|
|
} else {
|
|
|
|
++NumSpillsRemoved;
|
|
|
|
--NumSpills;
|
|
|
|
}
|
|
|
|
|
2010-08-05 06:35:11 +08:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2017-10-15 22:32:27 +08:00
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
2017-03-28 12:14:25 +08:00
|
|
|
LLVM_DUMP_METHOD
|
2013-08-15 07:50:16 +08:00
|
|
|
// Dump the range of instructions from B to E with their slot indexes.
|
|
|
|
static void dumpMachineInstrRangeWithSlotIndex(MachineBasicBlock::iterator B,
|
|
|
|
MachineBasicBlock::iterator E,
|
|
|
|
LiveIntervals const &LIS,
|
|
|
|
const char *const header,
|
|
|
|
unsigned VReg =0) {
|
|
|
|
char NextLine = '\n';
|
|
|
|
char SlotIndent = '\t';
|
|
|
|
|
2014-03-02 20:27:27 +08:00
|
|
|
if (std::next(B) == E) {
|
2013-08-15 07:50:16 +08:00
|
|
|
NextLine = ' ';
|
|
|
|
SlotIndent = ' ';
|
|
|
|
}
|
|
|
|
|
|
|
|
dbgs() << '\t' << header << ": " << NextLine;
|
|
|
|
|
|
|
|
for (MachineBasicBlock::iterator I = B; I != E; ++I) {
|
2016-02-27 14:40:41 +08:00
|
|
|
SlotIndex Idx = LIS.getInstructionIndex(*I).getRegSlot();
|
2013-08-15 07:50:16 +08:00
|
|
|
|
|
|
|
// If a register was passed in and this instruction has it as a
|
|
|
|
// destination that is marked as an early clobber, print the
|
|
|
|
// early-clobber slot index.
|
|
|
|
if (VReg) {
|
|
|
|
MachineOperand *MO = I->findRegisterDefOperand(VReg);
|
|
|
|
if (MO && MO->isEarlyClobber())
|
|
|
|
Idx = Idx.getRegSlot(true);
|
|
|
|
}
|
|
|
|
|
|
|
|
dbgs() << SlotIndent << Idx << '\t' << *I;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2012-03-01 09:43:25 +08:00
|
|
|
/// foldMemoryOperand - Try folding stack slot references in Ops into their
|
|
|
|
/// instructions.
|
|
|
|
///
|
|
|
|
/// @param Ops Operand indices from analyzeVirtReg().
|
2010-12-18 11:04:14 +08:00
|
|
|
/// @param LoadMI Load instruction to use instead of stack slot when non-null.
|
2012-03-01 09:43:25 +08:00
|
|
|
/// @return True on success.
|
|
|
|
bool InlineSpiller::
|
2017-08-30 06:32:07 +08:00
|
|
|
foldMemoryOperand(ArrayRef<std::pair<MachineInstr *, unsigned>> Ops,
|
2012-03-01 09:43:25 +08:00
|
|
|
MachineInstr *LoadMI) {
|
|
|
|
if (Ops.empty())
|
|
|
|
return false;
|
|
|
|
// Don't attempt folding in bundles.
|
|
|
|
MachineInstr *MI = Ops.front().first;
|
|
|
|
if (Ops.back().first != MI || MI->isBundled())
|
|
|
|
return false;
|
|
|
|
|
2011-09-16 02:22:52 +08:00
|
|
|
bool WasCopy = MI->isCopy();
|
2011-11-10 08:17:03 +08:00
|
|
|
unsigned ImpReg = 0;
|
|
|
|
|
2016-11-24 02:33:49 +08:00
|
|
|
// Spill subregs if the target allows it.
|
|
|
|
// We always want to spill subregs for stackmap/patchpoint pseudos.
|
|
|
|
bool SpillSubRegs = TII.isSubregFoldable() ||
|
|
|
|
MI->getOpcode() == TargetOpcode::STATEPOINT ||
|
|
|
|
MI->getOpcode() == TargetOpcode::PATCHPOINT ||
|
|
|
|
MI->getOpcode() == TargetOpcode::STACKMAP;
|
2013-11-17 09:36:23 +08:00
|
|
|
|
2010-07-01 08:13:04 +08:00
|
|
|
// TargetInstrInfo::foldMemoryOperand only expects explicit, non-tied
|
|
|
|
// operands.
|
|
|
|
SmallVector<unsigned, 8> FoldOps;
|
2015-12-24 13:20:40 +08:00
|
|
|
for (const auto &OpPair : Ops) {
|
|
|
|
unsigned Idx = OpPair.second;
|
|
|
|
assert(MI == OpPair.first && "Instruction conflict during operand folding");
|
2010-07-01 08:13:04 +08:00
|
|
|
MachineOperand &MO = MI->getOperand(Idx);
|
2011-11-10 08:17:03 +08:00
|
|
|
if (MO.isImplicit()) {
|
|
|
|
ImpReg = MO.getReg();
|
2010-07-01 08:13:04 +08:00
|
|
|
continue;
|
2011-11-10 08:17:03 +08:00
|
|
|
}
|
2016-11-24 02:33:49 +08:00
|
|
|
|
2013-11-17 09:36:23 +08:00
|
|
|
if (!SpillSubRegs && MO.getSubReg())
|
2010-07-01 08:13:04 +08:00
|
|
|
return false;
|
2011-02-09 03:33:55 +08:00
|
|
|
// We cannot fold a load instruction into a def.
|
|
|
|
if (LoadMI && MO.isDef())
|
|
|
|
return false;
|
2010-07-01 08:13:04 +08:00
|
|
|
// Tied use operands should not be passed to foldMemoryOperand.
|
|
|
|
if (!MI->isRegTiedToDefOperand(Idx))
|
|
|
|
FoldOps.push_back(Idx);
|
|
|
|
}
|
|
|
|
|
2016-12-08 08:06:51 +08:00
|
|
|
// If we only have implicit uses, we won't be able to fold that.
|
|
|
|
// Moreover, TargetInstrInfo::foldMemoryOperand will assert if we try!
|
|
|
|
if (FoldOps.empty())
|
|
|
|
return false;
|
|
|
|
|
2013-08-15 07:50:16 +08:00
|
|
|
MachineInstrSpan MIS(MI);
|
|
|
|
|
2010-12-18 11:04:14 +08:00
|
|
|
MachineInstr *FoldMI =
|
2016-06-30 08:01:54 +08:00
|
|
|
LoadMI ? TII.foldMemoryOperand(*MI, FoldOps, *LoadMI, &LIS)
|
|
|
|
: TII.foldMemoryOperand(*MI, FoldOps, StackSlot, &LIS);
|
2010-07-01 08:13:04 +08:00
|
|
|
if (!FoldMI)
|
|
|
|
return false;
|
2013-06-22 02:33:26 +08:00
|
|
|
|
|
|
|
// Remove LIS for any dead defs in the original MI not in FoldMI.
|
2016-02-28 01:05:33 +08:00
|
|
|
for (MIBundleOperands MO(*MI); MO.isValid(); ++MO) {
|
2013-06-22 02:33:26 +08:00
|
|
|
if (!MO->isReg())
|
|
|
|
continue;
|
|
|
|
unsigned Reg = MO->getReg();
|
|
|
|
if (!Reg || TargetRegisterInfo::isVirtualRegister(Reg) ||
|
|
|
|
MRI.isReserved(Reg)) {
|
|
|
|
continue;
|
|
|
|
}
|
2014-01-07 15:31:10 +08:00
|
|
|
// Skip non-Defs, including undef uses and internal reads.
|
|
|
|
if (MO->isUse())
|
|
|
|
continue;
|
2013-06-22 02:33:26 +08:00
|
|
|
MIBundleOperands::PhysRegInfo RI =
|
2016-02-28 01:05:33 +08:00
|
|
|
MIBundleOperands(*FoldMI).analyzePhysReg(Reg, &TRI);
|
2015-12-12 03:42:09 +08:00
|
|
|
if (RI.FullyDefined)
|
2013-06-22 02:33:26 +08:00
|
|
|
continue;
|
|
|
|
// FoldMI does not define this physreg. Remove the LI segment.
|
|
|
|
assert(MO->isDead() && "Cannot fold physreg def");
|
2016-02-27 14:40:41 +08:00
|
|
|
SlotIndex Idx = LIS.getInstructionIndex(*MI).getRegSlot();
|
2015-01-22 02:50:21 +08:00
|
|
|
LIS.removePhysRegDefAt(Reg, Idx);
|
2013-06-22 02:33:26 +08:00
|
|
|
}
|
2013-08-15 07:50:16 +08:00
|
|
|
|
2016-04-13 11:08:27 +08:00
|
|
|
int FI;
|
2016-07-01 07:28:15 +08:00
|
|
|
if (TII.isStoreToStackSlot(*MI, FI) &&
|
|
|
|
HSpiller.rmFromMergeableSpills(*MI, FI))
|
2016-04-13 11:08:27 +08:00
|
|
|
--NumSpills;
|
2016-02-27 14:40:41 +08:00
|
|
|
LIS.ReplaceMachineInstrInMaps(*MI, *FoldMI);
|
2010-07-10 01:29:08 +08:00
|
|
|
MI->eraseFromParent();
|
2011-11-10 08:17:03 +08:00
|
|
|
|
2013-08-15 07:50:16 +08:00
|
|
|
// Insert any new instructions other than FoldMI into the LIS maps.
|
|
|
|
assert(!MIS.empty() && "Unexpected empty span of instructions!");
|
2015-12-24 13:20:40 +08:00
|
|
|
for (MachineInstr &MI : MIS)
|
|
|
|
if (&MI != FoldMI)
|
2016-02-27 14:40:41 +08:00
|
|
|
LIS.InsertMachineInstrInMaps(MI);
|
2013-08-15 07:50:16 +08:00
|
|
|
|
2011-11-10 08:17:03 +08:00
|
|
|
// TII.foldMemoryOperand may have left some implicit operands on the
|
|
|
|
// instruction. Strip them.
|
|
|
|
if (ImpReg)
|
|
|
|
for (unsigned i = FoldMI->getNumOperands(); i; --i) {
|
|
|
|
MachineOperand &MO = FoldMI->getOperand(i - 1);
|
|
|
|
if (!MO.isReg() || !MO.isImplicit())
|
|
|
|
break;
|
|
|
|
if (MO.getReg() == ImpReg)
|
|
|
|
FoldMI->RemoveOperand(i - 1);
|
|
|
|
}
|
|
|
|
|
2013-08-15 07:50:16 +08:00
|
|
|
DEBUG(dumpMachineInstrRangeWithSlotIndex(MIS.begin(), MIS.end(), LIS,
|
|
|
|
"folded"));
|
|
|
|
|
2011-09-16 02:22:52 +08:00
|
|
|
if (!WasCopy)
|
|
|
|
++NumFolded;
|
2016-04-13 11:08:27 +08:00
|
|
|
else if (Ops.front().second == 0) {
|
2011-09-16 02:22:52 +08:00
|
|
|
++NumSpills;
|
2016-07-01 07:28:15 +08:00
|
|
|
HSpiller.addToMergeableSpills(*FoldMI, StackSlot, Original);
|
2016-04-13 11:08:27 +08:00
|
|
|
} else
|
2011-09-16 02:22:52 +08:00
|
|
|
++NumReloads;
|
2010-07-01 08:13:04 +08:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2013-08-15 07:50:16 +08:00
|
|
|
void InlineSpiller::insertReload(unsigned NewVReg,
|
2011-04-19 04:23:27 +08:00
|
|
|
SlotIndex Idx,
|
2010-07-01 07:03:52 +08:00
|
|
|
MachineBasicBlock::iterator MI) {
|
|
|
|
MachineBasicBlock &MBB = *MI->getParent();
|
2013-08-15 07:50:16 +08:00
|
|
|
|
|
|
|
MachineInstrSpan MIS(MI);
|
|
|
|
TII.loadRegFromStackSlot(MBB, MI, NewVReg, StackSlot,
|
|
|
|
MRI.getRegClass(NewVReg), &TRI);
|
|
|
|
|
|
|
|
LIS.InsertMachineInstrRangeInMaps(MIS.begin(), MI);
|
|
|
|
|
|
|
|
DEBUG(dumpMachineInstrRangeWithSlotIndex(MIS.begin(), MI, LIS, "reload",
|
|
|
|
NewVReg));
|
2011-05-06 01:22:53 +08:00
|
|
|
++NumReloads;
|
2010-07-01 07:03:52 +08:00
|
|
|
}
|
|
|
|
|
2017-06-06 07:51:27 +08:00
|
|
|
/// Check if \p Def fully defines a VReg with an undefined value.
|
|
|
|
/// If that's the case, that means the value of VReg is actually
|
|
|
|
/// not relevant.
|
|
|
|
static bool isFullUndefDef(const MachineInstr &Def) {
|
|
|
|
if (!Def.isImplicitDef())
|
|
|
|
return false;
|
|
|
|
assert(Def.getNumOperands() == 1 &&
|
|
|
|
"Implicit def with more than one definition");
|
|
|
|
// We can say that the VReg defined by Def is undef, only if it is
|
|
|
|
// fully defined by Def. Otherwise, some of the lanes may not be
|
|
|
|
// undef and the value of the VReg matters.
|
|
|
|
return !Def.getOperand(0).getSubReg();
|
|
|
|
}
|
|
|
|
|
2013-08-15 07:50:16 +08:00
|
|
|
/// insertSpill - Insert a spill of NewVReg after MI.
|
|
|
|
void InlineSpiller::insertSpill(unsigned NewVReg, bool isKill,
|
|
|
|
MachineBasicBlock::iterator MI) {
|
2010-07-01 07:03:52 +08:00
|
|
|
MachineBasicBlock &MBB = *MI->getParent();
|
2013-08-15 07:50:16 +08:00
|
|
|
|
|
|
|
MachineInstrSpan MIS(MI);
|
2017-06-07 08:22:07 +08:00
|
|
|
bool IsRealSpill = true;
|
|
|
|
if (isFullUndefDef(*MI)) {
|
2017-06-06 07:51:27 +08:00
|
|
|
// Don't spill undef value.
|
|
|
|
// Anything works for undef, in particular keeping the memory
|
|
|
|
// uninitialized is a viable option and it saves code size and
|
|
|
|
// run time.
|
|
|
|
BuildMI(MBB, std::next(MI), MI->getDebugLoc(), TII.get(TargetOpcode::KILL))
|
|
|
|
.addReg(NewVReg, getKillRegState(isKill));
|
2017-06-07 08:22:07 +08:00
|
|
|
IsRealSpill = false;
|
|
|
|
} else
|
2017-06-06 07:51:27 +08:00
|
|
|
TII.storeRegToStackSlot(MBB, std::next(MI), NewVReg, isKill, StackSlot,
|
|
|
|
MRI.getRegClass(NewVReg), &TRI);
|
2013-08-15 07:50:16 +08:00
|
|
|
|
2014-03-02 20:27:27 +08:00
|
|
|
LIS.InsertMachineInstrRangeInMaps(std::next(MI), MIS.end());
|
2013-08-15 07:50:16 +08:00
|
|
|
|
2014-03-02 20:27:27 +08:00
|
|
|
DEBUG(dumpMachineInstrRangeWithSlotIndex(std::next(MI), MIS.end(), LIS,
|
2013-08-15 07:50:16 +08:00
|
|
|
"spill"));
|
2011-05-06 01:22:53 +08:00
|
|
|
++NumSpills;
|
2017-06-07 08:22:07 +08:00
|
|
|
if (IsRealSpill)
|
|
|
|
HSpiller.addToMergeableSpills(*std::next(MI), StackSlot, Original);
|
2010-07-01 07:03:52 +08:00
|
|
|
}
|
|
|
|
|
2011-03-12 12:17:20 +08:00
|
|
|
/// spillAroundUses - insert spill code around each use of Reg.
|
|
|
|
void InlineSpiller::spillAroundUses(unsigned Reg) {
|
2017-11-28 20:42:37 +08:00
|
|
|
DEBUG(dbgs() << "spillAroundUses " << printReg(Reg) << '\n');
|
2011-03-15 03:56:43 +08:00
|
|
|
LiveInterval &OldLI = LIS.getInterval(Reg);
|
2010-07-03 01:44:57 +08:00
|
|
|
|
2011-03-12 12:17:20 +08:00
|
|
|
// Iterate over instructions using Reg.
|
2014-03-13 14:02:25 +08:00
|
|
|
for (MachineRegisterInfo::reg_bundle_iterator
|
|
|
|
RegI = MRI.reg_bundle_begin(Reg), E = MRI.reg_bundle_end();
|
|
|
|
RegI != E; ) {
|
2014-03-14 13:02:18 +08:00
|
|
|
MachineInstr *MI = &*(RegI++);
|
2010-06-30 07:58:39 +08:00
|
|
|
|
2010-07-03 03:54:40 +08:00
|
|
|
// Debug values are not allowed to affect codegen.
|
|
|
|
if (MI->isDebugValue()) {
|
|
|
|
// Modify DBG_VALUE now that the value is in a spill slot.
|
2013-06-17 04:34:15 +08:00
|
|
|
MachineBasicBlock *MBB = MI->getParent();
|
2017-04-18 09:21:53 +08:00
|
|
|
DEBUG(dbgs() << "Modifying debug info due to spill:\t" << *MI);
|
|
|
|
buildDbgValueForSpill(*MBB, MI, *MI, StackSlot);
|
|
|
|
MBB->erase(MI);
|
2010-07-03 03:54:40 +08:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2011-03-12 12:17:20 +08:00
|
|
|
// Ignore copies to/from snippets. We'll delete them.
|
|
|
|
if (SnippetCopies.count(MI))
|
|
|
|
continue;
|
|
|
|
|
2010-08-05 06:35:11 +08:00
|
|
|
// Stack slot accesses may coalesce away.
|
2011-03-12 12:17:20 +08:00
|
|
|
if (coalesceStackAccess(MI, Reg))
|
2010-08-05 06:35:11 +08:00
|
|
|
continue;
|
|
|
|
|
2010-06-30 07:58:39 +08:00
|
|
|
// Analyze instruction.
|
2012-03-01 09:43:25 +08:00
|
|
|
SmallVector<std::pair<MachineInstr*, unsigned>, 8> Ops;
|
2012-09-12 18:03:31 +08:00
|
|
|
MIBundleOperands::VirtRegInfo RI =
|
2016-02-28 01:05:33 +08:00
|
|
|
MIBundleOperands(*MI).analyzeVirtReg(Reg, &Ops);
|
2010-06-30 07:58:39 +08:00
|
|
|
|
2011-04-19 04:23:27 +08:00
|
|
|
// Find the slot index where this instruction reads and writes OldLI.
|
|
|
|
// This is usually the def slot, except for tied early clobbers.
|
2016-02-27 14:40:41 +08:00
|
|
|
SlotIndex Idx = LIS.getInstructionIndex(*MI).getRegSlot();
|
2011-11-14 04:45:27 +08:00
|
|
|
if (VNInfo *VNI = OldLI.getVNInfoAt(Idx.getRegSlot(true)))
|
2011-04-19 04:23:27 +08:00
|
|
|
if (SlotIndex::isSameInstr(Idx, VNI->def))
|
|
|
|
Idx = VNI->def;
|
|
|
|
|
2011-03-18 12:23:06 +08:00
|
|
|
// Check for a sibling copy.
|
2016-06-30 08:01:54 +08:00
|
|
|
unsigned SibReg = isFullCopyOf(*MI, Reg);
|
2011-03-20 13:44:58 +08:00
|
|
|
if (SibReg && isSibling(SibReg)) {
|
2011-05-12 02:25:10 +08:00
|
|
|
// This may actually be a copy between snippets.
|
|
|
|
if (isRegToSpill(SibReg)) {
|
|
|
|
DEBUG(dbgs() << "Found new snippet copy: " << *MI);
|
|
|
|
SnippetCopies.insert(MI);
|
|
|
|
continue;
|
|
|
|
}
|
2012-03-01 09:43:25 +08:00
|
|
|
if (RI.Writes) {
|
2016-04-13 11:08:27 +08:00
|
|
|
if (hoistSpillInsideBB(OldLI, *MI)) {
|
2011-03-20 13:44:58 +08:00
|
|
|
// This COPY is now dead, the value is already in the stack slot.
|
|
|
|
MI->getOperand(0).setIsDead();
|
|
|
|
DeadDefs.push_back(MI);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// This is a reload for a sib-reg copy. Drop spills downstream.
|
|
|
|
LiveInterval &SibLI = LIS.getInterval(SibReg);
|
|
|
|
eliminateRedundantSpills(SibLI, SibLI.getVNInfoAt(Idx));
|
|
|
|
// The COPY will fold to a reload below.
|
|
|
|
}
|
2011-03-18 12:23:06 +08:00
|
|
|
}
|
|
|
|
|
2010-07-03 01:44:57 +08:00
|
|
|
// Attempt to fold memory ops.
|
2012-03-01 09:43:25 +08:00
|
|
|
if (foldMemoryOperand(Ops))
|
2010-07-03 01:44:57 +08:00
|
|
|
continue;
|
|
|
|
|
2013-08-15 07:50:16 +08:00
|
|
|
// Create a new virtual register for spill/fill.
|
2010-06-30 07:58:39 +08:00
|
|
|
// FIXME: Infer regclass from instruction alone.
|
2013-08-15 07:50:16 +08:00
|
|
|
unsigned NewVReg = Edit->createFrom(Reg);
|
2010-06-30 07:58:39 +08:00
|
|
|
|
2012-03-01 09:43:25 +08:00
|
|
|
if (RI.Reads)
|
2013-08-15 07:50:16 +08:00
|
|
|
insertReload(NewVReg, Idx, MI);
|
2010-06-30 07:58:39 +08:00
|
|
|
|
|
|
|
// Rewrite instruction operands.
|
|
|
|
bool hasLiveDef = false;
|
2015-12-24 13:20:40 +08:00
|
|
|
for (const auto &OpPair : Ops) {
|
|
|
|
MachineOperand &MO = OpPair.first->getOperand(OpPair.second);
|
2013-08-15 07:50:16 +08:00
|
|
|
MO.setReg(NewVReg);
|
2010-06-30 07:58:39 +08:00
|
|
|
if (MO.isUse()) {
|
2015-12-24 13:20:40 +08:00
|
|
|
if (!OpPair.first->isRegTiedToDefOperand(OpPair.second))
|
2010-06-30 07:58:39 +08:00
|
|
|
MO.setIsKill();
|
|
|
|
} else {
|
|
|
|
if (!MO.isDead())
|
|
|
|
hasLiveDef = true;
|
|
|
|
}
|
|
|
|
}
|
2013-08-15 07:50:16 +08:00
|
|
|
DEBUG(dbgs() << "\trewrite: " << Idx << '\t' << *MI << '\n');
|
2010-06-30 07:58:39 +08:00
|
|
|
|
|
|
|
// FIXME: Use a second vreg if instruction has no tied ops.
|
2013-08-15 07:50:16 +08:00
|
|
|
if (RI.Writes)
|
2012-03-01 09:43:25 +08:00
|
|
|
if (hasLiveDef)
|
2013-08-15 07:50:16 +08:00
|
|
|
insertSpill(NewVReg, true, MI);
|
2010-06-30 07:58:39 +08:00
|
|
|
}
|
|
|
|
}
|
2011-03-12 12:17:20 +08:00
|
|
|
|
2011-03-30 05:20:19 +08:00
|
|
|
/// spillAll - Spill all registers remaining after rematerialization.
|
|
|
|
void InlineSpiller::spillAll() {
|
2011-03-27 06:16:41 +08:00
|
|
|
// Update LiveStacks now that we are committed to spilling.
|
|
|
|
if (StackSlot == VirtRegMap::NO_STACK_SLOT) {
|
2011-03-16 05:13:25 +08:00
|
|
|
StackSlot = VRM.assignVirt2StackSlot(Original);
|
2011-03-27 06:16:41 +08:00
|
|
|
StackInt = &LSS.getOrCreateInterval(StackSlot, MRI.getRegClass(Original));
|
2012-02-04 13:20:49 +08:00
|
|
|
StackInt->getNextValue(SlotIndex(), LSS.getVNInfoAllocator());
|
2011-03-27 06:16:41 +08:00
|
|
|
} else
|
|
|
|
StackInt = &LSS.getInterval(StackSlot);
|
2011-03-12 12:17:20 +08:00
|
|
|
|
2011-03-30 05:20:19 +08:00
|
|
|
if (Original != Edit->getReg())
|
|
|
|
VRM.assignVirt2StackSlot(Edit->getReg(), StackSlot);
|
2011-03-12 12:17:20 +08:00
|
|
|
|
2011-03-27 06:16:41 +08:00
|
|
|
assert(StackInt->getNumValNums() == 1 && "Bad stack interval values");
|
2015-12-24 13:20:40 +08:00
|
|
|
for (unsigned Reg : RegsToSpill)
|
|
|
|
StackInt->MergeSegmentsInAsValue(LIS.getInterval(Reg),
|
2013-10-11 05:28:43 +08:00
|
|
|
StackInt->getValNumInfo(0));
|
2011-03-27 06:16:41 +08:00
|
|
|
DEBUG(dbgs() << "Merged spilled regs: " << *StackInt << '\n');
|
2011-03-12 12:17:20 +08:00
|
|
|
|
|
|
|
// Spill around uses of all RegsToSpill.
|
2015-12-24 13:20:40 +08:00
|
|
|
for (unsigned Reg : RegsToSpill)
|
|
|
|
spillAroundUses(Reg);
|
2011-03-12 12:17:20 +08:00
|
|
|
|
2011-03-18 12:23:06 +08:00
|
|
|
// Hoisted spills may cause dead code.
|
|
|
|
if (!DeadDefs.empty()) {
|
|
|
|
DEBUG(dbgs() << "Eliminating " << DeadDefs.size() << " dead defs\n");
|
2016-07-09 05:08:09 +08:00
|
|
|
Edit->eliminateDeadDefs(DeadDefs, RegsToSpill, AA);
|
2011-03-18 12:23:06 +08:00
|
|
|
}
|
|
|
|
|
2011-03-12 12:17:20 +08:00
|
|
|
// Finally delete the SnippetCopies.
|
2015-12-24 13:20:40 +08:00
|
|
|
for (unsigned Reg : RegsToSpill) {
|
2014-03-13 14:02:25 +08:00
|
|
|
for (MachineRegisterInfo::reg_instr_iterator
|
2015-12-24 13:20:40 +08:00
|
|
|
RI = MRI.reg_instr_begin(Reg), E = MRI.reg_instr_end();
|
2014-03-13 14:02:25 +08:00
|
|
|
RI != E; ) {
|
2016-02-27 14:40:41 +08:00
|
|
|
MachineInstr &MI = *(RI++);
|
|
|
|
assert(SnippetCopies.count(&MI) && "Remaining use wasn't a snippet copy");
|
2011-05-12 02:25:10 +08:00
|
|
|
// FIXME: Do this with a LiveRangeEdit callback.
|
|
|
|
LIS.RemoveMachineInstrFromMaps(MI);
|
2016-02-27 14:40:41 +08:00
|
|
|
MI.eraseFromParent();
|
2011-05-12 02:25:10 +08:00
|
|
|
}
|
2011-03-12 12:17:20 +08:00
|
|
|
}
|
|
|
|
|
2011-03-30 05:20:19 +08:00
|
|
|
// Delete all spilled registers.
|
2015-12-24 13:20:40 +08:00
|
|
|
for (unsigned Reg : RegsToSpill)
|
|
|
|
Edit->eraseVirtReg(Reg);
|
2011-03-30 05:20:19 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void InlineSpiller::spill(LiveRangeEdit &edit) {
|
2011-05-06 01:22:53 +08:00
|
|
|
++NumSpilledRanges;
|
2011-03-30 05:20:19 +08:00
|
|
|
Edit = &edit;
|
|
|
|
assert(!TargetRegisterInfo::isStackSlot(edit.getReg())
|
|
|
|
&& "Trying to spill a stack slot.");
|
|
|
|
// Share a stack slot among all descendants of Original.
|
|
|
|
Original = VRM.getOriginal(edit.getReg());
|
|
|
|
StackSlot = VRM.getStackSlot(Original);
|
2014-04-14 08:51:57 +08:00
|
|
|
StackInt = nullptr;
|
2011-03-30 05:20:19 +08:00
|
|
|
|
|
|
|
DEBUG(dbgs() << "Inline spilling "
|
2014-11-17 13:50:14 +08:00
|
|
|
<< TRI.getRegClassName(MRI.getRegClass(edit.getReg()))
|
2013-10-11 05:29:05 +08:00
|
|
|
<< ':' << edit.getParent()
|
2017-11-28 20:42:37 +08:00
|
|
|
<< "\nFrom original " << printReg(Original) << '\n');
|
2011-03-30 05:20:19 +08:00
|
|
|
assert(edit.getParent().isSpillable() &&
|
|
|
|
"Attempting to spill already spilled value.");
|
|
|
|
assert(DeadDefs.empty() && "Previous spill didn't remove dead defs");
|
|
|
|
|
|
|
|
collectRegsToSpill();
|
|
|
|
reMaterializeAll();
|
|
|
|
|
|
|
|
// Remat may handle everything.
|
|
|
|
if (!RegsToSpill.empty())
|
|
|
|
spillAll();
|
|
|
|
|
2013-06-18 03:00:36 +08:00
|
|
|
Edit->calculateRegClassAndHint(MF, Loops, MBFI);
|
2011-03-12 12:17:20 +08:00
|
|
|
}
|
2016-04-13 11:08:27 +08:00
|
|
|
|
|
|
|
/// Optimizations after all the reg selections and spills are done.
|
2016-04-16 07:16:44 +08:00
|
|
|
void InlineSpiller::postOptimization() { HSpiller.hoistAllSpills(); }
|
2016-04-13 11:08:27 +08:00
|
|
|
|
|
|
|
/// When a spill is inserted, add the spill to MergeableSpills map.
|
2016-07-01 07:28:15 +08:00
|
|
|
void HoistSpillHelper::addToMergeableSpills(MachineInstr &Spill, int StackSlot,
|
2016-04-13 11:08:27 +08:00
|
|
|
unsigned Original) {
|
2017-09-14 05:41:30 +08:00
|
|
|
BumpPtrAllocator &Allocator = LIS.getVNInfoAllocator();
|
|
|
|
LiveInterval &OrigLI = LIS.getInterval(Original);
|
|
|
|
// save a copy of LiveInterval in StackSlotToOrigLI because the original
|
|
|
|
// LiveInterval may be cleared after all its references are spilled.
|
|
|
|
if (StackSlotToOrigLI.find(StackSlot) == StackSlotToOrigLI.end()) {
|
|
|
|
auto LI = llvm::make_unique<LiveInterval>(OrigLI.reg, OrigLI.weight);
|
|
|
|
LI->assign(OrigLI, Allocator);
|
|
|
|
StackSlotToOrigLI[StackSlot] = std::move(LI);
|
|
|
|
}
|
2016-07-01 07:28:15 +08:00
|
|
|
SlotIndex Idx = LIS.getInstructionIndex(Spill);
|
2017-09-14 05:41:30 +08:00
|
|
|
VNInfo *OrigVNI = StackSlotToOrigLI[StackSlot]->getVNInfoAt(Idx.getRegSlot());
|
2016-04-13 11:08:27 +08:00
|
|
|
std::pair<int, VNInfo *> MIdx = std::make_pair(StackSlot, OrigVNI);
|
2016-07-01 07:28:15 +08:00
|
|
|
MergeableSpills[MIdx].insert(&Spill);
|
2016-04-13 11:08:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/// When a spill is removed, remove the spill from MergeableSpills map.
|
|
|
|
/// Return true if the spill is removed successfully.
|
2016-07-01 07:28:15 +08:00
|
|
|
bool HoistSpillHelper::rmFromMergeableSpills(MachineInstr &Spill,
|
2016-04-13 11:08:27 +08:00
|
|
|
int StackSlot) {
|
2017-09-14 05:41:30 +08:00
|
|
|
auto It = StackSlotToOrigLI.find(StackSlot);
|
|
|
|
if (It == StackSlotToOrigLI.end())
|
2016-04-13 11:08:27 +08:00
|
|
|
return false;
|
2016-07-01 07:28:15 +08:00
|
|
|
SlotIndex Idx = LIS.getInstructionIndex(Spill);
|
2017-09-14 05:41:30 +08:00
|
|
|
VNInfo *OrigVNI = It->second->getVNInfoAt(Idx.getRegSlot());
|
2016-04-13 11:08:27 +08:00
|
|
|
std::pair<int, VNInfo *> MIdx = std::make_pair(StackSlot, OrigVNI);
|
2016-07-01 07:28:15 +08:00
|
|
|
return MergeableSpills[MIdx].erase(&Spill);
|
2016-04-13 11:08:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Check BB to see if it is a possible target BB to place a hoisted spill,
|
|
|
|
/// i.e., there should be a living sibling of OrigReg at the insert point.
|
2017-09-14 05:41:30 +08:00
|
|
|
bool HoistSpillHelper::isSpillCandBB(LiveInterval &OrigLI, VNInfo &OrigVNI,
|
2016-04-13 11:08:27 +08:00
|
|
|
MachineBasicBlock &BB, unsigned &LiveReg) {
|
|
|
|
SlotIndex Idx;
|
2017-09-14 05:41:30 +08:00
|
|
|
unsigned OrigReg = OrigLI.reg;
|
2016-05-24 03:39:19 +08:00
|
|
|
MachineBasicBlock::iterator MI = IPA.getLastInsertPointIter(OrigLI, BB);
|
2016-04-13 11:08:27 +08:00
|
|
|
if (MI != BB.end())
|
|
|
|
Idx = LIS.getInstructionIndex(*MI);
|
|
|
|
else
|
|
|
|
Idx = LIS.getMBBEndIdx(&BB).getPrevSlot();
|
|
|
|
SmallSetVector<unsigned, 16> &Siblings = Virt2SiblingsMap[OrigReg];
|
2017-09-14 05:41:30 +08:00
|
|
|
assert(OrigLI.getVNInfoAt(Idx) == &OrigVNI && "Unexpected VNI");
|
2016-04-13 11:08:27 +08:00
|
|
|
|
|
|
|
for (auto const SibReg : Siblings) {
|
|
|
|
LiveInterval &LI = LIS.getInterval(SibReg);
|
|
|
|
VNInfo *VNI = LI.getVNInfoAt(Idx);
|
|
|
|
if (VNI) {
|
|
|
|
LiveReg = SibReg;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2016-05-05 05:45:36 +08:00
|
|
|
/// Remove redundant spills in the same BB. Save those redundant spills in
|
2016-04-13 11:08:27 +08:00
|
|
|
/// SpillsToRm, and save the spill to keep and its BB in SpillBBToSpill map.
|
|
|
|
void HoistSpillHelper::rmRedundantSpills(
|
|
|
|
SmallPtrSet<MachineInstr *, 16> &Spills,
|
|
|
|
SmallVectorImpl<MachineInstr *> &SpillsToRm,
|
|
|
|
DenseMap<MachineDomTreeNode *, MachineInstr *> &SpillBBToSpill) {
|
|
|
|
// For each spill saw, check SpillBBToSpill[] and see if its BB already has
|
|
|
|
// another spill inside. If a BB contains more than one spill, only keep the
|
|
|
|
// earlier spill with smaller SlotIndex.
|
|
|
|
for (const auto CurrentSpill : Spills) {
|
|
|
|
MachineBasicBlock *Block = CurrentSpill->getParent();
|
2017-01-04 17:41:56 +08:00
|
|
|
MachineDomTreeNode *Node = MDT.getBase().getNode(Block);
|
2016-04-13 11:08:27 +08:00
|
|
|
MachineInstr *PrevSpill = SpillBBToSpill[Node];
|
|
|
|
if (PrevSpill) {
|
|
|
|
SlotIndex PIdx = LIS.getInstructionIndex(*PrevSpill);
|
|
|
|
SlotIndex CIdx = LIS.getInstructionIndex(*CurrentSpill);
|
|
|
|
MachineInstr *SpillToRm = (CIdx > PIdx) ? CurrentSpill : PrevSpill;
|
|
|
|
MachineInstr *SpillToKeep = (CIdx > PIdx) ? PrevSpill : CurrentSpill;
|
|
|
|
SpillsToRm.push_back(SpillToRm);
|
2017-01-04 17:41:56 +08:00
|
|
|
SpillBBToSpill[MDT.getBase().getNode(Block)] = SpillToKeep;
|
2016-04-13 11:08:27 +08:00
|
|
|
} else {
|
2017-01-04 17:41:56 +08:00
|
|
|
SpillBBToSpill[MDT.getBase().getNode(Block)] = CurrentSpill;
|
2016-04-13 11:08:27 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
for (const auto SpillToRm : SpillsToRm)
|
|
|
|
Spills.erase(SpillToRm);
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Starting from \p Root find a top-down traversal order of the dominator
|
|
|
|
/// tree to visit all basic blocks containing the elements of \p Spills.
|
|
|
|
/// Redundant spills will be found and put into \p SpillsToRm at the same
|
|
|
|
/// time. \p SpillBBToSpill will be populated as part of the process and
|
|
|
|
/// maps a basic block to the first store occurring in the basic block.
|
|
|
|
/// \post SpillsToRm.union(Spills\@post) == Spills\@pre
|
|
|
|
void HoistSpillHelper::getVisitOrders(
|
|
|
|
MachineBasicBlock *Root, SmallPtrSet<MachineInstr *, 16> &Spills,
|
|
|
|
SmallVectorImpl<MachineDomTreeNode *> &Orders,
|
|
|
|
SmallVectorImpl<MachineInstr *> &SpillsToRm,
|
|
|
|
DenseMap<MachineDomTreeNode *, unsigned> &SpillsToKeep,
|
|
|
|
DenseMap<MachineDomTreeNode *, MachineInstr *> &SpillBBToSpill) {
|
|
|
|
// The set contains all the possible BB nodes to which we may hoist
|
|
|
|
// original spills.
|
|
|
|
SmallPtrSet<MachineDomTreeNode *, 8> WorkSet;
|
|
|
|
// Save the BB nodes on the path from the first BB node containing
|
2016-05-05 05:45:36 +08:00
|
|
|
// non-redundant spill to the Root node.
|
2016-04-13 11:08:27 +08:00
|
|
|
SmallPtrSet<MachineDomTreeNode *, 8> NodesOnPath;
|
|
|
|
// All the spills to be hoisted must originate from a single def instruction
|
|
|
|
// to the OrigReg. It means the def instruction should dominate all the spills
|
|
|
|
// to be hoisted. We choose the BB where the def instruction is located as
|
|
|
|
// the Root.
|
|
|
|
MachineDomTreeNode *RootIDomNode = MDT[Root]->getIDom();
|
|
|
|
// For every node on the dominator tree with spill, walk up on the dominator
|
|
|
|
// tree towards the Root node until it is reached. If there is other node
|
|
|
|
// containing spill in the middle of the path, the previous spill saw will
|
2016-05-05 05:45:36 +08:00
|
|
|
// be redundant and the node containing it will be removed. All the nodes on
|
|
|
|
// the path starting from the first node with non-redundant spill to the Root
|
2016-04-13 11:08:27 +08:00
|
|
|
// node will be added to the WorkSet, which will contain all the possible
|
|
|
|
// locations where spills may be hoisted to after the loop below is done.
|
|
|
|
for (const auto Spill : Spills) {
|
|
|
|
MachineBasicBlock *Block = Spill->getParent();
|
|
|
|
MachineDomTreeNode *Node = MDT[Block];
|
|
|
|
MachineInstr *SpillToRm = nullptr;
|
|
|
|
while (Node != RootIDomNode) {
|
|
|
|
// If Node dominates Block, and it already contains a spill, the spill in
|
2016-05-05 05:45:36 +08:00
|
|
|
// Block will be redundant.
|
2016-04-13 11:08:27 +08:00
|
|
|
if (Node != MDT[Block] && SpillBBToSpill[Node]) {
|
|
|
|
SpillToRm = SpillBBToSpill[MDT[Block]];
|
|
|
|
break;
|
|
|
|
/// If we see the Node already in WorkSet, the path from the Node to
|
|
|
|
/// the Root node must already be traversed by another spill.
|
|
|
|
/// Then no need to repeat.
|
|
|
|
} else if (WorkSet.count(Node)) {
|
|
|
|
break;
|
|
|
|
} else {
|
|
|
|
NodesOnPath.insert(Node);
|
|
|
|
}
|
|
|
|
Node = Node->getIDom();
|
|
|
|
}
|
|
|
|
if (SpillToRm) {
|
|
|
|
SpillsToRm.push_back(SpillToRm);
|
|
|
|
} else {
|
|
|
|
// Add a BB containing the original spills to SpillsToKeep -- i.e.,
|
|
|
|
// set the initial status before hoisting start. The value of BBs
|
|
|
|
// containing original spills is set to 0, in order to descriminate
|
|
|
|
// with BBs containing hoisted spills which will be inserted to
|
|
|
|
// SpillsToKeep later during hoisting.
|
|
|
|
SpillsToKeep[MDT[Block]] = 0;
|
|
|
|
WorkSet.insert(NodesOnPath.begin(), NodesOnPath.end());
|
|
|
|
}
|
|
|
|
NodesOnPath.clear();
|
|
|
|
}
|
|
|
|
|
|
|
|
// Sort the nodes in WorkSet in top-down order and save the nodes
|
|
|
|
// in Orders. Orders will be used for hoisting in runHoistSpills.
|
|
|
|
unsigned idx = 0;
|
2017-01-04 17:41:56 +08:00
|
|
|
Orders.push_back(MDT.getBase().getNode(Root));
|
2016-04-13 11:08:27 +08:00
|
|
|
do {
|
|
|
|
MachineDomTreeNode *Node = Orders[idx++];
|
|
|
|
const std::vector<MachineDomTreeNode *> &Children = Node->getChildren();
|
|
|
|
unsigned NumChildren = Children.size();
|
|
|
|
for (unsigned i = 0; i != NumChildren; ++i) {
|
|
|
|
MachineDomTreeNode *Child = Children[i];
|
|
|
|
if (WorkSet.count(Child))
|
|
|
|
Orders.push_back(Child);
|
|
|
|
}
|
|
|
|
} while (idx != Orders.size());
|
|
|
|
assert(Orders.size() == WorkSet.size() &&
|
|
|
|
"Orders have different size with WorkSet");
|
|
|
|
|
|
|
|
#ifndef NDEBUG
|
|
|
|
DEBUG(dbgs() << "Orders size is " << Orders.size() << "\n");
|
|
|
|
SmallVector<MachineDomTreeNode *, 32>::reverse_iterator RIt = Orders.rbegin();
|
|
|
|
for (; RIt != Orders.rend(); RIt++)
|
|
|
|
DEBUG(dbgs() << "BB" << (*RIt)->getBlock()->getNumber() << ",");
|
|
|
|
DEBUG(dbgs() << "\n");
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Try to hoist spills according to BB hotness. The spills to removed will
|
|
|
|
/// be saved in \p SpillsToRm. The spills to be inserted will be saved in
|
|
|
|
/// \p SpillsToIns.
|
|
|
|
void HoistSpillHelper::runHoistSpills(
|
2017-09-14 05:41:30 +08:00
|
|
|
LiveInterval &OrigLI, VNInfo &OrigVNI,
|
|
|
|
SmallPtrSet<MachineInstr *, 16> &Spills,
|
2016-04-13 11:08:27 +08:00
|
|
|
SmallVectorImpl<MachineInstr *> &SpillsToRm,
|
|
|
|
DenseMap<MachineBasicBlock *, unsigned> &SpillsToIns) {
|
|
|
|
// Visit order of dominator tree nodes.
|
|
|
|
SmallVector<MachineDomTreeNode *, 32> Orders;
|
|
|
|
// SpillsToKeep contains all the nodes where spills are to be inserted
|
|
|
|
// during hoisting. If the spill to be inserted is an original spill
|
|
|
|
// (not a hoisted one), the value of the map entry is 0. If the spill
|
|
|
|
// is a hoisted spill, the value of the map entry is the VReg to be used
|
|
|
|
// as the source of the spill.
|
|
|
|
DenseMap<MachineDomTreeNode *, unsigned> SpillsToKeep;
|
|
|
|
// Map from BB to the first spill inside of it.
|
|
|
|
DenseMap<MachineDomTreeNode *, MachineInstr *> SpillBBToSpill;
|
|
|
|
|
|
|
|
rmRedundantSpills(Spills, SpillsToRm, SpillBBToSpill);
|
|
|
|
|
|
|
|
MachineBasicBlock *Root = LIS.getMBBFromIndex(OrigVNI.def);
|
|
|
|
getVisitOrders(Root, Spills, Orders, SpillsToRm, SpillsToKeep,
|
|
|
|
SpillBBToSpill);
|
|
|
|
|
|
|
|
// SpillsInSubTreeMap keeps the map from a dom tree node to a pair of
|
|
|
|
// nodes set and the cost of all the spills inside those nodes.
|
|
|
|
// The nodes set are the locations where spills are to be inserted
|
|
|
|
// in the subtree of current node.
|
2017-08-30 06:32:07 +08:00
|
|
|
using NodesCostPair =
|
|
|
|
std::pair<SmallPtrSet<MachineDomTreeNode *, 16>, BlockFrequency>;
|
2016-04-13 11:08:27 +08:00
|
|
|
DenseMap<MachineDomTreeNode *, NodesCostPair> SpillsInSubTreeMap;
|
2017-08-30 06:32:07 +08:00
|
|
|
|
2016-04-13 11:08:27 +08:00
|
|
|
// Iterate Orders set in reverse order, which will be a bottom-up order
|
|
|
|
// in the dominator tree. Once we visit a dom tree node, we know its
|
|
|
|
// children have already been visited and the spill locations in the
|
|
|
|
// subtrees of all the children have been determined.
|
|
|
|
SmallVector<MachineDomTreeNode *, 32>::reverse_iterator RIt = Orders.rbegin();
|
|
|
|
for (; RIt != Orders.rend(); RIt++) {
|
|
|
|
MachineBasicBlock *Block = (*RIt)->getBlock();
|
|
|
|
|
|
|
|
// If Block contains an original spill, simply continue.
|
|
|
|
if (SpillsToKeep.find(*RIt) != SpillsToKeep.end() && !SpillsToKeep[*RIt]) {
|
|
|
|
SpillsInSubTreeMap[*RIt].first.insert(*RIt);
|
|
|
|
// SpillsInSubTreeMap[*RIt].second contains the cost of spill.
|
|
|
|
SpillsInSubTreeMap[*RIt].second = MBFI.getBlockFreq(Block);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Collect spills in subtree of current node (*RIt) to
|
|
|
|
// SpillsInSubTreeMap[*RIt].first.
|
|
|
|
const std::vector<MachineDomTreeNode *> &Children = (*RIt)->getChildren();
|
|
|
|
unsigned NumChildren = Children.size();
|
|
|
|
for (unsigned i = 0; i != NumChildren; ++i) {
|
|
|
|
MachineDomTreeNode *Child = Children[i];
|
|
|
|
if (SpillsInSubTreeMap.find(Child) == SpillsInSubTreeMap.end())
|
|
|
|
continue;
|
|
|
|
// The stmt "SpillsInSubTree = SpillsInSubTreeMap[*RIt].first" below
|
|
|
|
// should be placed before getting the begin and end iterators of
|
|
|
|
// SpillsInSubTreeMap[Child].first, or else the iterators may be
|
|
|
|
// invalidated when SpillsInSubTreeMap[*RIt] is seen the first time
|
|
|
|
// and the map grows and then the original buckets in the map are moved.
|
|
|
|
SmallPtrSet<MachineDomTreeNode *, 16> &SpillsInSubTree =
|
|
|
|
SpillsInSubTreeMap[*RIt].first;
|
|
|
|
BlockFrequency &SubTreeCost = SpillsInSubTreeMap[*RIt].second;
|
|
|
|
SubTreeCost += SpillsInSubTreeMap[Child].second;
|
|
|
|
auto BI = SpillsInSubTreeMap[Child].first.begin();
|
|
|
|
auto EI = SpillsInSubTreeMap[Child].first.end();
|
|
|
|
SpillsInSubTree.insert(BI, EI);
|
|
|
|
SpillsInSubTreeMap.erase(Child);
|
|
|
|
}
|
|
|
|
|
|
|
|
SmallPtrSet<MachineDomTreeNode *, 16> &SpillsInSubTree =
|
|
|
|
SpillsInSubTreeMap[*RIt].first;
|
|
|
|
BlockFrequency &SubTreeCost = SpillsInSubTreeMap[*RIt].second;
|
|
|
|
// No spills in subtree, simply continue.
|
|
|
|
if (SpillsInSubTree.empty())
|
|
|
|
continue;
|
|
|
|
|
|
|
|
// Check whether Block is a possible candidate to insert spill.
|
|
|
|
unsigned LiveReg = 0;
|
2017-09-14 05:41:30 +08:00
|
|
|
if (!isSpillCandBB(OrigLI, OrigVNI, *Block, LiveReg))
|
2016-04-13 11:08:27 +08:00
|
|
|
continue;
|
|
|
|
|
|
|
|
// If there are multiple spills that could be merged, bias a little
|
|
|
|
// to hoist the spill.
|
|
|
|
BranchProbability MarginProb = (SpillsInSubTree.size() > 1)
|
|
|
|
? BranchProbability(9, 10)
|
|
|
|
: BranchProbability(1, 1);
|
|
|
|
if (SubTreeCost > MBFI.getBlockFreq(Block) * MarginProb) {
|
|
|
|
// Hoist: Move spills to current Block.
|
|
|
|
for (const auto SpillBB : SpillsInSubTree) {
|
|
|
|
// When SpillBB is a BB contains original spill, insert the spill
|
|
|
|
// to SpillsToRm.
|
|
|
|
if (SpillsToKeep.find(SpillBB) != SpillsToKeep.end() &&
|
|
|
|
!SpillsToKeep[SpillBB]) {
|
|
|
|
MachineInstr *SpillToRm = SpillBBToSpill[SpillBB];
|
|
|
|
SpillsToRm.push_back(SpillToRm);
|
|
|
|
}
|
|
|
|
// SpillBB will not contain spill anymore, remove it from SpillsToKeep.
|
|
|
|
SpillsToKeep.erase(SpillBB);
|
|
|
|
}
|
|
|
|
// Current Block is the BB containing the new hoisted spill. Add it to
|
|
|
|
// SpillsToKeep. LiveReg is the source of the new spill.
|
|
|
|
SpillsToKeep[*RIt] = LiveReg;
|
|
|
|
DEBUG({
|
|
|
|
dbgs() << "spills in BB: ";
|
|
|
|
for (const auto Rspill : SpillsInSubTree)
|
|
|
|
dbgs() << Rspill->getBlock()->getNumber() << " ";
|
|
|
|
dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber()
|
|
|
|
<< "\n";
|
|
|
|
});
|
|
|
|
SpillsInSubTree.clear();
|
|
|
|
SpillsInSubTree.insert(*RIt);
|
|
|
|
SubTreeCost = MBFI.getBlockFreq(Block);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// For spills in SpillsToKeep with LiveReg set (i.e., not original spill),
|
|
|
|
// save them to SpillsToIns.
|
|
|
|
for (const auto Ent : SpillsToKeep) {
|
|
|
|
if (Ent.second)
|
|
|
|
SpillsToIns[Ent.first->getBlock()] = Ent.second;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-05-05 05:45:36 +08:00
|
|
|
/// For spills with equal values, remove redundant spills and hoist those left
|
2016-04-13 11:08:27 +08:00
|
|
|
/// to less hot spots.
|
|
|
|
///
|
|
|
|
/// Spills with equal values will be collected into the same set in
|
|
|
|
/// MergeableSpills when spill is inserted. These equal spills are originated
|
2016-05-05 05:45:36 +08:00
|
|
|
/// from the same defining instruction and are dominated by the instruction.
|
|
|
|
/// Before hoisting all the equal spills, redundant spills inside in the same
|
|
|
|
/// BB are first marked to be deleted. Then starting from the spills left, walk
|
|
|
|
/// up on the dominator tree towards the Root node where the define instruction
|
2016-04-13 11:08:27 +08:00
|
|
|
/// is located, mark the dominated spills to be deleted along the way and
|
|
|
|
/// collect the BB nodes on the path from non-dominated spills to the define
|
|
|
|
/// instruction into a WorkSet. The nodes in WorkSet are the candidate places
|
2016-05-05 05:45:36 +08:00
|
|
|
/// where we are considering to hoist the spills. We iterate the WorkSet in
|
|
|
|
/// bottom-up order, and for each node, we will decide whether to hoist spills
|
|
|
|
/// inside its subtree to that node. In this way, we can get benefit locally
|
|
|
|
/// even if hoisting all the equal spills to one cold place is impossible.
|
2016-04-16 07:16:44 +08:00
|
|
|
void HoistSpillHelper::hoistAllSpills() {
|
|
|
|
SmallVector<unsigned, 4> NewVRegs;
|
|
|
|
LiveRangeEdit Edit(nullptr, NewVRegs, MF, LIS, &VRM, this);
|
|
|
|
|
2016-04-13 11:08:27 +08:00
|
|
|
for (unsigned i = 0, e = MRI.getNumVirtRegs(); i != e; ++i) {
|
|
|
|
unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
|
|
|
|
unsigned Original = VRM.getPreSplitReg(Reg);
|
|
|
|
if (!MRI.def_empty(Reg))
|
|
|
|
Virt2SiblingsMap[Original].insert(Reg);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Each entry in MergeableSpills contains a spill set with equal values.
|
|
|
|
for (auto &Ent : MergeableSpills) {
|
|
|
|
int Slot = Ent.first.first;
|
2017-09-14 05:41:30 +08:00
|
|
|
LiveInterval &OrigLI = *StackSlotToOrigLI[Slot];
|
2016-04-13 11:08:27 +08:00
|
|
|
VNInfo *OrigVNI = Ent.first.second;
|
|
|
|
SmallPtrSet<MachineInstr *, 16> &EqValSpills = Ent.second;
|
|
|
|
if (Ent.second.empty())
|
|
|
|
continue;
|
|
|
|
|
|
|
|
DEBUG({
|
|
|
|
dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n"
|
|
|
|
<< "Equal spills in BB: ";
|
|
|
|
for (const auto spill : EqValSpills)
|
|
|
|
dbgs() << spill->getParent()->getNumber() << " ";
|
|
|
|
dbgs() << "\n";
|
|
|
|
});
|
|
|
|
|
|
|
|
// SpillsToRm is the spill set to be removed from EqValSpills.
|
|
|
|
SmallVector<MachineInstr *, 16> SpillsToRm;
|
|
|
|
// SpillsToIns is the spill set to be newly inserted after hoisting.
|
|
|
|
DenseMap<MachineBasicBlock *, unsigned> SpillsToIns;
|
|
|
|
|
2017-09-14 05:41:30 +08:00
|
|
|
runHoistSpills(OrigLI, *OrigVNI, EqValSpills, SpillsToRm, SpillsToIns);
|
2016-04-13 11:08:27 +08:00
|
|
|
|
|
|
|
DEBUG({
|
|
|
|
dbgs() << "Finally inserted spills in BB: ";
|
|
|
|
for (const auto Ispill : SpillsToIns)
|
|
|
|
dbgs() << Ispill.first->getNumber() << " ";
|
|
|
|
dbgs() << "\nFinally removed spills in BB: ";
|
|
|
|
for (const auto Rspill : SpillsToRm)
|
|
|
|
dbgs() << Rspill->getParent()->getNumber() << " ";
|
|
|
|
dbgs() << "\n";
|
|
|
|
});
|
|
|
|
|
|
|
|
// Stack live range update.
|
|
|
|
LiveInterval &StackIntvl = LSS.getInterval(Slot);
|
2016-05-12 06:37:43 +08:00
|
|
|
if (!SpillsToIns.empty() || !SpillsToRm.empty())
|
2016-04-13 11:08:27 +08:00
|
|
|
StackIntvl.MergeValueInAsValue(OrigLI, OrigVNI,
|
|
|
|
StackIntvl.getValNumInfo(0));
|
|
|
|
|
|
|
|
// Insert hoisted spills.
|
|
|
|
for (auto const Insert : SpillsToIns) {
|
|
|
|
MachineBasicBlock *BB = Insert.first;
|
|
|
|
unsigned LiveReg = Insert.second;
|
2016-05-24 03:39:19 +08:00
|
|
|
MachineBasicBlock::iterator MI = IPA.getLastInsertPointIter(OrigLI, *BB);
|
2016-04-13 11:08:27 +08:00
|
|
|
TII.storeRegToStackSlot(*BB, MI, LiveReg, false, Slot,
|
|
|
|
MRI.getRegClass(LiveReg), &TRI);
|
|
|
|
LIS.InsertMachineInstrRangeInMaps(std::prev(MI), MI);
|
|
|
|
++NumSpills;
|
|
|
|
}
|
|
|
|
|
2016-05-05 05:45:36 +08:00
|
|
|
// Remove redundant spills or change them to dead instructions.
|
2016-04-13 11:08:27 +08:00
|
|
|
NumSpills -= SpillsToRm.size();
|
|
|
|
for (auto const RMEnt : SpillsToRm) {
|
|
|
|
RMEnt->setDesc(TII.get(TargetOpcode::KILL));
|
|
|
|
for (unsigned i = RMEnt->getNumOperands(); i; --i) {
|
|
|
|
MachineOperand &MO = RMEnt->getOperand(i - 1);
|
|
|
|
if (MO.isReg() && MO.isImplicit() && MO.isDef() && !MO.isDead())
|
|
|
|
RMEnt->RemoveOperand(i - 1);
|
|
|
|
}
|
|
|
|
}
|
2016-07-09 05:08:09 +08:00
|
|
|
Edit.eliminateDeadDefs(SpillsToRm, None, AA);
|
2016-04-13 11:08:27 +08:00
|
|
|
}
|
|
|
|
}
|
2016-04-16 07:16:44 +08:00
|
|
|
|
|
|
|
/// For VirtReg clone, the \p New register should have the same physreg or
|
|
|
|
/// stackslot as the \p old register.
|
|
|
|
void HoistSpillHelper::LRE_DidCloneVirtReg(unsigned New, unsigned Old) {
|
|
|
|
if (VRM.hasPhys(Old))
|
|
|
|
VRM.assignVirt2Phys(New, VRM.getPhys(Old));
|
|
|
|
else if (VRM.getStackSlot(Old) != VirtRegMap::NO_STACK_SLOT)
|
|
|
|
VRM.assignVirt2StackSlot(New, VRM.getStackSlot(Old));
|
|
|
|
else
|
|
|
|
llvm_unreachable("VReg should be assigned either physreg or stackslot");
|
|
|
|
}
|