2017-09-14 05:15:20 +08:00
|
|
|
//===- llvm/CodeGen/VirtRegMap.cpp - Virtual Register Map -----------------===//
|
2004-02-24 07:08:11 +08:00
|
|
|
//
|
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
//
|
2007-12-30 04:36:04 +08:00
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
// License. See LICENSE.TXT for details.
|
2004-02-24 07:08:11 +08:00
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
2004-09-30 09:54:45 +08:00
|
|
|
// This file implements the VirtRegMap class.
|
|
|
|
//
|
2010-02-11 00:03:48 +08:00
|
|
|
// It also contains implementations of the Spiller interface, which, given a
|
2004-09-30 09:54:45 +08:00
|
|
|
// virtual register map and a machine function, eliminates all virtual
|
|
|
|
// references by replacing them with physical register references - adding spill
|
2004-02-24 16:58:30 +08:00
|
|
|
// code as necessary.
|
2004-02-24 07:08:11 +08:00
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
2012-11-29 03:13:06 +08:00
|
|
|
#include "llvm/CodeGen/VirtRegMap.h"
|
2012-06-09 07:44:45 +08:00
|
|
|
#include "LiveDebugVariables.h"
|
2017-09-14 05:15:20 +08:00
|
|
|
#include "llvm/ADT/SmallVector.h"
|
2014-03-04 18:07:28 +08:00
|
|
|
#include "llvm/ADT/Statistic.h"
|
2017-09-14 05:15:20 +08:00
|
|
|
#include "llvm/CodeGen/LiveInterval.h"
|
2012-06-09 07:44:45 +08:00
|
|
|
#include "llvm/CodeGen/LiveIntervalAnalysis.h"
|
2012-09-22 04:04:28 +08:00
|
|
|
#include "llvm/CodeGen/LiveStackAnalysis.h"
|
2017-09-14 05:15:20 +08:00
|
|
|
#include "llvm/CodeGen/MachineBasicBlock.h"
|
2004-02-24 07:08:11 +08:00
|
|
|
#include "llvm/CodeGen/MachineFrameInfo.h"
|
2004-09-30 09:54:45 +08:00
|
|
|
#include "llvm/CodeGen/MachineFunction.h"
|
2017-09-14 05:15:20 +08:00
|
|
|
#include "llvm/CodeGen/MachineFunctionPass.h"
|
|
|
|
#include "llvm/CodeGen/MachineInstr.h"
|
|
|
|
#include "llvm/CodeGen/MachineOperand.h"
|
2007-12-31 12:13:23 +08:00
|
|
|
#include "llvm/CodeGen/MachineRegisterInfo.h"
|
2017-09-14 05:15:20 +08:00
|
|
|
#include "llvm/CodeGen/SlotIndexes.h"
|
|
|
|
#include "llvm/MC/LaneBitmask.h"
|
|
|
|
#include "llvm/Pass.h"
|
2006-08-27 20:54:02 +08:00
|
|
|
#include "llvm/Support/Compiler.h"
|
2009-02-11 16:24:21 +08:00
|
|
|
#include "llvm/Support/Debug.h"
|
2009-07-24 18:36:58 +08:00
|
|
|
#include "llvm/Support/raw_ostream.h"
|
2012-12-04 00:50:05 +08:00
|
|
|
#include "llvm/Target/TargetInstrInfo.h"
|
2017-09-14 05:15:20 +08:00
|
|
|
#include "llvm/Target/TargetOpcodes.h"
|
2012-12-04 00:50:05 +08:00
|
|
|
#include "llvm/Target/TargetRegisterInfo.h"
|
2014-08-05 05:25:23 +08:00
|
|
|
#include "llvm/Target/TargetSubtargetInfo.h"
|
2017-09-14 05:15:20 +08:00
|
|
|
#include <cassert>
|
|
|
|
#include <iterator>
|
|
|
|
#include <utility>
|
|
|
|
|
2004-02-24 07:08:11 +08:00
|
|
|
using namespace llvm;
|
|
|
|
|
2014-04-22 10:02:50 +08:00
|
|
|
#define DEBUG_TYPE "regalloc"
|
|
|
|
|
2011-09-16 02:31:13 +08:00
|
|
|
STATISTIC(NumSpillSlots, "Number of spill slots allocated");
|
|
|
|
STATISTIC(NumIdCopies, "Number of identity moves eliminated after rewriting");
|
2008-05-13 08:00:25 +08:00
|
|
|
|
2004-09-30 09:54:45 +08:00
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
// VirtRegMap implementation
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
2009-03-13 13:55:11 +08:00
|
|
|
char VirtRegMap::ID = 0;
|
|
|
|
|
2010-10-08 06:25:06 +08:00
|
|
|
INITIALIZE_PASS(VirtRegMap, "virtregmap", "Virtual Register Map", false, false)
|
2009-03-13 13:55:11 +08:00
|
|
|
|
|
|
|
bool VirtRegMap::runOnMachineFunction(MachineFunction &mf) {
|
2009-06-15 04:22:55 +08:00
|
|
|
MRI = &mf.getRegInfo();
|
2014-08-05 10:39:49 +08:00
|
|
|
TII = mf.getSubtarget().getInstrInfo();
|
|
|
|
TRI = mf.getSubtarget().getRegisterInfo();
|
2009-03-13 13:55:11 +08:00
|
|
|
MF = &mf;
|
2009-11-04 07:52:08 +08:00
|
|
|
|
2009-03-13 13:55:11 +08:00
|
|
|
Virt2PhysMap.clear();
|
|
|
|
Virt2StackSlotMap.clear();
|
|
|
|
Virt2SplitMap.clear();
|
2009-05-04 11:30:11 +08:00
|
|
|
|
2006-09-05 10:12:02 +08:00
|
|
|
grow();
|
2009-03-13 13:55:11 +08:00
|
|
|
return false;
|
2006-09-05 10:12:02 +08:00
|
|
|
}
|
|
|
|
|
2004-09-30 09:54:45 +08:00
|
|
|
void VirtRegMap::grow() {
|
2011-01-10 05:58:20 +08:00
|
|
|
unsigned NumRegs = MF->getRegInfo().getNumVirtRegs();
|
|
|
|
Virt2PhysMap.resize(NumRegs);
|
|
|
|
Virt2StackSlotMap.resize(NumRegs);
|
|
|
|
Virt2SplitMap.resize(NumRegs);
|
2004-02-24 07:08:11 +08:00
|
|
|
}
|
|
|
|
|
2017-06-09 05:30:54 +08:00
|
|
|
void VirtRegMap::assignVirt2Phys(unsigned virtReg, MCPhysReg physReg) {
|
|
|
|
assert(TargetRegisterInfo::isVirtualRegister(virtReg) &&
|
|
|
|
TargetRegisterInfo::isPhysicalRegister(physReg));
|
|
|
|
assert(Virt2PhysMap[virtReg] == NO_PHYS_REG &&
|
|
|
|
"attempt to assign physical register to already mapped "
|
|
|
|
"virtual register");
|
|
|
|
assert(!getRegInfo().isReserved(physReg) &&
|
|
|
|
"Attempt to map virtReg to a reserved physReg");
|
|
|
|
Virt2PhysMap[virtReg] = physReg;
|
|
|
|
}
|
|
|
|
|
2010-11-16 08:41:01 +08:00
|
|
|
unsigned VirtRegMap::createSpillSlot(const TargetRegisterClass *RC) {
|
2017-04-25 02:55:33 +08:00
|
|
|
unsigned Size = TRI->getSpillSize(*RC);
|
|
|
|
unsigned Align = TRI->getSpillAlignment(*RC);
|
|
|
|
int SS = MF->getFrameInfo().CreateSpillStackObject(Size, Align);
|
2011-09-16 02:31:13 +08:00
|
|
|
++NumSpillSlots;
|
2010-11-16 08:41:01 +08:00
|
|
|
return SS;
|
|
|
|
}
|
|
|
|
|
2012-12-04 08:30:22 +08:00
|
|
|
bool VirtRegMap::hasPreferredPhys(unsigned VirtReg) {
|
|
|
|
unsigned Hint = MRI->getSimpleHint(VirtReg);
|
|
|
|
if (!Hint)
|
2016-06-03 02:37:21 +08:00
|
|
|
return false;
|
2012-12-04 08:30:22 +08:00
|
|
|
if (TargetRegisterInfo::isVirtualRegister(Hint))
|
|
|
|
Hint = getPhys(Hint);
|
|
|
|
return getPhys(VirtReg) == Hint;
|
|
|
|
}
|
|
|
|
|
2012-12-04 07:23:50 +08:00
|
|
|
bool VirtRegMap::hasKnownPreference(unsigned VirtReg) {
|
|
|
|
std::pair<unsigned, unsigned> Hint = MRI->getRegAllocationHint(VirtReg);
|
|
|
|
if (TargetRegisterInfo::isPhysicalRegister(Hint.second))
|
|
|
|
return true;
|
|
|
|
if (TargetRegisterInfo::isVirtualRegister(Hint.second))
|
|
|
|
return hasPhys(Hint.second);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2004-09-30 09:54:45 +08:00
|
|
|
int VirtRegMap::assignVirt2StackSlot(unsigned virtReg) {
|
2008-02-11 02:45:23 +08:00
|
|
|
assert(TargetRegisterInfo::isVirtualRegister(virtReg));
|
2004-09-30 10:15:18 +08:00
|
|
|
assert(Virt2StackSlotMap[virtReg] == NO_STACK_SLOT &&
|
2004-09-30 09:54:45 +08:00
|
|
|
"attempt to assign stack slot to already spilled register");
|
2009-03-13 13:55:11 +08:00
|
|
|
const TargetRegisterClass* RC = MF->getRegInfo().getRegClass(virtReg);
|
2010-11-16 08:41:01 +08:00
|
|
|
return Virt2StackSlotMap[virtReg] = createSpillSlot(RC);
|
2004-02-24 07:08:11 +08:00
|
|
|
}
|
|
|
|
|
2008-02-27 11:04:06 +08:00
|
|
|
void VirtRegMap::assignVirt2StackSlot(unsigned virtReg, int SS) {
|
2008-02-11 02:45:23 +08:00
|
|
|
assert(TargetRegisterInfo::isVirtualRegister(virtReg));
|
2004-09-30 10:15:18 +08:00
|
|
|
assert(Virt2StackSlotMap[virtReg] == NO_STACK_SLOT &&
|
2004-09-30 09:54:45 +08:00
|
|
|
"attempt to assign stack slot to already spilled register");
|
2008-02-27 11:04:06 +08:00
|
|
|
assert((SS >= 0 ||
|
2016-07-29 02:40:00 +08:00
|
|
|
(SS >= MF->getFrameInfo().getObjectIndexBegin())) &&
|
2007-04-04 15:40:01 +08:00
|
|
|
"illegal fixed frame index");
|
2008-02-27 11:04:06 +08:00
|
|
|
Virt2StackSlotMap[virtReg] = SS;
|
2004-05-30 04:38:05 +08:00
|
|
|
}
|
|
|
|
|
2012-06-09 07:44:45 +08:00
|
|
|
void VirtRegMap::print(raw_ostream &OS, const Module*) const {
|
|
|
|
OS << "********** REGISTER MAP **********\n";
|
|
|
|
for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
|
|
|
|
unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
|
|
|
|
if (Virt2PhysMap[Reg] != (unsigned)VirtRegMap::NO_PHYS_REG) {
|
|
|
|
OS << '[' << PrintReg(Reg, TRI) << " -> "
|
|
|
|
<< PrintReg(Virt2PhysMap[Reg], TRI) << "] "
|
2014-11-17 13:50:14 +08:00
|
|
|
<< TRI->getRegClassName(MRI->getRegClass(Reg)) << "\n";
|
2012-06-09 07:44:45 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
|
|
|
|
unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
|
|
|
|
if (Virt2StackSlotMap[Reg] != VirtRegMap::NO_STACK_SLOT) {
|
|
|
|
OS << '[' << PrintReg(Reg, TRI) << " -> fi#" << Virt2StackSlotMap[Reg]
|
2014-11-17 13:50:14 +08:00
|
|
|
<< "] " << TRI->getRegClassName(MRI->getRegClass(Reg)) << "\n";
|
2012-06-09 07:44:45 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
OS << '\n';
|
|
|
|
}
|
|
|
|
|
2017-10-15 22:32:27 +08:00
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
2016-01-30 04:50:44 +08:00
|
|
|
LLVM_DUMP_METHOD void VirtRegMap::dump() const {
|
2012-06-09 07:44:45 +08:00
|
|
|
print(dbgs());
|
|
|
|
}
|
2012-09-07 03:06:06 +08:00
|
|
|
#endif
|
2012-06-09 07:44:45 +08:00
|
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
// VirtRegRewriter
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
|
|
|
// The VirtRegRewriter is the last of the register allocator passes.
|
|
|
|
// It rewrites virtual registers to physical registers as specified in the
|
|
|
|
// VirtRegMap analysis. It also updates live-in information on basic blocks
|
|
|
|
// according to LiveIntervals.
|
|
|
|
//
|
|
|
|
namespace {
|
2017-09-14 05:15:20 +08:00
|
|
|
|
2012-06-09 07:44:45 +08:00
|
|
|
class VirtRegRewriter : public MachineFunctionPass {
|
|
|
|
MachineFunction *MF;
|
|
|
|
const TargetRegisterInfo *TRI;
|
|
|
|
const TargetInstrInfo *TII;
|
|
|
|
MachineRegisterInfo *MRI;
|
|
|
|
SlotIndexes *Indexes;
|
|
|
|
LiveIntervals *LIS;
|
|
|
|
VirtRegMap *VRM;
|
|
|
|
|
|
|
|
void rewrite();
|
|
|
|
void addMBBLiveIns();
|
2015-06-17 02:22:28 +08:00
|
|
|
bool readsUndefSubreg(const MachineOperand &MO) const;
|
2015-09-10 02:07:54 +08:00
|
|
|
void addLiveInsForSubRanges(const LiveInterval &LI, unsigned PhysReg) const;
|
2016-07-09 08:19:07 +08:00
|
|
|
void handleIdentityCopy(MachineInstr &MI) const;
|
2017-03-17 08:41:39 +08:00
|
|
|
void expandCopyBundle(MachineInstr &MI) const;
|
[VirtRegRewriter] Properly model the register liveness on undef subreg definition
Undef subreg definition means that the content of the super register
doesn't matter at this point. While that's true for virtual registers,
this may not hold when replacing them with actual physical registers.
Indeed, some part of the physical register may be coalesced with the
related virtual register and thus, the values for those parts matter and
must be live.
The fix consists in checking whether or not subregs of the physical register
being assigned to an undef subreg definition are live through that def and
insert an implicit use if they are. Doing so, will keep them alive until
that point like they should be.
E.g., let vreg14 being assigned to R0_R1 then
%vreg14:gsub_0<def,read-undef> = COPY %R0 ; <-- R1 is still live here
%vreg14:gsub_1<def> = COPY %R1
Before this changes, the rewriter would change the code into:
%R0<def> = KILL %R0, %R0_R1<imp-def> ; <-- this tells R1 is redefined
%R1<def> = KILL %R1, %R0_R1<imp-def>, %R0_R1<imp-use> ; this value of this R1
; is believed to come
; from the previous
; instruction
Because of this invalid liveness, later pass could make wrong choices and in
particular clobber live register as it happened with the register scavenger in
llvm.org/PR34107
Now we would generate:
%R0<def> = KILL %R0, %R0_R1<imp-def>, %R0_R1<imp-use> ; This tells R1 needs to
; reach this point
%R1<def> = KILL %R1, %R0_R1<imp-def>, %R0_R1<imp-use>
The bug has been here forever, it got exposed recently because the register
scavenger got smarter.
Fixes llvm.org/PR34107
llvm-svn: 310979
2017-08-16 08:17:05 +08:00
|
|
|
bool subRegLiveThrough(const MachineInstr &MI, unsigned SuperPhysReg) const;
|
2015-09-10 02:07:54 +08:00
|
|
|
|
2012-06-09 07:44:45 +08:00
|
|
|
public:
|
|
|
|
static char ID;
|
2017-09-14 05:15:20 +08:00
|
|
|
|
2012-06-09 07:44:45 +08:00
|
|
|
VirtRegRewriter() : MachineFunctionPass(ID) {}
|
|
|
|
|
2014-03-07 17:26:03 +08:00
|
|
|
void getAnalysisUsage(AnalysisUsage &AU) const override;
|
2012-06-09 07:44:45 +08:00
|
|
|
|
2014-03-07 17:26:03 +08:00
|
|
|
bool runOnMachineFunction(MachineFunction&) override;
|
2017-09-14 05:15:20 +08:00
|
|
|
|
2016-03-30 01:40:22 +08:00
|
|
|
MachineFunctionProperties getSetProperties() const override {
|
|
|
|
return MachineFunctionProperties().set(
|
2016-08-25 09:27:13 +08:00
|
|
|
MachineFunctionProperties::Property::NoVRegs);
|
2016-03-30 01:40:22 +08:00
|
|
|
}
|
2012-06-09 07:44:45 +08:00
|
|
|
};
|
2017-09-14 05:15:20 +08:00
|
|
|
|
2012-06-09 07:44:45 +08:00
|
|
|
} // end anonymous namespace
|
|
|
|
|
2017-09-14 05:15:20 +08:00
|
|
|
char VirtRegRewriter::ID = 0;
|
|
|
|
|
2012-06-09 07:44:45 +08:00
|
|
|
char &llvm::VirtRegRewriterID = VirtRegRewriter::ID;
|
|
|
|
|
|
|
|
INITIALIZE_PASS_BEGIN(VirtRegRewriter, "virtregrewriter",
|
|
|
|
"Virtual Register Rewriter", false, false)
|
|
|
|
INITIALIZE_PASS_DEPENDENCY(SlotIndexes)
|
|
|
|
INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
|
|
|
|
INITIALIZE_PASS_DEPENDENCY(LiveDebugVariables)
|
2012-09-22 04:04:28 +08:00
|
|
|
INITIALIZE_PASS_DEPENDENCY(LiveStacks)
|
2012-06-09 07:44:45 +08:00
|
|
|
INITIALIZE_PASS_DEPENDENCY(VirtRegMap)
|
|
|
|
INITIALIZE_PASS_END(VirtRegRewriter, "virtregrewriter",
|
|
|
|
"Virtual Register Rewriter", false, false)
|
|
|
|
|
|
|
|
void VirtRegRewriter::getAnalysisUsage(AnalysisUsage &AU) const {
|
|
|
|
AU.setPreservesCFG();
|
|
|
|
AU.addRequired<LiveIntervals>();
|
|
|
|
AU.addRequired<SlotIndexes>();
|
|
|
|
AU.addPreserved<SlotIndexes>();
|
|
|
|
AU.addRequired<LiveDebugVariables>();
|
2012-09-22 04:04:28 +08:00
|
|
|
AU.addRequired<LiveStacks>();
|
|
|
|
AU.addPreserved<LiveStacks>();
|
2012-06-09 07:44:45 +08:00
|
|
|
AU.addRequired<VirtRegMap>();
|
|
|
|
MachineFunctionPass::getAnalysisUsage(AU);
|
|
|
|
}
|
|
|
|
|
|
|
|
bool VirtRegRewriter::runOnMachineFunction(MachineFunction &fn) {
|
|
|
|
MF = &fn;
|
2014-10-14 05:57:44 +08:00
|
|
|
TRI = MF->getSubtarget().getRegisterInfo();
|
|
|
|
TII = MF->getSubtarget().getInstrInfo();
|
2012-06-09 07:44:45 +08:00
|
|
|
MRI = &MF->getRegInfo();
|
|
|
|
Indexes = &getAnalysis<SlotIndexes>();
|
|
|
|
LIS = &getAnalysis<LiveIntervals>();
|
|
|
|
VRM = &getAnalysis<VirtRegMap>();
|
2011-02-19 06:03:18 +08:00
|
|
|
DEBUG(dbgs() << "********** REWRITE VIRTUAL REGISTERS **********\n"
|
|
|
|
<< "********** Function: "
|
2012-08-22 14:07:19 +08:00
|
|
|
<< MF->getName() << '\n');
|
2012-06-09 07:44:45 +08:00
|
|
|
DEBUG(VRM->dump());
|
|
|
|
|
|
|
|
// Add kill flags while we still have virtual registers.
|
2012-09-07 02:15:18 +08:00
|
|
|
LIS->addKillFlags(VRM);
|
2012-06-09 07:44:45 +08:00
|
|
|
|
2012-06-09 08:14:47 +08:00
|
|
|
// Live-in lists on basic blocks are required for physregs.
|
|
|
|
addMBBLiveIns();
|
|
|
|
|
2012-06-09 07:44:45 +08:00
|
|
|
// Rewrite virtual registers.
|
|
|
|
rewrite();
|
|
|
|
|
|
|
|
// Write out new DBG_VALUE instructions.
|
|
|
|
getAnalysis<LiveDebugVariables>().emitDebugValues(VRM);
|
|
|
|
|
|
|
|
// All machine operands and other references to virtual registers have been
|
|
|
|
// replaced. Remove the virtual registers and release all the transient data.
|
|
|
|
VRM->clearAllVirt();
|
|
|
|
MRI->clearVirtRegs();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2015-09-10 02:07:54 +08:00
|
|
|
void VirtRegRewriter::addLiveInsForSubRanges(const LiveInterval &LI,
|
|
|
|
unsigned PhysReg) const {
|
|
|
|
assert(!LI.empty());
|
|
|
|
assert(LI.hasSubRanges());
|
|
|
|
|
2017-09-14 05:15:20 +08:00
|
|
|
using SubRangeIteratorPair =
|
|
|
|
std::pair<const LiveInterval::SubRange *, LiveInterval::const_iterator>;
|
|
|
|
|
2015-09-10 02:07:54 +08:00
|
|
|
SmallVector<SubRangeIteratorPair, 4> SubRanges;
|
|
|
|
SlotIndex First;
|
|
|
|
SlotIndex Last;
|
|
|
|
for (const LiveInterval::SubRange &SR : LI.subranges()) {
|
|
|
|
SubRanges.push_back(std::make_pair(&SR, SR.begin()));
|
|
|
|
if (!First.isValid() || SR.segments.front().start < First)
|
|
|
|
First = SR.segments.front().start;
|
|
|
|
if (!Last.isValid() || SR.segments.back().end > Last)
|
|
|
|
Last = SR.segments.back().end;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check all mbb start positions between First and Last while
|
|
|
|
// simulatenously advancing an iterator for each subrange.
|
|
|
|
for (SlotIndexes::MBBIndexIterator MBBI = Indexes->findMBBIndex(First);
|
|
|
|
MBBI != Indexes->MBBIndexEnd() && MBBI->first <= Last; ++MBBI) {
|
|
|
|
SlotIndex MBBBegin = MBBI->first;
|
|
|
|
// Advance all subrange iterators so that their end position is just
|
|
|
|
// behind MBBBegin (or the iterator is at the end).
|
2016-12-15 22:36:06 +08:00
|
|
|
LaneBitmask LaneMask;
|
2015-09-10 02:07:54 +08:00
|
|
|
for (auto &RangeIterPair : SubRanges) {
|
|
|
|
const LiveInterval::SubRange *SR = RangeIterPair.first;
|
|
|
|
LiveInterval::const_iterator &SRI = RangeIterPair.second;
|
|
|
|
while (SRI != SR->end() && SRI->end <= MBBBegin)
|
|
|
|
++SRI;
|
|
|
|
if (SRI == SR->end())
|
|
|
|
continue;
|
|
|
|
if (SRI->start <= MBBBegin)
|
|
|
|
LaneMask |= SR->LaneMask;
|
|
|
|
}
|
2016-12-15 22:36:06 +08:00
|
|
|
if (LaneMask.none())
|
2015-09-10 02:07:54 +08:00
|
|
|
continue;
|
|
|
|
MachineBasicBlock *MBB = MBBI->second;
|
2015-09-10 02:08:03 +08:00
|
|
|
MBB->addLiveIn(PhysReg, LaneMask);
|
2015-09-10 02:07:54 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-06-09 08:14:47 +08:00
|
|
|
// Compute MBB live-in lists from virtual register live ranges and their
|
|
|
|
// assignments.
|
|
|
|
void VirtRegRewriter::addMBBLiveIns() {
|
|
|
|
for (unsigned Idx = 0, IdxE = MRI->getNumVirtRegs(); Idx != IdxE; ++Idx) {
|
|
|
|
unsigned VirtReg = TargetRegisterInfo::index2VirtReg(Idx);
|
|
|
|
if (MRI->reg_nodbg_empty(VirtReg))
|
|
|
|
continue;
|
|
|
|
LiveInterval &LI = LIS->getInterval(VirtReg);
|
|
|
|
if (LI.empty() || LIS->intervalIsInOneMBB(LI))
|
|
|
|
continue;
|
|
|
|
// This is a virtual register that is live across basic blocks. Its
|
|
|
|
// assigned PhysReg must be marked as live-in to those blocks.
|
|
|
|
unsigned PhysReg = VRM->getPhys(VirtReg);
|
|
|
|
assert(PhysReg != VirtRegMap::NO_PHYS_REG && "Unmapped virtual register.");
|
|
|
|
|
2014-12-10 09:13:08 +08:00
|
|
|
if (LI.hasSubRanges()) {
|
2015-09-10 02:07:54 +08:00
|
|
|
addLiveInsForSubRanges(LI, PhysReg);
|
2014-12-10 09:13:08 +08:00
|
|
|
} else {
|
2015-09-10 02:07:54 +08:00
|
|
|
// Go over MBB begin positions and see if we have segments covering them.
|
|
|
|
// The following works because segments and the MBBIndex list are both
|
|
|
|
// sorted by slot indexes.
|
|
|
|
SlotIndexes::MBBIndexIterator I = Indexes->MBBIndexBegin();
|
|
|
|
for (const auto &Seg : LI) {
|
|
|
|
I = Indexes->advanceMBBIndex(I, Seg.start);
|
|
|
|
for (; I != Indexes->MBBIndexEnd() && I->first < Seg.end; ++I) {
|
|
|
|
MachineBasicBlock *MBB = I->second;
|
|
|
|
MBB->addLiveIn(PhysReg);
|
|
|
|
}
|
2014-12-10 09:13:08 +08:00
|
|
|
}
|
2012-06-09 08:14:47 +08:00
|
|
|
}
|
|
|
|
}
|
2015-05-22 16:11:26 +08:00
|
|
|
|
|
|
|
// Sort and unique MBB LiveIns as we've not checked if SubReg/PhysReg were in
|
|
|
|
// each MBB's LiveIns set before calling addLiveIn on them.
|
|
|
|
for (MachineBasicBlock &MBB : *MF)
|
|
|
|
MBB.sortUniqueLiveIns();
|
2012-06-09 08:14:47 +08:00
|
|
|
}
|
|
|
|
|
2015-06-17 02:22:28 +08:00
|
|
|
/// Returns true if the given machine operand \p MO only reads undefined lanes.
|
|
|
|
/// The function only works for use operands with a subregister set.
|
|
|
|
bool VirtRegRewriter::readsUndefSubreg(const MachineOperand &MO) const {
|
|
|
|
// Shortcut if the operand is already marked undef.
|
|
|
|
if (MO.isUndef())
|
|
|
|
return true;
|
|
|
|
|
|
|
|
unsigned Reg = MO.getReg();
|
|
|
|
const LiveInterval &LI = LIS->getInterval(Reg);
|
|
|
|
const MachineInstr &MI = *MO.getParent();
|
2016-02-27 14:40:41 +08:00
|
|
|
SlotIndex BaseIndex = LIS->getInstructionIndex(MI);
|
2015-06-17 02:22:28 +08:00
|
|
|
// This code is only meant to handle reading undefined subregisters which
|
|
|
|
// we couldn't properly detect before.
|
|
|
|
assert(LI.liveAt(BaseIndex) &&
|
|
|
|
"Reads of completely dead register should be marked undef already");
|
|
|
|
unsigned SubRegIdx = MO.getSubReg();
|
2016-08-24 21:37:55 +08:00
|
|
|
assert(SubRegIdx != 0 && LI.hasSubRanges());
|
2015-09-26 05:51:14 +08:00
|
|
|
LaneBitmask UseMask = TRI->getSubRegIndexLaneMask(SubRegIdx);
|
2015-06-17 02:22:28 +08:00
|
|
|
// See if any of the relevant subregister liveranges is defined at this point.
|
|
|
|
for (const LiveInterval::SubRange &SR : LI.subranges()) {
|
2016-12-17 03:11:56 +08:00
|
|
|
if ((SR.LaneMask & UseMask).any() && SR.liveAt(BaseIndex))
|
2015-06-17 02:22:28 +08:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2016-07-09 08:19:07 +08:00
|
|
|
void VirtRegRewriter::handleIdentityCopy(MachineInstr &MI) const {
|
|
|
|
if (!MI.isIdentityCopy())
|
|
|
|
return;
|
|
|
|
DEBUG(dbgs() << "Identity copy: " << MI);
|
|
|
|
++NumIdCopies;
|
|
|
|
|
|
|
|
// Copies like:
|
|
|
|
// %R0 = COPY %R0<undef>
|
|
|
|
// %AL = COPY %AL, %EAX<imp-def>
|
|
|
|
// give us additional liveness information: The target (super-)register
|
|
|
|
// must not be valid before this point. Replace the COPY with a KILL
|
|
|
|
// instruction to maintain this information.
|
|
|
|
if (MI.getOperand(0).isUndef() || MI.getNumOperands() > 2) {
|
|
|
|
MI.setDesc(TII->get(TargetOpcode::KILL));
|
|
|
|
DEBUG(dbgs() << " replace by: " << MI);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (Indexes)
|
2017-03-17 08:41:33 +08:00
|
|
|
Indexes->removeSingleMachineInstrFromMaps(MI);
|
|
|
|
MI.eraseFromBundle();
|
2016-07-09 08:19:07 +08:00
|
|
|
DEBUG(dbgs() << " deleted.\n");
|
|
|
|
}
|
|
|
|
|
2017-03-17 08:41:39 +08:00
|
|
|
/// The liverange splitting logic sometimes produces bundles of copies when
|
|
|
|
/// subregisters are involved. Expand these into a sequence of copy instructions
|
|
|
|
/// after processing the last in the bundle. Does not update LiveIntervals
|
|
|
|
/// which we shouldn't need for this instruction anymore.
|
|
|
|
void VirtRegRewriter::expandCopyBundle(MachineInstr &MI) const {
|
|
|
|
if (!MI.isCopy())
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (MI.isBundledWithPred() && !MI.isBundledWithSucc()) {
|
|
|
|
// Only do this when the complete bundle is made out of COPYs.
|
2017-03-22 05:58:08 +08:00
|
|
|
MachineBasicBlock &MBB = *MI.getParent();
|
2017-03-17 08:41:39 +08:00
|
|
|
for (MachineBasicBlock::reverse_instr_iterator I =
|
2017-03-22 05:58:08 +08:00
|
|
|
std::next(MI.getReverseIterator()), E = MBB.instr_rend();
|
|
|
|
I != E && I->isBundledWithSucc(); ++I) {
|
2017-03-17 08:41:39 +08:00
|
|
|
if (!I->isCopy())
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (MachineBasicBlock::reverse_instr_iterator I = MI.getReverseIterator();
|
|
|
|
I->isBundledWithPred(); ) {
|
|
|
|
MachineInstr &MI = *I;
|
|
|
|
++I;
|
|
|
|
|
|
|
|
MI.unbundleFromPred();
|
|
|
|
if (Indexes)
|
|
|
|
Indexes->insertMachineInstrInMaps(MI);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
[VirtRegRewriter] Properly model the register liveness on undef subreg definition
Undef subreg definition means that the content of the super register
doesn't matter at this point. While that's true for virtual registers,
this may not hold when replacing them with actual physical registers.
Indeed, some part of the physical register may be coalesced with the
related virtual register and thus, the values for those parts matter and
must be live.
The fix consists in checking whether or not subregs of the physical register
being assigned to an undef subreg definition are live through that def and
insert an implicit use if they are. Doing so, will keep them alive until
that point like they should be.
E.g., let vreg14 being assigned to R0_R1 then
%vreg14:gsub_0<def,read-undef> = COPY %R0 ; <-- R1 is still live here
%vreg14:gsub_1<def> = COPY %R1
Before this changes, the rewriter would change the code into:
%R0<def> = KILL %R0, %R0_R1<imp-def> ; <-- this tells R1 is redefined
%R1<def> = KILL %R1, %R0_R1<imp-def>, %R0_R1<imp-use> ; this value of this R1
; is believed to come
; from the previous
; instruction
Because of this invalid liveness, later pass could make wrong choices and in
particular clobber live register as it happened with the register scavenger in
llvm.org/PR34107
Now we would generate:
%R0<def> = KILL %R0, %R0_R1<imp-def>, %R0_R1<imp-use> ; This tells R1 needs to
; reach this point
%R1<def> = KILL %R1, %R0_R1<imp-def>, %R0_R1<imp-use>
The bug has been here forever, it got exposed recently because the register
scavenger got smarter.
Fixes llvm.org/PR34107
llvm-svn: 310979
2017-08-16 08:17:05 +08:00
|
|
|
/// Check whether (part of) \p SuperPhysReg is live through \p MI.
|
|
|
|
/// \pre \p MI defines a subregister of a virtual register that
|
|
|
|
/// has been assigned to \p SuperPhysReg.
|
|
|
|
bool VirtRegRewriter::subRegLiveThrough(const MachineInstr &MI,
|
|
|
|
unsigned SuperPhysReg) const {
|
|
|
|
SlotIndex MIIndex = LIS->getInstructionIndex(MI);
|
|
|
|
SlotIndex BeforeMIUses = MIIndex.getBaseIndex();
|
|
|
|
SlotIndex AfterMIDefs = MIIndex.getBoundaryIndex();
|
|
|
|
for (MCRegUnitIterator Unit(SuperPhysReg, TRI); Unit.isValid(); ++Unit) {
|
|
|
|
const LiveRange &UnitRange = LIS->getRegUnit(*Unit);
|
|
|
|
// If the regunit is live both before and after MI,
|
|
|
|
// we assume it is live through.
|
|
|
|
// Generally speaking, this is not true, because something like
|
|
|
|
// "RU = op RU" would match that description.
|
|
|
|
// However, we know that we are trying to assess whether
|
|
|
|
// a def of a virtual reg, vreg, is live at the same time of RU.
|
|
|
|
// If we are in the "RU = op RU" situation, that means that vreg
|
|
|
|
// is defined at the same time as RU (i.e., "vreg, RU = op RU").
|
|
|
|
// Thus, vreg and RU interferes and vreg cannot be assigned to
|
|
|
|
// SuperPhysReg. Therefore, this situation cannot happen.
|
|
|
|
if (UnitRange.liveAt(AfterMIDefs) && UnitRange.liveAt(BeforeMIUses))
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2012-06-09 07:44:45 +08:00
|
|
|
void VirtRegRewriter::rewrite() {
|
2015-03-19 08:21:58 +08:00
|
|
|
bool NoSubRegLiveness = !MRI->subRegLivenessEnabled();
|
2011-04-28 01:42:31 +08:00
|
|
|
SmallVector<unsigned, 8> SuperDeads;
|
|
|
|
SmallVector<unsigned, 8> SuperDefs;
|
2011-02-19 06:03:18 +08:00
|
|
|
SmallVector<unsigned, 8> SuperKills;
|
2014-02-26 00:57:28 +08:00
|
|
|
|
2011-02-19 06:03:18 +08:00
|
|
|
for (MachineFunction::iterator MBBI = MF->begin(), MBBE = MF->end();
|
|
|
|
MBBI != MBBE; ++MBBI) {
|
|
|
|
DEBUG(MBBI->print(dbgs(), Indexes));
|
2012-01-19 15:46:36 +08:00
|
|
|
for (MachineBasicBlock::instr_iterator
|
|
|
|
MII = MBBI->instr_begin(), MIE = MBBI->instr_end(); MII != MIE;) {
|
2015-10-10 06:56:24 +08:00
|
|
|
MachineInstr *MI = &*MII;
|
2011-02-19 06:03:18 +08:00
|
|
|
++MII;
|
|
|
|
|
|
|
|
for (MachineInstr::mop_iterator MOI = MI->operands_begin(),
|
|
|
|
MOE = MI->operands_end(); MOI != MOE; ++MOI) {
|
|
|
|
MachineOperand &MO = *MOI;
|
2012-02-18 03:07:56 +08:00
|
|
|
|
|
|
|
// Make sure MRI knows about registers clobbered by regmasks.
|
|
|
|
if (MO.isRegMask())
|
|
|
|
MRI->addPhysRegsUsedFromRegMask(MO.getRegMask());
|
|
|
|
|
2011-02-19 06:03:18 +08:00
|
|
|
if (!MO.isReg() || !TargetRegisterInfo::isVirtualRegister(MO.getReg()))
|
|
|
|
continue;
|
|
|
|
unsigned VirtReg = MO.getReg();
|
2012-06-09 07:44:45 +08:00
|
|
|
unsigned PhysReg = VRM->getPhys(VirtReg);
|
|
|
|
assert(PhysReg != VirtRegMap::NO_PHYS_REG &&
|
|
|
|
"Instruction uses unmapped VirtReg");
|
2012-10-16 05:57:41 +08:00
|
|
|
assert(!MRI->isReserved(PhysReg) && "Reserved register assignment");
|
2011-02-19 06:03:18 +08:00
|
|
|
|
|
|
|
// Preserve semantics of sub-register operands.
|
2015-06-17 02:22:28 +08:00
|
|
|
unsigned SubReg = MO.getSubReg();
|
|
|
|
if (SubReg != 0) {
|
|
|
|
if (NoSubRegLiveness) {
|
|
|
|
// A virtual register kill refers to the whole register, so we may
|
|
|
|
// have to add <imp-use,kill> operands for the super-register. A
|
|
|
|
// partial redef always kills and redefines the super-register.
|
[VirtRegRewriter] Properly model the register liveness on undef subreg definition
Undef subreg definition means that the content of the super register
doesn't matter at this point. While that's true for virtual registers,
this may not hold when replacing them with actual physical registers.
Indeed, some part of the physical register may be coalesced with the
related virtual register and thus, the values for those parts matter and
must be live.
The fix consists in checking whether or not subregs of the physical register
being assigned to an undef subreg definition are live through that def and
insert an implicit use if they are. Doing so, will keep them alive until
that point like they should be.
E.g., let vreg14 being assigned to R0_R1 then
%vreg14:gsub_0<def,read-undef> = COPY %R0 ; <-- R1 is still live here
%vreg14:gsub_1<def> = COPY %R1
Before this changes, the rewriter would change the code into:
%R0<def> = KILL %R0, %R0_R1<imp-def> ; <-- this tells R1 is redefined
%R1<def> = KILL %R1, %R0_R1<imp-def>, %R0_R1<imp-use> ; this value of this R1
; is believed to come
; from the previous
; instruction
Because of this invalid liveness, later pass could make wrong choices and in
particular clobber live register as it happened with the register scavenger in
llvm.org/PR34107
Now we would generate:
%R0<def> = KILL %R0, %R0_R1<imp-def>, %R0_R1<imp-use> ; This tells R1 needs to
; reach this point
%R1<def> = KILL %R1, %R0_R1<imp-def>, %R0_R1<imp-use>
The bug has been here forever, it got exposed recently because the register
scavenger got smarter.
Fixes llvm.org/PR34107
llvm-svn: 310979
2017-08-16 08:17:05 +08:00
|
|
|
if ((MO.readsReg() && (MO.isDef() || MO.isKill())) ||
|
|
|
|
(MO.isDef() && subRegLiveThrough(*MI, PhysReg)))
|
2015-06-17 02:22:28 +08:00
|
|
|
SuperKills.push_back(PhysReg);
|
|
|
|
|
|
|
|
if (MO.isDef()) {
|
|
|
|
// Also add implicit defs for the super-register.
|
2014-12-10 09:13:04 +08:00
|
|
|
if (MO.isDead())
|
|
|
|
SuperDeads.push_back(PhysReg);
|
|
|
|
else
|
|
|
|
SuperDefs.push_back(PhysReg);
|
|
|
|
}
|
2015-06-17 02:22:28 +08:00
|
|
|
} else {
|
|
|
|
if (MO.isUse()) {
|
|
|
|
if (readsUndefSubreg(MO))
|
|
|
|
// We need to add an <undef> flag if the subregister is
|
|
|
|
// completely undefined (and we are not adding super-register
|
|
|
|
// defs).
|
|
|
|
MO.setIsUndef(true);
|
|
|
|
} else if (!MO.isDead()) {
|
|
|
|
assert(MO.isDef());
|
|
|
|
}
|
2011-10-05 08:01:48 +08:00
|
|
|
}
|
2011-02-19 06:03:18 +08:00
|
|
|
|
2017-03-17 08:41:33 +08:00
|
|
|
// The <def,undef> and <def,internal> flags only make sense for
|
|
|
|
// sub-register defs, and we are substituting a full physreg. An
|
|
|
|
// <imp-use,kill> operand from the SuperKills list will represent the
|
|
|
|
// partial read of the super-register.
|
|
|
|
if (MO.isDef()) {
|
2015-06-17 02:22:28 +08:00
|
|
|
MO.setIsUndef(false);
|
2017-03-17 08:41:33 +08:00
|
|
|
MO.setIsInternalRead(false);
|
|
|
|
}
|
2015-06-17 02:22:28 +08:00
|
|
|
|
2011-02-19 06:03:18 +08:00
|
|
|
// PhysReg operands cannot have subregister indexes.
|
2015-06-17 02:22:28 +08:00
|
|
|
PhysReg = TRI->getSubReg(PhysReg, SubReg);
|
2011-02-19 06:03:18 +08:00
|
|
|
assert(PhysReg && "Invalid SubReg for physical register");
|
|
|
|
MO.setSubReg(0);
|
|
|
|
}
|
|
|
|
// Rewrite. Note we could have used MachineOperand::substPhysReg(), but
|
|
|
|
// we need the inlining here.
|
|
|
|
MO.setReg(PhysReg);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Add any missing super-register kills after rewriting the whole
|
|
|
|
// instruction.
|
|
|
|
while (!SuperKills.empty())
|
|
|
|
MI->addRegisterKilled(SuperKills.pop_back_val(), TRI, true);
|
|
|
|
|
2011-04-28 01:42:31 +08:00
|
|
|
while (!SuperDeads.empty())
|
|
|
|
MI->addRegisterDead(SuperDeads.pop_back_val(), TRI, true);
|
|
|
|
|
|
|
|
while (!SuperDefs.empty())
|
|
|
|
MI->addRegisterDefined(SuperDefs.pop_back_val(), TRI);
|
|
|
|
|
2011-02-19 06:03:18 +08:00
|
|
|
DEBUG(dbgs() << "> " << *MI);
|
|
|
|
|
2017-03-17 08:41:39 +08:00
|
|
|
expandCopyBundle(*MI);
|
|
|
|
|
2016-07-09 08:19:07 +08:00
|
|
|
// We can remove identity copies right now.
|
|
|
|
handleIdentityCopy(*MI);
|
2011-02-19 06:03:18 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|