diff --git a/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/llvm/include/llvm/CodeGen/MachineBasicBlock.h index cc651ca77b77..82f9c0f25f0f 100644 --- a/llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ b/llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -19,6 +19,7 @@ namespace llvm { +class Pass; class BasicBlock; class MachineFunction; class MCSymbol; @@ -281,6 +282,14 @@ public: /// it returns end() iterator getFirstTerminator(); + /// SplitCriticalEdge - Split the critical edge from this block to the + /// given successor block, and return the newly created block, or null + /// if splitting is not possible. + /// + /// This function updates LiveVariables, MachineDominatorTree, and + /// MachineLoopInfo, as applicable. + MachineBasicBlock *SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P); + void pop_front() { Insts.pop_front(); } void pop_back() { Insts.pop_back(); } void push_back(MachineInstr *MI) { Insts.push_back(MI); } diff --git a/llvm/include/llvm/CodeGen/MachineLoopInfo.h b/llvm/include/llvm/CodeGen/MachineLoopInfo.h index 8459a8db9a30..3b3e31e02afa 100644 --- a/llvm/include/llvm/CodeGen/MachineLoopInfo.h +++ b/llvm/include/llvm/CodeGen/MachineLoopInfo.h @@ -64,13 +64,13 @@ class MachineLoopInfo : public MachineFunctionPass { void operator=(const MachineLoopInfo &); // do not implement MachineLoopInfo(const MachineLoopInfo &); // do not implement - LoopInfoBase& getBase() { return LI; } - public: static char ID; // Pass identification, replacement for typeid MachineLoopInfo() : MachineFunctionPass(&ID) {} + LoopInfoBase& getBase() { return LI; } + /// iterator/begin/end - The interface to the top-level loops in the current /// function. /// diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp index 9c3c76df77ff..69ab593798db 100644 --- a/llvm/lib/CodeGen/MachineBasicBlock.cpp +++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp @@ -13,7 +13,10 @@ #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/BasicBlock.h" +#include "llvm/CodeGen/LiveVariables.h" +#include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/Target/TargetRegisterInfo.h" @@ -396,6 +399,82 @@ bool MachineBasicBlock::canFallThrough() { return FBB == 0; } +MachineBasicBlock * +MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) { + MachineFunction *MF = getParent(); + DebugLoc dl; // FIXME: this is nowhere + + // We may need to update this's terminator, but we can't do that if AnalyzeBranch + // fails. If this uses a jump table, we won't touch it. + const TargetInstrInfo *TII = MF->getTarget().getInstrInfo(); + MachineBasicBlock *TBB = 0, *FBB = 0; + SmallVector Cond; + if (TII->AnalyzeBranch(*this, TBB, FBB, Cond)) + return NULL; + + MachineBasicBlock *NMBB = MF->CreateMachineBasicBlock(); + MF->insert(llvm::next(MachineFunction::iterator(this)), NMBB); + DEBUG(dbgs() << "PHIElimination splitting critical edge:" + " BB#" << getNumber() + << " -- BB#" << NMBB->getNumber() + << " -- BB#" << Succ->getNumber() << '\n'); + + ReplaceUsesOfBlockWith(Succ, NMBB); + updateTerminator(); + + // Insert unconditional "jump Succ" instruction in NMBB if necessary. + NMBB->addSuccessor(Succ); + if (!NMBB->isLayoutSuccessor(Succ)) { + Cond.clear(); + MF->getTarget().getInstrInfo()->InsertBranch(*NMBB, Succ, NULL, Cond, dl); + } + + // Fix PHI nodes in Succ so they refer to NMBB instead of this + for (MachineBasicBlock::iterator i = Succ->begin(), e = Succ->end(); + i != e && i->isPHI(); ++i) + for (unsigned ni = 1, ne = i->getNumOperands(); ni != ne; ni += 2) + if (i->getOperand(ni+1).getMBB() == this) + i->getOperand(ni+1).setMBB(NMBB); + + if (LiveVariables *LV = + P->getAnalysisIfAvailable()) + LV->addNewBlock(NMBB, this, Succ); + + if (MachineDominatorTree *MDT = + P->getAnalysisIfAvailable()) + MDT->addNewBlock(NMBB, this); + + if (MachineLoopInfo *MLI = + P->getAnalysisIfAvailable()) + if (MachineLoop *TIL = MLI->getLoopFor(this)) { + // If one or the other blocks were not in a loop, the new block is not + // either, and thus LI doesn't need to be updated. + if (MachineLoop *DestLoop = MLI->getLoopFor(Succ)) { + if (TIL == DestLoop) { + // Both in the same loop, the NMBB joins loop. + DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase()); + } else if (TIL->contains(DestLoop)) { + // Edge from an outer loop to an inner loop. Add to the outer loop. + TIL->addBasicBlockToLoop(NMBB, MLI->getBase()); + } else if (DestLoop->contains(TIL)) { + // Edge from an inner loop to an outer loop. Add to the outer loop. + DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase()); + } else { + // Edge from two loops with no containment relation. Because these + // are natural loops, we know that the destination block must be the + // header of its loop (adding a branch into a loop elsewhere would + // create an irreducible loop). + assert(DestLoop->getHeader() == Succ && + "Should not create irreducible loops!"); + if (MachineLoop *P = DestLoop->getParentLoop()) + P->addBasicBlockToLoop(NMBB, MLI->getBase()); + } + } + } + + return NMBB; +} + /// removeFromParent - This method unlinks 'this' from the containing function, /// and returns it, but does not delete it. MachineBasicBlock *MachineBasicBlock::removeFromParent() { diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp index d9623ab45c9e..9b24a9a2391b 100644 --- a/llvm/lib/CodeGen/MachineLICM.cpp +++ b/llvm/lib/CodeGen/MachineLICM.cpp @@ -83,7 +83,6 @@ namespace { const char *getPassName() const { return "Machine Instruction LICM"; } - // FIXME: Loop preheaders? virtual void getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesCFG(); AU.addRequired(); @@ -182,6 +181,10 @@ namespace { /// current loop preheader that may become duplicates of instructions that /// are hoisted out of the loop. void InitCSEMap(MachineBasicBlock *BB); + + /// getCurPreheader - Get the preheader for the current loop, splitting + /// a critical edge if needed. + MachineBasicBlock *getCurPreheader(); }; } // end anonymous namespace @@ -193,11 +196,11 @@ FunctionPass *llvm::createMachineLICMPass(bool PreRegAlloc) { return new MachineLICM(PreRegAlloc); } -/// LoopIsOuterMostWithPreheader - Test if the given loop is the outer-most -/// loop that has a preheader. -static bool LoopIsOuterMostWithPreheader(MachineLoop *CurLoop) { +/// LoopIsOuterMostWithPredecessor - Test if the given loop is the outer-most +/// loop that has a unique predecessor. +static bool LoopIsOuterMostWithPredecessor(MachineLoop *CurLoop) { for (MachineLoop *L = CurLoop->getParentLoop(); L; L = L->getParentLoop()) - if (L->getLoopPreheader()) + if (L->getLoopPredecessor()) return false; return true; } @@ -223,20 +226,11 @@ bool MachineLICM::runOnMachineFunction(MachineFunction &MF) { for (MachineLoopInfo::iterator I = MLI->begin(), E = MLI->end(); I != E; ++I){ CurLoop = *I; + CurPreheader = 0; // If this is done before regalloc, only visit outer-most preheader-sporting // loops. - if (PreRegAlloc && !LoopIsOuterMostWithPreheader(CurLoop)) - continue; - - // Determine the block to which to hoist instructions. If we can't find a - // suitable loop preheader, we can't do any hoisting. - // - // FIXME: We are only hoisting if the basic block coming into this loop - // has only one successor. This isn't the case in general because we haven't - // broken critical edges or added preheaders. - CurPreheader = CurLoop->getLoopPreheader(); - if (!CurPreheader) + if (PreRegAlloc && !LoopIsOuterMostWithPredecessor(CurLoop)) continue; if (!PreRegAlloc) @@ -438,13 +432,16 @@ void MachineLICM::AddToLiveIns(unsigned Reg) { /// operands that is safe to hoist, this instruction is called to do the /// dirty work. void MachineLICM::HoistPostRA(MachineInstr *MI, unsigned Def) { + MachineBasicBlock *Preheader = getCurPreheader(); + if (!Preheader) return; + // Now move the instructions to the predecessor, inserting it before any // terminator instructions. DEBUG({ dbgs() << "Hoisting " << *MI; - if (CurPreheader->getBasicBlock()) + if (Preheader->getBasicBlock()) dbgs() << " to MachineBasicBlock " - << CurPreheader->getName(); + << Preheader->getName(); if (MI->getParent()->getBasicBlock()) dbgs() << " from MachineBasicBlock " << MI->getParent()->getName(); @@ -453,7 +450,7 @@ void MachineLICM::HoistPostRA(MachineInstr *MI, unsigned Def) { // Splice the instruction to the preheader. MachineBasicBlock *MBB = MI->getParent(); - CurPreheader->splice(CurPreheader->getFirstTerminator(), MBB, MI); + Preheader->splice(Preheader->getFirstTerminator(), MBB, MI); // Add register to livein list to all the BBs in the current loop since a // loop invariant must be kept live throughout the whole loop. This is @@ -756,6 +753,9 @@ bool MachineLICM::EliminateCSE(MachineInstr *MI, /// that are safe to hoist, this instruction is called to do the dirty work. /// void MachineLICM::Hoist(MachineInstr *MI) { + MachineBasicBlock *Preheader = getCurPreheader(); + if (!Preheader) return; + // First check whether we should hoist this instruction. if (!IsLoopInvariantInst(*MI) || !IsProfitableToHoist(*MI)) { // If not, try unfolding a hoistable load. @@ -767,9 +767,9 @@ void MachineLICM::Hoist(MachineInstr *MI) { // terminator instructions. DEBUG({ dbgs() << "Hoisting " << *MI; - if (CurPreheader->getBasicBlock()) + if (Preheader->getBasicBlock()) dbgs() << " to MachineBasicBlock " - << CurPreheader->getName(); + << Preheader->getName(); if (MI->getParent()->getBasicBlock()) dbgs() << " from MachineBasicBlock " << MI->getParent()->getName(); @@ -779,7 +779,7 @@ void MachineLICM::Hoist(MachineInstr *MI) { // If this is the first instruction being hoisted to the preheader, // initialize the CSE map with potential common expressions. if (FirstInLoop) { - InitCSEMap(CurPreheader); + InitCSEMap(Preheader); FirstInLoop = false; } @@ -789,7 +789,7 @@ void MachineLICM::Hoist(MachineInstr *MI) { CI = CSEMap.find(Opcode); if (!EliminateCSE(MI, CI)) { // Otherwise, splice the instruction to the preheader. - CurPreheader->splice(CurPreheader->getFirstTerminator(),MI->getParent(),MI); + Preheader->splice(Preheader->getFirstTerminator(),MI->getParent(),MI); // Clear the kill flags of any register this instruction defines, // since they may need to be live throughout the entire loop @@ -813,3 +813,30 @@ void MachineLICM::Hoist(MachineInstr *MI) { ++NumHoisted; Changed = true; } + +MachineBasicBlock *MachineLICM::getCurPreheader() { + // Determine the block to which to hoist instructions. If we can't find a + // suitable loop predecessor, we can't do any hoisting. + + // If we've tried to get a preheader and failed, don't try again. + if (CurPreheader == reinterpret_cast(-1)) + return 0; + + if (!CurPreheader) { + CurPreheader = CurLoop->getLoopPreheader(); + if (!CurPreheader) { + MachineBasicBlock *Pred = CurLoop->getLoopPredecessor(); + if (!Pred) { + CurPreheader = reinterpret_cast(-1); + return 0; + } + + CurPreheader = Pred->SplitCriticalEdge(CurLoop->getHeader(), this); + if (!CurPreheader) { + CurPreheader = reinterpret_cast(-1); + return 0; + } + } + } + return CurPreheader; +} diff --git a/llvm/lib/CodeGen/PHIElimination.cpp b/llvm/lib/CodeGen/PHIElimination.cpp index 4271a142309c..007d4d9611c0 100644 --- a/llvm/lib/CodeGen/PHIElimination.cpp +++ b/llvm/lib/CodeGen/PHIElimination.cpp @@ -34,7 +34,6 @@ using namespace llvm; STATISTIC(NumAtomic, "Number of atomic phis lowered"); -STATISTIC(NumSplits, "Number of critical edges split on demand"); STATISTIC(NumReused, "Number of reused lowered phis"); char PHIElimination::ID = 0; @@ -391,58 +390,8 @@ bool llvm::PHIElimination::SplitPHIEdges(MachineFunction &MF, // (not considering PHI nodes). If the register is live in to this block // anyway, we would gain nothing from splitting. if (!LV.isLiveIn(Reg, MBB) && LV.isLiveOut(Reg, *PreMBB)) - SplitCriticalEdge(PreMBB, &MBB); + PreMBB->SplitCriticalEdge(&MBB, this); } } return true; } - -MachineBasicBlock *PHIElimination::SplitCriticalEdge(MachineBasicBlock *A, - MachineBasicBlock *B) { - assert(A && B && "Missing MBB end point"); - - MachineFunction *MF = A->getParent(); - DebugLoc dl; // FIXME: this is nowhere - - // We may need to update A's terminator, but we can't do that if AnalyzeBranch - // fails. If A uses a jump table, we won't touch it. - const TargetInstrInfo *TII = MF->getTarget().getInstrInfo(); - MachineBasicBlock *TBB = 0, *FBB = 0; - SmallVector Cond; - if (TII->AnalyzeBranch(*A, TBB, FBB, Cond)) - return NULL; - - ++NumSplits; - - MachineBasicBlock *NMBB = MF->CreateMachineBasicBlock(); - MF->insert(llvm::next(MachineFunction::iterator(A)), NMBB); - DEBUG(dbgs() << "PHIElimination splitting critical edge:" - " BB#" << A->getNumber() - << " -- BB#" << NMBB->getNumber() - << " -- BB#" << B->getNumber() << '\n'); - - A->ReplaceUsesOfBlockWith(B, NMBB); - A->updateTerminator(); - - // Insert unconditional "jump B" instruction in NMBB if necessary. - NMBB->addSuccessor(B); - if (!NMBB->isLayoutSuccessor(B)) { - Cond.clear(); - MF->getTarget().getInstrInfo()->InsertBranch(*NMBB, B, NULL, Cond, dl); - } - - // Fix PHI nodes in B so they refer to NMBB instead of A - for (MachineBasicBlock::iterator i = B->begin(), e = B->end(); - i != e && i->isPHI(); ++i) - for (unsigned ni = 1, ne = i->getNumOperands(); ni != ne; ni += 2) - if (i->getOperand(ni+1).getMBB() == A) - i->getOperand(ni+1).setMBB(NMBB); - - if (LiveVariables *LV=getAnalysisIfAvailable()) - LV->addNewBlock(NMBB, A, B); - - if (MachineDominatorTree *MDT=getAnalysisIfAvailable()) - MDT->addNewBlock(NMBB, A); - - return NMBB; -}