2012-12-20 06:10:31 +08:00
|
|
|
//===-- SIAnnotateControlFlow.cpp - ------------------===//
|
|
|
|
//
|
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
//
|
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
// License. See LICENSE.TXT for details.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
|
|
|
/// \file
|
|
|
|
/// Annotates the control flow with hardware specific intrinsics.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
#include "AMDGPU.h"
|
2013-01-02 18:22:59 +08:00
|
|
|
#include "llvm/ADT/DepthFirstIterator.h"
|
R600/SI: Fix bug from insertion of llvm.SI.end.cf into loop headers
The llvm.SI.end.cf intrinsic is used to mark the end of if-then blocks,
if-then-else blocks, and loops. It is responsible for updating the
exec mask to re-enable threads that had been masked during the preceding
control flow block. For example:
s_mov_b64 exec, 0x3 ; Initial exec mask
s_mov_b64 s[0:1], exec ; Saved exec mask
v_cmpx_gt_u32 exec, s[2:3], v0, 0 ; llvm.SI.if
do_stuff()
s_or_b64 exec, exec, s[0:1] ; llvm.SI.end.cf
The bug fixed by this patch was one where the llvm.SI.end.cf intrinsic
was being inserted into the header of loops. This would happen when
an if block terminated in a loop header and we would end up with
code like this:
s_mov_b64 exec, 0x3 ; Initial exec mask
s_mov_b64 s[0:1], exec ; Saved exec mask
v_cmpx_gt_u32 exec, s[2:3], v0, 0 ; llvm.SI.if
do_stuff()
LOOP: ; Start of loop header
s_or_b64 exec, exec, s[0:1] ; llvm.SI.end.cf <-BUG: The exec mask has the
same value at the beginning of each loop
iteration.
do_stuff();
s_cbranch_execnz LOOP
The fix is to create a new basic block before the loop and insert the
llvm.SI.end.cf there. This way the exec mask is restored before the
start of the loop instead of at the beginning of each iteration.
llvm-svn: 228302
2015-02-05 23:32:15 +08:00
|
|
|
#include "llvm/Analysis/LoopInfo.h"
|
2013-06-08 04:28:43 +08:00
|
|
|
#include "llvm/IR/Constants.h"
|
2014-01-13 17:26:24 +08:00
|
|
|
#include "llvm/IR/Dominators.h"
|
2013-06-08 04:28:43 +08:00
|
|
|
#include "llvm/IR/Instructions.h"
|
2013-01-02 19:36:10 +08:00
|
|
|
#include "llvm/IR/Module.h"
|
2013-01-02 18:22:59 +08:00
|
|
|
#include "llvm/Pass.h"
|
2012-12-20 06:10:31 +08:00
|
|
|
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
|
|
|
|
#include "llvm/Transforms/Utils/SSAUpdater.h"
|
|
|
|
|
|
|
|
using namespace llvm;
|
|
|
|
|
2014-04-22 10:41:26 +08:00
|
|
|
#define DEBUG_TYPE "si-annotate-control-flow"
|
|
|
|
|
2012-12-20 06:10:31 +08:00
|
|
|
namespace {
|
|
|
|
|
|
|
|
// Complex types used in this pass
|
|
|
|
typedef std::pair<BasicBlock *, Value *> StackEntry;
|
|
|
|
typedef SmallVector<StackEntry, 16> StackVector;
|
|
|
|
|
|
|
|
// Intrinsic names the control flow is annotated with
|
2013-07-16 09:17:10 +08:00
|
|
|
static const char *const IfIntrinsic = "llvm.SI.if";
|
|
|
|
static const char *const ElseIntrinsic = "llvm.SI.else";
|
|
|
|
static const char *const BreakIntrinsic = "llvm.SI.break";
|
|
|
|
static const char *const IfBreakIntrinsic = "llvm.SI.if.break";
|
|
|
|
static const char *const ElseBreakIntrinsic = "llvm.SI.else.break";
|
|
|
|
static const char *const LoopIntrinsic = "llvm.SI.loop";
|
|
|
|
static const char *const EndCfIntrinsic = "llvm.SI.end.cf";
|
2012-12-20 06:10:31 +08:00
|
|
|
|
|
|
|
class SIAnnotateControlFlow : public FunctionPass {
|
|
|
|
|
|
|
|
static char ID;
|
|
|
|
|
|
|
|
Type *Boolean;
|
|
|
|
Type *Void;
|
|
|
|
Type *Int64;
|
|
|
|
Type *ReturnStruct;
|
|
|
|
|
|
|
|
ConstantInt *BoolTrue;
|
|
|
|
ConstantInt *BoolFalse;
|
|
|
|
UndefValue *BoolUndef;
|
|
|
|
Constant *Int64Zero;
|
|
|
|
|
|
|
|
Constant *If;
|
|
|
|
Constant *Else;
|
|
|
|
Constant *Break;
|
|
|
|
Constant *IfBreak;
|
|
|
|
Constant *ElseBreak;
|
|
|
|
Constant *Loop;
|
|
|
|
Constant *EndCf;
|
|
|
|
|
|
|
|
DominatorTree *DT;
|
|
|
|
StackVector Stack;
|
|
|
|
|
R600/SI: Fix bug from insertion of llvm.SI.end.cf into loop headers
The llvm.SI.end.cf intrinsic is used to mark the end of if-then blocks,
if-then-else blocks, and loops. It is responsible for updating the
exec mask to re-enable threads that had been masked during the preceding
control flow block. For example:
s_mov_b64 exec, 0x3 ; Initial exec mask
s_mov_b64 s[0:1], exec ; Saved exec mask
v_cmpx_gt_u32 exec, s[2:3], v0, 0 ; llvm.SI.if
do_stuff()
s_or_b64 exec, exec, s[0:1] ; llvm.SI.end.cf
The bug fixed by this patch was one where the llvm.SI.end.cf intrinsic
was being inserted into the header of loops. This would happen when
an if block terminated in a loop header and we would end up with
code like this:
s_mov_b64 exec, 0x3 ; Initial exec mask
s_mov_b64 s[0:1], exec ; Saved exec mask
v_cmpx_gt_u32 exec, s[2:3], v0, 0 ; llvm.SI.if
do_stuff()
LOOP: ; Start of loop header
s_or_b64 exec, exec, s[0:1] ; llvm.SI.end.cf <-BUG: The exec mask has the
same value at the beginning of each loop
iteration.
do_stuff();
s_cbranch_execnz LOOP
The fix is to create a new basic block before the loop and insert the
llvm.SI.end.cf there. This way the exec mask is restored before the
start of the loop instead of at the beginning of each iteration.
llvm-svn: 228302
2015-02-05 23:32:15 +08:00
|
|
|
LoopInfo *LI;
|
|
|
|
|
2012-12-20 06:10:31 +08:00
|
|
|
bool isTopOfStack(BasicBlock *BB);
|
|
|
|
|
|
|
|
Value *popSaved();
|
|
|
|
|
|
|
|
void push(BasicBlock *BB, Value *Saved);
|
|
|
|
|
|
|
|
bool isElse(PHINode *Phi);
|
|
|
|
|
|
|
|
void eraseIfUnused(PHINode *Phi);
|
|
|
|
|
|
|
|
void openIf(BranchInst *Term);
|
|
|
|
|
|
|
|
void insertElse(BranchInst *Term);
|
|
|
|
|
2014-06-21 01:06:02 +08:00
|
|
|
Value *handleLoopCondition(Value *Cond, PHINode *Broken);
|
2012-12-20 06:10:31 +08:00
|
|
|
|
|
|
|
void handleLoop(BranchInst *Term);
|
|
|
|
|
|
|
|
void closeControlFlow(BasicBlock *BB);
|
|
|
|
|
|
|
|
public:
|
|
|
|
SIAnnotateControlFlow():
|
|
|
|
FunctionPass(ID) { }
|
|
|
|
|
2014-04-29 15:57:24 +08:00
|
|
|
bool doInitialization(Module &M) override;
|
2012-12-20 06:10:31 +08:00
|
|
|
|
2014-04-29 15:57:24 +08:00
|
|
|
bool runOnFunction(Function &F) override;
|
2012-12-20 06:10:31 +08:00
|
|
|
|
2014-04-29 15:57:24 +08:00
|
|
|
const char *getPassName() const override {
|
2012-12-20 06:10:31 +08:00
|
|
|
return "SI annotate control flow";
|
|
|
|
}
|
|
|
|
|
2014-04-29 15:57:24 +08:00
|
|
|
void getAnalysisUsage(AnalysisUsage &AU) const override {
|
R600/SI: Fix bug from insertion of llvm.SI.end.cf into loop headers
The llvm.SI.end.cf intrinsic is used to mark the end of if-then blocks,
if-then-else blocks, and loops. It is responsible for updating the
exec mask to re-enable threads that had been masked during the preceding
control flow block. For example:
s_mov_b64 exec, 0x3 ; Initial exec mask
s_mov_b64 s[0:1], exec ; Saved exec mask
v_cmpx_gt_u32 exec, s[2:3], v0, 0 ; llvm.SI.if
do_stuff()
s_or_b64 exec, exec, s[0:1] ; llvm.SI.end.cf
The bug fixed by this patch was one where the llvm.SI.end.cf intrinsic
was being inserted into the header of loops. This would happen when
an if block terminated in a loop header and we would end up with
code like this:
s_mov_b64 exec, 0x3 ; Initial exec mask
s_mov_b64 s[0:1], exec ; Saved exec mask
v_cmpx_gt_u32 exec, s[2:3], v0, 0 ; llvm.SI.if
do_stuff()
LOOP: ; Start of loop header
s_or_b64 exec, exec, s[0:1] ; llvm.SI.end.cf <-BUG: The exec mask has the
same value at the beginning of each loop
iteration.
do_stuff();
s_cbranch_execnz LOOP
The fix is to create a new basic block before the loop and insert the
llvm.SI.end.cf there. This way the exec mask is restored before the
start of the loop instead of at the beginning of each iteration.
llvm-svn: 228302
2015-02-05 23:32:15 +08:00
|
|
|
AU.addRequired<LoopInfoWrapperPass>();
|
2014-01-13 21:07:17 +08:00
|
|
|
AU.addRequired<DominatorTreeWrapperPass>();
|
|
|
|
AU.addPreserved<DominatorTreeWrapperPass>();
|
2012-12-20 06:10:31 +08:00
|
|
|
FunctionPass::getAnalysisUsage(AU);
|
|
|
|
}
|
|
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
} // end anonymous namespace
|
|
|
|
|
|
|
|
char SIAnnotateControlFlow::ID = 0;
|
|
|
|
|
|
|
|
/// \brief Initialize all the types and constants used in the pass
|
|
|
|
bool SIAnnotateControlFlow::doInitialization(Module &M) {
|
|
|
|
LLVMContext &Context = M.getContext();
|
|
|
|
|
|
|
|
Void = Type::getVoidTy(Context);
|
|
|
|
Boolean = Type::getInt1Ty(Context);
|
|
|
|
Int64 = Type::getInt64Ty(Context);
|
2014-04-25 13:30:21 +08:00
|
|
|
ReturnStruct = StructType::get(Boolean, Int64, (Type *)nullptr);
|
2012-12-20 06:10:31 +08:00
|
|
|
|
|
|
|
BoolTrue = ConstantInt::getTrue(Context);
|
|
|
|
BoolFalse = ConstantInt::getFalse(Context);
|
|
|
|
BoolUndef = UndefValue::get(Boolean);
|
|
|
|
Int64Zero = ConstantInt::get(Int64, 0);
|
|
|
|
|
|
|
|
If = M.getOrInsertFunction(
|
2014-04-25 13:30:21 +08:00
|
|
|
IfIntrinsic, ReturnStruct, Boolean, (Type *)nullptr);
|
2012-12-20 06:10:31 +08:00
|
|
|
|
|
|
|
Else = M.getOrInsertFunction(
|
2014-04-25 13:30:21 +08:00
|
|
|
ElseIntrinsic, ReturnStruct, Int64, (Type *)nullptr);
|
2012-12-20 06:10:31 +08:00
|
|
|
|
|
|
|
Break = M.getOrInsertFunction(
|
2014-04-25 13:30:21 +08:00
|
|
|
BreakIntrinsic, Int64, Int64, (Type *)nullptr);
|
2012-12-20 06:10:31 +08:00
|
|
|
|
|
|
|
IfBreak = M.getOrInsertFunction(
|
2014-04-25 13:30:21 +08:00
|
|
|
IfBreakIntrinsic, Int64, Boolean, Int64, (Type *)nullptr);
|
2012-12-20 06:10:31 +08:00
|
|
|
|
|
|
|
ElseBreak = M.getOrInsertFunction(
|
2014-04-25 13:30:21 +08:00
|
|
|
ElseBreakIntrinsic, Int64, Int64, Int64, (Type *)nullptr);
|
2012-12-20 06:10:31 +08:00
|
|
|
|
|
|
|
Loop = M.getOrInsertFunction(
|
2014-04-25 13:30:21 +08:00
|
|
|
LoopIntrinsic, Boolean, Int64, (Type *)nullptr);
|
2012-12-20 06:10:31 +08:00
|
|
|
|
|
|
|
EndCf = M.getOrInsertFunction(
|
2014-04-25 13:30:21 +08:00
|
|
|
EndCfIntrinsic, Void, Int64, (Type *)nullptr);
|
2012-12-20 06:10:31 +08:00
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// \brief Is BB the last block saved on the stack ?
|
|
|
|
bool SIAnnotateControlFlow::isTopOfStack(BasicBlock *BB) {
|
2013-02-14 16:00:33 +08:00
|
|
|
return !Stack.empty() && Stack.back().first == BB;
|
2012-12-20 06:10:31 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/// \brief Pop the last saved value from the control flow stack
|
|
|
|
Value *SIAnnotateControlFlow::popSaved() {
|
|
|
|
return Stack.pop_back_val().second;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// \brief Push a BB and saved value to the control flow stack
|
|
|
|
void SIAnnotateControlFlow::push(BasicBlock *BB, Value *Saved) {
|
|
|
|
Stack.push_back(std::make_pair(BB, Saved));
|
|
|
|
}
|
|
|
|
|
|
|
|
/// \brief Can the condition represented by this PHI node treated like
|
|
|
|
/// an "Else" block?
|
|
|
|
bool SIAnnotateControlFlow::isElse(PHINode *Phi) {
|
|
|
|
BasicBlock *IDom = DT->getNode(Phi->getParent())->getIDom()->getBlock();
|
|
|
|
for (unsigned i = 0, e = Phi->getNumIncomingValues(); i != e; ++i) {
|
|
|
|
if (Phi->getIncomingBlock(i) == IDom) {
|
|
|
|
|
|
|
|
if (Phi->getIncomingValue(i) != BoolTrue)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
} else {
|
|
|
|
if (Phi->getIncomingValue(i) != BoolFalse)
|
|
|
|
return false;
|
2014-06-21 01:06:02 +08:00
|
|
|
|
2012-12-20 06:10:31 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
// \brief Erase "Phi" if it is not used any more
|
|
|
|
void SIAnnotateControlFlow::eraseIfUnused(PHINode *Phi) {
|
|
|
|
if (!Phi->hasNUsesOrMore(1))
|
|
|
|
Phi->eraseFromParent();
|
|
|
|
}
|
|
|
|
|
|
|
|
/// \brief Open a new "If" block
|
|
|
|
void SIAnnotateControlFlow::openIf(BranchInst *Term) {
|
|
|
|
Value *Ret = CallInst::Create(If, Term->getCondition(), "", Term);
|
|
|
|
Term->setCondition(ExtractValueInst::Create(Ret, 0, "", Term));
|
|
|
|
push(Term->getSuccessor(1), ExtractValueInst::Create(Ret, 1, "", Term));
|
|
|
|
}
|
|
|
|
|
|
|
|
/// \brief Close the last "If" block and open a new "Else" block
|
|
|
|
void SIAnnotateControlFlow::insertElse(BranchInst *Term) {
|
|
|
|
Value *Ret = CallInst::Create(Else, popSaved(), "", Term);
|
|
|
|
Term->setCondition(ExtractValueInst::Create(Ret, 0, "", Term));
|
|
|
|
push(Term->getSuccessor(1), ExtractValueInst::Create(Ret, 1, "", Term));
|
|
|
|
}
|
|
|
|
|
|
|
|
/// \brief Recursively handle the condition leading to a loop
|
2014-06-21 01:06:02 +08:00
|
|
|
Value *SIAnnotateControlFlow::handleLoopCondition(Value *Cond, PHINode *Broken) {
|
2012-12-20 06:10:31 +08:00
|
|
|
if (PHINode *Phi = dyn_cast<PHINode>(Cond)) {
|
2014-06-21 01:06:02 +08:00
|
|
|
BasicBlock *Parent = Phi->getParent();
|
|
|
|
PHINode *NewPhi = PHINode::Create(Int64, 0, "", &Parent->front());
|
|
|
|
Value *Ret = NewPhi;
|
2012-12-20 06:10:31 +08:00
|
|
|
|
2013-12-05 13:44:44 +08:00
|
|
|
// Handle all non-constant incoming values first
|
2012-12-20 06:10:31 +08:00
|
|
|
for (unsigned i = 0, e = Phi->getNumIncomingValues(); i != e; ++i) {
|
|
|
|
Value *Incoming = Phi->getIncomingValue(i);
|
2014-06-21 01:06:02 +08:00
|
|
|
BasicBlock *From = Phi->getIncomingBlock(i);
|
|
|
|
if (isa<ConstantInt>(Incoming)) {
|
|
|
|
NewPhi->addIncoming(Broken, From);
|
2012-12-20 06:10:31 +08:00
|
|
|
continue;
|
2014-06-21 01:06:02 +08:00
|
|
|
}
|
2012-12-20 06:10:31 +08:00
|
|
|
|
|
|
|
Phi->setIncomingValue(i, BoolFalse);
|
2014-06-21 01:06:02 +08:00
|
|
|
Value *PhiArg = handleLoopCondition(Incoming, Broken);
|
|
|
|
NewPhi->addIncoming(PhiArg, From);
|
2012-12-20 06:10:31 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
BasicBlock *IDom = DT->getNode(Parent)->getIDom()->getBlock();
|
|
|
|
|
|
|
|
for (unsigned i = 0, e = Phi->getNumIncomingValues(); i != e; ++i) {
|
|
|
|
|
|
|
|
Value *Incoming = Phi->getIncomingValue(i);
|
|
|
|
if (Incoming != BoolTrue)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
BasicBlock *From = Phi->getIncomingBlock(i);
|
|
|
|
if (From == IDom) {
|
|
|
|
CallInst *OldEnd = dyn_cast<CallInst>(Parent->getFirstInsertionPt());
|
|
|
|
if (OldEnd && OldEnd->getCalledFunction() == EndCf) {
|
2014-06-21 01:06:02 +08:00
|
|
|
Value *Args[] = { OldEnd->getArgOperand(0), NewPhi };
|
|
|
|
Ret = CallInst::Create(ElseBreak, Args, "", OldEnd);
|
2012-12-20 06:10:31 +08:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
TerminatorInst *Insert = From->getTerminator();
|
2014-06-21 01:06:02 +08:00
|
|
|
Value *PhiArg = CallInst::Create(Break, Broken, "", Insert);
|
|
|
|
NewPhi->setIncomingValue(i, PhiArg);
|
2012-12-20 06:10:31 +08:00
|
|
|
}
|
|
|
|
eraseIfUnused(Phi);
|
2014-06-21 01:06:02 +08:00
|
|
|
return Ret;
|
2012-12-20 06:10:31 +08:00
|
|
|
|
|
|
|
} else if (Instruction *Inst = dyn_cast<Instruction>(Cond)) {
|
|
|
|
BasicBlock *Parent = Inst->getParent();
|
|
|
|
TerminatorInst *Insert = Parent->getTerminator();
|
2014-06-21 01:06:02 +08:00
|
|
|
Value *Args[] = { Cond, Broken };
|
|
|
|
return CallInst::Create(IfBreak, Args, "", Insert);
|
2012-12-20 06:10:31 +08:00
|
|
|
|
|
|
|
} else {
|
2013-12-11 05:37:42 +08:00
|
|
|
llvm_unreachable("Unhandled loop condition!");
|
2012-12-20 06:10:31 +08:00
|
|
|
}
|
2014-06-21 01:06:02 +08:00
|
|
|
return 0;
|
2012-12-20 06:10:31 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/// \brief Handle a back edge (loop)
|
|
|
|
void SIAnnotateControlFlow::handleLoop(BranchInst *Term) {
|
|
|
|
BasicBlock *Target = Term->getSuccessor(1);
|
|
|
|
PHINode *Broken = PHINode::Create(Int64, 0, "", &Target->front());
|
|
|
|
|
|
|
|
Value *Cond = Term->getCondition();
|
|
|
|
Term->setCondition(BoolTrue);
|
2014-06-21 01:06:02 +08:00
|
|
|
Value *Arg = handleLoopCondition(Cond, Broken);
|
2012-12-20 06:10:31 +08:00
|
|
|
|
|
|
|
BasicBlock *BB = Term->getParent();
|
|
|
|
for (pred_iterator PI = pred_begin(Target), PE = pred_end(Target);
|
|
|
|
PI != PE; ++PI) {
|
|
|
|
|
|
|
|
Broken->addIncoming(*PI == BB ? Arg : Int64Zero, *PI);
|
|
|
|
}
|
|
|
|
|
|
|
|
Term->setCondition(CallInst::Create(Loop, Arg, "", Term));
|
|
|
|
push(Term->getSuccessor(0), Arg);
|
R600/SI: Fix bug from insertion of llvm.SI.end.cf into loop headers
The llvm.SI.end.cf intrinsic is used to mark the end of if-then blocks,
if-then-else blocks, and loops. It is responsible for updating the
exec mask to re-enable threads that had been masked during the preceding
control flow block. For example:
s_mov_b64 exec, 0x3 ; Initial exec mask
s_mov_b64 s[0:1], exec ; Saved exec mask
v_cmpx_gt_u32 exec, s[2:3], v0, 0 ; llvm.SI.if
do_stuff()
s_or_b64 exec, exec, s[0:1] ; llvm.SI.end.cf
The bug fixed by this patch was one where the llvm.SI.end.cf intrinsic
was being inserted into the header of loops. This would happen when
an if block terminated in a loop header and we would end up with
code like this:
s_mov_b64 exec, 0x3 ; Initial exec mask
s_mov_b64 s[0:1], exec ; Saved exec mask
v_cmpx_gt_u32 exec, s[2:3], v0, 0 ; llvm.SI.if
do_stuff()
LOOP: ; Start of loop header
s_or_b64 exec, exec, s[0:1] ; llvm.SI.end.cf <-BUG: The exec mask has the
same value at the beginning of each loop
iteration.
do_stuff();
s_cbranch_execnz LOOP
The fix is to create a new basic block before the loop and insert the
llvm.SI.end.cf there. This way the exec mask is restored before the
start of the loop instead of at the beginning of each iteration.
llvm-svn: 228302
2015-02-05 23:32:15 +08:00
|
|
|
}/// \brief Close the last opened control flow
|
2012-12-20 06:10:31 +08:00
|
|
|
void SIAnnotateControlFlow::closeControlFlow(BasicBlock *BB) {
|
R600/SI: Fix bug from insertion of llvm.SI.end.cf into loop headers
The llvm.SI.end.cf intrinsic is used to mark the end of if-then blocks,
if-then-else blocks, and loops. It is responsible for updating the
exec mask to re-enable threads that had been masked during the preceding
control flow block. For example:
s_mov_b64 exec, 0x3 ; Initial exec mask
s_mov_b64 s[0:1], exec ; Saved exec mask
v_cmpx_gt_u32 exec, s[2:3], v0, 0 ; llvm.SI.if
do_stuff()
s_or_b64 exec, exec, s[0:1] ; llvm.SI.end.cf
The bug fixed by this patch was one where the llvm.SI.end.cf intrinsic
was being inserted into the header of loops. This would happen when
an if block terminated in a loop header and we would end up with
code like this:
s_mov_b64 exec, 0x3 ; Initial exec mask
s_mov_b64 s[0:1], exec ; Saved exec mask
v_cmpx_gt_u32 exec, s[2:3], v0, 0 ; llvm.SI.if
do_stuff()
LOOP: ; Start of loop header
s_or_b64 exec, exec, s[0:1] ; llvm.SI.end.cf <-BUG: The exec mask has the
same value at the beginning of each loop
iteration.
do_stuff();
s_cbranch_execnz LOOP
The fix is to create a new basic block before the loop and insert the
llvm.SI.end.cf there. This way the exec mask is restored before the
start of the loop instead of at the beginning of each iteration.
llvm-svn: 228302
2015-02-05 23:32:15 +08:00
|
|
|
llvm::Loop *L = LI->getLoopFor(BB);
|
|
|
|
|
|
|
|
if (L && L->getHeader() == BB) {
|
|
|
|
// We can't insert an EndCF call into a loop header, because it will
|
|
|
|
// get executed on every iteration of the loop, when it should be
|
|
|
|
// executed only once before the loop.
|
|
|
|
SmallVector <BasicBlock*, 8> Latches;
|
|
|
|
L->getLoopLatches(Latches);
|
|
|
|
|
|
|
|
std::vector<BasicBlock*> Preds;
|
|
|
|
for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB); PI != PE; ++PI) {
|
|
|
|
if (std::find(Latches.begin(), Latches.end(), *PI) == Latches.end())
|
|
|
|
Preds.push_back(*PI);
|
|
|
|
}
|
|
|
|
BB = llvm::SplitBlockPredecessors(BB, Preds, "endcf.split", nullptr, DT,
|
|
|
|
LI, false);
|
|
|
|
}
|
|
|
|
|
2012-12-20 06:10:31 +08:00
|
|
|
CallInst::Create(EndCf, popSaved(), "", BB->getFirstInsertionPt());
|
|
|
|
}
|
|
|
|
|
|
|
|
/// \brief Annotate the control flow with intrinsics so the backend can
|
|
|
|
/// recognize if/then/else and loops.
|
|
|
|
bool SIAnnotateControlFlow::runOnFunction(Function &F) {
|
2014-01-13 21:07:17 +08:00
|
|
|
DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
|
R600/SI: Fix bug from insertion of llvm.SI.end.cf into loop headers
The llvm.SI.end.cf intrinsic is used to mark the end of if-then blocks,
if-then-else blocks, and loops. It is responsible for updating the
exec mask to re-enable threads that had been masked during the preceding
control flow block. For example:
s_mov_b64 exec, 0x3 ; Initial exec mask
s_mov_b64 s[0:1], exec ; Saved exec mask
v_cmpx_gt_u32 exec, s[2:3], v0, 0 ; llvm.SI.if
do_stuff()
s_or_b64 exec, exec, s[0:1] ; llvm.SI.end.cf
The bug fixed by this patch was one where the llvm.SI.end.cf intrinsic
was being inserted into the header of loops. This would happen when
an if block terminated in a loop header and we would end up with
code like this:
s_mov_b64 exec, 0x3 ; Initial exec mask
s_mov_b64 s[0:1], exec ; Saved exec mask
v_cmpx_gt_u32 exec, s[2:3], v0, 0 ; llvm.SI.if
do_stuff()
LOOP: ; Start of loop header
s_or_b64 exec, exec, s[0:1] ; llvm.SI.end.cf <-BUG: The exec mask has the
same value at the beginning of each loop
iteration.
do_stuff();
s_cbranch_execnz LOOP
The fix is to create a new basic block before the loop and insert the
llvm.SI.end.cf there. This way the exec mask is restored before the
start of the loop instead of at the beginning of each iteration.
llvm-svn: 228302
2015-02-05 23:32:15 +08:00
|
|
|
LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
|
2012-12-20 06:10:31 +08:00
|
|
|
|
|
|
|
for (df_iterator<BasicBlock *> I = df_begin(&F.getEntryBlock()),
|
|
|
|
E = df_end(&F.getEntryBlock()); I != E; ++I) {
|
|
|
|
|
|
|
|
BranchInst *Term = dyn_cast<BranchInst>((*I)->getTerminator());
|
|
|
|
|
|
|
|
if (!Term || Term->isUnconditional()) {
|
|
|
|
if (isTopOfStack(*I))
|
|
|
|
closeControlFlow(*I);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (I.nodeVisited(Term->getSuccessor(1))) {
|
|
|
|
if (isTopOfStack(*I))
|
|
|
|
closeControlFlow(*I);
|
|
|
|
handleLoop(Term);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (isTopOfStack(*I)) {
|
|
|
|
PHINode *Phi = dyn_cast<PHINode>(Term->getCondition());
|
|
|
|
if (Phi && Phi->getParent() == *I && isElse(Phi)) {
|
|
|
|
insertElse(Term);
|
|
|
|
eraseIfUnused(Phi);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
closeControlFlow(*I);
|
|
|
|
}
|
|
|
|
openIf(Term);
|
|
|
|
}
|
|
|
|
|
|
|
|
assert(Stack.empty());
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// \brief Create the annotation pass
|
|
|
|
FunctionPass *llvm::createSIAnnotateControlFlowPass() {
|
|
|
|
return new SIAnnotateControlFlow();
|
|
|
|
}
|