2019-06-07 15:35:30 +08:00
|
|
|
//===-- HardwareLoops.cpp - Target Independent Hardware Loops --*- C++ -*-===//
|
|
|
|
//
|
|
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
/// \file
|
|
|
|
/// Insert hardware loop intrinsics into loops which are deemed profitable by
|
|
|
|
/// the target, by querying TargetTransformInfo. A hardware loop comprises of
|
|
|
|
/// two intrinsics: one, outside the loop, to set the loop iteration count and
|
|
|
|
/// another, in the exit block, to decrement the counter. The decremented value
|
|
|
|
/// can either be carried through the loop via a phi or handled in some opaque
|
|
|
|
/// way by the target.
|
|
|
|
///
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
2019-07-10 01:53:09 +08:00
|
|
|
#include "llvm/Pass.h"
|
2019-06-07 15:35:30 +08:00
|
|
|
#include "llvm/PassRegistry.h"
|
|
|
|
#include "llvm/PassSupport.h"
|
|
|
|
#include "llvm/ADT/Statistic.h"
|
|
|
|
#include "llvm/Analysis/AssumptionCache.h"
|
|
|
|
#include "llvm/Analysis/LoopInfo.h"
|
|
|
|
#include "llvm/Analysis/ScalarEvolution.h"
|
|
|
|
#include "llvm/Analysis/ScalarEvolutionExpander.h"
|
|
|
|
#include "llvm/Analysis/TargetTransformInfo.h"
|
|
|
|
#include "llvm/CodeGen/Passes.h"
|
|
|
|
#include "llvm/CodeGen/TargetPassConfig.h"
|
|
|
|
#include "llvm/IR/BasicBlock.h"
|
|
|
|
#include "llvm/IR/DataLayout.h"
|
|
|
|
#include "llvm/IR/Dominators.h"
|
|
|
|
#include "llvm/IR/Constants.h"
|
|
|
|
#include "llvm/IR/IRBuilder.h"
|
|
|
|
#include "llvm/IR/Instructions.h"
|
|
|
|
#include "llvm/IR/IntrinsicInst.h"
|
|
|
|
#include "llvm/IR/Value.h"
|
|
|
|
#include "llvm/Support/Debug.h"
|
|
|
|
#include "llvm/Transforms/Scalar.h"
|
2019-07-10 01:53:09 +08:00
|
|
|
#include "llvm/Transforms/Utils.h"
|
2019-06-07 15:35:30 +08:00
|
|
|
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
|
|
|
|
#include "llvm/Transforms/Utils/Local.h"
|
2019-07-10 01:53:09 +08:00
|
|
|
#include "llvm/Transforms/Utils/LoopUtils.h"
|
2019-06-07 15:35:30 +08:00
|
|
|
|
|
|
|
#define DEBUG_TYPE "hardware-loops"
|
|
|
|
|
|
|
|
#define HW_LOOPS_NAME "Hardware Loop Insertion"
|
|
|
|
|
|
|
|
using namespace llvm;
|
|
|
|
|
|
|
|
static cl::opt<bool>
|
|
|
|
ForceHardwareLoops("force-hardware-loops", cl::Hidden, cl::init(false),
|
|
|
|
cl::desc("Force hardware loops intrinsics to be inserted"));
|
|
|
|
|
|
|
|
static cl::opt<bool>
|
|
|
|
ForceHardwareLoopPHI(
|
|
|
|
"force-hardware-loop-phi", cl::Hidden, cl::init(false),
|
|
|
|
cl::desc("Force hardware loop counter to be updated through a phi"));
|
|
|
|
|
|
|
|
static cl::opt<bool>
|
|
|
|
ForceNestedLoop("force-nested-hardware-loop", cl::Hidden, cl::init(false),
|
|
|
|
cl::desc("Force allowance of nested hardware loops"));
|
|
|
|
|
|
|
|
static cl::opt<unsigned>
|
|
|
|
LoopDecrement("hardware-loop-decrement", cl::Hidden, cl::init(1),
|
|
|
|
cl::desc("Set the loop decrement value"));
|
|
|
|
|
|
|
|
static cl::opt<unsigned>
|
|
|
|
CounterBitWidth("hardware-loop-counter-bitwidth", cl::Hidden, cl::init(32),
|
|
|
|
cl::desc("Set the loop counter bitwidth"));
|
|
|
|
|
2019-06-28 15:38:16 +08:00
|
|
|
static cl::opt<bool>
|
|
|
|
ForceGuardLoopEntry(
|
|
|
|
"force-hardware-loop-guard", cl::Hidden, cl::init(false),
|
|
|
|
cl::desc("Force generation of loop guard intrinsic"));
|
|
|
|
|
2019-06-07 15:35:30 +08:00
|
|
|
STATISTIC(NumHWLoops, "Number of loops converted to hardware loops");
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
|
|
|
using TTI = TargetTransformInfo;
|
|
|
|
|
|
|
|
class HardwareLoops : public FunctionPass {
|
|
|
|
public:
|
|
|
|
static char ID;
|
|
|
|
|
|
|
|
HardwareLoops() : FunctionPass(ID) {
|
|
|
|
initializeHardwareLoopsPass(*PassRegistry::getPassRegistry());
|
|
|
|
}
|
|
|
|
|
|
|
|
bool runOnFunction(Function &F) override;
|
|
|
|
|
|
|
|
void getAnalysisUsage(AnalysisUsage &AU) const override {
|
|
|
|
AU.addRequired<LoopInfoWrapperPass>();
|
|
|
|
AU.addPreserved<LoopInfoWrapperPass>();
|
|
|
|
AU.addRequired<DominatorTreeWrapperPass>();
|
|
|
|
AU.addPreserved<DominatorTreeWrapperPass>();
|
|
|
|
AU.addRequired<ScalarEvolutionWrapperPass>();
|
|
|
|
AU.addRequired<AssumptionCacheTracker>();
|
|
|
|
AU.addRequired<TargetTransformInfoWrapperPass>();
|
|
|
|
}
|
|
|
|
|
|
|
|
// Try to convert the given Loop into a hardware loop.
|
|
|
|
bool TryConvertLoop(Loop *L);
|
|
|
|
|
|
|
|
// Given that the target believes the loop to be profitable, try to
|
|
|
|
// convert it.
|
2019-06-19 09:26:31 +08:00
|
|
|
bool TryConvertLoop(HardwareLoopInfo &HWLoopInfo);
|
2019-06-07 15:35:30 +08:00
|
|
|
|
|
|
|
private:
|
|
|
|
ScalarEvolution *SE = nullptr;
|
|
|
|
LoopInfo *LI = nullptr;
|
|
|
|
const DataLayout *DL = nullptr;
|
|
|
|
const TargetTransformInfo *TTI = nullptr;
|
|
|
|
DominatorTree *DT = nullptr;
|
2019-07-10 01:53:09 +08:00
|
|
|
bool PreserveLCSSA = false;
|
2019-06-07 15:35:30 +08:00
|
|
|
AssumptionCache *AC = nullptr;
|
|
|
|
TargetLibraryInfo *LibInfo = nullptr;
|
|
|
|
Module *M = nullptr;
|
|
|
|
bool MadeChange = false;
|
|
|
|
};
|
|
|
|
|
|
|
|
class HardwareLoop {
|
|
|
|
// Expand the trip count scev into a value that we can use.
|
2019-06-28 15:38:16 +08:00
|
|
|
Value *InitLoopCount();
|
2019-06-07 15:35:30 +08:00
|
|
|
|
|
|
|
// Insert the set_loop_iteration intrinsic.
|
2019-06-28 15:38:16 +08:00
|
|
|
void InsertIterationSetup(Value *LoopCountInit);
|
2019-06-07 15:35:30 +08:00
|
|
|
|
|
|
|
// Insert the loop_decrement intrinsic.
|
|
|
|
void InsertLoopDec();
|
|
|
|
|
|
|
|
// Insert the loop_decrement_reg intrinsic.
|
|
|
|
Instruction *InsertLoopRegDec(Value *EltsRem);
|
|
|
|
|
|
|
|
// If the target requires the counter value to be updated in the loop,
|
|
|
|
// insert a phi to hold the value. The intended purpose is for use by
|
|
|
|
// loop_decrement_reg.
|
|
|
|
PHINode *InsertPHICounter(Value *NumElts, Value *EltsRem);
|
|
|
|
|
|
|
|
// Create a new cmp, that checks the returned value of loop_decrement*,
|
|
|
|
// and update the exit branch to use it.
|
|
|
|
void UpdateBranch(Value *EltsRem);
|
|
|
|
|
|
|
|
public:
|
2019-06-19 09:26:31 +08:00
|
|
|
HardwareLoop(HardwareLoopInfo &Info, ScalarEvolution &SE,
|
2019-10-16 18:55:06 +08:00
|
|
|
const DataLayout &DL) :
|
|
|
|
SE(SE), DL(DL), L(Info.L), M(L->getHeader()->getModule()),
|
2019-06-07 15:35:30 +08:00
|
|
|
ExitCount(Info.ExitCount),
|
|
|
|
CountType(Info.CountType),
|
|
|
|
ExitBranch(Info.ExitBranch),
|
|
|
|
LoopDecrement(Info.LoopDecrement),
|
2019-06-28 15:38:16 +08:00
|
|
|
UsePHICounter(Info.CounterInReg),
|
|
|
|
UseLoopGuard(Info.PerformEntryTest) { }
|
2019-06-07 15:35:30 +08:00
|
|
|
|
|
|
|
void Create();
|
|
|
|
|
|
|
|
private:
|
|
|
|
ScalarEvolution &SE;
|
|
|
|
const DataLayout &DL;
|
|
|
|
Loop *L = nullptr;
|
|
|
|
Module *M = nullptr;
|
|
|
|
const SCEV *ExitCount = nullptr;
|
|
|
|
Type *CountType = nullptr;
|
|
|
|
BranchInst *ExitBranch = nullptr;
|
2019-06-28 15:38:16 +08:00
|
|
|
Value *LoopDecrement = nullptr;
|
2019-06-07 15:35:30 +08:00
|
|
|
bool UsePHICounter = false;
|
2019-06-28 15:38:16 +08:00
|
|
|
bool UseLoopGuard = false;
|
|
|
|
BasicBlock *BeginBB = nullptr;
|
2019-06-07 15:35:30 +08:00
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
char HardwareLoops::ID = 0;
|
|
|
|
|
|
|
|
bool HardwareLoops::runOnFunction(Function &F) {
|
|
|
|
if (skipFunction(F))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
LLVM_DEBUG(dbgs() << "HWLoops: Running on " << F.getName() << "\n");
|
|
|
|
|
|
|
|
LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
|
|
|
|
SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
|
|
|
|
DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
|
|
|
|
TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
|
|
|
|
DL = &F.getParent()->getDataLayout();
|
|
|
|
auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
|
Change TargetLibraryInfo analysis passes to always require Function
Summary:
This is the first change to enable the TLI to be built per-function so
that -fno-builtin* handling can be migrated to use function attributes.
See discussion on D61634 for background. This is an enabler for fixing
handling of these options for LTO, for example.
This change should not affect behavior, as the provided function is not
yet used to build a specifically per-function TLI, but rather enables
that migration.
Most of the changes were very mechanical, e.g. passing a Function to the
legacy analysis pass's getTLI interface, or in Module level cases,
adding a callback. This is similar to the way the per-function TTI
analysis works.
There was one place where we were looking for builtins but not in the
context of a specific function. See FindCXAAtExit in
lib/Transforms/IPO/GlobalOpt.cpp. I'm somewhat concerned my workaround
could provide the wrong behavior in some corner cases. Suggestions
welcome.
Reviewers: chandlerc, hfinkel
Subscribers: arsenm, dschuff, jvesely, nhaehnle, mehdi_amini, javed.absar, sbc100, jgravelle-google, eraman, aheejin, steven_wu, george.burgess.iv, dexonsmith, jfb, asbirlea, gchatelet, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D66428
llvm-svn: 371284
2019-09-07 11:09:36 +08:00
|
|
|
LibInfo = TLIP ? &TLIP->getTLI(F) : nullptr;
|
2019-07-10 01:53:09 +08:00
|
|
|
PreserveLCSSA = mustPreserveAnalysisID(LCSSAID);
|
2019-06-07 15:35:30 +08:00
|
|
|
AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
|
|
|
|
M = F.getParent();
|
|
|
|
|
|
|
|
for (LoopInfo::iterator I = LI->begin(), E = LI->end(); I != E; ++I) {
|
|
|
|
Loop *L = *I;
|
|
|
|
if (!L->getParentLoop())
|
|
|
|
TryConvertLoop(L);
|
|
|
|
}
|
|
|
|
|
|
|
|
return MadeChange;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Return true if the search should stop, which will be when an inner loop is
|
|
|
|
// converted and the parent loop doesn't support containing a hardware loop.
|
|
|
|
bool HardwareLoops::TryConvertLoop(Loop *L) {
|
|
|
|
// Process nested loops first.
|
2019-10-16 18:55:06 +08:00
|
|
|
for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I)
|
|
|
|
if (TryConvertLoop(*I))
|
2019-06-07 15:35:30 +08:00
|
|
|
return true; // Stop search.
|
|
|
|
|
2019-06-19 09:26:31 +08:00
|
|
|
HardwareLoopInfo HWLoopInfo(L);
|
2019-10-16 18:55:06 +08:00
|
|
|
if (!HWLoopInfo.canAnalyze(*LI))
|
2019-06-26 20:02:43 +08:00
|
|
|
return false;
|
|
|
|
|
2019-10-16 18:55:06 +08:00
|
|
|
if (TTI->isHardwareLoopProfitable(L, *SE, *AC, LibInfo, HWLoopInfo) ||
|
|
|
|
ForceHardwareLoops) {
|
|
|
|
|
|
|
|
// Allow overriding of the counter width and loop decrement value.
|
|
|
|
if (CounterBitWidth.getNumOccurrences())
|
|
|
|
HWLoopInfo.CountType =
|
|
|
|
IntegerType::get(M->getContext(), CounterBitWidth);
|
2019-06-07 15:35:30 +08:00
|
|
|
|
2019-10-16 18:55:06 +08:00
|
|
|
if (LoopDecrement.getNumOccurrences())
|
|
|
|
HWLoopInfo.LoopDecrement =
|
|
|
|
ConstantInt::get(HWLoopInfo.CountType, LoopDecrement);
|
2019-06-07 15:35:30 +08:00
|
|
|
|
2019-10-16 18:55:06 +08:00
|
|
|
MadeChange |= TryConvertLoop(HWLoopInfo);
|
|
|
|
return MadeChange && (!HWLoopInfo.IsNestingLegal && !ForceNestedLoop);
|
|
|
|
}
|
2019-06-07 15:35:30 +08:00
|
|
|
|
2019-10-16 18:55:06 +08:00
|
|
|
return false;
|
2019-06-07 15:35:30 +08:00
|
|
|
}
|
|
|
|
|
2019-06-19 09:26:31 +08:00
|
|
|
bool HardwareLoops::TryConvertLoop(HardwareLoopInfo &HWLoopInfo) {
|
2019-06-07 15:35:30 +08:00
|
|
|
|
2019-07-10 01:53:09 +08:00
|
|
|
Loop *L = HWLoopInfo.L;
|
|
|
|
LLVM_DEBUG(dbgs() << "HWLoops: Try to convert profitable loop: " << *L);
|
2019-06-07 15:35:30 +08:00
|
|
|
|
2019-06-19 09:26:31 +08:00
|
|
|
if (!HWLoopInfo.isHardwareLoopCandidate(*SE, *LI, *DT, ForceNestedLoop,
|
2019-10-16 18:55:06 +08:00
|
|
|
ForceHardwareLoopPHI))
|
2019-06-07 15:35:30 +08:00
|
|
|
return false;
|
|
|
|
|
2019-06-19 09:26:31 +08:00
|
|
|
assert(
|
|
|
|
(HWLoopInfo.ExitBlock && HWLoopInfo.ExitBranch && HWLoopInfo.ExitCount) &&
|
|
|
|
"Hardware Loop must have set exit info.");
|
|
|
|
|
2019-07-10 01:53:09 +08:00
|
|
|
BasicBlock *Preheader = L->getLoopPreheader();
|
|
|
|
|
|
|
|
// If we don't have a preheader, then insert one.
|
|
|
|
if (!Preheader)
|
|
|
|
Preheader = InsertPreheaderForLoop(L, DT, LI, nullptr, PreserveLCSSA);
|
|
|
|
if (!Preheader)
|
|
|
|
return false;
|
|
|
|
|
2019-10-16 18:55:06 +08:00
|
|
|
HardwareLoop HWLoop(HWLoopInfo, *SE, *DL);
|
2019-06-07 15:35:30 +08:00
|
|
|
HWLoop.Create();
|
|
|
|
++NumHWLoops;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
void HardwareLoop::Create() {
|
|
|
|
LLVM_DEBUG(dbgs() << "HWLoops: Converting loop..\n");
|
2019-10-16 18:55:06 +08:00
|
|
|
|
2019-06-28 15:38:16 +08:00
|
|
|
Value *LoopCountInit = InitLoopCount();
|
2019-10-16 18:55:06 +08:00
|
|
|
if (!LoopCountInit)
|
2019-07-10 01:53:09 +08:00
|
|
|
return;
|
2019-06-07 15:35:30 +08:00
|
|
|
|
2019-06-28 15:38:16 +08:00
|
|
|
InsertIterationSetup(LoopCountInit);
|
2019-06-07 15:35:30 +08:00
|
|
|
|
|
|
|
if (UsePHICounter || ForceHardwareLoopPHI) {
|
|
|
|
Instruction *LoopDec = InsertLoopRegDec(LoopCountInit);
|
|
|
|
Value *EltsRem = InsertPHICounter(LoopCountInit, LoopDec);
|
|
|
|
LoopDec->setOperand(0, EltsRem);
|
|
|
|
UpdateBranch(LoopDec);
|
|
|
|
} else
|
|
|
|
InsertLoopDec();
|
|
|
|
|
|
|
|
// Run through the basic blocks of the loop and see if any of them have dead
|
|
|
|
// PHIs that can be removed.
|
|
|
|
for (auto I : L->blocks())
|
|
|
|
DeleteDeadPHIs(I);
|
|
|
|
}
|
|
|
|
|
2019-06-28 15:38:16 +08:00
|
|
|
static bool CanGenerateTest(Loop *L, Value *Count) {
|
|
|
|
BasicBlock *Preheader = L->getLoopPreheader();
|
|
|
|
if (!Preheader->getSinglePredecessor())
|
|
|
|
return false;
|
|
|
|
|
|
|
|
BasicBlock *Pred = Preheader->getSinglePredecessor();
|
|
|
|
if (!isa<BranchInst>(Pred->getTerminator()))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
auto *BI = cast<BranchInst>(Pred->getTerminator());
|
|
|
|
if (BI->isUnconditional() || !isa<ICmpInst>(BI->getCondition()))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
// Check that the icmp is checking for equality of Count and zero and that
|
|
|
|
// a non-zero value results in entering the loop.
|
|
|
|
auto ICmp = cast<ICmpInst>(BI->getCondition());
|
2019-07-01 16:21:28 +08:00
|
|
|
LLVM_DEBUG(dbgs() << " - Found condition: " << *ICmp << "\n");
|
2019-06-28 15:38:16 +08:00
|
|
|
if (!ICmp->isEquality())
|
|
|
|
return false;
|
|
|
|
|
|
|
|
auto IsCompareZero = [](ICmpInst *ICmp, Value *Count, unsigned OpIdx) {
|
|
|
|
if (auto *Const = dyn_cast<ConstantInt>(ICmp->getOperand(OpIdx)))
|
|
|
|
return Const->isZero() && ICmp->getOperand(OpIdx ^ 1) == Count;
|
|
|
|
return false;
|
|
|
|
};
|
|
|
|
|
|
|
|
if (!IsCompareZero(ICmp, Count, 0) && !IsCompareZero(ICmp, Count, 1))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
unsigned SuccIdx = ICmp->getPredicate() == ICmpInst::ICMP_NE ? 0 : 1;
|
|
|
|
if (BI->getSuccessor(SuccIdx) != Preheader)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
Value *HardwareLoop::InitLoopCount() {
|
|
|
|
LLVM_DEBUG(dbgs() << "HWLoops: Initialising loop counter value:\n");
|
|
|
|
// Can we replace a conditional branch with an intrinsic that sets the
|
|
|
|
// loop counter and tests that is not zero?
|
|
|
|
|
2019-06-07 15:35:30 +08:00
|
|
|
SCEVExpander SCEVE(SE, DL, "loopcnt");
|
2019-07-10 01:53:09 +08:00
|
|
|
if (!ExitCount->getType()->isPointerTy() &&
|
|
|
|
ExitCount->getType() != CountType)
|
|
|
|
ExitCount = SE.getZeroExtendExpr(ExitCount, CountType);
|
|
|
|
|
|
|
|
ExitCount = SE.getAddExpr(ExitCount, SE.getOne(CountType));
|
2019-06-07 15:35:30 +08:00
|
|
|
|
2019-06-28 15:38:16 +08:00
|
|
|
// If we're trying to use the 'test and set' form of the intrinsic, we need
|
|
|
|
// to replace a conditional branch that is controlling entry to the loop. It
|
|
|
|
// is likely (guaranteed?) that the preheader has an unconditional branch to
|
|
|
|
// the loop header, so also check if it has a single predecessor.
|
|
|
|
if (SE.isLoopEntryGuardedByCond(L, ICmpInst::ICMP_NE, ExitCount,
|
2019-07-10 01:53:09 +08:00
|
|
|
SE.getZero(ExitCount->getType()))) {
|
|
|
|
LLVM_DEBUG(dbgs() << " - Attempting to use test.set counter.\n");
|
2019-06-28 15:38:16 +08:00
|
|
|
UseLoopGuard |= ForceGuardLoopEntry;
|
2019-07-10 01:53:09 +08:00
|
|
|
} else
|
2019-06-28 15:38:16 +08:00
|
|
|
UseLoopGuard = false;
|
|
|
|
|
|
|
|
BasicBlock *BB = L->getLoopPreheader();
|
|
|
|
if (UseLoopGuard && BB->getSinglePredecessor() &&
|
2019-07-10 01:53:09 +08:00
|
|
|
cast<BranchInst>(BB->getTerminator())->isUnconditional())
|
2019-06-28 15:38:16 +08:00
|
|
|
BB = BB->getSinglePredecessor();
|
2019-07-10 01:53:09 +08:00
|
|
|
|
|
|
|
if (!isSafeToExpandAt(ExitCount, BB->getTerminator(), SE)) {
|
|
|
|
LLVM_DEBUG(dbgs() << "- Bailing, unsafe to expand ExitCount "
|
|
|
|
<< *ExitCount << "\n");
|
|
|
|
return nullptr;
|
2019-06-07 15:35:30 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
Value *Count = SCEVE.expandCodeFor(ExitCount, CountType,
|
|
|
|
BB->getTerminator());
|
2019-06-28 15:38:16 +08:00
|
|
|
|
|
|
|
// FIXME: We've expanded Count where we hope to insert the counter setting
|
|
|
|
// intrinsic. But, in the case of the 'test and set' form, we may fallback to
|
|
|
|
// the just 'set' form and in which case the insertion block is most likely
|
|
|
|
// different. It means there will be instruction(s) in a block that possibly
|
|
|
|
// aren't needed. The isLoopEntryGuardedByCond is trying to avoid this issue,
|
|
|
|
// but it's doesn't appear to work in all cases.
|
|
|
|
|
|
|
|
UseLoopGuard = UseLoopGuard && CanGenerateTest(L, Count);
|
|
|
|
BeginBB = UseLoopGuard ? BB : L->getLoopPreheader();
|
|
|
|
LLVM_DEBUG(dbgs() << " - Loop Count: " << *Count << "\n"
|
|
|
|
<< " - Expanded Count in " << BB->getName() << "\n"
|
|
|
|
<< " - Will insert set counter intrinsic into: "
|
|
|
|
<< BeginBB->getName() << "\n");
|
2019-06-07 15:35:30 +08:00
|
|
|
return Count;
|
|
|
|
}
|
|
|
|
|
2019-06-28 15:38:16 +08:00
|
|
|
void HardwareLoop::InsertIterationSetup(Value *LoopCountInit) {
|
|
|
|
IRBuilder<> Builder(BeginBB->getTerminator());
|
2019-06-07 15:35:30 +08:00
|
|
|
Type *Ty = LoopCountInit->getType();
|
2019-06-28 15:38:16 +08:00
|
|
|
Intrinsic::ID ID = UseLoopGuard ?
|
|
|
|
Intrinsic::test_set_loop_iterations : Intrinsic::set_loop_iterations;
|
|
|
|
Function *LoopIter = Intrinsic::getDeclaration(M, ID, Ty);
|
|
|
|
Value *SetCount = Builder.CreateCall(LoopIter, LoopCountInit);
|
|
|
|
|
|
|
|
// Use the return value of the intrinsic to control the entry of the loop.
|
|
|
|
if (UseLoopGuard) {
|
|
|
|
assert((isa<BranchInst>(BeginBB->getTerminator()) &&
|
|
|
|
cast<BranchInst>(BeginBB->getTerminator())->isConditional()) &&
|
|
|
|
"Expected conditional branch");
|
|
|
|
auto *LoopGuard = cast<BranchInst>(BeginBB->getTerminator());
|
|
|
|
LoopGuard->setCondition(SetCount);
|
|
|
|
if (LoopGuard->getSuccessor(0) != L->getLoopPreheader())
|
|
|
|
LoopGuard->swapSuccessors();
|
|
|
|
}
|
|
|
|
LLVM_DEBUG(dbgs() << "HWLoops: Inserted loop counter: "
|
|
|
|
<< *SetCount << "\n");
|
2019-06-07 15:35:30 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void HardwareLoop::InsertLoopDec() {
|
|
|
|
IRBuilder<> CondBuilder(ExitBranch);
|
|
|
|
|
|
|
|
Function *DecFunc =
|
|
|
|
Intrinsic::getDeclaration(M, Intrinsic::loop_decrement,
|
|
|
|
LoopDecrement->getType());
|
|
|
|
Value *Ops[] = { LoopDecrement };
|
|
|
|
Value *NewCond = CondBuilder.CreateCall(DecFunc, Ops);
|
|
|
|
Value *OldCond = ExitBranch->getCondition();
|
|
|
|
ExitBranch->setCondition(NewCond);
|
|
|
|
|
|
|
|
// The false branch must exit the loop.
|
|
|
|
if (!L->contains(ExitBranch->getSuccessor(0)))
|
|
|
|
ExitBranch->swapSuccessors();
|
|
|
|
|
|
|
|
// The old condition may be dead now, and may have even created a dead PHI
|
|
|
|
// (the original induction variable).
|
|
|
|
RecursivelyDeleteTriviallyDeadInstructions(OldCond);
|
|
|
|
|
|
|
|
LLVM_DEBUG(dbgs() << "HWLoops: Inserted loop dec: " << *NewCond << "\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
Instruction* HardwareLoop::InsertLoopRegDec(Value *EltsRem) {
|
|
|
|
IRBuilder<> CondBuilder(ExitBranch);
|
|
|
|
|
|
|
|
Function *DecFunc =
|
|
|
|
Intrinsic::getDeclaration(M, Intrinsic::loop_decrement_reg,
|
|
|
|
{ EltsRem->getType(), EltsRem->getType(),
|
|
|
|
LoopDecrement->getType()
|
|
|
|
});
|
|
|
|
Value *Ops[] = { EltsRem, LoopDecrement };
|
|
|
|
Value *Call = CondBuilder.CreateCall(DecFunc, Ops);
|
|
|
|
|
|
|
|
LLVM_DEBUG(dbgs() << "HWLoops: Inserted loop dec: " << *Call << "\n");
|
|
|
|
return cast<Instruction>(Call);
|
|
|
|
}
|
|
|
|
|
|
|
|
PHINode* HardwareLoop::InsertPHICounter(Value *NumElts, Value *EltsRem) {
|
|
|
|
BasicBlock *Preheader = L->getLoopPreheader();
|
|
|
|
BasicBlock *Header = L->getHeader();
|
|
|
|
BasicBlock *Latch = ExitBranch->getParent();
|
|
|
|
IRBuilder<> Builder(Header->getFirstNonPHI());
|
|
|
|
PHINode *Index = Builder.CreatePHI(NumElts->getType(), 2);
|
|
|
|
Index->addIncoming(NumElts, Preheader);
|
|
|
|
Index->addIncoming(EltsRem, Latch);
|
|
|
|
LLVM_DEBUG(dbgs() << "HWLoops: PHI Counter: " << *Index << "\n");
|
|
|
|
return Index;
|
|
|
|
}
|
|
|
|
|
|
|
|
void HardwareLoop::UpdateBranch(Value *EltsRem) {
|
|
|
|
IRBuilder<> CondBuilder(ExitBranch);
|
|
|
|
Value *NewCond =
|
|
|
|
CondBuilder.CreateICmpNE(EltsRem, ConstantInt::get(EltsRem->getType(), 0));
|
|
|
|
Value *OldCond = ExitBranch->getCondition();
|
|
|
|
ExitBranch->setCondition(NewCond);
|
|
|
|
|
|
|
|
// The false branch must exit the loop.
|
|
|
|
if (!L->contains(ExitBranch->getSuccessor(0)))
|
|
|
|
ExitBranch->swapSuccessors();
|
|
|
|
|
|
|
|
// The old condition may be dead now, and may have even created a dead PHI
|
|
|
|
// (the original induction variable).
|
|
|
|
RecursivelyDeleteTriviallyDeadInstructions(OldCond);
|
|
|
|
}
|
|
|
|
|
|
|
|
INITIALIZE_PASS_BEGIN(HardwareLoops, DEBUG_TYPE, HW_LOOPS_NAME, false, false)
|
|
|
|
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
|
|
|
|
INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
|
|
|
|
INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
|
|
|
|
INITIALIZE_PASS_END(HardwareLoops, DEBUG_TYPE, HW_LOOPS_NAME, false, false)
|
|
|
|
|
|
|
|
FunctionPass *llvm::createHardwareLoopsPass() { return new HardwareLoops(); }
|