[Transforms] Fix some Clang-tidy modernize and Include What You Use warnings; other minor fixes (NFC).

llvm-svn: 316034
This commit is contained in:
Eugene Zelenko 2017-10-17 21:27:42 +00:00
parent 7cd4e2c96f
commit 6cadde7f40
14 changed files with 409 additions and 206 deletions

View File

@ -1,4 +1,4 @@
//===- NaryReassociate.h - Reassociate n-ary expressions ------------------===// //===- NaryReassociate.h - Reassociate n-ary expressions --------*- C++ -*-===//
// //
// The LLVM Compiler Infrastructure // The LLVM Compiler Infrastructure
// //
@ -81,15 +81,25 @@
#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/PassManager.h" #include "llvm/IR/PassManager.h"
#include "llvm/IR/ValueHandle.h"
namespace llvm { namespace llvm {
class AssumptionCache;
class BinaryOperator;
class DataLayout;
class DominatorTree;
class Function;
class GetElementPtrInst;
class Instruction;
class ScalarEvolution;
class SCEV;
class TargetLibraryInfo;
class TargetTransformInfo;
class Type;
class Value;
class NaryReassociatePass : public PassInfoMixin<NaryReassociatePass> { class NaryReassociatePass : public PassInfoMixin<NaryReassociatePass> {
public: public:
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
@ -109,6 +119,7 @@ private:
// Reassociate GEP for better CSE. // Reassociate GEP for better CSE.
Instruction *tryReassociateGEP(GetElementPtrInst *GEP); Instruction *tryReassociateGEP(GetElementPtrInst *GEP);
// Try splitting GEP at the I-th index and see whether either part can be // Try splitting GEP at the I-th index and see whether either part can be
// CSE'ed. This is a helper function for tryReassociateGEP. // CSE'ed. This is a helper function for tryReassociateGEP.
// //
@ -118,6 +129,7 @@ private:
// ..., i-th index). // ..., i-th index).
GetElementPtrInst *tryReassociateGEPAtIndex(GetElementPtrInst *GEP, GetElementPtrInst *tryReassociateGEPAtIndex(GetElementPtrInst *GEP,
unsigned I, Type *IndexedType); unsigned I, Type *IndexedType);
// Given GEP's I-th index = LHS + RHS, see whether &Base[..][LHS][..] or // Given GEP's I-th index = LHS + RHS, see whether &Base[..][LHS][..] or
// &Base[..][RHS][..] can be CSE'ed and rewrite GEP accordingly. // &Base[..][RHS][..] can be CSE'ed and rewrite GEP accordingly.
GetElementPtrInst *tryReassociateGEPAtIndex(GetElementPtrInst *GEP, GetElementPtrInst *tryReassociateGEPAtIndex(GetElementPtrInst *GEP,
@ -146,6 +158,7 @@ private:
// \c CandidateExpr. Returns null if not found. // \c CandidateExpr. Returns null if not found.
Instruction *findClosestMatchingDominator(const SCEV *CandidateExpr, Instruction *findClosestMatchingDominator(const SCEV *CandidateExpr,
Instruction *Dominatee); Instruction *Dominatee);
// GetElementPtrInst implicitly sign-extends an index if the index is shorter // GetElementPtrInst implicitly sign-extends an index if the index is shorter
// than the pointer size. This function returns whether Index is shorter than // than the pointer size. This function returns whether Index is shorter than
// GEP's pointer size, i.e., whether Index needs to be sign-extended in order // GEP's pointer size, i.e., whether Index needs to be sign-extended in order
@ -158,6 +171,7 @@ private:
ScalarEvolution *SE; ScalarEvolution *SE;
TargetLibraryInfo *TLI; TargetLibraryInfo *TLI;
TargetTransformInfo *TTI; TargetTransformInfo *TTI;
// A lookup table quickly telling which instructions compute the given SCEV. // A lookup table quickly telling which instructions compute the given SCEV.
// Note that there can be multiple instructions at different locations // Note that there can be multiple instructions at different locations
// computing to the same SCEV, so we map a SCEV to an instruction list. For // computing to the same SCEV, so we map a SCEV to an instruction list. For
@ -169,6 +183,7 @@ private:
// bar(a + b); // bar(a + b);
DenseMap<const SCEV *, SmallVector<WeakTrackingVH, 2>> SeenExprs; DenseMap<const SCEV *, SmallVector<WeakTrackingVH, 2>> SeenExprs;
}; };
} // namespace llvm
} // end namespace llvm
#endif // LLVM_TRANSFORMS_SCALAR_NARYREASSOCIATE_H #endif // LLVM_TRANSFORMS_SCALAR_NARYREASSOCIATE_H

View File

@ -1,4 +1,4 @@
//===- AddDiscriminators.h -------------------------------------*- C++ -*-===// //===- AddDiscriminators.h --------------------------------------*- C++ -*-===//
// //
// The LLVM Compiler Infrastructure // The LLVM Compiler Infrastructure
// //
@ -20,10 +20,13 @@
namespace llvm { namespace llvm {
class Function;
class AddDiscriminatorsPass : public PassInfoMixin<AddDiscriminatorsPass> { class AddDiscriminatorsPass : public PassInfoMixin<AddDiscriminatorsPass> {
public: public:
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
}; };
} // end namespace llvm } // end namespace llvm
#endif // LLVM_TRANSFORMS_UTILS_ADDDISCRIMINATORS_H #endif // LLVM_TRANSFORMS_UTILS_ADDDISCRIMINATORS_H

View File

@ -1,4 +1,4 @@
//===- llvm/Transforms/Utils/BypassSlowDivision.h --------------*- C++ -*-===// //===- llvm/Transforms/Utils/BypassSlowDivision.h ---------------*- C++ -*-===//
// //
// The LLVM Compiler Infrastructure // The LLVM Compiler Infrastructure
// //
@ -19,10 +19,14 @@
#define LLVM_TRANSFORMS_UTILS_BYPASSSLOWDIVISION_H #define LLVM_TRANSFORMS_UTILS_BYPASSSLOWDIVISION_H
#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMap.h"
#include "llvm/IR/Function.h" #include "llvm/ADT/DenseMapInfo.h"
#include <cstdint>
namespace llvm { namespace llvm {
class BasicBlock;
class Value;
struct DivRemMapKey { struct DivRemMapKey {
bool SignedOp; bool SignedOp;
Value *Dividend; Value *Dividend;
@ -61,6 +65,6 @@ template <> struct DenseMapInfo<DivRemMapKey> {
bool bypassSlowDivision( bool bypassSlowDivision(
BasicBlock *BB, const DenseMap<unsigned int, unsigned int> &BypassWidth); BasicBlock *BB, const DenseMap<unsigned int, unsigned int> &BypassWidth);
} // End llvm namespace } // end namespace llvm
#endif #endif // LLVM_TRANSFORMS_UTILS_BYPASSSLOWDIVISION_H

View File

@ -1,4 +1,4 @@
//===-- Local.h - Functions to perform local transformations ----*- C++ -*-===// //===- Local.h - Functions to perform local transformations -----*- C++ -*-===//
// //
// The LLVM Compiler Infrastructure // The LLVM Compiler Infrastructure
// //
@ -15,41 +15,44 @@
#ifndef LLVM_TRANSFORMS_UTILS_LOCAL_H #ifndef LLVM_TRANSFORMS_UTILS_LOCAL_H
#define LLVM_TRANSFORMS_UTILS_LOCAL_H #define LLVM_TRANSFORMS_UTILS_LOCAL_H
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/TinyPtrVector.h" #include "llvm/ADT/TinyPtrVector.h"
#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h" #include "llvm/IR/DataLayout.h"
#include "llvm/IR/Dominators.h" #include "llvm/IR/Dominators.h"
#include "llvm/IR/GetElementPtrTypeIterator.h" #include "llvm/IR/GetElementPtrTypeIterator.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Operator.h" #include "llvm/IR/Operator.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Casting.h"
#include <cstdint>
#include <limits>
namespace llvm { namespace llvm {
class User;
class BasicBlock;
class Function;
class BranchInst;
class Instruction;
class CallInst;
class DbgDeclareInst;
class DbgInfoIntrinsic;
class DbgValueInst;
class StoreInst;
class LoadInst;
class Value;
class PHINode;
class AllocaInst; class AllocaInst;
class AssumptionCache; class AssumptionCache;
class ConstantExpr; class BasicBlock;
class DataLayout; class BranchInst;
class CallInst;
class DbgInfoIntrinsic;
class DbgValueInst;
class DIBuilder;
class Function;
class Instruction;
class LazyValueInfo;
class LoadInst;
class MDNode;
class PHINode;
class StoreInst;
class TargetLibraryInfo; class TargetLibraryInfo;
class TargetTransformInfo; class TargetTransformInfo;
class DIBuilder;
class DominatorTree;
class LazyValueInfo;
template<typename T> class SmallVectorImpl;
/// A set of parameters used to control the transforms in the SimplifyCFG pass. /// A set of parameters used to control the transforms in the SimplifyCFG pass.
/// Options may change depending on the position in the optimization pipeline. /// Options may change depending on the position in the optimization pipeline.
@ -66,8 +69,7 @@ struct SimplifyCFGOptions {
AssumptionCache *AssumpCache = nullptr) AssumptionCache *AssumpCache = nullptr)
: BonusInstThreshold(BonusThreshold), : BonusInstThreshold(BonusThreshold),
ConvertSwitchToLookupTable(SwitchToLookup), ConvertSwitchToLookupTable(SwitchToLookup),
NeedCanonicalLoop(CanonicalLoops), NeedCanonicalLoop(CanonicalLoops), AC(AssumpCache) {}
AC(AssumpCache) {}
}; };
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
@ -229,7 +231,8 @@ Value *EmitGEPOffset(IRBuilderTy *Builder, const DataLayout &DL, User *GEP,
// Build a mask for high order bits. // Build a mask for high order bits.
unsigned IntPtrWidth = IntPtrTy->getScalarType()->getIntegerBitWidth(); unsigned IntPtrWidth = IntPtrTy->getScalarType()->getIntegerBitWidth();
uint64_t PtrSizeMask = ~0ULL >> (64 - IntPtrWidth); uint64_t PtrSizeMask =
std::numeric_limits<uint64_t>::max() >> (64 - IntPtrWidth);
gep_type_iterator GTI = gep_type_begin(GEP); gep_type_iterator GTI = gep_type_begin(GEP);
for (User::op_iterator i = GEP->op_begin() + 1, e = GEP->op_end(); i != e; for (User::op_iterator i = GEP->op_begin() + 1, e = GEP->op_end(); i != e;
@ -390,7 +393,6 @@ unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT,
unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT, unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT,
const BasicBlock *BB); const BasicBlock *BB);
/// Return true if the CallSite CS calls a gc leaf function. /// Return true if the CallSite CS calls a gc leaf function.
/// ///
/// A leaf function is a function that does not safepoint the thread during its /// A leaf function is a function that does not safepoint the thread during its
@ -452,6 +454,6 @@ void maybeMarkSanitizerLibraryCallNoBuiltin(CallInst *CI,
/// value? /// value?
bool canReplaceOperandWithVariable(const Instruction *I, unsigned OpIdx); bool canReplaceOperandWithVariable(const Instruction *I, unsigned OpIdx);
} // End llvm namespace } // end namespace llvm
#endif #endif // LLVM_TRANSFORMS_UTILS_LOCAL_H

View File

@ -21,18 +21,26 @@
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
#include "AMDGPU.h" #include "AMDGPU.h"
#include "llvm/ADT/DepthFirstIterator.h" #include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/DivergenceAnalysis.h" #include "llvm/Analysis/DivergenceAnalysis.h"
#include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/PostDominators.h"
#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/BasicBlock.h" #include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h" #include "llvm/IR/CFG.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Function.h" #include "llvm/IR/Function.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instructions.h" #include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Type.h" #include "llvm/IR/Type.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/Local.h"
using namespace llvm; using namespace llvm;
#define DEBUG_TYPE "amdgpu-unify-divergent-exit-nodes" #define DEBUG_TYPE "amdgpu-unify-divergent-exit-nodes"
@ -42,6 +50,7 @@ namespace {
class AMDGPUUnifyDivergentExitNodes : public FunctionPass { class AMDGPUUnifyDivergentExitNodes : public FunctionPass {
public: public:
static char ID; // Pass identification, replacement for typeid static char ID; // Pass identification, replacement for typeid
AMDGPUUnifyDivergentExitNodes() : FunctionPass(ID) { AMDGPUUnifyDivergentExitNodes() : FunctionPass(ID) {
initializeAMDGPUUnifyDivergentExitNodesPass(*PassRegistry::getPassRegistry()); initializeAMDGPUUnifyDivergentExitNodesPass(*PassRegistry::getPassRegistry());
} }
@ -51,9 +60,12 @@ public:
bool runOnFunction(Function &F) override; bool runOnFunction(Function &F) override;
}; };
} } // end anonymous namespace
char AMDGPUUnifyDivergentExitNodes::ID = 0; char AMDGPUUnifyDivergentExitNodes::ID = 0;
char &llvm::AMDGPUUnifyDivergentExitNodesID = AMDGPUUnifyDivergentExitNodes::ID;
INITIALIZE_PASS_BEGIN(AMDGPUUnifyDivergentExitNodes, DEBUG_TYPE, INITIALIZE_PASS_BEGIN(AMDGPUUnifyDivergentExitNodes, DEBUG_TYPE,
"Unify divergent function exit nodes", false, false) "Unify divergent function exit nodes", false, false)
INITIALIZE_PASS_DEPENDENCY(PostDominatorTreeWrapperPass) INITIALIZE_PASS_DEPENDENCY(PostDominatorTreeWrapperPass)
@ -61,8 +73,6 @@ INITIALIZE_PASS_DEPENDENCY(DivergenceAnalysis)
INITIALIZE_PASS_END(AMDGPUUnifyDivergentExitNodes, DEBUG_TYPE, INITIALIZE_PASS_END(AMDGPUUnifyDivergentExitNodes, DEBUG_TYPE,
"Unify divergent function exit nodes", false, false) "Unify divergent function exit nodes", false, false)
char &llvm::AMDGPUUnifyDivergentExitNodesID = AMDGPUUnifyDivergentExitNodes::ID;
void AMDGPUUnifyDivergentExitNodes::getAnalysisUsage(AnalysisUsage &AU) const{ void AMDGPUUnifyDivergentExitNodes::getAnalysisUsage(AnalysisUsage &AU) const{
// TODO: Preserve dominator tree. // TODO: Preserve dominator tree.
AU.addRequired<PostDominatorTreeWrapperPass>(); AU.addRequired<PostDominatorTreeWrapperPass>();
@ -113,7 +123,6 @@ static BasicBlock *unifyReturnBlockSet(Function &F,
// Otherwise, we need to insert a new basic block into the function, add a PHI // Otherwise, we need to insert a new basic block into the function, add a PHI
// nodes (if the function returns values), and convert all of the return // nodes (if the function returns values), and convert all of the return
// instructions into unconditional branches. // instructions into unconditional branches.
//
BasicBlock *NewRetBlock = BasicBlock::Create(F.getContext(), Name, &F); BasicBlock *NewRetBlock = BasicBlock::Create(F.getContext(), Name, &F);
PHINode *PN = nullptr; PHINode *PN = nullptr;
@ -129,7 +138,6 @@ static BasicBlock *unifyReturnBlockSet(Function &F,
// Loop over all of the blocks, replacing the return instruction with an // Loop over all of the blocks, replacing the return instruction with an
// unconditional branch. // unconditional branch.
//
for (BasicBlock *BB : ReturningBlocks) { for (BasicBlock *BB : ReturningBlocks) {
// Add an incoming element to the PHI node for every return instruction that // Add an incoming element to the PHI node for every return instruction that
// is merging into this new block... // is merging into this new block...
@ -157,7 +165,6 @@ bool AMDGPUUnifyDivergentExitNodes::runOnFunction(Function &F) {
// Loop over all of the blocks in a function, tracking all of the blocks that // Loop over all of the blocks in a function, tracking all of the blocks that
// return. // return.
//
SmallVector<BasicBlock *, 4> ReturningBlocks; SmallVector<BasicBlock *, 4> ReturningBlocks;
SmallVector<BasicBlock *, 4> UnreachableBlocks; SmallVector<BasicBlock *, 4> UnreachableBlocks;

View File

@ -77,19 +77,45 @@
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
#include "llvm/Transforms/Scalar/NaryReassociate.h" #include "llvm/Transforms/Scalar/NaryReassociate.h"
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h" #include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GetElementPtrTypeIterator.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Module.h" #include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PatternMatch.h" #include "llvm/IR/PatternMatch.h"
#include "llvm/Support/Debug.h" #include "llvm/IR/Type.h"
#include "llvm/Support/raw_ostream.h" #include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/Local.h"
#include <cassert>
#include <cstdint>
using namespace llvm; using namespace llvm;
using namespace PatternMatch; using namespace PatternMatch;
#define DEBUG_TYPE "nary-reassociate" #define DEBUG_TYPE "nary-reassociate"
namespace { namespace {
class NaryReassociateLegacyPass : public FunctionPass { class NaryReassociateLegacyPass : public FunctionPass {
public: public:
static char ID; static char ID;
@ -101,6 +127,7 @@ public:
bool doInitialization(Module &M) override { bool doInitialization(Module &M) override {
return false; return false;
} }
bool runOnFunction(Function &F) override; bool runOnFunction(Function &F) override;
void getAnalysisUsage(AnalysisUsage &AU) const override { void getAnalysisUsage(AnalysisUsage &AU) const override {
@ -118,9 +145,11 @@ public:
private: private:
NaryReassociatePass Impl; NaryReassociatePass Impl;
}; };
} // anonymous namespace
} // end anonymous namespace
char NaryReassociateLegacyPass::ID = 0; char NaryReassociateLegacyPass::ID = 0;
INITIALIZE_PASS_BEGIN(NaryReassociateLegacyPass, "nary-reassociate", INITIALIZE_PASS_BEGIN(NaryReassociateLegacyPass, "nary-reassociate",
"Nary reassociation", false, false) "Nary reassociation", false, false)
INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)

View File

@ -50,31 +50,45 @@
// //
// For more details about DWARF discriminators, please visit // For more details about DWARF discriminators, please visit
// http://wiki.dwarfstd.org/index.php?title=Path_Discriminators // http://wiki.dwarfstd.org/index.php?title=Path_Discriminators
//
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
#include "llvm/Transforms/Utils/AddDiscriminators.h" #include "llvm/Transforms/Utils/AddDiscriminators.h"
#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/IR/BasicBlock.h" #include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DebugInfo.h" #include "llvm/IR/Function.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h" #include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/LLVMContext.h" #include "llvm/IR/PassManager.h"
#include "llvm/Pass.h" #include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h" #include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h" #include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h" #include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Scalar.h"
#include <utility>
using namespace llvm; using namespace llvm;
#define DEBUG_TYPE "add-discriminators" #define DEBUG_TYPE "add-discriminators"
// Command line option to disable discriminator generation even in the
// presence of debug information. This is only needed when debugging
// debug info generation issues.
static cl::opt<bool> NoDiscriminators(
"no-discriminators", cl::init(false),
cl::desc("Disable generation of discriminator information."));
namespace { namespace {
// The legacy pass of AddDiscriminators. // The legacy pass of AddDiscriminators.
struct AddDiscriminatorsLegacyPass : public FunctionPass { struct AddDiscriminatorsLegacyPass : public FunctionPass {
static char ID; // Pass identification, replacement for typeid static char ID; // Pass identification, replacement for typeid
AddDiscriminatorsLegacyPass() : FunctionPass(ID) { AddDiscriminatorsLegacyPass() : FunctionPass(ID) {
initializeAddDiscriminatorsLegacyPassPass(*PassRegistry::getPassRegistry()); initializeAddDiscriminatorsLegacyPassPass(*PassRegistry::getPassRegistry());
} }
@ -85,18 +99,12 @@ struct AddDiscriminatorsLegacyPass : public FunctionPass {
} // end anonymous namespace } // end anonymous namespace
char AddDiscriminatorsLegacyPass::ID = 0; char AddDiscriminatorsLegacyPass::ID = 0;
INITIALIZE_PASS_BEGIN(AddDiscriminatorsLegacyPass, "add-discriminators", INITIALIZE_PASS_BEGIN(AddDiscriminatorsLegacyPass, "add-discriminators",
"Add DWARF path discriminators", false, false) "Add DWARF path discriminators", false, false)
INITIALIZE_PASS_END(AddDiscriminatorsLegacyPass, "add-discriminators", INITIALIZE_PASS_END(AddDiscriminatorsLegacyPass, "add-discriminators",
"Add DWARF path discriminators", false, false) "Add DWARF path discriminators", false, false)
// Command line option to disable discriminator generation even in the
// presence of debug information. This is only needed when debugging
// debug info generation issues.
static cl::opt<bool> NoDiscriminators(
"no-discriminators", cl::init(false),
cl::desc("Disable generation of discriminator information."));
// Create the legacy AddDiscriminatorsPass. // Create the legacy AddDiscriminatorsPass.
FunctionPass *llvm::createAddDiscriminatorsPass() { FunctionPass *llvm::createAddDiscriminatorsPass() {
return new AddDiscriminatorsLegacyPass(); return new AddDiscriminatorsLegacyPass();
@ -166,11 +174,11 @@ static bool addDiscriminators(Function &F) {
bool Changed = false; bool Changed = false;
typedef std::pair<StringRef, unsigned> Location; using Location = std::pair<StringRef, unsigned>;
typedef DenseSet<const BasicBlock *> BBSet; using BBSet = DenseSet<const BasicBlock *>;
typedef DenseMap<Location, BBSet> LocationBBMap; using LocationBBMap = DenseMap<Location, BBSet>;
typedef DenseMap<Location, unsigned> LocationDiscriminatorMap; using LocationDiscriminatorMap = DenseMap<Location, unsigned>;
typedef DenseSet<Location> LocationSet; using LocationSet = DenseSet<Location>;
LocationBBMap LBM; LocationBBMap LBM;
LocationDiscriminatorMap LDM; LocationDiscriminatorMap LDM;
@ -242,6 +250,7 @@ static bool addDiscriminators(Function &F) {
bool AddDiscriminatorsLegacyPass::runOnFunction(Function &F) { bool AddDiscriminatorsLegacyPass::runOnFunction(Function &F) {
return addDiscriminators(F); return addDiscriminators(F);
} }
PreservedAnalyses AddDiscriminatorsPass::run(Function &F, PreservedAnalyses AddDiscriminatorsPass::run(Function &F,
FunctionAnalysisManager &AM) { FunctionAnalysisManager &AM) {
if (!addDiscriminators(F)) if (!addDiscriminators(F))

View File

@ -1,4 +1,4 @@
//===-- BypassSlowDivision.cpp - Bypass slow division ---------------------===// //===- BypassSlowDivision.cpp - Bypass slow division ----------------------===//
// //
// The LLVM Compiler Infrastructure // The LLVM Compiler Infrastructure
// //
@ -17,19 +17,33 @@
#include "llvm/Transforms/Utils/BypassSlowDivision.h" #include "llvm/Transforms/Utils/BypassSlowDivision.h"
#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallPtrSet.h"
#include "llvm/Analysis/ValueTracking.h" #include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h" #include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h" #include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h" #include "llvm/IR/Instructions.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/KnownBits.h" #include "llvm/Support/KnownBits.h"
#include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/Local.h"
#include <cassert>
#include <cstdint>
using namespace llvm; using namespace llvm;
#define DEBUG_TYPE "bypass-slow-division" #define DEBUG_TYPE "bypass-slow-division"
namespace { namespace {
struct QuotRemPair { struct QuotRemPair {
Value *Quotient; Value *Quotient;
Value *Remainder; Value *Remainder;
@ -46,15 +60,11 @@ namespace {
Value *Quotient = nullptr; Value *Quotient = nullptr;
Value *Remainder = nullptr; Value *Remainder = nullptr;
}; };
}
namespace llvm { using DivCacheTy = DenseMap<DivRemMapKey, QuotRemPair>;
typedef DenseMap<DivRemMapKey, QuotRemPair> DivCacheTy; using BypassWidthsTy = DenseMap<unsigned, unsigned>;
typedef DenseMap<unsigned, unsigned> BypassWidthsTy; using VisitedSetTy = SmallPtrSet<Instruction *, 4>;
typedef SmallPtrSet<Instruction *, 4> VisitedSetTy;
}
namespace {
enum ValueRange { enum ValueRange {
/// Operand definitely fits into BypassType. No runtime checks are needed. /// Operand definitely fits into BypassType. No runtime checks are needed.
VALRNG_KNOWN_SHORT, VALRNG_KNOWN_SHORT,
@ -84,17 +94,21 @@ class FastDivInsertionTask {
return SlowDivOrRem->getOpcode() == Instruction::SDiv || return SlowDivOrRem->getOpcode() == Instruction::SDiv ||
SlowDivOrRem->getOpcode() == Instruction::SRem; SlowDivOrRem->getOpcode() == Instruction::SRem;
} }
bool isDivisionOp() { bool isDivisionOp() {
return SlowDivOrRem->getOpcode() == Instruction::SDiv || return SlowDivOrRem->getOpcode() == Instruction::SDiv ||
SlowDivOrRem->getOpcode() == Instruction::UDiv; SlowDivOrRem->getOpcode() == Instruction::UDiv;
} }
Type *getSlowType() { return SlowDivOrRem->getType(); } Type *getSlowType() { return SlowDivOrRem->getType(); }
public: public:
FastDivInsertionTask(Instruction *I, const BypassWidthsTy &BypassWidths); FastDivInsertionTask(Instruction *I, const BypassWidthsTy &BypassWidths);
Value *getReplacement(DivCacheTy &Cache); Value *getReplacement(DivCacheTy &Cache);
}; };
} // anonymous namespace
} // end anonymous namespace
FastDivInsertionTask::FastDivInsertionTask(Instruction *I, FastDivInsertionTask::FastDivInsertionTask(Instruction *I,
const BypassWidthsTy &BypassWidths) { const BypassWidthsTy &BypassWidths) {
@ -193,7 +207,7 @@ bool FastDivInsertionTask::isHashLikeValue(Value *V, VisitedSetTy &Visited) {
C = dyn_cast<ConstantInt>(cast<BitCastInst>(Op1)->getOperand(0)); C = dyn_cast<ConstantInt>(cast<BitCastInst>(Op1)->getOperand(0));
return C && C->getValue().getMinSignedBits() > BypassType->getBitWidth(); return C && C->getValue().getMinSignedBits() > BypassType->getBitWidth();
} }
case Instruction::PHI: { case Instruction::PHI:
// Stop IR traversal in case of a crazy input code. This limits recursion // Stop IR traversal in case of a crazy input code. This limits recursion
// depth. // depth.
if (Visited.size() >= 16) if (Visited.size() >= 16)
@ -209,7 +223,6 @@ bool FastDivInsertionTask::isHashLikeValue(Value *V, VisitedSetTy &Visited) {
return getValueRange(V, Visited) == VALRNG_LIKELY_LONG || return getValueRange(V, Visited) == VALRNG_LIKELY_LONG ||
isa<UndefValue>(V); isa<UndefValue>(V);
}); });
}
default: default:
return false; return false;
} }

View File

@ -12,11 +12,15 @@
// //
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h" #include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BlockFrequencyInfo.h"
@ -26,25 +30,46 @@
#include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/ValueTracking.h" #include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Attributes.h" #include "llvm/IR/Argument.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h" #include "llvm/IR/CFG.h"
#include "llvm/IR/CallSite.h" #include "llvm/IR/CallSite.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h" #include "llvm/IR/Constants.h"
#include "llvm/IR/DIBuilder.h" #include "llvm/IR/DIBuilder.h"
#include "llvm/IR/DataLayout.h" #include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugInfo.h" #include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h" #include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h" #include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h" #include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h" #include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h" #include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h" #include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h" #include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Transforms/Utils/Cloning.h" #include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/ValueMapper.h"
#include <algorithm> #include <algorithm>
#include <cassert>
#include <cstdint>
#include <iterator>
#include <limits>
#include <string>
#include <utility>
#include <vector>
using namespace llvm; using namespace llvm;
@ -62,28 +87,37 @@ bool llvm::InlineFunction(CallInst *CI, InlineFunctionInfo &IFI,
AAResults *CalleeAAR, bool InsertLifetime) { AAResults *CalleeAAR, bool InsertLifetime) {
return InlineFunction(CallSite(CI), IFI, CalleeAAR, InsertLifetime); return InlineFunction(CallSite(CI), IFI, CalleeAAR, InsertLifetime);
} }
bool llvm::InlineFunction(InvokeInst *II, InlineFunctionInfo &IFI, bool llvm::InlineFunction(InvokeInst *II, InlineFunctionInfo &IFI,
AAResults *CalleeAAR, bool InsertLifetime) { AAResults *CalleeAAR, bool InsertLifetime) {
return InlineFunction(CallSite(II), IFI, CalleeAAR, InsertLifetime); return InlineFunction(CallSite(II), IFI, CalleeAAR, InsertLifetime);
} }
namespace { namespace {
/// A class for recording information about inlining a landing pad. /// A class for recording information about inlining a landing pad.
class LandingPadInliningInfo { class LandingPadInliningInfo {
BasicBlock *OuterResumeDest; ///< Destination of the invoke's unwind. /// Destination of the invoke's unwind.
BasicBlock *InnerResumeDest; ///< Destination for the callee's resume. BasicBlock *OuterResumeDest;
LandingPadInst *CallerLPad; ///< LandingPadInst associated with the invoke.
PHINode *InnerEHValuesPHI; ///< PHI for EH values from landingpad insts. /// Destination for the callee's resume.
BasicBlock *InnerResumeDest = nullptr;
/// LandingPadInst associated with the invoke.
LandingPadInst *CallerLPad = nullptr;
/// PHI for EH values from landingpad insts.
PHINode *InnerEHValuesPHI = nullptr;
SmallVector<Value*, 8> UnwindDestPHIValues; SmallVector<Value*, 8> UnwindDestPHIValues;
public: public:
LandingPadInliningInfo(InvokeInst *II) LandingPadInliningInfo(InvokeInst *II)
: OuterResumeDest(II->getUnwindDest()), InnerResumeDest(nullptr), : OuterResumeDest(II->getUnwindDest()) {
CallerLPad(nullptr), InnerEHValuesPHI(nullptr) {
// If there are PHI nodes in the unwind destination block, we need to keep // If there are PHI nodes in the unwind destination block, we need to keep
// track of which values came into them from the invoke before removing // track of which values came into them from the invoke before removing
// the edge from this block. // the edge from this block.
llvm::BasicBlock *InvokeBB = II->getParent(); BasicBlock *InvokeBB = II->getParent();
BasicBlock::iterator I = OuterResumeDest->begin(); BasicBlock::iterator I = OuterResumeDest->begin();
for (; isa<PHINode>(I); ++I) { for (; isa<PHINode>(I); ++I) {
// Save the value to use for this edge. // Save the value to use for this edge.
@ -126,7 +160,8 @@ namespace {
} }
} }
}; };
} // anonymous namespace
} // end anonymous namespace
/// Get or create a target for the branch from ResumeInsts. /// Get or create a target for the branch from ResumeInsts.
BasicBlock *LandingPadInliningInfo::getInnerResumeDest() { BasicBlock *LandingPadInliningInfo::getInnerResumeDest() {
@ -189,7 +224,7 @@ static Value *getParentPad(Value *EHPad) {
return cast<CatchSwitchInst>(EHPad)->getParentPad(); return cast<CatchSwitchInst>(EHPad)->getParentPad();
} }
typedef DenseMap<Instruction *, Value *> UnwindDestMemoTy; using UnwindDestMemoTy = DenseMap<Instruction *, Value *>;
/// Helper for getUnwindDestToken that does the descendant-ward part of /// Helper for getUnwindDestToken that does the descendant-ward part of
/// the search. /// the search.
@ -617,7 +652,7 @@ static void HandleInlinedEHPad(InvokeInst *II, BasicBlock *FirstNewBlock,
// track of which values came into them from the invoke before removing the // track of which values came into them from the invoke before removing the
// edge from this block. // edge from this block.
SmallVector<Value *, 8> UnwindDestPHIValues; SmallVector<Value *, 8> UnwindDestPHIValues;
llvm::BasicBlock *InvokeBB = II->getParent(); BasicBlock *InvokeBB = II->getParent();
for (Instruction &I : *UnwindDest) { for (Instruction &I : *UnwindDest) {
// Save the value to use for this edge. // Save the value to use for this edge.
PHINode *PHI = dyn_cast<PHINode>(&I); PHINode *PHI = dyn_cast<PHINode>(&I);
@ -1359,6 +1394,7 @@ static void fixupLineNumbers(Function *Fn, Function::iterator FI,
} }
} }
} }
/// Update the block frequencies of the caller after a callee has been inlined. /// Update the block frequencies of the caller after a callee has been inlined.
/// ///
/// Each block cloned into the caller has its block frequency scaled by the /// Each block cloned into the caller has its block frequency scaled by the
@ -1848,8 +1884,9 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
// Check that array size doesn't saturate uint64_t and doesn't // Check that array size doesn't saturate uint64_t and doesn't
// overflow when it's multiplied by type size. // overflow when it's multiplied by type size.
if (AllocaArraySize != ~0ULL && if (AllocaArraySize != std::numeric_limits<uint64_t>::max() &&
UINT64_MAX / AllocaArraySize >= AllocaTypeSize) { std::numeric_limits<uint64_t>::max() / AllocaArraySize >=
AllocaTypeSize) {
AllocaSize = ConstantInt::get(Type::getInt64Ty(AI->getContext()), AllocaSize = ConstantInt::get(Type::getInt64Ty(AI->getContext()),
AllocaArraySize * AllocaTypeSize); AllocaArraySize * AllocaTypeSize);
} }
@ -1980,7 +2017,7 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
// match the callee's return type, we also need to change the return type of // match the callee's return type, we also need to change the return type of
// the intrinsic. // the intrinsic.
if (Caller->getReturnType() == TheCall->getType()) { if (Caller->getReturnType() == TheCall->getType()) {
auto NewEnd = remove_if(Returns, [](ReturnInst *RI) { auto NewEnd = llvm::remove_if(Returns, [](ReturnInst *RI) {
return RI->getParent()->getTerminatingDeoptimizeCall() != nullptr; return RI->getParent()->getTerminatingDeoptimizeCall() != nullptr;
}); });
Returns.erase(NewEnd, Returns.end()); Returns.erase(NewEnd, Returns.end());

View File

@ -1,4 +1,4 @@
//===-- Local.cpp - Functions to perform local transformations ------------===// //===- Local.cpp - Functions to perform local transformations -------------===//
// //
// The LLVM Compiler Infrastructure // The LLVM Compiler Infrastructure
// //
@ -13,42 +13,74 @@
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
#include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/Local.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseMapInfo.h"
#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/Hashing.h" #include "llvm/ADT/Hashing.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h" #include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h" #include "llvm/ADT/Statistic.h"
#include "llvm/ADT/TinyPtrVector.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/EHPersonalities.h"
#include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/LazyValueInfo.h" #include "llvm/Analysis/LazyValueInfo.h"
#include "llvm/Analysis/MemoryBuiltins.h" #include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h" #include "llvm/Analysis/ValueTracking.h"
#include "llvm/BinaryFormat/Dwarf.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h" #include "llvm/IR/CFG.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/ConstantRange.h" #include "llvm/IR/ConstantRange.h"
#include "llvm/IR/Constants.h" #include "llvm/IR/Constants.h"
#include "llvm/IR/DIBuilder.h" #include "llvm/IR/DIBuilder.h"
#include "llvm/IR/DataLayout.h" #include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugInfo.h" #include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h" #include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GetElementPtrTypeIterator.h" #include "llvm/IR/GetElementPtrTypeIterator.h"
#include "llvm/IR/GlobalAlias.h" #include "llvm/IR/GlobalObject.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h" #include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h" #include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h" #include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h" #include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Metadata.h" #include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h" #include "llvm/IR/Operator.h"
#include "llvm/IR/PatternMatch.h" #include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h" #include "llvm/IR/ValueHandle.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h" #include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h" #include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h" #include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cassert>
#include <climits>
#include <cstdint>
#include <iterator>
#include <map>
#include <utility>
using namespace llvm; using namespace llvm;
using namespace llvm::PatternMatch; using namespace llvm::PatternMatch;
@ -282,7 +314,6 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions,
return false; return false;
} }
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
// Local dead code elimination. // Local dead code elimination.
// //
@ -541,7 +572,6 @@ bool llvm::SimplifyInstructionsInBlock(BasicBlock *BB,
// Control Flow Graph Restructuring. // Control Flow Graph Restructuring.
// //
/// RemovePredecessorAndSimplify - Like BasicBlock::removePredecessor, this /// RemovePredecessorAndSimplify - Like BasicBlock::removePredecessor, this
/// method is called when we're about to delete Pred as a predecessor of BB. If /// method is called when we're about to delete Pred as a predecessor of BB. If
/// BB contains any PHI nodes, this drops the entries in the PHI nodes for Pred. /// BB contains any PHI nodes, this drops the entries in the PHI nodes for Pred.
@ -578,12 +608,10 @@ void llvm::RemovePredecessorAndSimplify(BasicBlock *BB, BasicBlock *Pred) {
} }
} }
/// MergeBasicBlockIntoOnlyPred - DestBB is a block with one predecessor and its /// MergeBasicBlockIntoOnlyPred - DestBB is a block with one predecessor and its
/// predecessor is known to have one successor (DestBB!). Eliminate the edge /// predecessor is known to have one successor (DestBB!). Eliminate the edge
/// between them, moving the instructions in the predecessor into DestBB and /// between them, moving the instructions in the predecessor into DestBB and
/// deleting the predecessor block. /// deleting the predecessor block.
///
void llvm::MergeBasicBlockIntoOnlyPred(BasicBlock *DestBB, DominatorTree *DT) { void llvm::MergeBasicBlockIntoOnlyPred(BasicBlock *DestBB, DominatorTree *DT) {
// If BB has single-entry PHI nodes, fold them. // If BB has single-entry PHI nodes, fold them.
while (PHINode *PN = dyn_cast<PHINode>(DestBB->begin())) { while (PHINode *PN = dyn_cast<PHINode>(DestBB->begin())) {
@ -602,7 +630,7 @@ void llvm::MergeBasicBlockIntoOnlyPred(BasicBlock *DestBB, DominatorTree *DT) {
if (DestBB->hasAddressTaken()) { if (DestBB->hasAddressTaken()) {
BlockAddress *BA = BlockAddress::get(DestBB); BlockAddress *BA = BlockAddress::get(DestBB);
Constant *Replacement = Constant *Replacement =
ConstantInt::get(llvm::Type::getInt32Ty(BA->getContext()), 1); ConstantInt::get(Type::getInt32Ty(BA->getContext()), 1);
BA->replaceAllUsesWith(ConstantExpr::getIntToPtr(Replacement, BA->replaceAllUsesWith(ConstantExpr::getIntToPtr(Replacement,
BA->getType())); BA->getType()));
BA->destroyConstant(); BA->destroyConstant();
@ -640,7 +668,6 @@ static bool CanMergeValues(Value *First, Value *Second) {
/// almost-empty BB ending in an unconditional branch to Succ, into Succ. /// almost-empty BB ending in an unconditional branch to Succ, into Succ.
/// ///
/// Assumption: Succ is the single successor for BB. /// Assumption: Succ is the single successor for BB.
///
static bool CanPropagatePredecessorsForPHIs(BasicBlock *BB, BasicBlock *Succ) { static bool CanPropagatePredecessorsForPHIs(BasicBlock *BB, BasicBlock *Succ) {
assert(*succ_begin(BB) == Succ && "Succ is not successor of BB!"); assert(*succ_begin(BB) == Succ && "Succ is not successor of BB!");
@ -696,8 +723,8 @@ static bool CanPropagatePredecessorsForPHIs(BasicBlock *BB, BasicBlock *Succ) {
return true; return true;
} }
typedef SmallVector<BasicBlock *, 16> PredBlockVector; using PredBlockVector = SmallVector<BasicBlock *, 16>;
typedef DenseMap<BasicBlock *, Value *> IncomingValueMap; using IncomingValueMap = DenseMap<BasicBlock *, Value *>;
/// \brief Determines the value to use as the phi node input for a block. /// \brief Determines the value to use as the phi node input for a block.
/// ///
@ -927,7 +954,6 @@ bool llvm::TryToSimplifyUncondBranchFromEmptyBlock(BasicBlock *BB) {
/// nodes in this block. This doesn't try to be clever about PHI nodes /// nodes in this block. This doesn't try to be clever about PHI nodes
/// which differ only in the order of the incoming values, but instcombine /// which differ only in the order of the incoming values, but instcombine
/// orders them so it usually won't matter. /// orders them so it usually won't matter.
///
bool llvm::EliminateDuplicatePHINodes(BasicBlock *BB) { bool llvm::EliminateDuplicatePHINodes(BasicBlock *BB) {
// This implementation doesn't currently consider undef operands // This implementation doesn't currently consider undef operands
// specially. Theoretically, two phis which are identical except for // specially. Theoretically, two phis which are identical except for
@ -937,9 +963,11 @@ bool llvm::EliminateDuplicatePHINodes(BasicBlock *BB) {
static PHINode *getEmptyKey() { static PHINode *getEmptyKey() {
return DenseMapInfo<PHINode *>::getEmptyKey(); return DenseMapInfo<PHINode *>::getEmptyKey();
} }
static PHINode *getTombstoneKey() { static PHINode *getTombstoneKey() {
return DenseMapInfo<PHINode *>::getTombstoneKey(); return DenseMapInfo<PHINode *>::getTombstoneKey();
} }
static unsigned getHashValue(PHINode *PN) { static unsigned getHashValue(PHINode *PN) {
// Compute a hash value on the operands. Instcombine will likely have // Compute a hash value on the operands. Instcombine will likely have
// sorted them, which helps expose duplicates, but we have to check all // sorted them, which helps expose duplicates, but we have to check all
@ -948,6 +976,7 @@ bool llvm::EliminateDuplicatePHINodes(BasicBlock *BB) {
hash_combine_range(PN->value_op_begin(), PN->value_op_end()), hash_combine_range(PN->value_op_begin(), PN->value_op_end()),
hash_combine_range(PN->block_begin(), PN->block_end()))); hash_combine_range(PN->block_begin(), PN->block_end())));
} }
static bool isEqual(PHINode *LHS, PHINode *RHS) { static bool isEqual(PHINode *LHS, PHINode *RHS) {
if (LHS == getEmptyKey() || LHS == getTombstoneKey() || if (LHS == getEmptyKey() || LHS == getTombstoneKey() ||
RHS == getEmptyKey() || RHS == getTombstoneKey()) RHS == getEmptyKey() || RHS == getTombstoneKey())
@ -984,7 +1013,6 @@ bool llvm::EliminateDuplicatePHINodes(BasicBlock *BB) {
/// often possible though. If alignment is important, a more reliable approach /// often possible though. If alignment is important, a more reliable approach
/// is to simply align all global variables and allocation instructions to /// is to simply align all global variables and allocation instructions to
/// their preferred alignment from the beginning. /// their preferred alignment from the beginning.
///
static unsigned enforceKnownAlignment(Value *V, unsigned Align, static unsigned enforceKnownAlignment(Value *V, unsigned Align,
unsigned PrefAlign, unsigned PrefAlign,
const DataLayout &DL) { const DataLayout &DL) {
@ -1068,7 +1096,7 @@ static bool LdStHasDebugValue(DILocalVariable *DIVar, DIExpression *DIExpr,
// Since we can't guarantee that the original dbg.declare instrinsic // Since we can't guarantee that the original dbg.declare instrinsic
// is removed by LowerDbgDeclare(), we need to make sure that we are // is removed by LowerDbgDeclare(), we need to make sure that we are
// not inserting the same dbg.value intrinsic over and over. // not inserting the same dbg.value intrinsic over and over.
llvm::BasicBlock::InstListType::iterator PrevI(I); BasicBlock::InstListType::iterator PrevI(I);
if (PrevI != I->getParent()->getInstList().begin()) { if (PrevI != I->getParent()->getInstList().begin()) {
--PrevI; --PrevI;
if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(PrevI)) if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(PrevI))
@ -1488,7 +1516,6 @@ BasicBlock *llvm::changeToInvokeAndSplitBasicBlock(CallInst *CI,
static bool markAliveBlocks(Function &F, static bool markAliveBlocks(Function &F,
SmallPtrSetImpl<BasicBlock*> &Reachable) { SmallPtrSetImpl<BasicBlock*> &Reachable) {
SmallVector<BasicBlock*, 128> Worklist; SmallVector<BasicBlock*, 128> Worklist;
BasicBlock *BB = &F.front(); BasicBlock *BB = &F.front();
Worklist.push_back(BB); Worklist.push_back(BB);
@ -1594,13 +1621,16 @@ static bool markAliveBlocks(Function &F,
static CatchPadInst *getEmptyKey() { static CatchPadInst *getEmptyKey() {
return DenseMapInfo<CatchPadInst *>::getEmptyKey(); return DenseMapInfo<CatchPadInst *>::getEmptyKey();
} }
static CatchPadInst *getTombstoneKey() { static CatchPadInst *getTombstoneKey() {
return DenseMapInfo<CatchPadInst *>::getTombstoneKey(); return DenseMapInfo<CatchPadInst *>::getTombstoneKey();
} }
static unsigned getHashValue(CatchPadInst *CatchPad) { static unsigned getHashValue(CatchPadInst *CatchPad) {
return static_cast<unsigned>(hash_combine_range( return static_cast<unsigned>(hash_combine_range(
CatchPad->value_op_begin(), CatchPad->value_op_end())); CatchPad->value_op_begin(), CatchPad->value_op_end()));
} }
static bool isEqual(CatchPadInst *LHS, CatchPadInst *RHS) { static bool isEqual(CatchPadInst *LHS, CatchPadInst *RHS) {
if (LHS == getEmptyKey() || LHS == getTombstoneKey() || if (LHS == getEmptyKey() || LHS == getTombstoneKey() ||
RHS == getEmptyKey() || RHS == getTombstoneKey()) RHS == getEmptyKey() || RHS == getTombstoneKey())
@ -1910,6 +1940,7 @@ void llvm::copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
} }
namespace { namespace {
/// A potential constituent of a bitreverse or bswap expression. See /// A potential constituent of a bitreverse or bswap expression. See
/// collectBitParts for a fuller explanation. /// collectBitParts for a fuller explanation.
struct BitPart { struct BitPart {
@ -1919,12 +1950,14 @@ struct BitPart {
/// The Value that this is a bitreverse/bswap of. /// The Value that this is a bitreverse/bswap of.
Value *Provider; Value *Provider;
/// The "provenance" of each bit. Provenance[A] = B means that bit A /// The "provenance" of each bit. Provenance[A] = B means that bit A
/// in Provider becomes bit B in the result of this expression. /// in Provider becomes bit B in the result of this expression.
SmallVector<int8_t, 32> Provenance; // int8_t means max size is i128. SmallVector<int8_t, 32> Provenance; // int8_t means max size is i128.
enum { Unset = -1 }; enum { Unset = -1 };
}; };
} // end anonymous namespace } // end anonymous namespace
/// Analyze the specified subexpression and see if it is capable of providing /// Analyze the specified subexpression and see if it is capable of providing
@ -1950,7 +1983,6 @@ struct BitPart {
/// ///
/// Because we pass around references into \c BPS, we must use a container that /// Because we pass around references into \c BPS, we must use a container that
/// does not invalidate internal references (std::map instead of DenseMap). /// does not invalidate internal references (std::map instead of DenseMap).
///
static const Optional<BitPart> & static const Optional<BitPart> &
collectBitParts(Value *V, bool MatchBSwaps, bool MatchBitReversals, collectBitParts(Value *V, bool MatchBSwaps, bool MatchBitReversals,
std::map<Value *, Optional<BitPart>> &BPS) { std::map<Value *, Optional<BitPart>> &BPS) {

View File

@ -1,4 +1,4 @@
//===----- LoadStoreVectorizer.cpp - GPU Load & Store Vectorizer ----------===// //===- LoadStoreVectorizer.cpp - GPU Load & Store Vectorizer --------------===//
// //
// The LLVM Compiler Infrastructure // The LLVM Compiler Infrastructure
// //
@ -6,47 +6,66 @@
// License. See LICENSE.TXT for details. // License. See LICENSE.TXT for details.
// //
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
//
//===----------------------------------------------------------------------===//
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/MapVector.h" #include "llvm/ADT/MapVector.h"
#include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SetVector.h" #include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h" #include "llvm/ADT/Statistic.h"
#include "llvm/ADT/Triple.h" #include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/OrderedBasicBlock.h" #include "llvm/Analysis/OrderedBasicBlock.h"
#include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h" #include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h" #include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h" #include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h" #include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h" #include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h" #include "llvm/IR/Instructions.h"
#include "llvm/IR/Module.h" #include "llvm/IR/Module.h"
#include "llvm/IR/Type.h" #include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h" #include "llvm/IR/Value.h"
#include "llvm/Support/CommandLine.h" #include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h" #include "llvm/Support/Debug.h"
#include "llvm/Support/KnownBits.h" #include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h" #include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Vectorize.h" #include "llvm/Transforms/Vectorize.h"
#include <algorithm>
#include <cassert>
#include <cstdlib>
#include <tuple>
#include <utility>
using namespace llvm; using namespace llvm;
#define DEBUG_TYPE "load-store-vectorizer" #define DEBUG_TYPE "load-store-vectorizer"
STATISTIC(NumVectorInstructions, "Number of vector accesses generated"); STATISTIC(NumVectorInstructions, "Number of vector accesses generated");
STATISTIC(NumScalarsVectorized, "Number of scalar accesses vectorized"); STATISTIC(NumScalarsVectorized, "Number of scalar accesses vectorized");
namespace {
// FIXME: Assuming stack alignment of 4 is always good enough // FIXME: Assuming stack alignment of 4 is always good enough
static const unsigned StackAdjustedAlignment = 4; static const unsigned StackAdjustedAlignment = 4;
typedef SmallVector<Instruction *, 8> InstrList;
typedef MapVector<Value *, InstrList> InstrListMap; namespace {
using InstrList = SmallVector<Instruction *, 8>;
using InstrListMap = MapVector<Value *, InstrList>;
class Vectorizer { class Vectorizer {
Function &F; Function &F;
@ -163,7 +182,10 @@ public:
AU.setPreservesCFG(); AU.setPreservesCFG();
} }
}; };
}
} // end anonymous namespace
char LoadStoreVectorizer::ID = 0;
INITIALIZE_PASS_BEGIN(LoadStoreVectorizer, DEBUG_TYPE, INITIALIZE_PASS_BEGIN(LoadStoreVectorizer, DEBUG_TYPE,
"Vectorize load and Store instructions", false, false) "Vectorize load and Store instructions", false, false)
@ -175,8 +197,6 @@ INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
INITIALIZE_PASS_END(LoadStoreVectorizer, DEBUG_TYPE, INITIALIZE_PASS_END(LoadStoreVectorizer, DEBUG_TYPE,
"Vectorize load and store instructions", false, false) "Vectorize load and store instructions", false, false)
char LoadStoreVectorizer::ID = 0;
Pass *llvm::createLoadStoreVectorizerPass() { Pass *llvm::createLoadStoreVectorizerPass() {
return new LoadStoreVectorizer(); return new LoadStoreVectorizer();
} }
@ -605,7 +625,7 @@ Vectorizer::collectInstructions(BasicBlock *BB) {
continue; continue;
// Make sure all the users of a vector are constant-index extracts. // Make sure all the users of a vector are constant-index extracts.
if (isa<VectorType>(Ty) && !all_of(LI->users(), [](const User *U) { if (isa<VectorType>(Ty) && !llvm::all_of(LI->users(), [](const User *U) {
const ExtractElementInst *EEI = dyn_cast<ExtractElementInst>(U); const ExtractElementInst *EEI = dyn_cast<ExtractElementInst>(U);
return EEI && isa<ConstantInt>(EEI->getOperand(1)); return EEI && isa<ConstantInt>(EEI->getOperand(1));
})) }))
@ -614,7 +634,6 @@ Vectorizer::collectInstructions(BasicBlock *BB) {
// Save the load locations. // Save the load locations.
Value *ObjPtr = GetUnderlyingObject(Ptr, DL); Value *ObjPtr = GetUnderlyingObject(Ptr, DL);
LoadRefs[ObjPtr].push_back(LI); LoadRefs[ObjPtr].push_back(LI);
} else if (StoreInst *SI = dyn_cast<StoreInst>(&I)) { } else if (StoreInst *SI = dyn_cast<StoreInst>(&I)) {
if (!SI->isSimple()) if (!SI->isSimple())
continue; continue;
@ -639,7 +658,7 @@ Vectorizer::collectInstructions(BasicBlock *BB) {
if (TySize > VecRegSize / 2) if (TySize > VecRegSize / 2)
continue; continue;
if (isa<VectorType>(Ty) && !all_of(SI->users(), [](const User *U) { if (isa<VectorType>(Ty) && !llvm::all_of(SI->users(), [](const User *U) {
const ExtractElementInst *EEI = dyn_cast<ExtractElementInst>(U); const ExtractElementInst *EEI = dyn_cast<ExtractElementInst>(U);
return EEI && isa<ConstantInt>(EEI->getOperand(1)); return EEI && isa<ConstantInt>(EEI->getOperand(1));
})) }))

View File

@ -6,6 +6,7 @@
// License. See LICENSE.TXT for details. // License. See LICENSE.TXT for details.
// //
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
//
// This pass implements the Bottom Up SLP vectorizer. It detects consecutive // This pass implements the Bottom Up SLP vectorizer. It detects consecutive
// stores that can be put together into vector-stores. Next, it attempts to // stores that can be put together into vector-stores. Next, it attempts to
// construct vectorizable tree using the use-def chains. If a profitable tree // construct vectorizable tree using the use-def chains. If a profitable tree
@ -361,14 +362,17 @@ static Value *isOneOf(Value *OpValue, Value *Op) {
} }
namespace { namespace {
/// Contains data for the instructions going to be vectorized. /// Contains data for the instructions going to be vectorized.
struct RawInstructionsData { struct RawInstructionsData {
/// Main Opcode of the instructions going to be vectorized. /// Main Opcode of the instructions going to be vectorized.
unsigned Opcode = 0; unsigned Opcode = 0;
/// The list of instructions have some instructions with alternate opcodes. /// The list of instructions have some instructions with alternate opcodes.
bool HasAltOpcodes = false; bool HasAltOpcodes = false;
}; };
} // namespace
} // end anonymous namespace
/// Checks the list of the vectorized instructions \p VL and returns info about /// Checks the list of the vectorized instructions \p VL and returns info about
/// this list. /// this list.
@ -392,19 +396,24 @@ static RawInstructionsData getMainOpcode(ArrayRef<Value *> VL) {
} }
namespace { namespace {
/// Main data required for vectorization of instructions. /// Main data required for vectorization of instructions.
struct InstructionsState { struct InstructionsState {
/// The very first instruction in the list with the main opcode. /// The very first instruction in the list with the main opcode.
Value *OpValue = nullptr; Value *OpValue = nullptr;
/// The main opcode for the list of instructions. /// The main opcode for the list of instructions.
unsigned Opcode = 0; unsigned Opcode = 0;
/// Some of the instructions in the list have alternate opcodes. /// Some of the instructions in the list have alternate opcodes.
bool IsAltShuffle = false; bool IsAltShuffle = false;
InstructionsState() = default; InstructionsState() = default;
InstructionsState(Value *OpValue, unsigned Opcode, bool IsAltShuffle) InstructionsState(Value *OpValue, unsigned Opcode, bool IsAltShuffle)
: OpValue(OpValue), Opcode(Opcode), IsAltShuffle(IsAltShuffle) {} : OpValue(OpValue), Opcode(Opcode), IsAltShuffle(IsAltShuffle) {}
}; };
} // namespace
} // end anonymous namespace
/// \returns analysis of the Instructions in \p VL described in /// \returns analysis of the Instructions in \p VL described in
/// InstructionsState, the Opcode that we suppose the whole list /// InstructionsState, the Opcode that we suppose the whole list
@ -973,6 +982,7 @@ private:
return os; return os;
} }
#endif #endif
friend struct GraphTraits<BoUpSLP *>; friend struct GraphTraits<BoUpSLP *>;
friend struct DOTGraphTraits<BoUpSLP *>; friend struct DOTGraphTraits<BoUpSLP *>;
@ -1176,9 +1186,9 @@ private:
/// The ID of the scheduling region. For a new vectorization iteration this /// The ID of the scheduling region. For a new vectorization iteration this
/// is incremented which "removes" all ScheduleData from the region. /// is incremented which "removes" all ScheduleData from the region.
int SchedulingRegionID = 1;
// Make sure that the initial SchedulingRegionID is greater than the // Make sure that the initial SchedulingRegionID is greater than the
// initial SchedulingRegionID in ScheduleData (which is 0). // initial SchedulingRegionID in ScheduleData (which is 0).
int SchedulingRegionID = 1;
}; };
/// Attaches the BlockScheduling structures to basic blocks. /// Attaches the BlockScheduling structures to basic blocks.
@ -1212,6 +1222,7 @@ private:
unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt. unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
unsigned MinVecRegSize; // Set by cl::opt (default: 128). unsigned MinVecRegSize; // Set by cl::opt (default: 128).
/// Instruction builder to construct the vectorized tree. /// Instruction builder to construct the vectorized tree.
IRBuilder<> Builder; IRBuilder<> Builder;
@ -4662,6 +4673,7 @@ class HorizontalReduction {
RK_Max, /// Maximum reduction data. RK_Max, /// Maximum reduction data.
RK_UMax, /// Unsigned maximum reduction data. RK_UMax, /// Unsigned maximum reduction data.
}; };
/// Contains info about operation, like its opcode, left and right operands. /// Contains info about operation, like its opcode, left and right operands.
class OperationData { class OperationData {
/// Opcode of the instruction. /// Opcode of the instruction.
@ -4672,8 +4684,10 @@ class HorizontalReduction {
/// Right operand of the reduction operation. /// Right operand of the reduction operation.
Value *RHS = nullptr; Value *RHS = nullptr;
/// Kind of the reduction operation. /// Kind of the reduction operation.
ReductionKind Kind = RK_None; ReductionKind Kind = RK_None;
/// True if float point min/max reduction has no NaNs. /// True if float point min/max reduction has no NaNs.
bool NoNaN = false; bool NoNaN = false;
@ -4725,7 +4739,7 @@ class HorizontalReduction {
/// Construction for reduced values. They are identified by opcode only and /// Construction for reduced values. They are identified by opcode only and
/// don't have associated LHS/RHS values. /// don't have associated LHS/RHS values.
explicit OperationData(Value *V) : Kind(RK_None) { explicit OperationData(Value *V) {
if (auto *I = dyn_cast<Instruction>(V)) if (auto *I = dyn_cast<Instruction>(V))
Opcode = I->getOpcode(); Opcode = I->getOpcode();
} }
@ -4737,6 +4751,7 @@ class HorizontalReduction {
: Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind), NoNaN(NoNaN) { : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind), NoNaN(NoNaN) {
assert(Kind != RK_None && "One of the reduction operations is expected."); assert(Kind != RK_None && "One of the reduction operations is expected.");
} }
explicit operator bool() const { return Opcode; } explicit operator bool() const { return Opcode; }
/// Get the index of the first operand. /// Get the index of the first operand.
@ -5421,7 +5436,6 @@ private:
/// starting from the last insertelement instruction. /// starting from the last insertelement instruction.
/// ///
/// Returns true if it matches /// Returns true if it matches
///
static bool findBuildVector(InsertElementInst *LastInsertElem, static bool findBuildVector(InsertElementInst *LastInsertElem,
SmallVectorImpl<Value *> &BuildVector, SmallVectorImpl<Value *> &BuildVector,
SmallVectorImpl<Value *> &BuildVectorOpds) { SmallVectorImpl<Value *> &BuildVectorOpds) {

View File

@ -18,12 +18,29 @@
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
#include "VPlan.h" #include "VPlan.h"
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/BasicBlock.h" #include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Dominators.h" #include "llvm/IR/Dominators.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/GraphWriter.h" #include "llvm/Support/GraphWriter.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include <cassert>
#include <iterator>
#include <string>
#include <vector>
using namespace llvm; using namespace llvm;
@ -138,7 +155,6 @@ void VPBasicBlock::execute(VPTransformState *State) {
SingleHPred->getExitBasicBlock() == PrevVPBB && SingleHPred->getExitBasicBlock() == PrevVPBB &&
PrevVPBB->getSingleHierarchicalSuccessor()) && /* B */ PrevVPBB->getSingleHierarchicalSuccessor()) && /* B */
!(Replica && getPredecessors().empty())) { /* C */ !(Replica && getPredecessors().empty())) { /* C */
NewBB = createEmptyBasicBlock(State->CFG); NewBB = createEmptyBasicBlock(State->CFG);
State->Builder.SetInsertPoint(NewBB); State->Builder.SetInsertPoint(NewBB);
// Temporarily terminate with unreachable until CFG is rewired. // Temporarily terminate with unreachable until CFG is rewired.

View File

@ -1,4 +1,4 @@
//===- VPlan.h - Represent A Vectorizer Plan ------------------------------===// //===- VPlan.h - Represent A Vectorizer Plan --------------------*- C++ -*-===//
// //
// The LLVM Compiler Infrastructure // The LLVM Compiler Infrastructure
// //
@ -6,7 +6,7 @@
// License. See LICENSE.TXT for details. // License. See LICENSE.TXT for details.
// //
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
/// //
/// \file /// \file
/// This file contains the declarations of the Vectorization Plan base classes: /// This file contains the declarations of the Vectorization Plan base classes:
/// 1. VPBasicBlock and VPRegionBlock that inherit from a common pure virtual /// 1. VPBasicBlock and VPRegionBlock that inherit from a common pure virtual
@ -18,34 +18,37 @@
/// 4. The VPlan class holding a candidate for vectorization; /// 4. The VPlan class holding a candidate for vectorization;
/// 5. The VPlanPrinter class providing a way to print a plan in dot format. /// 5. The VPlanPrinter class providing a way to print a plan in dot format.
/// These are documented in docs/VectorizationPlan.rst. /// These are documented in docs/VectorizationPlan.rst.
/// //
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H #ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
#define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H #define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/GraphTraits.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist.h"
#include "llvm/ADT/ilist_node.h" #include "llvm/ADT/ilist_node.h"
#include "llvm/IR/IRBuilder.h" #include "llvm/IR/IRBuilder.h"
#include "llvm/Support/raw_ostream.h" #include <algorithm>
#include <cassert>
// The (re)use of existing LoopVectorize classes is subject to future VPlan #include <cstddef>
// refactoring. #include <map>
namespace { #include <string>
// Forward declarations.
//class InnerLoopVectorizer;
class LoopVectorizationLegality;
class LoopVectorizationCostModel;
} // namespace
namespace llvm { namespace llvm {
// Forward declarations.
class BasicBlock; class BasicBlock;
class DominatorTree;
class InnerLoopVectorizer; class InnerLoopVectorizer;
class LoopInfo;
class raw_ostream;
class Value;
class VPBasicBlock; class VPBasicBlock;
class VPRegionBlock;
/// In what follows, the term "input IR" refers to code that is fed into the /// In what follows, the term "input IR" refers to code that is fed into the
/// vectorizer whereas the term "output IR" refers to code that is generated by /// vectorizer whereas the term "output IR" refers to code that is generated by
@ -54,8 +57,11 @@ class VPBasicBlock;
/// VPIteration represents a single point in the iteration space of the output /// VPIteration represents a single point in the iteration space of the output
/// (vectorized and/or unrolled) IR loop. /// (vectorized and/or unrolled) IR loop.
struct VPIteration { struct VPIteration {
unsigned Part; ///< in [0..UF) /// in [0..UF)
unsigned Lane; ///< in [0..VF) unsigned Part;
/// in [0..VF)
unsigned Lane;
}; };
/// This is a helper struct for maintaining vectorization state. It's used for /// This is a helper struct for maintaining vectorization state. It's used for
@ -75,7 +81,6 @@ struct VPIteration {
/// ///
/// Entries from either map can be retrieved using the getVectorValue and /// Entries from either map can be retrieved using the getVectorValue and
/// getScalarValue functions, which assert that the desired value exists. /// getScalarValue functions, which assert that the desired value exists.
struct VectorizerValueMap { struct VectorizerValueMap {
private: private:
/// The unroll factor. Each entry in the vector map contains UF vector values. /// The unroll factor. Each entry in the vector map contains UF vector values.
@ -87,8 +92,8 @@ private:
/// The vector and scalar map storage. We use std::map and not DenseMap /// The vector and scalar map storage. We use std::map and not DenseMap
/// because insertions to DenseMap invalidate its iterators. /// because insertions to DenseMap invalidate its iterators.
typedef SmallVector<Value *, 2> VectorParts; using VectorParts = SmallVector<Value *, 2>;
typedef SmallVector<SmallVector<Value *, 4>, 2> ScalarParts; using ScalarParts = SmallVector<SmallVector<Value *, 4>, 2>;
std::map<Value *, VectorParts> VectorMapStorage; std::map<Value *, VectorParts> VectorMapStorage;
std::map<Value *, ScalarParts> ScalarMapStorage; std::map<Value *, ScalarParts> ScalarMapStorage;
@ -193,12 +198,11 @@ public:
/// VPTransformState holds information passed down when "executing" a VPlan, /// VPTransformState holds information passed down when "executing" a VPlan,
/// needed for generating the output IR. /// needed for generating the output IR.
struct VPTransformState { struct VPTransformState {
VPTransformState(unsigned VF, unsigned UF, LoopInfo *LI, DominatorTree *DT,
VPTransformState(unsigned VF, unsigned UF, class LoopInfo *LI, IRBuilder<> &Builder, VectorizerValueMap &ValueMap,
class DominatorTree *DT, IRBuilder<> &Builder, InnerLoopVectorizer *ILV)
VectorizerValueMap &ValueMap, InnerLoopVectorizer *ILV) : VF(VF), UF(UF), LI(LI), DT(DT), Builder(Builder), ValueMap(ValueMap),
: VF(VF), UF(UF), Instance(), LI(LI), DT(DT), Builder(Builder), ILV(ILV) {}
ValueMap(ValueMap), ILV(ILV) {}
/// The chosen Vectorization and Unroll Factors of the loop being vectorized. /// The chosen Vectorization and Unroll Factors of the loop being vectorized.
unsigned VF; unsigned VF;
@ -213,25 +217,28 @@ struct VPTransformState {
/// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks. /// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks.
struct CFGState { struct CFGState {
/// The previous VPBasicBlock visited. Initially set to null. /// The previous VPBasicBlock visited. Initially set to null.
VPBasicBlock *PrevVPBB; VPBasicBlock *PrevVPBB = nullptr;
/// The previous IR BasicBlock created or used. Initially set to the new /// The previous IR BasicBlock created or used. Initially set to the new
/// header BasicBlock. /// header BasicBlock.
BasicBlock *PrevBB; BasicBlock *PrevBB = nullptr;
/// The last IR BasicBlock in the output IR. Set to the new latch /// The last IR BasicBlock in the output IR. Set to the new latch
/// BasicBlock, used for placing the newly created BasicBlocks. /// BasicBlock, used for placing the newly created BasicBlocks.
BasicBlock *LastBB; BasicBlock *LastBB = nullptr;
/// A mapping of each VPBasicBlock to the corresponding BasicBlock. In case /// A mapping of each VPBasicBlock to the corresponding BasicBlock. In case
/// of replication, maps the BasicBlock of the last replica created. /// of replication, maps the BasicBlock of the last replica created.
SmallDenseMap<VPBasicBlock *, BasicBlock *> VPBB2IRBB; SmallDenseMap<VPBasicBlock *, BasicBlock *> VPBB2IRBB;
CFGState() : PrevVPBB(nullptr), PrevBB(nullptr), LastBB(nullptr) {} CFGState() = default;
} CFG; } CFG;
/// Hold a pointer to LoopInfo to register new basic blocks in the loop. /// Hold a pointer to LoopInfo to register new basic blocks in the loop.
class LoopInfo *LI; LoopInfo *LI;
/// Hold a pointer to Dominator Tree to register new basic blocks in the loop. /// Hold a pointer to Dominator Tree to register new basic blocks in the loop.
class DominatorTree *DT; DominatorTree *DT;
/// Hold a reference to the IRBuilder used to generate output IR code. /// Hold a reference to the IRBuilder used to generate output IR code.
IRBuilder<> &Builder; IRBuilder<> &Builder;
@ -241,7 +248,7 @@ struct VPTransformState {
VectorizerValueMap &ValueMap; VectorizerValueMap &ValueMap;
/// Hold a pointer to InnerLoopVectorizer to reuse its IR generation methods. /// Hold a pointer to InnerLoopVectorizer to reuse its IR generation methods.
class InnerLoopVectorizer *ILV; InnerLoopVectorizer *ILV;
}; };
/// VPBlockBase is the building block of the Hierarchical Control-Flow Graph. /// VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
@ -255,7 +262,7 @@ private:
/// The immediate VPRegionBlock which this VPBlockBase belongs to, or null if /// The immediate VPRegionBlock which this VPBlockBase belongs to, or null if
/// it is a topmost VPBlockBase. /// it is a topmost VPBlockBase.
class VPRegionBlock *Parent; VPRegionBlock *Parent = nullptr;
/// List of predecessor blocks. /// List of predecessor blocks.
SmallVector<VPBlockBase *, 1> Predecessors; SmallVector<VPBlockBase *, 1> Predecessors;
@ -291,18 +298,18 @@ private:
protected: protected:
VPBlockBase(const unsigned char SC, const std::string &N) VPBlockBase(const unsigned char SC, const std::string &N)
: SubclassID(SC), Name(N), Parent(nullptr) {} : SubclassID(SC), Name(N) {}
public: public:
/// An enumeration for keeping track of the concrete subclass of VPBlockBase /// An enumeration for keeping track of the concrete subclass of VPBlockBase
/// that are actually instantiated. Values of this enumeration are kept in the /// that are actually instantiated. Values of this enumeration are kept in the
/// SubclassID field of the VPBlockBase objects. They are used for concrete /// SubclassID field of the VPBlockBase objects. They are used for concrete
/// type identification. /// type identification.
typedef enum { VPBasicBlockSC, VPRegionBlockSC } VPBlockTy; using VPBlockTy = enum { VPBasicBlockSC, VPRegionBlockSC };
typedef SmallVectorImpl<VPBlockBase *> VPBlocksTy; using VPBlocksTy = SmallVectorImpl<VPBlockBase *>;
virtual ~VPBlockBase() {} virtual ~VPBlockBase() = default;
const std::string &getName() const { return Name; } const std::string &getName() const { return Name; }
@ -437,14 +444,14 @@ private:
const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast). const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
/// Each VPRecipe belongs to a single VPBasicBlock. /// Each VPRecipe belongs to a single VPBasicBlock.
VPBasicBlock *Parent; VPBasicBlock *Parent = nullptr;
public: public:
/// An enumeration for keeping track of the concrete subclass of VPRecipeBase /// An enumeration for keeping track of the concrete subclass of VPRecipeBase
/// that is actually instantiated. Values of this enumeration are kept in the /// that is actually instantiated. Values of this enumeration are kept in the
/// SubclassID field of the VPRecipeBase objects. They are used for concrete /// SubclassID field of the VPRecipeBase objects. They are used for concrete
/// type identification. /// type identification.
typedef enum { using VPRecipeTy = enum {
VPBranchOnMaskSC, VPBranchOnMaskSC,
VPInterleaveSC, VPInterleaveSC,
VPPredInstPHISC, VPPredInstPHISC,
@ -452,11 +459,10 @@ public:
VPWidenIntOrFpInductionSC, VPWidenIntOrFpInductionSC,
VPWidenPHISC, VPWidenPHISC,
VPWidenSC, VPWidenSC,
} VPRecipeTy; };
VPRecipeBase(const unsigned char SC) : SubclassID(SC), Parent(nullptr) {} VPRecipeBase(const unsigned char SC) : SubclassID(SC) {}
virtual ~VPRecipeBase() = default;
virtual ~VPRecipeBase() {}
/// \return an ID for the concrete type of this object. /// \return an ID for the concrete type of this object.
/// This is used to implement the classof checks. This should not be used /// This is used to implement the classof checks. This should not be used
@ -480,18 +486,26 @@ public:
/// output IR instructions. /// output IR instructions.
class VPBasicBlock : public VPBlockBase { class VPBasicBlock : public VPBlockBase {
public: public:
typedef iplist<VPRecipeBase> RecipeListTy; using RecipeListTy = iplist<VPRecipeBase>;
private: private:
/// The VPRecipes held in the order of output instructions to generate. /// The VPRecipes held in the order of output instructions to generate.
RecipeListTy Recipes; RecipeListTy Recipes;
public: public:
VPBasicBlock(const Twine &Name = "", VPRecipeBase *Recipe = nullptr)
: VPBlockBase(VPBasicBlockSC, Name.str()) {
if (Recipe)
appendRecipe(Recipe);
}
~VPBasicBlock() override { Recipes.clear(); }
/// Instruction iterators... /// Instruction iterators...
typedef RecipeListTy::iterator iterator; using iterator = RecipeListTy::iterator;
typedef RecipeListTy::const_iterator const_iterator; using const_iterator = RecipeListTy::const_iterator;
typedef RecipeListTy::reverse_iterator reverse_iterator; using reverse_iterator = RecipeListTy::reverse_iterator;
typedef RecipeListTy::const_reverse_iterator const_reverse_iterator; using const_reverse_iterator = RecipeListTy::const_reverse_iterator;
//===--------------------------------------------------------------------===// //===--------------------------------------------------------------------===//
/// Recipe iterator methods /// Recipe iterator methods
@ -518,14 +532,6 @@ public:
return &VPBasicBlock::Recipes; return &VPBasicBlock::Recipes;
} }
VPBasicBlock(const Twine &Name = "", VPRecipeBase *Recipe = nullptr)
: VPBlockBase(VPBasicBlockSC, Name.str()) {
if (Recipe)
appendRecipe(Recipe);
}
~VPBasicBlock() { Recipes.clear(); }
/// Method to support type inquiry through isa, cast, and dyn_cast. /// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPBlockBase *V) { static inline bool classof(const VPBlockBase *V) {
return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC; return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC;
@ -581,7 +587,7 @@ public:
Exit->setParent(this); Exit->setParent(this);
} }
~VPRegionBlock() { ~VPRegionBlock() override {
if (Entry) if (Entry)
deleteCFG(Entry); deleteCFG(Entry);
} }
@ -649,7 +655,7 @@ public:
private: private:
/// Add to the given dominator tree the header block and every new basic block /// Add to the given dominator tree the header block and every new basic block
/// that was created between it and the latch block, inclusive. /// that was created between it and the latch block, inclusive.
static void updateDominatorTree(class DominatorTree *DT, static void updateDominatorTree(DominatorTree *DT,
BasicBlock *LoopPreHeaderBB, BasicBlock *LoopPreHeaderBB,
BasicBlock *LoopLatchBB); BasicBlock *LoopLatchBB);
}; };
@ -667,11 +673,11 @@ private:
unsigned Depth; unsigned Depth;
unsigned TabWidth = 2; unsigned TabWidth = 2;
std::string Indent; std::string Indent;
unsigned BID = 0; unsigned BID = 0;
SmallDenseMap<const VPBlockBase *, unsigned> BlockID; SmallDenseMap<const VPBlockBase *, unsigned> BlockID;
VPlanPrinter(raw_ostream &O, VPlan &P) : OS(O), Plan(P) {}
/// Handle indentation. /// Handle indentation.
void bumpIndent(int b) { Indent = std::string((Depth += b) * TabWidth, ' '); } void bumpIndent(int b) { Indent = std::string((Depth += b) * TabWidth, ' '); }
@ -701,8 +707,6 @@ private:
void drawEdge(const VPBlockBase *From, const VPBlockBase *To, bool Hidden, void drawEdge(const VPBlockBase *From, const VPBlockBase *To, bool Hidden,
const Twine &Label); const Twine &Label);
VPlanPrinter(raw_ostream &O, VPlan &P) : OS(O), Plan(P) {}
void dump(); void dump();
static void printAsIngredient(raw_ostream &O, Value *V); static void printAsIngredient(raw_ostream &O, Value *V);
@ -710,6 +714,7 @@ private:
struct VPlanIngredient { struct VPlanIngredient {
Value *V; Value *V;
VPlanIngredient(Value *V) : V(V) {} VPlanIngredient(Value *V) : V(V) {}
}; };
@ -732,8 +737,8 @@ inline raw_ostream &operator<<(raw_ostream &OS, VPlan &Plan) {
// graph of VPBlockBase nodes... // graph of VPBlockBase nodes...
template <> struct GraphTraits<VPBlockBase *> { template <> struct GraphTraits<VPBlockBase *> {
typedef VPBlockBase *NodeRef; using NodeRef = VPBlockBase *;
typedef SmallVectorImpl<VPBlockBase *>::iterator ChildIteratorType; using ChildIteratorType = SmallVectorImpl<VPBlockBase *>::iterator;
static NodeRef getEntryNode(NodeRef N) { return N; } static NodeRef getEntryNode(NodeRef N) { return N; }
@ -747,8 +752,8 @@ template <> struct GraphTraits<VPBlockBase *> {
}; };
template <> struct GraphTraits<const VPBlockBase *> { template <> struct GraphTraits<const VPBlockBase *> {
typedef const VPBlockBase *NodeRef; using NodeRef = const VPBlockBase *;
typedef SmallVectorImpl<VPBlockBase *>::const_iterator ChildIteratorType; using ChildIteratorType = SmallVectorImpl<VPBlockBase *>::const_iterator;
static NodeRef getEntryNode(NodeRef N) { return N; } static NodeRef getEntryNode(NodeRef N) { return N; }
@ -765,11 +770,9 @@ template <> struct GraphTraits<const VPBlockBase *> {
// graph of VPBlockBase nodes... and to walk it in inverse order. Inverse order // graph of VPBlockBase nodes... and to walk it in inverse order. Inverse order
// for a VPBlockBase is considered to be when traversing the predecessors of a // for a VPBlockBase is considered to be when traversing the predecessors of a
// VPBlockBase instead of its successors. // VPBlockBase instead of its successors.
//
template <> struct GraphTraits<Inverse<VPBlockBase *>> { template <> struct GraphTraits<Inverse<VPBlockBase *>> {
typedef VPBlockBase *NodeRef; using NodeRef = VPBlockBase *;
typedef SmallVectorImpl<VPBlockBase *>::iterator ChildIteratorType; using ChildIteratorType = SmallVectorImpl<VPBlockBase *>::iterator;
static Inverse<VPBlockBase *> getEntryNode(Inverse<VPBlockBase *> B) { static Inverse<VPBlockBase *> getEntryNode(Inverse<VPBlockBase *> B) {
return B; return B;
@ -784,6 +787,6 @@ template <> struct GraphTraits<Inverse<VPBlockBase *>> {
} }
}; };
} // namespace llvm } // end namespace llvm
#endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H #endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H