diff --git a/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h b/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h index f35707eeb3f0..e835bd5f0761 100644 --- a/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h +++ b/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h @@ -1,4 +1,4 @@ -//===- NaryReassociate.h - Reassociate n-ary expressions ------------------===// +//===- NaryReassociate.h - Reassociate n-ary expressions --------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -81,15 +81,25 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/Analysis/AssumptionCache.h" -#include "llvm/Analysis/ScalarEvolution.h" -#include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/Analysis/TargetTransformInfo.h" -#include "llvm/IR/Dominators.h" -#include "llvm/IR/Function.h" #include "llvm/IR/PassManager.h" +#include "llvm/IR/ValueHandle.h" namespace llvm { + +class AssumptionCache; +class BinaryOperator; +class DataLayout; +class DominatorTree; +class Function; +class GetElementPtrInst; +class Instruction; +class ScalarEvolution; +class SCEV; +class TargetLibraryInfo; +class TargetTransformInfo; +class Type; +class Value; + class NaryReassociatePass : public PassInfoMixin { public: PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); @@ -109,6 +119,7 @@ private: // Reassociate GEP for better CSE. Instruction *tryReassociateGEP(GetElementPtrInst *GEP); + // Try splitting GEP at the I-th index and see whether either part can be // CSE'ed. This is a helper function for tryReassociateGEP. // @@ -118,6 +129,7 @@ private: // ..., i-th index). GetElementPtrInst *tryReassociateGEPAtIndex(GetElementPtrInst *GEP, unsigned I, Type *IndexedType); + // Given GEP's I-th index = LHS + RHS, see whether &Base[..][LHS][..] or // &Base[..][RHS][..] can be CSE'ed and rewrite GEP accordingly. GetElementPtrInst *tryReassociateGEPAtIndex(GetElementPtrInst *GEP, @@ -146,6 +158,7 @@ private: // \c CandidateExpr. Returns null if not found. Instruction *findClosestMatchingDominator(const SCEV *CandidateExpr, Instruction *Dominatee); + // GetElementPtrInst implicitly sign-extends an index if the index is shorter // than the pointer size. This function returns whether Index is shorter than // GEP's pointer size, i.e., whether Index needs to be sign-extended in order @@ -158,6 +171,7 @@ private: ScalarEvolution *SE; TargetLibraryInfo *TLI; TargetTransformInfo *TTI; + // A lookup table quickly telling which instructions compute the given SCEV. // Note that there can be multiple instructions at different locations // computing to the same SCEV, so we map a SCEV to an instruction list. For @@ -169,6 +183,7 @@ private: // bar(a + b); DenseMap> SeenExprs; }; -} // namespace llvm + +} // end namespace llvm #endif // LLVM_TRANSFORMS_SCALAR_NARYREASSOCIATE_H diff --git a/llvm/include/llvm/Transforms/Utils/AddDiscriminators.h b/llvm/include/llvm/Transforms/Utils/AddDiscriminators.h index a87758300992..4dad06e6c125 100644 --- a/llvm/include/llvm/Transforms/Utils/AddDiscriminators.h +++ b/llvm/include/llvm/Transforms/Utils/AddDiscriminators.h @@ -1,4 +1,4 @@ -//===- AddDiscriminators.h -------------------------------------*- C++ -*-===// +//===- AddDiscriminators.h --------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -20,10 +20,13 @@ namespace llvm { +class Function; + class AddDiscriminatorsPass : public PassInfoMixin { public: PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; + } // end namespace llvm #endif // LLVM_TRANSFORMS_UTILS_ADDDISCRIMINATORS_H diff --git a/llvm/include/llvm/Transforms/Utils/BypassSlowDivision.h b/llvm/include/llvm/Transforms/Utils/BypassSlowDivision.h index 663bef2594b0..6eca5ed2154e 100644 --- a/llvm/include/llvm/Transforms/Utils/BypassSlowDivision.h +++ b/llvm/include/llvm/Transforms/Utils/BypassSlowDivision.h @@ -1,4 +1,4 @@ -//===- llvm/Transforms/Utils/BypassSlowDivision.h --------------*- C++ -*-===// +//===- llvm/Transforms/Utils/BypassSlowDivision.h ---------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -19,10 +19,14 @@ #define LLVM_TRANSFORMS_UTILS_BYPASSSLOWDIVISION_H #include "llvm/ADT/DenseMap.h" -#include "llvm/IR/Function.h" +#include "llvm/ADT/DenseMapInfo.h" +#include namespace llvm { +class BasicBlock; +class Value; + struct DivRemMapKey { bool SignedOp; Value *Dividend; @@ -61,6 +65,6 @@ template <> struct DenseMapInfo { bool bypassSlowDivision( BasicBlock *BB, const DenseMap &BypassWidth); -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_TRANSFORMS_UTILS_BYPASSSLOWDIVISION_H diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h index f91ebe1e9f81..af00a6c2cce3 100644 --- a/llvm/include/llvm/Transforms/Utils/Local.h +++ b/llvm/include/llvm/Transforms/Utils/Local.h @@ -1,4 +1,4 @@ -//===-- Local.h - Functions to perform local transformations ----*- C++ -*-===// +//===- Local.h - Functions to perform local transformations -----*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -15,41 +15,44 @@ #ifndef LLVM_TRANSFORMS_UTILS_LOCAL_H #define LLVM_TRANSFORMS_UTILS_LOCAL_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/TinyPtrVector.h" #include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/IR/CallSite.h" +#include "llvm/IR/Constant.h" +#include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/GetElementPtrTypeIterator.h" -#include "llvm/IR/IRBuilder.h" #include "llvm/IR/Operator.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/User.h" +#include "llvm/IR/Value.h" +#include "llvm/Support/Casting.h" +#include +#include namespace llvm { -class User; -class BasicBlock; -class Function; -class BranchInst; -class Instruction; -class CallInst; -class DbgDeclareInst; -class DbgInfoIntrinsic; -class DbgValueInst; -class StoreInst; -class LoadInst; -class Value; -class PHINode; class AllocaInst; class AssumptionCache; -class ConstantExpr; -class DataLayout; +class BasicBlock; +class BranchInst; +class CallInst; +class DbgInfoIntrinsic; +class DbgValueInst; +class DIBuilder; +class Function; +class Instruction; +class LazyValueInfo; +class LoadInst; +class MDNode; +class PHINode; +class StoreInst; class TargetLibraryInfo; class TargetTransformInfo; -class DIBuilder; -class DominatorTree; -class LazyValueInfo; - -template class SmallVectorImpl; /// A set of parameters used to control the transforms in the SimplifyCFG pass. /// Options may change depending on the position in the optimization pipeline. @@ -66,8 +69,7 @@ struct SimplifyCFGOptions { AssumptionCache *AssumpCache = nullptr) : BonusInstThreshold(BonusThreshold), ConvertSwitchToLookupTable(SwitchToLookup), - NeedCanonicalLoop(CanonicalLoops), - AC(AssumpCache) {} + NeedCanonicalLoop(CanonicalLoops), AC(AssumpCache) {} }; //===----------------------------------------------------------------------===// @@ -229,7 +231,8 @@ Value *EmitGEPOffset(IRBuilderTy *Builder, const DataLayout &DL, User *GEP, // Build a mask for high order bits. unsigned IntPtrWidth = IntPtrTy->getScalarType()->getIntegerBitWidth(); - uint64_t PtrSizeMask = ~0ULL >> (64 - IntPtrWidth); + uint64_t PtrSizeMask = + std::numeric_limits::max() >> (64 - IntPtrWidth); gep_type_iterator GTI = gep_type_begin(GEP); for (User::op_iterator i = GEP->op_begin() + 1, e = GEP->op_end(); i != e; @@ -390,7 +393,6 @@ unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT, unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT, const BasicBlock *BB); - /// Return true if the CallSite CS calls a gc leaf function. /// /// A leaf function is a function that does not safepoint the thread during its @@ -452,6 +454,6 @@ void maybeMarkSanitizerLibraryCallNoBuiltin(CallInst *CI, /// value? bool canReplaceOperandWithVariable(const Instruction *I, unsigned OpIdx); -} // End llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_TRANSFORMS_UTILS_LOCAL_H diff --git a/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp b/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp index f999d6caf9c0..6107f3a7dd18 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp @@ -21,18 +21,26 @@ //===----------------------------------------------------------------------===// #include "AMDGPU.h" -#include "llvm/ADT/DepthFirstIterator.h" -#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Analysis/DivergenceAnalysis.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" +#include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" +#include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" #include "llvm/IR/Type.h" +#include "llvm/Pass.h" +#include "llvm/Support/Casting.h" #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Utils/Local.h" + using namespace llvm; #define DEBUG_TYPE "amdgpu-unify-divergent-exit-nodes" @@ -42,6 +50,7 @@ namespace { class AMDGPUUnifyDivergentExitNodes : public FunctionPass { public: static char ID; // Pass identification, replacement for typeid + AMDGPUUnifyDivergentExitNodes() : FunctionPass(ID) { initializeAMDGPUUnifyDivergentExitNodesPass(*PassRegistry::getPassRegistry()); } @@ -51,9 +60,12 @@ public: bool runOnFunction(Function &F) override; }; -} +} // end anonymous namespace char AMDGPUUnifyDivergentExitNodes::ID = 0; + +char &llvm::AMDGPUUnifyDivergentExitNodesID = AMDGPUUnifyDivergentExitNodes::ID; + INITIALIZE_PASS_BEGIN(AMDGPUUnifyDivergentExitNodes, DEBUG_TYPE, "Unify divergent function exit nodes", false, false) INITIALIZE_PASS_DEPENDENCY(PostDominatorTreeWrapperPass) @@ -61,8 +73,6 @@ INITIALIZE_PASS_DEPENDENCY(DivergenceAnalysis) INITIALIZE_PASS_END(AMDGPUUnifyDivergentExitNodes, DEBUG_TYPE, "Unify divergent function exit nodes", false, false) -char &llvm::AMDGPUUnifyDivergentExitNodesID = AMDGPUUnifyDivergentExitNodes::ID; - void AMDGPUUnifyDivergentExitNodes::getAnalysisUsage(AnalysisUsage &AU) const{ // TODO: Preserve dominator tree. AU.addRequired(); @@ -113,7 +123,6 @@ static BasicBlock *unifyReturnBlockSet(Function &F, // Otherwise, we need to insert a new basic block into the function, add a PHI // nodes (if the function returns values), and convert all of the return // instructions into unconditional branches. - // BasicBlock *NewRetBlock = BasicBlock::Create(F.getContext(), Name, &F); PHINode *PN = nullptr; @@ -129,7 +138,6 @@ static BasicBlock *unifyReturnBlockSet(Function &F, // Loop over all of the blocks, replacing the return instruction with an // unconditional branch. - // for (BasicBlock *BB : ReturningBlocks) { // Add an incoming element to the PHI node for every return instruction that // is merging into this new block... @@ -157,7 +165,6 @@ bool AMDGPUUnifyDivergentExitNodes::runOnFunction(Function &F) { // Loop over all of the blocks in a function, tracking all of the blocks that // return. - // SmallVector ReturningBlocks; SmallVector UnreachableBlocks; diff --git a/llvm/lib/Transforms/Scalar/NaryReassociate.cpp b/llvm/lib/Transforms/Scalar/NaryReassociate.cpp index d0bfe3603897..b026c8d692c3 100644 --- a/llvm/lib/Transforms/Scalar/NaryReassociate.cpp +++ b/llvm/lib/Transforms/Scalar/NaryReassociate.cpp @@ -77,19 +77,45 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/Scalar/NaryReassociate.h" +#include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/AssumptionCache.h" +#include "llvm/Analysis/ScalarEvolution.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GetElementPtrTypeIterator.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/Module.h" +#include "llvm/IR/Operator.h" #include "llvm/IR/PatternMatch.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Value.h" +#include "llvm/IR/ValueHandle.h" +#include "llvm/Pass.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Utils/Local.h" +#include +#include + using namespace llvm; using namespace PatternMatch; #define DEBUG_TYPE "nary-reassociate" namespace { + class NaryReassociateLegacyPass : public FunctionPass { public: static char ID; @@ -101,6 +127,7 @@ public: bool doInitialization(Module &M) override { return false; } + bool runOnFunction(Function &F) override; void getAnalysisUsage(AnalysisUsage &AU) const override { @@ -118,9 +145,11 @@ public: private: NaryReassociatePass Impl; }; -} // anonymous namespace + +} // end anonymous namespace char NaryReassociateLegacyPass::ID = 0; + INITIALIZE_PASS_BEGIN(NaryReassociateLegacyPass, "nary-reassociate", "Nary reassociation", false, false) INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) diff --git a/llvm/lib/Transforms/Utils/AddDiscriminators.cpp b/llvm/lib/Transforms/Utils/AddDiscriminators.cpp index 4c9746b8c691..0f0668f24db5 100644 --- a/llvm/lib/Transforms/Utils/AddDiscriminators.cpp +++ b/llvm/lib/Transforms/Utils/AddDiscriminators.cpp @@ -50,31 +50,45 @@ // // For more details about DWARF discriminators, please visit // http://wiki.dwarfstd.org/index.php?title=Path_Discriminators +// //===----------------------------------------------------------------------===// #include "llvm/Transforms/Utils/AddDiscriminators.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StringRef.h" #include "llvm/IR/BasicBlock.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/PassManager.h" #include "llvm/Pass.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Scalar.h" +#include using namespace llvm; #define DEBUG_TYPE "add-discriminators" +// Command line option to disable discriminator generation even in the +// presence of debug information. This is only needed when debugging +// debug info generation issues. +static cl::opt NoDiscriminators( + "no-discriminators", cl::init(false), + cl::desc("Disable generation of discriminator information.")); + namespace { + // The legacy pass of AddDiscriminators. struct AddDiscriminatorsLegacyPass : public FunctionPass { static char ID; // Pass identification, replacement for typeid + AddDiscriminatorsLegacyPass() : FunctionPass(ID) { initializeAddDiscriminatorsLegacyPassPass(*PassRegistry::getPassRegistry()); } @@ -85,18 +99,12 @@ struct AddDiscriminatorsLegacyPass : public FunctionPass { } // end anonymous namespace char AddDiscriminatorsLegacyPass::ID = 0; + INITIALIZE_PASS_BEGIN(AddDiscriminatorsLegacyPass, "add-discriminators", "Add DWARF path discriminators", false, false) INITIALIZE_PASS_END(AddDiscriminatorsLegacyPass, "add-discriminators", "Add DWARF path discriminators", false, false) -// Command line option to disable discriminator generation even in the -// presence of debug information. This is only needed when debugging -// debug info generation issues. -static cl::opt NoDiscriminators( - "no-discriminators", cl::init(false), - cl::desc("Disable generation of discriminator information.")); - // Create the legacy AddDiscriminatorsPass. FunctionPass *llvm::createAddDiscriminatorsPass() { return new AddDiscriminatorsLegacyPass(); @@ -166,11 +174,11 @@ static bool addDiscriminators(Function &F) { bool Changed = false; - typedef std::pair Location; - typedef DenseSet BBSet; - typedef DenseMap LocationBBMap; - typedef DenseMap LocationDiscriminatorMap; - typedef DenseSet LocationSet; + using Location = std::pair; + using BBSet = DenseSet; + using LocationBBMap = DenseMap; + using LocationDiscriminatorMap = DenseMap; + using LocationSet = DenseSet; LocationBBMap LBM; LocationDiscriminatorMap LDM; @@ -242,6 +250,7 @@ static bool addDiscriminators(Function &F) { bool AddDiscriminatorsLegacyPass::runOnFunction(Function &F) { return addDiscriminators(F); } + PreservedAnalyses AddDiscriminatorsPass::run(Function &F, FunctionAnalysisManager &AM) { if (!addDiscriminators(F)) diff --git a/llvm/lib/Transforms/Utils/BypassSlowDivision.cpp b/llvm/lib/Transforms/Utils/BypassSlowDivision.cpp index d6c31f282e87..e9c14c93a9ad 100644 --- a/llvm/lib/Transforms/Utils/BypassSlowDivision.cpp +++ b/llvm/lib/Transforms/Utils/BypassSlowDivision.cpp @@ -1,4 +1,4 @@ -//===-- BypassSlowDivision.cpp - Bypass slow division ---------------------===// +//===- BypassSlowDivision.cpp - Bypass slow division ----------------------===// // // The LLVM Compiler Infrastructure // @@ -17,19 +17,33 @@ #include "llvm/Transforms/Utils/BypassSlowDivision.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Value.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/KnownBits.h" #include "llvm/Transforms/Utils/Local.h" +#include +#include using namespace llvm; #define DEBUG_TYPE "bypass-slow-division" namespace { + struct QuotRemPair { Value *Quotient; Value *Remainder; @@ -46,15 +60,11 @@ namespace { Value *Quotient = nullptr; Value *Remainder = nullptr; }; -} -namespace llvm { - typedef DenseMap DivCacheTy; - typedef DenseMap BypassWidthsTy; - typedef SmallPtrSet VisitedSetTy; -} +using DivCacheTy = DenseMap; +using BypassWidthsTy = DenseMap; +using VisitedSetTy = SmallPtrSet; -namespace { enum ValueRange { /// Operand definitely fits into BypassType. No runtime checks are needed. VALRNG_KNOWN_SHORT, @@ -84,17 +94,21 @@ class FastDivInsertionTask { return SlowDivOrRem->getOpcode() == Instruction::SDiv || SlowDivOrRem->getOpcode() == Instruction::SRem; } + bool isDivisionOp() { return SlowDivOrRem->getOpcode() == Instruction::SDiv || SlowDivOrRem->getOpcode() == Instruction::UDiv; } + Type *getSlowType() { return SlowDivOrRem->getType(); } public: FastDivInsertionTask(Instruction *I, const BypassWidthsTy &BypassWidths); + Value *getReplacement(DivCacheTy &Cache); }; -} // anonymous namespace + +} // end anonymous namespace FastDivInsertionTask::FastDivInsertionTask(Instruction *I, const BypassWidthsTy &BypassWidths) { @@ -193,7 +207,7 @@ bool FastDivInsertionTask::isHashLikeValue(Value *V, VisitedSetTy &Visited) { C = dyn_cast(cast(Op1)->getOperand(0)); return C && C->getValue().getMinSignedBits() > BypassType->getBitWidth(); } - case Instruction::PHI: { + case Instruction::PHI: // Stop IR traversal in case of a crazy input code. This limits recursion // depth. if (Visited.size() >= 16) @@ -209,7 +223,6 @@ bool FastDivInsertionTask::isHashLikeValue(Value *V, VisitedSetTy &Visited) { return getValueRange(V, Visited) == VALRNG_LIKELY_LONG || isa(V); }); - } default: return false; } diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp index 2a18c140c788..6b1391e0c80e 100644 --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -12,11 +12,15 @@ // //===----------------------------------------------------------------------===// +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/BlockFrequencyInfo.h" @@ -26,25 +30,46 @@ #include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/Analysis/ValueTracking.h" -#include "llvm/IR/Attributes.h" +#include "llvm/IR/Argument.h" +#include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" #include "llvm/IR/CallSite.h" +#include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DebugLoc.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Dominators.h" +#include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/User.h" +#include "llvm/IR/Value.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Transforms/Utils/Cloning.h" #include "llvm/Transforms/Utils/Local.h" +#include "llvm/Transforms/Utils/ValueMapper.h" #include +#include +#include +#include +#include +#include +#include +#include using namespace llvm; @@ -62,28 +87,37 @@ bool llvm::InlineFunction(CallInst *CI, InlineFunctionInfo &IFI, AAResults *CalleeAAR, bool InsertLifetime) { return InlineFunction(CallSite(CI), IFI, CalleeAAR, InsertLifetime); } + bool llvm::InlineFunction(InvokeInst *II, InlineFunctionInfo &IFI, AAResults *CalleeAAR, bool InsertLifetime) { return InlineFunction(CallSite(II), IFI, CalleeAAR, InsertLifetime); } namespace { + /// A class for recording information about inlining a landing pad. class LandingPadInliningInfo { - BasicBlock *OuterResumeDest; ///< Destination of the invoke's unwind. - BasicBlock *InnerResumeDest; ///< Destination for the callee's resume. - LandingPadInst *CallerLPad; ///< LandingPadInst associated with the invoke. - PHINode *InnerEHValuesPHI; ///< PHI for EH values from landingpad insts. + /// Destination of the invoke's unwind. + BasicBlock *OuterResumeDest; + + /// Destination for the callee's resume. + BasicBlock *InnerResumeDest = nullptr; + + /// LandingPadInst associated with the invoke. + LandingPadInst *CallerLPad = nullptr; + + /// PHI for EH values from landingpad insts. + PHINode *InnerEHValuesPHI = nullptr; + SmallVector UnwindDestPHIValues; public: LandingPadInliningInfo(InvokeInst *II) - : OuterResumeDest(II->getUnwindDest()), InnerResumeDest(nullptr), - CallerLPad(nullptr), InnerEHValuesPHI(nullptr) { + : OuterResumeDest(II->getUnwindDest()) { // If there are PHI nodes in the unwind destination block, we need to keep // track of which values came into them from the invoke before removing // the edge from this block. - llvm::BasicBlock *InvokeBB = II->getParent(); + BasicBlock *InvokeBB = II->getParent(); BasicBlock::iterator I = OuterResumeDest->begin(); for (; isa(I); ++I) { // Save the value to use for this edge. @@ -126,7 +160,8 @@ namespace { } } }; -} // anonymous namespace + +} // end anonymous namespace /// Get or create a target for the branch from ResumeInsts. BasicBlock *LandingPadInliningInfo::getInnerResumeDest() { @@ -189,7 +224,7 @@ static Value *getParentPad(Value *EHPad) { return cast(EHPad)->getParentPad(); } -typedef DenseMap UnwindDestMemoTy; +using UnwindDestMemoTy = DenseMap; /// Helper for getUnwindDestToken that does the descendant-ward part of /// the search. @@ -617,7 +652,7 @@ static void HandleInlinedEHPad(InvokeInst *II, BasicBlock *FirstNewBlock, // track of which values came into them from the invoke before removing the // edge from this block. SmallVector UnwindDestPHIValues; - llvm::BasicBlock *InvokeBB = II->getParent(); + BasicBlock *InvokeBB = II->getParent(); for (Instruction &I : *UnwindDest) { // Save the value to use for this edge. PHINode *PHI = dyn_cast(&I); @@ -1359,6 +1394,7 @@ static void fixupLineNumbers(Function *Fn, Function::iterator FI, } } } + /// Update the block frequencies of the caller after a callee has been inlined. /// /// Each block cloned into the caller has its block frequency scaled by the @@ -1848,8 +1884,9 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI, // Check that array size doesn't saturate uint64_t and doesn't // overflow when it's multiplied by type size. - if (AllocaArraySize != ~0ULL && - UINT64_MAX / AllocaArraySize >= AllocaTypeSize) { + if (AllocaArraySize != std::numeric_limits::max() && + std::numeric_limits::max() / AllocaArraySize >= + AllocaTypeSize) { AllocaSize = ConstantInt::get(Type::getInt64Ty(AI->getContext()), AllocaArraySize * AllocaTypeSize); } @@ -1980,7 +2017,7 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI, // match the callee's return type, we also need to change the return type of // the intrinsic. if (Caller->getReturnType() == TheCall->getType()) { - auto NewEnd = remove_if(Returns, [](ReturnInst *RI) { + auto NewEnd = llvm::remove_if(Returns, [](ReturnInst *RI) { return RI->getParent()->getTerminatingDeoptimizeCall() != nullptr; }); Returns.erase(NewEnd, Returns.end()); diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp index 21412dcf68e4..bf382191c6c6 100644 --- a/llvm/lib/Transforms/Utils/Local.cpp +++ b/llvm/lib/Transforms/Utils/Local.cpp @@ -1,4 +1,4 @@ -//===-- Local.cpp - Functions to perform local transformations ------------===// +//===- Local.cpp - Functions to perform local transformations -------------===// // // The LLVM Compiler Infrastructure // @@ -13,42 +13,74 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/Utils/Local.h" +#include "llvm/ADT/APInt.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/Hashing.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" +#include "llvm/ADT/TinyPtrVector.h" +#include "llvm/Analysis/ConstantFolding.h" #include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/LazyValueInfo.h" #include "llvm/Analysis/MemoryBuiltins.h" +#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/IR/Argument.h" +#include "llvm/IR/Attributes.h" +#include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" +#include "llvm/IR/CallSite.h" +#include "llvm/IR/Constant.h" #include "llvm/IR/ConstantRange.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DebugLoc.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Dominators.h" +#include "llvm/IR/Function.h" #include "llvm/IR/GetElementPtrTypeIterator.h" -#include "llvm/IR/GlobalAlias.h" -#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/GlobalObject.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" #include "llvm/IR/Metadata.h" +#include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" #include "llvm/IR/PatternMatch.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Use.h" +#include "llvm/IR/User.h" +#include "llvm/IR/Value.h" #include "llvm/IR/ValueHandle.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/KnownBits.h" -#include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" +#include +#include +#include +#include +#include +#include +#include + using namespace llvm; using namespace llvm::PatternMatch; @@ -282,7 +314,6 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions, return false; } - //===----------------------------------------------------------------------===// // Local dead code elimination. // @@ -541,7 +572,6 @@ bool llvm::SimplifyInstructionsInBlock(BasicBlock *BB, // Control Flow Graph Restructuring. // - /// RemovePredecessorAndSimplify - Like BasicBlock::removePredecessor, this /// method is called when we're about to delete Pred as a predecessor of BB. If /// BB contains any PHI nodes, this drops the entries in the PHI nodes for Pred. @@ -578,12 +608,10 @@ void llvm::RemovePredecessorAndSimplify(BasicBlock *BB, BasicBlock *Pred) { } } - /// MergeBasicBlockIntoOnlyPred - DestBB is a block with one predecessor and its /// predecessor is known to have one successor (DestBB!). Eliminate the edge /// between them, moving the instructions in the predecessor into DestBB and /// deleting the predecessor block. -/// void llvm::MergeBasicBlockIntoOnlyPred(BasicBlock *DestBB, DominatorTree *DT) { // If BB has single-entry PHI nodes, fold them. while (PHINode *PN = dyn_cast(DestBB->begin())) { @@ -602,7 +630,7 @@ void llvm::MergeBasicBlockIntoOnlyPred(BasicBlock *DestBB, DominatorTree *DT) { if (DestBB->hasAddressTaken()) { BlockAddress *BA = BlockAddress::get(DestBB); Constant *Replacement = - ConstantInt::get(llvm::Type::getInt32Ty(BA->getContext()), 1); + ConstantInt::get(Type::getInt32Ty(BA->getContext()), 1); BA->replaceAllUsesWith(ConstantExpr::getIntToPtr(Replacement, BA->getType())); BA->destroyConstant(); @@ -640,7 +668,6 @@ static bool CanMergeValues(Value *First, Value *Second) { /// almost-empty BB ending in an unconditional branch to Succ, into Succ. /// /// Assumption: Succ is the single successor for BB. -/// static bool CanPropagatePredecessorsForPHIs(BasicBlock *BB, BasicBlock *Succ) { assert(*succ_begin(BB) == Succ && "Succ is not successor of BB!"); @@ -696,8 +723,8 @@ static bool CanPropagatePredecessorsForPHIs(BasicBlock *BB, BasicBlock *Succ) { return true; } -typedef SmallVector PredBlockVector; -typedef DenseMap IncomingValueMap; +using PredBlockVector = SmallVector; +using IncomingValueMap = DenseMap; /// \brief Determines the value to use as the phi node input for a block. /// @@ -927,7 +954,6 @@ bool llvm::TryToSimplifyUncondBranchFromEmptyBlock(BasicBlock *BB) { /// nodes in this block. This doesn't try to be clever about PHI nodes /// which differ only in the order of the incoming values, but instcombine /// orders them so it usually won't matter. -/// bool llvm::EliminateDuplicatePHINodes(BasicBlock *BB) { // This implementation doesn't currently consider undef operands // specially. Theoretically, two phis which are identical except for @@ -937,9 +963,11 @@ bool llvm::EliminateDuplicatePHINodes(BasicBlock *BB) { static PHINode *getEmptyKey() { return DenseMapInfo::getEmptyKey(); } + static PHINode *getTombstoneKey() { return DenseMapInfo::getTombstoneKey(); } + static unsigned getHashValue(PHINode *PN) { // Compute a hash value on the operands. Instcombine will likely have // sorted them, which helps expose duplicates, but we have to check all @@ -948,6 +976,7 @@ bool llvm::EliminateDuplicatePHINodes(BasicBlock *BB) { hash_combine_range(PN->value_op_begin(), PN->value_op_end()), hash_combine_range(PN->block_begin(), PN->block_end()))); } + static bool isEqual(PHINode *LHS, PHINode *RHS) { if (LHS == getEmptyKey() || LHS == getTombstoneKey() || RHS == getEmptyKey() || RHS == getTombstoneKey()) @@ -984,7 +1013,6 @@ bool llvm::EliminateDuplicatePHINodes(BasicBlock *BB) { /// often possible though. If alignment is important, a more reliable approach /// is to simply align all global variables and allocation instructions to /// their preferred alignment from the beginning. -/// static unsigned enforceKnownAlignment(Value *V, unsigned Align, unsigned PrefAlign, const DataLayout &DL) { @@ -1068,7 +1096,7 @@ static bool LdStHasDebugValue(DILocalVariable *DIVar, DIExpression *DIExpr, // Since we can't guarantee that the original dbg.declare instrinsic // is removed by LowerDbgDeclare(), we need to make sure that we are // not inserting the same dbg.value intrinsic over and over. - llvm::BasicBlock::InstListType::iterator PrevI(I); + BasicBlock::InstListType::iterator PrevI(I); if (PrevI != I->getParent()->getInstList().begin()) { --PrevI; if (DbgValueInst *DVI = dyn_cast(PrevI)) @@ -1488,7 +1516,6 @@ BasicBlock *llvm::changeToInvokeAndSplitBasicBlock(CallInst *CI, static bool markAliveBlocks(Function &F, SmallPtrSetImpl &Reachable) { - SmallVector Worklist; BasicBlock *BB = &F.front(); Worklist.push_back(BB); @@ -1594,13 +1621,16 @@ static bool markAliveBlocks(Function &F, static CatchPadInst *getEmptyKey() { return DenseMapInfo::getEmptyKey(); } + static CatchPadInst *getTombstoneKey() { return DenseMapInfo::getTombstoneKey(); } + static unsigned getHashValue(CatchPadInst *CatchPad) { return static_cast(hash_combine_range( CatchPad->value_op_begin(), CatchPad->value_op_end())); } + static bool isEqual(CatchPadInst *LHS, CatchPadInst *RHS) { if (LHS == getEmptyKey() || LHS == getTombstoneKey() || RHS == getEmptyKey() || RHS == getTombstoneKey()) @@ -1910,6 +1940,7 @@ void llvm::copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI, } namespace { + /// A potential constituent of a bitreverse or bswap expression. See /// collectBitParts for a fuller explanation. struct BitPart { @@ -1919,12 +1950,14 @@ struct BitPart { /// The Value that this is a bitreverse/bswap of. Value *Provider; + /// The "provenance" of each bit. Provenance[A] = B means that bit A /// in Provider becomes bit B in the result of this expression. SmallVector Provenance; // int8_t means max size is i128. enum { Unset = -1 }; }; + } // end anonymous namespace /// Analyze the specified subexpression and see if it is capable of providing @@ -1950,7 +1983,6 @@ struct BitPart { /// /// Because we pass around references into \c BPS, we must use a container that /// does not invalidate internal references (std::map instead of DenseMap). -/// static const Optional & collectBitParts(Value *V, bool MatchBSwaps, bool MatchBitReversals, std::map> &BPS) { diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp index 9cf66382b581..2ec4f6ca9e7f 100644 --- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp @@ -1,4 +1,4 @@ -//===----- LoadStoreVectorizer.cpp - GPU Load & Store Vectorizer ----------===// +//===- LoadStoreVectorizer.cpp - GPU Load & Store Vectorizer --------------===// // // The LLVM Compiler Infrastructure // @@ -6,47 +6,66 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// -// -//===----------------------------------------------------------------------===// +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/PostOrderIterator.h" -#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" -#include "llvm/ADT/Triple.h" +#include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/MemoryLocation.h" #include "llvm/Analysis/OrderedBasicBlock.h" #include "llvm/Analysis/ScalarEvolution.h" -#include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/Analysis/VectorUtils.h" +#include "llvm/IR/Attributes.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Dominators.h" +#include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" +#include "llvm/IR/User.h" #include "llvm/IR/Value.h" -#include "llvm/Support/CommandLine.h" +#include "llvm/Pass.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/Debug.h" #include "llvm/Support/KnownBits.h" +#include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Vectorize.h" +#include +#include +#include +#include +#include using namespace llvm; #define DEBUG_TYPE "load-store-vectorizer" + STATISTIC(NumVectorInstructions, "Number of vector accesses generated"); STATISTIC(NumScalarsVectorized, "Number of scalar accesses vectorized"); -namespace { - // FIXME: Assuming stack alignment of 4 is always good enough static const unsigned StackAdjustedAlignment = 4; -typedef SmallVector InstrList; -typedef MapVector InstrListMap; + +namespace { + +using InstrList = SmallVector; +using InstrListMap = MapVector; class Vectorizer { Function &F; @@ -163,7 +182,10 @@ public: AU.setPreservesCFG(); } }; -} + +} // end anonymous namespace + +char LoadStoreVectorizer::ID = 0; INITIALIZE_PASS_BEGIN(LoadStoreVectorizer, DEBUG_TYPE, "Vectorize load and Store instructions", false, false) @@ -175,8 +197,6 @@ INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) INITIALIZE_PASS_END(LoadStoreVectorizer, DEBUG_TYPE, "Vectorize load and store instructions", false, false) -char LoadStoreVectorizer::ID = 0; - Pass *llvm::createLoadStoreVectorizerPass() { return new LoadStoreVectorizer(); } @@ -605,7 +625,7 @@ Vectorizer::collectInstructions(BasicBlock *BB) { continue; // Make sure all the users of a vector are constant-index extracts. - if (isa(Ty) && !all_of(LI->users(), [](const User *U) { + if (isa(Ty) && !llvm::all_of(LI->users(), [](const User *U) { const ExtractElementInst *EEI = dyn_cast(U); return EEI && isa(EEI->getOperand(1)); })) @@ -614,7 +634,6 @@ Vectorizer::collectInstructions(BasicBlock *BB) { // Save the load locations. Value *ObjPtr = GetUnderlyingObject(Ptr, DL); LoadRefs[ObjPtr].push_back(LI); - } else if (StoreInst *SI = dyn_cast(&I)) { if (!SI->isSimple()) continue; @@ -639,7 +658,7 @@ Vectorizer::collectInstructions(BasicBlock *BB) { if (TySize > VecRegSize / 2) continue; - if (isa(Ty) && !all_of(SI->users(), [](const User *U) { + if (isa(Ty) && !llvm::all_of(SI->users(), [](const User *U) { const ExtractElementInst *EEI = dyn_cast(U); return EEI && isa(EEI->getOperand(1)); })) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 6055aff8b9b0..5dcf5528ac92 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -6,6 +6,7 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +// // This pass implements the Bottom Up SLP vectorizer. It detects consecutive // stores that can be put together into vector-stores. Next, it attempts to // construct vectorizable tree using the use-def chains. If a profitable tree @@ -361,14 +362,17 @@ static Value *isOneOf(Value *OpValue, Value *Op) { } namespace { + /// Contains data for the instructions going to be vectorized. struct RawInstructionsData { /// Main Opcode of the instructions going to be vectorized. unsigned Opcode = 0; + /// The list of instructions have some instructions with alternate opcodes. bool HasAltOpcodes = false; }; -} // namespace + +} // end anonymous namespace /// Checks the list of the vectorized instructions \p VL and returns info about /// this list. @@ -392,19 +396,24 @@ static RawInstructionsData getMainOpcode(ArrayRef VL) { } namespace { + /// Main data required for vectorization of instructions. struct InstructionsState { /// The very first instruction in the list with the main opcode. Value *OpValue = nullptr; + /// The main opcode for the list of instructions. unsigned Opcode = 0; + /// Some of the instructions in the list have alternate opcodes. bool IsAltShuffle = false; + InstructionsState() = default; InstructionsState(Value *OpValue, unsigned Opcode, bool IsAltShuffle) : OpValue(OpValue), Opcode(Opcode), IsAltShuffle(IsAltShuffle) {} }; -} // namespace + +} // end anonymous namespace /// \returns analysis of the Instructions in \p VL described in /// InstructionsState, the Opcode that we suppose the whole list @@ -973,6 +982,7 @@ private: return os; } #endif + friend struct GraphTraits; friend struct DOTGraphTraits; @@ -1176,9 +1186,9 @@ private: /// The ID of the scheduling region. For a new vectorization iteration this /// is incremented which "removes" all ScheduleData from the region. - int SchedulingRegionID = 1; // Make sure that the initial SchedulingRegionID is greater than the // initial SchedulingRegionID in ScheduleData (which is 0). + int SchedulingRegionID = 1; }; /// Attaches the BlockScheduling structures to basic blocks. @@ -1212,6 +1222,7 @@ private: unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt. unsigned MinVecRegSize; // Set by cl::opt (default: 128). + /// Instruction builder to construct the vectorized tree. IRBuilder<> Builder; @@ -4662,6 +4673,7 @@ class HorizontalReduction { RK_Max, /// Maximum reduction data. RK_UMax, /// Unsigned maximum reduction data. }; + /// Contains info about operation, like its opcode, left and right operands. class OperationData { /// Opcode of the instruction. @@ -4672,8 +4684,10 @@ class HorizontalReduction { /// Right operand of the reduction operation. Value *RHS = nullptr; + /// Kind of the reduction operation. ReductionKind Kind = RK_None; + /// True if float point min/max reduction has no NaNs. bool NoNaN = false; @@ -4725,7 +4739,7 @@ class HorizontalReduction { /// Construction for reduced values. They are identified by opcode only and /// don't have associated LHS/RHS values. - explicit OperationData(Value *V) : Kind(RK_None) { + explicit OperationData(Value *V) { if (auto *I = dyn_cast(V)) Opcode = I->getOpcode(); } @@ -4737,6 +4751,7 @@ class HorizontalReduction { : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind), NoNaN(NoNaN) { assert(Kind != RK_None && "One of the reduction operations is expected."); } + explicit operator bool() const { return Opcode; } /// Get the index of the first operand. @@ -5421,7 +5436,6 @@ private: /// starting from the last insertelement instruction. /// /// Returns true if it matches -/// static bool findBuildVector(InsertElementInst *LastInsertElem, SmallVectorImpl &BuildVector, SmallVectorImpl &BuildVectorOpds) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 498f4c4f7f31..f74426e5f301 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -18,12 +18,29 @@ //===----------------------------------------------------------------------===// #include "VPlan.h" +#include "llvm/ADT/DepthFirstIterator.h" #include "llvm/ADT/PostOrderIterator.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Twine.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CFG.h" #include "llvm/IR/Dominators.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Value.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/GraphWriter.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include +#include +#include +#include using namespace llvm; @@ -138,7 +155,6 @@ void VPBasicBlock::execute(VPTransformState *State) { SingleHPred->getExitBasicBlock() == PrevVPBB && PrevVPBB->getSingleHierarchicalSuccessor()) && /* B */ !(Replica && getPredecessors().empty())) { /* C */ - NewBB = createEmptyBasicBlock(State->CFG); State->Builder.SetInsertPoint(NewBB); // Temporarily terminate with unreachable until CFG is rewired. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 3c11fdeb0763..d43774dd36eb 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1,4 +1,4 @@ -//===- VPlan.h - Represent A Vectorizer Plan ------------------------------===// +//===- VPlan.h - Represent A Vectorizer Plan --------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -6,7 +6,7 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// -/// +// /// \file /// This file contains the declarations of the Vectorization Plan base classes: /// 1. VPBasicBlock and VPRegionBlock that inherit from a common pure virtual @@ -18,34 +18,37 @@ /// 4. The VPlan class holding a candidate for vectorization; /// 5. The VPlanPrinter class providing a way to print a plan in dot format. /// These are documented in docs/VectorizationPlan.rst. -/// +// //===----------------------------------------------------------------------===// #ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H #define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/GraphTraits.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Twine.h" #include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist_node.h" #include "llvm/IR/IRBuilder.h" -#include "llvm/Support/raw_ostream.h" - -// The (re)use of existing LoopVectorize classes is subject to future VPlan -// refactoring. -namespace { -// Forward declarations. -//class InnerLoopVectorizer; -class LoopVectorizationLegality; -class LoopVectorizationCostModel; -} // namespace +#include +#include +#include +#include +#include namespace llvm { -// Forward declarations. class BasicBlock; +class DominatorTree; class InnerLoopVectorizer; +class LoopInfo; +class raw_ostream; +class Value; class VPBasicBlock; +class VPRegionBlock; /// In what follows, the term "input IR" refers to code that is fed into the /// vectorizer whereas the term "output IR" refers to code that is generated by @@ -54,8 +57,11 @@ class VPBasicBlock; /// VPIteration represents a single point in the iteration space of the output /// (vectorized and/or unrolled) IR loop. struct VPIteration { - unsigned Part; ///< in [0..UF) - unsigned Lane; ///< in [0..VF) + /// in [0..UF) + unsigned Part; + + /// in [0..VF) + unsigned Lane; }; /// This is a helper struct for maintaining vectorization state. It's used for @@ -75,7 +81,6 @@ struct VPIteration { /// /// Entries from either map can be retrieved using the getVectorValue and /// getScalarValue functions, which assert that the desired value exists. - struct VectorizerValueMap { private: /// The unroll factor. Each entry in the vector map contains UF vector values. @@ -87,8 +92,8 @@ private: /// The vector and scalar map storage. We use std::map and not DenseMap /// because insertions to DenseMap invalidate its iterators. - typedef SmallVector VectorParts; - typedef SmallVector, 2> ScalarParts; + using VectorParts = SmallVector; + using ScalarParts = SmallVector, 2>; std::map VectorMapStorage; std::map ScalarMapStorage; @@ -193,12 +198,11 @@ public: /// VPTransformState holds information passed down when "executing" a VPlan, /// needed for generating the output IR. struct VPTransformState { - - VPTransformState(unsigned VF, unsigned UF, class LoopInfo *LI, - class DominatorTree *DT, IRBuilder<> &Builder, - VectorizerValueMap &ValueMap, InnerLoopVectorizer *ILV) - : VF(VF), UF(UF), Instance(), LI(LI), DT(DT), Builder(Builder), - ValueMap(ValueMap), ILV(ILV) {} + VPTransformState(unsigned VF, unsigned UF, LoopInfo *LI, DominatorTree *DT, + IRBuilder<> &Builder, VectorizerValueMap &ValueMap, + InnerLoopVectorizer *ILV) + : VF(VF), UF(UF), LI(LI), DT(DT), Builder(Builder), ValueMap(ValueMap), + ILV(ILV) {} /// The chosen Vectorization and Unroll Factors of the loop being vectorized. unsigned VF; @@ -213,25 +217,28 @@ struct VPTransformState { /// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks. struct CFGState { /// The previous VPBasicBlock visited. Initially set to null. - VPBasicBlock *PrevVPBB; + VPBasicBlock *PrevVPBB = nullptr; + /// The previous IR BasicBlock created or used. Initially set to the new /// header BasicBlock. - BasicBlock *PrevBB; + BasicBlock *PrevBB = nullptr; + /// The last IR BasicBlock in the output IR. Set to the new latch /// BasicBlock, used for placing the newly created BasicBlocks. - BasicBlock *LastBB; + BasicBlock *LastBB = nullptr; + /// A mapping of each VPBasicBlock to the corresponding BasicBlock. In case /// of replication, maps the BasicBlock of the last replica created. SmallDenseMap VPBB2IRBB; - CFGState() : PrevVPBB(nullptr), PrevBB(nullptr), LastBB(nullptr) {} + CFGState() = default; } CFG; /// Hold a pointer to LoopInfo to register new basic blocks in the loop. - class LoopInfo *LI; + LoopInfo *LI; /// Hold a pointer to Dominator Tree to register new basic blocks in the loop. - class DominatorTree *DT; + DominatorTree *DT; /// Hold a reference to the IRBuilder used to generate output IR code. IRBuilder<> &Builder; @@ -241,7 +248,7 @@ struct VPTransformState { VectorizerValueMap &ValueMap; /// Hold a pointer to InnerLoopVectorizer to reuse its IR generation methods. - class InnerLoopVectorizer *ILV; + InnerLoopVectorizer *ILV; }; /// VPBlockBase is the building block of the Hierarchical Control-Flow Graph. @@ -255,7 +262,7 @@ private: /// The immediate VPRegionBlock which this VPBlockBase belongs to, or null if /// it is a topmost VPBlockBase. - class VPRegionBlock *Parent; + VPRegionBlock *Parent = nullptr; /// List of predecessor blocks. SmallVector Predecessors; @@ -291,18 +298,18 @@ private: protected: VPBlockBase(const unsigned char SC, const std::string &N) - : SubclassID(SC), Name(N), Parent(nullptr) {} + : SubclassID(SC), Name(N) {} public: /// An enumeration for keeping track of the concrete subclass of VPBlockBase /// that are actually instantiated. Values of this enumeration are kept in the /// SubclassID field of the VPBlockBase objects. They are used for concrete /// type identification. - typedef enum { VPBasicBlockSC, VPRegionBlockSC } VPBlockTy; + using VPBlockTy = enum { VPBasicBlockSC, VPRegionBlockSC }; - typedef SmallVectorImpl VPBlocksTy; + using VPBlocksTy = SmallVectorImpl; - virtual ~VPBlockBase() {} + virtual ~VPBlockBase() = default; const std::string &getName() const { return Name; } @@ -437,14 +444,14 @@ private: const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast). /// Each VPRecipe belongs to a single VPBasicBlock. - VPBasicBlock *Parent; + VPBasicBlock *Parent = nullptr; public: /// An enumeration for keeping track of the concrete subclass of VPRecipeBase /// that is actually instantiated. Values of this enumeration are kept in the /// SubclassID field of the VPRecipeBase objects. They are used for concrete /// type identification. - typedef enum { + using VPRecipeTy = enum { VPBranchOnMaskSC, VPInterleaveSC, VPPredInstPHISC, @@ -452,11 +459,10 @@ public: VPWidenIntOrFpInductionSC, VPWidenPHISC, VPWidenSC, - } VPRecipeTy; + }; - VPRecipeBase(const unsigned char SC) : SubclassID(SC), Parent(nullptr) {} - - virtual ~VPRecipeBase() {} + VPRecipeBase(const unsigned char SC) : SubclassID(SC) {} + virtual ~VPRecipeBase() = default; /// \return an ID for the concrete type of this object. /// This is used to implement the classof checks. This should not be used @@ -480,18 +486,26 @@ public: /// output IR instructions. class VPBasicBlock : public VPBlockBase { public: - typedef iplist RecipeListTy; + using RecipeListTy = iplist; private: /// The VPRecipes held in the order of output instructions to generate. RecipeListTy Recipes; public: + VPBasicBlock(const Twine &Name = "", VPRecipeBase *Recipe = nullptr) + : VPBlockBase(VPBasicBlockSC, Name.str()) { + if (Recipe) + appendRecipe(Recipe); + } + + ~VPBasicBlock() override { Recipes.clear(); } + /// Instruction iterators... - typedef RecipeListTy::iterator iterator; - typedef RecipeListTy::const_iterator const_iterator; - typedef RecipeListTy::reverse_iterator reverse_iterator; - typedef RecipeListTy::const_reverse_iterator const_reverse_iterator; + using iterator = RecipeListTy::iterator; + using const_iterator = RecipeListTy::const_iterator; + using reverse_iterator = RecipeListTy::reverse_iterator; + using const_reverse_iterator = RecipeListTy::const_reverse_iterator; //===--------------------------------------------------------------------===// /// Recipe iterator methods @@ -518,14 +532,6 @@ public: return &VPBasicBlock::Recipes; } - VPBasicBlock(const Twine &Name = "", VPRecipeBase *Recipe = nullptr) - : VPBlockBase(VPBasicBlockSC, Name.str()) { - if (Recipe) - appendRecipe(Recipe); - } - - ~VPBasicBlock() { Recipes.clear(); } - /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPBlockBase *V) { return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC; @@ -581,7 +587,7 @@ public: Exit->setParent(this); } - ~VPRegionBlock() { + ~VPRegionBlock() override { if (Entry) deleteCFG(Entry); } @@ -649,7 +655,7 @@ public: private: /// Add to the given dominator tree the header block and every new basic block /// that was created between it and the latch block, inclusive. - static void updateDominatorTree(class DominatorTree *DT, + static void updateDominatorTree(DominatorTree *DT, BasicBlock *LoopPreHeaderBB, BasicBlock *LoopLatchBB); }; @@ -667,11 +673,11 @@ private: unsigned Depth; unsigned TabWidth = 2; std::string Indent; - unsigned BID = 0; - SmallDenseMap BlockID; + VPlanPrinter(raw_ostream &O, VPlan &P) : OS(O), Plan(P) {} + /// Handle indentation. void bumpIndent(int b) { Indent = std::string((Depth += b) * TabWidth, ' '); } @@ -701,8 +707,6 @@ private: void drawEdge(const VPBlockBase *From, const VPBlockBase *To, bool Hidden, const Twine &Label); - VPlanPrinter(raw_ostream &O, VPlan &P) : OS(O), Plan(P) {} - void dump(); static void printAsIngredient(raw_ostream &O, Value *V); @@ -710,6 +714,7 @@ private: struct VPlanIngredient { Value *V; + VPlanIngredient(Value *V) : V(V) {} }; @@ -732,8 +737,8 @@ inline raw_ostream &operator<<(raw_ostream &OS, VPlan &Plan) { // graph of VPBlockBase nodes... template <> struct GraphTraits { - typedef VPBlockBase *NodeRef; - typedef SmallVectorImpl::iterator ChildIteratorType; + using NodeRef = VPBlockBase *; + using ChildIteratorType = SmallVectorImpl::iterator; static NodeRef getEntryNode(NodeRef N) { return N; } @@ -747,8 +752,8 @@ template <> struct GraphTraits { }; template <> struct GraphTraits { - typedef const VPBlockBase *NodeRef; - typedef SmallVectorImpl::const_iterator ChildIteratorType; + using NodeRef = const VPBlockBase *; + using ChildIteratorType = SmallVectorImpl::const_iterator; static NodeRef getEntryNode(NodeRef N) { return N; } @@ -765,11 +770,9 @@ template <> struct GraphTraits { // graph of VPBlockBase nodes... and to walk it in inverse order. Inverse order // for a VPBlockBase is considered to be when traversing the predecessors of a // VPBlockBase instead of its successors. -// - template <> struct GraphTraits> { - typedef VPBlockBase *NodeRef; - typedef SmallVectorImpl::iterator ChildIteratorType; + using NodeRef = VPBlockBase *; + using ChildIteratorType = SmallVectorImpl::iterator; static Inverse getEntryNode(Inverse B) { return B; @@ -784,6 +787,6 @@ template <> struct GraphTraits> { } }; -} // namespace llvm +} // end namespace llvm #endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H