From 63182c2ac0b643a60d397274e8a31166fc7243fa Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sun, 13 Sep 2020 00:07:31 -0700 Subject: [PATCH] [gcov] Add spanning tree optimization gcov is an "Edge Profiling with Edge Counters" application according to Optimally Profiling and Tracing Programs (1994). The minimum number of counters necessary is |E|-(|V|-1). The unmeasured edges form a spanning tree. Both GCC --coverage and clang -fprofile-generate leverage this optimization. This patch implements the optimization for clang --coverage. The produced .gcda files are much smaller now. --- clang/test/CodeGen/code-coverage-tsan.c | 1 - compiler-rt/test/profile/Posix/gcov-fork.c | 2 +- .../test/profile/gcov-dump-and-remove.c | 8 +- .../Instrumentation/GCOVProfiling.cpp | 404 +++++++++++------- .../GCOVProfiling/atomic-counter.ll | 3 +- .../split-indirectbr-critical-edges.ll | 61 +++ 6 files changed, 327 insertions(+), 152 deletions(-) create mode 100644 llvm/test/Transforms/GCOVProfiling/split-indirectbr-critical-edges.ll diff --git a/clang/test/CodeGen/code-coverage-tsan.c b/clang/test/CodeGen/code-coverage-tsan.c index 023a99598075..17f6596aa83d 100644 --- a/clang/test/CodeGen/code-coverage-tsan.c +++ b/clang/test/CodeGen/code-coverage-tsan.c @@ -5,7 +5,6 @@ // CHECK-LABEL: void @foo() /// Two counters are incremented by __tsan_atomic64_fetch_add. // CHECK: call i64 @__tsan_atomic64_fetch_add -// CHECK-NEXT: call i64 @__tsan_atomic64_fetch_add // CHECK-NEXT: call i32 @__tsan_atomic32_fetch_sub _Atomic(int) cnt; diff --git a/compiler-rt/test/profile/Posix/gcov-fork.c b/compiler-rt/test/profile/Posix/gcov-fork.c index b89eb64922f0..e66690a961e2 100644 --- a/compiler-rt/test/profile/Posix/gcov-fork.c +++ b/compiler-rt/test/profile/Posix/gcov-fork.c @@ -17,7 +17,7 @@ int main(void) { // CHECK-NEXT: 1: [[#@LINE]]: int status; // CHECK-NEXT: -: [[#@LINE]]: func1(); // CHECK-NEXT: 1: [[#@LINE]]: pid_t pid = fork(); // CHECK-NEXT: 1: [[#@LINE]]: - if (pid == -1) return 1; // CHECK-NEXT: 2: [[#@LINE]]: + if (pid == -1) return 1; // CHECK-NEXT: 1: [[#@LINE]]: if (pid) // CHECK-NEXT: 2: [[#@LINE]]: wait(&status); // CHECK-NEXT: 1: [[#@LINE]]: func2(); // CHECK-NEXT: 2: [[#@LINE]]: diff --git a/compiler-rt/test/profile/gcov-dump-and-remove.c b/compiler-rt/test/profile/gcov-dump-and-remove.c index b7f80535aada..c35640f93b3d 100644 --- a/compiler-rt/test/profile/gcov-dump-and-remove.c +++ b/compiler-rt/test/profile/gcov-dump-and-remove.c @@ -11,10 +11,10 @@ extern void __gcov_dump(void); extern void __gcov_reset(void); extern int remove(const char *); // CHECK: -: [[#@LINE]]:extern int remove -int main(void) { // CHECK-NEXT: #####: [[#@LINE]]: - __gcov_dump(); // CHECK-NEXT: #####: [[#@LINE]]: - __gcov_reset(); // CHECK-NEXT: #####: [[#@LINE]]: - if (remove("gcov-dump-and-remove.gcda") != 0) // CHECK-NEXT: #####: [[#@LINE]]: +int main(void) { // CHECK-NEXT: 1: [[#@LINE]]: + __gcov_dump(); // CHECK-NEXT: 1: [[#@LINE]]: + __gcov_reset(); // CHECK-NEXT: 1: [[#@LINE]]: + if (remove("gcov-dump-and-remove.gcda") != 0) // CHECK-NEXT: 1: [[#@LINE]]: return 1; // CHECK-NEXT: #####: [[#@LINE]]: return 1; // CHECK-NEXT: -: [[#@LINE]]: __gcov_dump(); // CHECK-NEXT: 1: [[#@LINE]]: diff --git a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp index 734deda99707..437063eef6f9 100644 --- a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp @@ -13,6 +13,7 @@ // //===----------------------------------------------------------------------===// +#include "CFGMST.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Hashing.h" #include "llvm/ADT/STLExtras.h" @@ -20,6 +21,8 @@ #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringMap.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/CFG.h" @@ -53,6 +56,8 @@ namespace endian = llvm::support::endian; #define DEBUG_TYPE "insert-gcov-profiling" enum : uint32_t { + GCOV_ARC_ON_TREE = 1 << 0, + GCOV_TAG_FUNCTION = 0x01000000, GCOV_TAG_BLOCKS = 0x01410000, GCOV_TAG_ARCS = 0x01430000, @@ -94,9 +99,10 @@ class GCOVProfiler { public: GCOVProfiler() : GCOVProfiler(GCOVOptions::getDefault()) {} GCOVProfiler(const GCOVOptions &Opts) : Options(Opts) {} - bool - runOnModule(Module &M, - std::function GetTLI); + bool runOnModule(Module &M, + function_ref GetBFI, + function_ref GetBPI, + function_ref GetTLI); void write(uint32_t i) { char Bytes[4]; @@ -112,13 +118,12 @@ public: private: // Create the .gcno files for the Module based on DebugInfo. - void emitProfileNotes(NamedMDNode *CUNode); + bool + emitProfileNotes(NamedMDNode *CUNode, bool HasExecOrFork, + function_ref GetBFI, + function_ref GetBPI, + function_ref GetTLI); - // Modify the program to track transitions along edges and call into the - // profiling runtime to emit .gcda files when run. - void instrumentFunction( - Function &F, - SmallVectorImpl> &CountersBySP); void emitGlobalConstructor( SmallVectorImpl> &CountersBySP); @@ -158,6 +163,7 @@ private: SmallVector, 16> Funcs; std::vector FilterRe; std::vector ExcludeRe; + DenseSet ExecBlocks; StringMap InstrumentedFiles; }; @@ -173,24 +179,69 @@ public: StringRef getPassName() const override { return "GCOV Profiler"; } bool runOnModule(Module &M) override { - return Profiler.runOnModule(M, [this](Function &F) -> TargetLibraryInfo & { - return getAnalysis().getTLI(F); - }); + auto GetBFI = [this](Function &F) { + return &this->getAnalysis(F).getBFI(); + }; + auto GetBPI = [this](Function &F) { + return &this->getAnalysis(F).getBPI(); + }; + auto GetTLI = [this](Function &F) -> const TargetLibraryInfo & { + return this->getAnalysis().getTLI(F); + }; + return Profiler.runOnModule(M, GetBFI, GetBPI, GetTLI); } void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.addRequired(); AU.addRequired(); } private: GCOVProfiler Profiler; }; + +struct BBInfo { + BBInfo *Group; + uint32_t Index; + uint32_t Rank = 0; + + BBInfo(unsigned Index) : Group(this), Index(Index) {} + const std::string infoString() const { + return (Twine("Index=") + Twine(Index)).str(); + } +}; + +struct Edge { + // This class implements the CFG edges. Note the CFG can be a multi-graph. + // So there might be multiple edges with same SrcBB and DestBB. + const BasicBlock *SrcBB; + const BasicBlock *DestBB; + uint64_t Weight; + BasicBlock *Place = nullptr; + uint32_t SrcNumber, DstNumber; + bool InMST = false; + bool Removed = false; + bool IsCritical = false; + + Edge(const BasicBlock *Src, const BasicBlock *Dest, uint64_t W = 1) + : SrcBB(Src), DestBB(Dest), Weight(W) {} + + // Return the information string of an edge. + const std::string infoString() const { + return (Twine(Removed ? "-" : " ") + (InMST ? " " : "*") + + (IsCritical ? "c" : " ") + " W=" + Twine(Weight)) + .str(); + } +}; } char GCOVProfilerLegacyPass::ID = 0; INITIALIZE_PASS_BEGIN( GCOVProfilerLegacyPass, "insert-gcov-profiling", "Insert instrumentation for GCOV profiling", false, false) +INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) INITIALIZE_PASS_END( GCOVProfilerLegacyPass, "insert-gcov-profiling", @@ -275,8 +326,8 @@ namespace { return LinesByFile.try_emplace(Filename, P, Filename).first->second; } - void addEdge(GCOVBlock &Successor) { - OutEdges.push_back(&Successor); + void addEdge(GCOVBlock &Successor, uint32_t Flags) { + OutEdges.emplace_back(&Successor, Flags); } void writeOut() { @@ -310,9 +361,9 @@ namespace { } uint32_t Number; - SmallVector OutEdges; + SmallVector, 4> OutEdges; - private: + private: friend class GCOVFunction; GCOVBlock(GCOVProfiler *P, uint32_t Number) @@ -345,7 +396,7 @@ namespace { FuncChecksum = hash_value(FunctionNameAndLine); } - GCOVBlock &getBlock(BasicBlock *BB) { + GCOVBlock &getBlock(const BasicBlock *BB) { return Blocks.find(BB)->second; } @@ -402,33 +453,41 @@ namespace { LLVM_DEBUG(dbgs() << (Blocks.size() + 1) << " blocks\n"); // Emit edges between blocks. - Function *F = Blocks.begin()->first->getParent(); - write(GCOV_TAG_ARCS); - write(3); - write(0); - write(getBlock(&F->getEntryBlock()).Number); - write(0); // no flags - for (BasicBlock &I : *F) { - GCOVBlock &Block = getBlock(&I); + const uint32_t Outgoing = EntryBlock.OutEdges.size(); + if (Outgoing) { + write(GCOV_TAG_ARCS); + write(Outgoing * 2 + 1); + write(EntryBlock.Number); + for (const auto &E : EntryBlock.OutEdges) { + write(E.first->Number); + write(E.second); + } + } + std::vector Sorted; + Sorted.reserve(Blocks.size()); + for (auto &It : Blocks) + Sorted.push_back(&It.second); + llvm::sort(Sorted, [](GCOVBlock *x, GCOVBlock *y) { + return x->Number < y->Number; + }); + for (GCOVBlock &Block : make_pointee_range(Sorted)) { if (Block.OutEdges.empty()) continue; write(GCOV_TAG_ARCS); write(Block.OutEdges.size() * 2 + 1); write(Block.Number); - for (int i = 0, e = Block.OutEdges.size(); i != e; ++i) { - LLVM_DEBUG(dbgs() << Block.Number << " -> " - << Block.OutEdges[i]->Number << "\n"); - write(Block.OutEdges[i]->Number); - write(0); // no flags + for (const auto &E : Block.OutEdges) { + write(E.first->Number); + write(E.second); } } // Emit lines for each block. - for (BasicBlock &I : *F) - getBlock(&I).writeOut(); + for (GCOVBlock &Block : make_pointee_range(Sorted)) + Block.writeOut(); } - private: + public: const DISubprogram *SP; unsigned EndLine; uint32_t Ident; @@ -549,7 +608,9 @@ std::string GCOVProfiler::mangleName(const DICompileUnit *CU, } bool GCOVProfiler::runOnModule( - Module &M, std::function GetTLI) { + Module &M, function_ref GetBFI, + function_ref GetBPI, + function_ref GetTLI) { this->M = &M; this->GetTLI = std::move(GetTLI); Ctx = &M.getContext(); @@ -558,12 +619,12 @@ bool GCOVProfiler::runOnModule( if (!CUNode || (!Options.EmitNotes && !Options.EmitData)) return false; - bool Modified = AddFlushBeforeForkAndExec(); + bool HasExecOrFork = AddFlushBeforeForkAndExec(); FilterRe = createRegexesFromString(Options.Filter); ExcludeRe = createRegexesFromString(Options.Exclude); - emitProfileNotes(CUNode); - return Modified || Options.EmitData; + emitProfileNotes(CUNode, HasExecOrFork, GetBFI, GetBPI, GetTLI); + return true; } PreservedAnalyses GCOVProfilerPass::run(Module &M, @@ -573,9 +634,17 @@ PreservedAnalyses GCOVProfilerPass::run(Module &M, FunctionAnalysisManager &FAM = AM.getResult(M).getManager(); - if (!Profiler.runOnModule(M, [&](Function &F) -> TargetLibraryInfo & { - return FAM.getResult(F); - })) + auto GetBFI = [&FAM](Function &F) { + return &FAM.getResult(F); + }; + auto GetBPI = [&FAM](Function &F) { + return &FAM.getResult(F); + }; + auto GetTLI = [&FAM](Function &F) -> const TargetLibraryInfo & { + return FAM.getResult(F); + }; + + if (!Profiler.runOnModule(M, GetBFI, GetBPI, GetTLI)) return PreservedAnalyses::all(); return PreservedAnalyses::none(); @@ -681,6 +750,7 @@ bool GCOVProfiler::AddFlushBeforeForkAndExec() { // dumped FunctionCallee ResetF = M->getOrInsertFunction("llvm_reset_counters", FTy); Builder.CreateCall(ResetF)->setDebugLoc(Loc); + ExecBlocks.insert(Parent); Parent->splitBasicBlock(NextInst); Parent->back().setDebugLoc(Loc); } @@ -688,7 +758,67 @@ bool GCOVProfiler::AddFlushBeforeForkAndExec() { return !Forks.empty() || !Execs.empty(); } -void GCOVProfiler::emitProfileNotes(NamedMDNode *CUNode) { +static BasicBlock *getInstrBB(CFGMST &MST, Edge &E, + const DenseSet &ExecBlocks) { + if (E.InMST || E.Removed) + return nullptr; + + BasicBlock *SrcBB = const_cast(E.SrcBB); + BasicBlock *DestBB = const_cast(E.DestBB); + // For a fake edge, instrument the real BB. + if (SrcBB == nullptr) + return DestBB; + if (DestBB == nullptr) + return SrcBB; + + auto CanInstrument = [](BasicBlock *BB) -> BasicBlock * { + // There are basic blocks (such as catchswitch) cannot be instrumented. + // If the returned first insertion point is the end of BB, skip this BB. + if (BB->getFirstInsertionPt() == BB->end()) + return nullptr; + return BB; + }; + + // Instrument the SrcBB if it has a single successor, + // otherwise, the DestBB if this is not a critical edge. + Instruction *TI = SrcBB->getTerminator(); + if (TI->getNumSuccessors() <= 1 && !ExecBlocks.count(SrcBB)) + return CanInstrument(SrcBB); + if (!E.IsCritical) + return CanInstrument(DestBB); + + // Some IndirectBr critical edges cannot be split by the previous + // SplitIndirectBrCriticalEdges call. Bail out. + const unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB); + BasicBlock *InstrBB = + isa(TI) ? nullptr : SplitCriticalEdge(TI, SuccNum); + if (!InstrBB) + return nullptr; + + MST.addEdge(SrcBB, InstrBB, 0); + MST.addEdge(InstrBB, DestBB, 0).InMST = true; + E.Removed = true; + + return CanInstrument(InstrBB); +} + +#ifndef NDEBUG +static void dumpEdges(CFGMST &MST, GCOVFunction &GF) { + size_t ID = 0; + for (auto &E : make_pointee_range(MST.AllEdges)) { + GCOVBlock &Src = E.SrcBB ? GF.getBlock(E.SrcBB) : GF.getEntryBlock(); + GCOVBlock &Dst = E.DestBB ? GF.getBlock(E.DestBB) : GF.getReturnBlock(); + dbgs() << " Edge " << ID++ << ": " << Src.Number << "->" << Dst.Number + << E.infoString() << "\n"; + } +} +#endif + +bool GCOVProfiler::emitProfileNotes( + NamedMDNode *CUNode, bool HasExecOrFork, + function_ref GetBFI, + function_ref GetBPI, + function_ref GetTLI) { int Version; { uint8_t c3 = Options.Version[0]; @@ -725,36 +855,79 @@ void GCOVProfiler::emitProfileNotes(NamedMDNode *CUNode) { // TODO: Functions using scope-based EH are currently not supported. if (isUsingScopeBasedEH(F)) continue; - // gcov expects every function to start with an entry block that has a - // single successor, so split the entry block to make sure of that. - BasicBlock &EntryBlock = F.getEntryBlock(); - - Funcs.push_back(std::make_unique(this, &F, SP, EndLine, - FunctionIdent++, Version)); - GCOVFunction &Func = *Funcs.back(); - // Add the function line number to the lines of the entry block // to have a counter for the function definition. uint32_t Line = SP->getLine(); auto Filename = getFilename(SP); + BranchProbabilityInfo *BPI = GetBPI(F); + BlockFrequencyInfo *BFI = GetBFI(F); + + // Split indirectbr critical edges here before computing the MST rather + // than later in getInstrBB() to avoid invalidating it. + SplitIndirectBrCriticalEdges(F, BPI, BFI); + + CFGMST MST(F, /*InstrumentFuncEntry_=*/false, BPI, BFI); + + // getInstrBB can split basic blocks and push elements to AllEdges. + for (size_t I : llvm::seq(0, MST.AllEdges.size())) { + auto &E = *MST.AllEdges[I]; + // For now, disable spanning tree optimization when fork or exec* is + // used. + if (HasExecOrFork) + E.InMST = false; + E.Place = getInstrBB(MST, E, ExecBlocks); + } + // Basic blocks in F are finalized at this point. + BasicBlock &EntryBlock = F.getEntryBlock(); + Funcs.push_back(std::make_unique(this, &F, SP, EndLine, + FunctionIdent++, Version)); + GCOVFunction &Func = *Funcs.back(); + + // Some non-tree edges are IndirectBr which cannot be split. Ignore them + // as well. + llvm::erase_if(MST.AllEdges, [](std::unique_ptr &E) { + return E->Removed || (!E->InMST && !E->Place); + }); + const size_t Measured = + llvm::partition(MST.AllEdges, + [](std::unique_ptr &E) { return E->Place; }) - + MST.AllEdges.begin(); + for (size_t I : llvm::seq(0, Measured)) { + Edge &E = *MST.AllEdges[I]; + GCOVBlock &Src = + E.SrcBB ? Func.getBlock(E.SrcBB) : Func.getEntryBlock(); + GCOVBlock &Dst = + E.DestBB ? Func.getBlock(E.DestBB) : Func.getReturnBlock(); + E.SrcNumber = Src.Number; + E.DstNumber = Dst.Number; + } + std::stable_sort( + MST.AllEdges.begin(), MST.AllEdges.begin() + Measured, + [](const std::unique_ptr &L, const std::unique_ptr &R) { + return L->SrcNumber != R->SrcNumber ? L->SrcNumber < R->SrcNumber + : L->DstNumber < R->DstNumber; + }); + + for (const Edge &E : make_pointee_range(MST.AllEdges)) { + GCOVBlock &Src = + E.SrcBB ? Func.getBlock(E.SrcBB) : Func.getEntryBlock(); + GCOVBlock &Dst = + E.DestBB ? Func.getBlock(E.DestBB) : Func.getReturnBlock(); + Src.addEdge(Dst, E.Place ? 0 : uint32_t(GCOV_ARC_ON_TREE)); + } + // Artificial functions such as global initializers if (!SP->isArtificial()) Func.getBlock(&EntryBlock).getFile(Filename).addLine(Line); - Func.getEntryBlock().addEdge(Func.getBlock(&EntryBlock)); - for (auto &BB : F) { - GCOVBlock &Block = Func.getBlock(&BB); - Instruction *TI = BB.getTerminator(); - if (int successors = TI->getNumSuccessors()) { - for (int i = 0; i != successors; ++i) { - Block.addEdge(Func.getBlock(TI->getSuccessor(i))); - } - } else if (isa(TI)) { - Block.addEdge(Func.getReturnBlock()); - } - for (GCOVBlock *Succ : Block.OutEdges) { - uint32_t Idx = Succ->Number; + LLVM_DEBUG(dumpEdges(MST, Func)); + + for (auto &GB : Func.Blocks) { + const BasicBlock &BB = *GB.first; + auto &Block = GB.second; + for (auto Succ : Block.OutEdges) { + uint32_t Idx = Succ.first->Number; do EdgeDestinations.push_back(Idx & 255); while ((Idx >>= 8) > 0); } @@ -782,8 +955,30 @@ void GCOVProfiler::emitProfileNotes(NamedMDNode *CUNode) { } Line = 0; } - if (EmitGCDA) - instrumentFunction(F, CountersBySP); + if (EmitGCDA) { + DISubprogram *SP = F.getSubprogram(); + ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(*Ctx), Measured); + GlobalVariable *Counters = new GlobalVariable( + *M, CounterTy, false, GlobalValue::InternalLinkage, + Constant::getNullValue(CounterTy), "__llvm_gcov_ctr"); + CountersBySP.emplace_back(Counters, SP); + + for (size_t I : llvm::seq(0, Measured)) { + const Edge &E = *MST.AllEdges[I]; + IRBuilder<> Builder(E.Place, E.Place->getFirstInsertionPt()); + Value *V = Builder.CreateConstInBoundsGEP2_64( + Counters->getValueType(), Counters, 0, I); + if (Options.Atomic) { + Builder.CreateAtomicRMW(AtomicRMWInst::Add, V, Builder.getInt64(1), + AtomicOrdering::Monotonic); + } else { + Value *Count = + Builder.CreateLoad(Builder.getInt64Ty(), V, "gcov_ctr"); + Count = Builder.CreateAdd(Count, Builder.getInt64(1)); + Builder.CreateStore(Count, V); + } + } + } } char Tmp[4]; @@ -830,86 +1025,7 @@ void GCOVProfiler::emitProfileNotes(NamedMDNode *CUNode) { EmitGCDA = false; } } -} - -void GCOVProfiler::instrumentFunction( - Function &F, - SmallVectorImpl> &CountersBySP) { - DISubprogram *SP = F.getSubprogram(); - DenseMap, unsigned> EdgeToCounter; - unsigned Edges = 0; - EdgeToCounter[{nullptr, &F.getEntryBlock()}] = Edges++; - for (auto &BB : F) { - Instruction *TI = BB.getTerminator(); - if (isa(TI)) { - EdgeToCounter[{&BB, nullptr}] = Edges++; - } else { - for (BasicBlock *Succ : successors(TI)) { - EdgeToCounter[{&BB, Succ}] = Edges++; - } - } - } - - ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(*Ctx), Edges); - GlobalVariable *Counters = - new GlobalVariable(*M, CounterTy, false, GlobalValue::InternalLinkage, - Constant::getNullValue(CounterTy), "__llvm_gcov_ctr"); - CountersBySP.push_back(std::make_pair(Counters, SP)); - - // If a BB has several predecessors, use a PHINode to select - // the correct counter. - for (auto &BB : F) { - // The phi node must be at the begin of the BB. - IRBuilder<> BuilderForPhi(&*BB.begin()); - IRBuilder<> Builder(&*BB.getFirstInsertionPt()); - Type *Int64PtrTy = Type::getInt64PtrTy(*Ctx); - Value *V; - if (&BB == &F.getEntryBlock()) { - auto It = EdgeToCounter.find({nullptr, &BB}); - V = Builder.CreateConstInBoundsGEP2_64(Counters->getValueType(), Counters, - 0, It->second); - } else { - const unsigned EdgeCount = std::distance(pred_begin(&BB), pred_end(&BB)); - if (EdgeCount == 0) - continue; - PHINode *Phi = BuilderForPhi.CreatePHI(Int64PtrTy, EdgeCount); - for (BasicBlock *Pred : predecessors(&BB)) { - auto It = EdgeToCounter.find({Pred, &BB}); - assert(It != EdgeToCounter.end()); - const unsigned Edge = It->second; - Value *EdgeCounter = BuilderForPhi.CreateConstInBoundsGEP2_64( - Counters->getValueType(), Counters, 0, Edge); - Phi->addIncoming(EdgeCounter, Pred); - V = Phi; - } - } - - if (Options.Atomic) { - Builder.CreateAtomicRMW(AtomicRMWInst::Add, V, Builder.getInt64(1), - AtomicOrdering::Monotonic); - } else { - Value *Count = Builder.CreateLoad(Builder.getInt64Ty(), V, "gcov_ctr"); - Count = Builder.CreateAdd(Count, Builder.getInt64(1)); - Builder.CreateStore(Count, V); - } - - Instruction *TI = BB.getTerminator(); - if (isa(TI)) { - auto It = EdgeToCounter.find({&BB, nullptr}); - assert(It != EdgeToCounter.end()); - const unsigned Edge = It->second; - Value *Counter = Builder.CreateConstInBoundsGEP2_64( - Counters->getValueType(), Counters, 0, Edge); - if (Options.Atomic) { - Builder.CreateAtomicRMW(AtomicRMWInst::Add, Counter, - Builder.getInt64(1), AtomicOrdering::Monotonic); - } else { - Value *Count = Builder.CreateLoad(Builder.getInt64Ty(), Counter); - Count = Builder.CreateAdd(Count, Builder.getInt64(1)); - Builder.CreateStore(Count, Counter); - } - } - } + return true; } void GCOVProfiler::emitGlobalConstructor( diff --git a/llvm/test/Transforms/GCOVProfiling/atomic-counter.ll b/llvm/test/Transforms/GCOVProfiling/atomic-counter.ll index 61ee30a4414b..2c5ea41b6fd8 100644 --- a/llvm/test/Transforms/GCOVProfiling/atomic-counter.ll +++ b/llvm/test/Transforms/GCOVProfiling/atomic-counter.ll @@ -4,8 +4,7 @@ ; CHECK-LABEL: void @empty() ; CHECK-NEXT: entry: -; CHECK-NEXT: %0 = atomicrmw add i64* getelementptr inbounds ([2 x i64], [2 x i64]* @__llvm_gcov_ctr, i64 0, i64 0), i64 1 monotonic, !dbg [[DBG:![0-9]+]] -; CHECK-NEXT: %1 = atomicrmw add i64* getelementptr inbounds ([2 x i64], [2 x i64]* @__llvm_gcov_ctr, i64 0, i64 1), i64 1 monotonic, !dbg [[DBG]] +; CHECK-NEXT: %0 = atomicrmw add i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__llvm_gcov_ctr, i64 0, i64 0), i64 1 monotonic, !dbg [[DBG:![0-9]+]] ; CHECK-NEXT: ret void, !dbg [[DBG]] define dso_local void @empty() !dbg !5 { diff --git a/llvm/test/Transforms/GCOVProfiling/split-indirectbr-critical-edges.ll b/llvm/test/Transforms/GCOVProfiling/split-indirectbr-critical-edges.ll new file mode 100644 index 000000000000..4d4ffe4021fa --- /dev/null +++ b/llvm/test/Transforms/GCOVProfiling/split-indirectbr-critical-edges.ll @@ -0,0 +1,61 @@ +; RUN: mkdir -p %t && cd %t +; RUN: opt < %s -passes=insert-gcov-profiling -S | FileCheck %s + +; CHECK: @__llvm_gcov_ctr = internal global [1 x i64] zeroinitializer + +;; If an indirectbr critical edge cannot be split, ignore it. +;; The edge will not be profiled. +; CHECK-LABEL: @cannot_split( +; CHECK: indirect.preheader: +; CHECK-NEXT: load {{.*}} @__llvm_gcov_ctr +; CHECK-NOT: load {{.*}} @__llvm_gcov_ctr + +define dso_local i32 @cannot_split(i8* nocapture readonly %p) #0 !dbg !7 { +entry: + %targets = alloca <2 x i8*>, align 16 + store <2 x i8*> , <2 x i8*>* %targets, align 16, !dbg !9 + br label %for.cond, !dbg !14 + +for.cond: ; preds = %for.cond, %entry + %p.addr.0 = phi i8* [ %p, %entry ], [ %incdec.ptr, %for.cond ] + %0 = load i8, i8* %p.addr.0, align 1, !dbg !15 + %cmp = icmp eq i8 %0, 7, !dbg !17 + %incdec.ptr = getelementptr inbounds i8, i8* %p.addr.0, i64 1, !dbg !18 + br i1 %cmp, label %indirect.preheader, label %for.cond, !dbg !15, !llvm.loop !19 + +indirect.preheader: ; preds = %for.cond + %1 = load i8, i8* %incdec.ptr, align 1, !dbg !21 + %idxprom = sext i8 %1 to i64, !dbg !21 + %arrayidx4 = getelementptr inbounds <2 x i8*>, <2 x i8*>* %targets, i64 0, i64 %idxprom, !dbg !21 + %2 = load i8*, i8** %arrayidx4, align 8, !dbg !21 + br label %indirect + +indirect: ; preds = %indirect.preheader, %indirect + indirectbr i8* %2, [label %indirect, label %end] + +end: ; preds = %indirect + ret i32 0, !dbg !22 +} + +attributes #0 = { norecurse nounwind readonly uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 12.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "a.c", directory: "/tmp/c") +!2 = !{} +!3 = !{i32 7, !"Dwarf Version", i32 4} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!7 = distinct !DISubprogram(name: "cannot_split", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2) +!8 = !DISubroutineType(types: !2) +!9 = !DILocation(line: 3, column: 14, scope: !7) +!14 = !DILocation(line: 5, column: 3, scope: !7) +!15 = !DILocation(line: 6, column: 9, scope: !7) +!17 = !DILocation(line: 6, column: 12, scope: !7) +!18 = !DILocation(line: 5, column: 12, scope: !7) +!19 = distinct !{!19, !14, !20} +!20 = !DILocation(line: 9, column: 5, scope: !7) +!21 = !DILocation(line: 0, scope: !7) +!22 = !DILocation(line: 11, column: 3, scope: !7)