maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

Revert "[PGO] Context sensitive PGO (part 1)" 

This reverts commit r354930, it was causing UBSan failures.

llvm-svn: 354953
This commit is contained in:
Vlad Tsyrklevich 2019-02-27 03:45:28 +00:00
parent 2ffa170515
commit c01643087e
14 changed files with 75 additions and 309 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
llvm/include/llvm/InitializePasses.h
View File

@ -299,7 +299,6 @@ void initializePEIPass(PassRegistry&);
void initializePGOIndirectCallPromotionLegacyPassPass(PassRegistry&);
void initializePGOInstrumentationGenLegacyPassPass(PassRegistry&);
void initializePGOInstrumentationUseLegacyPassPass(PassRegistry&);
void initializePGOInstrumentationGenCreateVarLegacyPassPass(PassRegistry&);
void initializePGOMemOPSizeOptLegacyPassPass(PassRegistry&);
void initializePHIEliminationPass(PassRegistry&);
void initializePartialInlinerLegacyPassPass(PassRegistry&);

6
llvm/include/llvm/LTO/Config.h
View File

@ -55,9 +55,6 @@ struct Config {
/// Disable entirely the optimizer, including importing for ThinLTO
bool CodeGenOnly = false;
/// Run PGO context sensitive IR instrumentation.
bool RunCSIRInstr = false;
/// If this field is set, the set of passes run in the middle-end optimizer
/// will be the one specified by the string. Only works with the new pass
/// manager as the old one doesn't have this ability.
@ -76,9 +73,6 @@ struct Config {
/// with this triple.
std::string DefaultTriple;
/// Context Sensitive PGO profile path.
std::string CSIRProfile;
/// Sample PGO profile path.
std::string SampleProfile;

1
llvm/include/llvm/LinkAllPasses.h
View File

@ -102,7 +102,6 @@ namespace {
(void) llvm::createGCOVProfilerPass();
(void) llvm::createPGOInstrumentationGenLegacyPass();
(void) llvm::createPGOInstrumentationUseLegacyPass();
(void) llvm::createPGOInstrumentationGenCreateVarLegacyPass();
(void) llvm::createPGOIndirectCallPromotionLegacyPass();
(void) llvm::createPGOMemOPSizeOptLegacyPass();
(void) llvm::createInstrProfilingLegacyPass();

16
llvm/include/llvm/ProfileData/InstrProf.h
View File

@ -767,20 +767,10 @@ struct NamedInstrProfRecord : InstrProfRecord {
StringRef Name;
uint64_t Hash;
// We reserve this bit as the flag for context sensitive profile record.
static const int CS_FLAG_IN_FUNC_HASH = 60;
NamedInstrProfRecord() = default;
NamedInstrProfRecord(StringRef Name, uint64_t Hash,
std::vector<uint64_t> Counts)
: InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {}
static bool hasCSFlagInHash(uint64_t FuncHash) {
return ((FuncHash >> CS_FLAG_IN_FUNC_HASH) & 1);
}
static void setCSFlagInHash(uint64_t &FuncHash) {
FuncHash |= ((uint64_t)1 << CS_FLAG_IN_FUNC_HASH);
}
};
uint32_t InstrProfRecord::getNumValueKinds() const {
@ -1014,8 +1004,6 @@ namespace RawInstrProf {
// from control data struct is changed from raw pointer to Name's MD5 value.
// Version 4: ValueDataBegin and ValueDataSizes fields are removed from the
// raw header.
// Version 5: Bit 60 of FuncHash is reserved for the flag for the context
// sensitive records.
const uint64_t Version = INSTR_PROF_RAW_VERSION;
template <class IntPtrT> inline uint64_t getMagic();
@ -1052,10 +1040,6 @@ struct Header {
void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart,
int64_t &RangeLast);
// Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
// aware this is an ir_level profile so it can set the version flag.
void createIRLevelProfileFlagVar(Module &M, bool IsCS);
// Create the variable for the profile file name.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput);

2
llvm/include/llvm/ProfileData/InstrProfData.inc
View File

@ -635,12 +635,10 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
* version for other variants of profile. We set the lowest bit of the upper 8
* bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton
* generated profile, and 0 if this is a Clang FE generated profile.
* 1 in bit 57 indicates there are context-sensitive records in the profile.
*/
#define VARIANT_MASKS_ALL 0xff00000000000000ULL
#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime

18
llvm/include/llvm/Transforms/Instrumentation.h
View File

@ -87,14 +87,10 @@ struct GCOVOptions {
ModulePass *createGCOVProfilerPass(const GCOVOptions &Options =
GCOVOptions::getDefault());
// PGO Instrumention. Parameter IsCS indicates if this is the context senstive
// instrumentation.
ModulePass *createPGOInstrumentationGenLegacyPass(bool IsCS = false);
// PGO Instrumention
ModulePass *createPGOInstrumentationGenLegacyPass();
ModulePass *
createPGOInstrumentationUseLegacyPass(StringRef Filename = StringRef(""),
bool IsCS = false);
ModulePass *createPGOInstrumentationGenCreateVarLegacyPass(
StringRef CSInstrName = StringRef(""));
createPGOInstrumentationUseLegacyPass(StringRef Filename = StringRef(""));
ModulePass *createPGOIndirectCallPromotionLegacyPass(bool InLTO = false,
bool SamplePGO = false);
FunctionPass *createPGOMemOPSizeOptLegacyPass();
@ -136,19 +132,15 @@ struct InstrProfOptions {
// Use atomic profile counter increments.
bool Atomic = false;
// Use BFI to guide register promotion
bool UseBFIInPromotion = false;
// Name of the profile file to use as output
std::string InstrProfileOutput;
InstrProfOptions() = default;
};
/// Insert frontend instrumentation based profiling. Parameter IsCS indicates if
// this is the context senstive instrumentation.
/// Insert frontend instrumentation based profiling.
ModulePass *createInstrProfilingLegacyPass(
const InstrProfOptions &Options = InstrProfOptions(), bool IsCS = false);
const InstrProfOptions &Options = InstrProfOptions());
FunctionPass *createHWAddressSanitizerPass(bool CompileKernel = false,
bool Recover = false);

6
llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
View File

@ -35,8 +35,7 @@ using LoadStorePair = std::pair<Instruction *, Instruction *>;
class InstrProfiling : public PassInfoMixin<InstrProfiling> {
public:
InstrProfiling() = default;
InstrProfiling(const InstrProfOptions &Options, bool IsCS)
: Options(Options), IsCS(IsCS) {}
InstrProfiling(const InstrProfOptions &Options) : Options(Options) {}
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
bool run(Module &M, const TargetLibraryInfo &TLI);
@ -61,9 +60,6 @@ private:
GlobalVariable *NamesVar;
size_t NamesSize;
// Is this lowering for the context-sensitive instrumentation.
bool IsCS;
// vector of counter load/store pairs to be register promoted.
std::vector<LoadStorePair> PromotionCandidates;

31
llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
View File

@ -17,7 +17,6 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/IR/PassManager.h"
#include "llvm/ProfileData/InstrProf.h"
#include <cstdint>
#include <string>
@ -27,51 +26,23 @@ class Function;
class Instruction;
class Module;
/// The instrumentation (profile-instr-gen) pass for IR based PGO.
// We use this pass to create COMDAT profile variables for context
// sensitive PGO (CSPGO). The reason to have a pass for this is CSPGO
// can be run after LTO/ThinLTO linking. Lld linker needs to see
// all the COMDAT variables before linking. So we have this pass
// always run before linking for CSPGO.
class PGOInstrumentationGenCreateVar
: public PassInfoMixin<PGOInstrumentationGenCreateVar> {
public:
PGOInstrumentationGenCreateVar(std::string CSInstrName = "")
: CSInstrName(CSInstrName) {}
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM) {
createProfileFileNameVar(M, CSInstrName);
createIRLevelProfileFlagVar(M, /* IsCS */ true);
return PreservedAnalyses::all();
}
private:
std::string CSInstrName;
};
/// The instrumentation (profile-instr-gen) pass for IR based PGO.
class PGOInstrumentationGen : public PassInfoMixin<PGOInstrumentationGen> {
public:
PGOInstrumentationGen(bool IsCS = false) : IsCS(IsCS) {}
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
private:
// If this is a context sensitive instrumentation.
bool IsCS;
};
/// The profile annotation (profile-instr-use) pass for IR based PGO.
class PGOInstrumentationUse : public PassInfoMixin<PGOInstrumentationUse> {
public:
PGOInstrumentationUse(std::string Filename = "",
std::string RemappingFilename = "", bool IsCS = false);
std::string RemappingFilename = "");
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
private:
std::string ProfileFileName;
std::string ProfileRemappingFileName;
// If this is a context sensitive instrumentation.
bool IsCS;
};
/// The indirect function call promotion pass.

3
llvm/lib/Passes/PassBuilder.cpp
View File

@ -569,8 +569,7 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
if (!ProfileGenFile.empty())
Options.InstrProfileOutput = ProfileGenFile;
Options.DoCounterPromotion = true;
Options.UseBFIInPromotion = false;
MPM.addPass(InstrProfiling(Options, false));
MPM.addPass(InstrProfiling(Options));
}
if (!ProfileUseFile.empty())

19
llvm/lib/ProfileData/InstrProf.cpp
View File

@ -1011,25 +1011,6 @@ void getMemOPSizeRangeFromOption(StringRef MemOPSizeRange, int64_t &RangeStart,
assert(RangeLast >= RangeStart);
}
// Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
// aware this is an ir_level profile so it can set the version flag.
void createIRLevelProfileFlagVar(Module &M, bool IsCS) {
const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
Type *IntTy64 = Type::getInt64Ty(M.getContext());
uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
if (IsCS)
ProfileVersion |= VARIANT_MASK_CSIR_PROF;
auto IRLevelVersionVariable = new GlobalVariable(
M, IntTy64, true, GlobalValue::WeakAnyLinkage,
Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), VarName);
IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility);
Triple TT(M.getTargetTriple());
if (TT.supportsCOMDAT()) {
IRLevelVersionVariable->setLinkage(GlobalValue::ExternalLinkage);
IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName));
}
}
// Create the variable for the profile file name.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) {
if (InstrProfileOutput.empty())

53
llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
View File

@ -18,8 +18,6 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Triple.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/IR/Attributes.h"
@ -149,8 +147,8 @@ public:
static char ID;
InstrProfilingLegacyPass() : ModulePass(ID) {}
InstrProfilingLegacyPass(const InstrProfOptions &Options, bool IsCS)
: ModulePass(ID), InstrProf(Options, IsCS) {}
InstrProfilingLegacyPass(const InstrProfOptions &Options)
: ModulePass(ID), InstrProf(Options) {}
StringRef getPassName() const override {
return "Frontend instrumentation-based coverage lowering";
@ -234,9 +232,9 @@ class PGOCounterPromoter {
public:
PGOCounterPromoter(
DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
Loop &CurLoop, LoopInfo &LI)
: LoopToCandidates(LoopToCands), ExitBlocks(), InsertPts(), L(CurLoop),
LI(LI), BFI(BFI) {
LI(LI) {
SmallVector<BasicBlock *, 8> LoopExitBlocks;
SmallPtrSet<BasicBlock *, 8> BlockSet;
@ -265,20 +263,6 @@ public:
SSAUpdater SSA(&NewPHIs);
Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
// If BFI is set, we will use it to guide the promotions.
if (BFI) {
auto *BB = Cand.first->getParent();
auto InstrCount = BFI->getBlockProfileCount(BB);
if (!InstrCount)
continue;
auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
// If the average loop trip count is not greater than 1.5, we skip
// promotion.
if (PreheaderCount &&
(PreheaderCount.getValue() * 3) >= (InstrCount.getValue() * 2))
continue;
}
PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
L.getLoopPreheader(), ExitBlocks,
InsertPts, LoopToCandidates, LI);
@ -328,11 +312,6 @@ private:
SmallVector<BasicBlock *, 8> ExitingBlocks;
LP->getExitingBlocks(ExitingBlocks);
// If BFI is set, we do more aggressive promotions based on BFI.
if (BFI)
return (unsigned)-1;
// Not considierered speculative.
if (ExitingBlocks.size() == 1)
return MaxNumOfPromotionsPerLoop;
@ -364,7 +343,6 @@ private:
SmallVector<Instruction *, 8> InsertPts;
Loop &L;
LoopInfo &LI;
BlockFrequencyInfo *BFI;
};
} // end anonymous namespace
@ -387,9 +365,8 @@ INITIALIZE_PASS_END(
"Frontend instrumentation-based coverage lowering.", false, false)
ModulePass *
llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options,
bool IsCS) {
return new InstrProfilingLegacyPass(Options, IsCS);
llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options) {
return new InstrProfilingLegacyPass(Options);
}
static InstrProfIncrementInst *castToIncrementInst(Instruction *Instr) {
@ -438,13 +415,6 @@ void InstrProfiling::promoteCounterLoadStores(Function *F) {
LoopInfo LI(DT);
DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
std::unique_ptr<BlockFrequencyInfo> BFI;
if (Options.UseBFIInPromotion) {
std::unique_ptr<BranchProbabilityInfo> BPI;
BPI.reset(new BranchProbabilityInfo(*F, LI, TLI));
BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
}
for (const auto &LoadStore : PromotionCandidates) {
auto *CounterLoad = LoadStore.first;
auto *CounterStore = LoadStore.second;
@ -460,7 +430,7 @@ void InstrProfiling::promoteCounterLoadStores(Function *F) {
// Do a post-order traversal of the loops so that counter updates can be
// iteratively hoisted outside the loop nest.
for (auto *Loop : llvm::reverse(Loops)) {
PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI);
Promoter.run(&TotalCountersPromoted);
}
}
@ -711,6 +681,7 @@ static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
// Don't do this for Darwin. compiler-rt uses linker magic.
if (TT.isOSDarwin())
return false;
// Use linker script magic to get data/cnts/name start/end.
if (TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
TT.isOSFuchsia() || TT.isPS4CPU() || TT.isOSWindows())
@ -1014,12 +985,8 @@ void InstrProfiling::emitUses() {
}
void InstrProfiling::emitInitialization() {
// Create ProfileFileName variable. Don't don't this for the
// context-sensitive instrumentation lowering: This lowering is after
// LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
// have already create the variable before LTO/ThinLTO linking.
if (!IsCS)
createProfileFileNameVar(*M, Options.InstrProfileOutput);
// Create variable for profile name.
createProfileFileNameVar(*M, Options.InstrProfileOutput);
Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
if (!RegisterF)
return;

203
llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
View File

@ -65,7 +65,6 @@
#include "llvm/Analysis/IndirectCallVisitor.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
@ -133,19 +132,6 @@ STATISTIC(NumOfPGOFunc, "Number of functions having valid profile counts.");
STATISTIC(NumOfPGOMismatch, "Number of functions having mismatch profile.");
STATISTIC(NumOfPGOMissing, "Number of functions without profile.");
STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentations.");
STATISTIC(NumOfCSPGOInstrument, "Number of edges instrumented in CSPGO.");
STATISTIC(NumOfCSPGOSelectInsts,
"Number of select instruction instrumented in CSPGO.");
STATISTIC(NumOfCSPGOMemIntrinsics,
"Number of mem intrinsics instrumented in CSPGO.");
STATISTIC(NumOfCSPGOEdge, "Number of edges in CSPGO.");
STATISTIC(NumOfCSPGOBB, "Number of basic-blocks in CSPGO.");
STATISTIC(NumOfCSPGOSplit, "Number of critical edge splits in CSPGO.");
STATISTIC(NumOfCSPGOFunc,
"Number of functions having valid profile counts in CSPGO.");
STATISTIC(NumOfCSPGOMismatch,
"Number of functions having mismatch profile in CSPGO.");
STATISTIC(NumOfCSPGOMissing, "Number of functions without profile in CSPGO.");
// Command line option to specify the file to read profile from. This is
// mainly used for testing.
@ -397,8 +383,7 @@ class PGOInstrumentationGenLegacyPass : public ModulePass {
public:
static char ID;
PGOInstrumentationGenLegacyPass(bool IsCS = false)
: ModulePass(ID), IsCS(IsCS) {
PGOInstrumentationGenLegacyPass() : ModulePass(ID) {
initializePGOInstrumentationGenLegacyPassPass(
*PassRegistry::getPassRegistry());
}
@ -406,8 +391,6 @@ public:
StringRef getPassName() const override { return "PGOInstrumentationGenPass"; }
private:
// Is this is context-sensitive instrumentation.
bool IsCS;
bool runOnModule(Module &M) override;
void getAnalysisUsage(AnalysisUsage &AU) const override {
@ -420,8 +403,8 @@ public:
static char ID;
// Provide the profile filename as the parameter.
PGOInstrumentationUseLegacyPass(std::string Filename = "", bool IsCS = false)
: ModulePass(ID), ProfileFileName(std::move(Filename)), IsCS(IsCS) {
PGOInstrumentationUseLegacyPass(std::string Filename = "")
: ModulePass(ID), ProfileFileName(std::move(Filename)) {
if (!PGOTestProfileFile.empty())
ProfileFileName = PGOTestProfileFile;
initializePGOInstrumentationUseLegacyPassPass(
@ -432,38 +415,14 @@ public:
private:
std::string ProfileFileName;
// Is this is context-sensitive instrumentation use.
bool IsCS;
bool runOnModule(Module &M) override;
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<ProfileSummaryInfoWrapperPass>();
AU.addRequired<BlockFrequencyInfoWrapperPass>();
}
};
class PGOInstrumentationGenCreateVarLegacyPass : public ModulePass {
public:
static char ID;
StringRef getPassName() const override {
return "PGOInstrumentationGenCreateVarPass";
}
PGOInstrumentationGenCreateVarLegacyPass(std::string CSInstrName = "")
: ModulePass(ID), InstrProfileOutput(CSInstrName) {
initializePGOInstrumentationGenCreateVarLegacyPassPass(
*PassRegistry::getPassRegistry());
}
private:
bool runOnModule(Module &M) override {
createProfileFileNameVar(M, InstrProfileOutput);
createIRLevelProfileFlagVar(M, true);
return false;
}
std::string InstrProfileOutput;
};
} // end anonymous namespace
char PGOInstrumentationGenLegacyPass::ID = 0;
@ -475,8 +434,8 @@ INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)
INITIALIZE_PASS_END(PGOInstrumentationGenLegacyPass, "pgo-instr-gen",
"PGO instrumentation.", false, false)
ModulePass *llvm::createPGOInstrumentationGenLegacyPass(bool IsCS) {
return new PGOInstrumentationGenLegacyPass(IsCS);
ModulePass *llvm::createPGOInstrumentationGenLegacyPass() {
return new PGOInstrumentationGenLegacyPass();
}
char PGOInstrumentationUseLegacyPass::ID = 0;
@ -485,25 +444,11 @@ INITIALIZE_PASS_BEGIN(PGOInstrumentationUseLegacyPass, "pgo-instr-use",
"Read PGO instrumentation profile.", false, false)
INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
INITIALIZE_PASS_END(PGOInstrumentationUseLegacyPass, "pgo-instr-use",
"Read PGO instrumentation profile.", false, false)
ModulePass *llvm::createPGOInstrumentationUseLegacyPass(StringRef Filename,
bool IsCS) {
return new PGOInstrumentationUseLegacyPass(Filename.str(), IsCS);
}
char PGOInstrumentationGenCreateVarLegacyPass::ID = 0;
INITIALIZE_PASS(PGOInstrumentationGenCreateVarLegacyPass,
"pgo-instr-gen-create-var",
"Create PGO instrumentation version variable for CSPGO.", false,
false)
ModulePass *
llvm::createPGOInstrumentationGenCreateVarLegacyPass(StringRef CSInstrName) {
return new PGOInstrumentationGenCreateVarLegacyPass(CSInstrName);
ModulePass *llvm::createPGOInstrumentationUseLegacyPass(StringRef Filename) {
return new PGOInstrumentationUseLegacyPass(Filename.str());
}
namespace {
@ -551,9 +496,6 @@ template <class Edge, class BBInfo> class FuncPGOInstrumentation {
private:
Function &F;
// Is this is context-sensitive instrumentation.
bool IsCS;
// A map that stores the Comdat group in function F.
std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;
@ -593,23 +535,15 @@ public:
Function &Func,
std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
bool CreateGlobalVar = false, BranchProbabilityInfo *BPI = nullptr,
BlockFrequencyInfo *BFI = nullptr, bool IsCS = false)
: F(Func), IsCS(IsCS), ComdatMembers(ComdatMembers),
ValueSites(IPVK_Last + 1), SIVisitor(Func), MIVisitor(Func),
MST(F, BPI, BFI) {
BlockFrequencyInfo *BFI = nullptr)
: F(Func), ComdatMembers(ComdatMembers), ValueSites(IPVK_Last + 1),
SIVisitor(Func), MIVisitor(Func), MST(F, BPI, BFI) {
// This should be done before CFG hash computation.
SIVisitor.countSelects(Func);
MIVisitor.countMemIntrinsics(Func);
if (!IsCS) {
NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
NumOfPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics();
NumOfPGOBB += MST.BBInfos.size();
ValueSites[IPVK_IndirectCallTarget] = findIndirectCalls(Func);
} else {
NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
NumOfCSPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics();
NumOfCSPGOBB += MST.BBInfos.size();
}
NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
NumOfPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics();
ValueSites[IPVK_IndirectCallTarget] = findIndirectCalls(Func);
ValueSites[IPVK_MemOPSize] = MIVisitor.findMemIntrinsics(Func);
FuncName = getPGOFuncName(F);
@ -618,12 +552,13 @@ public:
renameComdatFunction();
LLVM_DEBUG(dumpInfo("after CFGMST"));
NumOfPGOBB += MST.BBInfos.size();
for (auto &E : MST.AllEdges) {
if (E->Removed)
continue;
IsCS ? NumOfCSPGOEdge++ : NumOfPGOEdge++;
NumOfPGOEdge++;
if (!E->InMST)
IsCS ? NumOfCSPGOInstrument++ : NumOfPGOInstrument++;
NumOfPGOInstrument++;
}
if (CreateGlobalVar)
@ -662,17 +597,9 @@ void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
}
}
JC.update(Indexes);
// Hash format for context sensitive profile. Reserve 4 bits for other
// information.
FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 |
(uint64_t)ValueSites[IPVK_IndirectCallTarget].size() << 48 |
//(uint64_t)ValueSites[IPVK_MemOPSize].size() << 40 |
(uint64_t)MST.AllEdges.size() << 32 | JC.getCRC();
// Reserve bit 60-63 for other information purpose.
FunctionHash &= 0x0FFFFFFFFFFFFFFF;
if (IsCS)
NamedInstrProfRecord::setCSFlagInHash(FunctionHash);
LLVM_DEBUG(dbgs() << "Function Hash Computation for " << F.getName() << ":\n"
<< " CRC = " << JC.getCRC()
<< ", Selects = " << SIVisitor.getNumOfSelectInsts()
@ -778,7 +705,7 @@ BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) {
// For a critical edge, we have to split. Instrument the newly
// created BB.
IsCS ? NumOfCSPGOSplit++ : NumOfPGOSplit++;
NumOfPGOSplit++;
LLVM_DEBUG(dbgs() << "Split critical edge: " << getBBInfo(SrcBB).Index
<< " --> " << getBBInfo(DestBB).Index << "\n");
unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB);
@ -793,14 +720,12 @@ BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) {
// Critical edges will be split.
static void instrumentOneFunc(
Function &F, Module *M, BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFI,
std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
bool IsCS) {
std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
// Split indirectbr critical edges here before computing the MST rather than
// later in getInstrBB() to avoid invalidating it.
SplitIndirectBrCriticalEdges(F, BPI, BFI);
FuncPGOInstrumentation<PGOEdge, BBInfo> FuncInfo(F, ComdatMembers, true, BPI,
BFI, IsCS);
BFI);
unsigned NumCounters = FuncInfo.getNumCounters();
uint32_t I = 0;
@ -927,10 +852,10 @@ public:
PGOUseFunc(Function &Func, Module *Modu,
std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
BranchProbabilityInfo *BPI = nullptr,
BlockFrequencyInfo *BFIin = nullptr, bool IsCS = false)
BlockFrequencyInfo *BFIin = nullptr)
: F(Func), M(Modu), BFI(BFIin),
FuncInfo(Func, ComdatMembers, false, BPI, BFIin, IsCS),
FreqAttr(FFA_Normal), IsCS(IsCS) {}
FuncInfo(Func, ComdatMembers, false, BPI, BFIin),
FreqAttr(FFA_Normal) {}
// Read counts for the instrumented BB from profile.
bool readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros);
@ -1003,9 +928,6 @@ private:
// Function hotness info derived from profile.
FuncFreqAttr FreqAttr;
// Is to use the context sensitive profile.
bool IsCS;
// Find the Instrumented BB and set the value.
void setInstrumentedCounts(const std::vector<uint64_t> &CountFromProfile);
@ -1099,31 +1021,23 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros)
handleAllErrors(std::move(E), [&](const InstrProfError &IPE) {
auto Err = IPE.get();
bool SkipWarning = false;
LLVM_DEBUG(dbgs() << "Error in reading profile for Func "
<< FuncInfo.FuncName << ": ");
if (Err == instrprof_error::unknown_function) {
IsCS ? NumOfCSPGOMissing++ : NumOfPGOMissing++;
NumOfPGOMissing++;
SkipWarning = !PGOWarnMissing;
LLVM_DEBUG(dbgs() << "unknown function");
} else if (Err == instrprof_error::hash_mismatch ||
Err == instrprof_error::malformed) {
IsCS ? NumOfCSPGOMismatch++ : NumOfPGOMismatch++;
NumOfPGOMismatch++;
SkipWarning =
NoPGOWarnMismatch ||
(NoPGOWarnMismatchComdat &&
(F.hasComdat() ||
F.getLinkage() == GlobalValue::AvailableExternallyLinkage));
LLVM_DEBUG(dbgs() << "hash mismatch (skip=" << SkipWarning << ")");
}
LLVM_DEBUG(dbgs() << " IsCS=" << IsCS << "\n");
if (SkipWarning)
return;
std::string Msg = IPE.message() + std::string(" ") + F.getName().str() +
std::string(" Hash = ") +
std::to_string(FuncInfo.FunctionHash);
std::string Msg = IPE.message() + std::string(" ") + F.getName().str();
Ctx.diagnose(
DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
});
@ -1132,7 +1046,7 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros)
ProfileRecord = std::move(Result.get());
std::vector<uint64_t> &CountFromProfile = ProfileRecord.Counts;
IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;
NumOfPGOFunc++;
LLVM_DEBUG(dbgs() << CountFromProfile.size() << " counts\n");
uint64_t ValueSum = 0;
for (unsigned I = 0, S = CountFromProfile.size(); I < S; I++) {
@ -1147,11 +1061,7 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros)
getBBInfo(nullptr).UnknownCountInEdge = 2;
setInstrumentedCounts(CountFromProfile);
#if 0
ProgramMaxCount = PGOReader->getMaximumFunctionCount(IsCS);
#else
ProgramMaxCount = PGOReader->getMaximumFunctionCount();
#endif
return true;
}
@ -1256,8 +1166,7 @@ void PGOUseFunc::populateCounters() {
// Assign the scaled count values to the BB with multiple out edges.
void PGOUseFunc::setBranchWeights() {
// Generate MD_prof metadata for every branch instruction.
LLVM_DEBUG(dbgs() << "\nSetting branch weights for func " << F.getName()
<< " IsCS=" << IsCS << "\n");
LLVM_DEBUG(dbgs() << "\nSetting branch weights.\n");
for (auto &BB : F) {
Instruction *TI = BB.getTerminator();
if (TI->getNumSuccessors() < 2)
@ -1265,7 +1174,6 @@ void PGOUseFunc::setBranchWeights() {
if (!(isa<BranchInst>(TI) || isa<SwitchInst>(TI) ||
isa<IndirectBrInst>(TI)))
continue;
if (getBBInfo(&BB).CountValue == 0)
continue;
@ -1443,6 +1351,24 @@ void PGOUseFunc::annotateValueSites(uint32_t Kind) {
}
}
// Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
// aware this is an ir_level profile so it can set the version flag.
static void createIRLevelProfileFlagVariable(Module &M) {
Type *IntTy64 = Type::getInt64Ty(M.getContext());
uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
auto IRLevelVersionVariable = new GlobalVariable(
M, IntTy64, true, GlobalVariable::ExternalLinkage,
Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)),
INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility);
Triple TT(M.getTargetTriple());
if (!TT.supportsCOMDAT())
IRLevelVersionVariable->setLinkage(GlobalValue::WeakAnyLinkage);
else
IRLevelVersionVariable->setComdat(M.getOrInsertComdat(
StringRef(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR))));
}
// Collect the set of members for each Comdat in module M and store
// in ComdatMembers.
static void collectComdatMembers(
@ -1463,11 +1389,8 @@ static void collectComdatMembers(
static bool InstrumentAllFunctions(
Module &M, function_ref<BranchProbabilityInfo *(Function &)> LookupBPI,
function_ref<BlockFrequencyInfo *(Function &)> LookupBFI, bool IsCS) {
// For the context-sensitve instrumentation, we should have a separated pass
// (before LTO/ThinLTO linking) to create these variables.
if (!IsCS)
createIRLevelProfileFlagVar(M, /* IsCS */ false);
function_ref<BlockFrequencyInfo *(Function &)> LookupBFI) {
createIRLevelProfileFlagVariable(M);
std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
collectComdatMembers(M, ComdatMembers);
@ -1476,7 +1399,7 @@ static bool InstrumentAllFunctions(
continue;
auto *BPI = LookupBPI(F);
auto *BFI = LookupBFI(F);
instrumentOneFunc(F, &M, BPI, BFI, ComdatMembers, IsCS);
instrumentOneFunc(F, &M, BPI, BFI, ComdatMembers);
}
return true;
}
@ -1491,7 +1414,7 @@ bool PGOInstrumentationGenLegacyPass::runOnModule(Module &M) {
auto LookupBFI = [this](Function &F) {
return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI();
};
return InstrumentAllFunctions(M, LookupBPI, LookupBFI, IsCS);
return InstrumentAllFunctions(M, LookupBPI, LookupBFI);
}
PreservedAnalyses PGOInstrumentationGen::run(Module &M,
@ -1505,7 +1428,7 @@ PreservedAnalyses PGOInstrumentationGen::run(Module &M,
return &FAM.getResult<BlockFrequencyAnalysis>(F);
};
if (!InstrumentAllFunctions(M, LookupBPI, LookupBFI, IsCS))
if (!InstrumentAllFunctions(M, LookupBPI, LookupBFI))
return PreservedAnalyses::all();
return PreservedAnalyses::none();
@ -1514,7 +1437,7 @@ PreservedAnalyses PGOInstrumentationGen::run(Module &M,
static bool annotateAllFunctions(
Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName,
function_ref<BranchProbabilityInfo *(Function &)> LookupBPI,
function_ref<BlockFrequencyInfo *(Function &)> LookupBFI, bool IsCS) {
function_ref<BlockFrequencyInfo *(Function &)> LookupBFI) {
LLVM_DEBUG(dbgs() << "Read in profile counters: ");
auto &Ctx = M.getContext();
// Read the counter array from file.
@ -1535,11 +1458,6 @@ static bool annotateAllFunctions(
StringRef("Cannot get PGOReader")));
return false;
}
#if 0
if (!PGOReader->hasCSIRLevelProfile() && IsCS)
return false;
#endif
// TODO: might need to change the warning once the clang option is finalized.
if (!PGOReader->isIRLevelProfile()) {
Ctx.diagnose(DiagnosticInfoPGOProfile(
@ -1559,7 +1477,7 @@ static bool annotateAllFunctions(
// Split indirectbr critical edges here before computing the MST rather than
// later in getInstrBB() to avoid invalidating it.
SplitIndirectBrCriticalEdges(F, BPI, BFI);
PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI, IsCS);
PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI);
bool AllZeros = false;
if (!Func.readCounters(PGOReader.get(), AllZeros))
continue;
@ -1607,14 +1525,7 @@ static bool annotateAllFunctions(
}
}
}
#if 0
M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()),
IsCS ? ProfileSummary::PSK_CSInstr
: ProfileSummary::PSK_Instr);
#else
M.setProfileSummary(PGOReader->getSummary().getMD(M.getContext()));
#endif
// Set function hotness attribute from the profile.
// We have to apply these attributes at the end because their presence
// can affect the BranchProbabilityInfo of any callers, resulting in an
@ -1633,10 +1544,9 @@ static bool annotateAllFunctions(
}
PGOInstrumentationUse::PGOInstrumentationUse(std::string Filename,
std::string RemappingFilename,
bool IsCS)
std::string RemappingFilename)
: ProfileFileName(std::move(Filename)),
ProfileRemappingFileName(std::move(RemappingFilename)), IsCS(IsCS) {
ProfileRemappingFileName(std::move(RemappingFilename)) {
if (!PGOTestProfileFile.empty())
ProfileFileName = PGOTestProfileFile;
if (!PGOTestProfileRemappingFile.empty())
@ -1656,7 +1566,7 @@ PreservedAnalyses PGOInstrumentationUse::run(Module &M,
};
if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName,
LookupBPI, LookupBFI, IsCS))
LookupBPI, LookupBFI))
return PreservedAnalyses::all();
return PreservedAnalyses::none();
@ -1673,8 +1583,7 @@ bool PGOInstrumentationUseLegacyPass::runOnModule(Module &M) {
return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI();
};
return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI,
IsCS);
return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI);
}
static std::string getSimpleNodeName(const BasicBlock *Node) {

14
llvm/tools/gold/gold-plugin.cpp
View File

@ -209,10 +209,6 @@ namespace options {
static std::string OptRemarksFilename;
static bool OptRemarksWithHotness = false;
// Context sensitive PGO options.
static std::string cs_profile_path;
static bool cs_pgo_gen = false;
static void process_plugin_option(const char *opt_)
{
if (opt_ == nullptr)
@ -272,11 +268,7 @@ namespace options {
} else if (opt == "disable-verify") {
DisableVerify = true;
} else if (opt.startswith("sample-profile=")) {
sample_profile = opt.substr(strlen("sample-profile="));
} else if (opt == "cs-profile-generate") {
cs_pgo_gen = true;
} else if (opt.startswith("cs-profile-path=")) {
cs_profile_path = opt.substr(strlen("cs-profile-path="));
sample_profile= opt.substr(strlen("sample-profile="));
} else if (opt == "new-pass-manager") {
new_pass_manager = true;
} else if (opt == "debug-pass-manager") {
@ -900,10 +892,6 @@ static std::unique_ptr<LTO> createLTO(IndexWriteCallback OnIndexWrite,
if (!options::sample_profile.empty())
Conf.SampleProfile = options::sample_profile;
if (!options::cs_profile_path.empty())
Conf.CSIRProfile = options::cs_profile_path;
Conf.RunCSIRInstr = options::cs_pgo_gen;
Conf.DwoDir = options::dwo_dir;
// Set up optimization remarks handling.

11
llvm/tools/llvm-lto2/llvm-lto2.cpp
View File

@ -104,15 +104,6 @@ static cl::opt<std::string>
SamplePGOFile("lto-sample-profile-file",
cl::desc("Specify a SamplePGO profile file"));
static cl::opt<std::string>
CSPGOFile("lto-cspgo-profile-file",
cl::desc("Specify a context sensitive PGO profile file"));
static cl::opt<bool>
RunCSIRInstr("lto-cspgo-gen",
cl::desc("Run PGO context sensitive IR instrumentation"),
cl::init(false), cl::Hidden);
static cl::opt<bool>
UseNewPM("use-new-pm",
cl::desc("Run LTO passes using the new pass manager"),
@ -223,8 +214,6 @@ static int run(int argc, char **argv) {
Conf.RemarksWithHotness = OptRemarksWithHotness;
Conf.SampleProfile = SamplePGOFile;
Conf.CSIRProfile = CSPGOFile;
Conf.RunCSIRInstr = RunCSIRInstr;
// Run a custom pipeline, if asked for.
Conf.OptPipeline = OptPipeline;