forked from OSchip/llvm-project
[PGO] Context sensitive PGO (part 3)
Part 3 of CSPGO changes (mostly related to PassMananger). Differential Revision: https://reviews.llvm.org/D54175 llvm-svn: 355330
This commit is contained in:
parent
bb4d4e2d76
commit
db29a3a438
|
@ -31,25 +31,38 @@ class ModuleSummaryIndex;
|
|||
|
||||
/// A struct capturing PGO tunables.
|
||||
struct PGOOptions {
|
||||
PGOOptions(std::string ProfileGenFile = "", std::string ProfileUseFile = "",
|
||||
std::string SampleProfileFile = "",
|
||||
std::string ProfileRemappingFile = "",
|
||||
bool RunProfileGen = false, bool SamplePGOSupport = false)
|
||||
: ProfileGenFile(ProfileGenFile), ProfileUseFile(ProfileUseFile),
|
||||
SampleProfileFile(SampleProfileFile),
|
||||
ProfileRemappingFile(ProfileRemappingFile),
|
||||
RunProfileGen(RunProfileGen),
|
||||
SamplePGOSupport(SamplePGOSupport || !SampleProfileFile.empty()) {
|
||||
assert((RunProfileGen ||
|
||||
!SampleProfileFile.empty() ||
|
||||
!ProfileUseFile.empty() ||
|
||||
SamplePGOSupport) && "Illegal PGOOptions.");
|
||||
enum PGOAction { NoAction, IRInstr, IRUse, SampleUse };
|
||||
enum CSPGOAction { NoCSAction, CSIRInstr, CSIRUse };
|
||||
PGOOptions(std::string ProfileFile = "", std::string CSProfileGenFile = "",
|
||||
std::string ProfileRemappingFile = "", PGOAction Action = NoAction,
|
||||
CSPGOAction CSAction = NoCSAction, bool SamplePGOSupport = false)
|
||||
: ProfileFile(ProfileFile), CSProfileGenFile(CSProfileGenFile),
|
||||
ProfileRemappingFile(ProfileRemappingFile), Action(Action),
|
||||
CSAction(CSAction),
|
||||
SamplePGOSupport(SamplePGOSupport || Action == SampleUse) {
|
||||
// Note, we do allow ProfileFile.empty() for Action=IRUse LTO can
|
||||
// callback with IRUse action without ProfileFile.
|
||||
|
||||
// If there is a CSAction, PGOAction cannot be IRInstr or SampleUse.
|
||||
assert(this->CSAction == NoCSAction ||
|
||||
(this->Action != IRInstr && this->Action != SampleUse));
|
||||
|
||||
// For CSIRInstr, CSProfileGenFile also needs to be nonempty.
|
||||
assert(this->CSAction != CSIRInstr || !this->CSProfileGenFile.empty());
|
||||
|
||||
// If CSAction is CSIRUse, PGOAction needs to be IRUse as they share
|
||||
// a profile.
|
||||
assert(this->CSAction != CSIRUse || this->Action == IRUse);
|
||||
|
||||
// If neither CSAction nor CSAction, SamplePGOSupport needs to be true.
|
||||
assert(this->Action != NoAction || this->CSAction != NoCSAction ||
|
||||
this->SamplePGOSupport);
|
||||
}
|
||||
std::string ProfileGenFile;
|
||||
std::string ProfileUseFile;
|
||||
std::string SampleProfileFile;
|
||||
std::string ProfileFile;
|
||||
std::string CSProfileGenFile;
|
||||
std::string ProfileRemappingFile;
|
||||
bool RunProfileGen;
|
||||
PGOAction Action;
|
||||
CSPGOAction CSAction;
|
||||
bool SamplePGOSupport;
|
||||
};
|
||||
|
||||
|
@ -607,9 +620,8 @@ private:
|
|||
bool VerifyEachPass, bool DebugLogging);
|
||||
|
||||
void addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
|
||||
OptimizationLevel Level, bool RunProfileGen,
|
||||
std::string ProfileGenFile,
|
||||
std::string ProfileUseFile,
|
||||
OptimizationLevel Level, bool RunProfileGen, bool IsCS,
|
||||
std::string ProfileFile,
|
||||
std::string ProfileRemappingFile);
|
||||
|
||||
void invokePeepholeEPCallbacks(FunctionPassManager &, OptimizationLevel);
|
||||
|
|
|
@ -159,6 +159,10 @@ public:
|
|||
|
||||
/// Enable profile instrumentation pass.
|
||||
bool EnablePGOInstrGen;
|
||||
/// Enable profile context sensitive instrumentation pass.
|
||||
bool EnablePGOCSInstrGen;
|
||||
/// Enable profile context sensitive profile use pass.
|
||||
bool EnablePGOCSInstrUse;
|
||||
/// Profile data file name that the instrumentation will be written to.
|
||||
std::string PGOInstrGen;
|
||||
/// Path of the profile data file.
|
||||
|
@ -185,7 +189,7 @@ private:
|
|||
void addInitialAliasAnalysisPasses(legacy::PassManagerBase &PM) const;
|
||||
void addLTOOptimizationPasses(legacy::PassManagerBase &PM);
|
||||
void addLateLTOOptimizationPasses(legacy::PassManagerBase &PM);
|
||||
void addPGOInstrPasses(legacy::PassManagerBase &MPM);
|
||||
void addPGOInstrPasses(legacy::PassManagerBase &MPM, bool IsCS);
|
||||
void addFunctionSimplificationPasses(legacy::PassManagerBase &MPM);
|
||||
void addInstructionCombiningPass(legacy::PassManagerBase &MPM) const;
|
||||
|
||||
|
|
|
@ -154,8 +154,15 @@ static void runNewPMPasses(Config &Conf, Module &Mod, TargetMachine *TM,
|
|||
const ModuleSummaryIndex *ImportSummary) {
|
||||
Optional<PGOOptions> PGOOpt;
|
||||
if (!Conf.SampleProfile.empty())
|
||||
PGOOpt = PGOOptions("", "", Conf.SampleProfile, Conf.ProfileRemapping,
|
||||
false, true);
|
||||
PGOOpt = PGOOptions(Conf.SampleProfile, "", Conf.ProfileRemapping,
|
||||
PGOOptions::SampleUse, PGOOptions::NoCSAction, true);
|
||||
else if (Conf.RunCSIRInstr) {
|
||||
PGOOpt = PGOOptions("", Conf.CSIRProfile, Conf.ProfileRemapping,
|
||||
PGOOptions::IRUse, PGOOptions::CSIRInstr);
|
||||
} else if (!Conf.CSIRProfile.empty()) {
|
||||
PGOOpt = PGOOptions(Conf.CSIRProfile, "", Conf.ProfileRemapping,
|
||||
PGOOptions::IRUse, PGOOptions::CSIRUse);
|
||||
}
|
||||
|
||||
PassBuilder PB(TM, PGOOpt);
|
||||
AAManager AA;
|
||||
|
@ -273,6 +280,11 @@ static void runOldPMPasses(Config &Conf, Module &Mod, TargetMachine *TM,
|
|||
PMB.SLPVectorize = true;
|
||||
PMB.OptLevel = Conf.OptLevel;
|
||||
PMB.PGOSampleUse = Conf.SampleProfile;
|
||||
PMB.EnablePGOCSInstrGen = Conf.RunCSIRInstr;
|
||||
if (!Conf.RunCSIRInstr && !Conf.CSIRProfile.empty()) {
|
||||
PMB.EnablePGOCSInstrUse = true;
|
||||
PMB.PGOInstrUse = Conf.CSIRProfile;
|
||||
}
|
||||
if (IsThinLTO)
|
||||
PMB.populateThinLTOPassManager(passes);
|
||||
else
|
||||
|
|
|
@ -406,7 +406,7 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
|
|||
|
||||
// For PGO use pipeline, try to optimize memory intrinsics such as memcpy
|
||||
// using the size value profile. Don't perform this when optimizing for size.
|
||||
if (PGOOpt && !PGOOpt->ProfileUseFile.empty() &&
|
||||
if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&
|
||||
!isOptimizingForSize(Level))
|
||||
FPM.addPass(PGOMemOPSizeOpt());
|
||||
|
||||
|
@ -449,8 +449,8 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
|
|||
// Do not enable unrolling in PreLinkThinLTO phase during sample PGO
|
||||
// because it changes IR to makes profile annotation in back compile
|
||||
// inaccurate.
|
||||
if (Phase != ThinLTOPhase::PreLink ||
|
||||
!PGOOpt || PGOOpt->SampleProfileFile.empty())
|
||||
if (Phase != ThinLTOPhase::PreLink || !PGOOpt ||
|
||||
PGOOpt->Action != PGOOptions::SampleUse)
|
||||
LPM2.addPass(LoopFullUnrollPass(Level));
|
||||
|
||||
for (auto &C : LoopOptimizerEndEPCallbacks)
|
||||
|
@ -510,7 +510,8 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
|
|||
invokePeepholeEPCallbacks(FPM, Level);
|
||||
|
||||
if (EnableCHR && Level == O3 && PGOOpt &&
|
||||
(!PGOOpt->ProfileUseFile.empty() || !PGOOpt->SampleProfileFile.empty()))
|
||||
(PGOOpt->Action == PGOOptions::IRUse ||
|
||||
PGOOpt->Action == PGOOptions::SampleUse))
|
||||
FPM.addPass(ControlHeightReductionPass());
|
||||
|
||||
return FPM;
|
||||
|
@ -518,15 +519,15 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
|
|||
|
||||
void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
|
||||
PassBuilder::OptimizationLevel Level,
|
||||
bool RunProfileGen,
|
||||
std::string ProfileGenFile,
|
||||
std::string ProfileUseFile,
|
||||
bool RunProfileGen, bool IsCS,
|
||||
std::string ProfileFile,
|
||||
std::string ProfileRemappingFile) {
|
||||
// Generally running simplification passes and the inliner with an high
|
||||
// threshold results in smaller executables, but there may be cases where
|
||||
// the size grows, so let's be conservative here and skip this simplification
|
||||
// at -Os/Oz.
|
||||
if (!isOptimizingForSize(Level)) {
|
||||
// at -Os/Oz. We will not do this inline for context sensistive PGO (when
|
||||
// IsCS is true).
|
||||
if (!isOptimizingForSize(Level) && !IsCS) {
|
||||
InlineParams IP;
|
||||
|
||||
// In the old pass manager, this is a cl::opt. Should still this be one?
|
||||
|
@ -559,7 +560,7 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
|
|||
MPM.addPass(GlobalDCEPass());
|
||||
|
||||
if (RunProfileGen) {
|
||||
MPM.addPass(PGOInstrumentationGen());
|
||||
MPM.addPass(PGOInstrumentationGen(IsCS));
|
||||
|
||||
FunctionPassManager FPM;
|
||||
FPM.addPass(
|
||||
|
@ -568,15 +569,13 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
|
|||
|
||||
// Add the profile lowering pass.
|
||||
InstrProfOptions Options;
|
||||
if (!ProfileGenFile.empty())
|
||||
Options.InstrProfileOutput = ProfileGenFile;
|
||||
if (!ProfileFile.empty())
|
||||
Options.InstrProfileOutput = ProfileFile;
|
||||
Options.DoCounterPromotion = true;
|
||||
Options.UseBFIInPromotion = false;
|
||||
MPM.addPass(InstrProfiling(Options, false));
|
||||
}
|
||||
|
||||
if (!ProfileUseFile.empty())
|
||||
MPM.addPass(PGOInstrumentationUse(ProfileUseFile, ProfileRemappingFile));
|
||||
Options.UseBFIInPromotion = IsCS;
|
||||
MPM.addPass(InstrProfiling(Options, IsCS));
|
||||
} else if (!ProfileFile.empty())
|
||||
MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
|
||||
}
|
||||
|
||||
static InlineParams
|
||||
|
@ -593,7 +592,7 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
|
|||
bool DebugLogging) {
|
||||
ModulePassManager MPM(DebugLogging);
|
||||
|
||||
bool HasSampleProfile = PGOOpt && !PGOOpt->SampleProfileFile.empty();
|
||||
bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
|
||||
|
||||
// In ThinLTO mode, when flattened profile is used, all the available
|
||||
// profile information will be annotated in PreLink phase so there is
|
||||
|
@ -646,7 +645,7 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
|
|||
if (LoadSampleProfile) {
|
||||
// Annotate sample profile right after early FPM to ensure freshness of
|
||||
// the debug info.
|
||||
MPM.addPass(SampleProfileLoaderPass(PGOOpt->SampleProfileFile,
|
||||
MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
|
||||
PGOOpt->ProfileRemappingFile,
|
||||
Phase == ThinLTOPhase::PreLink));
|
||||
// Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard
|
||||
|
@ -695,12 +694,17 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
|
|||
|
||||
// Add all the requested passes for instrumentation PGO, if requested.
|
||||
if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
|
||||
(!PGOOpt->ProfileGenFile.empty() || !PGOOpt->ProfileUseFile.empty())) {
|
||||
addPGOInstrPasses(MPM, DebugLogging, Level, PGOOpt->RunProfileGen,
|
||||
PGOOpt->ProfileGenFile, PGOOpt->ProfileUseFile,
|
||||
(PGOOpt->Action == PGOOptions::IRInstr ||
|
||||
PGOOpt->Action == PGOOptions::IRUse)) {
|
||||
addPGOInstrPasses(MPM, DebugLogging, Level,
|
||||
/* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
|
||||
/* IsCS */ false, PGOOpt->ProfileFile,
|
||||
PGOOpt->ProfileRemappingFile);
|
||||
MPM.addPass(PGOIndirectCallPromotion(false, false));
|
||||
}
|
||||
if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
|
||||
PGOOpt->CSAction == PGOOptions::CSIRInstr)
|
||||
MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
|
||||
|
||||
// Synthesize function entry counts for non-PGO compilation.
|
||||
if (EnableSyntheticCounts && !PGOOpt)
|
||||
|
@ -731,8 +735,8 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
|
|||
// For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
|
||||
// because it makes profile annotation in the backend inaccurate.
|
||||
InlineParams IP = getInlineParamsFromOptLevel(Level);
|
||||
if (Phase == ThinLTOPhase::PreLink &&
|
||||
PGOOpt && !PGOOpt->SampleProfileFile.empty())
|
||||
if (Phase == ThinLTOPhase::PreLink && PGOOpt &&
|
||||
PGOOpt->Action == PGOOptions::SampleUse)
|
||||
IP.HotCallSiteThreshold = 0;
|
||||
MainCGPipeline.addPass(InlinerPass(IP));
|
||||
|
||||
|
@ -795,6 +799,21 @@ ModulePassManager PassBuilder::buildModuleOptimizationPipeline(
|
|||
// FIXME: Is this really an optimization rather than a canonicalization?
|
||||
MPM.addPass(ReversePostOrderFunctionAttrsPass());
|
||||
|
||||
// Do a post inline PGO instrumentation and use pass. This is a context
|
||||
// sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
|
||||
// cross-module inline has not been done yet. The context sensitive
|
||||
// instrumentation is after all the inlines are done.
|
||||
if (!LTOPreLink && PGOOpt) {
|
||||
if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
|
||||
addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
|
||||
/* IsCS */ true, PGOOpt->CSProfileGenFile,
|
||||
PGOOpt->ProfileRemappingFile);
|
||||
else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
|
||||
addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
|
||||
/* IsCS */ true, PGOOpt->ProfileFile,
|
||||
PGOOpt->ProfileRemappingFile);
|
||||
}
|
||||
|
||||
// Re-require GloblasAA here prior to function passes. This is particularly
|
||||
// useful as the above will have inlined, DCE'ed, and function-attr
|
||||
// propagated everything. We should at this point have a reasonably minimal
|
||||
|
@ -1031,7 +1050,7 @@ PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level,
|
|||
assert(Level != O0 && "Must request optimizations for the default pipeline!");
|
||||
// FIXME: We should use a customized pre-link pipeline!
|
||||
return buildPerModuleDefaultPipeline(Level, DebugLogging,
|
||||
/*LTOPreLink=*/true);
|
||||
/* LTOPreLink */true);
|
||||
}
|
||||
|
||||
ModulePassManager
|
||||
|
@ -1040,9 +1059,9 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging,
|
|||
assert(Level != O0 && "Must request optimizations for the default pipeline!");
|
||||
ModulePassManager MPM(DebugLogging);
|
||||
|
||||
if (PGOOpt && !PGOOpt->SampleProfileFile.empty()) {
|
||||
if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
|
||||
// Load sample profile before running the LTO optimization pipeline.
|
||||
MPM.addPass(SampleProfileLoaderPass(PGOOpt->SampleProfileFile,
|
||||
MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
|
||||
PGOOpt->ProfileRemappingFile,
|
||||
false /* ThinLTOPhase::PreLink */));
|
||||
}
|
||||
|
@ -1068,7 +1087,7 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging,
|
|||
// This two-step promotion is to save the compile time. For LTO, it should
|
||||
// produce the same result as if we only do promotion here.
|
||||
MPM.addPass(PGOIndirectCallPromotion(
|
||||
true /* InLTO */, PGOOpt && !PGOOpt->SampleProfileFile.empty()));
|
||||
true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
|
||||
// Propagate constants at call sites into the functions they call. This
|
||||
// opens opportunities for globalopt (and inlining) by substituting function
|
||||
// pointers passed as arguments to direct uses of functions.
|
||||
|
@ -1150,6 +1169,19 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging,
|
|||
|
||||
FPM.addPass(JumpThreadingPass());
|
||||
|
||||
// Do a post inline PGO instrumentation and use pass. This is a context
|
||||
// sensitive PGO pass.
|
||||
if (PGOOpt) {
|
||||
if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
|
||||
addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
|
||||
/* IsCS */ true, PGOOpt->CSProfileGenFile,
|
||||
PGOOpt->ProfileRemappingFile);
|
||||
else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
|
||||
addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
|
||||
/* IsCS */ true, PGOOpt->ProfileFile,
|
||||
PGOOpt->ProfileRemappingFile);
|
||||
}
|
||||
|
||||
// Break up allocas
|
||||
FPM.addPass(SROA());
|
||||
|
||||
|
|
|
@ -174,6 +174,8 @@ PassManagerBuilder::PassManagerBuilder() {
|
|||
MergeFunctions = false;
|
||||
PrepareForLTO = false;
|
||||
EnablePGOInstrGen = false;
|
||||
EnablePGOCSInstrGen = false;
|
||||
EnablePGOCSInstrUse = false;
|
||||
PGOInstrGen = "";
|
||||
PGOInstrUse = "";
|
||||
PGOSampleUse = "";
|
||||
|
@ -271,13 +273,19 @@ void PassManagerBuilder::populateFunctionPassManager(
|
|||
}
|
||||
|
||||
// Do PGO instrumentation generation or use pass as the option specified.
|
||||
void PassManagerBuilder::addPGOInstrPasses(legacy::PassManagerBase &MPM) {
|
||||
if (!EnablePGOInstrGen && PGOInstrUse.empty() && PGOSampleUse.empty())
|
||||
void PassManagerBuilder::addPGOInstrPasses(legacy::PassManagerBase &MPM,
|
||||
bool IsCS = false) {
|
||||
if (IsCS) {
|
||||
if (!EnablePGOCSInstrGen && !EnablePGOCSInstrUse)
|
||||
return;
|
||||
} else if (!EnablePGOInstrGen && PGOInstrUse.empty() && PGOSampleUse.empty())
|
||||
return;
|
||||
|
||||
// Perform the preinline and cleanup passes for O1 and above.
|
||||
// And avoid doing them if optimizing for size.
|
||||
// We will not do this inline for context sensitive PGO (when IsCS is true).
|
||||
if (OptLevel > 0 && SizeLevel == 0 && !DisablePreInliner &&
|
||||
PGOSampleUse.empty()) {
|
||||
PGOSampleUse.empty() && !IsCS) {
|
||||
// Create preinline pass. We construct an InlineParams object and specify
|
||||
// the threshold here to avoid the command line options of the regular
|
||||
// inliner to influence pre-inlining. The only fields of InlineParams we
|
||||
|
@ -295,22 +303,23 @@ void PassManagerBuilder::addPGOInstrPasses(legacy::PassManagerBase &MPM) {
|
|||
MPM.add(createInstructionCombiningPass()); // Combine silly seq's
|
||||
addExtensionsToPM(EP_Peephole, MPM);
|
||||
}
|
||||
if (EnablePGOInstrGen) {
|
||||
MPM.add(createPGOInstrumentationGenLegacyPass());
|
||||
if ((EnablePGOInstrGen && !IsCS) || (EnablePGOCSInstrGen && IsCS)) {
|
||||
MPM.add(createPGOInstrumentationGenLegacyPass(IsCS));
|
||||
// Add the profile lowering pass.
|
||||
InstrProfOptions Options;
|
||||
if (!PGOInstrGen.empty())
|
||||
Options.InstrProfileOutput = PGOInstrGen;
|
||||
Options.DoCounterPromotion = true;
|
||||
Options.UseBFIInPromotion = IsCS;
|
||||
MPM.add(createLoopRotatePass());
|
||||
MPM.add(createInstrProfilingLegacyPass(Options));
|
||||
MPM.add(createInstrProfilingLegacyPass(Options, IsCS));
|
||||
}
|
||||
if (!PGOInstrUse.empty())
|
||||
MPM.add(createPGOInstrumentationUseLegacyPass(PGOInstrUse));
|
||||
MPM.add(createPGOInstrumentationUseLegacyPass(PGOInstrUse, IsCS));
|
||||
// Indirect call promotion that promotes intra-module targets only.
|
||||
// For ThinLTO this is done earlier due to interactions with globalopt
|
||||
// for imported functions. We don't run this at -O0.
|
||||
if (OptLevel > 0)
|
||||
if (OptLevel > 0 && !IsCS)
|
||||
MPM.add(
|
||||
createPGOIndirectCallPromotionLegacyPass(false, !PGOSampleUse.empty()));
|
||||
}
|
||||
|
@ -418,7 +427,7 @@ void PassManagerBuilder::addFunctionSimplificationPasses(
|
|||
addExtensionsToPM(EP_Peephole, MPM);
|
||||
|
||||
if (EnableCHR && OptLevel >= 3 &&
|
||||
(!PGOInstrUse.empty() || !PGOSampleUse.empty()))
|
||||
(!PGOInstrUse.empty() || !PGOSampleUse.empty() || EnablePGOCSInstrGen))
|
||||
MPM.add(createControlHeightReductionLegacyPass());
|
||||
}
|
||||
|
||||
|
@ -533,6 +542,11 @@ void PassManagerBuilder::populateModulePassManager(
|
|||
if (DefaultOrPreLinkPipeline && !PrepareForThinLTOUsingPGOSampleProfile)
|
||||
addPGOInstrPasses(MPM);
|
||||
|
||||
// Create profile COMDAT variables. Lld linker wants to see all variables
|
||||
// before the LTO/ThinLTO link since it needs to resolve symbols/comdats.
|
||||
if (!PerformThinLTO && EnablePGOCSInstrGen)
|
||||
MPM.add(createPGOInstrumentationGenCreateVarLegacyPass(PGOInstrGen));
|
||||
|
||||
// We add a module alias analysis pass here. In part due to bugs in the
|
||||
// analysis infrastructure this "works" in that the analysis stays alive
|
||||
// for the entire SCC pass run below.
|
||||
|
@ -574,6 +588,14 @@ void PassManagerBuilder::populateModulePassManager(
|
|||
// and saves running remaining passes on the eliminated functions.
|
||||
MPM.add(createEliminateAvailableExternallyPass());
|
||||
|
||||
// CSFDO instrumentation and use pass. Don't invoke this for Prepare pass
|
||||
// for LTO and ThinLTO -- The actual pass will be called after all inlines
|
||||
// are performed.
|
||||
// Need to do this after COMDAT variables have been eliminated,
|
||||
// (i.e. after EliminateAvailableExternallyPass).
|
||||
if (!(PrepareForLTO || PrepareForThinLTO))
|
||||
addPGOInstrPasses(MPM, /* IsCS */ true);
|
||||
|
||||
if (EnableOrderFileInstrumentation)
|
||||
MPM.add(createInstrOrderFilePass());
|
||||
|
||||
|
@ -854,6 +876,9 @@ void PassManagerBuilder::addLTOOptimizationPasses(legacy::PassManagerBase &PM) {
|
|||
|
||||
PM.add(createPruneEHPass()); // Remove dead EH info.
|
||||
|
||||
// CSFDO instrumentation and use pass.
|
||||
addPGOInstrPasses(PM, /* IsCS */ true);
|
||||
|
||||
// Optimize globals again if we ran the inliner.
|
||||
if (RunInliner)
|
||||
PM.add(createGlobalOptimizerPass());
|
||||
|
|
|
@ -102,6 +102,9 @@ static cl::opt<std::string> OptimizerLastEPPipeline(
|
|||
|
||||
extern cl::opt<PGOKind> PGOKindFlag;
|
||||
extern cl::opt<std::string> ProfileFile;
|
||||
extern cl::opt<CSPGOKind> CSPGOKindFlag;
|
||||
extern cl::opt<std::string> CSProfileGenFile;
|
||||
|
||||
static cl::opt<std::string>
|
||||
ProfileRemappingFile("profile-remapping-file",
|
||||
cl::desc("Path to the profile remapping file."),
|
||||
|
@ -219,20 +222,41 @@ bool llvm::runPassPipeline(StringRef Arg0, Module &M, TargetMachine *TM,
|
|||
Optional<PGOOptions> P;
|
||||
switch (PGOKindFlag) {
|
||||
case InstrGen:
|
||||
P = PGOOptions(ProfileFile, "", "", "", true);
|
||||
P = PGOOptions(ProfileFile, "", "", PGOOptions::IRInstr);
|
||||
break;
|
||||
case InstrUse:
|
||||
P = PGOOptions("", ProfileFile, "", ProfileRemappingFile, false);
|
||||
P = PGOOptions(ProfileFile, "", ProfileRemappingFile, PGOOptions::IRUse);
|
||||
break;
|
||||
case SampleUse:
|
||||
P = PGOOptions("", "", ProfileFile, ProfileRemappingFile, false);
|
||||
P = PGOOptions(ProfileFile, "", ProfileRemappingFile,
|
||||
PGOOptions::SampleUse);
|
||||
break;
|
||||
case NoPGO:
|
||||
if (DebugInfoForProfiling)
|
||||
P = PGOOptions("", "", "", "", false, true);
|
||||
P = PGOOptions("", "", "", PGOOptions::NoAction, PGOOptions::NoCSAction,
|
||||
true);
|
||||
else
|
||||
P = None;
|
||||
}
|
||||
}
|
||||
if (CSPGOKindFlag != NoCSPGO) {
|
||||
if (P && (P->Action == PGOOptions::IRInstr ||
|
||||
P->Action == PGOOptions::SampleUse))
|
||||
errs() << "CSPGOKind cannot be used with IRInstr or SampleUse";
|
||||
if (CSPGOKindFlag == CSInstrGen) {
|
||||
if (CSProfileGenFile.empty())
|
||||
errs() << "CSInstrGen needs to specify CSProfileGenFile";
|
||||
if (P) {
|
||||
P->CSAction = PGOOptions::CSIRInstr;
|
||||
P->CSProfileGenFile = CSProfileGenFile;
|
||||
} else
|
||||
P = PGOOptions("", CSProfileGenFile, ProfileRemappingFile,
|
||||
PGOOptions::NoAction, PGOOptions::CSIRInstr);
|
||||
} else /* CSPGOKindFlag == CSInstrUse */ {
|
||||
if (!P)
|
||||
errs() << "CSInstrUse needs to be together with InstrUse";
|
||||
P->CSAction = PGOOptions::CSIRUse;
|
||||
}
|
||||
}
|
||||
PassInstrumentationCallbacks PIC;
|
||||
StandardInstrumentations SI;
|
||||
SI.registerCallbacks(PIC);
|
||||
|
|
|
@ -45,6 +45,7 @@ enum PGOKind {
|
|||
InstrUse,
|
||||
SampleUse
|
||||
};
|
||||
enum CSPGOKind { NoCSPGO, CSInstrGen, CSInstrUse };
|
||||
}
|
||||
|
||||
/// Driver function to run the new pass manager over a module.
|
||||
|
|
|
@ -287,6 +287,22 @@ cl::opt<PGOKind>
|
|||
cl::opt<std::string> ProfileFile("profile-file",
|
||||
cl::desc("Path to the profile."), cl::Hidden);
|
||||
|
||||
cl::opt<CSPGOKind> CSPGOKindFlag(
|
||||
"cspgo-kind", cl::init(NoCSPGO), cl::Hidden,
|
||||
cl::desc("The kind of context sensitive profile guided optimization"),
|
||||
cl::values(
|
||||
clEnumValN(NoCSPGO, "nocspgo", "Do not use CSPGO."),
|
||||
clEnumValN(
|
||||
CSInstrGen, "cspgo-instr-gen-pipeline",
|
||||
"Instrument (context sensitive) the IR to generate profile."),
|
||||
clEnumValN(
|
||||
CSInstrUse, "cspgo-instr-use-pipeline",
|
||||
"Use instrumented (context sensitive) profile to guide PGO.")));
|
||||
cl::opt<std::string> CSProfileGenFile(
|
||||
"cs-profilegen-file",
|
||||
cl::desc("Path to the instrumented context sensitive profile."),
|
||||
cl::Hidden);
|
||||
|
||||
class OptCustomPassManager : public legacy::PassManager {
|
||||
DebugifyStatsMap DIStatsMap;
|
||||
|
||||
|
@ -396,6 +412,17 @@ static void AddOptimizationPasses(legacy::PassManagerBase &MPM,
|
|||
break;
|
||||
}
|
||||
|
||||
switch (CSPGOKindFlag) {
|
||||
case CSInstrGen:
|
||||
Builder.EnablePGOCSInstrGen = true;
|
||||
break;
|
||||
case InstrUse:
|
||||
Builder.EnablePGOCSInstrUse = true;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
Builder.populateFunctionPassManager(FPM);
|
||||
Builder.populateModulePassManager(MPM);
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue