2016-04-11 21:58:45 +08:00
|
|
|
//===- ModuleSummaryAnalysis.cpp - Module summary index builder -----------===//
|
|
|
|
//
|
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
//
|
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
// License. See LICENSE.TXT for details.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
|
|
|
// This pass builds a ModuleSummaryIndex object for the module, to be written
|
|
|
|
// to bitcode or LLVM assembly.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
#include "llvm/Analysis/ModuleSummaryAnalysis.h"
|
2017-08-17 06:07:40 +08:00
|
|
|
#include "llvm/ADT/ArrayRef.h"
|
|
|
|
#include "llvm/ADT/DenseSet.h"
|
2016-12-21 05:12:28 +08:00
|
|
|
#include "llvm/ADT/MapVector.h"
|
2017-08-17 06:07:40 +08:00
|
|
|
#include "llvm/ADT/STLExtras.h"
|
2016-12-21 05:12:28 +08:00
|
|
|
#include "llvm/ADT/SetVector.h"
|
2017-08-17 06:07:40 +08:00
|
|
|
#include "llvm/ADT/SmallPtrSet.h"
|
|
|
|
#include "llvm/ADT/SmallVector.h"
|
|
|
|
#include "llvm/ADT/StringRef.h"
|
2016-04-11 21:58:45 +08:00
|
|
|
#include "llvm/Analysis/BlockFrequencyInfo.h"
|
|
|
|
#include "llvm/Analysis/BranchProbabilityInfo.h"
|
2016-07-17 22:47:01 +08:00
|
|
|
#include "llvm/Analysis/IndirectCallPromotionAnalysis.h"
|
2016-04-11 21:58:45 +08:00
|
|
|
#include "llvm/Analysis/LoopInfo.h"
|
[thinlto] Basic thinlto fdo heuristic
Summary:
This patch improves thinlto importer
by importing 3x larger functions that are called from hot block.
I compared performance with the trunk on spec, and there
were about 2% on povray and 3.33% on milc. These results seems
to be consistant and match the results Teresa got with her simple
heuristic. Some benchmarks got slower but I think they are just
noisy (mcf, xalancbmki, omnetpp)- running the benchmarks again with
more iterations to confirm. Geomean of all benchmarks including the noisy ones
were about +0.02%.
I see much better improvement on google branch with Easwaran patch
for pgo callsite inlining (the inliner actually inline those big functions)
Over all I see +0.5% improvement, and I get +8.65% on povray.
So I guess we will see much bigger change when Easwaran patch will land
(it depends on new pass manager), but it is still worth putting this to trunk
before it.
Implementation details changes:
- Removed CallsiteCount.
- ProfileCount got replaced by Hotness
- hot-import-multiplier is set to 3.0 for now,
didn't have time to tune it up, but I see that we get most of the interesting
functions with 3, so there is no much performance difference with higher, and
binary size doesn't grow as much as with 10.0.
Reviewers: eraman, mehdi_amini, tejohnson
Subscribers: mehdi_amini, llvm-commits
Differential Revision: https://reviews.llvm.org/D24638
llvm-svn: 282437
2016-09-27 04:37:32 +08:00
|
|
|
#include "llvm/Analysis/ProfileSummaryInfo.h"
|
2016-12-22 07:03:45 +08:00
|
|
|
#include "llvm/Analysis/TypeMetadataUtils.h"
|
2017-08-17 06:07:40 +08:00
|
|
|
#include "llvm/IR/Attributes.h"
|
|
|
|
#include "llvm/IR/BasicBlock.h"
|
2016-04-11 21:58:45 +08:00
|
|
|
#include "llvm/IR/CallSite.h"
|
2017-08-17 06:07:40 +08:00
|
|
|
#include "llvm/IR/Constant.h"
|
|
|
|
#include "llvm/IR/Constants.h"
|
2016-04-11 21:58:45 +08:00
|
|
|
#include "llvm/IR/Dominators.h"
|
2017-08-17 06:07:40 +08:00
|
|
|
#include "llvm/IR/Function.h"
|
|
|
|
#include "llvm/IR/GlobalAlias.h"
|
|
|
|
#include "llvm/IR/GlobalValue.h"
|
|
|
|
#include "llvm/IR/GlobalVariable.h"
|
|
|
|
#include "llvm/IR/Instructions.h"
|
2016-04-11 21:58:45 +08:00
|
|
|
#include "llvm/IR/IntrinsicInst.h"
|
2017-08-17 06:07:40 +08:00
|
|
|
#include "llvm/IR/Intrinsics.h"
|
|
|
|
#include "llvm/IR/Metadata.h"
|
|
|
|
#include "llvm/IR/Module.h"
|
|
|
|
#include "llvm/IR/ModuleSummaryIndex.h"
|
|
|
|
#include "llvm/IR/Use.h"
|
|
|
|
#include "llvm/IR/User.h"
|
2017-03-31 08:08:24 +08:00
|
|
|
#include "llvm/Object/ModuleSymbolTable.h"
|
2017-08-17 06:07:40 +08:00
|
|
|
#include "llvm/Object/SymbolicFile.h"
|
2016-04-11 21:58:45 +08:00
|
|
|
#include "llvm/Pass.h"
|
2017-08-17 06:07:40 +08:00
|
|
|
#include "llvm/Support/Casting.h"
|
2018-03-31 08:18:08 +08:00
|
|
|
#include "llvm/Support/CommandLine.h"
|
2017-08-17 06:07:40 +08:00
|
|
|
#include <algorithm>
|
|
|
|
#include <cassert>
|
|
|
|
#include <cstdint>
|
|
|
|
#include <vector>
|
|
|
|
|
2016-04-11 21:58:45 +08:00
|
|
|
using namespace llvm;
|
|
|
|
|
|
|
|
#define DEBUG_TYPE "module-summary-analysis"
|
|
|
|
|
2018-03-31 08:18:08 +08:00
|
|
|
// Option to force edges cold which will block importing when the
|
|
|
|
// -import-cold-multiplier is set to 0. Useful for debugging.
|
|
|
|
FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold =
|
|
|
|
FunctionSummary::FSHT_None;
|
|
|
|
cl::opt<FunctionSummary::ForceSummaryHotnessType, true> FSEC(
|
|
|
|
"force-summary-edges-cold", cl::Hidden, cl::location(ForceSummaryEdgesCold),
|
|
|
|
cl::desc("Force all edges in the function summary to cold"),
|
|
|
|
cl::values(clEnumValN(FunctionSummary::FSHT_None, "none", "None."),
|
|
|
|
clEnumValN(FunctionSummary::FSHT_AllNonCritical,
|
|
|
|
"all-non-critical", "All non-critical edges."),
|
|
|
|
clEnumValN(FunctionSummary::FSHT_All, "all", "All edges.")));
|
|
|
|
|
2016-04-11 21:58:45 +08:00
|
|
|
// Walk through the operands of a given User via worklist iteration and populate
|
|
|
|
// the set of GlobalValue references encountered. Invoked either on an
|
|
|
|
// Instruction or a GlobalVariable (which walks its initializer).
|
2018-10-12 15:24:02 +08:00
|
|
|
// Return true if any of the operands contains blockaddress. This is important
|
|
|
|
// to know when computing summary for global var, because if global variable
|
|
|
|
// references basic block address we can't import it separately from function
|
|
|
|
// containing that basic block. For simplicity we currently don't import such
|
|
|
|
// global vars at all. When importing function we aren't interested if any
|
|
|
|
// instruction in it takes an address of any basic block, because instruction
|
|
|
|
// can only take an address of basic block located in the same function.
|
|
|
|
static bool findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
|
2017-05-05 02:03:25 +08:00
|
|
|
SetVector<ValueInfo> &RefEdges,
|
2016-04-11 21:58:45 +08:00
|
|
|
SmallPtrSet<const User *, 8> &Visited) {
|
2018-10-12 15:24:02 +08:00
|
|
|
bool HasBlockAddress = false;
|
2016-04-11 21:58:45 +08:00
|
|
|
SmallVector<const User *, 32> Worklist;
|
|
|
|
Worklist.push_back(CurUser);
|
|
|
|
|
|
|
|
while (!Worklist.empty()) {
|
|
|
|
const User *U = Worklist.pop_back_val();
|
|
|
|
|
|
|
|
if (!Visited.insert(U).second)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
ImmutableCallSite CS(U);
|
|
|
|
|
|
|
|
for (const auto &OI : U->operands()) {
|
|
|
|
const User *Operand = dyn_cast<User>(OI);
|
|
|
|
if (!Operand)
|
|
|
|
continue;
|
2018-10-12 15:24:02 +08:00
|
|
|
if (isa<BlockAddress>(Operand)) {
|
|
|
|
HasBlockAddress = true;
|
2016-04-11 21:58:45 +08:00
|
|
|
continue;
|
2018-10-12 15:24:02 +08:00
|
|
|
}
|
2016-12-21 05:12:28 +08:00
|
|
|
if (auto *GV = dyn_cast<GlobalValue>(Operand)) {
|
2016-04-11 21:58:45 +08:00
|
|
|
// We have a reference to a global value. This should be added to
|
|
|
|
// the reference set unless it is a callee. Callees are handled
|
|
|
|
// specially by WriteFunction and are added to a separate list.
|
|
|
|
if (!(CS && CS.isCallee(&OI)))
|
2017-05-05 02:03:25 +08:00
|
|
|
RefEdges.insert(Index.getOrInsertValueInfo(GV));
|
2016-04-11 21:58:45 +08:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
Worklist.push_back(Operand);
|
|
|
|
}
|
|
|
|
}
|
2018-10-12 15:24:02 +08:00
|
|
|
return HasBlockAddress;
|
2016-04-11 21:58:45 +08:00
|
|
|
}
|
|
|
|
|
[thinlto] Basic thinlto fdo heuristic
Summary:
This patch improves thinlto importer
by importing 3x larger functions that are called from hot block.
I compared performance with the trunk on spec, and there
were about 2% on povray and 3.33% on milc. These results seems
to be consistant and match the results Teresa got with her simple
heuristic. Some benchmarks got slower but I think they are just
noisy (mcf, xalancbmki, omnetpp)- running the benchmarks again with
more iterations to confirm. Geomean of all benchmarks including the noisy ones
were about +0.02%.
I see much better improvement on google branch with Easwaran patch
for pgo callsite inlining (the inliner actually inline those big functions)
Over all I see +0.5% improvement, and I get +8.65% on povray.
So I guess we will see much bigger change when Easwaran patch will land
(it depends on new pass manager), but it is still worth putting this to trunk
before it.
Implementation details changes:
- Removed CallsiteCount.
- ProfileCount got replaced by Hotness
- hot-import-multiplier is set to 3.0 for now,
didn't have time to tune it up, but I see that we get most of the interesting
functions with 3, so there is no much performance difference with higher, and
binary size doesn't grow as much as with 10.0.
Reviewers: eraman, mehdi_amini, tejohnson
Subscribers: mehdi_amini, llvm-commits
Differential Revision: https://reviews.llvm.org/D24638
llvm-svn: 282437
2016-09-27 04:37:32 +08:00
|
|
|
static CalleeInfo::HotnessType getHotness(uint64_t ProfileCount,
|
|
|
|
ProfileSummaryInfo *PSI) {
|
|
|
|
if (!PSI)
|
|
|
|
return CalleeInfo::HotnessType::Unknown;
|
|
|
|
if (PSI->isHotCount(ProfileCount))
|
|
|
|
return CalleeInfo::HotnessType::Hot;
|
|
|
|
if (PSI->isColdCount(ProfileCount))
|
|
|
|
return CalleeInfo::HotnessType::Cold;
|
|
|
|
return CalleeInfo::HotnessType::None;
|
|
|
|
}
|
|
|
|
|
2017-01-05 22:32:16 +08:00
|
|
|
static bool isNonRenamableLocal(const GlobalValue &GV) {
|
|
|
|
return GV.hasSection() && GV.hasLocalLinkage();
|
|
|
|
}
|
|
|
|
|
2017-02-11 06:29:38 +08:00
|
|
|
/// Determine whether this call has all constant integer arguments (excluding
|
|
|
|
/// "this") and summarize it to VCalls or ConstVCalls as appropriate.
|
|
|
|
static void addVCallToSet(DevirtCallSite Call, GlobalValue::GUID Guid,
|
|
|
|
SetVector<FunctionSummary::VFuncId> &VCalls,
|
|
|
|
SetVector<FunctionSummary::ConstVCall> &ConstVCalls) {
|
|
|
|
std::vector<uint64_t> Args;
|
|
|
|
// Start from the second argument to skip the "this" pointer.
|
|
|
|
for (auto &Arg : make_range(Call.CS.arg_begin() + 1, Call.CS.arg_end())) {
|
|
|
|
auto *CI = dyn_cast<ConstantInt>(Arg);
|
|
|
|
if (!CI || CI->getBitWidth() > 64) {
|
|
|
|
VCalls.insert({Guid, Call.Offset});
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
Args.push_back(CI->getZExtValue());
|
|
|
|
}
|
|
|
|
ConstVCalls.insert({{Guid, Call.Offset}, std::move(Args)});
|
|
|
|
}
|
|
|
|
|
|
|
|
/// If this intrinsic call requires that we add information to the function
|
|
|
|
/// summary, do so via the non-constant reference arguments.
|
|
|
|
static void addIntrinsicToSummary(
|
|
|
|
const CallInst *CI, SetVector<GlobalValue::GUID> &TypeTests,
|
|
|
|
SetVector<FunctionSummary::VFuncId> &TypeTestAssumeVCalls,
|
|
|
|
SetVector<FunctionSummary::VFuncId> &TypeCheckedLoadVCalls,
|
|
|
|
SetVector<FunctionSummary::ConstVCall> &TypeTestAssumeConstVCalls,
|
2018-09-27 22:55:32 +08:00
|
|
|
SetVector<FunctionSummary::ConstVCall> &TypeCheckedLoadConstVCalls,
|
|
|
|
DominatorTree &DT) {
|
2017-02-11 06:29:38 +08:00
|
|
|
switch (CI->getCalledFunction()->getIntrinsicID()) {
|
|
|
|
case Intrinsic::type_test: {
|
|
|
|
auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(1));
|
|
|
|
auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
|
|
|
|
if (!TypeId)
|
|
|
|
break;
|
|
|
|
GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
|
|
|
|
|
|
|
|
// Produce a summary from type.test intrinsics. We only summarize type.test
|
|
|
|
// intrinsics that are used other than by an llvm.assume intrinsic.
|
|
|
|
// Intrinsics that are assumed are relevant only to the devirtualization
|
|
|
|
// pass, not the type test lowering pass.
|
|
|
|
bool HasNonAssumeUses = llvm::any_of(CI->uses(), [](const Use &CIU) {
|
|
|
|
auto *AssumeCI = dyn_cast<CallInst>(CIU.getUser());
|
|
|
|
if (!AssumeCI)
|
|
|
|
return true;
|
|
|
|
Function *F = AssumeCI->getCalledFunction();
|
|
|
|
return !F || F->getIntrinsicID() != Intrinsic::assume;
|
|
|
|
});
|
|
|
|
if (HasNonAssumeUses)
|
|
|
|
TypeTests.insert(Guid);
|
|
|
|
|
|
|
|
SmallVector<DevirtCallSite, 4> DevirtCalls;
|
|
|
|
SmallVector<CallInst *, 4> Assumes;
|
2018-09-27 22:55:32 +08:00
|
|
|
findDevirtualizableCallsForTypeTest(DevirtCalls, Assumes, CI, DT);
|
2017-02-11 06:29:38 +08:00
|
|
|
for (auto &Call : DevirtCalls)
|
|
|
|
addVCallToSet(Call, Guid, TypeTestAssumeVCalls,
|
|
|
|
TypeTestAssumeConstVCalls);
|
|
|
|
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
case Intrinsic::type_checked_load: {
|
|
|
|
auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(2));
|
|
|
|
auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
|
|
|
|
if (!TypeId)
|
|
|
|
break;
|
|
|
|
GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
|
|
|
|
|
|
|
|
SmallVector<DevirtCallSite, 4> DevirtCalls;
|
|
|
|
SmallVector<Instruction *, 4> LoadedPtrs;
|
|
|
|
SmallVector<Instruction *, 4> Preds;
|
|
|
|
bool HasNonCallUses = false;
|
|
|
|
findDevirtualizableCallsForTypeCheckedLoad(DevirtCalls, LoadedPtrs, Preds,
|
2018-09-27 22:55:32 +08:00
|
|
|
HasNonCallUses, CI, DT);
|
2017-02-11 06:29:38 +08:00
|
|
|
// Any non-call uses of the result of llvm.type.checked.load will
|
|
|
|
// prevent us from optimizing away the llvm.type.test.
|
|
|
|
if (HasNonCallUses)
|
|
|
|
TypeTests.insert(Guid);
|
|
|
|
for (auto &Call : DevirtCalls)
|
|
|
|
addVCallToSet(Call, Guid, TypeCheckedLoadVCalls,
|
|
|
|
TypeCheckedLoadConstVCalls);
|
|
|
|
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-11-16 15:08:00 +08:00
|
|
|
static bool isNonVolatileLoad(const Instruction *I) {
|
|
|
|
if (const auto *LI = dyn_cast<LoadInst>(I))
|
|
|
|
return !LI->isVolatile();
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
|
|
|
|
const Function &F, BlockFrequencyInfo *BFI,
|
|
|
|
ProfileSummaryInfo *PSI, DominatorTree &DT,
|
|
|
|
bool HasLocalsInUsedOrAsm,
|
|
|
|
DenseSet<GlobalValue::GUID> &CantBePromoted,
|
|
|
|
bool IsThinLTO) {
|
2016-10-28 10:39:38 +08:00
|
|
|
// Summary not currently supported for anonymous functions, they should
|
|
|
|
// have been named.
|
|
|
|
assert(F.hasName());
|
2016-04-11 21:58:45 +08:00
|
|
|
|
|
|
|
unsigned NumInsts = 0;
|
|
|
|
// Map from callee ValueId to profile count. Used to accumulate profile
|
|
|
|
// counts for all static calls to a given callee.
|
2016-12-21 05:12:28 +08:00
|
|
|
MapVector<ValueInfo, CalleeInfo> CallGraphEdges;
|
|
|
|
SetVector<ValueInfo> RefEdges;
|
2016-12-22 07:03:45 +08:00
|
|
|
SetVector<GlobalValue::GUID> TypeTests;
|
2017-02-11 06:29:38 +08:00
|
|
|
SetVector<FunctionSummary::VFuncId> TypeTestAssumeVCalls,
|
|
|
|
TypeCheckedLoadVCalls;
|
|
|
|
SetVector<FunctionSummary::ConstVCall> TypeTestAssumeConstVCalls,
|
|
|
|
TypeCheckedLoadConstVCalls;
|
2016-07-17 22:47:01 +08:00
|
|
|
ICallPromotionAnalysis ICallAnalysis;
|
2017-09-07 13:35:35 +08:00
|
|
|
SmallPtrSet<const User *, 8> Visited;
|
|
|
|
|
|
|
|
// Add personality function, prefix data and prologue data to function's ref
|
|
|
|
// list.
|
|
|
|
findRefEdges(Index, &F, RefEdges, Visited);
|
2018-11-16 15:08:00 +08:00
|
|
|
std::vector<const Instruction *> NonVolatileLoads;
|
2016-04-11 21:58:45 +08:00
|
|
|
|
2016-11-15 00:40:19 +08:00
|
|
|
bool HasInlineAsmMaybeReferencingInternal = false;
|
2016-06-27 01:27:42 +08:00
|
|
|
for (const BasicBlock &BB : F)
|
|
|
|
for (const Instruction &I : BB) {
|
2016-08-30 08:46:26 +08:00
|
|
|
if (isa<DbgInfoIntrinsic>(I))
|
|
|
|
continue;
|
|
|
|
++NumInsts;
|
2018-11-16 15:08:00 +08:00
|
|
|
if (isNonVolatileLoad(&I)) {
|
|
|
|
// Postpone processing of non-volatile load instructions
|
|
|
|
// See comments below
|
|
|
|
Visited.insert(&I);
|
|
|
|
NonVolatileLoads.push_back(&I);
|
|
|
|
continue;
|
|
|
|
}
|
2017-05-05 02:03:25 +08:00
|
|
|
findRefEdges(Index, &I, RefEdges, Visited);
|
2016-08-30 08:46:26 +08:00
|
|
|
auto CS = ImmutableCallSite(&I);
|
|
|
|
if (!CS)
|
|
|
|
continue;
|
2016-10-30 13:40:44 +08:00
|
|
|
|
|
|
|
const auto *CI = dyn_cast<CallInst>(&I);
|
|
|
|
// Since we don't know exactly which local values are referenced in inline
|
2016-11-15 00:40:19 +08:00
|
|
|
// assembly, conservatively mark the function as possibly referencing
|
|
|
|
// a local value from inline assembly to ensure we don't export a
|
|
|
|
// reference (which would require renaming and promotion of the
|
|
|
|
// referenced value).
|
2017-09-02 00:24:02 +08:00
|
|
|
if (HasLocalsInUsedOrAsm && CI && CI->isInlineAsm())
|
2016-11-15 00:40:19 +08:00
|
|
|
HasInlineAsmMaybeReferencingInternal = true;
|
2016-10-30 13:40:44 +08:00
|
|
|
|
2016-10-09 00:11:42 +08:00
|
|
|
auto *CalledValue = CS.getCalledValue();
|
2016-08-30 08:46:26 +08:00
|
|
|
auto *CalledFunction = CS.getCalledFunction();
|
2017-11-10 08:47:47 +08:00
|
|
|
if (CalledValue && !CalledFunction) {
|
|
|
|
CalledValue = CalledValue->stripPointerCastsNoFollowAliases();
|
|
|
|
// Stripping pointer casts can reveal a called function.
|
|
|
|
CalledFunction = dyn_cast<Function>(CalledValue);
|
|
|
|
}
|
2016-10-09 00:11:42 +08:00
|
|
|
// Check if this is an alias to a function. If so, get the
|
|
|
|
// called aliasee for the checks below.
|
|
|
|
if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
|
|
|
|
assert(!CalledFunction && "Expected null called function in callsite for alias");
|
|
|
|
CalledFunction = dyn_cast<Function>(GA->getBaseObject());
|
|
|
|
}
|
2016-12-22 07:03:45 +08:00
|
|
|
// Check if this is a direct call to a known function or a known
|
|
|
|
// intrinsic, or an indirect call with profile data.
|
2016-08-30 08:46:26 +08:00
|
|
|
if (CalledFunction) {
|
2017-02-11 06:29:38 +08:00
|
|
|
if (CI && CalledFunction->isIntrinsic()) {
|
|
|
|
addIntrinsicToSummary(
|
|
|
|
CI, TypeTests, TypeTestAssumeVCalls, TypeCheckedLoadVCalls,
|
2018-09-27 22:55:32 +08:00
|
|
|
TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls, DT);
|
2017-02-11 06:29:38 +08:00
|
|
|
continue;
|
2016-12-22 07:03:45 +08:00
|
|
|
}
|
2016-10-28 10:39:38 +08:00
|
|
|
// We should have named any anonymous globals
|
|
|
|
assert(CalledFunction->hasName());
|
2017-05-10 07:21:10 +08:00
|
|
|
auto ScaledCount = PSI->getProfileCount(&I, BFI);
|
2016-12-21 05:12:28 +08:00
|
|
|
auto Hotness = ScaledCount ? getHotness(ScaledCount.getValue(), PSI)
|
|
|
|
: CalleeInfo::HotnessType::Unknown;
|
2018-03-31 08:18:08 +08:00
|
|
|
if (ForceSummaryEdgesCold != FunctionSummary::FSHT_None)
|
|
|
|
Hotness = CalleeInfo::HotnessType::Cold;
|
2016-12-21 05:12:28 +08:00
|
|
|
|
2016-10-09 00:11:42 +08:00
|
|
|
// Use the original CalledValue, in case it was an alias. We want
|
|
|
|
// to record the call edge to the alias in that case. Eventually
|
|
|
|
// an alias summary will be created to associate the alias and
|
|
|
|
// aliasee.
|
2018-01-26 03:27:17 +08:00
|
|
|
auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo(
|
|
|
|
cast<GlobalValue>(CalledValue))];
|
|
|
|
ValueInfo.updateHotness(Hotness);
|
|
|
|
// Add the relative block frequency to CalleeInfo if there is no profile
|
|
|
|
// information.
|
|
|
|
if (BFI != nullptr && Hotness == CalleeInfo::HotnessType::Unknown) {
|
2018-02-23 03:44:08 +08:00
|
|
|
uint64_t BBFreq = BFI->getBlockFreq(&BB).getFrequency();
|
|
|
|
uint64_t EntryFreq = BFI->getEntryFreq();
|
|
|
|
ValueInfo.updateRelBlockFreq(BBFreq, EntryFreq);
|
2018-01-26 03:27:17 +08:00
|
|
|
}
|
2016-08-30 08:46:26 +08:00
|
|
|
} else {
|
|
|
|
// Skip inline assembly calls.
|
|
|
|
if (CI && CI->isInlineAsm())
|
|
|
|
continue;
|
2017-11-18 02:28:05 +08:00
|
|
|
// Skip direct calls.
|
|
|
|
if (!CalledValue || isa<Constant>(CalledValue))
|
|
|
|
continue;
|
2016-04-11 21:58:45 +08:00
|
|
|
|
[ThinLTO] Add funtions in callees metadata to CallGraphEdges
Summary:
If there's a callees metadata attached to the indirect call instruction, add CallGraphEdges to the callees mentioned in the metadata when computing FunctionSummary.
* Why this is necessary:
Consider following code example:
```
(foo.c)
static int f1(int x) {...}
static int f2(int x);
static int (*fptr)(int) = f2;
static int f2(int x) {
if (x) fptr=f1; return f1(x);
}
int foo(int x) {
(*fptr)(x); // !callees metadata of !{i32 (i32)* @f1, i32 (i32)* @f2} would be attached to this call.
}
(bar.c)
int bar(int x) {
return foo(x);
}
```
At LTO time when `foo.o` is imported into `bar.o`, function `foo` might be inlined into `bar` and PGO-guided indirect call promotion will run after that. If the profile data tells that the promotion of `@f1` or `@f2` is beneficial, the optimizer will check if the "promoted" `@f1` or `@f2` (such as `@f1.llvm.0` or `@f2.llvm.0`) is available. Without this patch, importing `!callees` metadata would only add promoted declarations of `@f1` and `@f2` to the `bar.o`, but still the optimizer will assume that the function is available and perform the promotion. The result of that is link failure with `undefined reference to @f1.llvm.0`.
This patch fixes this problem by adding callees in the `!callees` metadata to CallGraphEdges so that their definition would be properly imported into.
One may ask that there already is a logic to add indirect call promotion targets to be added to CallGraphEdges. However, if profile data says "indirect call promotion is only beneficial under a certain inline context", the logic wouldn't work. In the code example above, if profile data is like
```
bar:1000000:100000
1:100000
1: foo:100000
1: 100000 f1:100000
```
, Computing FunctionSummary for `foo.o` wouldn't add `foo->f1` to CallGraphEdges. (Also, it is at least "possible" that one can provide profile data to only link step but not to compilation step).
Reviewers: tejohnson, mehdi_amini, pcc
Reviewed By: tejohnson
Subscribers: inglorion, eraman, llvm-commits
Differential Revision: https://reviews.llvm.org/D44399
llvm-svn: 327358
2018-03-13 12:26:58 +08:00
|
|
|
// Check if the instruction has a callees metadata. If so, add callees
|
|
|
|
// to CallGraphEdges to reflect the references from the metadata, and
|
|
|
|
// to enable importing for subsequent indirect call promotion and
|
|
|
|
// inlining.
|
|
|
|
if (auto *MD = I.getMetadata(LLVMContext::MD_callees)) {
|
|
|
|
for (auto &Op : MD->operands()) {
|
|
|
|
Function *Callee = mdconst::extract_or_null<Function>(Op);
|
|
|
|
if (Callee)
|
|
|
|
CallGraphEdges[Index.getOrInsertValueInfo(Callee)];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-08-30 08:46:26 +08:00
|
|
|
uint32_t NumVals, NumCandidates;
|
|
|
|
uint64_t TotalCount;
|
|
|
|
auto CandidateProfileData =
|
|
|
|
ICallAnalysis.getPromotionCandidatesForInstruction(
|
|
|
|
&I, NumVals, TotalCount, NumCandidates);
|
|
|
|
for (auto &Candidate : CandidateProfileData)
|
2017-05-05 02:03:25 +08:00
|
|
|
CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)]
|
|
|
|
.updateHotness(getHotness(Candidate.Count, PSI));
|
2016-04-11 21:58:45 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-11-16 15:08:00 +08:00
|
|
|
// By now we processed all instructions in a function, except
|
|
|
|
// non-volatile loads. All new refs we add in a loop below
|
|
|
|
// are obviously constant. All constant refs are grouped in the
|
|
|
|
// end of RefEdges vector, so we can use a single integer value
|
|
|
|
// to identify them.
|
|
|
|
unsigned RefCnt = RefEdges.size();
|
|
|
|
for (const Instruction *I : NonVolatileLoads) {
|
|
|
|
Visited.erase(I);
|
|
|
|
findRefEdges(Index, I, RefEdges, Visited);
|
|
|
|
}
|
|
|
|
std::vector<ValueInfo> Refs = RefEdges.takeVector();
|
|
|
|
// Regular LTO module doesn't participate in ThinLTO import,
|
|
|
|
// so no reference from it can be readonly, since this would
|
|
|
|
// require importing variable as local copy
|
|
|
|
if (IsThinLTO)
|
|
|
|
for (; RefCnt < Refs.size(); ++RefCnt)
|
|
|
|
Refs[RefCnt].setReadOnly();
|
|
|
|
|
2017-03-01 02:09:44 +08:00
|
|
|
// Explicit add hot edges to enforce importing for designated GUIDs for
|
|
|
|
// sample PGO, to enable the same inlines as the profiled optimized binary.
|
|
|
|
for (auto &I : F.getImportGUIDs())
|
2017-05-05 02:03:25 +08:00
|
|
|
CallGraphEdges[Index.getOrInsertValueInfo(I)].updateHotness(
|
2018-03-31 08:18:08 +08:00
|
|
|
ForceSummaryEdgesCold == FunctionSummary::FSHT_All
|
|
|
|
? CalleeInfo::HotnessType::Cold
|
|
|
|
: CalleeInfo::HotnessType::Critical);
|
2017-03-01 02:09:44 +08:00
|
|
|
|
2017-01-05 22:32:16 +08:00
|
|
|
bool NonRenamableLocal = isNonRenamableLocal(F);
|
|
|
|
bool NotEligibleForImport =
|
2018-11-07 03:41:35 +08:00
|
|
|
NonRenamableLocal || HasInlineAsmMaybeReferencingInternal;
|
2017-01-06 05:34:18 +08:00
|
|
|
GlobalValueSummary::GVFlags Flags(F.getLinkage(), NotEligibleForImport,
|
2017-11-05 01:04:39 +08:00
|
|
|
/* Live = */ false, F.isDSOLocal());
|
2017-08-05 00:00:58 +08:00
|
|
|
FunctionSummary::FFlags FunFlags{
|
|
|
|
F.hasFnAttribute(Attribute::ReadNone),
|
|
|
|
F.hasFnAttribute(Attribute::ReadOnly),
|
2018-11-07 03:41:35 +08:00
|
|
|
F.hasFnAttribute(Attribute::NoRecurse), F.returnDoesNotAlias(),
|
|
|
|
// FIXME: refactor this to use the same code that inliner is using.
|
2018-12-01 13:11:46 +08:00
|
|
|
// Don't try to import functions with noinline attribute.
|
|
|
|
F.getAttributes().hasFnAttribute(Attribute::NoInline)};
|
2016-12-21 05:12:28 +08:00
|
|
|
auto FuncSummary = llvm::make_unique<FunctionSummary>(
|
2018-12-14 03:54:27 +08:00
|
|
|
Flags, NumInsts, FunFlags, /*EntryCount=*/0, std::move(Refs),
|
|
|
|
CallGraphEdges.takeVector(), TypeTests.takeVector(),
|
|
|
|
TypeTestAssumeVCalls.takeVector(), TypeCheckedLoadVCalls.takeVector(),
|
2017-02-11 06:29:38 +08:00
|
|
|
TypeTestAssumeConstVCalls.takeVector(),
|
|
|
|
TypeCheckedLoadConstVCalls.takeVector());
|
2017-01-05 22:32:16 +08:00
|
|
|
if (NonRenamableLocal)
|
|
|
|
CantBePromoted.insert(F.getGUID());
|
2018-06-26 08:20:49 +08:00
|
|
|
Index.addGlobalValueSummary(F, std::move(FuncSummary));
|
2016-04-11 21:58:45 +08:00
|
|
|
}
|
|
|
|
|
2019-01-18 00:05:04 +08:00
|
|
|
static void
|
|
|
|
computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V,
|
|
|
|
DenseSet<GlobalValue::GUID> &CantBePromoted) {
|
2016-12-21 05:12:28 +08:00
|
|
|
SetVector<ValueInfo> RefEdges;
|
2016-04-11 21:58:45 +08:00
|
|
|
SmallPtrSet<const User *, 8> Visited;
|
2018-10-12 15:24:02 +08:00
|
|
|
bool HasBlockAddress = findRefEdges(Index, &V, RefEdges, Visited);
|
2017-01-05 22:32:16 +08:00
|
|
|
bool NonRenamableLocal = isNonRenamableLocal(V);
|
2017-01-06 05:34:18 +08:00
|
|
|
GlobalValueSummary::GVFlags Flags(V.getLinkage(), NonRenamableLocal,
|
2017-11-05 01:04:39 +08:00
|
|
|
/* Live = */ false, V.isDSOLocal());
|
2018-11-16 15:08:00 +08:00
|
|
|
|
|
|
|
// Don't mark variables we won't be able to internalize as read-only.
|
|
|
|
GlobalVarSummary::GVarFlags VarFlags(
|
|
|
|
!V.hasComdat() && !V.hasAppendingLinkage() && !V.isInterposable() &&
|
|
|
|
!V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass());
|
|
|
|
auto GVarSummary = llvm::make_unique<GlobalVarSummary>(Flags, VarFlags,
|
|
|
|
RefEdges.takeVector());
|
2017-01-05 22:32:16 +08:00
|
|
|
if (NonRenamableLocal)
|
|
|
|
CantBePromoted.insert(V.getGUID());
|
2018-10-12 15:24:02 +08:00
|
|
|
if (HasBlockAddress)
|
|
|
|
GVarSummary->setNotEligibleToImport();
|
2018-06-26 08:20:49 +08:00
|
|
|
Index.addGlobalValueSummary(V, std::move(GVarSummary));
|
2016-04-11 21:58:45 +08:00
|
|
|
}
|
|
|
|
|
2017-01-05 22:32:16 +08:00
|
|
|
static void
|
|
|
|
computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A,
|
2017-01-05 22:59:56 +08:00
|
|
|
DenseSet<GlobalValue::GUID> &CantBePromoted) {
|
2017-01-05 22:32:16 +08:00
|
|
|
bool NonRenamableLocal = isNonRenamableLocal(A);
|
2017-01-06 05:34:18 +08:00
|
|
|
GlobalValueSummary::GVFlags Flags(A.getLinkage(), NonRenamableLocal,
|
2017-11-05 01:04:39 +08:00
|
|
|
/* Live = */ false, A.isDSOLocal());
|
2017-09-14 01:10:24 +08:00
|
|
|
auto AS = llvm::make_unique<AliasSummary>(Flags);
|
2016-10-28 10:39:38 +08:00
|
|
|
auto *Aliasee = A.getBaseObject();
|
|
|
|
auto *AliaseeSummary = Index.getGlobalValueSummary(*Aliasee);
|
|
|
|
assert(AliaseeSummary && "Alias expects aliasee summary to be parsed");
|
|
|
|
AS->setAliasee(AliaseeSummary);
|
2017-01-05 22:32:16 +08:00
|
|
|
if (NonRenamableLocal)
|
|
|
|
CantBePromoted.insert(A.getGUID());
|
2018-06-26 08:20:49 +08:00
|
|
|
Index.addGlobalValueSummary(A, std::move(AS));
|
2016-10-28 10:39:38 +08:00
|
|
|
}
|
|
|
|
|
2017-01-06 05:34:18 +08:00
|
|
|
// Set LiveRoot flag on entries matching the given value name.
|
|
|
|
static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) {
|
2017-05-05 02:03:25 +08:00
|
|
|
if (ValueInfo VI = Index.getValueInfo(GlobalValue::getGUID(Name)))
|
|
|
|
for (auto &Summary : VI.getSummaryList())
|
2017-06-02 04:30:06 +08:00
|
|
|
Summary->setLive(true);
|
2017-01-06 05:34:18 +08:00
|
|
|
}
|
|
|
|
|
2016-08-19 15:49:19 +08:00
|
|
|
ModuleSummaryIndex llvm::buildModuleSummaryIndex(
|
|
|
|
const Module &M,
|
[thinlto] Basic thinlto fdo heuristic
Summary:
This patch improves thinlto importer
by importing 3x larger functions that are called from hot block.
I compared performance with the trunk on spec, and there
were about 2% on povray and 3.33% on milc. These results seems
to be consistant and match the results Teresa got with her simple
heuristic. Some benchmarks got slower but I think they are just
noisy (mcf, xalancbmki, omnetpp)- running the benchmarks again with
more iterations to confirm. Geomean of all benchmarks including the noisy ones
were about +0.02%.
I see much better improvement on google branch with Easwaran patch
for pgo callsite inlining (the inliner actually inline those big functions)
Over all I see +0.5% improvement, and I get +8.65% on povray.
So I guess we will see much bigger change when Easwaran patch will land
(it depends on new pass manager), but it is still worth putting this to trunk
before it.
Implementation details changes:
- Removed CallsiteCount.
- ProfileCount got replaced by Hotness
- hot-import-multiplier is set to 3.0 for now,
didn't have time to tune it up, but I see that we get most of the interesting
functions with 3, so there is no much performance difference with higher, and
binary size doesn't grow as much as with 10.0.
Reviewers: eraman, mehdi_amini, tejohnson
Subscribers: mehdi_amini, llvm-commits
Differential Revision: https://reviews.llvm.org/D24638
llvm-svn: 282437
2016-09-27 04:37:32 +08:00
|
|
|
std::function<BlockFrequencyInfo *(const Function &F)> GetBFICallback,
|
|
|
|
ProfileSummaryInfo *PSI) {
|
2017-05-11 02:52:16 +08:00
|
|
|
assert(PSI);
|
[LTO] Record whether LTOUnit splitting is enabled in index
Summary:
Records in the module summary index whether the bitcode was compiled
with the option necessary to enable splitting the LTO unit
(e.g. -fsanitize=cfi, -fwhole-program-vtables, or -fsplit-lto-unit).
The information is passed down to the ModuleSummaryIndex builder via a
new module flag "EnableSplitLTOUnit", which is propagated onto a flag
on the summary index.
This is then used during the LTO link to check whether all linked
summaries were built with the same value of this flag. If not, an error
is issued when we detect a situation requiring whole program visibility
of the class hierarchy. This is the case when both of the following
conditions are met:
1) We are performing LowerTypeTests or Whole Program Devirtualization.
2) There are type tests or type checked loads in the code.
Note I have also changed the ThinLTOBitcodeWriter to also gate the
module splitting on the value of this flag.
Reviewers: pcc
Subscribers: ormris, mehdi_amini, Prazek, inglorion, eraman, steven_wu, dexonsmith, arphaman, dang, llvm-commits
Differential Revision: https://reviews.llvm.org/D53890
llvm-svn: 350948
2019-01-12 02:31:57 +08:00
|
|
|
bool EnableSplitLTOUnit = false;
|
|
|
|
if (auto *MD = mdconst::extract_or_null<ConstantInt>(
|
|
|
|
M.getModuleFlag("EnableSplitLTOUnit")))
|
|
|
|
EnableSplitLTOUnit = MD->getZExtValue();
|
|
|
|
ModuleSummaryIndex Index(/*HaveGVs=*/true, EnableSplitLTOUnit);
|
2016-10-30 13:40:44 +08:00
|
|
|
|
2016-11-11 00:57:32 +08:00
|
|
|
// Identify the local values in the llvm.used and llvm.compiler.used sets,
|
|
|
|
// which should not be exported as they would then require renaming and
|
|
|
|
// promotion, but we may have opaque uses e.g. in inline asm. We collect them
|
|
|
|
// here because we use this information to mark functions containing inline
|
|
|
|
// assembly calls as not importable.
|
|
|
|
SmallPtrSet<GlobalValue *, 8> LocalsUsed;
|
2016-10-30 13:40:44 +08:00
|
|
|
SmallPtrSet<GlobalValue *, 8> Used;
|
2016-11-11 00:57:32 +08:00
|
|
|
// First collect those in the llvm.used set.
|
2016-10-30 13:40:44 +08:00
|
|
|
collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ false);
|
2016-11-11 00:57:32 +08:00
|
|
|
// Next collect those in the llvm.compiler.used set.
|
|
|
|
collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ true);
|
2017-01-05 22:59:56 +08:00
|
|
|
DenseSet<GlobalValue::GUID> CantBePromoted;
|
2016-10-30 13:40:44 +08:00
|
|
|
for (auto *V : Used) {
|
2017-01-05 22:32:16 +08:00
|
|
|
if (V->hasLocalLinkage()) {
|
2016-10-30 13:40:44 +08:00
|
|
|
LocalsUsed.insert(V);
|
2017-01-05 22:32:16 +08:00
|
|
|
CantBePromoted.insert(V->getGUID());
|
|
|
|
}
|
2016-10-30 13:40:44 +08:00
|
|
|
}
|
2016-04-20 22:39:45 +08:00
|
|
|
|
2017-09-02 00:24:02 +08:00
|
|
|
bool HasLocalInlineAsmSymbol = false;
|
|
|
|
if (!M.getModuleInlineAsm().empty()) {
|
|
|
|
// Collect the local values defined by module level asm, and set up
|
|
|
|
// summaries for these symbols so that they can be marked as NoRename,
|
|
|
|
// to prevent export of any use of them in regular IR that would require
|
|
|
|
// renaming within the module level asm. Note we don't need to create a
|
|
|
|
// summary for weak or global defs, as they don't need to be flagged as
|
|
|
|
// NoRename, and defs in module level asm can't be imported anyway.
|
|
|
|
// Also, any values used but not defined within module level asm should
|
|
|
|
// be listed on the llvm.used or llvm.compiler.used global and marked as
|
|
|
|
// referenced from there.
|
|
|
|
ModuleSymbolTable::CollectAsmSymbols(
|
|
|
|
M, [&](StringRef Name, object::BasicSymbolRef::Flags Flags) {
|
|
|
|
// Symbols not marked as Weak or Global are local definitions.
|
|
|
|
if (Flags & (object::BasicSymbolRef::SF_Weak |
|
|
|
|
object::BasicSymbolRef::SF_Global))
|
|
|
|
return;
|
|
|
|
HasLocalInlineAsmSymbol = true;
|
|
|
|
GlobalValue *GV = M.getNamedValue(Name);
|
|
|
|
if (!GV)
|
|
|
|
return;
|
|
|
|
assert(GV->isDeclaration() && "Def in module asm already has definition");
|
|
|
|
GlobalValueSummary::GVFlags GVFlags(GlobalValue::InternalLinkage,
|
|
|
|
/* NotEligibleToImport = */ true,
|
2017-11-05 01:04:39 +08:00
|
|
|
/* Live = */ true,
|
|
|
|
/* Local */ GV->isDSOLocal());
|
2018-06-26 08:20:49 +08:00
|
|
|
CantBePromoted.insert(GV->getGUID());
|
2017-09-02 00:24:02 +08:00
|
|
|
// Create the appropriate summary type.
|
|
|
|
if (Function *F = dyn_cast<Function>(GV)) {
|
|
|
|
std::unique_ptr<FunctionSummary> Summary =
|
|
|
|
llvm::make_unique<FunctionSummary>(
|
2018-12-14 03:54:27 +08:00
|
|
|
GVFlags, /*InstCount=*/0,
|
2017-09-02 00:24:02 +08:00
|
|
|
FunctionSummary::FFlags{
|
|
|
|
F->hasFnAttribute(Attribute::ReadNone),
|
|
|
|
F->hasFnAttribute(Attribute::ReadOnly),
|
|
|
|
F->hasFnAttribute(Attribute::NoRecurse),
|
2018-11-07 03:41:35 +08:00
|
|
|
F->returnDoesNotAlias(),
|
|
|
|
/* NoInline = */ false},
|
2018-12-14 03:54:27 +08:00
|
|
|
/*EntryCount=*/0, ArrayRef<ValueInfo>{},
|
|
|
|
ArrayRef<FunctionSummary::EdgeTy>{},
|
2017-09-02 00:24:02 +08:00
|
|
|
ArrayRef<GlobalValue::GUID>{},
|
|
|
|
ArrayRef<FunctionSummary::VFuncId>{},
|
|
|
|
ArrayRef<FunctionSummary::VFuncId>{},
|
|
|
|
ArrayRef<FunctionSummary::ConstVCall>{},
|
|
|
|
ArrayRef<FunctionSummary::ConstVCall>{});
|
2018-06-26 08:20:49 +08:00
|
|
|
Index.addGlobalValueSummary(*GV, std::move(Summary));
|
2017-09-02 00:24:02 +08:00
|
|
|
} else {
|
|
|
|
std::unique_ptr<GlobalVarSummary> Summary =
|
2018-11-16 15:08:00 +08:00
|
|
|
llvm::make_unique<GlobalVarSummary>(
|
|
|
|
GVFlags, GlobalVarSummary::GVarFlags(),
|
|
|
|
ArrayRef<ValueInfo>{});
|
2018-06-26 08:20:49 +08:00
|
|
|
Index.addGlobalValueSummary(*GV, std::move(Summary));
|
2017-09-02 00:24:02 +08:00
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2018-11-16 15:08:00 +08:00
|
|
|
bool IsThinLTO = true;
|
|
|
|
if (auto *MD =
|
|
|
|
mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("ThinLTO")))
|
|
|
|
IsThinLTO = MD->getZExtValue();
|
|
|
|
|
2016-04-11 21:58:45 +08:00
|
|
|
// Compute summaries for all functions defined in module, and save in the
|
|
|
|
// index.
|
2016-08-19 15:49:19 +08:00
|
|
|
for (auto &F : M) {
|
2016-04-11 21:58:45 +08:00
|
|
|
if (F.isDeclaration())
|
|
|
|
continue;
|
|
|
|
|
2018-09-27 22:55:32 +08:00
|
|
|
DominatorTree DT(const_cast<Function &>(F));
|
2016-04-11 21:58:45 +08:00
|
|
|
BlockFrequencyInfo *BFI = nullptr;
|
|
|
|
std::unique_ptr<BlockFrequencyInfo> BFIPtr;
|
2016-08-19 15:49:19 +08:00
|
|
|
if (GetBFICallback)
|
|
|
|
BFI = GetBFICallback(F);
|
2017-12-22 09:33:52 +08:00
|
|
|
else if (F.hasProfileData()) {
|
2018-09-27 22:55:32 +08:00
|
|
|
LoopInfo LI{DT};
|
2016-04-11 21:58:45 +08:00
|
|
|
BranchProbabilityInfo BPI{F, LI};
|
|
|
|
BFIPtr = llvm::make_unique<BlockFrequencyInfo>(F, BPI, LI);
|
|
|
|
BFI = BFIPtr.get();
|
|
|
|
}
|
|
|
|
|
2018-09-27 22:55:32 +08:00
|
|
|
computeFunctionSummary(Index, M, F, BFI, PSI, DT,
|
2017-09-02 00:24:02 +08:00
|
|
|
!LocalsUsed.empty() || HasLocalInlineAsmSymbol,
|
2018-11-16 15:08:00 +08:00
|
|
|
CantBePromoted, IsThinLTO);
|
2016-04-11 21:58:45 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Compute summaries for all variables defined in module, and save in the
|
|
|
|
// index.
|
2016-08-19 15:49:19 +08:00
|
|
|
for (const GlobalVariable &G : M.globals()) {
|
2016-04-11 21:58:45 +08:00
|
|
|
if (G.isDeclaration())
|
|
|
|
continue;
|
2019-01-18 00:05:04 +08:00
|
|
|
computeVariableSummary(Index, G, CantBePromoted);
|
2016-04-11 21:58:45 +08:00
|
|
|
}
|
2016-10-28 10:39:38 +08:00
|
|
|
|
|
|
|
// Compute summaries for all aliases defined in module, and save in the
|
|
|
|
// index.
|
|
|
|
for (const GlobalAlias &A : M.aliases())
|
2017-01-05 22:32:16 +08:00
|
|
|
computeAliasSummary(Index, A, CantBePromoted);
|
2016-10-28 10:39:38 +08:00
|
|
|
|
2016-10-30 13:40:44 +08:00
|
|
|
for (auto *V : LocalsUsed) {
|
|
|
|
auto *Summary = Index.getGlobalValueSummary(*V);
|
|
|
|
assert(Summary && "Missing summary for global value");
|
2017-01-05 22:32:16 +08:00
|
|
|
Summary->setNotEligibleToImport();
|
2016-10-30 13:40:44 +08:00
|
|
|
}
|
|
|
|
|
2017-01-06 05:34:18 +08:00
|
|
|
// The linker doesn't know about these LLVM produced values, so we need
|
|
|
|
// to flag them as live in the index to ensure index-based dead value
|
|
|
|
// analysis treats them as live roots of the analysis.
|
|
|
|
setLiveRoot(Index, "llvm.used");
|
|
|
|
setLiveRoot(Index, "llvm.compiler.used");
|
|
|
|
setLiveRoot(Index, "llvm.global_ctors");
|
|
|
|
setLiveRoot(Index, "llvm.global_dtors");
|
|
|
|
setLiveRoot(Index, "llvm.global.annotations");
|
|
|
|
|
2017-01-05 22:32:16 +08:00
|
|
|
for (auto &GlobalList : Index) {
|
2017-05-05 02:03:25 +08:00
|
|
|
// Ignore entries for references that are undefined in the current module.
|
|
|
|
if (GlobalList.second.SummaryList.empty())
|
|
|
|
continue;
|
|
|
|
|
|
|
|
assert(GlobalList.second.SummaryList.size() == 1 &&
|
2017-01-05 22:32:16 +08:00
|
|
|
"Expected module's index to have one summary per GUID");
|
2017-05-05 02:03:25 +08:00
|
|
|
auto &Summary = GlobalList.second.SummaryList[0];
|
2017-06-09 07:01:49 +08:00
|
|
|
if (!IsThinLTO) {
|
|
|
|
Summary->setNotEligibleToImport();
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2017-01-05 22:32:16 +08:00
|
|
|
bool AllRefsCanBeExternallyReferenced =
|
|
|
|
llvm::all_of(Summary->refs(), [&](const ValueInfo &VI) {
|
2017-05-05 02:03:25 +08:00
|
|
|
return !CantBePromoted.count(VI.getGUID());
|
2017-01-05 22:32:16 +08:00
|
|
|
});
|
|
|
|
if (!AllRefsCanBeExternallyReferenced) {
|
|
|
|
Summary->setNotEligibleToImport();
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (auto *FuncSummary = dyn_cast<FunctionSummary>(Summary.get())) {
|
|
|
|
bool AllCallsCanBeExternallyReferenced = llvm::all_of(
|
|
|
|
FuncSummary->calls(), [&](const FunctionSummary::EdgeTy &Edge) {
|
2017-05-05 02:03:25 +08:00
|
|
|
return !CantBePromoted.count(Edge.first.getGUID());
|
2017-01-05 22:32:16 +08:00
|
|
|
});
|
|
|
|
if (!AllCallsCanBeExternallyReferenced)
|
|
|
|
Summary->setNotEligibleToImport();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-08-19 15:49:19 +08:00
|
|
|
return Index;
|
2016-04-11 21:58:45 +08:00
|
|
|
}
|
|
|
|
|
2016-11-24 01:53:26 +08:00
|
|
|
AnalysisKey ModuleSummaryIndexAnalysis::Key;
|
2016-08-12 21:53:02 +08:00
|
|
|
|
2016-08-19 15:49:19 +08:00
|
|
|
ModuleSummaryIndex
|
2016-08-12 21:53:02 +08:00
|
|
|
ModuleSummaryIndexAnalysis::run(Module &M, ModuleAnalysisManager &AM) {
|
[thinlto] Basic thinlto fdo heuristic
Summary:
This patch improves thinlto importer
by importing 3x larger functions that are called from hot block.
I compared performance with the trunk on spec, and there
were about 2% on povray and 3.33% on milc. These results seems
to be consistant and match the results Teresa got with her simple
heuristic. Some benchmarks got slower but I think they are just
noisy (mcf, xalancbmki, omnetpp)- running the benchmarks again with
more iterations to confirm. Geomean of all benchmarks including the noisy ones
were about +0.02%.
I see much better improvement on google branch with Easwaran patch
for pgo callsite inlining (the inliner actually inline those big functions)
Over all I see +0.5% improvement, and I get +8.65% on povray.
So I guess we will see much bigger change when Easwaran patch will land
(it depends on new pass manager), but it is still worth putting this to trunk
before it.
Implementation details changes:
- Removed CallsiteCount.
- ProfileCount got replaced by Hotness
- hot-import-multiplier is set to 3.0 for now,
didn't have time to tune it up, but I see that we get most of the interesting
functions with 3, so there is no much performance difference with higher, and
binary size doesn't grow as much as with 10.0.
Reviewers: eraman, mehdi_amini, tejohnson
Subscribers: mehdi_amini, llvm-commits
Differential Revision: https://reviews.llvm.org/D24638
llvm-svn: 282437
2016-09-27 04:37:32 +08:00
|
|
|
ProfileSummaryInfo &PSI = AM.getResult<ProfileSummaryAnalysis>(M);
|
2016-08-12 21:53:02 +08:00
|
|
|
auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
|
[thinlto] Basic thinlto fdo heuristic
Summary:
This patch improves thinlto importer
by importing 3x larger functions that are called from hot block.
I compared performance with the trunk on spec, and there
were about 2% on povray and 3.33% on milc. These results seems
to be consistant and match the results Teresa got with her simple
heuristic. Some benchmarks got slower but I think they are just
noisy (mcf, xalancbmki, omnetpp)- running the benchmarks again with
more iterations to confirm. Geomean of all benchmarks including the noisy ones
were about +0.02%.
I see much better improvement on google branch with Easwaran patch
for pgo callsite inlining (the inliner actually inline those big functions)
Over all I see +0.5% improvement, and I get +8.65% on povray.
So I guess we will see much bigger change when Easwaran patch will land
(it depends on new pass manager), but it is still worth putting this to trunk
before it.
Implementation details changes:
- Removed CallsiteCount.
- ProfileCount got replaced by Hotness
- hot-import-multiplier is set to 3.0 for now,
didn't have time to tune it up, but I see that we get most of the interesting
functions with 3, so there is no much performance difference with higher, and
binary size doesn't grow as much as with 10.0.
Reviewers: eraman, mehdi_amini, tejohnson
Subscribers: mehdi_amini, llvm-commits
Differential Revision: https://reviews.llvm.org/D24638
llvm-svn: 282437
2016-09-27 04:37:32 +08:00
|
|
|
return buildModuleSummaryIndex(
|
|
|
|
M,
|
|
|
|
[&FAM](const Function &F) {
|
|
|
|
return &FAM.getResult<BlockFrequencyAnalysis>(
|
|
|
|
*const_cast<Function *>(&F));
|
|
|
|
},
|
|
|
|
&PSI);
|
2016-08-12 21:53:02 +08:00
|
|
|
}
|
|
|
|
|
2016-04-11 21:58:45 +08:00
|
|
|
char ModuleSummaryIndexWrapperPass::ID = 0;
|
2017-08-17 06:07:40 +08:00
|
|
|
|
2016-04-11 21:58:45 +08:00
|
|
|
INITIALIZE_PASS_BEGIN(ModuleSummaryIndexWrapperPass, "module-summary-analysis",
|
|
|
|
"Module Summary Analysis", false, true)
|
|
|
|
INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
|
2017-01-21 14:01:22 +08:00
|
|
|
INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
|
2016-04-11 21:58:45 +08:00
|
|
|
INITIALIZE_PASS_END(ModuleSummaryIndexWrapperPass, "module-summary-analysis",
|
|
|
|
"Module Summary Analysis", false, true)
|
|
|
|
|
|
|
|
ModulePass *llvm::createModuleSummaryIndexWrapperPass() {
|
|
|
|
return new ModuleSummaryIndexWrapperPass();
|
|
|
|
}
|
|
|
|
|
|
|
|
ModuleSummaryIndexWrapperPass::ModuleSummaryIndexWrapperPass()
|
|
|
|
: ModulePass(ID) {
|
|
|
|
initializeModuleSummaryIndexWrapperPassPass(*PassRegistry::getPassRegistry());
|
|
|
|
}
|
|
|
|
|
|
|
|
bool ModuleSummaryIndexWrapperPass::runOnModule(Module &M) {
|
2018-11-19 13:23:16 +08:00
|
|
|
auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
|
2018-06-26 10:29:08 +08:00
|
|
|
Index.emplace(buildModuleSummaryIndex(
|
[thinlto] Basic thinlto fdo heuristic
Summary:
This patch improves thinlto importer
by importing 3x larger functions that are called from hot block.
I compared performance with the trunk on spec, and there
were about 2% on povray and 3.33% on milc. These results seems
to be consistant and match the results Teresa got with her simple
heuristic. Some benchmarks got slower but I think they are just
noisy (mcf, xalancbmki, omnetpp)- running the benchmarks again with
more iterations to confirm. Geomean of all benchmarks including the noisy ones
were about +0.02%.
I see much better improvement on google branch with Easwaran patch
for pgo callsite inlining (the inliner actually inline those big functions)
Over all I see +0.5% improvement, and I get +8.65% on povray.
So I guess we will see much bigger change when Easwaran patch will land
(it depends on new pass manager), but it is still worth putting this to trunk
before it.
Implementation details changes:
- Removed CallsiteCount.
- ProfileCount got replaced by Hotness
- hot-import-multiplier is set to 3.0 for now,
didn't have time to tune it up, but I see that we get most of the interesting
functions with 3, so there is no much performance difference with higher, and
binary size doesn't grow as much as with 10.0.
Reviewers: eraman, mehdi_amini, tejohnson
Subscribers: mehdi_amini, llvm-commits
Differential Revision: https://reviews.llvm.org/D24638
llvm-svn: 282437
2016-09-27 04:37:32 +08:00
|
|
|
M,
|
|
|
|
[this](const Function &F) {
|
|
|
|
return &(this->getAnalysis<BlockFrequencyInfoWrapperPass>(
|
|
|
|
*const_cast<Function *>(&F))
|
|
|
|
.getBFI());
|
|
|
|
},
|
2018-11-19 13:23:16 +08:00
|
|
|
PSI));
|
2016-04-11 21:58:45 +08:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool ModuleSummaryIndexWrapperPass::doFinalization(Module &M) {
|
2016-08-19 15:49:19 +08:00
|
|
|
Index.reset();
|
2016-04-11 21:58:45 +08:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
void ModuleSummaryIndexWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
|
|
|
|
AU.setPreservesAll();
|
|
|
|
AU.addRequired<BlockFrequencyInfoWrapperPass>();
|
[thinlto] Basic thinlto fdo heuristic
Summary:
This patch improves thinlto importer
by importing 3x larger functions that are called from hot block.
I compared performance with the trunk on spec, and there
were about 2% on povray and 3.33% on milc. These results seems
to be consistant and match the results Teresa got with her simple
heuristic. Some benchmarks got slower but I think they are just
noisy (mcf, xalancbmki, omnetpp)- running the benchmarks again with
more iterations to confirm. Geomean of all benchmarks including the noisy ones
were about +0.02%.
I see much better improvement on google branch with Easwaran patch
for pgo callsite inlining (the inliner actually inline those big functions)
Over all I see +0.5% improvement, and I get +8.65% on povray.
So I guess we will see much bigger change when Easwaran patch will land
(it depends on new pass manager), but it is still worth putting this to trunk
before it.
Implementation details changes:
- Removed CallsiteCount.
- ProfileCount got replaced by Hotness
- hot-import-multiplier is set to 3.0 for now,
didn't have time to tune it up, but I see that we get most of the interesting
functions with 3, so there is no much performance difference with higher, and
binary size doesn't grow as much as with 10.0.
Reviewers: eraman, mehdi_amini, tejohnson
Subscribers: mehdi_amini, llvm-commits
Differential Revision: https://reviews.llvm.org/D24638
llvm-svn: 282437
2016-09-27 04:37:32 +08:00
|
|
|
AU.addRequired<ProfileSummaryInfoWrapperPass>();
|
2016-04-11 21:58:45 +08:00
|
|
|
}
|