2014-01-07 06:27:43 +08:00
|
|
|
//===--- CodeGenPGO.h - PGO Instrumentation for LLVM CodeGen ----*- C++ -*-===//
|
|
|
|
//
|
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
//
|
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
// License. See LICENSE.TXT for details.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
|
|
|
// Instrumentation-based profile-guided optimization
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
2014-08-14 00:25:19 +08:00
|
|
|
#ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENPGO_H
|
|
|
|
#define LLVM_CLANG_LIB_CODEGEN_CODEGENPGO_H
|
2014-01-07 06:27:43 +08:00
|
|
|
|
|
|
|
#include "CGBuilder.h"
|
|
|
|
#include "CodeGenModule.h"
|
|
|
|
#include "CodeGenTypes.h"
|
|
|
|
#include "clang/Frontend/CodeGenOptions.h"
|
|
|
|
#include "llvm/ADT/StringMap.h"
|
|
|
|
#include "llvm/Support/MemoryBuffer.h"
|
2014-03-09 19:36:40 +08:00
|
|
|
#include <memory>
|
2014-01-07 06:27:43 +08:00
|
|
|
|
|
|
|
namespace clang {
|
|
|
|
namespace CodeGen {
|
|
|
|
|
2015-04-24 07:06:47 +08:00
|
|
|
/// Per-function PGO state.
|
2014-01-07 06:27:43 +08:00
|
|
|
class CodeGenPGO {
|
|
|
|
private:
|
|
|
|
CodeGenModule &CGM;
|
2014-12-03 07:15:30 +08:00
|
|
|
std::string FuncName;
|
2014-12-09 03:04:51 +08:00
|
|
|
llvm::GlobalVariable *FuncNameVar;
|
2014-01-07 06:27:43 +08:00
|
|
|
|
|
|
|
unsigned NumRegionCounters;
|
2014-03-19 05:58:06 +08:00
|
|
|
uint64_t FunctionHash;
|
2014-03-27 03:26:05 +08:00
|
|
|
std::unique_ptr<llvm::DenseMap<const Stmt *, unsigned>> RegionCounterMap;
|
|
|
|
std::unique_ptr<llvm::DenseMap<const Stmt *, uint64_t>> StmtCountMap;
|
2014-12-03 06:38:52 +08:00
|
|
|
std::vector<uint64_t> RegionCounts;
|
2014-01-07 06:27:43 +08:00
|
|
|
uint64_t CurrentRegionCount;
|
2014-08-05 02:41:51 +08:00
|
|
|
/// \brief A flag that is set to true when this function doesn't need
|
|
|
|
/// to have coverage mapping data.
|
|
|
|
bool SkipCoverageMapping;
|
2014-01-07 06:27:43 +08:00
|
|
|
|
|
|
|
public:
|
|
|
|
CodeGenPGO(CodeGenModule &CGM)
|
2014-12-09 03:04:51 +08:00
|
|
|
: CGM(CGM), NumRegionCounters(0), FunctionHash(0), CurrentRegionCount(0),
|
2014-08-05 02:41:51 +08:00
|
|
|
SkipCoverageMapping(false) {}
|
2014-01-07 06:27:43 +08:00
|
|
|
|
|
|
|
/// Whether or not we have PGO region data for the current function. This is
|
|
|
|
/// false both when we have no data at all and when our data has been
|
|
|
|
/// discarded.
|
2014-12-03 06:38:52 +08:00
|
|
|
bool haveRegionCounts() const { return !RegionCounts.empty(); }
|
2014-01-07 06:27:43 +08:00
|
|
|
|
|
|
|
/// Return the counter value of the current region.
|
|
|
|
uint64_t getCurrentRegionCount() const { return CurrentRegionCount; }
|
Change PGO instrumentation to compute counts in a separate AST traversal.
Previously, we made one traversal of the AST prior to codegen to assign
counters to the ASTs and then propagated the count values during codegen. This
patch now adds a separate AST traversal prior to codegen for the
-fprofile-instr-use option to propagate the count values. The counts are then
saved in a map from which they can be retrieved during codegen.
This new approach has several advantages:
1. It gets rid of a lot of extra PGO-related code that had previously been
added to codegen.
2. It fixes a serious bug. My original implementation (which was mailed to the
list but never committed) used 3 counters for every loop. Justin improved it to
move 2 of those counters into the less-frequently executed breaks and continues,
but that turned out to produce wrong count values in some cases. The solution
requires visiting a loop body before the condition so that the count for the
condition properly includes the break and continue counts. Changing codegen to
visit a loop body first would be a fairly invasive change, but with a separate
AST traversal, it is easy to control the order of traversal. I've added a
testcase (provided by Justin) to make sure this works correctly.
3. It improves the instrumentation overhead, reducing the number of counters for
a loop from 3 to 1. We no longer need dedicated counters for breaks and
continues, since we can just use the propagated count values when visiting
breaks and continues.
To make this work, I needed to make a change to the way we count case
statements, going back to my original approach of not including the fall-through
in the counter values. This was necessary because there isn't always an AST node
that can be used to record the fall-through count. Now case statements are
handled the same as default statements, with the fall-through paths branching
over the counter increments. While I was at it, I also went back to using this
approach for do-loops -- omitting the fall-through count into the loop body
simplifies some of the calculations and make them behave the same as other
loops. Whenever we start using this instrumentation for coverage, we'll need
to add the fall-through counts into the counter values.
llvm-svn: 201528
2014-02-18 03:21:09 +08:00
|
|
|
|
2014-01-07 06:27:43 +08:00
|
|
|
/// Set the counter value for the current region. This is used to keep track
|
|
|
|
/// of changes to the most recent counter from control flow and non-local
|
|
|
|
/// exits.
|
|
|
|
void setCurrentRegionCount(uint64_t Count) { CurrentRegionCount = Count; }
|
Change PGO instrumentation to compute counts in a separate AST traversal.
Previously, we made one traversal of the AST prior to codegen to assign
counters to the ASTs and then propagated the count values during codegen. This
patch now adds a separate AST traversal prior to codegen for the
-fprofile-instr-use option to propagate the count values. The counts are then
saved in a map from which they can be retrieved during codegen.
This new approach has several advantages:
1. It gets rid of a lot of extra PGO-related code that had previously been
added to codegen.
2. It fixes a serious bug. My original implementation (which was mailed to the
list but never committed) used 3 counters for every loop. Justin improved it to
move 2 of those counters into the less-frequently executed breaks and continues,
but that turned out to produce wrong count values in some cases. The solution
requires visiting a loop body before the condition so that the count for the
condition properly includes the break and continue counts. Changing codegen to
visit a loop body first would be a fairly invasive change, but with a separate
AST traversal, it is easy to control the order of traversal. I've added a
testcase (provided by Justin) to make sure this works correctly.
3. It improves the instrumentation overhead, reducing the number of counters for
a loop from 3 to 1. We no longer need dedicated counters for breaks and
continues, since we can just use the propagated count values when visiting
breaks and continues.
To make this work, I needed to make a change to the way we count case
statements, going back to my original approach of not including the fall-through
in the counter values. This was necessary because there isn't always an AST node
that can be used to record the fall-through count. Now case statements are
handled the same as default statements, with the fall-through paths branching
over the counter increments. While I was at it, I also went back to using this
approach for do-loops -- omitting the fall-through count into the loop body
simplifies some of the calculations and make them behave the same as other
loops. Whenever we start using this instrumentation for coverage, we'll need
to add the fall-through counts into the counter values.
llvm-svn: 201528
2014-02-18 03:21:09 +08:00
|
|
|
|
|
|
|
/// Check if an execution count is known for a given statement. If so, return
|
|
|
|
/// true and put the value in Count; else return false.
|
2015-04-13 20:23:19 +08:00
|
|
|
Optional<uint64_t> getStmtCount(const Stmt *S) {
|
Change PGO instrumentation to compute counts in a separate AST traversal.
Previously, we made one traversal of the AST prior to codegen to assign
counters to the ASTs and then propagated the count values during codegen. This
patch now adds a separate AST traversal prior to codegen for the
-fprofile-instr-use option to propagate the count values. The counts are then
saved in a map from which they can be retrieved during codegen.
This new approach has several advantages:
1. It gets rid of a lot of extra PGO-related code that had previously been
added to codegen.
2. It fixes a serious bug. My original implementation (which was mailed to the
list but never committed) used 3 counters for every loop. Justin improved it to
move 2 of those counters into the less-frequently executed breaks and continues,
but that turned out to produce wrong count values in some cases. The solution
requires visiting a loop body before the condition so that the count for the
condition properly includes the break and continue counts. Changing codegen to
visit a loop body first would be a fairly invasive change, but with a separate
AST traversal, it is easy to control the order of traversal. I've added a
testcase (provided by Justin) to make sure this works correctly.
3. It improves the instrumentation overhead, reducing the number of counters for
a loop from 3 to 1. We no longer need dedicated counters for breaks and
continues, since we can just use the propagated count values when visiting
breaks and continues.
To make this work, I needed to make a change to the way we count case
statements, going back to my original approach of not including the fall-through
in the counter values. This was necessary because there isn't always an AST node
that can be used to record the fall-through count. Now case statements are
handled the same as default statements, with the fall-through paths branching
over the counter increments. While I was at it, I also went back to using this
approach for do-loops -- omitting the fall-through count into the loop body
simplifies some of the calculations and make them behave the same as other
loops. Whenever we start using this instrumentation for coverage, we'll need
to add the fall-through counts into the counter values.
llvm-svn: 201528
2014-02-18 03:21:09 +08:00
|
|
|
if (!StmtCountMap)
|
2015-04-13 20:23:19 +08:00
|
|
|
return None;
|
|
|
|
auto I = StmtCountMap->find(S);
|
Change PGO instrumentation to compute counts in a separate AST traversal.
Previously, we made one traversal of the AST prior to codegen to assign
counters to the ASTs and then propagated the count values during codegen. This
patch now adds a separate AST traversal prior to codegen for the
-fprofile-instr-use option to propagate the count values. The counts are then
saved in a map from which they can be retrieved during codegen.
This new approach has several advantages:
1. It gets rid of a lot of extra PGO-related code that had previously been
added to codegen.
2. It fixes a serious bug. My original implementation (which was mailed to the
list but never committed) used 3 counters for every loop. Justin improved it to
move 2 of those counters into the less-frequently executed breaks and continues,
but that turned out to produce wrong count values in some cases. The solution
requires visiting a loop body before the condition so that the count for the
condition properly includes the break and continue counts. Changing codegen to
visit a loop body first would be a fairly invasive change, but with a separate
AST traversal, it is easy to control the order of traversal. I've added a
testcase (provided by Justin) to make sure this works correctly.
3. It improves the instrumentation overhead, reducing the number of counters for
a loop from 3 to 1. We no longer need dedicated counters for breaks and
continues, since we can just use the propagated count values when visiting
breaks and continues.
To make this work, I needed to make a change to the way we count case
statements, going back to my original approach of not including the fall-through
in the counter values. This was necessary because there isn't always an AST node
that can be used to record the fall-through count. Now case statements are
handled the same as default statements, with the fall-through paths branching
over the counter increments. While I was at it, I also went back to using this
approach for do-loops -- omitting the fall-through count into the loop body
simplifies some of the calculations and make them behave the same as other
loops. Whenever we start using this instrumentation for coverage, we'll need
to add the fall-through counts into the counter values.
llvm-svn: 201528
2014-02-18 03:21:09 +08:00
|
|
|
if (I == StmtCountMap->end())
|
2015-04-13 20:23:19 +08:00
|
|
|
return None;
|
|
|
|
return I->second;
|
Change PGO instrumentation to compute counts in a separate AST traversal.
Previously, we made one traversal of the AST prior to codegen to assign
counters to the ASTs and then propagated the count values during codegen. This
patch now adds a separate AST traversal prior to codegen for the
-fprofile-instr-use option to propagate the count values. The counts are then
saved in a map from which they can be retrieved during codegen.
This new approach has several advantages:
1. It gets rid of a lot of extra PGO-related code that had previously been
added to codegen.
2. It fixes a serious bug. My original implementation (which was mailed to the
list but never committed) used 3 counters for every loop. Justin improved it to
move 2 of those counters into the less-frequently executed breaks and continues,
but that turned out to produce wrong count values in some cases. The solution
requires visiting a loop body before the condition so that the count for the
condition properly includes the break and continue counts. Changing codegen to
visit a loop body first would be a fairly invasive change, but with a separate
AST traversal, it is easy to control the order of traversal. I've added a
testcase (provided by Justin) to make sure this works correctly.
3. It improves the instrumentation overhead, reducing the number of counters for
a loop from 3 to 1. We no longer need dedicated counters for breaks and
continues, since we can just use the propagated count values when visiting
breaks and continues.
To make this work, I needed to make a change to the way we count case
statements, going back to my original approach of not including the fall-through
in the counter values. This was necessary because there isn't always an AST node
that can be used to record the fall-through count. Now case statements are
handled the same as default statements, with the fall-through paths branching
over the counter increments. While I was at it, I also went back to using this
approach for do-loops -- omitting the fall-through count into the loop body
simplifies some of the calculations and make them behave the same as other
loops. Whenever we start using this instrumentation for coverage, we'll need
to add the fall-through counts into the counter values.
llvm-svn: 201528
2014-02-18 03:21:09 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/// If the execution count for the current statement is known, record that
|
|
|
|
/// as the current count.
|
|
|
|
void setCurrentStmt(const Stmt *S) {
|
2015-04-13 20:23:19 +08:00
|
|
|
if (auto Count = getStmtCount(S))
|
|
|
|
setCurrentRegionCount(*Count);
|
Change PGO instrumentation to compute counts in a separate AST traversal.
Previously, we made one traversal of the AST prior to codegen to assign
counters to the ASTs and then propagated the count values during codegen. This
patch now adds a separate AST traversal prior to codegen for the
-fprofile-instr-use option to propagate the count values. The counts are then
saved in a map from which they can be retrieved during codegen.
This new approach has several advantages:
1. It gets rid of a lot of extra PGO-related code that had previously been
added to codegen.
2. It fixes a serious bug. My original implementation (which was mailed to the
list but never committed) used 3 counters for every loop. Justin improved it to
move 2 of those counters into the less-frequently executed breaks and continues,
but that turned out to produce wrong count values in some cases. The solution
requires visiting a loop body before the condition so that the count for the
condition properly includes the break and continue counts. Changing codegen to
visit a loop body first would be a fairly invasive change, but with a separate
AST traversal, it is easy to control the order of traversal. I've added a
testcase (provided by Justin) to make sure this works correctly.
3. It improves the instrumentation overhead, reducing the number of counters for
a loop from 3 to 1. We no longer need dedicated counters for breaks and
continues, since we can just use the propagated count values when visiting
breaks and continues.
To make this work, I needed to make a change to the way we count case
statements, going back to my original approach of not including the fall-through
in the counter values. This was necessary because there isn't always an AST node
that can be used to record the fall-through count. Now case statements are
handled the same as default statements, with the fall-through paths branching
over the counter increments. While I was at it, I also went back to using this
approach for do-loops -- omitting the fall-through count into the loop body
simplifies some of the calculations and make them behave the same as other
loops. Whenever we start using this instrumentation for coverage, we'll need
to add the fall-through counts into the counter values.
llvm-svn: 201528
2014-02-18 03:21:09 +08:00
|
|
|
}
|
|
|
|
|
2014-08-05 02:41:51 +08:00
|
|
|
/// Check if we need to emit coverage mapping for a given declaration
|
|
|
|
void checkGlobalDecl(GlobalDecl GD);
|
2014-01-07 06:27:43 +08:00
|
|
|
/// Assign counters to regions and configure them for PGO of a given
|
|
|
|
/// function. Does nothing if instrumentation is not enabled and either
|
|
|
|
/// generates global variables or associates PGO data with each of the
|
|
|
|
/// counters depending on whether we are generating or using instrumentation.
|
2014-03-06 12:55:41 +08:00
|
|
|
void assignRegionCounters(const Decl *D, llvm::Function *Fn);
|
2014-08-05 02:41:51 +08:00
|
|
|
/// Emit a coverage mapping range with a counter zero
|
|
|
|
/// for an unused declaration.
|
|
|
|
void emitEmptyCounterMapping(const Decl *D, StringRef FuncName,
|
|
|
|
llvm::GlobalValue::LinkageTypes Linkage);
|
2014-01-07 06:27:43 +08:00
|
|
|
private:
|
2014-03-06 12:55:41 +08:00
|
|
|
void setFuncName(llvm::Function *Fn);
|
2014-08-05 02:41:51 +08:00
|
|
|
void setFuncName(StringRef Name, llvm::GlobalValue::LinkageTypes Linkage);
|
2014-12-09 03:04:51 +08:00
|
|
|
void createFuncNameVar(llvm::GlobalValue::LinkageTypes Linkage);
|
2014-01-07 06:27:43 +08:00
|
|
|
void mapRegionCounters(const Decl *D);
|
Change PGO instrumentation to compute counts in a separate AST traversal.
Previously, we made one traversal of the AST prior to codegen to assign
counters to the ASTs and then propagated the count values during codegen. This
patch now adds a separate AST traversal prior to codegen for the
-fprofile-instr-use option to propagate the count values. The counts are then
saved in a map from which they can be retrieved during codegen.
This new approach has several advantages:
1. It gets rid of a lot of extra PGO-related code that had previously been
added to codegen.
2. It fixes a serious bug. My original implementation (which was mailed to the
list but never committed) used 3 counters for every loop. Justin improved it to
move 2 of those counters into the less-frequently executed breaks and continues,
but that turned out to produce wrong count values in some cases. The solution
requires visiting a loop body before the condition so that the count for the
condition properly includes the break and continue counts. Changing codegen to
visit a loop body first would be a fairly invasive change, but with a separate
AST traversal, it is easy to control the order of traversal. I've added a
testcase (provided by Justin) to make sure this works correctly.
3. It improves the instrumentation overhead, reducing the number of counters for
a loop from 3 to 1. We no longer need dedicated counters for breaks and
continues, since we can just use the propagated count values when visiting
breaks and continues.
To make this work, I needed to make a change to the way we count case
statements, going back to my original approach of not including the fall-through
in the counter values. This was necessary because there isn't always an AST node
that can be used to record the fall-through count. Now case statements are
handled the same as default statements, with the fall-through paths branching
over the counter increments. While I was at it, I also went back to using this
approach for do-loops -- omitting the fall-through count into the loop body
simplifies some of the calculations and make them behave the same as other
loops. Whenever we start using this instrumentation for coverage, we'll need
to add the fall-through counts into the counter values.
llvm-svn: 201528
2014-02-18 03:21:09 +08:00
|
|
|
void computeRegionCounts(const Decl *D);
|
2014-04-19 05:52:00 +08:00
|
|
|
void applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader,
|
|
|
|
llvm::Function *Fn);
|
2014-06-26 09:45:07 +08:00
|
|
|
void loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader,
|
|
|
|
bool IsInMainFile);
|
2014-01-07 06:27:43 +08:00
|
|
|
void emitCounterVariables();
|
2014-08-05 02:41:51 +08:00
|
|
|
void emitCounterRegionMapping(const Decl *D);
|
2014-01-07 06:27:43 +08:00
|
|
|
|
2015-04-24 07:06:47 +08:00
|
|
|
public:
|
|
|
|
void emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S);
|
2014-01-07 06:27:43 +08:00
|
|
|
|
2015-04-24 07:06:47 +08:00
|
|
|
/// Return the region count for the counter at the given index.
|
|
|
|
uint64_t getRegionCount(const Stmt *S) {
|
2014-05-21 13:09:00 +08:00
|
|
|
if (!RegionCounterMap)
|
2014-01-07 06:27:43 +08:00
|
|
|
return 0;
|
|
|
|
if (!haveRegionCounts())
|
|
|
|
return 0;
|
2015-04-24 07:06:47 +08:00
|
|
|
return RegionCounts[(*RegionCounterMap)[S]];
|
2014-01-07 06:27:43 +08:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
} // end namespace CodeGen
|
|
|
|
} // end namespace clang
|
|
|
|
|
|
|
|
#endif
|