maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[BOLT][NFC] Move ICF pass into a separate file 

Summary:
Consolidate code used by identical code folding under
Passes/IdenticalCodeFolding.cpp.

(cherry picked from FBD8109916)
This commit is contained in:
Maksim Panchenko 2018-05-22 15:52:21 -07:00
parent 6302e18f94
commit 929b0908f7
10 changed files with 495 additions and 436 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bolt/src/BinaryBasicBlock.h
View File

@ -825,9 +825,9 @@ public:
LayoutIndex = Index;
}
/// FIXME
/// Needed by graph traits.
BinaryFunction *getParent() const {
return nullptr;
return getFunction();
}
private:

170
bolt/src/BinaryFunction.cpp
View File

@ -2981,136 +2981,6 @@ BinaryFunction::BasicBlockOrderType BinaryFunction::dfs() const {
return DFS;
}
bool BinaryFunction::isIdenticalWith(const BinaryFunction &OtherBF,
bool IgnoreSymbols,
bool UseDFS) const {
assert(hasCFG() && OtherBF.hasCFG() && "both functions should have CFG");
// Compare the two functions, one basic block at a time.
// Currently we require two identical basic blocks to have identical
// instruction sequences and the same index in their corresponding
// functions. The latter is important for CFG equality.
if (layout_size() != OtherBF.layout_size())
return false;
// Comparing multi-entry functions could be non-trivial.
if (isMultiEntry() || OtherBF.isMultiEntry())
return false;
// Process both functions in either DFS or existing order.
const auto &Order = UseDFS ? dfs() : BasicBlocksLayout;
const auto &OtherOrder = UseDFS ? OtherBF.dfs() : OtherBF.BasicBlocksLayout;
auto BBI = OtherOrder.begin();
for (const auto *BB : Order) {
const auto *OtherBB = *BBI;
if (BB->getLayoutIndex() != OtherBB->getLayoutIndex())
return false;
// Compare successor basic blocks.
// NOTE: the comparison for jump tables is only partially verified here.
if (BB->succ_size() != OtherBB->succ_size())
return false;
auto SuccBBI = OtherBB->succ_begin();
for (const auto *SuccBB : BB->successors()) {
const auto *SuccOtherBB = *SuccBBI;
if (SuccBB->getLayoutIndex() != SuccOtherBB->getLayoutIndex())
return false;
++SuccBBI;
}
// Compare all instructions including pseudos.
auto I = BB->begin(), E = BB->end();
auto OtherI = OtherBB->begin(), OtherE = OtherBB->end();
while (I != E && OtherI != OtherE) {
bool Identical;
if (IgnoreSymbols) {
Identical =
isInstrEquivalentWith(*I, *BB, *OtherI, *OtherBB, OtherBF,
[](const MCSymbol *A, const MCSymbol *B) {
return true;
});
} else {
// Compare symbols.
auto AreSymbolsIdentical = [&] (const MCSymbol *A, const MCSymbol *B) {
if (A == B)
return true;
// All local symbols are considered identical since they affect a
// control flow and we check the control flow separately.
// If a local symbol is escaped, then the function (potentially) has
// multiple entry points and we exclude such functions from
// comparison.
if (A->isTemporary() && B->isTemporary())
return true;
// Compare symbols as functions.
const auto *FunctionA = BC.getFunctionForSymbol(A);
const auto *FunctionB = BC.getFunctionForSymbol(B);
if (FunctionA && FunctionB) {
// Self-referencing functions and recursive calls.
if (FunctionA == this && FunctionB == &OtherBF)
return true;
return FunctionA == FunctionB;
}
// Check if symbols are jump tables.
auto *SIA = BC.getBinaryDataByName(A->getName());
if (!SIA)
return false;
auto *SIB = BC.getBinaryDataByName(B->getName());
if (!SIB)
return false;
assert((SIA->getAddress() != SIB->getAddress()) &&
"different symbols should not have the same value");
const auto *JumpTableA =
getJumpTableContainingAddress(SIA->getAddress());
if (!JumpTableA)
return false;
const auto *JumpTableB =
OtherBF.getJumpTableContainingAddress(SIB->getAddress());
if (!JumpTableB)
return false;
if ((SIA->getAddress() - JumpTableA->getAddress()) !=
(SIB->getAddress() - JumpTableB->getAddress()))
return false;
return equalJumpTables(JumpTableA, JumpTableB, OtherBF);
};
Identical =
isInstrEquivalentWith(*I, *BB, *OtherI, *OtherBB, OtherBF,
AreSymbolsIdentical);
}
if (!Identical)
return false;
++I; ++OtherI;
}
// One of the identical blocks may have a trailing unconditional jump that
// is ignored for CFG purposes.
auto *TrailingInstr = (I != E ? &(*I)
: (OtherI != OtherE ? &(*OtherI) : 0));
if (TrailingInstr && !BC.MIB->isUnconditionalBranch(*TrailingInstr)) {
return false;
}
++BBI;
}
return true;
}
std::string BinaryFunction::generateJumpTableName(uint64_t Address) const {
auto *JT = getJumpTableContainingAddress(Address);
size_t Id;
@ -3129,46 +2999,6 @@ std::string BinaryFunction::generateJumpTableName(uint64_t Address) const {
(Offset ? ("." + std::to_string(Offset)) : ""));
}
bool BinaryFunction::equalJumpTables(const JumpTable *JumpTableA,
const JumpTable *JumpTableB,
const BinaryFunction &BFB) const {
if (JumpTableA->EntrySize != JumpTableB->EntrySize)
return false;
if (JumpTableA->Type != JumpTableB->Type)
return false;
if (JumpTableA->getSize() != JumpTableB->getSize())
return false;
for (uint64_t Index = 0; Index < JumpTableA->Entries.size(); ++Index) {
const auto *LabelA = JumpTableA->Entries[Index];
const auto *LabelB = JumpTableB->Entries[Index];
const auto *TargetA = getBasicBlockForLabel(LabelA);
const auto *TargetB = BFB.getBasicBlockForLabel(LabelB);
if (!TargetA || !TargetB) {
assert((TargetA || LabelA == getFunctionEndLabel()) &&
"no target basic block found");
assert((TargetB || LabelB == BFB.getFunctionEndLabel()) &&
"no target basic block found");
if (TargetA != TargetB)
return false;
continue;
}
assert(TargetA && TargetB && "cannot locate target block(s)");
if (TargetA->getLayoutIndex() != TargetB->getLayoutIndex())
return false;
}
return true;
}
std::size_t BinaryFunction::hash(bool Recompute, bool UseDFS) const {
if (size() == 0)
return 0;

123
bolt/src/BinaryFunction.h
View File

@ -424,60 +424,6 @@ private:
/// Synchronize branch instructions with CFG.
void postProcessBranches();
/// Helper function that compares an instruction of this function to the
/// given instruction of the given function. The functions should have
/// identical CFG.
template <class Compare>
bool isInstrEquivalentWith(
const MCInst &InstA, const BinaryBasicBlock &BBA,
const MCInst &InstB, const BinaryBasicBlock &BBB,
const BinaryFunction &BFB, Compare Comp) const {
if (InstA.getOpcode() != InstB.getOpcode()) {
return false;
}
// In this function we check for special conditions:
//
// * instructions with landing pads
//
// Most of the common cases should be handled by MCPlus::equals()
// that compares regular instruction operands.
//
// NB: there's no need to compare jump table indirect jump instructions
// separately as jump tables are handled by comparing corresponding
// symbols.
const auto EHInfoA = BC.MIB->getEHInfo(InstA);
const auto EHInfoB = BC.MIB->getEHInfo(InstB);
if (EHInfoA || EHInfoB) {
if (!EHInfoA && (EHInfoB->first || EHInfoB->second))
return false;
if (!EHInfoB && (EHInfoA->first || EHInfoA->second))
return false;
if (EHInfoA && EHInfoB) {
// Action indices should match.
if (EHInfoA->second != EHInfoB->second)
return false;
if (!EHInfoA->first != !EHInfoB->first)
return false;
if (EHInfoA->first && EHInfoB->first) {
const auto *LPA = BBA.getLandingPad(EHInfoA->first);
const auto *LPB = BBB.getLandingPad(EHInfoB->first);
assert(LPA && LPB && "cannot locate landing pad(s)");
if (LPA->getLayoutIndex() != LPB->getLayoutIndex())
return false;
}
}
}
return BC.MIB->equals(InstA, InstB, Comp);
}
/// Recompute landing pad information for the function and all its blocks.
void recomputeLandingPads();
@ -583,45 +529,16 @@ private:
/// jump table names.
mutable std::map<uint64_t, size_t> JumpTableIds;
/// Generate a unique name for this jump table at the given address that should
/// be repeatable no matter what the start address of the table is.
/// Generate a unique name for this jump table at the given address that
/// should be repeatable no matter what the start address of the table is.
std::string generateJumpTableName(uint64_t Address) const;
/// Return jump table that covers a given \p Address in memory.
JumpTable *getJumpTableContainingAddress(uint64_t Address) {
auto JTI = JumpTables.upper_bound(Address);
if (JTI == JumpTables.begin())
return nullptr;
--JTI;
if (JTI->first + JTI->second->getSize() > Address) {
return JTI->second;
}
return nullptr;
}
const JumpTable *getJumpTableContainingAddress(uint64_t Address) const {
auto JTI = JumpTables.upper_bound(Address);
if (JTI == JumpTables.begin())
return nullptr;
--JTI;
if (JTI->first + JTI->second->getSize() > Address) {
return JTI->second;
}
return nullptr;
}
/// Iterate over all jump tables associated with this function.
iterator_range<std::map<uint64_t, JumpTable *>::const_iterator>
jumpTables() const {
return make_range(JumpTables.begin(), JumpTables.end());
}
/// Compare two jump tables in 2 functions. The function relies on consistent
/// ordering of basic blocks in both binary functions (e.g. DFS).
bool equalJumpTables(const JumpTable *JumpTableA,
const JumpTable *JumpTableB,
const BinaryFunction &BFB) const;
/// All jump table sites in the function.
std::vector<std::pair<uint64_t, uint64_t>> JTSites;
@ -767,7 +684,6 @@ private:
}
public:
BinaryFunction(BinaryFunction &&) = default;
using iterator = pointee_iterator<BasicBlockListType::iterator>;
@ -872,6 +788,11 @@ public:
}
}
/// Return current basic block layout.
const BasicBlockOrderType &getLayout() const {
return BasicBlocksLayout;
}
/// Return a list of basic blocks sorted using DFS and update layout indices
/// using the same order. Does not modify the current layout.
BasicBlockOrderType dfs() const;
@ -985,6 +906,23 @@ public:
/// CFG is constructed or while instruction offsets are available in CFG.
MCInst *getInstructionAtOffset(uint64_t Offset);
/// Return jump table that covers a given \p Address in memory.
JumpTable *getJumpTableContainingAddress(uint64_t Address) {
auto JTI = JumpTables.upper_bound(Address);
if (JTI == JumpTables.begin())
return nullptr;
--JTI;
if (JTI->first + JTI->second->getSize() > Address) {
return JTI->second;
}
return nullptr;
}
const JumpTable *getJumpTableContainingAddress(uint64_t Address) const {
return const_cast<BinaryFunction *>(this)->
getJumpTableContainingAddress(Address);
}
/// Return the name of the function as extracted from the binary file.
/// If the function has multiple names - return the last one
/// followed by "(*#<numnames>)".
@ -2125,19 +2063,6 @@ public:
/// Convert function-level branch data into instruction annotations.
void convertBranchData();
/// Returns true if this function has identical code and CFG with
/// the given function \p BF.
///
/// If \p IgnoreSymbols is set to true, then symbolic operands are ignored
/// during comparison.
///
/// If \p UseDFS is set to true, then compute DFS of each function and use
/// is for CFG equivalency. Potentially it will help to catch more cases,
/// but is slower.
bool isIdenticalWith(const BinaryFunction &BF,
bool IgnoreSymbols = false,
bool UseDFS = false) const;
/// Returns a hash value for the function. To be used for ICF. Two congruent
/// functions (functions with different symbolic references but identical
/// otherwise) are required to have identical hashes.

1
bolt/src/BinaryPassManager.cpp
View File

@ -13,6 +13,7 @@
#include "Passes/Aligner.h"
#include "Passes/AllocCombiner.h"
#include "Passes/FrameOptimizer.h"
#include "Passes/IdenticalCodeFolding.h"
#include "Passes/IndirectCallPromotion.h"
#include "Passes/Inliner.h"
#include "Passes/LongJmp.h"

2
bolt/src/BoltDiff.cpp
View File

@ -13,7 +13,7 @@
//===----------------------------------------------------------------------===//
#include "RewriteInstance.h"
#include "Passes/BinaryPasses.h"
#include "Passes/IdenticalCodeFolding.h"
#include "llvm/Support/CommandLine.h"
#undef DEBUG_TYPE

148
bolt/src/Passes/BinaryPasses.cpp
View File

@ -75,13 +75,6 @@ DynoStatsSortOrderOpt("print-sorted-by-order",
cl::init(DynoStatsSortOrder::Descending),
cl::cat(BoltOptCategory));
static cl::opt<bool>
ICFUseDFS("icf-dfs",
cl::desc("use DFS ordering when using -icf option"),
cl::ReallyHidden,
cl::ZeroOrMore,
cl::cat(BoltOptCategory));
static cl::opt<bool>
MinBranchClusters("min-branch-clusters",
cl::desc("use a modified clustering algorithm geared towards minimizing "
@ -1259,147 +1252,6 @@ void SimplifyRODataLoads::runOnFunctions(
<< "BOLT-INFO: dynamic loads found: " << NumDynamicLoadsFound << "\n";
}
void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC,
std::map<uint64_t, BinaryFunction> &BFs,
std::set<uint64_t> &) {
const auto OriginalFunctionCount = BFs.size();
uint64_t NumFunctionsFolded = 0;
uint64_t NumJTFunctionsFolded = 0;
uint64_t BytesSavedEstimate = 0;
uint64_t CallsSavedEstimate = 0;
static bool UseDFS = opts::ICFUseDFS;
// This hash table is used to identify identical functions. It maps
// a function to a bucket of functions identical to it.
struct KeyHash {
std::size_t operator()(const BinaryFunction *F) const {
return F->hash(/*Recompute=*/false);
}
};
struct KeyCongruent {
bool operator()(const BinaryFunction *A, const BinaryFunction *B) const {
return A->isIdenticalWith(*B, /*IgnoreSymbols=*/true, /*UseDFS=*/UseDFS);
}
};
struct KeyEqual {
bool operator()(const BinaryFunction *A, const BinaryFunction *B) const {
return A->isIdenticalWith(*B, /*IgnoreSymbols=*/false, /*UseDFS=*/UseDFS);
}
};
// Create buckets with congruent functions - functions that potentially could
// be folded.
std::unordered_map<BinaryFunction *, std::set<BinaryFunction *>,
KeyHash, KeyCongruent> CongruentBuckets;
for (auto &BFI : BFs) {
auto &BF = BFI.second;
if (!shouldOptimize(BF) || BF.isFolded())
continue;
// Make sure indices are in-order.
BF.updateLayoutIndices();
// Pre-compute hash before pushing into hashtable.
BF.hash(/*Recompute=*/true, /*UseDFS=*/UseDFS);
CongruentBuckets[&BF].emplace(&BF);
}
// We repeat the pass until no new modifications happen.
unsigned Iteration = 1;
uint64_t NumFoldedLastIteration;
do {
NumFoldedLastIteration = 0;
DEBUG(dbgs() << "BOLT-DEBUG: ICF iteration " << Iteration << "...\n");
for (auto &CBI : CongruentBuckets) {
auto &Candidates = CBI.second;
if (Candidates.size() < 2)
continue;
// Identical functions go into the same bucket.
std::unordered_map<BinaryFunction *, std::vector<BinaryFunction *>,
KeyHash, KeyEqual> IdenticalBuckets;
for (auto *BF : Candidates) {
IdenticalBuckets[BF].emplace_back(BF);
}
for (auto &IBI : IdenticalBuckets) {
// Functions identified as identical.
auto &Twins = IBI.second;
if (Twins.size() < 2)
continue;
// Fold functions. Keep the order consistent across invocations with
// different options.
std::stable_sort(Twins.begin(), Twins.end(),
[](const BinaryFunction *A, const BinaryFunction *B) {
return A->getFunctionNumber() < B->getFunctionNumber();
});
BinaryFunction *ParentBF = Twins[0];
for (unsigned i = 1; i < Twins.size(); ++i) {
auto *ChildBF = Twins[i];
DEBUG(dbgs() << "BOLT-DEBUG: folding " << *ChildBF << " into "
<< *ParentBF << '\n');
// Remove child function from the list of candidates.
auto FI = Candidates.find(ChildBF);
assert(FI != Candidates.end() &&
"function expected to be in the set");
Candidates.erase(FI);
// Fold the function and remove from the list of processed functions.
BytesSavedEstimate += ChildBF->getSize();
CallsSavedEstimate += std::min(ChildBF->getKnownExecutionCount(),
ParentBF->getKnownExecutionCount());
BC.foldFunction(*ChildBF, *ParentBF, BFs);
++NumFoldedLastIteration;
if (ParentBF->hasJumpTables())
++NumJTFunctionsFolded;
}
}
}
NumFunctionsFolded += NumFoldedLastIteration;
++Iteration;
} while (NumFoldedLastIteration > 0);
DEBUG(
// Print functions that are congruent but not identical.
for (auto &CBI : CongruentBuckets) {
auto &Candidates = CBI.second;
if (Candidates.size() < 2)
continue;
dbgs() << "BOLT-DEBUG: the following " << Candidates.size()
<< " functions (each of size " << (*Candidates.begin())->getSize()
<< " bytes) are congruent but not identical:\n";
for (auto *BF : Candidates) {
dbgs() << " " << *BF;
if (BF->getKnownExecutionCount()) {
dbgs() << " (executed " << BF->getKnownExecutionCount() << " times)";
}
dbgs() << '\n';
}
}
);
if (NumFunctionsFolded) {
outs() << "BOLT-INFO: ICF folded " << NumFunctionsFolded
<< " out of " << OriginalFunctionCount << " functions in "
<< Iteration << " passes. "
<< NumJTFunctionsFolded << " functions had jump tables.\n"
<< "BOLT-INFO: Removing all identical functions will save "
<< format("%.2lf", (double) BytesSavedEstimate / 1024)
<< " KB of code space. Folded functions were called "
<< CallsSavedEstimate << " times based on profile.\n";
}
}
void
PrintProgramStats::runOnFunctions(BinaryContext &BC,
std::map<uint64_t, BinaryFunction> &BFs,

16
bolt/src/Passes/BinaryPasses.h
View File

@ -384,22 +384,6 @@ public:
std::set<uint64_t> &LargeFunctions) override;
};
/// An optimization that replaces references to identical functions with
/// references to a single one of them.
///
class IdenticalCodeFolding : public BinaryFunctionPass {
public:
explicit IdenticalCodeFolding(const cl::opt<bool> &PrintPass)
: BinaryFunctionPass(PrintPass) { }
const char *getName() const override {
return "identical-code-folding";
}
void runOnFunctions(BinaryContext &BC,
std::map<uint64_t, BinaryFunction> &BFs,
std::set<uint64_t> &LargeFunctions) override;
};
/// Prints a list of the top 100 functions sorted by a set of
/// dyno stats categories.
class PrintProgramStats : public BinaryFunctionPass {

1
bolt/src/Passes/CMakeLists.txt
View File

@ -12,6 +12,7 @@ add_llvm_library(LLVMBOLTPasses
FrameOptimizer.cpp
HFSort.cpp
HFSortPlus.cpp
IdenticalCodeFolding.cpp
IndirectCallPromotion.cpp
Inliner.cpp
JTFootprintReduction.cpp

425
bolt/src/Passes/IdenticalCodeFolding.cpp Normal file
View File

@ -0,0 +1,425 @@
//===--- IdenticalCodeFolding.cpp -----------------------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
//===----------------------------------------------------------------------===//
#include "Passes/IdenticalCodeFolding.h"
#include "llvm/Support/Options.h"
#include <map>
#include <set>
#include <unordered_map>
#define DEBUG_TYPE "bolt-icf"
using namespace llvm;
using namespace bolt;
namespace opts {
extern cl::OptionCategory BoltOptCategory;
static cl::opt<bool>
UseDFS("icf-dfs",
cl::desc("use DFS ordering when using -icf option"),
cl::ReallyHidden,
cl::ZeroOrMore,
cl::cat(BoltOptCategory));
} // namespace opts
namespace {
/// Compare two jump tables in 2 functions. The function relies on consistent
/// ordering of basic blocks in both binary functions (e.g. DFS).
bool equalJumpTables(const JumpTable &JumpTableA,
const JumpTable &JumpTableB,
const BinaryFunction &FunctionA,
const BinaryFunction &FunctionB) {
if (JumpTableA.EntrySize != JumpTableB.EntrySize)
return false;
if (JumpTableA.Type != JumpTableB.Type)
return false;
if (JumpTableA.getSize() != JumpTableB.getSize())
return false;
for (uint64_t Index = 0; Index < JumpTableA.Entries.size(); ++Index) {
const auto *LabelA = JumpTableA.Entries[Index];
const auto *LabelB = JumpTableB.Entries[Index];
const auto *TargetA = FunctionA.getBasicBlockForLabel(LabelA);
const auto *TargetB = FunctionB.getBasicBlockForLabel(LabelB);
if (!TargetA || !TargetB) {
assert((TargetA || LabelA == FunctionA.getFunctionEndLabel()) &&
"no target basic block found");
assert((TargetB || LabelB == FunctionB.getFunctionEndLabel()) &&
"no target basic block found");
if (TargetA != TargetB)
return false;
continue;
}
assert(TargetA && TargetB && "cannot locate target block(s)");
if (TargetA->getLayoutIndex() != TargetB->getLayoutIndex())
return false;
}
return true;
}
/// Helper function that compares an instruction of this function to the
/// given instruction of the given function. The functions should have
/// identical CFG.
template <class Compare>
bool isInstrEquivalentWith(const MCInst &InstA, const BinaryBasicBlock &BBA,
const MCInst &InstB, const BinaryBasicBlock &BBB,
Compare Comp) {
if (InstA.getOpcode() != InstB.getOpcode()) {
return false;
}
const auto &BC = BBA.getFunction()->getBinaryContext();
// In this function we check for special conditions:
//
// * instructions with landing pads
//
// Most of the common cases should be handled by MCPlus::equals()
// that compares regular instruction operands.
//
// NB: there's no need to compare jump table indirect jump instructions
// separately as jump tables are handled by comparing corresponding
// symbols.
const auto EHInfoA = BC.MIB->getEHInfo(InstA);
const auto EHInfoB = BC.MIB->getEHInfo(InstB);
if (EHInfoA || EHInfoB) {
if (!EHInfoA && (EHInfoB->first || EHInfoB->second))
return false;
if (!EHInfoB && (EHInfoA->first || EHInfoA->second))
return false;
if (EHInfoA && EHInfoB) {
// Action indices should match.
if (EHInfoA->second != EHInfoB->second)
return false;
if (!EHInfoA->first != !EHInfoB->first)
return false;
if (EHInfoA->first && EHInfoB->first) {
const auto *LPA = BBA.getLandingPad(EHInfoA->first);
const auto *LPB = BBB.getLandingPad(EHInfoB->first);
assert(LPA && LPB && "cannot locate landing pad(s)");
if (LPA->getLayoutIndex() != LPB->getLayoutIndex())
return false;
}
}
}
return BC.MIB->equals(InstA, InstB, Comp);
}
/// Returns true if this function has identical code and CFG with
/// the given function \p BF.
///
/// If \p IgnoreSymbols is set to true, then symbolic operands are ignored
/// during comparison.
///
/// If \p UseDFS is set to true, then compute DFS of each function and use
/// is for CFG equivalency. Potentially it will help to catch more cases,
/// but is slower.
bool isIdenticalWith(const BinaryFunction &A, const BinaryFunction &B,
bool IgnoreSymbols, bool UseDFS) {
assert(A.hasCFG() && B.hasCFG() && "both functions should have CFG");
// Compare the two functions, one basic block at a time.
// Currently we require two identical basic blocks to have identical
// instruction sequences and the same index in their corresponding
// functions. The latter is important for CFG equality.
if (A.layout_size() != B.layout_size())
return false;
// Comparing multi-entry functions could be non-trivial.
if (A.isMultiEntry() || B.isMultiEntry())
return false;
// Process both functions in either DFS or existing order.
const auto &OrderA = UseDFS ? A.dfs() : A.getLayout();
const auto &OrderB = UseDFS ? B.dfs() : B.getLayout();
const auto &BC = A.getBinaryContext();
auto BBI = OrderB.begin();
for (const auto *BB : OrderA) {
const auto *OtherBB = *BBI;
if (BB->getLayoutIndex() != OtherBB->getLayoutIndex())
return false;
// Compare successor basic blocks.
// NOTE: the comparison for jump tables is only partially verified here.
if (BB->succ_size() != OtherBB->succ_size())
return false;
auto SuccBBI = OtherBB->succ_begin();
for (const auto *SuccBB : BB->successors()) {
const auto *SuccOtherBB = *SuccBBI;
if (SuccBB->getLayoutIndex() != SuccOtherBB->getLayoutIndex())
return false;
++SuccBBI;
}
// Compare all instructions including pseudos.
auto I = BB->begin(), E = BB->end();
auto OtherI = OtherBB->begin(), OtherE = OtherBB->end();
while (I != E && OtherI != OtherE) {
bool Identical;
if (IgnoreSymbols) {
Identical =
isInstrEquivalentWith(*I, *BB, *OtherI, *OtherBB,
[](const MCSymbol *A, const MCSymbol *B) {
return true;
});
} else {
// Compare symbols.
auto AreSymbolsIdentical = [&] (const MCSymbol *SymbolA,
const MCSymbol *SymbolB) {
if (SymbolA == SymbolB)
return true;
// All local symbols are considered identical since they affect a
// control flow and we check the control flow separately.
// If a local symbol is escaped, then the function (potentially) has
// multiple entry points and we exclude such functions from
// comparison.
if (SymbolA->isTemporary() && SymbolB->isTemporary())
return true;
// Compare symbols as functions.
const auto *FunctionA = BC.getFunctionForSymbol(SymbolA);
const auto *FunctionB = BC.getFunctionForSymbol(SymbolB);
if (FunctionA && FunctionB) {
// Self-referencing functions and recursive calls.
if (FunctionA == &A && FunctionB == &B)
return true;
return FunctionA == FunctionB;
}
// Check if symbols are jump tables.
auto *SIA = BC.getBinaryDataByName(SymbolA->getName());
if (!SIA)
return false;
auto *SIB = BC.getBinaryDataByName(SymbolB->getName());
if (!SIB)
return false;
assert((SIA->getAddress() != SIB->getAddress()) &&
"different symbols should not have the same value");
const auto *JumpTableA =
A.getJumpTableContainingAddress(SIA->getAddress());
if (!JumpTableA)
return false;
const auto *JumpTableB =
B.getJumpTableContainingAddress(SIB->getAddress());
if (!JumpTableB)
return false;
if ((SIA->getAddress() - JumpTableA->getAddress()) !=
(SIB->getAddress() - JumpTableB->getAddress()))
return false;
return equalJumpTables(*JumpTableA, *JumpTableB, A, B);
};
Identical =
isInstrEquivalentWith(*I, *BB, *OtherI, *OtherBB,
AreSymbolsIdentical);
}
if (!Identical) {
return false;
}
++I; ++OtherI;
}
// One of the identical blocks may have a trailing unconditional jump that
// is ignored for CFG purposes.
auto *TrailingInstr = (I != E ? &(*I)
: (OtherI != OtherE ? &(*OtherI) : 0));
if (TrailingInstr && !BC.MIB->isUnconditionalBranch(*TrailingInstr)) {
return false;
}
++BBI;
}
return true;
}
}
namespace llvm {
namespace bolt {
void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC,
std::map<uint64_t, BinaryFunction> &BFs,
std::set<uint64_t> &) {
const auto OriginalFunctionCount = BFs.size();
uint64_t NumFunctionsFolded = 0;
uint64_t NumJTFunctionsFolded = 0;
uint64_t BytesSavedEstimate = 0;
uint64_t CallsSavedEstimate = 0;
// This hash table is used to identify identical functions. It maps
// a function to a bucket of functions identical to it.
struct KeyHash {
std::size_t operator()(const BinaryFunction *F) const {
return F->hash(/*Recompute=*/false);
}
};
struct KeyCongruent {
bool operator()(const BinaryFunction *A, const BinaryFunction *B) const {
return isIdenticalWith(*A, *B, /*IgnoreSymbols=*/true, opts::UseDFS);
}
};
struct KeyEqual {
bool operator()(const BinaryFunction *A, const BinaryFunction *B) const {
return isIdenticalWith(*A, *B, /*IgnoreSymbols=*/false, opts::UseDFS);
}
};
// Create buckets with congruent functions - functions that potentially could
// be folded.
std::unordered_map<BinaryFunction *, std::set<BinaryFunction *>,
KeyHash, KeyCongruent> CongruentBuckets;
for (auto &BFI : BFs) {
auto &BF = BFI.second;
if (!shouldOptimize(BF) || BF.isFolded())
continue;
// Make sure indices are in-order.
BF.updateLayoutIndices();
// Pre-compute hash before pushing into hashtable.
BF.hash(/*Recompute=*/true, opts::UseDFS);
CongruentBuckets[&BF].emplace(&BF);
}
// We repeat the pass until no new modifications happen.
unsigned Iteration = 1;
uint64_t NumFoldedLastIteration;
do {
NumFoldedLastIteration = 0;
DEBUG(dbgs() << "BOLT-DEBUG: ICF iteration " << Iteration << "...\n");
for (auto &CBI : CongruentBuckets) {
auto &Candidates = CBI.second;
if (Candidates.size() < 2)
continue;
// Identical functions go into the same bucket.
std::unordered_map<BinaryFunction *, std::vector<BinaryFunction *>,
KeyHash, KeyEqual> IdenticalBuckets;
for (auto *BF : Candidates) {
IdenticalBuckets[BF].emplace_back(BF);
}
for (auto &IBI : IdenticalBuckets) {
// Functions identified as identical.
auto &Twins = IBI.second;
if (Twins.size() < 2)
continue;
// Fold functions. Keep the order consistent across invocations with
// different options.
std::stable_sort(Twins.begin(), Twins.end(),
[](const BinaryFunction *A, const BinaryFunction *B) {
return A->getFunctionNumber() < B->getFunctionNumber();
});
BinaryFunction *ParentBF = Twins[0];
for (unsigned i = 1; i < Twins.size(); ++i) {
auto *ChildBF = Twins[i];
DEBUG(dbgs() << "BOLT-DEBUG: folding " << *ChildBF << " into "
<< *ParentBF << '\n');
// Remove child function from the list of candidates.
auto FI = Candidates.find(ChildBF);
assert(FI != Candidates.end() &&
"function expected to be in the set");
Candidates.erase(FI);
// Fold the function and remove from the list of processed functions.
BytesSavedEstimate += ChildBF->getSize();
CallsSavedEstimate += std::min(ChildBF->getKnownExecutionCount(),
ParentBF->getKnownExecutionCount());
BC.foldFunction(*ChildBF, *ParentBF, BFs);
++NumFoldedLastIteration;
if (ParentBF->hasJumpTables())
++NumJTFunctionsFolded;
}
}
}
NumFunctionsFolded += NumFoldedLastIteration;
++Iteration;
} while (NumFoldedLastIteration > 0);
DEBUG(
// Print functions that are congruent but not identical.
for (auto &CBI : CongruentBuckets) {
auto &Candidates = CBI.second;
if (Candidates.size() < 2)
continue;
dbgs() << "BOLT-DEBUG: the following " << Candidates.size()
<< " functions (each of size " << (*Candidates.begin())->getSize()
<< " bytes) are congruent but not identical:\n";
for (auto *BF : Candidates) {
dbgs() << " " << *BF;
if (BF->getKnownExecutionCount()) {
dbgs() << " (executed " << BF->getKnownExecutionCount() << " times)";
}
dbgs() << '\n';
}
}
);
if (NumFunctionsFolded) {
outs() << "BOLT-INFO: ICF folded " << NumFunctionsFolded
<< " out of " << OriginalFunctionCount << " functions in "
<< Iteration << " passes. "
<< NumJTFunctionsFolded << " functions had jump tables.\n"
<< "BOLT-INFO: Removing all identical functions will save "
<< format("%.2lf", (double) BytesSavedEstimate / 1024)
<< " KB of code space. Folded functions were called "
<< CallsSavedEstimate << " times based on profile.\n";
}
}
} // namespace bolt
} // namespace llvm

41
bolt/src/Passes/IdenticalCodeFolding.h Normal file
View File

@ -0,0 +1,41 @@
//===--- IdenticalCodeFolding.h -------------------------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_TOOLS_LLVM_BOLT_PASSES_IDENTICAL_CODE_FOLDING_H
#define LLVM_TOOLS_LLVM_BOLT_PASSES_IDENTICAL_CODE_FOLDING_H
#include "BinaryContext.h"
#include "BinaryFunction.h"
#include "Passes/BinaryPasses.h"
namespace llvm {
namespace bolt {
/// An optimization that replaces references to identical functions with
/// references to a single one of them.
///
class IdenticalCodeFolding : public BinaryFunctionPass {
public:
explicit IdenticalCodeFolding(const cl::opt<bool> &PrintPass)
: BinaryFunctionPass(PrintPass) { }
const char *getName() const override {
return "identical-code-folding";
}
void runOnFunctions(BinaryContext &BC,
std::map<uint64_t, BinaryFunction> &BFs,
std::set<uint64_t> &LargeFunctions) override;
};
} // namespace bolt
} // namespace llvm
#endif