maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

Rebase: [BOLT][NFC] Expand auto types 

Summary:
Expanded auto types across BOLT semi-automatically with the aid
of clangd LSP

(cherry picked from FBD33289309)
This commit is contained in:
Amir Ayupov 2021-04-08 00:19:26 -07:00 committed by Maksim Panchenko
parent dc2673a039
commit c7306cc219
96 changed files with 2931 additions and 2780 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
bolt/runtime/instr.cpp
View File

@ -294,8 +294,8 @@ private:
template <typename... Args>
void forEachElement(void (*Callback)(MapEntry &, Args...),
uint32_t NumEntries, MapEntry *Entries, Args... args) {
for (int I = 0; I < NumEntries; ++I) {
auto &Entry = Entries[I];
for (uint32_t I = 0; I < NumEntries; ++I) {
MapEntry &Entry = Entries[I];
if (Entry.Key == VacantMarker)
continue;
if (Entry.Key & FollowUpTableMarker) {
@ -311,7 +311,7 @@ private:
MapEntry &firstAllocation(uint64_t Key, BumpPtrAllocator &Alloc) {
TableRoot = new (Alloc, 0) MapEntry[InitialSize];
auto &Entry = TableRoot[Key % InitialSize];
MapEntry &Entry = TableRoot[Key % InitialSize];
Entry.Key = Key;
return Entry;
}
@ -321,7 +321,7 @@ private:
const uint32_t NumEntries = CurLevel == 0 ? InitialSize : IncSize;
uint64_t Remainder = Selector / NumEntries;
Selector = Selector % NumEntries;
auto &Entry = Entries[Selector];
MapEntry &Entry = Entries[Selector];
// A hit
if (Entry.Key == Key) {
@ -907,7 +907,7 @@ struct NodeToCallsMap {
Entries[I].NumCalls = 0;
}
for (int I = 0; I < D.NumCalls; ++I) {
auto &Entry = Entries[D.Calls[I].FromNode];
MapEntry &Entry = Entries[D.Calls[I].FromNode];
Entry.Calls[Entry.NumCalls++] = I;
}
}
@ -923,17 +923,17 @@ struct NodeToCallsMap {
const FunctionDescription &D,
const uint64_t *Counters,
ProfileWriterContext &Ctx) const {
const auto &Entry = Entries[NodeID];
const MapEntry &Entry = Entries[NodeID];
uint64_t MaxValue = 0ull;
for (int I = 0, E = Entry.NumCalls; I != E; ++I) {
const auto CallID = Entry.Calls[I];
const uint32_t CallID = Entry.Calls[I];
DEBUG(reportNumber(" Setting freq for call ID: ", CallID, 10));
auto &CallDesc = D.Calls[CallID];
const CallDescription &CallDesc = D.Calls[CallID];
if (CallDesc.Counter == 0xffffffff) {
CallFreqs[CallID] = Freq;
DEBUG(reportNumber(" with : ", Freq, 10));
} else {
const auto CounterVal = Counters[CallDesc.Counter];
const uint64_t CounterVal = Counters[CallDesc.Counter];
CallFreqs[CallID] = CounterVal;
MaxValue = CounterVal > MaxValue ? CounterVal : MaxValue;
DEBUG(reportNumber(" with (private counter) : ", CounterVal, 10));
@ -1481,7 +1481,7 @@ extern "C" void __bolt_instr_setup() {
if (__bolt_instr_wait_forks)
__setpgid(0, 0);
if (auto PID = __fork())
if (long PID = __fork())
return;
watchProcess();
}

67
bolt/src/BinaryBasicBlock.cpp
View File

@ -46,13 +46,13 @@ bool BinaryBasicBlock::hasInstructions() const {
}
void BinaryBasicBlock::adjustNumPseudos(const MCInst &Inst, int Sign) {
auto &BC = Function->getBinaryContext();
BinaryContext &BC = Function->getBinaryContext();
if (BC.MII->get(Inst.getOpcode()).isPseudo())
NumPseudos += Sign;
}
BinaryBasicBlock::iterator BinaryBasicBlock::getFirstNonPseudo() {
const auto &BC = Function->getBinaryContext();
const BinaryContext &BC = Function->getBinaryContext();
for (auto II = Instructions.begin(), E = Instructions.end(); II != E; ++II) {
if (!BC.MII->get(II->getOpcode()).isPseudo())
return II;
@ -61,7 +61,7 @@ BinaryBasicBlock::iterator BinaryBasicBlock::getFirstNonPseudo() {
}
BinaryBasicBlock::reverse_iterator BinaryBasicBlock::getLastNonPseudo() {
const auto &BC = Function->getBinaryContext();
const BinaryContext &BC = Function->getBinaryContext();
for (auto RII = Instructions.rbegin(), E = Instructions.rend();
RII != E; ++RII) {
if (!BC.MII->get(RII->getOpcode()).isPseudo())
@ -71,9 +71,9 @@ BinaryBasicBlock::reverse_iterator BinaryBasicBlock::getLastNonPseudo() {
}
bool BinaryBasicBlock::validateSuccessorInvariants() {
const auto *Inst = getLastNonPseudoInstr();
const auto *JT = Inst ? Function->getJumpTable(*Inst) : nullptr;
auto &BC = Function->getBinaryContext();
const MCInst *Inst = getLastNonPseudoInstr();
const JumpTable *JT = Inst ? Function->getJumpTable(*Inst) : nullptr;
BinaryContext &BC = Function->getBinaryContext();
bool Valid = true;
if (JT) {
@ -84,7 +84,7 @@ bool BinaryBasicBlock::validateSuccessorInvariants() {
const std::vector<const MCSymbol *> Entries(&JT->Entries[Range.first],
&JT->Entries[Range.second]);
std::set<const MCSymbol *> UniqueSyms(Entries.begin(), Entries.end());
for (auto *Succ : Successors) {
for (BinaryBasicBlock *Succ : Successors) {
auto Itr = UniqueSyms.find(Succ->getLabel());
if (Itr != UniqueSyms.end()) {
UniqueSyms.erase(Itr);
@ -100,7 +100,7 @@ bool BinaryBasicBlock::validateSuccessorInvariants() {
// If there are any leftover entries in the jump table, they
// must be one of the function end labels.
if (Valid) {
for (auto *Sym : UniqueSyms) {
for (const MCSymbol *Sym : UniqueSyms) {
Valid &= (Sym == Function->getFunctionEndLabel() ||
Sym == Function->getFunctionColdEndLabel());
if (!Valid) {
@ -195,7 +195,8 @@ int32_t BinaryBasicBlock::getCFIStateAtInstr(const MCInst *Instr) const {
getFunction()->getState() >= BinaryFunction::State::CFG &&
"can only calculate CFI state when function is in or past the CFG state");
const auto &FDEProgram = getFunction()->getFDEProgram();
const std::vector<MCCFIInstruction> &FDEProgram =
getFunction()->getFDEProgram();
// Find the last CFI preceding Instr in this basic block.
const MCInst *LastCFI = nullptr;
@ -240,7 +241,7 @@ int32_t BinaryBasicBlock::getCFIStateAtInstr(const MCInst *Instr) const {
--State;
assert(State >= 0 && "first CFI cannot be RestoreState");
while (Depth && State >= 0) {
const auto &CFIInstr = FDEProgram[State];
const MCCFIInstruction &CFIInstr = FDEProgram[State];
if (CFIInstr.getOperation() == MCCFIInstruction::OpRestoreState) {
++Depth;
} else if (CFIInstr.getOperation() == MCCFIInstruction::OpRememberState) {
@ -290,7 +291,7 @@ void BinaryBasicBlock::replaceSuccessor(BinaryBasicBlock *Succ,
}
void BinaryBasicBlock::removeAllSuccessors() {
for (auto *SuccessorBB : successors()) {
for (BinaryBasicBlock *SuccessorBB : successors()) {
SuccessorBB->removePredecessor(this);
}
Successors.clear();
@ -338,9 +339,9 @@ void BinaryBasicBlock::removeDuplicateConditionalSuccessor(MCInst *CondBranch) {
assert(succ_size() == 2 && Successors[0] == Successors[1] &&
"conditional successors expected");
auto *Succ = Successors[0];
const auto CondBI = BranchInfo[0];
const auto UncondBI = BranchInfo[1];
BinaryBasicBlock *Succ = Successors[0];
const BinaryBranchInfo CondBI = BranchInfo[0];
const BinaryBranchInfo UncondBI = BranchInfo[1];
eraseInstruction(findInstruction(CondBranch));
@ -357,14 +358,14 @@ void BinaryBasicBlock::removeDuplicateConditionalSuccessor(MCInst *CondBranch) {
void BinaryBasicBlock::adjustExecutionCount(double Ratio) {
auto adjustedCount = [&](uint64_t Count) -> uint64_t {
auto NewCount = Count * Ratio;
double NewCount = Count * Ratio;
if (!NewCount && Count && (Ratio > 0.0))
NewCount = 1;
return NewCount;
};
setExecutionCount(adjustedCount(getKnownExecutionCount()));
for (auto &BI : branch_info()) {
for (BinaryBranchInfo &BI : branch_info()) {
if (BI.Count != COUNT_NO_PROFILE)
BI.Count = adjustedCount(BI.Count);
if (BI.MispredictedCount != COUNT_INFERRED)
@ -402,7 +403,7 @@ BinaryBasicBlock::getMacroOpFusionPair() const {
auto RI = getLastNonPseudo();
assert(RI != rend() && "cannot have an empty block with 2 successors");
auto &BC = Function->getBinaryContext();
BinaryContext &BC = Function->getBinaryContext();
// Skip instruction if it's an unconditional branch following
// a conditional one.
@ -425,7 +426,7 @@ BinaryBasicBlock::getMacroOpFusionPair() const {
}
MCInst *BinaryBasicBlock::getTerminatorBefore(MCInst *Pos) {
auto &BC = Function->getBinaryContext();
BinaryContext &BC = Function->getBinaryContext();
auto Itr = rbegin();
bool Check = Pos ? false : true;
MCInst *FirstTerminator{nullptr};
@ -444,7 +445,7 @@ MCInst *BinaryBasicBlock::getTerminatorBefore(MCInst *Pos) {
}
bool BinaryBasicBlock::hasTerminatorAfter(MCInst *Pos) {
auto &BC = Function->getBinaryContext();
BinaryContext &BC = Function->getBinaryContext();
auto Itr = rbegin();
while (Itr != rend()) {
if (&*Itr == Pos)
@ -467,7 +468,7 @@ bool BinaryBasicBlock::swapConditionalSuccessors() {
void BinaryBasicBlock::addBranchInstruction(const BinaryBasicBlock *Successor) {
assert(isSuccessor(Successor));
auto &BC = Function->getBinaryContext();
BinaryContext &BC = Function->getBinaryContext();
MCInst NewInst;
std::unique_lock<std::shared_timed_mutex> Lock(BC.CtxMutex);
BC.MIB->createUncondBranch(NewInst, Successor->getLabel(), BC.Ctx.get());
@ -475,7 +476,7 @@ void BinaryBasicBlock::addBranchInstruction(const BinaryBasicBlock *Successor) {
}
void BinaryBasicBlock::addTailCallInstruction(const MCSymbol *Target) {
auto &BC = Function->getBinaryContext();
BinaryContext &BC = Function->getBinaryContext();
MCInst NewInst;
BC.MIB->createTailCall(NewInst, Target, BC.Ctx.get());
Instructions.emplace_back(std::move(NewInst));
@ -483,8 +484,8 @@ void BinaryBasicBlock::addTailCallInstruction(const MCSymbol *Target) {
uint32_t BinaryBasicBlock::getNumCalls() const {
uint32_t N{0};
auto &BC = Function->getBinaryContext();
for (auto &Instr : Instructions) {
BinaryContext &BC = Function->getBinaryContext();
for (const MCInst &Instr : Instructions) {
if (BC.MIB->isCall(Instr))
++N;
}
@ -493,9 +494,9 @@ uint32_t BinaryBasicBlock::getNumCalls() const {
uint32_t BinaryBasicBlock::getNumPseudos() const {
#ifndef NDEBUG
auto &BC = Function->getBinaryContext();
BinaryContext &BC = Function->getBinaryContext();
uint32_t N = 0;
for (auto &Instr : Instructions) {
for (const MCInst &Instr : Instructions) {
if (BC.MII->get(Instr.getOpcode()).isPseudo())
++N;
}
@ -515,7 +516,7 @@ BinaryBasicBlock::getBranchStats(const BinaryBasicBlock *Succ) const {
if (Function->hasValidProfile()) {
uint64_t TotalCount = 0;
uint64_t TotalMispreds = 0;
for (const auto &BI : BranchInfo) {
for (const BinaryBranchInfo &BI : BranchInfo) {
if (BI.Count != COUNT_NO_PROFILE) {
TotalCount += BI.Count;
TotalMispreds += BI.MispredictedCount;
@ -525,7 +526,7 @@ BinaryBasicBlock::getBranchStats(const BinaryBasicBlock *Succ) const {
if (TotalCount > 0) {
auto Itr = std::find(Successors.begin(), Successors.end(), Succ);
assert(Itr != Successors.end());
const auto &BI = BranchInfo[Itr - Successors.begin()];
const BinaryBranchInfo &BI = BranchInfo[Itr - Successors.begin()];
if (BI.Count && BI.Count != COUNT_NO_PROFILE) {
if (TotalMispreds == 0) TotalMispreds = 1;
return std::make_pair(double(BI.Count) / TotalCount,
@ -537,7 +538,7 @@ BinaryBasicBlock::getBranchStats(const BinaryBasicBlock *Succ) const {
}
void BinaryBasicBlock::dump() const {
auto &BC = Function->getBinaryContext();
BinaryContext &BC = Function->getBinaryContext();
if (Label) outs() << Label->getName() << ":\n";
BC.printInstructions(outs(), Instructions.begin(), Instructions.end(),
getOffset());
@ -559,7 +560,7 @@ uint64_t BinaryBasicBlock::estimateSize(const MCCodeEmitter *Emitter) const {
BinaryBasicBlock::BinaryBranchInfo &
BinaryBasicBlock::getBranchInfo(const BinaryBasicBlock &Succ) {
auto BI = branch_info_begin();
for (auto BB : successors()) {
for (BinaryBasicBlock *BB : successors()) {
if (&Succ == BB)
return *BI;
++BI;
@ -572,7 +573,7 @@ BinaryBasicBlock::getBranchInfo(const BinaryBasicBlock &Succ) {
BinaryBasicBlock::BinaryBranchInfo &
BinaryBasicBlock::getBranchInfo(const MCSymbol *Label) {
auto BI = branch_info_begin();
for (auto BB : successors()) {
for (BinaryBasicBlock *BB : successors()) {
if (BB->getLabel() == Label)
return *BI;
++BI;
@ -585,7 +586,7 @@ BinaryBasicBlock::getBranchInfo(const MCSymbol *Label) {
BinaryBasicBlock *BinaryBasicBlock::splitAt(iterator II) {
assert(II != end() && "expected iterator pointing to instruction");
auto *NewBlock = getFunction()->addBasicBlock(0);
BinaryBasicBlock *NewBlock = getFunction()->addBasicBlock(0);
// Adjust successors/predecessors and propagate the execution count.
moveAllSuccessorsTo(NewBlock);
@ -606,8 +607,8 @@ void BinaryBasicBlock::updateOutputValues(const MCAsmLayout &Layout) {
if (!LocSyms)
return;
const auto BBAddress = getOutputAddressRange().first;
const auto BBOffset = Layout.getSymbolOffset(*getLabel());
const uint64_t BBAddress = getOutputAddressRange().first;
const uint64_t BBOffset = Layout.getSymbolOffset(*getLabel());
for (const auto &LocSymKV : *LocSyms) {
const uint32_t InputFunctionOffset = LocSymKV.first;
const uint32_t OutputOffset = static_cast<uint32_t>(

2
bolt/src/BinaryBasicBlock.h
View File

@ -434,7 +434,7 @@ public:
void setSuccessorBranchInfo(const BinaryBasicBlock &Succ,
uint64_t Count,
uint64_t MispredictedCount) {
auto &BI = getBranchInfo(Succ);
BinaryBranchInfo &BI = getBranchInfo(Succ);
BI.Count = Count;
BI.MispredictedCount = MispredictedCount;
}

307
bolt/src/BinaryContext.cpp
View File

@ -118,13 +118,13 @@ BinaryContext::BinaryContext(std::unique_ptr<MCContext> Ctx,
}
BinaryContext::~BinaryContext() {
for (auto *Section : Sections) {
for (BinarySection *Section : Sections) {
delete Section;
}
for (auto *InjectedFunction : InjectedBinaryFunctions) {
for (BinaryFunction *InjectedFunction : InjectedBinaryFunctions) {
delete InjectedFunction;
}
for (auto JTI : JumpTables) {
for (std::pair<const uint64_t, JumpTable *> JTI : JumpTables) {
delete JTI.second;
}
clearBinaryData();
@ -348,10 +348,10 @@ bool BinaryContext::validateObjectNesting() const {
bool BinaryContext::validateHoles() const {
bool Valid = true;
for (auto &Section : sections()) {
for (const auto &Rel : Section.relocations()) {
auto RelAddr = Rel.Offset + Section.getAddress();
auto *BD = getBinaryDataContainingAddress(RelAddr);
for (BinarySection &Section : sections()) {
for (const Relocation &Rel : Section.relocations()) {
uint64_t RelAddr = Rel.Offset + Section.getAddress();
const BinaryData *BD = getBinaryDataContainingAddress(RelAddr);
if (!BD) {
errs() << "BOLT-WARNING: no BinaryData found for relocation at address"
<< " 0x" << Twine::utohexstr(RelAddr) << " in "
@ -369,12 +369,12 @@ bool BinaryContext::validateHoles() const {
}
void BinaryContext::updateObjectNesting(BinaryDataMapType::iterator GAI) {
const auto Address = GAI->second->getAddress();
const auto Size = GAI->second->getSize();
const uint64_t Address = GAI->second->getAddress();
const uint64_t Size = GAI->second->getSize();
auto fixParents =
[&](BinaryDataMapType::iterator Itr, BinaryData *NewParent) {
auto *OldParent = Itr->second->Parent;
BinaryData *OldParent = Itr->second->Parent;
Itr->second->Parent = NewParent;
++Itr;
while (Itr != BinaryDataMap.end() && OldParent &&
@ -386,7 +386,7 @@ void BinaryContext::updateObjectNesting(BinaryDataMapType::iterator GAI) {
// Check if the previous symbol contains the newly added symbol.
if (GAI != BinaryDataMap.begin()) {
auto *Prev = std::prev(GAI)->second;
BinaryData *Prev = std::prev(GAI)->second;
while (Prev) {
if (Prev->getSection() == GAI->second->getSection() &&
Prev->containsRange(Address, Size)) {
@ -400,7 +400,7 @@ void BinaryContext::updateObjectNesting(BinaryDataMapType::iterator GAI) {
// Check if the newly added symbol contains any subsequent symbols.
if (Size != 0) {
auto *BD = GAI->second->Parent ? GAI->second->Parent : GAI->second;
BinaryData *BD = GAI->second->Parent ? GAI->second->Parent : GAI->second;
auto Itr = std::next(GAI);
while (Itr != BinaryDataMap.end() &&
BD->containsRange(Itr->second->getAddress(),
@ -439,7 +439,7 @@ BinaryContext::handleAddressRef(uint64_t Address, BinaryFunction &BF,
// too.
auto IslandIter = AddressToConstantIslandMap.lower_bound(Address);
if (IslandIter != AddressToConstantIslandMap.end()) {
if (auto *IslandSym =
if (MCSymbol *IslandSym =
IslandIter->second->getOrCreateProxyIslandAccess(Address, BF)) {
/// Make this function depend on IslandIter->second because we have
/// a reference to its constant island. When emitting this function,
@ -454,7 +454,7 @@ BinaryContext::handleAddressRef(uint64_t Address, BinaryFunction &BF,
// Note that the address does not necessarily have to reside inside
// a section, it could be an absolute address too.
auto Section = getSectionForAddress(Address);
ErrorOr<BinarySection &> Section = getSectionForAddress(Address);
if (Section && Section->isText()) {
if (BF.containsAddress(Address, /*UseMaxSize=*/ isAArch64())) {
if (Address != BF.getAddress()) {
@ -478,7 +478,7 @@ BinaryContext::handleAddressRef(uint64_t Address, BinaryFunction &BF,
// With relocations, catch jump table references outside of the basic block
// containing the indirect jump.
if (HasRelocations) {
const auto MemType = analyzeMemoryAt(Address, BF);
const MemoryContentsType MemType = analyzeMemoryAt(Address, BF);
if (MemType == MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE && IsPCRel) {
const MCSymbol *Symbol =
getOrCreateJumpTable(BF, Address, JumpTable::JTT_PIC);
@ -487,12 +487,12 @@ BinaryContext::handleAddressRef(uint64_t Address, BinaryFunction &BF,
}
}
if (auto *BD = getBinaryDataContainingAddress(Address)) {
if (BinaryData *BD = getBinaryDataContainingAddress(Address)) {
return std::make_pair(BD->getSymbol(), Address - BD->getAddress());
}
// TODO: use DWARF info to get size/alignment here?
auto *TargetSymbol = getOrCreateGlobalSymbol(Address, "DATAat");
MCSymbol *TargetSymbol = getOrCreateGlobalSymbol(Address, "DATAat");
LLVM_DEBUG(dbgs() << "Created symbol " << TargetSymbol->getName() << '\n');
return std::make_pair(TargetSymbol, Addend);
}
@ -502,7 +502,7 @@ BinaryContext::analyzeMemoryAt(uint64_t Address, BinaryFunction &BF) {
if (!isX86())
return MemoryContentsType::UNKNOWN;
auto Section = getSectionForAddress(Address);
ErrorOr<BinarySection &> Section = getSectionForAddress(Address);
if (!Section) {
// No section - possibly an absolute address. Since we don't allow
// internal function addresses to escape the function scope - we
@ -555,9 +555,10 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
auto isFragment = [](BinaryFunction &Fragment,
BinaryFunction &Parent) -> bool {
// Check if <fragment restored name> == <parent restored name>.cold(.\d+)?
for (auto BFName : Parent.getNames()) {
auto BFNamePrefix = Regex::escape(NameResolver::restore(BFName));
auto BFNameRegex = Twine(BFNamePrefix, "\\.cold(\\.[0-9]+)?").str();
for (StringRef BFName : Parent.getNames()) {
std::string BFNamePrefix = Regex::escape(NameResolver::restore(BFName));
std::string BFNameRegex =
Twine(BFNamePrefix, "\\.cold(\\.[0-9]+)?").str();
if (Fragment.hasRestoredNameRegex(BFNameRegex))
return true;
}
@ -594,14 +595,14 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
return false;
};
auto Section = getSectionForAddress(Address);
ErrorOr<BinarySection &> Section = getSectionForAddress(Address);
if (!Section)
return false;
// The upper bound is defined by containing object, section limits, and
// the next jump table in memory.
auto UpperBound = Section->getEndAddress();
const auto *JumpTableBD = getBinaryDataAtAddress(Address);
uint64_t UpperBound = Section->getEndAddress();
const BinaryData *JumpTableBD = getBinaryDataAtAddress(Address);
if (JumpTableBD && JumpTableBD->getSize()) {
assert(JumpTableBD->getEndAddress() <= UpperBound &&
"data object cannot cross a section boundary");
@ -613,8 +614,8 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: analyzeJumpTable in " << BF.getPrintName()
<< '\n');
const auto EntrySize = getJumpTableEntrySize(Type);
for (auto EntryAddress = Address; EntryAddress <= UpperBound - EntrySize;
const uint64_t EntrySize = getJumpTableEntrySize(Type);
for (uint64_t EntryAddress = Address; EntryAddress <= UpperBound - EntrySize;
EntryAddress += EntrySize) {
LLVM_DEBUG(dbgs() << " * Checking 0x" << Twine::utohexstr(EntryAddress)
<< " -> ");
@ -647,7 +648,7 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
}
// Function or one of its fragments.
auto TargetBF = getBinaryFunctionContainingAddress(Value);
BinaryFunction *TargetBF = getBinaryFunctionContainingAddress(Value);
// We assume that a jump table cannot have function start as an entry.
if (!doesBelongToFunction(Value, TargetBF) || Value == BF.getAddress()) {
@ -659,7 +660,7 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
<< TargetBF->getPrintName() << '\n';
if (TargetBF->isFragment())
dbgs() << " ! is a fragment\n";
auto TargetParent = TargetBF->getParentFragment();
BinaryFunction *TargetParent = TargetBF->getParentFragment();
dbgs() << " ! its parent is "
<< (TargetParent ? TargetParent->getPrintName() : "(none)")
<< '\n';
@ -705,8 +706,8 @@ void BinaryContext::populateJumpTables() {
<< '\n');
for (auto JTI = JumpTables.begin(), JTE = JumpTables.end(); JTI != JTE;
++JTI) {
auto *JT = JTI->second;
auto &BF = *JT->Parent;
JumpTable *JT = JTI->second;
BinaryFunction &BF = *JT->Parent;
if (!BF.isSimple())
continue;
@ -717,11 +718,8 @@ void BinaryContext::populateJumpTables() {
NextJTAddress = NextJTI->second->getAddress();
}
const auto Success = analyzeJumpTable(JT->getAddress(),
JT->Type,
BF,
NextJTAddress,
&JT->OffsetEntries);
const bool Success = analyzeJumpTable(JT->getAddress(), JT->Type, BF,
NextJTAddress, &JT->OffsetEntries);
if (!Success) {
dbgs() << "failed to analyze jump table in function " << BF << '\n';
JT->print(dbgs());
@ -734,7 +732,7 @@ void BinaryContext::populateJumpTables() {
llvm_unreachable("jump table heuristic failure");
}
for (auto EntryOffset : JT->OffsetEntries) {
for (uint64_t EntryOffset : JT->OffsetEntries) {
if (EntryOffset == BF.getSize())
BF.IgnoredBranches.emplace_back(EntryOffset, BF.getSize());
else
@ -744,7 +742,7 @@ void BinaryContext::populateJumpTables() {
// In strict mode, erase PC-relative relocation record. Later we check that
// all such records are erased and thus have been accounted for.
if (opts::StrictMode && JT->Type == JumpTable::JTT_PIC) {
for (auto Address = JT->getAddress();
for (uint64_t Address = JT->getAddress();
Address < JT->getAddress() + JT->getSize();
Address += JT->EntrySize) {
DataPCRelocations.erase(DataPCRelocations.find(Address));
@ -760,7 +758,7 @@ void BinaryContext::populateJumpTables() {
LLVM_DEBUG({
dbgs() << DataPCRelocations.size()
<< " unclaimed PC-relative relocations left in data:\n";
for (auto Reloc : DataPCRelocations)
for (uint64_t Reloc : DataPCRelocations)
dbgs() << Twine::utohexstr(Reloc) << '\n';
});
assert(0 && "unclaimed PC-relative relocations left in data\n");
@ -768,7 +766,7 @@ void BinaryContext::populateJumpTables() {
clearList(DataPCRelocations);
// Functions containing split jump tables need to be skipped with all
// fragments.
for (auto BF : FuncsToSkip) {
for (BinaryFunction *BF : FuncsToSkip) {
BinaryFunction *ParentBF =
const_cast<BinaryFunction *>(BF->getTopmostFragment());
LLVM_DEBUG(dbgs() << "Skipping " << ParentBF->getPrintName()
@ -800,7 +798,7 @@ BinaryFunction *BinaryContext::createBinaryFunction(
auto Result = BinaryFunctions.emplace(
Address, BinaryFunction(Name, Section, Address, Size, *this));
assert(Result.second == true && "unexpected duplicate function");
auto *BF = &Result.first->second;
BinaryFunction *BF = &Result.first->second;
registerNameAtAddress(Name, Address, SymbolSize ? SymbolSize : Size,
Alignment);
setSymbolToFunctionMap(BF->getSymbol(), BF);
@ -810,7 +808,7 @@ BinaryFunction *BinaryContext::createBinaryFunction(
const MCSymbol *
BinaryContext::getOrCreateJumpTable(BinaryFunction &Function, uint64_t Address,
JumpTable::JumpTableType Type) {
if (auto *JT = getJumpTableContainingAddress(Address)) {
if (JumpTable *JT = getJumpTableContainingAddress(Address)) {
assert(JT->Type == Type && "jump table types have to match");
assert(JT->Parent == &Function &&
"cannot re-use jump table of a different function");
@ -821,27 +819,23 @@ BinaryContext::getOrCreateJumpTable(BinaryFunction &Function, uint64_t Address,
// Re-use the existing symbol if possible.
MCSymbol *JTLabel{nullptr};
if (auto *Object = getBinaryDataAtAddress(Address)) {
if (BinaryData *Object = getBinaryDataAtAddress(Address)) {
if (!isInternalSymbolName(Object->getSymbol()->getName()))
JTLabel = Object->getSymbol();
}
const auto EntrySize = getJumpTableEntrySize(Type);
const uint64_t EntrySize = getJumpTableEntrySize(Type);
if (!JTLabel) {
const auto JumpTableName = generateJumpTableName(Function, Address);
const std::string JumpTableName = generateJumpTableName(Function, Address);
JTLabel = registerNameAtAddress(JumpTableName, Address, 0, EntrySize);
}
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: creating jump table " << JTLabel->getName()
<< " in function " << Function << '\n');
auto *JT = new JumpTable(*JTLabel,
Address,
EntrySize,
Type,
JumpTable::LabelMapType{{0, JTLabel}},
Function,
*getSectionForAddress(Address));
JumpTable *JT = new JumpTable(*JTLabel, Address, EntrySize, Type,
JumpTable::LabelMapType{{0, JTLabel}}, Function,
*getSectionForAddress(Address));
JumpTables.emplace(Address, JT);
// Duplicate the entry for the parent function for easy access.
@ -856,7 +850,7 @@ BinaryContext::duplicateJumpTable(BinaryFunction &Function, JumpTable *JT,
auto L = scopeLock();
unsigned Offset = 0;
bool Found = false;
for (auto Elmt : JT->Labels) {
for (std::pair<const unsigned, MCSymbol *> Elmt : JT->Labels) {
if (Elmt.second != OldLabel)
continue;
Offset = Elmt.first;
@ -864,14 +858,11 @@ BinaryContext::duplicateJumpTable(BinaryFunction &Function, JumpTable *JT,
break;
}
assert(Found && "Label not found");
auto *NewLabel = Ctx->createNamedTempSymbol("duplicatedJT");
auto *NewJT = new JumpTable(*NewLabel,
JT->getAddress(),
JT->EntrySize,
JT->Type,
JumpTable::LabelMapType{{Offset, NewLabel}},
Function,
*getSectionForAddress(JT->getAddress()));
MCSymbol *NewLabel = Ctx->createNamedTempSymbol("duplicatedJT");
JumpTable *NewJT =
new JumpTable(*NewLabel, JT->getAddress(), JT->EntrySize, JT->Type,
JumpTable::LabelMapType{{Offset, NewLabel}}, Function,
*getSectionForAddress(JT->getAddress()));
NewJT->Entries = JT->Entries;
NewJT->Counts = JT->Counts;
uint64_t JumpTableID = ++DuplicatedJumpTables;
@ -887,7 +878,7 @@ std::string BinaryContext::generateJumpTableName(const BinaryFunction &BF,
uint64_t Address) {
size_t Id;
uint64_t Offset = 0;
if (const auto *JT = BF.getJumpTableContainingAddress(Address)) {
if (const JumpTable *JT = BF.getJumpTableContainingAddress(Address)) {
Offset = Address - JT->getAddress();
auto Itr = JT->Labels.find(Offset);
if (Itr != JT->Labels.end()) {
@ -909,7 +900,7 @@ bool BinaryContext::hasValidCodePadding(const BinaryFunction &BF) {
if (BF.getSize() == BF.getMaxSize())
return true;
auto FunctionData = BF.getData();
ErrorOr<ArrayRef<unsigned char>> FunctionData = BF.getData();
assert(FunctionData && "cannot get function as data");
uint64_t Offset = BF.getSize();
@ -920,7 +911,7 @@ bool BinaryContext::hasValidCodePadding(const BinaryFunction &BF) {
// Skip instructions that satisfy the predicate condition.
auto skipInstructions = [&](std::function<bool(const MCInst &)> Predicate) {
const auto StartOffset = Offset;
const uint64_t StartOffset = Offset;
for (; Offset < BF.getMaxSize();
Offset += InstrSize, InstrAddress += InstrSize) {
if (!DisAsm->getInstruction(Instr,
@ -938,7 +929,7 @@ bool BinaryContext::hasValidCodePadding(const BinaryFunction &BF) {
// Skip a sequence of zero bytes.
auto skipZeros = [&]() {
const auto StartOffset = Offset;
const uint64_t StartOffset = Offset;
for (; Offset < BF.getMaxSize(); ++Offset)
if ((*FunctionData)[Offset] != 0)
break;
@ -991,7 +982,7 @@ bool BinaryContext::hasValidCodePadding(const BinaryFunction &BF) {
void BinaryContext::adjustCodePadding() {
for (auto &BFI : BinaryFunctions) {
auto &BF = BFI.second;
BinaryFunction &BF = BFI.second;
if (!shouldEmit(BF))
continue;
@ -1015,13 +1006,14 @@ MCSymbol *BinaryContext::registerNameAtAddress(StringRef Name,
uint16_t Alignment,
unsigned Flags) {
// Register the name with MCContext.
auto *Symbol = Ctx->getOrCreateSymbol(Name);
MCSymbol *Symbol = Ctx->getOrCreateSymbol(Name);
auto GAI = BinaryDataMap.find(Address);
BinaryData *BD;
if (GAI == BinaryDataMap.end()) {
auto SectionOrErr = getSectionForAddress(Address);
auto &Section = SectionOrErr ? SectionOrErr.get() : absoluteSection();
ErrorOr<BinarySection &> SectionOrErr = getSectionForAddress(Address);
BinarySection &Section =
SectionOrErr ? SectionOrErr.get() : absoluteSection();
BD = new BinaryData(*Symbol,
Address,
Size,
@ -1053,7 +1045,7 @@ BinaryContext::getBinaryDataContainingAddressImpl(uint64_t Address) const {
}
// If this is a sub-symbol, see if a parent data contains the address.
auto *BD = NI->second->getParent();
const BinaryData *BD = NI->second->getParent();
while (BD) {
if (BD->containsAddress(Address))
return BD;
@ -1087,16 +1079,16 @@ bool BinaryContext::setBinaryDataSize(uint64_t Address, uint64_t Size) {
void BinaryContext::generateSymbolHashes() {
auto isPadding = [](const BinaryData &BD) {
auto Contents = BD.getSection().getContents();
auto SymData = Contents.substr(BD.getOffset(), BD.getSize());
StringRef Contents = BD.getSection().getContents();
StringRef SymData = Contents.substr(BD.getOffset(), BD.getSize());
return (BD.getName().startswith("HOLEat") ||
SymData.find_first_not_of(0) == StringRef::npos);
};
uint64_t NumCollisions = 0;
for (auto &Entry : BinaryDataMap) {
auto &BD = *Entry.second;
auto Name = BD.getName();
BinaryData &BD = *Entry.second;
StringRef Name = BD.getName();
if (!isInternalSymbolName(Name))
continue;
@ -1109,7 +1101,7 @@ void BinaryContext::generateSymbolHashes() {
return !isInternalSymbolName(Symbol->getName());
});
if (Itr != BD.getSymbols().end()) {
auto Idx = std::distance(BD.getSymbols().begin(), Itr);
size_t Idx = std::distance(BD.getSymbols().begin(), Itr);
std::swap(BD.getSymbols()[0], BD.getSymbols()[Idx]);
continue;
}
@ -1120,8 +1112,8 @@ void BinaryContext::generateSymbolHashes() {
continue;
}
const auto Hash = BD.getSection().hash(BD);
const auto Idx = Name.find("0x");
const uint64_t Hash = BD.getSection().hash(BD);
const size_t Idx = Name.find("0x");
std::string NewName = (Twine(Name.substr(0, Idx)) +
"_" + Twine::utohexstr(Hash)).str();
if (getBinaryDataByName(NewName)) {
@ -1154,7 +1146,7 @@ void BinaryContext::registerFragment(BinaryFunction &TargetFunction,
// Only a parent function (or a sibling) can reach its fragment.
assert(!Function.IsFragment &&
"only one cold fragment is supported at this time");
if (auto *TargetParent = TargetFunction.getParentFragment()) {
if (BinaryFunction *TargetParent = TargetFunction.getParentFragment()) {
assert(TargetParent == &Function && "mismatching parent function");
return;
}
@ -1171,18 +1163,19 @@ void BinaryContext::registerFragment(BinaryFunction &TargetFunction,
}
void BinaryContext::processInterproceduralReferences(BinaryFunction &Function) {
for (auto Address : Function.InterproceduralReferences) {
for (uint64_t Address : Function.InterproceduralReferences) {
if (!Address)
continue;
auto *TargetFunction = getBinaryFunctionContainingAddress(Address);
BinaryFunction *TargetFunction =
getBinaryFunctionContainingAddress(Address);
if (&Function == TargetFunction)
continue;
if (TargetFunction) {
if (TargetFunction->IsFragment)
registerFragment(*TargetFunction, Function);
if (auto Offset = Address - TargetFunction->getAddress())
if (uint64_t Offset = Address - TargetFunction->getAddress())
TargetFunction->addEntryPointAtOffset(Offset);
continue;
@ -1190,7 +1183,7 @@ void BinaryContext::processInterproceduralReferences(BinaryFunction &Function) {
// Check if address falls in function padding space - this could be
// unmarked data in code. In this case adjust the padding space size.
auto Section = getSectionForAddress(Address);
ErrorOr<BinarySection &> Section = getSectionForAddress(Address);
assert(Section && "cannot get section for referenced address");
if (!Section->isText())
@ -1230,7 +1223,7 @@ void BinaryContext::postProcessSymbolTable() {
fixBinaryDataHoles();
bool Valid = true;
for (auto &Entry : BinaryDataMap) {
auto *BD = Entry.second;
BinaryData *BD = Entry.second;
if ((BD->getName().startswith("SYMBOLat") ||
BD->getName().startswith("DATAat")) &&
!BD->getParent() &&
@ -1255,10 +1248,10 @@ void BinaryContext::foldFunction(BinaryFunction &ChildBF,
std::unique_lock<std::shared_timed_mutex> WriteSymbolMapLock(
SymbolToFunctionMapMutex, std::defer_lock);
const auto ChildName = ChildBF.getOneName();
const StringRef ChildName = ChildBF.getOneName();
// Move symbols over and update bookkeeping info.
for (auto *Symbol : ChildBF.getSymbols()) {
for (MCSymbol *Symbol : ChildBF.getSymbols()) {
ParentBF.getSymbols().push_back(Symbol);
WriteSymbolMapLock.lock();
SymbolToFunctionMap[Symbol] = &ParentBF;
@ -1309,11 +1302,11 @@ void BinaryContext::foldFunction(BinaryFunction &ChildBF,
void BinaryContext::fixBinaryDataHoles() {
assert(validateObjectNesting() && "object nesting inconsitency detected");
for (auto &Section : allocatableSections()) {
for (BinarySection &Section : allocatableSections()) {
std::vector<std::pair<uint64_t, uint64_t>> Holes;
auto isNotHole = [&Section](const binary_data_iterator &Itr) {
auto *BD = Itr->second;
BinaryData *BD = Itr->second;
bool isHole = (!BD->getParent() &&
!BD->getSize() &&
BD->isObject() &&
@ -1332,7 +1325,7 @@ void BinaryContext::fixBinaryDataHoles() {
while (Itr != End) {
if (Itr->second->getAddress() > EndAddress) {
auto Gap = Itr->second->getAddress() - EndAddress;
uint64_t Gap = Itr->second->getAddress() - EndAddress;
Holes.push_back(std::make_pair(EndAddress, Gap));
}
EndAddress = Itr->second->getEndAddress();
@ -1346,8 +1339,8 @@ void BinaryContext::fixBinaryDataHoles() {
// If there is already a symbol at the start of the hole, grow that symbol
// to cover the rest. Otherwise, create a new symbol to cover the hole.
for (auto &Hole : Holes) {
auto *BD = getBinaryDataAtAddress(Hole.first);
for (std::pair<uint64_t, uint64_t> &Hole : Holes) {
BinaryData *BD = getBinaryDataAtAddress(Hole.first);
if (BD) {
// BD->getSection() can be != Section if there are sections that
// overlap. In this case it is probably safe to just skip the holes
@ -1369,8 +1362,8 @@ void BinaryContext::printGlobalSymbols(raw_ostream& OS) const {
bool FirstSection = true;
for (auto &Entry : BinaryDataMap) {
const auto *BD = Entry.second;
const auto &Section = BD->getSection();
const BinaryData *BD = Entry.second;
const BinarySection &Section = BD->getSection();
if (FirstSection || Section != *CurrentSection) {
uint64_t Address, Size;
StringRef Name = Section.getName();
@ -1390,7 +1383,7 @@ void BinaryContext::printGlobalSymbols(raw_ostream& OS) const {
}
OS << "BOLT-INFO: ";
auto *P = BD->getParent();
const BinaryData *P = BD->getParent();
while (P) {
OS << " ";
P = P->getParent();
@ -1402,23 +1395,26 @@ void BinaryContext::printGlobalSymbols(raw_ostream& OS) const {
unsigned BinaryContext::addDebugFilenameToUnit(const uint32_t DestCUID,
const uint32_t SrcCUID,
unsigned FileIndex) {
auto SrcUnit = DwCtx->getCompileUnitForOffset(SrcCUID);
auto LineTable = DwCtx->getLineTableForUnit(SrcUnit);
const auto &FileNames = LineTable->Prologue.FileNames;
DWARFCompileUnit *SrcUnit = DwCtx->getCompileUnitForOffset(SrcCUID);
const DWARFDebugLine::LineTable *LineTable =
DwCtx->getLineTableForUnit(SrcUnit);
const std::vector<DWARFDebugLine::FileNameEntry> &FileNames =
LineTable->Prologue.FileNames;
// Dir indexes start at 1, as DWARF file numbers, and a dir index 0
// means empty dir.
assert(FileIndex > 0 && FileIndex <= FileNames.size() &&
"FileIndex out of range for the compilation unit.");
StringRef Dir = "";
if (FileNames[FileIndex - 1].DirIdx != 0) {
if (auto DirName = dwarf::toString(
if (Optional<const char *> DirName = dwarf::toString(
LineTable->Prologue
.IncludeDirectories[FileNames[FileIndex - 1].DirIdx - 1])) {
Dir = *DirName;
}
}
StringRef FileName = "";
if (auto FName = dwarf::toString(FileNames[FileIndex - 1].Name))
if (Optional<const char *> FName =
dwarf::toString(FileNames[FileIndex - 1].Name))
FileName = *FName;
assert(FileName != "");
return cantFail(Ctx->getDwarfFile(Dir, FileName, 0, None, None, DestCUID));
@ -1471,13 +1467,14 @@ void BinaryContext::preprocessDebugInfo() {
// it to assign CU to functions.
std::vector<CURange> AllRanges;
AllRanges.reserve(DwCtx->getNumCompileUnits());
for (const auto &CU : DwCtx->compile_units()) {
auto RangesOrError = CU->getUnitDIE().getAddressRanges();
for (const std::unique_ptr<DWARFUnit> &CU : DwCtx->compile_units()) {
Expected<DWARFAddressRangesVector> RangesOrError =
CU->getUnitDIE().getAddressRanges();
if (!RangesOrError) {
consumeError(RangesOrError.takeError());
continue;
}
for (auto &Range : *RangesOrError) {
for (DWARFAddressRange &Range : *RangesOrError) {
// Parts of the debug info could be invalidated due to corresponding code
// being removed from the binary by the linker. Hence we check if the
// address is a valid one.
@ -1500,11 +1497,12 @@ void BinaryContext::preprocessDebugInfo() {
// Populate MCContext with DWARF files from all units.
StringRef GlobalPrefix = AsmInfo->getPrivateGlobalPrefix();
for (const auto &CU : DwCtx->compile_units()) {
for (const std::unique_ptr<DWARFUnit> &CU : DwCtx->compile_units()) {
const uint64_t CUID = CU->getOffset();
const DWARFDebugLine::LineTable *LineTable =
DwCtx->getLineTableForUnit(CU.get());
const auto &FileNames = LineTable->Prologue.FileNames;
const std::vector<DWARFDebugLine::FileNameEntry> &FileNames =
LineTable->Prologue.FileNames;
// Assign a unique label to every line table, one per CU.
Ctx->getMCDwarfLineTable(CUID).setLabel(
@ -1520,12 +1518,12 @@ void BinaryContext::preprocessDebugInfo() {
// means empty dir.
StringRef Dir = "";
if (FileNames[I].DirIdx != 0)
if (auto DirName = dwarf::toString(
if (Optional<const char *> DirName = dwarf::toString(
LineTable->Prologue
.IncludeDirectories[FileNames[I].DirIdx - 1]))
Dir = *DirName;
StringRef FileName = "";
if (auto FName = dwarf::toString(FileNames[I].Name))
if (Optional<const char *> FName = dwarf::toString(FileNames[I].Name))
FileName = *FName;
assert(FileName != "");
cantFail(Ctx->getDwarfFile(Dir, FileName, 0, None, None, CUID));
@ -1625,19 +1623,19 @@ void BinaryContext::printInstruction(raw_ostream &OS,
if (MIB->isTailCall(Instruction))
OS << " # TAILCALL ";
if (MIB->isInvoke(Instruction)) {
const auto EHInfo = MIB->getEHInfo(Instruction);
const Optional<MCPlus::MCLandingPad> EHInfo = MIB->getEHInfo(Instruction);
OS << " # handler: ";
if (EHInfo->first)
OS << *EHInfo->first;
else
OS << '0';
OS << "; action: " << EHInfo->second;
const auto GnuArgsSize = MIB->getGnuArgsSize(Instruction);
const int64_t GnuArgsSize = MIB->getGnuArgsSize(Instruction);
if (GnuArgsSize >= 0)
OS << "; GNU_args_size = " << GnuArgsSize;
}
} else if (MIB->isIndirectBranch(Instruction)) {
if (auto JTAddress = MIB->getJumpTable(Instruction)) {
if (uint64_t JTAddress = MIB->getJumpTable(Instruction)) {
OS << " # JUMPTABLE @0x" << Twine::utohexstr(JTAddress);
} else {
OS << " # UNKNOWN CONTROL FLOW";
@ -1651,12 +1649,13 @@ void BinaryContext::printInstruction(raw_ostream &OS,
: nullptr;
if (LineTable) {
auto RowRef = DebugLineTableRowRef::fromSMLoc(Instruction.getLoc());
DebugLineTableRowRef RowRef =
DebugLineTableRowRef::fromSMLoc(Instruction.getLoc());
if (RowRef != DebugLineTableRowRef::NULL_ROW) {
const auto &Row = LineTable->Rows[RowRef.RowIndex - 1];
const DWARFDebugLine::Row &Row = LineTable->Rows[RowRef.RowIndex - 1];
StringRef FileName = "";
if (auto FName =
if (Optional<const char *> FName =
dwarf::toString(LineTable->Prologue.FileNames[Row.File - 1].Name))
FileName = *FName;
OS << " # debug line " << FileName << ":" << Row.Line;
@ -1668,7 +1667,7 @@ void BinaryContext::printInstruction(raw_ostream &OS,
}
if ((opts::PrintRelocations || PrintRelocations) && Function) {
const auto Size = computeCodeSize(&Instruction, &Instruction + 1);
const uint64_t Size = computeCodeSize(&Instruction, &Instruction + 1);
Function->printRelocations(OS, Offset, Size);
}
@ -1684,7 +1683,7 @@ ErrorOr<BinarySection&> BinaryContext::getSectionForAddress(uint64_t Address) {
auto SI = AddressToSection.upper_bound(Address);
if (SI != AddressToSection.begin()) {
--SI;
auto UpperBound = SI->first + SI->second->getSize();
uint64_t UpperBound = SI->first + SI->second->getSize();
if (!SI->second->getSize())
UpperBound += 1;
if (UpperBound > Address)
@ -1695,7 +1694,7 @@ ErrorOr<BinarySection&> BinaryContext::getSectionForAddress(uint64_t Address) {
ErrorOr<StringRef>
BinaryContext::getSectionNameForAddress(uint64_t Address) const {
if (auto Section = getSectionForAddress(Address)) {
if (ErrorOr<const BinarySection &> Section = getSectionForAddress(Address)) {
return Section->getName();
}
return std::make_error_code(std::errc::bad_address);
@ -1736,10 +1735,10 @@ BinarySection &BinaryContext::registerOrUpdateSection(StringRef Name,
if (NamedSections.begin() != NamedSections.end()) {
assert(std::next(NamedSections.begin()) == NamedSections.end() &&
"can only update unique sections");
auto *Section = NamedSections.begin()->second;
BinarySection *Section = NamedSections.begin()->second;
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: updating " << *Section << " -> ");
const auto Flag = Section->isAllocatable();
const bool Flag = Section->isAllocatable();
Section->update(Data, Size, Alignment, ELFType, ELFFlags);
LLVM_DEBUG(dbgs() << *Section << "\n");
// FIXME: Fix section flags/attributes for MachO.
@ -1754,7 +1753,7 @@ BinarySection &BinaryContext::registerOrUpdateSection(StringRef Name,
}
bool BinaryContext::deregisterSection(BinarySection &Section) {
auto *SectionPtr = &Section;
BinarySection *SectionPtr = &Section;
auto Itr = Sections.find(SectionPtr);
if (Itr != Sections.end()) {
auto Range = AddressToSection.equal_range(SectionPtr->getAddress());
@ -1784,13 +1783,13 @@ bool BinaryContext::deregisterSection(BinarySection &Section) {
}
void BinaryContext::printSections(raw_ostream &OS) const {
for (auto &Section : Sections) {
for (BinarySection *const &Section : Sections) {
OS << "BOLT-INFO: " << *Section << "\n";
}
}
BinarySection &BinaryContext::absoluteSection() {
if (auto Section = getUniqueSectionByName("<absolute>"))
if (ErrorOr<BinarySection &> Section = getUniqueSectionByName("<absolute>"))
return *Section;
return registerOrUpdateSection("<absolute>", ELF::SHT_NULL, 0u);
}
@ -1798,7 +1797,7 @@ BinarySection &BinaryContext::absoluteSection() {
ErrorOr<uint64_t>
BinaryContext::getUnsignedValueAtAddress(uint64_t Address,
size_t Size) const {
const auto Section = getSectionForAddress(Address);
const ErrorOr<const BinarySection &> Section = getSectionForAddress(Address);
if (!Section)
return std::make_error_code(std::errc::bad_address);
@ -1814,7 +1813,7 @@ BinaryContext::getUnsignedValueAtAddress(uint64_t Address,
ErrorOr<uint64_t>
BinaryContext::getSignedValueAtAddress(uint64_t Address,
size_t Size) const {
const auto Section = getSectionForAddress(Address);
const ErrorOr<const BinarySection &> Section = getSectionForAddress(Address);
if (!Section)
return std::make_error_code(std::errc::bad_address);
@ -1832,7 +1831,7 @@ void BinaryContext::addRelocation(uint64_t Address,
uint64_t Type,
uint64_t Addend,
uint64_t Value) {
auto Section = getSectionForAddress(Address);
ErrorOr<BinarySection &> Section = getSectionForAddress(Address);
assert(Section && "cannot find section for address");
Section->addRelocation(Address - Section->getAddress(),
Symbol,
@ -1846,7 +1845,7 @@ void BinaryContext::addDynamicRelocation(uint64_t Address,
uint64_t Type,
uint64_t Addend,
uint64_t Value) {
auto Section = getSectionForAddress(Address);
ErrorOr<BinarySection &> Section = getSectionForAddress(Address);
assert(Section && "cannot find section for address");
Section->addDynamicRelocation(Address - Section->getAddress(),
Symbol,
@ -1856,13 +1855,13 @@ void BinaryContext::addDynamicRelocation(uint64_t Address,
}
bool BinaryContext::removeRelocationAt(uint64_t Address) {
auto Section = getSectionForAddress(Address);
ErrorOr<BinarySection &> Section = getSectionForAddress(Address);
assert(Section && "cannot find section for address");
return Section->removeRelocationAt(Address - Section->getAddress());
}
const Relocation *BinaryContext::getRelocationAt(uint64_t Address) {
auto Section = getSectionForAddress(Address);
ErrorOr<BinarySection &> Section = getSectionForAddress(Address);
if (!Section)
return nullptr;
@ -1870,7 +1869,7 @@ const Relocation *BinaryContext::getRelocationAt(uint64_t Address) {
}
const Relocation *BinaryContext::getDynamicRelocationAt(uint64_t Address) {
auto Section = getSectionForAddress(Address);
ErrorOr<BinarySection &> Section = getSectionForAddress(Address);
if (!Section)
return nullptr;
@ -1880,7 +1879,7 @@ const Relocation *BinaryContext::getDynamicRelocationAt(uint64_t Address) {
void BinaryContext::markAmbiguousRelocations(BinaryData &BD,
const uint64_t Address) {
auto setImmovable = [&](BinaryData &BD) {
auto *Root = BD.getAtomicRoot();
BinaryData *Root = BD.getAtomicRoot();
LLVM_DEBUG(if (Root->isMoveable()) {
dbgs() << "BOLT-DEBUG: setting " << *Root << " as immovable "
<< "due to ambiguous relocation referencing 0x"
@ -1893,7 +1892,7 @@ void BinaryContext::markAmbiguousRelocations(BinaryData &BD,
setImmovable(BD);
// Set previous symbol as immovable
auto *Prev = getBinaryDataContainingAddress(Address-1);
BinaryData *Prev = getBinaryDataContainingAddress(Address - 1);
if (Prev && Prev->getEndAddress() == BD.getAddress())
setImmovable(*Prev);
}
@ -1902,7 +1901,7 @@ void BinaryContext::markAmbiguousRelocations(BinaryData &BD,
setImmovable(BD);
// Set next symbol as immovable
auto *Next = getBinaryDataContainingAddress(BD.getEndAddress());
BinaryData *Next = getBinaryDataContainingAddress(BD.getEndAddress());
if (Next && Next->getAddress() == BD.getEndAddress())
setImmovable(*Next);
}
@ -1915,7 +1914,7 @@ BinaryFunction *BinaryContext::getFunctionForSymbol(const MCSymbol *Symbol,
if (BFI == SymbolToFunctionMap.end())
return nullptr;
auto *BF = BFI->second;
BinaryFunction *BF = BFI->second;
if (EntryDesc)
*EntryDesc = BF->getEntryIDForSymbol(Symbol);
@ -1945,7 +1944,7 @@ BinaryFunction *
BinaryContext::createInjectedBinaryFunction(const std::string &Name,
bool IsSimple) {
InjectedBinaryFunctions.push_back(new BinaryFunction(Name, *this, IsSimple));
auto *BF = InjectedBinaryFunctions.back();
BinaryFunction *BF = InjectedBinaryFunctions.back();
setSymbolToFunctionMap(BF->getSymbol(), BF);
BF->CurrentState = BinaryFunction::State::CFG;
return BF;
@ -1958,9 +1957,10 @@ BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) {
BF.fixBranches();
// Create local MC context to isolate the effect of ephemeral code emission.
auto MCEInstance = createIndependentMCCodeEmitter();
auto *LocalCtx = MCEInstance.LocalCtx.get();
auto *MAB = TheTarget->createMCAsmBackend(*STI, *MRI, MCTargetOptions());
IndependentCodeEmitter MCEInstance = createIndependentMCCodeEmitter();
MCContext *LocalCtx = MCEInstance.LocalCtx.get();
MCAsmBackend *MAB =
TheTarget->createMCAsmBackend(*STI, *MRI, MCTargetOptions());
SmallString<256> Code;
raw_svector_ostream VecOS(Code);
@ -1975,7 +1975,7 @@ BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) {
Streamer->initSections(false, *STI);
auto *Section = MCEInstance.LocalMOFI->getTextSection();
MCSection *Section = MCEInstance.LocalMOFI->getTextSection();
Section->setHasInstructions(true);
// Create symbols in the LocalCtx so that they get destroyed with it.
@ -1991,10 +1991,9 @@ BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) {
Streamer->emitLabel(EndLabel);
if (BF.isSplit()) {
auto *ColdSection =
LocalCtx->getELFSection(BF.getColdCodeSectionName(),
ELF::SHT_PROGBITS,
ELF::SHF_EXECINSTR | ELF::SHF_ALLOC);
MCSectionELF *ColdSection =
LocalCtx->getELFSection(BF.getColdCodeSectionName(), ELF::SHT_PROGBITS,
ELF::SHF_EXECINSTR | ELF::SHF_ALLOC);
ColdSection->setHasInstructions(true);
Streamer->SwitchSection(ColdSection);
@ -2011,20 +2010,21 @@ BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) {
// MCStreamer::Finish(), which does more than we want
Streamer->emitBytes(StringRef(""));
auto &Assembler =
MCAssembler &Assembler =
static_cast<MCObjectStreamer *>(Streamer.get())->getAssembler();
MCAsmLayout Layout(Assembler);
Assembler.layout(Layout);
const auto HotSize = Layout.getSymbolOffset(*EndLabel) -
Layout.getSymbolOffset(*StartLabel);
const auto ColdSize = BF.isSplit() ? Layout.getSymbolOffset(*ColdEndLabel) -
Layout.getSymbolOffset(*ColdStartLabel)
: 0ULL;
const uint64_t HotSize =
Layout.getSymbolOffset(*EndLabel) - Layout.getSymbolOffset(*StartLabel);
const uint64_t ColdSize = BF.isSplit()
? Layout.getSymbolOffset(*ColdEndLabel) -
Layout.getSymbolOffset(*ColdStartLabel)
: 0ULL;
// Clean-up the effect of the code emission.
for (const auto &Symbol : Assembler.symbols()) {
auto *MutableSymbol = const_cast<MCSymbol *>(&Symbol);
for (const MCSymbol &Symbol : Assembler.symbols()) {
MCSymbol *MutableSymbol = const_cast<MCSymbol *>(&Symbol);
MutableSymbol->setUndefined();
MutableSymbol->setIsRegistered(false);
}
@ -2070,8 +2070,8 @@ BinaryContext::getBinaryFunctionContainingAddress(uint64_t Address,
return nullptr;
--FI;
const auto UsedSize = UseMaxSize ? FI->second.getMaxSize()
: FI->second.getSize();
const uint64_t UsedSize =
UseMaxSize ? FI->second.getMaxSize() : FI->second.getSize();
if (Address >= FI->first + UsedSize + (CheckPastEnd ? 1 : 0))
return nullptr;
@ -2093,9 +2093,9 @@ BinaryContext::getBinaryFunctionAtAddress(uint64_t Address) {
// address. In such case, we look for a function matching the symbol
// registered at the original address. The new function (the one that the
// original was folded into) will hold the symbol.
if (const auto *BD = getBinaryDataAtAddress(Address)) {
if (const BinaryData *BD = getBinaryDataAtAddress(Address)) {
uint64_t EntryID{0};
auto *BF = getFunctionForSymbol(BD->getSymbol(), &EntryID);
BinaryFunction *BF = getFunctionForSymbol(BD->getSymbol(), &EntryID);
if (BF && EntryID == 0)
return BF;
}
@ -2106,13 +2106,14 @@ DebugAddressRangesVector BinaryContext::translateModuleAddressRanges(
const DWARFAddressRangesVector &InputRanges) const {
DebugAddressRangesVector OutputRanges;
for (const auto Range : InputRanges) {
for (const DWARFAddressRange Range : InputRanges) {
auto BFI = BinaryFunctions.lower_bound(Range.LowPC);
while (BFI != BinaryFunctions.end()) {
const auto &Function = BFI->second;
const BinaryFunction &Function = BFI->second;
if (Function.getAddress() >= Range.HighPC)
break;
const auto FunctionRanges = Function.getOutputAddressRanges();
const DebugAddressRangesVector FunctionRanges =
Function.getOutputAddressRanges();
std::move(std::begin(FunctionRanges),
std::end(FunctionRanges),
std::back_inserter(OutputRanges));

2
bolt/src/BinaryContext.h
View File

@ -691,7 +691,7 @@ public:
/// Return a value of the global \p Symbol or an error if the value
/// was not set.
ErrorOr<uint64_t> getSymbolValue(const MCSymbol &Symbol) const {
const auto *BD = getBinaryDataByName(Symbol.getName());
const BinaryData *BD = getBinaryDataByName(Symbol.getName());
if (!BD)
return std::make_error_code(std::errc::bad_address);
return BD->getAddress();

6
bolt/src/BinaryData.cpp
View File

@ -54,7 +54,7 @@ void BinaryData::merge(const BinaryData *Other) {
}
bool BinaryData::hasName(StringRef Name) const {
for (const auto *Symbol : Symbols) {
for (const MCSymbol *Symbol : Symbols) {
if (Name == Symbol->getName())
return true;
}
@ -63,7 +63,7 @@ bool BinaryData::hasName(StringRef Name) const {
bool BinaryData::hasNameRegex(StringRef NameRegex) const {
Regex MatchName(NameRegex);
for (const auto *Symbol : Symbols) {
for (const MCSymbol *Symbol : Symbols) {
if (MatchName.match(Symbol->getName()))
return true;
}
@ -71,7 +71,7 @@ bool BinaryData::hasNameRegex(StringRef NameRegex) const {
}
bool BinaryData::nameStartsWith(StringRef Prefix) const {
for (const auto *Symbol : Symbols) {
for (const MCSymbol *Symbol : Symbols) {
if (Symbol->getName().startswith(Prefix))
return true;
}

10
bolt/src/BinaryData.h
View File

@ -66,7 +66,7 @@ protected:
uint64_t OutputOffset{0};
BinaryData *getRootData() {
auto *BD = this;
BinaryData *BD = this;
while (BD->Parent)
BD = BD->Parent;
return BD;
@ -151,21 +151,21 @@ public:
}
const BinaryData *getRootData() const {
auto *BD = this;
const BinaryData *BD = this;
while (BD->Parent)
BD = BD->Parent;
return BD;
}
BinaryData *getAtomicRoot() {
auto *BD = this;
BinaryData *BD = this;
while (!BD->isAtomic() && BD->Parent)
BD = BD->Parent;
return BD;
}
const BinaryData *getAtomicRoot() const {
auto *BD = this;
const BinaryData *BD = this;
while (!BD->isAtomic() && BD->Parent)
BD = BD->Parent;
return BD;
@ -224,7 +224,7 @@ inline raw_ostream &operator<<(raw_ostream &OS,
const char *Sep = "\n ";
uint64_t TotalCount = 0;
for (auto &AccessInfo : MAP.AddressAccessInfo) {
for (const AddressAccess &AccessInfo : MAP.AddressAccessInfo) {
SS << Sep << "{ ";
if (AccessInfo.MemoryObject)
SS << AccessInfo.MemoryObject->getName() << " + ";

121
bolt/src/BinaryEmitter.cpp
View File

@ -98,19 +98,19 @@ size_t padFunction(const BinaryFunction &Function) {
static std::map<std::string, size_t> FunctionPadding;
if (FunctionPadding.empty() && !FunctionPadSpec.empty()) {
for (auto &Spec : FunctionPadSpec) {
auto N = Spec.find(':');
for (std::string &Spec : FunctionPadSpec) {
size_t N = Spec.find(':');
if (N == std::string::npos)
continue;
auto Name = Spec.substr(0, N);
auto Padding = std::stoull(Spec.substr(N+1));
std::string Name = Spec.substr(0, N);
size_t Padding = std::stoull(Spec.substr(N+1));
FunctionPadding[Name] = Padding;
}
}
for (auto &FPI : FunctionPadding) {
auto Name = FPI.first;
auto Padding = FPI.second;
std::string Name = FPI.first;
size_t Padding = FPI.second;
if (Function.hasNameRegex(Name)) {
return Padding;
}
@ -192,7 +192,7 @@ private:
void BinaryEmitter::emitAll(StringRef OrgSecPrefix) {
Streamer.initSections(false, *BC.STI);
if (auto *RtLibrary = BC.getRuntimeLibrary()) {
if (RuntimeLibrary *RtLibrary = BC.getRuntimeLibrary()) {
RtLibrary->emitBinary(BC, Streamer);
}
@ -210,9 +210,9 @@ void BinaryEmitter::emitAll(StringRef OrgSecPrefix) {
void BinaryEmitter::emitFunctions() {
auto emit = [&](const std::vector<BinaryFunction *> &Functions) {
const auto HasProfile = BC.NumProfiledFuncs > 0;
const bool HasProfile = BC.NumProfiledFuncs > 0;
const bool OriginalAllowAutoPadding = Streamer.getAllowAutoPadding();
for (auto *Function : Functions) {
for (BinaryFunction *Function : Functions) {
if (!BC.shouldEmit(*Function)) {
continue;
}
@ -278,7 +278,7 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) {
if (BC.HasRelocations) {
Streamer.emitCodeAlignment(BinaryFunction::MinAlign, &*BC.STI);
auto MaxAlignBytes = EmitColdPart
uint16_t MaxAlignBytes = EmitColdPart
? Function.getMaxColdAlignmentBytes()
: Function.getMaxAlignmentBytes();
if (MaxAlignBytes > 0)
@ -313,15 +313,15 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) {
Streamer.emitCFIPersonality(Function.getPersonalityFunction(),
Function.getPersonalityEncoding());
}
auto *LSDASymbol = EmitColdPart ? Function.getColdLSDASymbol()
: Function.getLSDASymbol();
MCSymbol *LSDASymbol =
EmitColdPart ? Function.getColdLSDASymbol() : Function.getLSDASymbol();
if (LSDASymbol) {
Streamer.emitCFILsda(LSDASymbol, BC.LSDAEncoding);
} else {
Streamer.emitCFILsda(0, dwarf::DW_EH_PE_omit);
}
// Emit CFI instructions relative to the CIE
for (const auto &CFIInstr : Function.cie()) {
for (const MCCFIInstruction &CFIInstr : Function.cie()) {
// Only write CIE CFI insns that LLVM will not already emit
const std::vector<MCCFIInstruction> &FrameInstrs =
MAI->getInitialFrameState();
@ -336,7 +336,7 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) {
// Emit UD2 at the beginning if requested by user.
if (!opts::BreakFunctionNames.empty()) {
for (auto &Name : opts::BreakFunctionNames) {
for (std::string &Name : opts::BreakFunctionNames) {
if (Function.hasNameRegex(Name)) {
Streamer.emitIntValue(0x0B0F, 2); // UD2: 0F 0B
break;
@ -348,7 +348,7 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) {
emitFunctionBody(Function, EmitColdPart, /*EmitCodeOnly=*/false);
// Emit padding if requested.
if (auto Padding = opts::padFunction(Function)) {
if (size_t Padding = opts::padFunction(Function)) {
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: padding function " << Function << " with "
<< Padding << " bytes\n");
Streamer.emitFill(Padding, MAI->getTextAlignFillValue());
@ -389,7 +389,7 @@ void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, bool EmitColdPart,
// Track the first emitted instruction with debug info.
bool FirstInstr = true;
for (auto BB : BF.layout()) {
for (BinaryBasicBlock *BB : BF.layout()) {
if (EmitColdPart != BB->isCold())
continue;
@ -400,7 +400,7 @@ void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, bool EmitColdPart,
}
Streamer.emitLabel(BB->getLabel());
if (!EmitCodeOnly) {
if (auto *EntrySymbol = BF.getSecondaryEntryPointSymbol(*BB)) {
if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(*BB)) {
Streamer.emitLabel(EntrySymbol);
}
}
@ -421,14 +421,14 @@ void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, bool EmitColdPart,
// Remember if the last instruction emitted was a prefix.
bool LastIsPrefix = false;
for (auto I = BB->begin(), E = BB->end(); I != E; ++I) {
auto &Instr = *I;
MCInst &Instr = *I;
if (EmitCodeOnly && BC.MII->get(Instr.getOpcode()).isPseudo())
continue;
// Handle pseudo instructions.
if (BC.MIB->isEHLabel(Instr)) {
const auto *Label = BC.MIB->getTargetSymbol(Instr);
const MCSymbol *Label = BC.MIB->getTargetSymbol(Instr);
assert(Instr.getNumOperands() >= 1 && Label &&
"bad EH_LABEL instruction");
Streamer.emitLabel(const_cast<MCSymbol *>(Label));
@ -578,7 +578,7 @@ void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
++IS;
}
if (RI != BF.getMoveRelocations().end() && FunctionOffset == RI->first) {
auto RelocationSize = RI->second.emit(&Streamer);
size_t RelocationSize = RI->second.emit(&Streamer);
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted relocation for symbol "
<< RI->second.Symbol->getName() << " at offset 0x"
<< Twine::utohexstr(RI->first) << " with size "
@ -598,7 +598,7 @@ void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
return;
// Now emit constant islands from other functions that we may have used in
// this function.
for (auto *ExternalFunc : Islands.Dependency) {
for (BinaryFunction *ExternalFunc : Islands.Dependency) {
emitConstantIslands(*ExternalFunc, EmitColdPart, &BF);
}
}
@ -624,8 +624,8 @@ SMLoc BinaryEmitter::emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc,
// have come across some inlined code. We must look up the CU
// for the instruction's original function and get the line table
// from that.
const auto FunctionUnitIndex = FunctionCU->getOffset();
const auto CurrentUnitIndex = RowReference.DwCompileUnitIndex;
const uint64_t FunctionUnitIndex = FunctionCU->getOffset();
const uint32_t CurrentUnitIndex = RowReference.DwCompileUnitIndex;
if (CurrentUnitIndex != FunctionUnitIndex) {
CurrentLineTable = BC.DwCtx->getLineTableForUnit(
BC.DwCtx->getCompileUnitForOffset(CurrentUnitIndex));
@ -635,7 +635,8 @@ SMLoc BinaryEmitter::emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc,
CurrentLineTable->Rows[RowReference.RowIndex - 1].File);
}
const auto &CurrentRow = CurrentLineTable->Rows[RowReference.RowIndex - 1];
const DWARFDebugLine::Row &CurrentRow =
CurrentLineTable->Rows[RowReference.RowIndex - 1];
if (!CurrentFilenum)
CurrentFilenum = CurrentRow.File;
@ -673,7 +674,7 @@ void BinaryEmitter::emitJumpTables(const BinaryFunction &BF) {
}
for (auto &JTI : BF.jumpTables()) {
auto &JT = *JTI.second;
JumpTable &JT = *JTI.second;
if (opts::PrintJumpTables)
JT.print(outs());
if ((opts::JumpTables == JTS_BASIC || !BF.isSimple()) &&
@ -684,9 +685,8 @@ void BinaryEmitter::emitJumpTables(const BinaryFunction &BF) {
if (opts::JumpTables == JTS_BASIC) {
std::string Name = ".local." + JT.Labels[0]->getName().str();
std::replace(Name.begin(), Name.end(), '/', '.');
auto &Section = BC.registerOrUpdateSection(Name,
ELF::SHT_PROGBITS,
ELF::SHF_ALLOC);
BinarySection &Section =
BC.registerOrUpdateSection(Name, ELF::SHT_PROGBITS, ELF::SHF_ALLOC);
Section.setAnonymous(true);
JT.setOutputSection(Section);
HotSection = BC.getDataSection(Name);
@ -730,7 +730,7 @@ void BinaryEmitter::emitJumpTable(const JumpTable &JT, MCSection *HotSection,
}
MCSymbol *LastLabel = nullptr;
uint64_t Offset = 0;
for (auto *Entry : JT.Entries) {
for (MCSymbol *Entry : JT.Entries) {
auto LI = JT.Labels.find(Offset);
if (LI != JT.Labels.end()) {
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitting jump table "
@ -754,9 +754,12 @@ void BinaryEmitter::emitJumpTable(const JumpTable &JT, MCSection *HotSection,
if (JT.Type == JumpTable::JTT_NORMAL) {
Streamer.emitSymbolValue(Entry, JT.OutputEntrySize);
} else { // JTT_PIC
auto JTExpr = MCSymbolRefExpr::create(LastLabel, Streamer.getContext());
auto E = MCSymbolRefExpr::create(Entry, Streamer.getContext());
auto Value = MCBinaryExpr::createSub(E, JTExpr, Streamer.getContext());
const MCSymbolRefExpr *JTExpr =
MCSymbolRefExpr::create(LastLabel, Streamer.getContext());
const MCSymbolRefExpr *E =
MCSymbolRefExpr::create(Entry, Streamer.getContext());
const MCBinaryExpr *Value =
MCBinaryExpr::createSub(E, JTExpr, Streamer.getContext());
Streamer.emitValue(Value, JT.EntrySize);
}
Offset += JT.EntrySize;
@ -812,8 +815,8 @@ void BinaryEmitter::emitCFIInstruction(const MCCFIInstruction &Inst) const {
// The code is based on EHStreamer::emitExceptionTable().
void BinaryEmitter::emitLSDA(BinaryFunction &BF, bool EmitColdPart) {
const auto *Sites =
EmitColdPart ? &BF.getColdCallSites() : &BF.getCallSites();
const std::vector<BinaryFunction::CallSite> *Sites =
EmitColdPart ? &BF.getColdCallSites() : &BF.getCallSites();
if (Sites->empty()) {
return;
}
@ -826,21 +829,22 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, bool EmitColdPart) {
//
// sizeof(dwarf::DW_EH_PE_data4) * 3 + sizeof(uleb128(action))
uint64_t CallSiteTableLength = Sites->size() * 4 * 3;
for (const auto &CallSite : *Sites) {
for (const BinaryFunction::CallSite &CallSite : *Sites) {
CallSiteTableLength += getULEB128Size(CallSite.Action);
}
Streamer.SwitchSection(BC.MOFI->getLSDASection());
const auto TTypeEncoding = BC.TTypeEncoding;
const auto TTypeEncodingSize = BC.getDWARFEncodingSize(TTypeEncoding);
const auto TTypeAlignment = 4;
const unsigned TTypeEncoding = BC.TTypeEncoding;
const unsigned TTypeEncodingSize = BC.getDWARFEncodingSize(TTypeEncoding);
const uint16_t TTypeAlignment = 4;
// Type tables have to be aligned at 4 bytes.
Streamer.emitValueToAlignment(TTypeAlignment);
// Emit the LSDA label.
auto *LSDASymbol = EmitColdPart ? BF.getColdLSDASymbol() : BF.getLSDASymbol();
MCSymbol *LSDASymbol =
EmitColdPart ? BF.getColdLSDASymbol() : BF.getLSDASymbol();
assert(LSDASymbol && "no LSDA symbol set");
Streamer.emitLabel(LSDASymbol);
@ -921,9 +925,9 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, bool EmitColdPart) {
Streamer.emitIntValue(dwarf::DW_EH_PE_sdata4, 1);
Streamer.emitULEB128IntValue(CallSiteTableLength);
for (const auto &CallSite : *Sites) {
const auto *BeginLabel = CallSite.Start;
const auto *EndLabel = CallSite.End;
for (const BinaryFunction::CallSite &CallSite : *Sites) {
const MCSymbol *BeginLabel = CallSite.Start;
const MCSymbol *EndLabel = CallSite.End;
assert(BeginLabel && "start EH label expected");
assert(EndLabel && "end EH label expected");
@ -944,13 +948,13 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, bool EmitColdPart) {
//
// For type table we (re-)encode the table using TTypeEncoding matching
// the current assembler mode.
for (auto const &Byte : BF.getLSDAActionTable()) {
for (uint8_t const &Byte : BF.getLSDAActionTable()) {
Streamer.emitIntValue(Byte, 1);
}
const auto &TypeTable = (TTypeEncoding & dwarf::DW_EH_PE_indirect)
? BF.getLSDATypeAddressTable()
: BF.getLSDATypeTable();
const BinaryFunction::LSDATypeTableTy &TypeTable =
(TTypeEncoding & dwarf::DW_EH_PE_indirect) ? BF.getLSDATypeAddressTable()
: BF.getLSDATypeTable();
assert(TypeTable.size() == BF.getLSDATypeTable().size() &&
"indirect type table size mismatch");
@ -968,10 +972,9 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, bool EmitColdPart) {
BC.getOrCreateGlobalSymbol(TypeAddress, "TI", 0, TTypeAlignment);
MCSymbol *DotSymbol = BC.Ctx->createNamedTempSymbol();
Streamer.emitLabel(DotSymbol);
const auto *SubDotExpr = MCBinaryExpr::createSub(
const MCBinaryExpr *SubDotExpr = MCBinaryExpr::createSub(
MCSymbolRefExpr::create(TypeSymbol, *BC.Ctx),
MCSymbolRefExpr::create(DotSymbol, *BC.Ctx),
*BC.Ctx);
MCSymbolRefExpr::create(DotSymbol, *BC.Ctx), *BC.Ctx);
Streamer.emitValue(SubDotExpr, TTypeEncodingSize);
} else {
Streamer.emitIntValue(0, TTypeEncodingSize);
@ -980,14 +983,14 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, bool EmitColdPart) {
}
}
}
for (auto const &Byte : BF.getLSDATypeIndexTable()) {
for (uint8_t const &Byte : BF.getLSDATypeIndexTable()) {
Streamer.emitIntValue(Byte, 1);
}
}
void BinaryEmitter::emitDebugLineInfoForOriginalFunctions() {
for (auto &It : BC.getBinaryFunctions()) {
const auto &Function = It.second;
const BinaryFunction &Function = It.second;
// If the function was emitted, its line info was emitted with it.
if (Function.isEmitted())
@ -1006,10 +1009,10 @@ void BinaryEmitter::emitDebugLineInfoForOriginalFunctions() {
uint64_t Address = It.first;
if (LineTable->lookupAddressRange({Address, 0}, Function.getMaxSize(),
Results)) {
auto &OutputLineTable =
MCLineSection &OutputLineTable =
BC.Ctx->getMCDwarfLineTable(Unit->getOffset()).getMCLineSections();
for (auto RowIndex : Results) {
const auto &Row = LineTable->Rows[RowIndex];
for (uint32_t RowIndex : Results) {
const DWARFDebugLine::Row &Row = LineTable->Rows[RowIndex];
BC.Ctx->setCurrentDwarfLoc(
Row.File,
Row.Line,
@ -1021,7 +1024,7 @@ void BinaryEmitter::emitDebugLineInfoForOriginalFunctions() {
Row.Isa,
Row.Discriminator,
Row.Address.Address);
auto Loc = BC.Ctx->getCurrentDwarfLoc();
MCDwarfLoc Loc = BC.Ctx->getCurrentDwarfLoc();
BC.Ctx->clearDwarfLocSeen();
OutputLineTable.addLineEntry(MCDwarfLineEntry{nullptr, Loc},
FunctionSection);
@ -1030,7 +1033,7 @@ void BinaryEmitter::emitDebugLineInfoForOriginalFunctions() {
// for end_sequence mark.
BC.Ctx->setCurrentDwarfLoc(0, 0, 0, 0, 0, 0,
Address + Function.getMaxSize());
auto Loc = BC.Ctx->getCurrentDwarfLoc();
MCDwarfLoc Loc = BC.Ctx->getCurrentDwarfLoc();
BC.Ctx->clearDwarfLocSeen();
OutputLineTable.addLineEntry(MCDwarfLineEntry{nullptr, Loc},
FunctionSection);
@ -1070,7 +1073,7 @@ void BinaryEmitter::emitFunctionBodyRaw(BinaryFunction &BF) {
(LI == BF.getLabels().end() ? BF.getSize() : LI->first);
uint64_t NextRelocationOffset =
(RI == BF.getMoveRelocations().end() ? BF.getSize() : RI->first);
auto NextStop = std::min(NextLabelOffset, NextRelocationOffset);
uint64_t NextStop = std::min(NextLabelOffset, NextRelocationOffset);
assert(NextStop <= BF.getSize() && "internal overflow error");
if (FunctionOffset < NextStop) {
Streamer.emitBytes(FunctionContents.slice(FunctionOffset, NextStop));
@ -1084,7 +1087,7 @@ void BinaryEmitter::emitFunctionBodyRaw(BinaryFunction &BF) {
++LI;
}
if (RI != BF.getMoveRelocations().end() && FunctionOffset == RI->first) {
auto RelocationSize = RI->second.emit(&Streamer);
size_t RelocationSize = RI->second.emit(&Streamer);
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted relocation for symbol "
<< RI->second.Symbol->getName() << " at offset 0x"
<< Twine::utohexstr(RI->first) << " with size "
@ -1100,7 +1103,7 @@ void BinaryEmitter::emitFunctionBodyRaw(BinaryFunction &BF) {
}
void BinaryEmitter::emitDataSections(StringRef OrgSecPrefix) {
for (auto &Section : BC.sections()) {
for (BinarySection &Section : BC.sections()) {
if (!Section.hasRelocations() || !Section.hasSectionRef())
continue;

609
bolt/src/BinaryFunction.cpp
View File

File diff suppressed because it is too large Load Diff

57
bolt/src/BinaryFunction.h
View File

@ -94,7 +94,7 @@ inline raw_ostream &operator<<(raw_ostream &OS,
const char *Sep = "\n ";
uint64_t TotalCount = 0;
uint64_t TotalMispreds = 0;
for (auto &CSP : ICSP) {
for (const IndirectCallProfile &CSP : ICSP) {
SS << Sep << "{ " << (CSP.Symbol ? CSP.Symbol->getName() : "<unknown>")
<< ": " << CSP.Count << " (" << CSP.Mispreds << " misses) }";
Sep = ",\n ";
@ -179,6 +179,8 @@ public:
/// Mark injected functions
bool IsInjected = false;
using LSDATypeTableTy = std::vector<uint64_t>;
private:
/// Current state of the function.
State CurrentState{State::Empty};
@ -496,8 +498,6 @@ private:
ArrayRef<uint8_t> LSDAActionTable;
ArrayRef<uint8_t> LSDATypeIndexTable;
using LSDATypeTableTy = std::vector<uint64_t>;
/// Vector of addresses of types referenced by LSDA.
LSDATypeTableTy LSDATypeTable;
@ -700,10 +700,10 @@ private:
/// Release memory allocated for CFG and instructions.
/// We still keep basic blocks for address translation/mapping purposes.
void releaseCFG() {
for (auto *BB : BasicBlocks) {
for (BinaryBasicBlock *BB : BasicBlocks) {
BB->releaseCFG();
}
for (auto BB : DeletedBasicBlocks) {
for (BinaryBasicBlock *BB : DeletedBasicBlocks) {
BB->releaseCFG();
}
@ -951,9 +951,9 @@ public:
BinaryBasicBlock *getLandingPadBBFor(const BinaryBasicBlock &BB,
const MCInst &InvokeInst) const {
assert(BC.MIB->isInvoke(InvokeInst) && "must be invoke instruction");
const auto LP = BC.MIB->getEHInfo(InvokeInst);
const Optional<MCPlus::MCLandingPad> LP = BC.MIB->getEHInfo(InvokeInst);
if (LP && LP->first) {
auto *LBB = BB.getLandingPad(LP->first);
BinaryBasicBlock *LBB = BB.getLandingPad(LP->first);
assert (LBB && "Landing pad should be defined");
return LBB;
}
@ -998,7 +998,7 @@ public:
/// primary: <function>/<id>
/// alternative: <function>/<file>/<id2>
std::string getPrintName() const {
const auto NumNames = Symbols.size() + Aliases.size();
const size_t NumNames = Symbols.size() + Aliases.size();
return NumNames == 1
? getOneName().str()
: (getOneName().str() + "(*" + std::to_string(NumNames) + ")");
@ -1022,11 +1022,11 @@ public:
/// Return the name for which the Callback returned true if any.
template <typename FType>
Optional<StringRef> forEachName(FType Callback) const {
for (auto *Symbol : Symbols)
for (MCSymbol *Symbol : Symbols)
if (Callback(Symbol->getName()))
return Symbol->getName();
for (auto &Name : Aliases)
for (const std::string &Name : Aliases)
if (Callback(StringRef(Name)))
return StringRef(Name);
@ -1151,7 +1151,7 @@ public:
/// Return the number of emitted instructions for this function.
uint32_t getNumNonPseudos() const {
uint32_t N = 0;
for (auto &BB : layout()) {
for (BinaryBasicBlock *const &BB : layout()) {
N += BB->getNumNonPseudos();
}
return N;
@ -1301,7 +1301,7 @@ public:
uint64_t Addend, uint64_t Value) {
assert(Address >= getAddress() && Address < getAddress() + getMaxSize() &&
"address is outside of the function");
auto Offset = Address - getAddress();
uint64_t Offset = Address - getAddress();
switch (RelType) {
case ELF::R_X86_64_8:
case ELF::R_X86_64_16:
@ -1492,12 +1492,12 @@ public:
}
const JumpTable *getJumpTable(const MCInst &Inst) const {
const auto Address = BC.MIB->getJumpTable(Inst);
const uint64_t Address = BC.MIB->getJumpTable(Inst);
return getJumpTableContainingAddress(Address);
}
JumpTable *getJumpTable(const MCInst &Inst) {
const auto Address = BC.MIB->getJumpTable(Inst);
const uint64_t Address = BC.MIB->getJumpTable(Inst);
return getJumpTableContainingAddress(Address);
}
@ -1609,10 +1609,11 @@ public:
std::unique_lock<std::shared_timed_mutex> Lock(BC.CtxMutex);
Label = BC.Ctx->createNamedTempSymbol("BB");
}
auto BBPtr = createBasicBlock(Offset, Label, DeriveAlignment);
std::unique_ptr<BinaryBasicBlock> BBPtr =
createBasicBlock(Offset, Label, DeriveAlignment);
BasicBlocks.emplace_back(BBPtr.release());
auto *BB = BasicBlocks.back();
BinaryBasicBlock *BB = BasicBlocks.back();
BB->setIndex(BasicBlocks.size() - 1);
if (CurrentState == State::Disassembled) {
@ -1654,7 +1655,7 @@ public:
/// Return basic block range that originally contained offset \p Offset
/// from the function start to the function end.
iterator_range<iterator> getBasicBlockRangeFromOffsetToEnd(uint64_t Offset) {
auto *BB = getBasicBlockContainingOffset(Offset);
BinaryBasicBlock *BB = getBasicBlockContainingOffset(Offset);
return BB
? iterator_range<iterator>(BasicBlocks.begin() + getIndex(BB), end())
: iterator_range<iterator>(end(), end());
@ -1687,7 +1688,7 @@ public:
/// Make sure basic blocks' indices match the current layout.
void updateLayoutIndices() const {
unsigned Index = 0;
for (auto *BB : layout()) {
for (BinaryBasicBlock *BB : layout()) {
BB->setLayoutIndex(Index++);
}
}
@ -1817,7 +1818,7 @@ public:
BinaryBasicBlock::iterator addCFIInstruction(BinaryBasicBlock *BB,
BinaryBasicBlock::iterator Pos,
MCCFIInstruction &&Inst) {
auto Idx = FrameInstructions.size();
size_t Idx = FrameInstructions.size();
FrameInstructions.emplace_back(std::forward<MCCFIInstruction>(Inst));
return addCFIPseudo(BB, Pos, Idx);
}
@ -2109,7 +2110,7 @@ public:
Symbol = BC.getOrCreateGlobalSymbol(Address, "ISLANDat");
// Internal bookkeeping
const auto Offset = Address - getAddress();
const uint64_t Offset = Address - getAddress();
assert((!Islands.Offsets.count(Offset) ||
Islands.Offsets[Offset] == Symbol) &&
"Inconsistent island symbol management");
@ -2126,7 +2127,7 @@ public:
/// function.
MCSymbol *
getOrCreateProxyIslandAccess(uint64_t Address, BinaryFunction &Referrer) {
auto Symbol = getOrCreateIslandAccess(Address);
MCSymbol *Symbol = getOrCreateIslandAccess(Address);
if (!Symbol)
return nullptr;
@ -2148,7 +2149,7 @@ public:
if (Address < getAddress())
return false;
auto Offset = Address - getAddress();
uint64_t Offset = Address - getAddress();
if (Offset >= getMaxSize())
return false;
@ -2191,7 +2192,7 @@ public:
}
if (!OnBehalfOf) {
for (auto *ExternalFunc : Islands.Dependency)
for (BinaryFunction *ExternalFunc : Islands.Dependency)
Size += ExternalFunc->estimateConstantIslandSize(this);
}
return Size;
@ -2431,13 +2432,13 @@ public:
size_t estimateHotSize(const bool UseSplitSize = true) const {
size_t Estimate = 0;
if (UseSplitSize && isSplit()) {
for (const auto *BB : BasicBlocksLayout) {
for (const BinaryBasicBlock *BB : BasicBlocksLayout) {
if (!BB->isCold()) {
Estimate += BC.computeCodeSize(BB->begin(), BB->end());
}
}
} else {
for (const auto *BB : BasicBlocksLayout) {
for (const BinaryBasicBlock *BB : BasicBlocksLayout) {
if (BB->getKnownExecutionCount() != 0) {
Estimate += BC.computeCodeSize(BB->begin(), BB->end());
}
@ -2450,7 +2451,7 @@ public:
if (!isSplit())
return estimateSize();
size_t Estimate = 0;
for (const auto *BB : BasicBlocksLayout) {
for (const BinaryBasicBlock *BB : BasicBlocksLayout) {
if (BB->isCold()) {
Estimate += BC.computeCodeSize(BB->begin(), BB->end());
}
@ -2460,7 +2461,7 @@ public:
size_t estimateSize() const {
size_t Estimate = 0;
for (const auto *BB : BasicBlocksLayout) {
for (const BinaryBasicBlock *BB : BasicBlocksLayout) {
Estimate += BC.computeCodeSize(BB->begin(), BB->end());
}
return Estimate;
@ -2519,7 +2520,7 @@ public:
/// Mark child fragments as ignored.
void ignoreFragments() {
for (auto *Fragment : Fragments)
for (BinaryFunction *Fragment : Fragments)
Fragment->setIgnored();
}
};

50
bolt/src/BinaryFunctionProfile.cpp
View File

@ -101,14 +101,14 @@ void BinaryFunction::postProcessProfile() {
}
// Compute preliminary execution count for each basic block.
for (auto *BB : BasicBlocks) {
for (BinaryBasicBlock *BB : BasicBlocks) {
if ((!BB->isEntryPoint() && !BB->isLandingPad()) ||
BB->ExecutionCount == BinaryBasicBlock::COUNT_NO_PROFILE)
BB->ExecutionCount = 0;
}
for (auto *BB : BasicBlocks) {
for (BinaryBasicBlock *BB : BasicBlocks) {
auto SuccBIIter = BB->branch_info_begin();
for (auto Succ : BB->successors()) {
for (BinaryBasicBlock *Succ : BB->successors()) {
// All incoming edges to the primary entry have been accounted for, thus
// we skip the update here.
if (SuccBIIter->Count != BinaryBasicBlock::COUNT_NO_PROFILE &&
@ -119,12 +119,12 @@ void BinaryFunction::postProcessProfile() {
}
if (opts::FixBlockCounts) {
for (auto *BB : BasicBlocks) {
for (BinaryBasicBlock *BB : BasicBlocks) {
// Make sure that execution count of a block is at least the branch count
// of an incoming/outgoing jump.
auto SuccBIIter = BB->branch_info_begin();
for (auto Succ : BB->successors()) {
auto Count = SuccBIIter->Count;
for (BinaryBasicBlock *Succ : BB->successors()) {
uint64_t Count = SuccBIIter->Count;
if (Count != BinaryBasicBlock::COUNT_NO_PROFILE && Count > 0) {
Succ->setExecutionCount(std::max(Succ->getExecutionCount(), Count));
BB->setExecutionCount(std::max(BB->getExecutionCount(), Count));
@ -133,7 +133,7 @@ void BinaryFunction::postProcessProfile() {
}
// Make sure that execution count of a block is at least the number of
// function calls from the block.
for (auto &Inst : *BB) {
for (MCInst &Inst : *BB) {
// Ignore non-call instruction
if (!BC.MIB->isCall(Inst))
continue;
@ -156,19 +156,20 @@ void BinaryFunction::postProcessProfile() {
}
// Update profile information for jump tables based on CFG branch data.
for (auto *BB : BasicBlocks) {
const auto *LastInstr = BB->getLastNonPseudoInstr();
for (BinaryBasicBlock *BB : BasicBlocks) {
const MCInst *LastInstr = BB->getLastNonPseudoInstr();
if (!LastInstr)
continue;
const auto JTAddress = BC.MIB->getJumpTable(*LastInstr);
const uint64_t JTAddress = BC.MIB->getJumpTable(*LastInstr);
if (!JTAddress)
continue;
auto *JT = getJumpTableContainingAddress(JTAddress);
JumpTable *JT = getJumpTableContainingAddress(JTAddress);
if (!JT)
continue;
uint64_t TotalBranchCount = 0;
for (const auto &BranchInfo : BB->branch_info()) {
for (const BinaryBasicBlock::BinaryBranchInfo &BranchInfo :
BB->branch_info()) {
TotalBranchCount += BranchInfo.Count;
}
JT->Count += TotalBranchCount;
@ -180,12 +181,13 @@ void BinaryFunction::postProcessProfile() {
if (JT->Counts.empty())
JT->Counts.resize(JT->Entries.size());
auto EI = JT->Entries.begin();
auto Delta = (JTAddress - JT->getAddress()) / JT->EntrySize;
uint64_t Delta = (JTAddress - JT->getAddress()) / JT->EntrySize;
EI += Delta;
while (EI != JT->Entries.end()) {
const auto *TargetBB = getBasicBlockForLabel(*EI);
const BinaryBasicBlock *TargetBB = getBasicBlockForLabel(*EI);
if (TargetBB) {
const auto &BranchInfo = BB->getBranchInfo(*TargetBB);
const BinaryBasicBlock::BinaryBranchInfo &BranchInfo =
BB->getBranchInfo(*TargetBB);
assert(Delta < JT->Counts.size());
JT->Counts[Delta].Count += BranchInfo.Count;
JT->Counts[Delta].Mispreds += BranchInfo.MispredictedCount;
@ -229,7 +231,7 @@ void BinaryFunction::mergeProfileDataInto(BinaryFunction &BF) const {
auto BBMergeSI = BBMerge->succ_begin();
auto BIMergeI = BBMerge->branch_info_begin();
auto BII = BB->branch_info_begin();
for (const auto *BBSucc : BB->successors()) {
for (const BinaryBasicBlock *BBSucc : BB->successors()) {
(void)BBSucc;
assert(getIndex(BBSucc) == BF.getIndex(*BBMergeSI));
@ -266,7 +268,7 @@ void BinaryFunction::mergeProfileDataInto(BinaryFunction &BF) const {
if (JTMergeI->second->Counts.empty())
JTMergeI->second->Counts.resize(JTEntry.second->Counts.size());
auto CountMergeI = JTMergeI->second->Counts.begin();
for (const auto &JI : JTEntry.second->Counts) {
for (const JumpTable::JumpInfo &JI : JTEntry.second->Counts) {
CountMergeI->Count += JI.Count;
CountMergeI->Mispreds += JI.Mispreds;
++CountMergeI;
@ -282,7 +284,7 @@ void BinaryFunction::inferFallThroughCounts() {
// Work on a basic block at a time, propagating frequency information
// forwards.
// It is important to walk in the layout order.
for (auto *BB : BasicBlocks) {
for (BinaryBasicBlock *BB : BasicBlocks) {
const uint64_t BBExecCount = BB->getExecutionCount();
// Propagate this information to successors, filling in fall-through edges
@ -293,14 +295,14 @@ void BinaryFunction::inferFallThroughCounts() {
// Calculate frequency of outgoing branches from this node according to
// LBR data.
uint64_t ReportedBranches = 0;
for (const auto &SuccBI : BB->branch_info()) {
for (const BinaryBasicBlock::BinaryBranchInfo &SuccBI : BB->branch_info()) {
if (SuccBI.Count != BinaryBasicBlock::COUNT_NO_PROFILE)
ReportedBranches += SuccBI.Count;
}
// Get taken count of conditional tail call if the block ends with one.
uint64_t CTCTakenCount = 0;
const auto CTCInstr = BB->getLastNonPseudoInstr();
const MCInst *CTCInstr = BB->getLastNonPseudoInstr();
if (CTCInstr && BC.MIB->getConditionalTailCall(*CTCInstr)) {
CTCTakenCount =
BC.MIB->getAnnotationWithDefault<uint64_t>(*CTCInstr, "CTCTakenCount");
@ -311,7 +313,7 @@ void BinaryFunction::inferFallThroughCounts() {
// for a landing pad to be associated with more than one basic blocks,
// we may overestimate the frequency of throws for such blocks.
uint64_t ReportedThrows = 0;
for (const auto *LP: BB->landing_pads()) {
for (const BinaryBasicBlock *LP : BB->landing_pads()) {
ReportedThrows += LP->getExecutionCount();
}
@ -336,7 +338,7 @@ void BinaryFunction::inferFallThroughCounts() {
if (BB->succ_size() <= 2) {
// Skip if the last instruction is an unconditional jump.
const auto *LastInstr = BB->getLastNonPseudoInstr();
const MCInst *LastInstr = BB->getLastNonPseudoInstr();
if (LastInstr &&
(BC.MIB->isUnconditionalBranch(*LastInstr) ||
BC.MIB->isIndirectBranch(*LastInstr)))
@ -358,9 +360,9 @@ void BinaryFunction::inferFallThroughCounts() {
void BinaryFunction::clearProfile() {
// Keep function execution profile the same. Only clear basic block and edge
// counts.
for (auto *BB : BasicBlocks) {
for (BinaryBasicBlock *BB : BasicBlocks) {
BB->ExecutionCount = 0;
for (auto &BI : BB->branch_info()) {
for (BinaryBasicBlock::BinaryBranchInfo &BI : BB->branch_info()) {
BI.Count = 0;
BI.MispredictedCount = 0;
}

10
bolt/src/BinaryPassManager.cpp
View File

@ -320,12 +320,14 @@ const char BinaryFunctionPassManager::TimerGroupDesc[] =
void BinaryFunctionPassManager::runPasses() {
auto &BFs = BC.getBinaryFunctions();
for (size_t PassIdx = 0; PassIdx < Passes.size(); PassIdx++) {
const auto &OptPassPair = Passes[PassIdx];
const std::pair<const bool, std::unique_ptr<BinaryFunctionPass>>
&OptPassPair = Passes[PassIdx];
if (!OptPassPair.first)
continue;
auto &Pass = OptPassPair.second;
auto PassIdName = formatv("{0:2}_{1}", PassIdx, Pass->getName()).str();
const std::unique_ptr<BinaryFunctionPass> &Pass = OptPassPair.second;
std::string PassIdName =
formatv("{0:2}_{1}", PassIdx, Pass->getName()).str();
if (opts::Verbosity > 0) {
outs() << "BOLT-INFO: Starting pass: " << Pass->getName() << "\n";
@ -382,7 +384,7 @@ void BinaryFunctionPassManager::runPasses() {
void BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
BinaryFunctionPassManager Manager(BC);
const auto InitialDynoStats = getDynoStats(BC.getBinaryFunctions());
const DynoStats InitialDynoStats = getDynoStats(BC.getBinaryFunctions());
if (opts::Instrument) {
Manager.registerPass(std::make_unique<Instrumentation>(NeverPrint));

43
bolt/src/BinarySection.cpp
View File

@ -41,21 +41,21 @@ BinarySection::hash(const BinaryData &BD,
Cache[&BD] = 0;
auto Offset = BD.getAddress() - getAddress();
const auto EndOffset = BD.getEndAddress() - getAddress();
uint64_t Offset = BD.getAddress() - getAddress();
const uint64_t EndOffset = BD.getEndAddress() - getAddress();
auto Begin = Relocations.lower_bound(Relocation{Offset, 0, 0, 0, 0});
auto End = Relocations.upper_bound(Relocation{EndOffset, 0, 0, 0, 0});
const auto Contents = getContents();
const StringRef Contents = getContents();
hash_code Hash = hash_combine(hash_value(BD.getSize()),
hash_value(BD.getSectionName()));
while (Begin != End) {
const auto &Rel = *Begin++;
const Relocation &Rel = *Begin++;
Hash = hash_combine(
Hash,
hash_value(Contents.substr(Offset, Begin->Offset - Offset)));
if (auto *RelBD = BC.getBinaryDataByName(Rel.Symbol->getName())) {
if (BinaryData *RelBD = BC.getBinaryDataByName(Rel.Symbol->getName())) {
Hash = hash_combine(Hash, hash(*RelBD, Cache));
}
Offset = Rel.Offset + Rel.getSize();
@ -73,9 +73,8 @@ BinarySection::hash(const BinaryData &BD,
void BinarySection::emitAsData(MCStreamer &Streamer, StringRef NewName) const {
StringRef SectionName = !NewName.empty() ? NewName : getName();
StringRef SectionContents = getContents();
auto *ELFSection = BC.Ctx->getELFSection(SectionName,
getELFType(),
getELFFlags());
MCSectionELF *ELFSection =
BC.Ctx->getELFSection(SectionName, getELFType(), getELFFlags());
Streamer.SwitchSection(ELFSection);
Streamer.emitValueToAlignment(getAlignment());
@ -91,7 +90,7 @@ void BinarySection::emitAsData(MCStreamer &Streamer, StringRef NewName) const {
Streamer.emitBytes(SectionContents);
} else {
uint64_t SectionOffset = 0;
for (auto &Relocation : relocations()) {
for (const Relocation &Relocation : relocations()) {
assert(Relocation.Offset < SectionContents.size() && "overflow detected");
// Skip undefined symbols.
if (BC.UndefinedSymbols.count(Relocation.Symbol))
@ -108,7 +107,7 @@ void BinarySection::emitAsData(MCStreamer &Streamer, StringRef NewName) const {
<< " at offset 0x"
<< Twine::utohexstr(Relocation.Offset) << " with size "
<< Relocation::getSizeForType(Relocation.Type) << '\n');
auto RelocationSize = Relocation.emit(&Streamer);
size_t RelocationSize = Relocation.emit(&Streamer);
SectionOffset += RelocationSize;
}
assert(SectionOffset <= SectionContents.size() && "overflow error");
@ -140,14 +139,14 @@ void BinarySection::flushPendingRelocations(raw_pwrite_stream &OS,
<< " address: 0x" << Twine::utohexstr(SectionAddress) << '\n'
<< " offset: 0x" << Twine::utohexstr(SectionFileOffset) << '\n');
for (auto &Patch : Patches) {
for (BinaryPatch &Patch : Patches) {
OS.pwrite(Patch.Bytes.data(),
Patch.Bytes.size(),
SectionFileOffset + Patch.Offset);
}
for (auto &Reloc : PendingRelocations) {
for (Relocation &Reloc : PendingRelocations) {
uint64_t Value = Reloc.Addend;
if (Reloc.Symbol)
Value += Resolver(Reloc.Symbol);
@ -227,7 +226,7 @@ void BinarySection::print(raw_ostream &OS) const {
OS << " (tls)";
if (opts::PrintRelocations) {
for (auto &R : relocations())
for (const Relocation &R : relocations())
OS << "\n " << R;
}
}
@ -236,17 +235,17 @@ std::set<Relocation> BinarySection::reorderRelocations(bool Inplace) const {
assert(PendingRelocations.empty() &&
"reodering pending relocations not supported");
std::set<Relocation> NewRelocations;
for (const auto &Rel : relocations()) {
auto RelAddr = Rel.Offset + getAddress();
auto *BD = BC.getBinaryDataContainingAddress(RelAddr);
for (const Relocation &Rel : relocations()) {
uint64_t RelAddr = Rel.Offset + getAddress();
BinaryData *BD = BC.getBinaryDataContainingAddress(RelAddr);
BD = BD->getAtomicRoot();
assert(BD);
if ((!BD->isMoved() && !Inplace) || BD->isJumpTable())
continue;
auto NewRel(Rel);
auto RelOffset = RelAddr - BD->getAddress();
Relocation NewRel(Rel);
uint64_t RelOffset = RelAddr - BD->getAddress();
NewRel.Offset = BD->getOutputOffset() + RelOffset;
assert(NewRel.Offset < getSize());
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: moving " << Rel << " -> " << NewRel
@ -266,12 +265,12 @@ void BinarySection::reorderContents(const std::vector<BinaryData *> &Order,
std::string Str;
raw_string_ostream OS(Str);
auto *Src = Contents.data();
const char *Src = Contents.data();
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: reorderContents for " << Name << "\n");
for (auto *BD : Order) {
for (BinaryData *BD : Order) {
assert((BD->isMoved() || !Inplace) && !BD->isJumpTable());
assert(BD->isAtomic() && BD->isMoveable());
const auto SrcOffset = BD->getAddress() - getAddress();
const uint64_t SrcOffset = BD->getAddress() - getAddress();
assert(SrcOffset < Contents.size());
assert(SrcOffset == BD->getOffset());
while (OS.tell() < BD->getOutputOffset()) {
@ -285,7 +284,7 @@ void BinarySection::reorderContents(const std::vector<BinaryData *> &Order,
// If there are no existing relocations, tack a phony one at the end
// of the reordered segment to force LLVM to recognize and map this
// section.
auto *ZeroSym = BC.registerNameAtAddress("Zero", 0, 0, 0);
MCSymbol *ZeroSym = BC.registerNameAtAddress("Zero", 0, 0, 0);
addRelocation(OS.tell(), ZeroSym, ELF::R_X86_64_64, 0xdeadbeef);
uint64_t Zero = 0;

6
bolt/src/BinarySection.h
View File

@ -107,9 +107,9 @@ class BinarySection {
ELFSectionRef(Section).getType() == ELF::SHT_NOBITS)
return StringRef();
auto ContentsOrErr = Section.getContents();
Expected<StringRef> ContentsOrErr = Section.getContents();
if (!ContentsOrErr) {
auto E = ContentsOrErr.takeError();
Error E = ContentsOrErr.takeError();
errs() << "BOLT-ERROR: cannot get section contents for "
<< getName(Section) << ": " << E << ".\n";
exit(1);
@ -501,7 +501,7 @@ public:
};
inline uint8_t *copyByteArray(const uint8_t *Data, uint64_t Size) {
auto Array = new uint8_t[Size];
auto *Array = new uint8_t[Size];
memcpy(Array, Data, Size);
return Array;
}

29
bolt/src/BoltAddressTranslation.cpp
View File

@ -23,8 +23,9 @@ const char* BoltAddressTranslation::SECTION_NAME = ".note.bolt_bat";
void BoltAddressTranslation::writeEntriesForBB(MapTy &Map,
const BinaryBasicBlock &BB,
uint64_t FuncAddress) {
const auto BBOutputOffset = BB.getOutputAddressRange().first - FuncAddress;
const auto BBInputOffset = BB.getInputOffset();
const uint64_t BBOutputOffset =
BB.getOutputAddressRange().first - FuncAddress;
const uint32_t BBInputOffset = BB.getInputOffset();
assert(BBInputOffset != BinaryBasicBlock::INVALID_OFFSET &&
"Every output BB must track back to an input BB for profile "
@ -43,8 +44,8 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map,
Map[BBOutputOffset] = BBInputOffset;
for (const auto &IOPair : BB.getOffsetTranslationTable()) {
const auto OutputOffset = IOPair.first + BBOutputOffset;
const auto InputOffset = IOPair.second;
const uint64_t OutputOffset = IOPair.first + BBOutputOffset;
const uint32_t InputOffset = IOPair.second;
// Is this the first instruction in the BB? No need to duplicate the entry.
if (OutputOffset == BBOutputOffset)
@ -60,7 +61,7 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map,
void BoltAddressTranslation::write(raw_ostream &OS) {
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Writing BOLT Address Translation Tables\n");
for (auto &BFI : BC.getBinaryFunctions()) {
auto &Function = BFI.second;
BinaryFunction &Function = BFI.second;
// We don't need a translation table if the body of the function hasn't
// changed
if (!BC.HasRelocations && !Function.isSimple())
@ -71,7 +72,7 @@ void BoltAddressTranslation::write(raw_ostream &OS) {
<< Twine::utohexstr(Function.getOutputAddress()) << "\n");
MapTy Map;
const bool IsSplit = Function.isSplit();
for (const auto &BB : Function.layout()) {
for (BinaryBasicBlock *&BB : Function.layout()) {
if (IsSplit && BB->isCold())
break;
writeEntriesForBB(Map, *BB, Function.getOutputAddress());
@ -84,7 +85,7 @@ void BoltAddressTranslation::write(raw_ostream &OS) {
// Cold map
Map.clear();
LLVM_DEBUG(dbgs() << " Cold part\n");
for (const auto &BB : Function.layout()) {
for (BinaryBasicBlock *&BB : Function.layout()) {
if (!BB->isCold())
continue;
writeEntriesForBB(Map, *BB, Function.cold().getAddress());
@ -105,7 +106,7 @@ void BoltAddressTranslation::write(raw_ostream &OS) {
<< Twine::utohexstr(Address) << ".\n");
OS.write(reinterpret_cast<const char *>(&Address), 8);
OS.write(reinterpret_cast<const char *>(&NumEntries), 4);
for (auto &KeyVal : Map) {
for (std::pair<const uint32_t, uint32_t> &KeyVal : Map) {
OS.write(reinterpret_cast<const char *>(&KeyVal.first), 4);
OS.write(reinterpret_cast<const char *>(&KeyVal.second), 4);
}
@ -114,7 +115,7 @@ void BoltAddressTranslation::write(raw_ostream &OS) {
LLVM_DEBUG(dbgs() << "Writing " << NumColdEntries
<< " cold part mappings.\n");
OS.write(reinterpret_cast<const char *>(&NumColdEntries), 4);
for (auto &ColdEntry : ColdPartSource) {
for (std::pair<const uint64_t, uint64_t> &ColdEntry : ColdPartSource) {
OS.write(reinterpret_cast<const char *>(&ColdEntry.first), 8);
OS.write(reinterpret_cast<const char *>(&ColdEntry.second), 8);
LLVM_DEBUG(dbgs() << " " << Twine::utohexstr(ColdEntry.first) << " -> "
@ -217,7 +218,7 @@ uint64_t BoltAddressTranslation::translate(const BinaryFunction &Func,
return Offset - KeyVal->first + Val;
}
Optional<SmallVector<std::pair<uint64_t, uint64_t>, 16>>
Optional<BoltAddressTranslation::FallthroughListTy>
BoltAddressTranslation::getFallthroughsInTrace(
const BinaryFunction &Func, uint64_t From, uint64_t To) const {
SmallVector<std::pair<uint64_t, uint64_t>, 16> Res;
@ -254,7 +255,7 @@ BoltAddressTranslation::getFallthroughsInTrace(
return Res;
for (auto Iter = FromIter; Iter != ToIter; ) {
const auto Src = Iter->first;
const uint32_t Src = Iter->first;
if (Iter->second & BRANCHENTRY) {
++Iter;
continue;
@ -281,9 +282,9 @@ uint64_t BoltAddressTranslation::fetchParentAddress(uint64_t Address) const {
bool BoltAddressTranslation::enabledFor(
llvm::object::ELFObjectFileBase *InputFile) const {
for (const auto &Section : InputFile->sections()) {
auto SectionNameOrErr = Section.getName();
if (auto E = SectionNameOrErr.takeError())
for (const SectionRef &Section : InputFile->sections()) {
Expected<StringRef> SectionNameOrErr = Section.getName();
if (Error E = SectionNameOrErr.takeError())
continue;
if (SectionNameOrErr.get() == SECTION_NAME)

5
bolt/src/BoltAddressTranslation.h
View File

@ -59,6 +59,9 @@ public:
// In-memory representation of the address translation table
using MapTy = std::map<uint32_t, uint32_t>;
// List of taken fall-throughs
using FallthroughListTy = SmallVector<std::pair<uint64_t, uint64_t>, 16>;
/// Name of the ELF section where the table will be serialized to in the
/// output binary
static const char *SECTION_NAME;
@ -82,7 +85,7 @@ public:
/// taken in the path started at FirstLBR.To and ending at SecondLBR.From.
/// Return NoneType if trace is invalid or the list of fall-throughs
/// otherwise.
Optional<SmallVector<std::pair<uint64_t, uint64_t>, 16>>
Optional<FallthroughListTy>
getFallthroughsInTrace(const BinaryFunction &Func, uint64_t From,
uint64_t To) const;

73
bolt/src/BoltDiff.cpp
View File

@ -213,11 +213,11 @@ class RewriteInstanceDiff {
void buildLookupMaps() {
for (const auto &BFI : RI1.BC->getBinaryFunctions()) {
StringRef LTOName;
const auto &Function = BFI.second;
const auto Score = getNormalizedScore(Function, RI1);
const BinaryFunction &Function = BFI.second;
const double Score = getNormalizedScore(Function, RI1);
LargestBin1.insert(std::make_pair<>(Score, &Function));
for (const auto Name : Function.getNames()) {
if (auto OptionalLTOName = getLTOCommonName(Name))
for (const StringRef Name : Function.getNames()) {
if (Optional<StringRef> OptionalLTOName = getLTOCommonName(Name))
LTOName = *OptionalLTOName;
NameLookup[Name] = &Function;
}
@ -233,11 +233,11 @@ class RewriteInstanceDiff {
// Compute LTONameLookup2 and LargestBin2
for (const auto &BFI : RI2.BC->getBinaryFunctions()) {
StringRef LTOName;
const auto &Function = BFI.second;
const auto Score = getNormalizedScore(Function, RI2);
const BinaryFunction &Function = BFI.second;
const double Score = getNormalizedScore(Function, RI2);
LargestBin2.insert(std::make_pair<>(Score, &Function));
for (const auto Name : Function.getNames()) {
if (auto OptionalLTOName = getLTOCommonName(Name))
for (const StringRef Name : Function.getNames()) {
if (Optional<StringRef> OptionalLTOName = getLTOCommonName(Name))
LTOName = *OptionalLTOName;
}
if (opts::IgnoreLTOSuffix && !LTOName.empty()) {
@ -255,12 +255,12 @@ class RewriteInstanceDiff {
std::set<const BinaryFunction *> Bin1ProfiledMapped;
for (const auto &BFI2 : RI2.BC->getBinaryFunctions()) {
const auto &Function2 = BFI2.second;
const BinaryFunction &Function2 = BFI2.second;
StringRef LTOName;
bool Match = false;
for (const auto Name : Function2.getNames()) {
for (const StringRef Name : Function2.getNames()) {
auto Iter = NameLookup.find(Name);
if (auto OptionalLTOName = getLTOCommonName(Name))
if (Optional<StringRef> OptionalLTOName = getLTOCommonName(Name))
LTOName = *OptionalLTOName;
if (Iter == NameLookup.end())
continue;
@ -312,7 +312,7 @@ class RewriteInstanceDiff {
outs() << "\nFunctions in profile 1 that are missing in the profile 2:\n";
std::vector<const BinaryFunction *> Unmapped;
for (const auto &BFI : RI1.BC->getBinaryFunctions()) {
const auto &Function = BFI.second;
const BinaryFunction &Function = BFI.second;
if (!Function.hasValidProfile() || Bin1ProfiledMapped.count(&Function))
continue;
Unmapped.emplace_back(&Function);
@ -321,7 +321,7 @@ class RewriteInstanceDiff {
[&](const BinaryFunction *A, const BinaryFunction *B) {
return A->getFunctionScore() > B->getFunctionScore();
});
for (auto Function : Unmapped) {
for (const BinaryFunction *Function : Unmapped) {
outs() << Function->getPrintName() << " : ";
outs() << Function->getFunctionScore() << "\n";
}
@ -357,8 +357,8 @@ class RewriteInstanceDiff {
/// Also match each edge in binary 2 to the corresponding ones in binary 1.
void matchBasicBlocks() {
for (const auto &MapEntry : FuncMap) {
const auto &Func1 = MapEntry.second;
const auto &Func2 = MapEntry.first;
const BinaryFunction *const &Func1 = MapEntry.second;
const BinaryFunction *const &Func2 = MapEntry.first;
auto Iter1 = Func1->layout_begin();
auto Iter2 = Func2->layout_begin();
@ -386,8 +386,8 @@ class RewriteInstanceDiff {
Match = false;
break;
}
const auto ScoreEdge1 = getNormalizedScore(BIIter1, RI1);
const auto ScoreEdge2 = getNormalizedScore(BIIter2, RI2);
const double ScoreEdge1 = getNormalizedScore(BIIter1, RI1);
const double ScoreEdge2 = getNormalizedScore(BIIter2, RI2);
EMap.insert(std::make_pair<>(
std::abs(ScoreEdge2 - ScoreEdge1),
std::make_pair<>(
@ -422,8 +422,8 @@ class RewriteInstanceDiff {
void reportHottestBBDiffs() {
std::map<double, const BinaryBasicBlock *> LargestDiffs;
for (const auto &MapEntry : BBMap) {
const auto *BB2 = MapEntry.first;
const auto *BB1 = MapEntry.second;
const BinaryBasicBlock *BB2 = MapEntry.first;
const BinaryBasicBlock *BB1 = MapEntry.second;
LargestDiffs.insert(
std::make_pair<>(std::abs(getNormalizedScore(*BB2, RI2) -
getNormalizedScore(*BB1, RI1)),
@ -439,9 +439,9 @@ class RewriteInstanceDiff {
setRegularColor();
outs() << " * Functions with different contents do not appear here\n\n";
for (auto I = LargestDiffs.rbegin(), E = LargestDiffs.rend(); I != E; ++I) {
const auto *BB2 = I->second;
const auto Score2 = getNormalizedScore(*BB2, RI2);
const auto Score1 = getNormalizedScore(*BBMap[BB2], RI1);
const BinaryBasicBlock *BB2 = I->second;
const double Score2 = getNormalizedScore(*BB2, RI2);
const double Score1 = getNormalizedScore(*BBMap[BB2], RI1);
outs() << "BB " << BB2->getName() << " from "
<< BBToFuncMap[BB2]->getDemangledName()
<< "\n\tScore bin1 = " << format("%.4f", Score1 * 100.0)
@ -470,10 +470,12 @@ class RewriteInstanceDiff {
setRegularColor();
outs() << " * Functions with different contents do not appear here\n";
for (auto I = EdgeMap.rbegin(), E = EdgeMap.rend(); I != E; ++I) {
auto &Edge2 = I->second.first;
auto &Edge1 = I->second.second;
const auto Score2 = std::get<2>(Edge2);
const auto Score1 = std::get<2>(Edge1);
std::tuple<const BinaryBasicBlock *, const BinaryBasicBlock *, double>
&Edge2 = I->second.first;
std::tuple<const BinaryBasicBlock *, const BinaryBasicBlock *, double>
&Edge1 = I->second.second;
const double Score2 = std::get<2>(Edge2);
const double Score1 = std::get<2>(Edge1);
outs() << "Edge (" << std::get<0>(Edge2)->getName() << " -> "
<< std::get<1>(Edge2)->getName() << ") in "
<< BBToFuncMap[std::get<0>(Edge2)]->getDemangledName()
@ -500,7 +502,7 @@ class RewriteInstanceDiff {
/// LTO variant 1 to variant 2, even though they represent the same function.
void computeAggregatedLTOScore() {
for (const auto &BFI : RI1.BC->getBinaryFunctions()) {
const auto &Function = BFI.second;
const BinaryFunction &Function = BFI.second;
double Score = getNormalizedScore(Function, RI1);
auto Iter = LTOMap1.find(&Function);
if (Iter == LTOMap1.end())
@ -510,7 +512,7 @@ class RewriteInstanceDiff {
double UnmappedScore{0};
for (const auto &BFI : RI2.BC->getBinaryFunctions()) {
const auto &Function = BFI.second;
const BinaryFunction &Function = BFI.second;
bool Matched = FuncMap.find(&Function) != FuncMap.end();
double Score = getNormalizedScore(Function, RI2);
auto Iter = LTOMap2.find(&Function);
@ -539,8 +541,8 @@ class RewriteInstanceDiff {
void reportHottestFuncDiffs() {
std::multimap<double, decltype(FuncMap)::value_type> LargestDiffs;
for (const auto &MapEntry : FuncMap) {
const auto &Func1 = MapEntry.second;
const auto &Func2 = MapEntry.first;
const BinaryFunction *const &Func1 = MapEntry.second;
const BinaryFunction *const &Func2 = MapEntry.first;
double Score1 = getNormalizedScore(*Func1, RI1);
auto Iter1 = LTOMap1.find(Func1);
if (Iter1 != LTOMap1.end()) {
@ -565,12 +567,13 @@ class RewriteInstanceDiff {
outs() << "=========================================================\n";
setRegularColor();
for (auto I = LargestDiffs.rbegin(), E = LargestDiffs.rend(); I != E; ++I) {
const auto &MapEntry = I->second;
const std::pair<const BinaryFunction *const, const BinaryFunction *>
&MapEntry = I->second;
if (opts::IgnoreUnchanged &&
MapEntry.second->computeHash(/*UseDFS=*/true) ==
MapEntry.first->computeHash(/*UseDFS=*/true))
continue;
const auto &Scores = ScoreMap[MapEntry.first];
const std::pair<double, double> &Scores = ScoreMap[MapEntry.first];
outs() << "Function " << MapEntry.first->getDemangledName();
if (MapEntry.first->getDemangledName() !=
MapEntry.second->getDemangledName())
@ -609,7 +612,7 @@ class RewriteInstanceDiff {
outs() << "=====================================\n";
setRegularColor();
for (auto I = LargestBin2.rbegin(), E = LargestBin2.rend(); I != E; ++I) {
const auto &MapEntry = *I;
const std::pair<const double, const BinaryFunction *> &MapEntry = *I;
outs() << "Function " << MapEntry.second->getDemangledName() << "\n";
auto Iter = ScoreMap.find(MapEntry.second);
if (Iter != ScoreMap.end()) {
@ -629,7 +632,7 @@ class RewriteInstanceDiff {
outs() << "=====================================\n";
setRegularColor();
for (auto I = LargestBin1.rbegin(), E = LargestBin1.rend(); I != E; ++I) {
const auto &MapEntry = *I;
const std::pair<const double, const BinaryFunction *> &MapEntry = *I;
outs() << "Function " << MapEntry.second->getDemangledName()
<< "\n\tScore bin1 = " << format("%.2f", MapEntry.first * 100.0)
<< "%\n";
@ -646,7 +649,7 @@ class RewriteInstanceDiff {
outs() << "List of functions from binary 2 that were not matched with any "
<< "function in binary 1:\n";
for (const auto &BFI2 : RI2.BC->getBinaryFunctions()) {
const auto &Function2 = BFI2.second;
const BinaryFunction &Function2 = BFI2.second;
if (Bin2MappedFuncs.count(&Function2))
continue;
outs() << Function2.getPrintName() << "\n";

59
bolt/src/CacheMetrics.cpp
View File

@ -37,9 +37,9 @@ void extractBasicBlockInfo(
std::unordered_map<BinaryBasicBlock *, uint64_t> &BBAddr,
std::unordered_map<BinaryBasicBlock *, uint64_t> &BBSize) {
for (auto BF : BinaryFunctions) {
const auto &BC = BF->getBinaryContext();
for (auto BB : BF->layout()) {
for (BinaryFunction *BF : BinaryFunctions) {
const BinaryContext &BC = BF->getBinaryContext();
for (BinaryBasicBlock *BB : BF->layout()) {
if (BF->isSimple() || BC.HasRelocations) {
// Use addresses/sizes as in the output binary
BBAddr[BB] = BB->getOutputAddressRange().first;
@ -61,12 +61,12 @@ double calcTSPScore(
const std::unordered_map<BinaryBasicBlock *, uint64_t> &BBSize) {
double Score = 0;
for (auto BF : BinaryFunctions) {
for (BinaryFunction *BF : BinaryFunctions) {
if (!BF->hasProfile())
continue;
for (auto SrcBB : BF->layout()) {
for (BinaryBasicBlock *SrcBB : BF->layout()) {
auto BI = SrcBB->branch_info_begin();
for (auto DstBB : SrcBB->successors()) {
for (BinaryBasicBlock *DstBB : SrcBB->successors()) {
if (SrcBB != DstBB && BI->Count != BinaryBasicBlock::COUNT_NO_PROFILE &&
BBAddr.at(SrcBB) + BBSize.at(SrcBB) == BBAddr.at(DstBB))
Score += BI->Count;
@ -85,12 +85,12 @@ double calcExtTSPScore(
const std::unordered_map<BinaryBasicBlock *, uint64_t> &BBSize) {
double Score = 0.0;
for (auto BF : BinaryFunctions) {
for (BinaryFunction *BF : BinaryFunctions) {
if (!BF->hasProfile())
continue;
for (auto SrcBB : BF->layout()) {
for (BinaryBasicBlock *SrcBB : BF->layout()) {
auto BI = SrcBB->branch_info_begin();
for (auto DstBB : SrcBB->successors()) {
for (BinaryBasicBlock *DstBB : SrcBB->successors()) {
if (DstBB != SrcBB) {
Score += CacheMetrics::extTSPScore(BBAddr.at(SrcBB),
BBSize.at(SrcBB),
@ -112,22 +112,22 @@ std::unordered_map<const BinaryFunction *, Predecessors>
extractFunctionCalls(const std::vector<BinaryFunction *> &BinaryFunctions) {
std::unordered_map<const BinaryFunction *, Predecessors> Calls;
for (auto SrcFunction : BinaryFunctions) {
const auto &BC = SrcFunction->getBinaryContext();
for (auto BB : SrcFunction->layout()) {
for (BinaryFunction *SrcFunction : BinaryFunctions) {
const BinaryContext &BC = SrcFunction->getBinaryContext();
for (BinaryBasicBlock *BB : SrcFunction->layout()) {
// Find call instructions and extract target symbols from each one
for (auto &Inst : *BB) {
for (MCInst &Inst : *BB) {
if (!BC.MIB->isCall(Inst))
continue;
// Call info
const MCSymbol* DstSym = BC.MIB->getTargetSymbol(Inst);
auto Count = BB->getKnownExecutionCount();
uint64_t Count = BB->getKnownExecutionCount();
// Ignore calls w/o information
if (DstSym == nullptr || Count == 0)
continue;
auto DstFunction = BC.getFunctionForSymbol(DstSym);
const BinaryFunction *DstFunction = BC.getFunctionForSymbol(DstSym);
// Ignore recursive calls
if (DstFunction == nullptr ||
DstFunction->layout_empty() ||
@ -161,13 +161,14 @@ double expectedCacheHitRatio(
const double PageSize = opts::ITLBPageSize;
const uint64_t CacheEntries = opts::ITLBEntries;
auto Calls = extractFunctionCalls(BinaryFunctions);
std::unordered_map<const BinaryFunction *, Predecessors> Calls =
extractFunctionCalls(BinaryFunctions);
// Compute 'hotness' of the functions
double TotalSamples = 0;
std::unordered_map<BinaryFunction *, double> FunctionSamples;
for (auto BF : BinaryFunctions) {
for (BinaryFunction *BF : BinaryFunctions) {
double Samples = 0;
for (auto Pair : Calls[BF]) {
for (std::pair<BinaryFunction *, uint64_t> Pair : Calls[BF]) {
Samples += Pair.second;
}
Samples = std::max(Samples, (double)BF->getKnownExecutionCount());
@ -177,28 +178,28 @@ double expectedCacheHitRatio(
// Compute 'hotness' of the pages
std::unordered_map<uint64_t, double> PageSamples;
for (auto BF : BinaryFunctions) {
for (BinaryFunction *BF : BinaryFunctions) {
if (BF->layout_empty())
continue;
auto Page = BBAddr.at(BF->layout_front()) / PageSize;
double Page = BBAddr.at(BF->layout_front()) / PageSize;
PageSamples[Page] += FunctionSamples.at(BF);
}
// Computing the expected number of misses for every function
double Misses = 0;
for (auto BF : BinaryFunctions) {
for (BinaryFunction *BF : BinaryFunctions) {
// Skip the function if it has no samples
if (BF->layout_empty() || FunctionSamples.at(BF) == 0.0)
continue;
double Samples = FunctionSamples.at(BF);
auto Page = BBAddr.at(BF->layout_front()) / PageSize;
double Page = BBAddr.at(BF->layout_front()) / PageSize;
// The probability that the page is not present in the cache
double MissProb = pow(1.0 - PageSamples[Page] / TotalSamples, CacheEntries);
// Processing all callers of the function
for (auto Pair : Calls[BF]) {
auto SrcFunction = Pair.first;
auto SrcPage = BBAddr.at(SrcFunction->layout_front()) / PageSize;
for (std::pair<BinaryFunction *, uint64_t> Pair : Calls[BF]) {
BinaryFunction *SrcFunction = Pair.first;
double SrcPage = BBAddr.at(SrcFunction->layout_front()) / PageSize;
// Is this a 'long' or a 'short' call?
if (Page != SrcPage) {
// This is a miss
@ -229,7 +230,7 @@ double CacheMetrics::extTSPScore(uint64_t SrcAddr,
}
// Forward
if (SrcAddr + SrcSize < DstAddr) {
const auto Dist = DstAddr - (SrcAddr + SrcSize);
const uint64_t Dist = DstAddr - (SrcAddr + SrcSize);
if (Dist <= opts::ForwardDistance) {
double Prob = 1.0 - static_cast<double>(Dist) / opts::ForwardDistance;
return opts::ForwardWeight * Prob * Count;
@ -237,7 +238,7 @@ double CacheMetrics::extTSPScore(uint64_t SrcAddr,
return 0;
}
// Backward
const auto Dist = SrcAddr + SrcSize - DstAddr;
const uint64_t Dist = SrcAddr + SrcSize - DstAddr;
if (Dist <= opts::BackwardDistance) {
double Prob = 1.0 - static_cast<double>(Dist) / opts::BackwardDistance;
return opts::BackwardWeight * Prob * Count;
@ -258,13 +259,13 @@ void CacheMetrics::printAll(const std::vector<BinaryFunction *> &BFs) {
size_t HotCodeMinAddr = std::numeric_limits<size_t>::max();
size_t HotCodeMaxAddr = 0;
for (auto BF : BFs) {
for (BinaryFunction *BF : BFs) {
NumFunctions++;
if (BF->hasProfile())
NumProfiledFunctions++;
if (BF->hasValidIndex())
NumHotFunctions++;
for (auto BB : BF->layout()) {
for (BinaryBasicBlock *BB : BF->layout()) {
NumBlocks++;
size_t BBAddrMin = BB->getOutputAddressRange().first;
size_t BBAddrMax = BB->getOutputAddressRange().second;

113
bolt/src/DWARFRewriter.cpp
View File

@ -68,8 +68,9 @@ DeterministicDebugInfo("deterministic-debuginfo",
} // namespace opts
void DWARFRewriter::updateDebugInfo() {
auto DebugAbbrev = BC.getUniqueSectionByName(".debug_abbrev");
auto DebugInfo = BC.getUniqueSectionByName(".debug_info");
ErrorOr<BinarySection &> DebugAbbrev =
BC.getUniqueSectionByName(".debug_abbrev");
ErrorOr<BinarySection &> DebugInfo = BC.getUniqueSectionByName(".debug_info");
if (DebugAbbrev) {
DebugAbbrev->registerPatcher(std::make_unique<DebugAbbrevPatcher>());
AbbrevPatcher =
@ -101,14 +102,14 @@ void DWARFRewriter::updateDebugInfo() {
};
if (opts::NoThreads || opts::DeterministicDebugInfo) {
for (auto &CU : BC.DwCtx->compile_units()) {
for (std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {
processUnitDIE(0, CU.get());
}
} else {
// Update unit debug info in parallel
auto &ThreadPool = ParallelUtilities::getThreadPool();
ThreadPool &ThreadPool = ParallelUtilities::getThreadPool();
size_t CUIndex = 0;
for (auto &CU : BC.DwCtx->compile_units()) {
for (std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {
ThreadPool.async(processUnitDIE, CUIndex, CU.get());
CUIndex++;
}
@ -170,7 +171,8 @@ void DWARFRewriter::updateUnitDebugInfo(size_t CUIndex, DWARFUnit *Unit) {
uint64_t Address;
uint64_t SectionIndex, HighPC;
if (!DIE.getLowAndHighPC(Address, HighPC, SectionIndex)) {
auto RangesOrError = DIE.getAddressRanges();
Expected<DWARFAddressRangesVector> RangesOrError =
DIE.getAddressRanges();
if (!RangesOrError) {
consumeError(RangesOrError.takeError());
break;
@ -198,7 +200,8 @@ void DWARFRewriter::updateUnitDebugInfo(size_t CUIndex, DWARFUnit *Unit) {
RangesSectionWriter->addRanges(FunctionRanges));
} else {
// Delay conversion of [LowPC, HighPC) into DW_AT_ranges if possible.
const auto *Abbrev = DIE.getAbbreviationDeclarationPtr();
const DWARFAbbreviationDeclaration *Abbrev =
DIE.getAbbreviationDeclarationPtr();
assert(Abbrev && "abbrev expected");
// Create a critical section.
@ -257,7 +260,8 @@ void DWARFRewriter::updateUnitDebugInfo(size_t CUIndex, DWARFUnit *Unit) {
// Handle any tag that can have DW_AT_location attribute.
DWARFFormValue Value;
uint64_t AttrOffset;
if (auto V = DIE.find(dwarf::DW_AT_location, &AttrOffset)) {
if (Optional<DWARFFormValue> V =
DIE.find(dwarf::DW_AT_location, &AttrOffset)) {
Value = *V;
if (Value.isFormClass(DWARFFormValue::FC_Constant) ||
Value.isFormClass(DWARFFormValue::FC_SectionOffset)) {
@ -327,9 +331,10 @@ void DWARFRewriter::updateUnitDebugInfo(size_t CUIndex, DWARFUnit *Unit) {
Value.isFormClass(DWARFFormValue::FC_Block)) &&
"unexpected DW_AT_location form");
}
} else if (auto V = DIE.find(dwarf::DW_AT_low_pc, &AttrOffset)) {
} else if (Optional<DWARFFormValue> V =
DIE.find(dwarf::DW_AT_low_pc, &AttrOffset)) {
Value = *V;
const auto Result = Value.getAsAddress();
const Optional<uint64_t> Result = Value.getAsAddress();
if (Result.hasValue()) {
const uint64_t Address = Result.getValue();
uint64_t NewAddress = 0;
@ -369,7 +374,8 @@ void DWARFRewriter::updateDWARFObjectAddressRanges(
return;
}
const auto *AbbreviationDecl = DIE.getAbbreviationDeclarationPtr();
const DWARFAbbreviationDeclaration *AbbreviationDecl =
DIE.getAbbreviationDeclarationPtr();
if (!AbbreviationDecl) {
if (opts::Verbosity >= 1) {
errs() << "BOLT-WARNING: object's DIE doesn't have an abbreviation: "
@ -421,8 +427,10 @@ void DWARFRewriter::updateLineTableOffsets() {
uint64_t CurrentOffset = 0;
uint64_t Offset = 0;
auto DbgInfoSection = BC.getUniqueSectionByName(".debug_info");
auto TypeInfoSection = BC.getUniqueSectionByName(".debug_types");
ErrorOr<BinarySection &> DbgInfoSection =
BC.getUniqueSectionByName(".debug_info");
ErrorOr<BinarySection &> TypeInfoSection =
BC.getUniqueSectionByName(".debug_types");
assert(((BC.DwCtx->getNumTypeUnits() > 0 && TypeInfoSection) ||
BC.DwCtx->getNumTypeUnits() == 0) &&
"Was not able to retrieve Debug Types section.");
@ -433,14 +441,15 @@ void DWARFRewriter::updateLineTableOffsets() {
// ones.
std::unordered_map<uint64_t, uint64_t> DebugLineOffsetMap;
auto getStatementListValue = [](DWARFUnit *Unit) {
auto StmtList = Unit->getUnitDIE().find(dwarf::DW_AT_stmt_list);
auto Offset = dwarf::toSectionOffset(StmtList);
auto GetStatementListValue = [](DWARFUnit *Unit) {
Optional<DWARFFormValue> StmtList =
Unit->getUnitDIE().find(dwarf::DW_AT_stmt_list);
Optional<uint64_t> Offset = dwarf::toSectionOffset(StmtList);
assert(Offset && "Was not able to retreive value of DW_AT_stmt_list.");
return *Offset;
};
for (const auto &CU : BC.DwCtx->compile_units()) {
for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {
const unsigned CUID = CU->getOffset();
MCSymbol *Label = BC.Ctx->getMCDwarfLineTable(CUID).getLabel();
if (!Label)
@ -477,7 +486,7 @@ void DWARFRewriter::updateLineTableOffsets() {
Offset += Label->getOffset() - CurrentOffset;
CurrentOffset = Label->getOffset();
DebugLineOffsetMap[getStatementListValue(CU.get())] = Offset;
DebugLineOffsetMap[GetStatementListValue(CU.get())] = Offset;
assert(DbgInfoSection && ".debug_info section must exist");
DbgInfoSection->addRelocation(LTOffset,
nullptr,
@ -490,13 +499,13 @@ void DWARFRewriter::updateLineTableOffsets() {
<< " has line table at " << Offset << "\n");
}
for (const auto &TU : BC.DwCtx->types_section_units()) {
auto *Unit = TU.get();
for (const std::unique_ptr<DWARFUnit> &TU : BC.DwCtx->types_section_units()) {
DWARFUnit *Unit = TU.get();
const uint64_t LTOffset =
BC.DwCtx->getAttrFieldOffsetForUnit(Unit, dwarf::DW_AT_stmt_list);
if (!LTOffset)
continue;
auto Iter = DebugLineOffsetMap.find(getStatementListValue(Unit));
auto Iter = DebugLineOffsetMap.find(GetStatementListValue(Unit));
assert(Iter != DebugLineOffsetMap.end() &&
"Type Unit Updated Line Number Entry does not exist.");
TypeInfoSection->addRelocation(LTOffset, nullptr, ELF::R_X86_64_32,
@ -523,19 +532,21 @@ void DWARFRewriter::finalizeDebugSections() {
*BC.STI, *BC.MRI, MCTargetOptions()));
ARangesSectionWriter->writeARangesSection(OS);
const auto &ARangesContents = OS.str();
const StringRef &ARangesContents = OS.str();
BC.registerOrUpdateNoteSection(".debug_aranges",
copyByteArray(ARangesContents),
ARangesContents.size());
}
auto RangesSectionContents = RangesSectionWriter->finalize();
std::unique_ptr<RangesBufferVector> RangesSectionContents =
RangesSectionWriter->finalize();
BC.registerOrUpdateNoteSection(".debug_ranges",
copyByteArray(*RangesSectionContents),
RangesSectionContents->size());
auto LocationListSectionContents = makeFinalLocListsSection();
std::unique_ptr<LocBufferVector> LocationListSectionContents =
makeFinalLocListsSection();
BC.registerOrUpdateNoteSection(".debug_loc",
copyByteArray(*LocationListSectionContents),
LocationListSectionContents->size());
@ -550,10 +561,10 @@ void DWARFRewriter::updateGdbIndexSection() {
StringRef GdbIndexContents = BC.getGdbIndexSection()->getContents();
const auto *Data = GdbIndexContents.data();
const char *Data = GdbIndexContents.data();
// Parse the header.
const auto Version = read32le(Data);
const uint32_t Version = read32le(Data);
if (Version != 7 && Version != 8) {
errs() << "BOLT-ERROR: can only process .gdb_index versions 7 and 8\n";
exit(1);
@ -562,24 +573,24 @@ void DWARFRewriter::updateGdbIndexSection() {
// Some .gdb_index generators use file offsets while others use section
// offsets. Hence we can only rely on offsets relative to each other,
// and ignore their absolute values.
const auto CUListOffset = read32le(Data + 4);
const auto CUTypesOffset = read32le(Data + 8);
const auto AddressTableOffset = read32le(Data + 12);
const auto SymbolTableOffset = read32le(Data + 16);
const auto ConstantPoolOffset = read32le(Data + 20);
const uint32_t CUListOffset = read32le(Data + 4);
const uint32_t CUTypesOffset = read32le(Data + 8);
const uint32_t AddressTableOffset = read32le(Data + 12);
const uint32_t SymbolTableOffset = read32le(Data + 16);
const uint32_t ConstantPoolOffset = read32le(Data + 20);
Data += 24;
// Map CUs offsets to indices and verify existing index table.
std::map<uint32_t, uint32_t> OffsetToIndexMap;
const auto CUListSize = CUTypesOffset - CUListOffset;
const auto NumCUs = BC.DwCtx->getNumCompileUnits();
const uint32_t CUListSize = CUTypesOffset - CUListOffset;
const unsigned NumCUs = BC.DwCtx->getNumCompileUnits();
if (CUListSize != NumCUs * 16) {
errs() << "BOLT-ERROR: .gdb_index: CU count mismatch\n";
exit(1);
}
for (unsigned Index = 0; Index < NumCUs; ++Index, Data += 16) {
const auto *CU = BC.DwCtx->getUnitAtIndex(Index);
const auto Offset = read64le(Data);
const DWARFUnit *CU = BC.DwCtx->getUnitAtIndex(Index);
const uint64_t Offset = read64le(Data);
if (CU->getOffset() != Offset) {
errs() << "BOLT-ERROR: .gdb_index CU offset mismatch\n";
exit(1);
@ -589,14 +600,14 @@ void DWARFRewriter::updateGdbIndexSection() {
}
// Ignore old address table.
const auto OldAddressTableSize = SymbolTableOffset - AddressTableOffset;
const uint32_t OldAddressTableSize = SymbolTableOffset - AddressTableOffset;
// Move Data to the beginning of symbol table.
Data += SymbolTableOffset - CUTypesOffset;
// Calculate the size of the new address table.
uint32_t NewAddressTableSize = 0;
for (const auto &CURangesPair : ARangesSectionWriter->getCUAddressRanges()) {
const auto &Ranges = CURangesPair.second;
const SmallVector<DebugAddressRange, 2> &Ranges = CURangesPair.second;
NewAddressTableSize += Ranges.size() * 20;
}
@ -608,7 +619,7 @@ void DWARFRewriter::updateGdbIndexSection() {
// Free'd by ExecutableFileMemoryManager.
auto *NewGdbIndexContents = new uint8_t[NewGdbIndexSize];
auto *Buffer = NewGdbIndexContents;
uint8_t *Buffer = NewGdbIndexContents;
write32le(Buffer, Version);
write32le(Buffer + 4, CUListOffset);
@ -624,10 +635,11 @@ void DWARFRewriter::updateGdbIndexSection() {
Buffer += AddressTableOffset - CUListOffset;
// Generate new address table.
for (const auto &CURangesPair : ARangesSectionWriter->getCUAddressRanges()) {
const auto CUIndex = OffsetToIndexMap[CURangesPair.first];
const auto &Ranges = CURangesPair.second;
for (const auto &Range : Ranges) {
for (const std::pair<const uint64_t, DebugAddressRangesVector> &CURangesPair :
ARangesSectionWriter->getCUAddressRanges()) {
const uint32_t CUIndex = OffsetToIndexMap[CURangesPair.first];
const DebugAddressRangesVector &Ranges = CURangesPair.second;
for (const DebugAddressRange &Range : Ranges) {
write64le(Buffer, Range.LowPC);
write64le(Buffer + 8, Range.HighPC);
write32le(Buffer + 16, CUIndex);
@ -635,8 +647,8 @@ void DWARFRewriter::updateGdbIndexSection() {
}
}
const auto TrailingSize =
GdbIndexContents.data() + GdbIndexContents.size() - Data;
const size_t TrailingSize =
GdbIndexContents.data() + GdbIndexContents.size() - Data;
assert(Buffer + TrailingSize == NewGdbIndexContents + NewGdbIndexSize &&
"size calculation error");
@ -697,7 +709,7 @@ void DWARFRewriter::convertPending(const DWARFAbbreviationDeclaration *Abbrev) {
auto I = PendingRanges.find(Abbrev);
if (I != PendingRanges.end()) {
for (auto &Pair : I->second) {
for (std::pair<DWARFDieWrapper, DebugAddressRange> &Pair : I->second) {
convertToRanges(Pair.first, {Pair.second});
}
PendingRanges.erase(I);
@ -723,12 +735,13 @@ std::unique_ptr<LocBufferVector> DWARFRewriter::makeFinalLocListsSection() {
for (size_t CUIndex = 0; CUIndex < LocListWritersByCU.size(); ++CUIndex) {
SectionOffsetByCU[CUIndex] = SectionOffset;
auto CurrCULocationLists = LocListWritersByCU[CUIndex]->finalize();
std::unique_ptr<LocBufferVector> CurrCULocationLists =
LocListWritersByCU[CUIndex]->finalize();
*LocStream << *CurrCULocationLists;
SectionOffset += CurrCULocationLists->size();
}
for (auto &Patch : LocListDebugInfoPatches) {
for (LocListDebugInfoPatchType &Patch : LocListDebugInfoPatches) {
DebugInfoPatcher
->addLE32Patch(Patch.DebugInfoOffset,
SectionOffsetByCU[Patch.CUIndex] + Patch.CUWriterOffset);
@ -738,8 +751,10 @@ std::unique_ptr<LocBufferVector> DWARFRewriter::makeFinalLocListsSection() {
}
void DWARFRewriter::flushPendingRanges() {
for (auto &I : PendingRanges) {
for (auto &RangePair : I.second) {
for (std::pair<const DWARFAbbreviationDeclaration *const,
std::vector<std::pair<DWARFDieWrapper, DebugAddressRange>>>
&I : PendingRanges) {
for (std::pair<DWARFDieWrapper, DebugAddressRange> &RangePair : I.second) {
patchLowHigh(RangePair.first, RangePair.second);
}
}

365
bolt/src/DataAggregator.cpp
View File

File diff suppressed because it is too large Load Diff

229
bolt/src/DataReader.cpp
View File

@ -41,7 +41,7 @@ namespace llvm {
namespace bolt {
Optional<StringRef> getLTOCommonName(const StringRef Name) {
auto LTOSuffixPos = Name.find(".lto_priv.");
size_t LTOSuffixPos = Name.find(".lto_priv.");
if (LTOSuffixPos != StringRef::npos) {
return Name.substr(0, LTOSuffixPos + 10);
} else if ((LTOSuffixPos = Name.find(".constprop.")) != StringRef::npos) {
@ -55,7 +55,7 @@ namespace {
/// Return true if the function name can change across compilations.
bool hasVolatileName(const BinaryFunction &BF) {
for (const auto Name : BF.getNames()) {
for (const StringRef Name : BF.getNames()) {
if (getLTOCommonName(Name))
return true;
}
@ -120,8 +120,8 @@ void FuncBranchData::appendFrom(const FuncBranchData &FBD, uint64_t Offset) {
uint64_t FuncBranchData::getNumExecutedBranches() const {
uint64_t ExecutedBranches{0};
for (const auto &BI : Data) {
auto BranchCount = BI.Branches;
for (const BranchInfo &BI : Data) {
int64_t BranchCount = BI.Branches;
assert(BranchCount >= 0 && "branch execution count should not be negative");
ExecutedBranches += BranchCount;
}
@ -164,7 +164,7 @@ void FuncSampleData::bumpCount(uint64_t Offset, uint64_t Count) {
Index[Offset] = Data.size() - 1;
return;
}
auto &SI = Data[Iter->second];
SampleInfo &SI = Data[Iter->second];
SI.Hits += Count;
}
@ -177,7 +177,7 @@ void FuncBranchData::bumpBranchCount(uint64_t OffsetFrom, uint64_t OffsetTo,
IntraIndex[OffsetFrom][OffsetTo] = Data.size() - 1;
return;
}
auto &BI = Data[Iter->second];
BranchInfo &BI = Data[Iter->second];
BI.Branches += Count;
BI.Mispreds += Mispreds;
}
@ -190,7 +190,7 @@ void FuncBranchData::bumpCallCount(uint64_t OffsetFrom, const Location &To,
InterIndex[OffsetFrom][To] = Data.size() - 1;
return;
}
auto &BI = Data[Iter->second];
BranchInfo &BI = Data[Iter->second];
BI.Branches += Count;
BI.Mispreds += Mispreds;
}
@ -204,7 +204,7 @@ void FuncBranchData::bumpEntryCount(const Location &From, uint64_t OffsetTo,
EntryIndex[OffsetTo][From] = EntryData.size() - 1;
return;
}
auto &BI = EntryData[Iter->second];
BranchInfo &BI = EntryData[Iter->second];
BI.Branches += Count;
BI.Mispreds += Mispreds;
}
@ -224,7 +224,7 @@ void BranchInfo::print(raw_ostream &OS) const {
ErrorOr<const BranchInfo &> FuncBranchData::getBranch(uint64_t From,
uint64_t To) const {
for (const auto &I : Data) {
for (const BranchInfo &I : Data) {
if (I.From.Offset == From && I.To.Offset == To &&
I.From.Name == I.To.Name)
return I;
@ -275,7 +275,7 @@ void FuncMemData::update(const Location &Offset, const Location &Addr) {
}
Error DataReader::preprocessProfile(BinaryContext &BC) {
if (auto EC = parseInput()) {
if (std::error_code EC = parseInput()) {
return errorCodeToError(EC);
}
@ -289,12 +289,12 @@ Error DataReader::preprocessProfile(BinaryContext &BC) {
}
for (auto &BFI : BC.getBinaryFunctions()) {
auto &Function = BFI.second;
if (auto *MemData = getMemDataForNames(Function.getNames())) {
BinaryFunction &Function = BFI.second;
if (FuncMemData *MemData = getMemDataForNames(Function.getNames())) {
setMemData(Function, MemData);
MemData->Used = true;
}
if (auto *FuncData = getBranchDataForNames(Function.getNames())) {
if (FuncBranchData *FuncData = getBranchDataForNames(Function.getNames())) {
setBranchData(Function, FuncData);
Function.ExecutionCount = FuncData->ExecutionCount;
FuncData->Used = true;
@ -302,7 +302,7 @@ Error DataReader::preprocessProfile(BinaryContext &BC) {
}
for (auto &BFI : BC.getBinaryFunctions()) {
auto &Function = BFI.second;
BinaryFunction &Function = BFI.second;
matchProfileMemData(Function);
}
@ -311,12 +311,12 @@ Error DataReader::preprocessProfile(BinaryContext &BC) {
Error DataReader::readProfilePreCFG(BinaryContext &BC) {
for (auto &BFI : BC.getBinaryFunctions()) {
auto &Function = BFI.second;
BinaryFunction &Function = BFI.second;
FuncMemData *MemoryData = getMemData(Function);
if (!MemoryData)
continue;
for (auto &MI : MemoryData->Data) {
for (MemInfo &MI : MemoryData->Data) {
const uint64_t Offset = MI.Offset.Offset;
auto II = Function.Instructions.find(Offset);
if (II == Function.Instructions.end()) {
@ -348,12 +348,12 @@ Error DataReader::readProfilePreCFG(BinaryContext &BC) {
Error DataReader::readProfile(BinaryContext &BC) {
for (auto &BFI : BC.getBinaryFunctions()) {
auto &Function = BFI.second;
BinaryFunction &Function = BFI.second;
readProfile(Function);
}
uint64_t NumUnused{0};
for (const auto &FuncData : NamesToBranches)
for (const StringMapEntry<FuncBranchData> &FuncData : NamesToBranches)
if (!FuncData.getValue().Used)
++NumUnused;
BC.setNumUnusedProfiledObjects(NumUnused);
@ -362,14 +362,15 @@ Error DataReader::readProfile(BinaryContext &BC) {
}
std::error_code DataReader::parseInput() {
auto MB = MemoryBuffer::getFileOrSTDIN(Filename);
if (auto EC = MB.getError()) {
ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
MemoryBuffer::getFileOrSTDIN(Filename);
if (std::error_code EC = MB.getError()) {
Diag << "cannot open " << Filename << ": " << EC.message() << "\n";
return EC;
}
FileBuf = std::move(MB.get());
ParsingBuf = FileBuf->getBuffer();
if (auto EC = parse()) {
if (std::error_code EC = parse()) {
return EC;
}
if (!ParsingBuf.empty()) {
@ -408,10 +409,10 @@ void DataReader::readProfile(BinaryFunction &BF) {
// may be accounted for in the execution count of an entry block if the last
// instruction in a predecessor fall-through block is a call. This situation
// should rarely happen because there are few multiple-entry functions.
for (const auto &BI : FBD->EntryData) {
for (const BranchInfo &BI : FBD->EntryData) {
BinaryBasicBlock *BB = BF.getBasicBlockAtOffset(BI.To.Offset);
if (BB && (BB->isEntryPoint() || BB->isLandingPad())) {
auto Count = BB->getExecutionCount();
uint64_t Count = BB->getExecutionCount();
if (Count == BinaryBasicBlock::COUNT_NO_PROFILE)
Count = 0;
BB->setExecutionCount(Count + BI.Branches);
@ -419,7 +420,7 @@ void DataReader::readProfile(BinaryFunction &BF) {
}
uint64_t MismatchedBranches = 0;
for (const auto &BI : FBD->Data) {
for (const BranchInfo &BI : FBD->Data) {
if (BI.From.Name != BI.To.Name) {
continue;
}
@ -464,8 +465,9 @@ void DataReader::matchProfileData(BinaryFunction &BF) {
return;
// Check for a profile that matches with 100% confidence.
const auto AllBranchData = getBranchDataForNamesRegex(BF.getNames());
for (auto *NewBranchData : AllBranchData) {
const std::vector<FuncBranchData *> AllBranchData =
getBranchDataForNamesRegex(BF.getNames());
for (FuncBranchData *NewBranchData : AllBranchData) {
// Prevent functions from sharing the same profile.
if (NewBranchData->Used)
continue;
@ -486,13 +488,14 @@ void DataReader::matchProfileData(BinaryFunction &BF) {
}
void DataReader::matchProfileMemData(BinaryFunction &BF) {
const auto AllMemData = getMemDataForNamesRegex(BF.getNames());
for (auto *NewMemData : AllMemData) {
const std::vector<FuncMemData *> AllMemData =
getMemDataForNamesRegex(BF.getNames());
for (FuncMemData *NewMemData : AllMemData) {
// Prevent functions from sharing the same profile.
if (NewMemData->Used)
continue;
if (auto *MD = getMemData(BF))
if (FuncMemData *MD = getMemData(BF))
MD->Used = false;
// Update function profile data with the new set.
@ -503,7 +506,7 @@ void DataReader::matchProfileMemData(BinaryFunction &BF) {
}
bool DataReader::fetchProfileForOtherEntryPoints(BinaryFunction &BF) {
auto &BC = BF.getBinaryContext();
BinaryContext &BC = BF.getBinaryContext();
FuncBranchData *FBD = getBranchData(BF);
if (!FBD)
@ -512,7 +515,7 @@ bool DataReader::fetchProfileForOtherEntryPoints(BinaryFunction &BF) {
// Check if we are missing profiling data for secondary entry points
bool First{true};
bool Updated{false};
for (auto BB : BF.BasicBlocks) {
for (BinaryBasicBlock *BB : BF.BasicBlocks) {
if (First) {
First = false;
continue;
@ -520,7 +523,7 @@ bool DataReader::fetchProfileForOtherEntryPoints(BinaryFunction &BF) {
if (BB->isEntryPoint()) {
uint64_t EntryAddress = BB->getOffset() + BF.getAddress();
// Look for branch data associated with this entry point
if (auto *BD = BC.getBinaryDataAtAddress(EntryAddress)) {
if (BinaryData *BD = BC.getBinaryDataAtAddress(EntryAddress)) {
if (FuncBranchData *Data = getBranchDataForSymbols(BD->getSymbols())) {
FBD->appendFrom(*Data, BB->getOffset());
Data->Used = true;
@ -535,7 +538,7 @@ bool DataReader::fetchProfileForOtherEntryPoints(BinaryFunction &BF) {
float DataReader::evaluateProfileData(BinaryFunction &BF,
const FuncBranchData &BranchData) const {
auto &BC = BF.getBinaryContext();
BinaryContext &BC = BF.getBinaryContext();
// Until we define a minimal profile, we consider an empty branch data to be
// a valid profile. It could happen to a function without branches when we
@ -545,7 +548,7 @@ float DataReader::evaluateProfileData(BinaryFunction &BF,
}
uint64_t NumMatchedBranches = 0;
for (const auto &BI : BranchData.Data) {
for (const BranchInfo &BI : BranchData.Data) {
bool IsValid = false;
if (BI.From.Name == BI.To.Name) {
// Try to record information with 0 count.
@ -561,7 +564,7 @@ float DataReader::evaluateProfileData(BinaryFunction &BF,
// When matching profiling info, we did not reach the stage
// when we identify tail calls, so they are still represented
// by regular branch instructions and we need isBranch() here.
auto *Instr = BF.getInstructionAtOffset(BI.From.Offset);
MCInst *Instr = BF.getInstructionAtOffset(BI.From.Offset);
// If it's a prefix - skip it.
if (Instr && BC.MIB->isPrefix(*Instr))
Instr = BF.getInstructionAtOffset(BI.From.Offset + 1);
@ -585,7 +588,7 @@ float DataReader::evaluateProfileData(BinaryFunction &BF,
else dbgs() << "<outbounds>\n";);
}
const auto MatchRatio = (float) NumMatchedBranches / BranchData.Data.size();
const float MatchRatio = (float)NumMatchedBranches / BranchData.Data.size();
if (opts::Verbosity >= 2 && NumMatchedBranches < BranchData.Data.size()) {
errs() << "BOLT-WARNING: profile branches match only "
<< format("%.1f%%", MatchRatio * 100.0f) << " ("
@ -597,7 +600,7 @@ float DataReader::evaluateProfileData(BinaryFunction &BF,
}
void DataReader::readSampleData(BinaryFunction &BF) {
auto SampleDataOrErr = getFuncSampleData(BF.getNames());
FuncSampleData *SampleDataOrErr = getFuncSampleData(BF.getNames());
if (!SampleDataOrErr)
return;
@ -645,7 +648,7 @@ void DataReader::readSampleData(BinaryFunction &BF) {
}
void DataReader::convertBranchData(BinaryFunction &BF) const {
auto &BC = BF.getBinaryContext();
BinaryContext &BC = BF.getBinaryContext();
if (BF.empty())
return;
@ -663,12 +666,12 @@ void DataReader::convertBranchData(BinaryFunction &BF) const {
// 3) Regular direct calls. The count could be different from containing
// basic block count. Keep this data in case we find it useful.
//
for (auto &BI : FBD->Data) {
for (BranchInfo &BI : FBD->Data) {
// Ignore internal branches.
if (BI.To.IsSymbol && BI.To.Name == BI.From.Name && BI.To.Offset != 0)
continue;
auto *Instr = BF.getInstructionAtOffset(BI.From.Offset);
MCInst *Instr = BF.getInstructionAtOffset(BI.From.Offset);
if (!Instr ||
(!BC.MIB->isCall(*Instr) && !BC.MIB->isIndirectBranch(*Instr)))
continue;
@ -689,7 +692,7 @@ void DataReader::convertBranchData(BinaryFunction &BF) const {
*Instr, "CallProfile");
MCSymbol *CalleeSymbol{nullptr};
if (BI.To.IsSymbol) {
if (auto *BD = BC.getBinaryDataByName(BI.To.Name)) {
if (BinaryData *BD = BC.getBinaryDataByName(BI.To.Name)) {
CalleeSymbol = BD->getSymbol();
}
}
@ -706,10 +709,10 @@ void DataReader::convertBranchData(BinaryFunction &BF) const {
bool DataReader::recordBranch(BinaryFunction &BF,
uint64_t From, uint64_t To,
uint64_t Count, uint64_t Mispreds) const {
auto &BC = BF.getBinaryContext();
BinaryContext &BC = BF.getBinaryContext();
auto *FromBB = BF.getBasicBlockContainingOffset(From);
auto *ToBB = BF.getBasicBlockContainingOffset(To);
BinaryBasicBlock *FromBB = BF.getBasicBlockContainingOffset(From);
BinaryBasicBlock *ToBB = BF.getBasicBlockContainingOffset(To);
if (!FromBB || !ToBB) {
LLVM_DEBUG(dbgs() << "failed to get block for recorded branch\n");
@ -731,7 +734,7 @@ bool DataReader::recordBranch(BinaryFunction &BF,
}
// Very rarely we will see ignored branches. Do a linear check.
for (auto &Branch : BF.IgnoredBranches) {
for (std::pair<uint32_t, uint32_t> &Branch : BF.IgnoredBranches) {
if (Branch == std::make_pair(static_cast<uint32_t>(From),
static_cast<uint32_t>(To)))
return true;
@ -742,10 +745,10 @@ bool DataReader::recordBranch(BinaryFunction &BF,
// While building the CFG we make sure these nops are attributed to the
// previous basic block, thus we check if the destination belongs to the
// gap past the last instruction.
const auto *LastInstr = ToBB->getLastNonPseudoInstr();
const MCInst *LastInstr = ToBB->getLastNonPseudoInstr();
if (LastInstr) {
const auto LastInstrOffset =
BC.MIB->getAnnotationWithDefault<uint32_t>(*LastInstr, "Offset");
const uint32_t LastInstrOffset =
BC.MIB->getAnnotationWithDefault<uint32_t>(*LastInstr, "Offset");
// With old .fdata we are getting FT branches for "jcc,jmp" sequences.
if (To == LastInstrOffset && BC.MIB->isUnconditionalBranch(*LastInstr)) {
@ -763,14 +766,14 @@ bool DataReader::recordBranch(BinaryFunction &BF,
// The real destination is the layout successor of the detected ToBB.
if (ToBB == BF.BasicBlocksLayout.back())
return false;
auto *NextBB = BF.BasicBlocksLayout[ToBB->getIndex() + 1];
BinaryBasicBlock *NextBB = BF.BasicBlocksLayout[ToBB->getIndex() + 1];
assert((NextBB && NextBB->getOffset() > ToBB->getOffset()) && "bad layout");
ToBB = NextBB;
}
// If there's no corresponding instruction for 'From', we have probably
// discarded it as a FT from __builtin_unreachable.
auto *FromInstruction = BF.getInstructionAtOffset(From);
MCInst *FromInstruction = BF.getInstructionAtOffset(From);
if (!FromInstruction) {
// If the data was collected in a bolted binary, the From addresses may be
// translated to the first instruction of the source BB if BOLT inserted
@ -813,7 +816,7 @@ bool DataReader::recordBranch(BinaryFunction &BF,
return false;
}
auto &BI = FromBB->getBranchInfo(*ToBB);
BinaryBasicBlock::BinaryBranchInfo &BI = FromBB->getBranchInfo(*ToBB);
BI.Count += Count;
// Only update mispredicted count if it the count was real.
if (Count) {
@ -859,7 +862,7 @@ ErrorOr<StringRef> DataReader::parseString(char EndChar, bool EndNl) {
std::string EndChars(1, EndChar);
if (EndNl)
EndChars.push_back('\n');
auto StringEnd = ParsingBuf.find_first_of(EndChars);
size_t StringEnd = ParsingBuf.find_first_of(EndChars);
if (StringEnd == StringRef::npos || StringEnd == 0) {
reportError("malformed field");
return make_error_code(llvm::errc::io_error);
@ -884,7 +887,7 @@ ErrorOr<StringRef> DataReader::parseString(char EndChar, bool EndNl) {
}
ErrorOr<int64_t> DataReader::parseNumberField(char EndChar, bool EndNl) {
auto NumStrRes = parseString(EndChar, EndNl);
ErrorOr<StringRef> NumStrRes = parseString(EndChar, EndNl);
if (std::error_code EC = NumStrRes.getError())
return EC;
StringRef NumStr = NumStrRes.get();
@ -898,7 +901,7 @@ ErrorOr<int64_t> DataReader::parseNumberField(char EndChar, bool EndNl) {
}
ErrorOr<uint64_t> DataReader::parseHexField(char EndChar, bool EndNl) {
auto NumStrRes = parseString(EndChar, EndNl);
ErrorOr<StringRef> NumStrRes = parseString(EndChar, EndNl);
if (std::error_code EC = NumStrRes.getError())
return EC;
StringRef NumStr = NumStrRes.get();
@ -942,14 +945,14 @@ ErrorOr<Location> DataReader::parseLocation(char EndChar,
consumeAllRemainingFS();
// Read the string containing the symbol or the DSO name
auto NameRes = parseString(FieldSeparator);
ErrorOr<StringRef> NameRes = parseString(FieldSeparator);
if (std::error_code EC = NameRes.getError())
return EC;
StringRef Name = NameRes.get();
consumeAllRemainingFS();
// Read the offset
auto Offset = parseHexField(EndChar, EndNl);
ErrorOr<uint64_t> Offset = parseHexField(EndChar, EndNl);
if (std::error_code EC = Offset.getError())
return EC;
@ -957,7 +960,7 @@ ErrorOr<Location> DataReader::parseLocation(char EndChar,
}
ErrorOr<BranchInfo> DataReader::parseBranchInfo() {
auto Res = parseLocation(FieldSeparator);
ErrorOr<Location> Res = parseLocation(FieldSeparator);
if (std::error_code EC = Res.getError())
return EC;
Location From = Res.get();
@ -969,13 +972,13 @@ ErrorOr<BranchInfo> DataReader::parseBranchInfo() {
Location To = Res.get();
consumeAllRemainingFS();
auto MRes = parseNumberField(FieldSeparator);
ErrorOr<int64_t> MRes = parseNumberField(FieldSeparator);
if (std::error_code EC = MRes.getError())
return EC;
int64_t NumMispreds = MRes.get();
consumeAllRemainingFS();
auto BRes = parseNumberField(FieldSeparator, /* EndNl = */ true);
ErrorOr<int64_t> BRes = parseNumberField(FieldSeparator, /* EndNl = */ true);
if (std::error_code EC = BRes.getError())
return EC;
int64_t NumBranches = BRes.get();
@ -990,7 +993,7 @@ ErrorOr<BranchInfo> DataReader::parseBranchInfo() {
}
ErrorOr<MemInfo> DataReader::parseMemInfo() {
auto Res = parseMemLocation(FieldSeparator);
ErrorOr<Location> Res = parseMemLocation(FieldSeparator);
if (std::error_code EC = Res.getError())
return EC;
Location Offset = Res.get();
@ -1002,7 +1005,7 @@ ErrorOr<MemInfo> DataReader::parseMemInfo() {
Location Addr = Res.get();
consumeAllRemainingFS();
auto CountRes = parseNumberField(FieldSeparator, true);
ErrorOr<int64_t> CountRes = parseNumberField(FieldSeparator, true);
if (std::error_code EC = CountRes.getError())
return EC;
@ -1016,13 +1019,13 @@ ErrorOr<MemInfo> DataReader::parseMemInfo() {
}
ErrorOr<SampleInfo> DataReader::parseSampleInfo() {
auto Res = parseLocation(FieldSeparator);
ErrorOr<Location> Res = parseLocation(FieldSeparator);
if (std::error_code EC = Res.getError())
return EC;
Location Address = Res.get();
consumeAllRemainingFS();
auto BRes = parseNumberField(FieldSeparator, /* EndNl = */ true);
ErrorOr<int64_t> BRes = parseNumberField(FieldSeparator, /* EndNl = */ true);
if (std::error_code EC = BRes.getError())
return EC;
int64_t Occurrences = BRes.get();
@ -1046,7 +1049,7 @@ ErrorOr<bool> DataReader::maybeParseNoLBRFlag() {
ParsingBuf = ParsingBuf.drop_front(1);
while (ParsingBuf.size() > 0 && ParsingBuf[0] != '\n') {
auto EventName = parseString(' ', true);
ErrorOr<StringRef> EventName = parseString(' ', true);
if (!EventName)
return make_error_code(llvm::errc::io_error);
EventNames.insert(EventName.get());
@ -1095,11 +1098,11 @@ std::error_code DataReader::parseInNoLBRMode() {
auto GetOrCreateFuncEntry = [&](StringRef Name) {
auto I = NamesToSamples.find(Name);
if (I == NamesToSamples.end()) {
bool success;
std::tie(I, success) = NamesToSamples.insert(std::make_pair(
bool Success;
std::tie(I, Success) = NamesToSamples.insert(std::make_pair(
Name, FuncSampleData(Name, FuncSampleData::ContainerTy())));
assert(success && "unexpected result of insert");
assert(Success && "unexpected result of insert");
}
return I;
};
@ -1107,16 +1110,16 @@ std::error_code DataReader::parseInNoLBRMode() {
auto GetOrCreateFuncMemEntry = [&](StringRef Name) {
auto I = NamesToMemEvents.find(Name);
if (I == NamesToMemEvents.end()) {
bool success;
std::tie(I, success) = NamesToMemEvents.insert(
bool Success;
std::tie(I, Success) = NamesToMemEvents.insert(
std::make_pair(Name, FuncMemData(Name, FuncMemData::ContainerTy())));
assert(success && "unexpected result of insert");
assert(Success && "unexpected result of insert");
}
return I;
};
while (hasBranchData()) {
auto Res = parseSampleInfo();
ErrorOr<SampleInfo> Res = parseSampleInfo();
if (std::error_code EC = Res.getError())
return EC;
@ -1126,12 +1129,12 @@ std::error_code DataReader::parseInNoLBRMode() {
if (!SI.Loc.IsSymbol)
continue;
auto I = GetOrCreateFuncEntry(SI.Loc.Name);
StringMapIterator<FuncSampleData> I = GetOrCreateFuncEntry(SI.Loc.Name);
I->getValue().Data.emplace_back(std::move(SI));
}
while (hasMemData()) {
auto Res = parseMemInfo();
ErrorOr<MemInfo> Res = parseMemInfo();
if (std::error_code EC = Res.getError())
return EC;
@ -1141,16 +1144,16 @@ std::error_code DataReader::parseInNoLBRMode() {
if (!MI.Offset.IsSymbol)
continue;
auto I = GetOrCreateFuncMemEntry(MI.Offset.Name);
StringMapIterator<FuncMemData> I = GetOrCreateFuncMemEntry(MI.Offset.Name);
I->getValue().Data.emplace_back(std::move(MI));
}
for (auto &FuncSamples : NamesToSamples) {
for (StringMapEntry<FuncSampleData> &FuncSamples : NamesToSamples) {
std::stable_sort(FuncSamples.second.Data.begin(),
FuncSamples.second.Data.end());
}
for (auto &MemEvents : NamesToMemEvents) {
for (StringMapEntry<FuncMemData> &MemEvents : NamesToMemEvents) {
std::stable_sort(MemEvents.second.Data.begin(),
MemEvents.second.Data.end());
}
@ -1162,12 +1165,11 @@ std::error_code DataReader::parse() {
auto GetOrCreateFuncEntry = [&](StringRef Name) {
auto I = NamesToBranches.find(Name);
if (I == NamesToBranches.end()) {
bool success;
std::tie(I, success) = NamesToBranches.insert(
std::make_pair(Name, FuncBranchData(Name,
FuncBranchData::ContainerTy(),
FuncBranchData::ContainerTy())));
assert(success && "unexpected result of insert");
bool Success;
std::tie(I, Success) = NamesToBranches.insert(std::make_pair(
Name, FuncBranchData(Name, FuncBranchData::ContainerTy(),
FuncBranchData::ContainerTy())));
assert(Success && "unexpected result of insert");
}
return I;
};
@ -1175,22 +1177,22 @@ std::error_code DataReader::parse() {
auto GetOrCreateFuncMemEntry = [&](StringRef Name) {
auto I = NamesToMemEvents.find(Name);
if (I == NamesToMemEvents.end()) {
bool success;
std::tie(I, success) = NamesToMemEvents.insert(
bool Success;
std::tie(I, Success) = NamesToMemEvents.insert(
std::make_pair(Name, FuncMemData(Name, FuncMemData::ContainerTy())));
assert(success && "unexpected result of insert");
assert(Success && "unexpected result of insert");
}
return I;
};
Col = 0;
Line = 1;
auto FlagOrErr = maybeParseNoLBRFlag();
ErrorOr<bool> FlagOrErr = maybeParseNoLBRFlag();
if (!FlagOrErr)
return FlagOrErr.getError();
NoLBRMode = *FlagOrErr;
auto BATFlagOrErr = maybeParseBATFlag();
ErrorOr<bool> BATFlagOrErr = maybeParseBATFlag();
if (!BATFlagOrErr)
return BATFlagOrErr.getError();
BATMode = *BATFlagOrErr;
@ -1204,7 +1206,7 @@ std::error_code DataReader::parse() {
return parseInNoLBRMode();
while (hasBranchData()) {
auto Res = parseBranchInfo();
ErrorOr<BranchInfo> Res = parseBranchInfo();
if (std::error_code EC = Res.getError())
return EC;
@ -1214,7 +1216,7 @@ std::error_code DataReader::parse() {
if (!BI.From.IsSymbol && !BI.To.IsSymbol)
continue;
auto I = GetOrCreateFuncEntry(BI.From.Name);
StringMapIterator<FuncBranchData> I = GetOrCreateFuncEntry(BI.From.Name);
I->getValue().Data.emplace_back(std::move(BI));
// Add entry data for branches to another function or branches
@ -1235,7 +1237,7 @@ std::error_code DataReader::parse() {
}
while (hasMemData()) {
auto Res = parseMemInfo();
ErrorOr<MemInfo> Res = parseMemInfo();
if (std::error_code EC = Res.getError())
return EC;
@ -1245,16 +1247,16 @@ std::error_code DataReader::parse() {
if (!MI.Offset.IsSymbol)
continue;
auto I = GetOrCreateFuncMemEntry(MI.Offset.Name);
StringMapIterator<FuncMemData> I = GetOrCreateFuncMemEntry(MI.Offset.Name);
I->getValue().Data.emplace_back(std::move(MI));
}
for (auto &FuncBranches : NamesToBranches) {
for (StringMapEntry<FuncBranchData> &FuncBranches : NamesToBranches) {
std::stable_sort(FuncBranches.second.Data.begin(),
FuncBranches.second.Data.end());
}
for (auto &MemEvents : NamesToMemEvents) {
for (StringMapEntry<FuncMemData> &MemEvents : NamesToMemEvents) {
std::stable_sort(MemEvents.second.Data.begin(),
MemEvents.second.Data.end());
}
@ -1263,16 +1265,16 @@ std::error_code DataReader::parse() {
}
void DataReader::buildLTONameMaps() {
for (auto &FuncData : NamesToBranches) {
const auto FuncName = FuncData.getKey();
const auto CommonName = getLTOCommonName(FuncName);
for (StringMapEntry<FuncBranchData> &FuncData : NamesToBranches) {
const StringRef FuncName = FuncData.getKey();
const Optional<StringRef> CommonName = getLTOCommonName(FuncName);
if (CommonName)
LTOCommonNameMap[*CommonName].push_back(&FuncData.getValue());
}
for (auto &FuncData : NamesToMemEvents) {
const auto FuncName = FuncData.getKey();
const auto CommonName = getLTOCommonName(FuncName);
for (StringMapEntry<FuncMemData> &FuncData : NamesToMemEvents) {
const StringRef FuncName = FuncData.getKey();
const Optional<StringRef> CommonName = getLTOCommonName(FuncName);
if (CommonName)
LTOCommonNameMemMap[*CommonName].push_back(&FuncData.getValue());
}
@ -1317,11 +1319,12 @@ fetchMapEntriesRegex(
for (auto FI = FuncNames.rbegin(), FE = FuncNames.rend(); FI != FE; ++FI) {
StringRef Name = *FI;
Name = normalizeName(Name);
const auto LTOCommonName = getLTOCommonName(Name);
const Optional<StringRef> LTOCommonName = getLTOCommonName(Name);
if (LTOCommonName) {
auto I = LTOCommonNameMap.find(*LTOCommonName);
if (I != LTOCommonNameMap.end()) {
auto &CommonData = I->getValue();
const std::vector<decltype(MapTy::MapEntryTy::second) *> &CommonData =
I->getValue();
AllData.insert(AllData.end(), CommonData.begin(), CommonData.end());
}
} else {
@ -1347,11 +1350,13 @@ bool DataReader::mayHaveProfileData(const BinaryFunction &Function) {
if (!hasVolatileName(Function))
return false;
const auto AllBranchData = getBranchDataForNamesRegex(Function.getNames());
const std::vector<FuncBranchData *> AllBranchData =
getBranchDataForNamesRegex(Function.getNames());
if (!AllBranchData.empty())
return true;
const auto AllMemData = getMemDataForNamesRegex(Function.getNames());
const std::vector<FuncMemData *> AllMemData =
getMemDataForNamesRegex(Function.getNames());
if (!AllMemData.empty())
return true;
@ -1390,8 +1395,8 @@ DataReader::getMemDataForNamesRegex(const std::vector<StringRef> &FuncNames) {
}
bool DataReader::hasLocalsWithFileName() const {
for (const auto &Func : NamesToBranches) {
const auto &FuncName = Func.getKey();
for (const StringMapEntry<FuncBranchData> &Func : NamesToBranches) {
const StringRef &FuncName = Func.getKey();
if (FuncName.count('/') == 2 && FuncName[0] != '/')
return true;
}
@ -1399,14 +1404,14 @@ bool DataReader::hasLocalsWithFileName() const {
}
void DataReader::dump() const {
for (const auto &Func : NamesToBranches) {
for (const StringMapEntry<FuncBranchData> &Func : NamesToBranches) {
Diag << Func.getKey() << " branches:\n";
for (const auto &BI : Func.getValue().Data) {
for (const BranchInfo &BI : Func.getValue().Data) {
Diag << BI.From.Name << " " << BI.From.Offset << " " << BI.To.Name << " "
<< BI.To.Offset << " " << BI.Mispreds << " " << BI.Branches << "\n";
}
Diag << Func.getKey() << " entry points:\n";
for (const auto &BI : Func.getValue().EntryData) {
for (const BranchInfo &BI : Func.getValue().EntryData) {
Diag << BI.From.Name << " " << BI.From.Offset << " " << BI.To.Name << " "
<< BI.To.Offset << " " << BI.Mispreds << " " << BI.Branches << "\n";
}
@ -1416,18 +1421,18 @@ void DataReader::dump() const {
StringRef Event = I->getKey();
Diag << "Data was collected with event: " << Event << "\n";
}
for (const auto &Func : NamesToSamples) {
for (const StringMapEntry<FuncSampleData> &Func : NamesToSamples) {
Diag << Func.getKey() << " samples:\n";
for (const auto &SI : Func.getValue().Data) {
for (const SampleInfo &SI : Func.getValue().Data) {
Diag << SI.Loc.Name << " " << SI.Loc.Offset << " "
<< SI.Hits << "\n";
}
}
for (const auto &Func : NamesToMemEvents) {
for (const StringMapEntry<FuncMemData> &Func : NamesToMemEvents) {
Diag << "Memory events for " << Func.getValue().Name;
Location LastOffset(0);
for (auto &MI : Func.getValue().Data) {
for (const MemInfo &MI : Func.getValue().Data) {
if (MI.Offset == LastOffset) {
Diag << ", " << MI.Addr << "/" << MI.Count;
} else {

16
bolt/src/DebugData.cpp
View File

@ -42,7 +42,7 @@ uint64_t writeAddressRanges(
raw_svector_ostream &Stream,
const DebugAddressRangesVector &AddressRanges,
const bool WriteRelativeRanges = false) {
for (auto &Range : AddressRanges) {
for (const DebugAddressRange &Range : AddressRanges) {
support::endian::write(Stream, Range.LowPC, support::little);
support::endian::write(
Stream, WriteRelativeRanges ? Range.HighPC - Range.LowPC : Range.HighPC,
@ -89,7 +89,7 @@ DebugRangesSectionWriter::addRanges(const DebugAddressRangesVector &Ranges) {
// Reading the SectionOffset and updating it should be atomic to guarantee
// unique and correct offsets in patches.
std::lock_guard<std::mutex> Lock(WriterMutex);
const auto EntryOffset = SectionOffset;
const uint32_t EntryOffset = SectionOffset;
SectionOffset += writeAddressRanges(*RangesStream.get(), Ranges);
return EntryOffset;
@ -107,8 +107,9 @@ void DebugARangesSectionWriter::writeARangesSection(
// specification, section 6.1.4 Lookup by Address
// http://www.dwarfstd.org/doc/DWARF4.pdf
for (const auto &CUOffsetAddressRangesPair : CUAddressRanges) {
const auto Offset = CUOffsetAddressRangesPair.first;
const auto &AddressRanges = CUOffsetAddressRangesPair.second;
const uint64_t Offset = CUOffsetAddressRangesPair.first;
const DebugAddressRangesVector &AddressRanges =
CUOffsetAddressRangesPair.second;
// Emit header.
@ -157,7 +158,7 @@ DebugLocWriter::addList(const DebugLocationsVector &LocList) {
// Since there is a separate DebugLocWriter for each thread,
// we don't need a lock to read the SectionOffset and update it.
const auto EntryOffset = SectionOffset;
const uint32_t EntryOffset = SectionOffset;
for (const DebugLocationEntry &Entry : LocList) {
support::endian::write(*LocStream, static_cast<uint64_t>(Entry.LowPC),
@ -236,8 +237,9 @@ void DebugAbbrevPatcher::addAttributePatch(
void DebugAbbrevPatcher::patchBinary(std::string &Contents) {
SimpleBinaryPatcher Patcher;
for (const auto &Patch : AbbrevPatches) {
const auto Attribute = Patch.Abbrev->findAttribute(Patch.Attr);
for (const AbbrevAttrPatch &Patch : AbbrevPatches) {
const DWARFAbbreviationDeclaration::AttributeSpec *const Attribute =
Patch.Abbrev->findAttribute(Patch.Attr);
assert(Attribute && "Specified attribute doesn't occur in abbreviation.");
// Because we're only handling standard values (i.e. no DW_FORM_GNU_* or

14
bolt/src/DynoStats.cpp
View File

@ -120,7 +120,7 @@ DynoStats getDynoStats(const BinaryFunction &BF) {
// direction.
BF.updateLayoutIndices();
for (const auto &BB : BF.layout()) {
for (BinaryBasicBlock *const &BB : BF.layout()) {
// The basic block execution count equals to the sum of incoming branch
// frequencies. This may deviate from the sum of outgoing branches of the
// basic block especially since the block may contain a function that
@ -136,7 +136,7 @@ DynoStats getDynoStats(const BinaryFunction &BF) {
Stats[DynoStats::VENEER_CALLS_AARCH64] += BF.getKnownExecutionCount();
// Count the number of calls by iterating through all instructions.
for (const auto &Instr : *BB) {
for (const MCInst &Instr : *BB) {
if (BC.MIB->isStore(Instr)) {
Stats[DynoStats::STORES] += BBExecutionCount;
}
@ -155,8 +155,8 @@ DynoStats getDynoStats(const BinaryFunction &BF) {
Stats[DynoStats::FUNCTION_CALLS] += CallFreq;
if (BC.MIB->isIndirectCall(Instr)) {
Stats[DynoStats::INDIRECT_CALLS] += CallFreq;
} else if (const auto *CallSymbol = BC.MIB->getTargetSymbol(Instr)) {
const auto *BF = BC.getFunctionForSymbol(CallSymbol);
} else if (const MCSymbol *CallSymbol = BC.MIB->getTargetSymbol(Instr)) {
const BinaryFunction *BF = BC.getFunctionForSymbol(CallSymbol);
if (BF && BF->isPLTFunction()) {
Stats[DynoStats::PLT_CALLS] += CallFreq;
@ -177,7 +177,7 @@ DynoStats getDynoStats(const BinaryFunction &BF) {
Stats[DynoStats::INSTRUCTIONS] += BB->getNumNonPseudos() * BBExecutionCount;
// Jump tables.
const auto *LastInstr = BB->getLastNonPseudoInstr();
const MCInst *LastInstr = BB->getLastNonPseudoInstr();
if (BC.MIB->getJumpTable(*LastInstr)) {
Stats[DynoStats::JUMP_TABLE_BRANCHES] += BBExecutionCount;
LLVM_DEBUG(
@ -225,11 +225,11 @@ DynoStats getDynoStats(const BinaryFunction &BF) {
}
// Conditional branch that could be followed by an unconditional branch.
auto TakenCount = BB->getTakenBranchInfo().Count;
uint64_t TakenCount = BB->getTakenBranchInfo().Count;
if (TakenCount == BinaryBasicBlock::COUNT_NO_PROFILE)
TakenCount = 0;
auto NonTakenCount = BB->getFallthroughBranchInfo().Count;
uint64_t NonTakenCount = BB->getFallthroughBranchInfo().Count;
if (NonTakenCount == BinaryBasicBlock::COUNT_NO_PROFILE)
NonTakenCount = 0;

4
bolt/src/DynoStats.h
View File

@ -160,8 +160,8 @@ callWithDynoStats(FnType &&Func,
Func();
if (Flag) {
const auto DynoStatsAfter = getDynoStats(Funcs);
const auto Changed = (DynoStatsAfter != DynoStatsBefore);
const DynoStats DynoStatsAfter = getDynoStats(Funcs);
const bool Changed = (DynoStatsAfter != DynoStatsBefore);
outs() << "BOLT-INFO: program-wide dynostats after running "
<< Phase << (Changed ? "" : " (no change)") << ":\n\n"
<< DynoStatsBefore << '\n';

58
bolt/src/Exceptions.cpp
View File

@ -119,13 +119,13 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
uint8_t LPStartEncoding = Data.getU8(&Offset);
uint64_t LPStart = 0;
if (auto MaybeLPStart = Data.getEncodedPointer(&Offset, LPStartEncoding,
Offset + LSDASectionAddress))
if (Optional<uint64_t> MaybeLPStart = Data.getEncodedPointer(
&Offset, LPStartEncoding, Offset + LSDASectionAddress))
LPStart = *MaybeLPStart;
assert(LPStart == 0 && "support for split functions not implemented");
const auto TTypeEncoding = Data.getU8(&Offset);
const uint8_t TTypeEncoding = Data.getU8(&Offset);
size_t TTypeEncodingSize = 0;
uintptr_t TTypeEnd = 0;
if (TTypeEncoding != DW_EH_PE_omit) {
@ -154,14 +154,14 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
// The actual type info table starts at the same location, but grows in
// opposite direction. TTypeEncoding is used to encode stored values.
const auto TypeTableStart = Offset + TTypeEnd;
const uint64_t TypeTableStart = Offset + TTypeEnd;
uint8_t CallSiteEncoding = Data.getU8(&Offset);
uint32_t CallSiteTableLength = Data.getULEB128(&Offset);
auto CallSiteTableStart = Offset;
auto CallSiteTableEnd = CallSiteTableStart + CallSiteTableLength;
auto CallSitePtr = CallSiteTableStart;
auto ActionTableStart = CallSiteTableEnd;
uint64_t CallSiteTableStart = Offset;
uint64_t CallSiteTableEnd = CallSiteTableStart + CallSiteTableLength;
uint64_t CallSitePtr = CallSiteTableStart;
uint64_t ActionTableStart = CallSiteTableEnd;
if (opts::PrintExceptions) {
outs() << "CallSite Encoding = " << (unsigned)CallSiteEncoding << '\n';
@ -214,7 +214,7 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
auto IE = Instructions.end();
assert(II != IE && "exception range not pointing to an instruction");
do {
auto &Instruction = II->second;
MCInst &Instruction = II->second;
if (BC.MIB->isCall(Instruction) &&
!BC.MIB->getConditionalTailCall(Instruction)) {
assert(!BC.MIB->isInvoke(Instruction) &&
@ -242,11 +242,11 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
return;
}
if (TTypeEncoding & DW_EH_PE_indirect) {
auto PointerOrErr = BC.getPointerAtAddress(TypeAddress);
ErrorOr<uint64_t> PointerOrErr = BC.getPointerAtAddress(TypeAddress);
assert(PointerOrErr && "failed to decode indirect address");
TypeAddress = *PointerOrErr;
}
if (auto *TypeSymBD = BC.getBinaryDataAtAddress(TypeAddress)) {
if (BinaryData *TypeSymBD = BC.getBinaryDataAtAddress(TypeAddress)) {
OS << TypeSymBD->getName();
} else {
OS << "0x" << Twine::utohexstr(TypeAddress);
@ -257,7 +257,7 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
uint64_t ActionPtr = ActionTableStart + ActionEntry - 1;
int64_t ActionType;
int64_t ActionNext;
auto Sep = "";
const char *Sep = "";
do {
ActionType = Data.getSLEB128(&ActionPtr);
const uint32_t Self = ActionPtr;
@ -278,13 +278,13 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
} else { // ActionType < 0
if (opts::PrintExceptions)
outs() << "filter exception types ";
auto TSep = "";
const char *TSep = "";
// ActionType is a negative *byte* offset into *uleb128-encoded* table
// of indices with base 1.
// E.g. -1 means offset 0, -2 is offset 1, etc. The indices are
// encoded using uleb128 thus we cannot directly dereference them.
uint64_t TypeIndexTablePtr = TypeIndexTableStart - ActionType - 1;
while (auto Index = Data.getULEB128(&TypeIndexTablePtr)) {
while (uint64_t Index = Data.getULEB128(&TypeIndexTablePtr)) {
MaxTypeIndex = std::max(MaxTypeIndex, static_cast<unsigned>(Index));
if (opts::PrintExceptions) {
outs() << TSep;
@ -319,7 +319,7 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
ActionTableStart);
for (unsigned Index = 1; Index <= MaxTypeIndex; ++Index) {
uint64_t TTEntry = TypeTableStart - Index * TTypeEncodingSize;
const auto TTEntryAddress = TTEntry + LSDASectionAddress;
const uint64_t TTEntryAddress = TTEntry + LSDASectionAddress;
uint64_t TypeAddress =
*Data.getEncodedPointer(&TTEntry, TTypeEncoding, TTEntryAddress);
if ((TTypeEncoding & DW_EH_PE_pcrel) && (TypeAddress == TTEntryAddress))
@ -327,7 +327,7 @@ void BinaryFunction::parseLSDA(ArrayRef<uint8_t> LSDASectionData,
if (TTypeEncoding & DW_EH_PE_indirect) {
LSDATypeAddressTable.emplace_back(TypeAddress);
if (TypeAddress) {
auto PointerOrErr = BC.getPointerAtAddress(TypeAddress);
ErrorOr<uint64_t> PointerOrErr = BC.getPointerAtAddress(TypeAddress);
assert(PointerOrErr && "failed to decode indirect address");
TypeAddress = *PointerOrErr;
}
@ -364,9 +364,9 @@ void BinaryFunction::updateEHRanges() {
bool SeenCold = false;
// Sites to update - either regular or cold.
auto *Sites = &CallSites;
std::vector<CallSite> *Sites = &CallSites;
for (auto &BB : BasicBlocksLayout) {
for (BinaryBasicBlock *&BB : BasicBlocksLayout) {
if (BB->isCold() && !SeenCold) {
SeenCold = true;
@ -397,7 +397,7 @@ void BinaryFunction::updateEHRanges() {
// Extract exception handling information from the instruction.
const MCSymbol *LP = nullptr;
uint64_t Action = 0;
if (const auto EHInfo = BC.MIB->getEHInfo(*II))
if (const Optional<MCPlus::MCLandingPad> EHInfo = BC.MIB->getEHInfo(*II))
std::tie(LP, Action) = *EHInfo;
// No action if the exception handler has not changed.
@ -457,8 +457,8 @@ void BinaryFunction::updateEHRanges() {
// Check if we need to close the range.
if (StartRange) {
assert((!isSplit() || Sites == &ColdCallSites) && "sites mismatch");
const auto *EndRange = IsStartInCold ? getFunctionColdEndLabel()
: getFunctionEndLabel();
const MCSymbol *EndRange =
IsStartInCold ? getFunctionColdEndLabel() : getFunctionEndLabel();
Sites->emplace_back(CallSite{StartRange, EndRange,
PreviousEH.LP, PreviousEH.Action});
}
@ -468,7 +468,7 @@ const uint8_t DWARF_CFI_PRIMARY_OPCODE_MASK = 0xc0;
CFIReaderWriter::CFIReaderWriter(const DWARFDebugFrame &EHFrame) {
// Prepare FDEs for fast lookup
for (const auto &Entry : EHFrame.entries()) {
for (const dwarf::FrameEntry &Entry : EHFrame.entries()) {
const auto *CurFDE = dyn_cast<dwarf::FDE>(&Entry);
// Skip CIEs.
if (!CurFDE)
@ -502,7 +502,7 @@ bool CFIReaderWriter::fillCFIInfoFor(BinaryFunction &Function) const {
return true;
const FDE &CurFDE = *I->second;
auto LSDA = CurFDE.getLSDAAddress();
Optional<uint64_t> LSDA = CurFDE.getLSDAAddress();
Function.setLSDAAddress(LSDA ? *LSDA : 0);
uint64_t Offset = 0;
@ -684,8 +684,9 @@ std::vector<char> CFIReaderWriter::generateEHFrameHeader(
const dwarf::FDE *FDE = dyn_cast<dwarf::FDE>(&Entry);
if (FDE == nullptr)
continue;
const auto FuncAddress = FDE->getInitialLocation();
const auto FDEAddress = NewEHFrame.getEHFrameAddress() + FDE->getOffset();
const uint64_t FuncAddress = FDE->getInitialLocation();
const uint64_t FDEAddress =
NewEHFrame.getEHFrameAddress() + FDE->getOffset();
// Ignore unused FDEs.
if (FuncAddress == 0)
@ -712,8 +713,9 @@ std::vector<char> CFIReaderWriter::generateEHFrameHeader(
const dwarf::FDE *FDE = dyn_cast<dwarf::FDE>(&Entry);
if (FDE == nullptr)
continue;
const auto FuncAddress = FDE->getInitialLocation();
const auto FDEAddress = OldEHFrame.getEHFrameAddress() + FDE->getOffset();
const uint64_t FuncAddress = FDE->getInitialLocation();
const uint64_t FDEAddress =
OldEHFrame.getEHFrameAddress() + FDE->getOffset();
// Add the address if we failed to write it.
if (PCToFDE.count(FuncAddress) == 0) {
@ -751,7 +753,7 @@ std::vector<char> CFIReaderWriter::generateEHFrameHeader(
support::ulittle32_t::ref(EHFrameHeader.data() + 8) = PCToFDE.size();
// Write the table at offset 12.
auto *Ptr = EHFrameHeader.data();
char *Ptr = EHFrameHeader.data();
uint32_t Offset = 12;
for (const auto &PCI : PCToFDE) {
int64_t InitialPCOffset = PCI.first - EHFrameHeaderAddress;

8
bolt/src/ExecutableFileMemoryManager.cpp
View File

@ -31,10 +31,8 @@ uint8_t *ExecutableFileMemoryManager::allocateSection(intptr_t Size,
// Register a debug section as a note section.
if (!ObjectsLoaded && RewriteInstance::isDebugSection(SectionName)) {
uint8_t *DataCopy = new uint8_t[Size];
auto &Section = BC.registerOrUpdateNoteSection(SectionName,
DataCopy,
Size,
Alignment);
BinarySection &Section =
BC.registerOrUpdateNoteSection(SectionName, DataCopy, Size, Alignment);
Section.setSectionID(SectionID);
assert(!Section.isAllocatable() && "note sections cannot be allocatable");
return DataCopy;
@ -73,7 +71,7 @@ uint8_t *ExecutableFileMemoryManager::allocateSection(intptr_t Size,
}
}
auto &Section = BC.registerOrUpdateSection(
BinarySection &Section = BC.registerOrUpdateSection(
SectionName, ELF::SHT_PROGBITS,
BinarySection::getFlags(IsReadOnly, IsCode, true), Ret, Size, Alignment);
Section.setSectionID(SectionID);

20
bolt/src/Heatmap.cpp
View File

@ -87,7 +87,7 @@ void Heatmap::print(raw_ostream &OS) const {
// Calculate the max value for scaling.
uint64_t MaxValue = 0;
for (auto &Entry : Map) {
for (const std::pair<const uint64_t, uint64_t> &Entry : Map) {
MaxValue = std::max<uint64_t>(MaxValue, Entry.second);
}
@ -95,7 +95,7 @@ void Heatmap::print(raw_ostream &OS) const {
// the Address.
auto startLine = [&](uint64_t Address, bool Empty = false) {
changeColor(DefaultColor);
const auto LineAddress = Address / BytesPerLine * BytesPerLine;
const uint64_t LineAddress = Address / BytesPerLine * BytesPerLine;
if (MaxAddress > 0xffffffff)
OS << format("0x%016" PRIx64 ": ", LineAddress);
@ -104,15 +104,15 @@ void Heatmap::print(raw_ostream &OS) const {
if (Empty)
Address = LineAddress + BytesPerLine;
for (auto Fill = LineAddress; Fill < Address; Fill += BucketSize) {
for (uint64_t Fill = LineAddress; Fill < Address; Fill += BucketSize) {
OS << FillChar;
}
};
// Finish line after \p Address was printed.
auto finishLine = [&](uint64_t Address) {
const auto End = alignTo(Address + 1, BytesPerLine);
for (auto Fill = Address + BucketSize; Fill < End; Fill += BucketSize)
const uint64_t End = alignTo(Address + 1, BytesPerLine);
for (uint64_t Fill = Address + BucketSize; Fill < End; Fill += BucketSize)
OS << FillChar;
OS << '\n';
};
@ -120,7 +120,7 @@ void Heatmap::print(raw_ostream &OS) const {
// Fill empty space in (Start, End) range.
auto fillRange = [&](uint64_t Start, uint64_t End) {
if ((Start / BytesPerLine) == (End / BytesPerLine)) {
for (auto Fill = Start + BucketSize; Fill < End; Fill += BucketSize) {
for (uint64_t Fill = Start + BucketSize; Fill < End; Fill += BucketSize) {
changeColor(DefaultColor);
OS << FillChar;
}
@ -189,7 +189,7 @@ void Heatmap::print(raw_ostream &OS) const {
OS << "Legend:\n";
uint64_t PrevValue = 0;
for (unsigned I = 0; I < sizeof(Range) / sizeof(Range[0]); ++I) {
const auto Value = Range[I];
const uint64_t Value = Range[I];
OS << " ";
printValue(Value, true);
OS << " : (" << PrevValue << ", " << Value << "]\n";
@ -203,7 +203,7 @@ void Heatmap::print(raw_ostream &OS) const {
OS << " ";
unsigned PrevValue = unsigned(-1);
for (unsigned I = 0; I < BytesPerLine; I += BucketSize) {
const auto Value = (I & ((1 << Pos * 4) - 1)) >> (Pos - 1) * 4;
const unsigned Value = (I & ((1 << Pos * 4) - 1)) >> (Pos - 1) * 4;
if (Value != PrevValue) {
OS << Twine::utohexstr(Value);
PrevValue = Value;
@ -218,7 +218,7 @@ void Heatmap::print(raw_ostream &OS) const {
uint64_t PrevAddress = 0;
for (auto MI = Map.begin(), ME = Map.end(); MI != ME; ++MI) {
auto &Entry = *MI;
const std::pair<const uint64_t, uint64_t> &Entry = *MI;
uint64_t Address = Entry.first * BucketSize;
if (PrevAddress) {
@ -252,7 +252,7 @@ void Heatmap::printCDF(raw_ostream &OS) const {
uint64_t NumTotalCounts{0};
std::vector<uint64_t> Counts;
for (const auto &KV : Map) {
for (const std::pair<const uint64_t, uint64_t> &KV : Map) {
Counts.push_back(KV.second);
NumTotalCounts += KV.second;
}

14
bolt/src/JumpTable.cpp
View File

@ -56,8 +56,8 @@ JumpTable::getEntriesForAddress(const uint64_t Addr) const {
auto LI = Labels.find(Offset);
if (LI != Labels.end()) {
const auto NextLI = std::next(LI);
const auto NextOffset =
NextLI == Labels.end() ? getSize() : NextLI->first;
const uint64_t NextOffset =
NextLI == Labels.end() ? getSize() : NextLI->first;
if (InstOffset >= LI->first && InstOffset < NextOffset) {
StartIndex = I;
EndIndex = I;
@ -77,10 +77,10 @@ JumpTable::getEntriesForAddress(const uint64_t Addr) const {
bool JumpTable::replaceDestination(uint64_t JTAddress, const MCSymbol *OldDest,
MCSymbol *NewDest) {
bool Patched{false};
const auto Range = getEntriesForAddress(JTAddress);
const std::pair<size_t, size_t> Range = getEntriesForAddress(JTAddress);
for (auto I = &Entries[Range.first], E = &Entries[Range.second]; I != E;
++I) {
auto &Entry = *I;
MCSymbol *&Entry = *I;
if (Entry == OldDest) {
Patched = true;
Entry = NewDest;
@ -95,7 +95,7 @@ void JumpTable::updateOriginal() {
// overwritten.
const uint64_t BaseOffset = getAddress() - getSection().getAddress();
uint64_t Offset = BaseOffset;
for (auto *Entry : Entries) {
for (MCSymbol *Entry : Entries) {
const auto RelType =
Type == JTT_NORMAL ? ELF::R_X86_64_64 : ELF::R_X86_64_PC32;
const uint64_t RelAddend = (Type == JTT_NORMAL ? 0 : Offset - BaseOffset);
@ -116,10 +116,10 @@ void JumpTable::print(raw_ostream &OS) const {
OS << "Jump table " << getName() << " for function " << *Parent << " at 0x"
<< Twine::utohexstr(getAddress()) << " with a total count of " << Count
<< ":\n";
for (const auto EntryOffset : OffsetEntries) {
for (const uint64_t EntryOffset : OffsetEntries) {
OS << " 0x" << Twine::utohexstr(EntryOffset) << '\n';
}
for (const auto *Entry : Entries) {
for (const MCSymbol *Entry : Entries) {
auto LI = Labels.find(Offset);
if (Offset && LI != Labels.end()) {
OS << "Jump Table " << LI->second->getName() << " at 0x"

57
bolt/src/MCPlusBuilder.cpp
View File

@ -123,10 +123,12 @@ bool MCPlusBuilder::equals(const MCTargetExpr &A, const MCTargetExpr &B,
Optional<MCLandingPad> MCPlusBuilder::getEHInfo(const MCInst &Inst) const {
if (!isCall(Inst))
return NoneType();
auto LPSym = getAnnotationOpValue(Inst, MCAnnotation::kEHLandingPad);
Optional<int64_t> LPSym =
getAnnotationOpValue(Inst, MCAnnotation::kEHLandingPad);
if (!LPSym)
return NoneType();
auto Action = getAnnotationOpValue(Inst, MCAnnotation::kEHAction);
Optional<int64_t> Action =
getAnnotationOpValue(Inst, MCAnnotation::kEHAction);
if (!Action)
return NoneType();
@ -145,7 +147,8 @@ void MCPlusBuilder::addEHInfo(MCInst &Inst, const MCLandingPad &LP) {
}
int64_t MCPlusBuilder::getGnuArgsSize(const MCInst &Inst) const {
auto Value = getAnnotationOpValue(Inst, MCAnnotation::kGnuArgsSize);
Optional<int64_t> Value =
getAnnotationOpValue(Inst, MCAnnotation::kGnuArgsSize);
if (!Value)
return -1LL;
return *Value;
@ -161,7 +164,8 @@ void MCPlusBuilder::addGnuArgsSize(MCInst &Inst, int64_t GnuArgsSize,
}
uint64_t MCPlusBuilder::getJumpTable(const MCInst &Inst) const {
auto Value = getAnnotationOpValue(Inst, MCAnnotation::kJumpTable);
Optional<int64_t> Value =
getAnnotationOpValue(Inst, MCAnnotation::kJumpTable);
if (!Value)
return 0;
return *Value;
@ -190,7 +194,8 @@ bool MCPlusBuilder::unsetJumpTable(MCInst &Inst) {
Optional<uint64_t>
MCPlusBuilder::getConditionalTailCall(const MCInst &Inst) const {
auto Value = getAnnotationOpValue(Inst, MCAnnotation::kConditionalTailCall);
Optional<int64_t> Value =
getAnnotationOpValue(Inst, MCAnnotation::kConditionalTailCall);
if (!Value)
return NoneType();
return static_cast<uint64_t>(*Value);
@ -213,7 +218,7 @@ bool MCPlusBuilder::unsetConditionalTailCall(MCInst &Inst) {
}
bool MCPlusBuilder::hasAnnotation(const MCInst &Inst, unsigned Index) const {
const auto *AnnotationInst = getAnnotationInst(Inst);
const MCInst *AnnotationInst = getAnnotationInst(Inst);
if (!AnnotationInst)
return false;
@ -221,12 +226,12 @@ bool MCPlusBuilder::hasAnnotation(const MCInst &Inst, unsigned Index) const {
}
bool MCPlusBuilder::removeAnnotation(MCInst &Inst, unsigned Index) {
auto *AnnotationInst = getAnnotationInst(Inst);
MCInst *AnnotationInst = getAnnotationInst(Inst);
if (!AnnotationInst)
return false;
for (int I = AnnotationInst->getNumOperands() - 1; I >= 0; --I) {
auto ImmValue = AnnotationInst->getOperand(I).getImm();
int64_t ImmValue = AnnotationInst->getOperand(I).getImm();
if (extractAnnotationIndex(ImmValue) == Index) {
AnnotationInst->erase(AnnotationInst->begin() + I);
return true;
@ -236,7 +241,7 @@ bool MCPlusBuilder::removeAnnotation(MCInst &Inst, unsigned Index) {
}
void MCPlusBuilder::stripAnnotations(MCInst &Inst, bool KeepTC) {
auto *AnnotationInst = getAnnotationInst(Inst);
MCInst *AnnotationInst = getAnnotationInst(Inst);
if (!AnnotationInst)
return;
// Preserve TailCall annotation.
@ -249,14 +254,14 @@ void MCPlusBuilder::stripAnnotations(MCInst &Inst, bool KeepTC) {
void
MCPlusBuilder::printAnnotations(const MCInst &Inst, raw_ostream &OS) const {
const auto *AnnotationInst = getAnnotationInst(Inst);
const MCInst *AnnotationInst = getAnnotationInst(Inst);
if (!AnnotationInst)
return;
for (unsigned I = 0; I < AnnotationInst->getNumOperands(); ++I) {
const auto Imm = AnnotationInst->getOperand(I).getImm();
const auto Index = extractAnnotationIndex(Imm);
const auto Value = extractAnnotationValue(Imm);
const int64_t Imm = AnnotationInst->getOperand(I).getImm();
const unsigned Index = extractAnnotationIndex(Imm);
const int64_t Value = extractAnnotationValue(Imm);
const auto *Annotation =
reinterpret_cast<const MCAnnotation *>(Value);
if (Index >= MCAnnotation::kGeneric) {
@ -277,15 +282,15 @@ void MCPlusBuilder::getClobberedRegs(const MCInst &Inst,
if (isPrefix(Inst) || isCFI(Inst))
return;
const auto &InstInfo = Info->get(Inst.getOpcode());
const MCInstrDesc &InstInfo = Info->get(Inst.getOpcode());
const auto *ImplicitDefs = InstInfo.getImplicitDefs();
const MCPhysReg *ImplicitDefs = InstInfo.getImplicitDefs();
for (unsigned I = 0, E = InstInfo.getNumImplicitDefs(); I != E; ++I) {
Regs |= getAliases(ImplicitDefs[I], /*OnlySmaller=*/false);
}
for (unsigned I = 0, E = InstInfo.getNumDefs(); I != E; ++I) {
const auto &Operand = Inst.getOperand(I);
const MCOperand &Operand = Inst.getOperand(I);
assert(Operand.isReg());
Regs |= getAliases(Operand.getReg(), /*OnlySmaller=*/false);
}
@ -296,13 +301,13 @@ void MCPlusBuilder::getTouchedRegs(const MCInst &Inst,
if (isPrefix(Inst) || isCFI(Inst))
return;
const auto &InstInfo = Info->get(Inst.getOpcode());
const MCInstrDesc &InstInfo = Info->get(Inst.getOpcode());
const auto *ImplicitDefs = InstInfo.getImplicitDefs();
const MCPhysReg *ImplicitDefs = InstInfo.getImplicitDefs();
for (unsigned I = 0, E = InstInfo.getNumImplicitDefs(); I != E; ++I) {
Regs |= getAliases(ImplicitDefs[I], /*OnlySmaller=*/false);
}
const auto *ImplicitUses = InstInfo.getImplicitUses();
const MCPhysReg *ImplicitUses = InstInfo.getImplicitUses();
for (unsigned I = 0, E = InstInfo.getNumImplicitUses(); I != E; ++I) {
Regs |= getAliases(ImplicitUses[I], /*OnlySmaller=*/false);
}
@ -319,15 +324,15 @@ void MCPlusBuilder::getWrittenRegs(const MCInst &Inst,
if (isPrefix(Inst) || isCFI(Inst))
return;
const auto &InstInfo = Info->get(Inst.getOpcode());
const MCInstrDesc &InstInfo = Info->get(Inst.getOpcode());
const auto *ImplicitDefs = InstInfo.getImplicitDefs();
const MCPhysReg *ImplicitDefs = InstInfo.getImplicitDefs();
for (unsigned I = 0, E = InstInfo.getNumImplicitDefs(); I != E; ++I) {
Regs |= getAliases(ImplicitDefs[I], /*OnlySmaller=*/true);
}
for (unsigned I = 0, E = InstInfo.getNumDefs(); I != E; ++I) {
const auto &Operand = Inst.getOperand(I);
const MCOperand &Operand = Inst.getOperand(I);
assert(Operand.isReg());
Regs |= getAliases(Operand.getReg(), /*OnlySmaller=*/true);
}
@ -337,9 +342,9 @@ void MCPlusBuilder::getUsedRegs(const MCInst &Inst, BitVector &Regs) const {
if (isPrefix(Inst) || isCFI(Inst))
return;
const auto &InstInfo = Info->get(Inst.getOpcode());
const MCInstrDesc &InstInfo = Info->get(Inst.getOpcode());
const auto *ImplicitUses = InstInfo.getImplicitUses();
const MCPhysReg *ImplicitUses = InstInfo.getImplicitUses();
for (unsigned I = 0, E = InstInfo.getNumImplicitUses(); I != E; ++I) {
Regs |= getAliases(ImplicitUses[I], /*OnlySmaller=*/true);
}
@ -352,12 +357,12 @@ void MCPlusBuilder::getUsedRegs(const MCInst &Inst, BitVector &Regs) const {
}
bool MCPlusBuilder::hasDefOfPhysReg(const MCInst &MI, unsigned Reg) const {
const auto &InstInfo = Info->get(MI.getOpcode());
const MCInstrDesc &InstInfo = Info->get(MI.getOpcode());
return InstInfo.hasDefOfPhysReg(MI, Reg, *RegInfo);
}
bool MCPlusBuilder::hasUseOfPhysReg(const MCInst &MI, unsigned Reg) const {
const auto &InstInfo = Info->get(MI.getOpcode());
const MCInstrDesc &InstInfo = Info->get(MI.getOpcode());
for (int I = InstInfo.NumDefs; I < InstInfo.NumOperands; ++I)
if (MI.getOperand(I).isReg() &&
RegInfo->isSubRegisterEq(Reg, MI.getOperand(I).getReg()))

42
bolt/src/MCPlusBuilder.h
View File

@ -99,11 +99,11 @@ private:
if (Inst.getNumOperands() == 0)
return nullptr;
const auto &LastOp = Inst.getOperand(Inst.getNumOperands() - 1);
const MCOperand &LastOp = Inst.getOperand(Inst.getNumOperands() - 1);
if (!LastOp.isInst())
return nullptr;
auto *AnnotationInst = const_cast<MCInst *>(LastOp.getInst());
MCInst *AnnotationInst = const_cast<MCInst *>(LastOp.getInst());
assert(AnnotationInst->getOpcode() == TargetOpcode::ANNOTATION_LABEL);
return AnnotationInst;
@ -111,17 +111,17 @@ private:
void setAnnotationOpValue(MCInst &Inst, unsigned Index, int64_t Value,
AllocatorIdTy AllocatorId = 0) {
auto *AnnotationInst = getAnnotationInst(Inst);
MCInst *AnnotationInst = getAnnotationInst(Inst);
if (!AnnotationInst) {
auto &Allocator = getAnnotationAllocator(AllocatorId);
AnnotationAllocator &Allocator = getAnnotationAllocator(AllocatorId);
AnnotationInst = new (Allocator.MCInstAllocator.Allocate()) MCInst();
AnnotationInst->setOpcode(TargetOpcode::ANNOTATION_LABEL);
Inst.addOperand(MCOperand::createInst(AnnotationInst));
}
const auto AnnotationValue = encodeAnnotationImm(Index, Value);
const int64_t AnnotationValue = encodeAnnotationImm(Index, Value);
for (int I = AnnotationInst->getNumOperands() - 1; I >= 0; --I) {
auto ImmValue = AnnotationInst->getOperand(I).getImm();
int64_t ImmValue = AnnotationInst->getOperand(I).getImm();
if (extractAnnotationIndex(ImmValue) == Index) {
AnnotationInst->getOperand(I).setImm(AnnotationValue);
return;
@ -133,12 +133,12 @@ private:
Optional<int64_t>
getAnnotationOpValue(const MCInst &Inst, unsigned Index) const {
const auto *AnnotationInst = getAnnotationInst(Inst);
const MCInst *AnnotationInst = getAnnotationInst(Inst);
if (!AnnotationInst)
return NoneType();
for (int I = AnnotationInst->getNumOperands() - 1; I >= 0; --I) {
auto ImmValue = AnnotationInst->getOperand(I).getImm();
int64_t ImmValue = AnnotationInst->getOperand(I).getImm();
if (extractAnnotationIndex(ImmValue) == Index) {
return extractAnnotationValue(ImmValue);
}
@ -307,8 +307,8 @@ public:
/// Free the values allocator within the annotation allocator
void freeValuesAllocator(AllocatorIdTy AllocatorId) {
auto &Allocator = getAnnotationAllocator(AllocatorId);
for (auto *Annotation : Allocator.AnnotationPool)
AnnotationAllocator &Allocator = getAnnotationAllocator(AllocatorId);
for (MCPlus::MCAnnotation *Annotation : Allocator.AnnotationPool)
Annotation->~MCAnnotation();
Allocator.AnnotationPool.clear();
@ -322,8 +322,8 @@ public:
/// Free all memory allocated for annotations
void freeAnnotations() {
for (auto &Element : AnnotationAllocators) {
auto &Allocator = Element.second;
for (auto *Annotation : Allocator.AnnotationPool)
AnnotationAllocator &Allocator = Element.second;
for (MCPlus::MCAnnotation *Annotation : Allocator.AnnotationPool)
Annotation->~MCAnnotation();
Allocator.AnnotationPool.clear();
@ -616,13 +616,13 @@ public:
MCPlus::getNumPrimeOperands(*CurInst))
return false;
const auto &Op = CurInst->getOperand(OpNum);
const MCOperand &Op = CurInst->getOperand(OpNum);
if (!Op.isReg())
return true;
MCPhysReg Reg = Op.getReg();
while (next()) {
const auto &InstrDesc = MIA.Info->get(CurInst->getOpcode());
const MCInstrDesc &InstrDesc = MIA.Info->get(CurInst->getOpcode());
if (InstrDesc.hasDefOfPhysReg(*CurInst, Reg, MRI)) {
InstrWindow = InstrWindow.slice(0, CurInst - InstrWindow.begin() + 1);
return true;
@ -679,7 +679,7 @@ public:
if (OpNum < 0)
return false;
const auto &Op = CurInst->getOperand(OpNum);
const MCOperand &Op = CurInst->getOperand(OpNum);
if (!Op.isImm())
return false;
Imm = Op.getImm();
@ -718,7 +718,7 @@ public:
if (OpNum < 0 || static_cast<unsigned int>(OpNum) >=
MCPlus::getNumPrimeOperands(*I))
return false;
const auto &Op = I->getOperand(OpNum);
const MCOperand &Op = I->getOperand(OpNum);
if (!Op.isReg())
return false;
Reg = Op.getReg();
@ -1552,7 +1552,7 @@ public:
if (AI != AnnotationNameIndexMap.end())
return AI->second;
const auto Index =
const unsigned Index =
AnnotationNameIndexMap.size() + MCPlus::MCAnnotation::kGeneric;
AnnotationNameIndexMap.insert(std::make_pair(Name, Index));
AnnotationNames.emplace_back(std::string(Name));
@ -1566,7 +1566,7 @@ public:
const ValueType &Val,
AllocatorIdTy AllocatorId = 0) {
assert(!hasAnnotation(Inst, Index));
auto &Allocator = getAnnotationAllocator(AllocatorId);
AnnotationAllocator &Allocator = getAnnotationAllocator(AllocatorId);
auto *A = new (Allocator.ValueAllocator)
MCPlus::MCSimpleAnnotation<ValueType>(Val);
@ -1609,7 +1609,7 @@ public:
template <typename ValueType>
ValueType &getOrCreateAnnotationAs(MCInst &Inst, StringRef Name,
AllocatorIdTy AllocatorId = 0) {
const auto Index = getOrCreateAnnotationIndex(Name);
const unsigned Index = getOrCreateAnnotationIndex(Name);
return getOrCreateAnnotationAs<ValueType>(Inst, Index, AllocatorId);
}
@ -1617,7 +1617,7 @@ public:
/// Use hasAnnotation() if the annotation may not exist.
template <typename ValueType>
ValueType &getAnnotationAs(const MCInst &Inst, unsigned Index) const {
auto Value = getAnnotationOpValue(Inst, Index);
Optional<int64_t> Value = getAnnotationOpValue(Inst, Index);
assert(Value && "annotation should exist");
return reinterpret_cast<MCPlus::MCSimpleAnnotation<ValueType> *>
(*Value)->getValue();
@ -1647,7 +1647,7 @@ public:
template <typename ValueType> const ValueType &
getAnnotationWithDefault(const MCInst &Inst, StringRef Name,
const ValueType &DefaultValue = ValueType()) {
const auto Index = getOrCreateAnnotationIndex(Name);
const unsigned Index = getOrCreateAnnotationIndex(Name);
return getAnnotationWithDefault<ValueType>(Inst, Index, DefaultValue);
}

18
bolt/src/MachORewriteInstance.cpp
View File

@ -85,26 +85,26 @@ Error MachORewriteInstance::setProfile(StringRef Filename) {
void MachORewriteInstance::preprocessProfileData() {
if (!ProfileReader)
return;
if (auto E = ProfileReader->preprocessProfile(*BC.get()))
if (Error E = ProfileReader->preprocessProfile(*BC.get()))
report_error("cannot pre-process profile", std::move(E));
}
void MachORewriteInstance::processProfileDataPreCFG() {
if (!ProfileReader)
return;
if (auto E = ProfileReader->readProfilePreCFG(*BC.get()))
if (Error E = ProfileReader->readProfilePreCFG(*BC.get()))
report_error("cannot read profile pre-CFG", std::move(E));
}
void MachORewriteInstance::processProfileData() {
if (!ProfileReader)
return;
if (auto E = ProfileReader->readProfile(*BC.get()))
if (Error E = ProfileReader->readProfile(*BC.get()))
report_error("cannot read profile", std::move(E));
}
void MachORewriteInstance::readSpecialSections() {
for (const auto &Section : InputFile->sections()) {
for (const object::SectionRef &Section : InputFile->sections()) {
Expected<StringRef> SectionName = Section.getName();;
check_error(SectionName.takeError(), "cannot get section name");
// Only register sections with names.
@ -157,7 +157,7 @@ std::vector<DataInCodeRegion> readDataInCode(const MachOObjectFile &O) {
Optional<uint64_t> readStartAddress(const MachOObjectFile &O) {
Optional<uint64_t> StartOffset;
Optional<uint64_t> TextVMAddr;
for (const auto &LC : O.load_commands()) {
for (const object::MachOObjectFile::LoadCommandInfo &LC : O.load_commands()) {
switch (LC.C.cmd) {
case MachO::LC_MAIN: {
MachO::entry_point_command LCMain = O.getEntryPointCommand(LC);
@ -206,7 +206,7 @@ void MachORewriteInstance::discoverFileObjects() {
});
for (size_t Index = 0; Index < FunctionSymbols.size(); ++Index) {
const uint64_t Address = cantFail(FunctionSymbols[Index].getValue());
auto Section = BC->getSectionForAddress(Address);
ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address);
// TODO: It happens for some symbols (e.g. __mh_execute_header).
// Add proper logic to handle them correctly.
if (!Section) {
@ -253,7 +253,7 @@ void MachORewriteInstance::discoverFileObjects() {
BinaryFunction &Function = BFI.second;
Function.setMaxSize(Function.getSize());
auto FunctionData = Function.getData();
ErrorOr<ArrayRef<uint8_t>> FunctionData = Function.getData();
if (!FunctionData) {
errs() << "BOLT-ERROR: corresponding section is non-executable or "
<< "empty for function " << Function << '\n';
@ -412,7 +412,7 @@ public:
JITSymbol findSymbol(const std::string &Name) override {
LLVM_DEBUG(dbgs() << "BOLT: looking for " << Name << "\n");
if (auto *I = BC.getBinaryDataByName(Name)) {
if (BinaryData *I = BC.getBinaryDataByName(Name)) {
const uint64_t Address = I->isMoved() && !I->isJumpTable()
? I->getOutputAddress()
: I->getAddress();
@ -462,7 +462,7 @@ void MachORewriteInstance::emitAndLink() {
"error creating in-memory object");
assert(Obj && "createObjectFile cannot return nullptr");
auto Resolver = BOLTSymbolResolver(*BC);
BOLTSymbolResolver Resolver = BOLTSymbolResolver(*BC);
MCAsmLayout FinalLayout(
static_cast<MCObjectStreamer *>(Streamer.get())->getAssembler());

2
bolt/src/NameResolver.h
View File

@ -30,7 +30,7 @@ class NameResolver {
public:
/// Return unique version of the \p Name in the form "Name<Sep><Number>".
std::string uniquify(StringRef Name) {
const auto ID = ++Counters[Name];
const uint64_t ID = ++Counters[Name];
return (Name + Twine(Sep) + Twine(ID)).str();
}

16
bolt/src/ParallelUtilities.cpp
View File

@ -79,7 +79,7 @@ inline unsigned estimateTotalCost(const BinaryContext &BC,
unsigned TotalCost = 0;
for (auto &BFI : BC.getBinaryFunctions()) {
auto &BF = BFI.second;
const BinaryFunction &BF = BFI.second;
TotalCost += computeCostFor(BF, SkipPredicate, SchedPolicy);
}
@ -118,7 +118,7 @@ void runOnEachFunction(BinaryContext &BC, SchedulingPolicy SchedPolicy,
LLVM_DEBUG(T.startTimer());
for (auto It = BlockBegin; It != BlockEnd; ++It) {
auto &BF = It->second;
BinaryFunction &BF = It->second;
if (SkipPredicate && SkipPredicate(BF))
continue;
@ -145,7 +145,7 @@ void runOnEachFunction(BinaryContext &BC, SchedulingPolicy SchedPolicy,
for (auto It = BC.getBinaryFunctions().begin();
It != BC.getBinaryFunctions().end(); ++It) {
auto &BF = It->second;
BinaryFunction &BF = It->second;
CurrentCost += computeCostFor(BF, SkipPredicate, SchedPolicy);
if (CurrentCost >= BlockCost) {
@ -173,7 +173,7 @@ void runOnEachFunctionWithUniqueAllocId(
LLVM_DEBUG(T.startTimer());
std::shared_lock<std::shared_timed_mutex> Lock(MainLock);
for (auto It = BlockBegin; It != BlockEnd; ++It) {
auto &BF = It->second;
BinaryFunction &BF = It->second;
if (SkipPredicate && SkipPredicate(BF))
continue;
@ -202,12 +202,13 @@ void runOnEachFunctionWithUniqueAllocId(
unsigned AllocId = 1;
for (auto It = BC.getBinaryFunctions().begin();
It != BC.getBinaryFunctions().end(); ++It) {
auto &BF = It->second;
BinaryFunction &BF = It->second;
CurrentCost += computeCostFor(BF, SkipPredicate, SchedPolicy);
if (CurrentCost >= BlockCost) {
if (!BC.MIB->checkAllocatorExists(AllocId)) {
auto Id = BC.MIB->initializeNewAnnotationAllocator();
MCPlusBuilder::AllocatorIdTy Id =
BC.MIB->initializeNewAnnotationAllocator();
assert(AllocId == Id && "unexpected allocator id created");
}
Pool.async(runBlock, BlockBegin, std::next(It), AllocId);
@ -218,7 +219,8 @@ void runOnEachFunctionWithUniqueAllocId(
}
if (!BC.MIB->checkAllocatorExists(AllocId)) {
auto Id = BC.MIB->initializeNewAnnotationAllocator();
MCPlusBuilder::AllocatorIdTy Id =
BC.MIB->initializeNewAnnotationAllocator();
assert(AllocId == Id && "unexpected allocator id created");
}

20
bolt/src/Passes/Aligner.cpp
View File

@ -89,11 +89,11 @@ void alignMaxBytes(BinaryFunction &Function) {
// -- the size of the function
// -- the specified number of bytes
void alignCompact(BinaryFunction &Function, const MCCodeEmitter *Emitter) {
const auto &BC = Function.getBinaryContext();
const BinaryContext &BC = Function.getBinaryContext();
size_t HotSize = 0;
size_t ColdSize = 0;
for (const auto *BB : Function.layout()) {
for (const BinaryBasicBlock *BB : Function.layout()) {
if (BB->isCold())
ColdSize += BC.computeCodeSize(BB->begin(), BB->end(), Emitter);
else
@ -119,13 +119,13 @@ void AlignerPass::alignBlocks(BinaryFunction &Function,
if (!Function.hasValidProfile() || !Function.isSimple())
return;
const auto &BC = Function.getBinaryContext();
const BinaryContext &BC = Function.getBinaryContext();
const auto FuncCount =
const uint64_t FuncCount =
std::max<uint64_t>(1, Function.getKnownExecutionCount());
BinaryBasicBlock *PrevBB{nullptr};
for (auto *BB : Function.layout()) {
auto Count = BB->getKnownExecutionCount();
for (BinaryBasicBlock *BB : Function.layout()) {
uint64_t Count = BB->getKnownExecutionCount();
if (Count <= FuncCount * opts::AlignBlocksThreshold / 100) {
PrevBB = BB;
@ -141,8 +141,9 @@ void AlignerPass::alignBlocks(BinaryFunction &Function,
if (Count < FTCount * 2)
continue;
const auto BlockSize = BC.computeCodeSize(BB->begin(), BB->end(), Emitter);
const auto BytesToUse =
const uint64_t BlockSize =
BC.computeCodeSize(BB->begin(), BB->end(), Emitter);
const uint64_t BytesToUse =
std::min<uint64_t>(opts::BlockAlignment - 1, BlockSize);
if (opts::AlignBlocksMinSize && BlockSize < opts::AlignBlocksMinSize)
@ -168,7 +169,8 @@ void AlignerPass::runOnFunctions(BinaryContext &BC) {
ParallelUtilities::WorkFuncTy WorkFun = [&](BinaryFunction &BF) {
// Create a separate MCCodeEmitter to allow lock free execution
auto Emitter = BC.createIndependentMCCodeEmitter();
BinaryContext::IndependentCodeEmitter Emitter =
BC.createIndependentMCCodeEmitter();
if (opts::UseCompactAligner)
alignCompact(BF, Emitter.MCE.get());

10
bolt/src/Passes/AllocCombiner.cpp
View File

@ -36,14 +36,14 @@ bool isIndifferentToSP(const MCInst &Inst, const BinaryContext &BC) {
if (BC.MIB->isCFI(Inst))
return true;
const auto II = BC.MII->get(Inst.getOpcode());
const MCInstrDesc II = BC.MII->get(Inst.getOpcode());
if (BC.MIB->isTerminator(Inst) ||
II.hasImplicitDefOfPhysReg(BC.MIB->getStackPointer(), BC.MRI.get()) ||
II.hasImplicitUseOfPhysReg(BC.MIB->getStackPointer()))
return false;
for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) {
const auto &Operand = Inst.getOperand(I);
const MCOperand &Operand = Inst.getOperand(I);
if (Operand.isReg() && Operand.getReg() == BC.MIB->getStackPointer()) {
return false;
}
@ -58,7 +58,7 @@ bool shouldProcess(const BinaryFunction &Function) {
void runForAllWeCare(std::map<uint64_t, BinaryFunction> &BFs,
std::function<void(BinaryFunction &)> Task) {
for (auto &It : BFs) {
auto &Function = It.second;
BinaryFunction &Function = It.second;
if (shouldProcess(Function))
Task(Function);
}
@ -68,10 +68,10 @@ void runForAllWeCare(std::map<uint64_t, BinaryFunction> &BFs,
void AllocCombinerPass::combineAdjustments(BinaryContext &BC,
BinaryFunction &BF) {
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
MCInst *Prev = nullptr;
for (auto I = BB.rbegin(), E = BB.rend(); I != E; ++I) {
auto &Inst = *I;
MCInst &Inst = *I;
if (isIndifferentToSP(Inst, BC))
continue; // Skip updating Prev

53
bolt/src/Passes/BinaryFunctionCallGraph.cpp
View File

@ -28,7 +28,7 @@ namespace bolt {
CallGraph::NodeId BinaryFunctionCallGraph::addNode(BinaryFunction *BF,
uint32_t Size,
uint64_t Samples) {
auto Id = CallGraph::addNode(Size, Samples);
NodeId Id = CallGraph::addNode(Size, Samples);
assert(size_t(Id) == Funcs.size());
Funcs.push_back(BF);
FuncToNodeId[BF] = Id;
@ -44,14 +44,14 @@ std::deque<BinaryFunction *> BinaryFunctionCallGraph::buildTraversalOrder() {
std::vector<NodeStatus> NodeStatus(Funcs.size());
std::stack<NodeId> Worklist;
for (auto *Func : Funcs) {
const auto Id = FuncToNodeId.at(Func);
for (BinaryFunction *Func : Funcs) {
const NodeId Id = FuncToNodeId.at(Func);
Worklist.push(Id);
NodeStatus[Id] = NEW;
}
while (!Worklist.empty()) {
const auto FuncId = Worklist.top();
const NodeId FuncId = Worklist.top();
Worklist.pop();
if (NodeStatus[FuncId] == VISITED)
@ -66,7 +66,7 @@ std::deque<BinaryFunction *> BinaryFunctionCallGraph::buildTraversalOrder() {
assert(NodeStatus[FuncId] == NEW);
NodeStatus[FuncId] = VISITING;
Worklist.push(FuncId);
for (const auto Callee : successors(FuncId)) {
for (const NodeId Callee : successors(FuncId)) {
if (NodeStatus[Callee] == VISITING || NodeStatus[Callee] == VISITED)
continue;
Worklist.push(Callee);
@ -87,7 +87,8 @@ BinaryFunctionCallGraph buildCallGraph(BinaryContext &BC,
NamedRegionTimer T1("buildcg", "Callgraph construction", "CG breakdown",
"CG breakdown", opts::TimeOpts);
BinaryFunctionCallGraph Cg;
static constexpr auto COUNT_NO_PROFILE = BinaryBasicBlock::COUNT_NO_PROFILE;
static constexpr uint64_t COUNT_NO_PROFILE =
BinaryBasicBlock::COUNT_NO_PROFILE;
// Compute function size
auto functionSize = [&](const BinaryFunction *Function) {
@ -98,13 +99,13 @@ BinaryFunctionCallGraph buildCallGraph(BinaryContext &BC,
// Add call graph nodes.
auto lookupNode = [&](BinaryFunction *Function) {
const auto Id = Cg.maybeGetNodeId(Function);
const CallGraph::NodeId Id = Cg.maybeGetNodeId(Function);
if (Id == CallGraph::InvalidId) {
// It's ok to use the hot size here when the function is split. This is
// because emitFunctions will emit the hot part first in the order that is
// computed by ReorderFunctions. The cold part will be emitted with the
// rest of the cold functions and code.
const auto Size = functionSize(Function);
const size_t Size = functionSize(Function);
// NOTE: for functions without a profile, we set the number of samples
// to zero. This will keep these functions from appearing in the hot
// section. This is a little weird because we wouldn't be trying to
@ -126,19 +127,19 @@ BinaryFunctionCallGraph buildCallGraph(BinaryContext &BC,
uint64_t NumFallbacks = 0;
uint64_t RecursiveCallsites = 0;
for (auto &It : BC.getBinaryFunctions()) {
auto *Function = &It.second;
BinaryFunction *Function = &It.second;
if (Filter(*Function)) {
continue;
}
const auto SrcId = lookupNode(Function);
const CallGraph::NodeId SrcId = lookupNode(Function);
// Offset of the current basic block from the beginning of the function
uint64_t Offset = 0;
auto recordCall = [&](const MCSymbol *DestSymbol, const uint64_t Count) {
if (auto *DstFunc =
DestSymbol ? BC.getFunctionForSymbol(DestSymbol) : nullptr) {
if (BinaryFunction *DstFunc =
DestSymbol ? BC.getFunctionForSymbol(DestSymbol) : nullptr) {
if (DstFunc == Function) {
LLVM_DEBUG(dbgs() << "BOLT-INFO: recursive call detected in "
<< *DstFunc << "\n");
@ -149,9 +150,9 @@ BinaryFunctionCallGraph buildCallGraph(BinaryContext &BC,
if (Filter(*DstFunc)) {
return false;
}
const auto DstId = lookupNode(DstFunc);
const CallGraph::NodeId DstId = lookupNode(DstFunc);
const bool IsValidCount = Count != COUNT_NO_PROFILE;
const auto AdjCount = UseEdgeCounts && IsValidCount ? Count : 1;
const uint64_t AdjCount = UseEdgeCounts && IsValidCount ? Count : 1;
if (!IsValidCount)
++NoProfileCallsites;
Cg.incArcWeight(SrcId, DstId, AdjCount, Offset);
@ -166,23 +167,27 @@ BinaryFunctionCallGraph buildCallGraph(BinaryContext &BC,
return false;
};
// Pairs of (symbol, count) for each target at this callsite.
using TargetDesc = std::pair<const MCSymbol *, uint64_t>;
using CallInfoTy = std::vector<TargetDesc>;
// Get pairs of (symbol, count) for each target at this callsite.
// If the call is to an unknown function the symbol will be nullptr.
// If there is no profiling data the count will be COUNT_NO_PROFILE.
auto getCallInfo = [&](const BinaryBasicBlock *BB, const MCInst &Inst) {
std::vector<std::pair<const MCSymbol *, uint64_t>> Counts;
const auto *DstSym = BC.MIB->getTargetSymbol(Inst);
CallInfoTy Counts;
const MCSymbol *DstSym = BC.MIB->getTargetSymbol(Inst);
// If this is an indirect call use perf data directly.
if (!DstSym && BC.MIB->hasAnnotation(Inst, "CallProfile")) {
const auto &ICSP =
BC.MIB->getAnnotationAs<IndirectCallSiteProfile>(Inst, "CallProfile");
for (const auto &CSI : ICSP) {
for (const IndirectCallProfile &CSI : ICSP) {
if (CSI.Symbol)
Counts.push_back(std::make_pair(CSI.Symbol, CSI.Count));
}
} else {
const auto Count = BB->getExecutionCount();
const uint64_t Count = BB->getExecutionCount();
Counts.push_back(std::make_pair(DstSym, Count));
}
@ -198,8 +203,8 @@ BinaryFunctionCallGraph buildCallGraph(BinaryContext &BC,
dbgs() << "BOLT-DEBUG: buildCallGraph: Falling back to perf data"
<< " for " << *Function << "\n");
++NumFallbacks;
const auto Size = functionSize(Function);
for (const auto &CSI : Function->getAllCallSites()) {
const size_t Size = functionSize(Function);
for (const IndirectCallProfile &CSI : Function->getAllCallSites()) {
++TotalCallsites;
if (!CSI.Symbol)
@ -216,7 +221,7 @@ BinaryFunctionCallGraph buildCallGraph(BinaryContext &BC,
}
}
} else {
for (auto *BB : Function->layout()) {
for (BinaryBasicBlock *BB : Function->layout()) {
// Don't count calls from cold blocks unless requested.
if (BB->isCold() && !IncludeColdCalls)
continue;
@ -233,13 +238,13 @@ BinaryFunctionCallGraph buildCallGraph(BinaryContext &BC,
BBIncludedInFunctionSize = true;
}
for (auto &Inst : *BB) {
for (MCInst &Inst : *BB) {
// Find call instructions and extract target symbols from each one.
if (BC.MIB->isCall(Inst)) {
const auto CallInfo = getCallInfo(BB, Inst);
const CallInfoTy CallInfo = getCallInfo(BB, Inst);
if (!CallInfo.empty()) {
for (const auto &CI : CallInfo) {
for (const TargetDesc &CI : CallInfo) {
++TotalCallsites;
if (!recordCall(CI.first, CI.second))
++NotProcessed;

194
bolt/src/Passes/BinaryPasses.cpp
View File

@ -277,7 +277,7 @@ void EliminateUnreachableBlocks::runOnFunction(BinaryFunction& Function) {
uint64_t Bytes;
Function.markUnreachableBlocks();
LLVM_DEBUG({
for (auto *BB : Function.layout()) {
for (BinaryBasicBlock *BB : Function.layout()) {
if (!BB->isValid()) {
dbgs() << "BOLT-INFO: UCE found unreachable block " << BB->getName()
<< " in function " << Function << "\n";
@ -301,7 +301,7 @@ void EliminateUnreachableBlocks::runOnFunction(BinaryFunction& Function) {
void EliminateUnreachableBlocks::runOnFunctions(BinaryContext &BC) {
for (auto &It : BC.getBinaryFunctions()) {
auto &Function = It.second;
BinaryFunction &Function = It.second;
if (shouldOptimize(Function)) {
runOnFunction(Function);
}
@ -371,7 +371,7 @@ void ReorderBasicBlocks::runOnFunctions(BinaryContext &BC) {
for (std::map<uint64_t, BinaryFunction &>::reverse_iterator
Rit = ScoreMap.rbegin();
Rit != ScoreMap.rend() && I < opts::PrintFuncStat; ++Rit, ++I) {
auto &Function = Rit->second;
BinaryFunction &Function = Rit->second;
OS << " Information for function of top: " << (I + 1) << ": \n";
OS << " Function Score is: " << Function.getFunctionScore()
@ -446,7 +446,7 @@ void ReorderBasicBlocks::modifyFunctionLayout(BinaryFunction &BF,
void FixupBranches::runOnFunctions(BinaryContext &BC) {
for (auto &It : BC.getBinaryFunctions()) {
auto &Function = It.second;
BinaryFunction &Function = It.second;
if (!BC.shouldEmit(Function) || !Function.isSimple())
continue;
@ -516,13 +516,13 @@ void LowerAnnotations::runOnFunctions(BinaryContext &BC) {
std::vector<std::pair<MCInst *, uint32_t>> PreservedOffsetAnnotations;
for (auto &It : BC.getBinaryFunctions()) {
auto &BF = It.second;
BinaryFunction &BF = It.second;
int64_t CurrentGnuArgsSize = 0;
// Have we crossed hot/cold border for split functions?
bool SeenCold = false;
for (auto *BB : BF.layout()) {
for (BinaryBasicBlock *BB : BF.layout()) {
if (BB->isCold() && !SeenCold) {
SeenCold = true;
CurrentGnuArgsSize = 0;
@ -534,7 +534,7 @@ void LowerAnnotations::runOnFunctions(BinaryContext &BC) {
for (auto II = BB->begin(); II != BB->end(); ++II) {
if (!BC.MIB->isInvoke(*II))
continue;
const auto NewGnuArgsSize = BC.MIB->getGnuArgsSize(*II);
const int64_t NewGnuArgsSize = BC.MIB->getGnuArgsSize(*II);
assert(NewGnuArgsSize >= 0 && "expected non-negative GNU_args_size");
if (NewGnuArgsSize != CurrentGnuArgsSize) {
auto InsertII = BF.addCFIInstruction(BB, II,
@ -568,7 +568,7 @@ void LowerAnnotations::runOnFunctions(BinaryContext &BC) {
BC.MIB->freeAnnotations();
// Reinsert preserved annotations we need during code emission.
for (const auto &Item : PreservedOffsetAnnotations)
for (const std::pair<MCInst *, uint32_t> &Item : PreservedOffsetAnnotations)
BC.MIB->addAnnotation<uint32_t>(*Item.first, "Offset", Item.second);
}
@ -592,7 +592,7 @@ uint64_t fixDoubleJumps(BinaryContext &BC,
bool MarkInvalid) {
uint64_t NumDoubleJumps = 0;
for (auto &BB : Function) {
for (BinaryBasicBlock &BB : Function) {
auto checkAndPatch = [&](BinaryBasicBlock *Pred,
BinaryBasicBlock *Succ,
const MCSymbol *SuccSym) {
@ -605,7 +605,7 @@ uint64_t fixDoubleJumps(BinaryContext &BC,
const MCSymbol *FBB = nullptr;
MCInst *CondBranch = nullptr;
MCInst *UncondBranch = nullptr;
auto Res = Pred->analyzeBranch(TBB, FBB, CondBranch, UncondBranch);
bool Res = Pred->analyzeBranch(TBB, FBB, CondBranch, UncondBranch);
if(!Res) {
LLVM_DEBUG(dbgs() << "analyzeBranch failed in peepholes in block:\n";
Pred->dump());
@ -615,7 +615,7 @@ uint64_t fixDoubleJumps(BinaryContext &BC,
// We must patch up any existing branch instructions to match up
// with the new successor.
auto *Ctx = BC.Ctx.get();
MCContext *Ctx = BC.Ctx.get();
assert((CondBranch || (!CondBranch && Pred->succ_size() == 1)) &&
"Predecessor block has inconsistent number of successors");
if (CondBranch &&
@ -632,7 +632,7 @@ uint64_t fixDoubleJumps(BinaryContext &BC,
} else {
// Succ will be null in the tail call case. In this case we
// need to explicitly add a tail call instruction.
auto *Branch = Pred->getLastNonPseudoInstr();
MCInst *Branch = Pred->getLastNonPseudoInstr();
if (Branch && BC.MIB->isUnconditionalBranch(*Branch)) {
assert(BC.MIB->getTargetSymbol(*Branch) == BB.getLabel());
Pred->removeSuccessor(&BB);
@ -655,7 +655,7 @@ uint64_t fixDoubleJumps(BinaryContext &BC,
if (BB.getNumNonPseudos() != 1 || BB.isLandingPad())
continue;
auto *Inst = BB.getFirstNonPseudoInstr();
MCInst *Inst = BB.getFirstNonPseudoInstr();
const bool IsTailCall = BC.MIB->isTailCall(*Inst);
if (!BC.MIB->isUnconditionalBranch(*Inst) && !IsTailCall)
@ -665,15 +665,15 @@ uint64_t fixDoubleJumps(BinaryContext &BC,
if (IsTailCall && BC.MIB->isConditionalBranch(*Inst))
continue;
const auto *SuccSym = BC.MIB->getTargetSymbol(*Inst);
auto *Succ = BB.getSuccessor();
const MCSymbol *SuccSym = BC.MIB->getTargetSymbol(*Inst);
BinaryBasicBlock *Succ = BB.getSuccessor();
if (((!Succ || &BB == Succ) && !IsTailCall) || (IsTailCall && !SuccSym))
continue;
std::vector<BinaryBasicBlock *> Preds{BB.pred_begin(), BB.pred_end()};
for (auto *Pred : Preds) {
for (BinaryBasicBlock *Pred : Preds) {
if (Pred->isLandingPad())
continue;
@ -716,7 +716,8 @@ bool SimplifyConditionalTailCalls::shouldRewriteBranch(
if (opts::SctcMode == opts::SctcPreserveDirection)
return IsForward == DirectionFlag;
const auto Frequency = PredBB->getBranchStats(BB);
const ErrorOr<std::pair<double, double>> Frequency =
PredBB->getBranchStats(BB);
// It's ok to rewrite the conditional branch if the new target will be
// a backward branch.
@ -755,16 +756,16 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryContext &BC,
BB->isEntryPoint());
};
for (auto *BB : BF.layout()) {
for (BinaryBasicBlock *BB : BF.layout()) {
// Locate BB with a single direct tail-call instruction.
if (BB->getNumNonPseudos() != 1)
continue;
auto *Instr = BB->getFirstNonPseudoInstr();
MCInst *Instr = BB->getFirstNonPseudoInstr();
if (!MIB->isTailCall(*Instr) || BC.MIB->isConditionalBranch(*Instr))
continue;
auto *CalleeSymbol = MIB->getTargetSymbol(*Instr);
const MCSymbol *CalleeSymbol = MIB->getTargetSymbol(*Instr);
if (!CalleeSymbol)
continue;
@ -772,8 +773,8 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryContext &BC,
const bool IsForwardCTC = BF.isForwardCall(CalleeSymbol);
// Iterate through all predecessors.
for (auto *PredBB : BB->predecessors()) {
auto *CondSucc = PredBB->getConditionalSuccessor(true);
for (BinaryBasicBlock *PredBB : BB->predecessors()) {
BinaryBasicBlock *CondSucc = PredBB->getConditionalSuccessor(true);
if (!CondSucc)
continue;
@ -783,7 +784,7 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryContext &BC,
const MCSymbol *FBB = nullptr;
MCInst *CondBranch = nullptr;
MCInst *UncondBranch = nullptr;
auto Result = PredBB->analyzeBranch(TBB, FBB, CondBranch, UncondBranch);
bool Result = PredBB->analyzeBranch(TBB, FBB, CondBranch, UncondBranch);
// analyzeBranch() can fail due to unusual branch instructions, e.g. jrcxz
if (!Result) {
@ -862,8 +863,8 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryContext &BC,
// Add unconditional branches at the end of BBs to new successors
// as long as the successor is not a fallthrough.
for (auto &Entry : NeedsUncondBranch) {
auto *PredBB = Entry.first;
auto *CondSucc = Entry.second;
BinaryBasicBlock *PredBB = Entry.first;
const BinaryBasicBlock *CondSucc = Entry.second;
const MCSymbol *TBB = nullptr;
const MCSymbol *FBB = nullptr;
@ -873,13 +874,13 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryContext &BC,
// Find the next valid block. Invalid blocks will be deleted
// so they shouldn't be considered fallthrough targets.
const auto *NextBlock = BF.getBasicBlockAfter(PredBB, false);
const BinaryBasicBlock *NextBlock = BF.getBasicBlockAfter(PredBB, false);
while (NextBlock && !isValid(NextBlock)) {
NextBlock = BF.getBasicBlockAfter(NextBlock, false);
}
// Get the unconditional successor to this block.
const auto *PredSucc = PredBB->getSuccessor();
const BinaryBasicBlock *PredSucc = PredBB->getSuccessor();
assert(PredSucc && "The other branch should be a tail call");
const bool HasFallthrough = (NextBlock && PredSucc == NextBlock);
@ -901,7 +902,7 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryContext &BC,
if (NumLocalCTCs > 0) {
NumDoubleJumps += fixDoubleJumps(BC, BF, true);
// Clean-up unreachable tail-call blocks.
const auto Stats = BF.eraseInvalidBBs();
const std::pair<unsigned, uint64_t> Stats = BF.eraseInvalidBBs();
DeletedBlocks += Stats.first;
DeletedBytes += Stats.second;
@ -928,7 +929,7 @@ void SimplifyConditionalTailCalls::runOnFunctions(BinaryContext &BC) {
return;
for (auto &It : BC.getBinaryFunctions()) {
auto &Function = It.second;
BinaryFunction &Function = It.second;
if (!shouldOptimize(Function))
continue;
@ -955,8 +956,8 @@ uint64_t Peepholes::shortenInstructions(BinaryContext &BC,
BinaryFunction &Function) {
MCInst DebugInst;
uint64_t Count = 0;
for (auto &BB : Function) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : Function) {
for (MCInst &Inst : BB) {
if (opts::Verbosity > 1) {
DebugInst = Inst;
}
@ -977,8 +978,8 @@ uint64_t Peepholes::shortenInstructions(BinaryContext &BC,
void Peepholes::addTailcallTraps(BinaryContext &BC,
BinaryFunction &Function) {
for (auto &BB : Function) {
auto *Inst = BB.getLastNonPseudoInstr();
for (BinaryBasicBlock &BB : Function) {
MCInst *Inst = BB.getLastNonPseudoInstr();
if (Inst && BC.MIB->isTailCall(*Inst) && BC.MIB->isIndirectBranch(*Inst)) {
MCInst Trap;
if (BC.MIB->createTrap(Trap)) {
@ -991,12 +992,12 @@ void Peepholes::addTailcallTraps(BinaryContext &BC,
void Peepholes::removeUselessCondBranches(BinaryContext &BC,
BinaryFunction &Function) {
for (auto &BB : Function) {
for (BinaryBasicBlock &BB : Function) {
if (BB.succ_size() != 2)
continue;
auto *CondBB = BB.getConditionalSuccessor(true);
auto *UncondBB = BB.getConditionalSuccessor(false);
BinaryBasicBlock *CondBB = BB.getConditionalSuccessor(true);
BinaryBasicBlock *UncondBB = BB.getConditionalSuccessor(false);
if (CondBB != UncondBB)
continue;
@ -1004,7 +1005,7 @@ void Peepholes::removeUselessCondBranches(BinaryContext &BC,
const MCSymbol *FBB = nullptr;
MCInst *CondBranch = nullptr;
MCInst *UncondBranch = nullptr;
auto Result = BB.analyzeBranch(TBB, FBB, CondBranch, UncondBranch);
bool Result = BB.analyzeBranch(TBB, FBB, CondBranch, UncondBranch);
// analyzeBranch() can fail due to unusual branch instructions,
// e.g. jrcxz, or jump tables (indirect jump).
@ -1028,7 +1029,7 @@ void Peepholes::runOnFunctions(BinaryContext &BC) {
return;
for (auto &It : BC.getBinaryFunctions()) {
auto &Function = It.second;
BinaryFunction &Function = It.second;
if (shouldOptimize(Function)) {
if (Opts & opts::PEEP_SHORTEN)
NumShortened += shortenInstructions(BC, Function);
@ -1060,8 +1061,8 @@ bool SimplifyRODataLoads::simplifyRODataLoads(
uint64_t NumLocalLoadsFound = 0;
uint64_t NumDynamicLocalLoadsFound = 0;
for (auto *BB : BF.layout()) {
for (auto &Inst : *BB) {
for (BinaryBasicBlock *BB : BF.layout()) {
for (MCInst &Inst : *BB) {
unsigned Opcode = Inst.getOpcode();
const MCInstrDesc &Desc = BC.MII->get(Opcode);
@ -1074,7 +1075,7 @@ bool SimplifyRODataLoads::simplifyRODataLoads(
if (MIB->hasPCRelOperand(Inst)) {
// Try to find the symbol that corresponds to the PC-relative operand.
auto DispOpI = MIB->getMemOperandDisp(Inst);
MCOperand *DispOpI = MIB->getMemOperandDisp(Inst);
assert(DispOpI != Inst.end() && "expected PC-relative displacement");
assert(DispOpI->isExpr() &&
"found PC-relative with non-symbolic displacement");
@ -1091,7 +1092,7 @@ bool SimplifyRODataLoads::simplifyRODataLoads(
// Look up the symbol address in the global symbols map of the binary
// context object.
auto *BD = BC.getBinaryDataByName(DisplSymbol->getName());
BinaryData *BD = BC.getBinaryDataByName(DisplSymbol->getName());
if (!BD)
continue;
TargetAddress = BD->getAddress() + DisplOffset;
@ -1101,7 +1102,8 @@ bool SimplifyRODataLoads::simplifyRODataLoads(
// Get the contents of the section containing the target address of the
// memory operand. We are only interested in read-only sections.
auto DataSection = BC.getSectionForAddress(TargetAddress);
ErrorOr<BinarySection &> DataSection =
BC.getSectionForAddress(TargetAddress);
if (!DataSection || !DataSection->isReadOnly())
continue;
@ -1133,7 +1135,7 @@ bool SimplifyRODataLoads::simplifyRODataLoads(
void SimplifyRODataLoads::runOnFunctions(BinaryContext &BC) {
for (auto &It : BC.getBinaryFunctions()) {
auto &Function = It.second;
BinaryFunction &Function = It.second;
if (shouldOptimize(Function) && simplifyRODataLoads(BC, Function)) {
Modified.insert(&Function);
}
@ -1147,7 +1149,7 @@ void SimplifyRODataLoads::runOnFunctions(BinaryContext &BC) {
}
void AssignSections::runOnFunctions(BinaryContext &BC) {
for (auto *Function : BC.getInjectedBinaryFunctions()) {
for (BinaryFunction *Function : BC.getInjectedBinaryFunctions()) {
Function->setCodeSectionName(BC.getInjectedCodeSectionName());
Function->setColdCodeSectionName(BC.getInjectedColdCodeSectionName());
}
@ -1156,11 +1158,11 @@ void AssignSections::runOnFunctions(BinaryContext &BC) {
if (!BC.HasRelocations)
return;
const auto UseColdSection =
const bool UseColdSection =
BC.NumProfiledFuncs > 0 ||
opts::ReorderFunctions == ReorderFunctions::RT_USER;
for (auto &BFI : BC.getBinaryFunctions()) {
auto &Function = BFI.second;
BinaryFunction &Function = BFI.second;
if (opts::isHotTextMover(Function)) {
Function.setCodeSectionName(BC.getHotTextMoverSectionName());
Function.setColdCodeSectionName(BC.getHotTextMoverSectionName());
@ -1200,11 +1202,11 @@ void PrintProfileStats::runOnFunctions(BinaryContext &BC) {
continue;
FlowMapTy &IncomingMap = TotalIncomingMaps[&Function];
FlowMapTy &OutgoingMap = TotalOutgoingMaps[&Function];
for (const auto &BB : Function) {
auto TotalOutgoing = 0ULL;
for (const BinaryBasicBlock &BB : Function) {
uint64_t TotalOutgoing = 0ULL;
auto SuccBIIter = BB.branch_info_begin();
for (auto Succ : BB.successors()) {
auto Count = SuccBIIter->Count;
for (BinaryBasicBlock *Succ : BB.successors()) {
uint64_t Count = SuccBIIter->Count;
if (Count == BinaryBasicBlock::COUNT_NO_PROFILE || Count == 0) {
++SuccBIIter;
continue;
@ -1218,7 +1220,7 @@ void PrintProfileStats::runOnFunctions(BinaryContext &BC) {
size_t NumBlocks = 0;
double Mean = 0.0;
for (const auto &BB : Function) {
for (const BinaryBasicBlock &BB : Function) {
// Do not compute score for low frequency blocks, entry or exit blocks
if (IncomingMap[&BB] < 100 || OutgoingMap[&BB] == 0 || BB.isEntryPoint())
continue;
@ -1249,7 +1251,7 @@ void PrintProfileStats::runOnFunctions(BinaryContext &BC) {
continue;
FlowMapTy &IncomingMap = TotalIncomingMaps[&Function];
FlowMapTy &OutgoingMap = TotalOutgoingMaps[&Function];
for (const auto &BB : Function) {
for (const BinaryBasicBlock &BB : Function) {
if (IncomingMap[&BB] < 100 || OutgoingMap[&BB] == 0)
continue;
++NumBlocksConsidered;
@ -1284,7 +1286,7 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) {
std::vector<BinaryFunction *> ProfiledFunctions;
const char *StaleFuncsHeader = "BOLT-INFO: Functions with stale profile:\n";
for (auto &BFI : BC.getBinaryFunctions()) {
auto &Function = BFI.second;
BinaryFunction &Function = BFI.second;
// Ignore PLT functions for stats.
if (Function.isPLTFunction())
@ -1311,7 +1313,7 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) {
if (!Function.hasProfile())
continue;
auto SampleCount = Function.getRawBranchCount();
uint64_t SampleCount = Function.getRawBranchCount();
TotalSampleCount += SampleCount;
if (Function.hasValidProfile()) {
@ -1328,8 +1330,8 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) {
}
BC.NumProfiledFuncs = ProfiledFunctions.size();
const auto NumAllProfiledFunctions =
ProfiledFunctions.size() + NumStaleProfileFunctions;
const size_t NumAllProfiledFunctions =
ProfiledFunctions.size() + NumStaleProfileFunctions;
outs() << "BOLT-INFO: " << NumAllProfiledFunctions
<< " out of " << NumRegularFunctions << " functions in the binary ("
<< format("%.1f", NumAllProfiledFunctions /
@ -1371,7 +1373,7 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) {
}
}
if (const auto NumUnusedObjects = BC.getNumUnusedProfiledObjects()) {
if (const uint64_t NumUnusedObjects = BC.getNumUnusedProfiledObjects()) {
outs() << "BOLT-INFO: profile for " << NumUnusedObjects
<< " objects was ignored\n";
}
@ -1386,7 +1388,8 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) {
);
auto SFI = ProfiledFunctions.begin();
auto SFIend = ProfiledFunctions.end();
for (auto I = 0u; I < opts::TopCalledLimit && SFI != SFIend; ++SFI, ++I) {
for (unsigned I = 0u; I < opts::TopCalledLimit && SFI != SFIend;
++SFI, ++I) {
outs() << " " << **SFI << " : "
<< (*SFI)->getExecutionCount() << '\n';
}
@ -1402,7 +1405,7 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) {
std::map<const BinaryFunction *, DynoStats> Stats;
for (const auto &BFI : BC.getBinaryFunctions()) {
const auto &BF = BFI.second;
const BinaryFunction &BF = BFI.second;
if (shouldOptimize(BF) && BF.hasValidProfile()) {
Functions.push_back(&BF);
Stats.emplace(&BF, getDynoStats(BF));
@ -1431,8 +1434,8 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) {
Functions.begin(),
Functions.end(),
[Ascending,&Stats](const BinaryFunction *A, const BinaryFunction *B) {
const auto &StatsA = Stats.at(A);
const auto &StatsB = Stats.at(B);
const DynoStats &StatsA = Stats.at(A);
const DynoStats &StatsB = Stats.at(B);
return Ascending
? StatsA.lessThan(StatsB, opts::PrintSortedBy)
: StatsB.lessThan(StatsA, opts::PrintSortedBy);
@ -1446,7 +1449,7 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) {
} else {
outs() << "(";
bool PrintComma = false;
for (const auto Category : opts::PrintSortedBy) {
for (const DynoStats::Category Category : opts::PrintSortedBy) {
if (PrintComma) outs() << ", ";
outs() << DynoStats::Description(Category);
PrintComma = true;
@ -1457,12 +1460,12 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) {
outs() << " are:\n";
auto SFI = Functions.begin();
for (unsigned I = 0; I < 100 && SFI != Functions.end(); ++SFI, ++I) {
const auto Stats = getDynoStats(**SFI);
const DynoStats Stats = getDynoStats(**SFI);
outs() << " " << **SFI;
if (!SortAll) {
outs() << " (";
bool PrintComma = false;
for (const auto Category : opts::PrintSortedBy) {
for (const DynoStats::Category Category : opts::PrintSortedBy) {
if (PrintComma) outs() << ", ";
outs() << dynoStatsOptName(Category) << "=" << Stats[Category];
PrintComma = true;
@ -1480,7 +1483,7 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) {
" traps.";
if (opts::Verbosity >= 1 || BC.TrappedFunctions.size() <= 5) {
errs() << '\n';
for (const auto *Function : BC.TrappedFunctions)
for (const BinaryFunction *Function : BC.TrappedFunctions)
errs() << " " << *Function << '\n';
} else {
errs() << " Use -v=1 to see the list.\n";
@ -1510,14 +1513,14 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) {
if (opts::ReportBadLayout) {
std::vector<const BinaryFunction *> SuboptimalFuncs;
for (auto &BFI : BC.getBinaryFunctions()) {
const auto &BF = BFI.second;
const BinaryFunction &BF = BFI.second;
if (!BF.hasValidProfile())
continue;
const auto HotThreshold =
const uint64_t HotThreshold =
std::max<uint64_t>(BF.getKnownExecutionCount(), 1);
bool HotSeen = false;
for (const auto *BB : BF.rlayout()) {
for (const BinaryBasicBlock *BB : BF.rlayout()) {
if (!HotSeen && BB->getKnownExecutionCount() > HotThreshold) {
HotSeen = true;
continue;
@ -1559,8 +1562,8 @@ PrintProgramStats::runOnFunctions(BinaryContext &BC) {
void InstructionLowering::runOnFunctions(BinaryContext &BC) {
for (auto &BFI : BC.getBinaryFunctions()) {
for (auto &BB : BFI.second) {
for (auto &Instruction : BB) {
for (BinaryBasicBlock &BB : BFI.second) {
for (MCInst &Instruction : BB) {
BC.MIB->lowerTailCall(Instruction);
}
}
@ -1571,7 +1574,7 @@ void StripRepRet::runOnFunctions(BinaryContext &BC) {
uint64_t NumPrefixesRemoved = 0;
uint64_t NumBytesSaved = 0;
for (auto &BFI : BC.getBinaryFunctions()) {
for (auto &BB : BFI.second) {
for (BinaryBasicBlock &BB : BFI.second) {
auto LastInstRIter = BB.getLastNonPseudo();
if (LastInstRIter == BB.rend() ||
!BC.MIB->isReturn(*LastInstRIter) ||
@ -1597,24 +1600,25 @@ void InlineMemcpy::runOnFunctions(BinaryContext &BC) {
uint64_t NumInlined = 0;
uint64_t NumInlinedDyno = 0;
for (auto &BFI : BC.getBinaryFunctions()) {
for (auto &BB : BFI.second) {
for (BinaryBasicBlock &BB : BFI.second) {
for (auto II = BB.begin(); II != BB.end(); ++II) {
auto &Inst = *II;
MCInst &Inst = *II;
if (!BC.MIB->isCall(Inst) || MCPlus::getNumPrimeOperands(Inst) != 1 ||
!Inst.getOperand(0).isExpr())
continue;
const auto *CalleeSymbol = BC.MIB->getTargetSymbol(Inst);
const MCSymbol *CalleeSymbol = BC.MIB->getTargetSymbol(Inst);
if (CalleeSymbol->getName() != "memcpy" &&
CalleeSymbol->getName() != "memcpy@PLT" &&
CalleeSymbol->getName() != "_memcpy8")
continue;
const auto IsMemcpy8 = (CalleeSymbol->getName() == "_memcpy8");
const auto IsTailCall = BC.MIB->isTailCall(Inst);
const bool IsMemcpy8 = (CalleeSymbol->getName() == "_memcpy8");
const bool IsTailCall = BC.MIB->isTailCall(Inst);
const auto NewCode = BC.MIB->createInlineMemcpy(IsMemcpy8);
const std::vector<MCInst> NewCode =
BC.MIB->createInlineMemcpy(IsMemcpy8);
II = BB.replaceInstruction(II, NewCode);
std::advance(II, NewCode.size() - 1);
if (IsTailCall) {
@ -1642,8 +1646,8 @@ bool SpecializeMemcpy1::shouldOptimize(const BinaryFunction &Function) const {
if (!BinaryFunctionPass::shouldOptimize(Function))
return false;
for (auto &FunctionSpec : Spec) {
auto FunctionName = StringRef(FunctionSpec).split(':').first;
for (const std::string &FunctionSpec : Spec) {
StringRef FunctionName = StringRef(FunctionSpec).split(':').first;
if (Function.hasNameRegex(FunctionName))
return true;
}
@ -1654,7 +1658,7 @@ bool SpecializeMemcpy1::shouldOptimize(const BinaryFunction &Function) const {
std::set<size_t>
SpecializeMemcpy1::getCallSitesToOptimize(const BinaryFunction &Function) const{
StringRef SitesString;
for (auto &FunctionSpec : Spec) {
for (const std::string &FunctionSpec : Spec) {
StringRef FunctionName;
std::tie(FunctionName, SitesString) = StringRef(FunctionSpec).split(':');
if (Function.hasNameRegex(FunctionName))
@ -1665,7 +1669,7 @@ SpecializeMemcpy1::getCallSitesToOptimize(const BinaryFunction &Function) const{
std::set<size_t> Sites;
SmallVector<StringRef, 4> SitesVec;
SitesString.split(SitesVec, ':');
for (auto SiteString : SitesVec) {
for (StringRef SiteString : SitesVec) {
if (SiteString.empty())
continue;
size_t Result;
@ -1683,26 +1687,26 @@ void SpecializeMemcpy1::runOnFunctions(BinaryContext &BC) {
uint64_t NumSpecialized = 0;
uint64_t NumSpecializedDyno = 0;
for (auto &BFI : BC.getBinaryFunctions()) {
auto &Function = BFI.second;
BinaryFunction &Function = BFI.second;
if (!shouldOptimize(Function))
continue;
auto CallsToOptimize = getCallSitesToOptimize(Function);
std::set<size_t> CallsToOptimize = getCallSitesToOptimize(Function);
auto shouldOptimize = [&](size_t N) {
return CallsToOptimize.empty() || CallsToOptimize.count(N);
};
std::vector<BinaryBasicBlock *> Blocks(Function.pbegin(), Function.pend());
size_t CallSiteID = 0;
for (auto *CurBB : Blocks) {
for (BinaryBasicBlock *CurBB : Blocks) {
for (auto II = CurBB->begin(); II != CurBB->end(); ++II) {
auto &Inst = *II;
MCInst &Inst = *II;
if (!BC.MIB->isCall(Inst) || MCPlus::getNumPrimeOperands(Inst) != 1 ||
!Inst.getOperand(0).isExpr())
continue;
const auto *CalleeSymbol = BC.MIB->getTargetSymbol(Inst);
const MCSymbol *CalleeSymbol = BC.MIB->getTargetSymbol(Inst);
if (CalleeSymbol->getName() != "memcpy" &&
CalleeSymbol->getName() != "memcpy@PLT")
continue;
@ -1716,9 +1720,9 @@ void SpecializeMemcpy1::runOnFunctions(BinaryContext &BC) {
continue;
// Create a copy of a call to memcpy(dest, src, size).
auto MemcpyInstr = Inst;
MCInst MemcpyInstr = Inst;
auto *OneByteMemcpyBB = CurBB->splitAt(II);
BinaryBasicBlock *OneByteMemcpyBB = CurBB->splitAt(II);
BinaryBasicBlock *NextBB{nullptr};
if (OneByteMemcpyBB->getNumNonPseudos() > 1) {
@ -1730,11 +1734,11 @@ void SpecializeMemcpy1::runOnFunctions(BinaryContext &BC) {
assert(NextBB && "unexpected call to memcpy() with no return");
}
auto *MemcpyBB = Function.addBasicBlock(CurBB->getInputOffset());
auto CmpJCC = BC.MIB->createCmpJE(BC.MIB->getIntArgRegister(2),
1,
OneByteMemcpyBB->getLabel(),
BC.Ctx.get());
BinaryBasicBlock *MemcpyBB =
Function.addBasicBlock(CurBB->getInputOffset());
std::vector<MCInst> CmpJCC =
BC.MIB->createCmpJE(BC.MIB->getIntArgRegister(2), 1,
OneByteMemcpyBB->getLabel(), BC.Ctx.get());
CurBB->addInstructions(CmpJCC);
CurBB->addSuccessor(MemcpyBB);
@ -1748,7 +1752,7 @@ void SpecializeMemcpy1::runOnFunctions(BinaryContext &BC) {
if (CurBB->getKnownExecutionCount() > 0)
MemcpyBB->setExecutionCount(1);
auto OneByteMemcpy = BC.MIB->createOneByteMemcpy();
std::vector<MCInst> OneByteMemcpy = BC.MIB->createOneByteMemcpy();
OneByteMemcpyBB->addInstructions(OneByteMemcpy);
++NumSpecialized;

4
bolt/src/Passes/BinaryPasses.h
View File

@ -78,8 +78,8 @@ public:
}
void runOnFunctions(BinaryContext &BC) override {
const auto NewDynoStats = getDynoStats(BC.getBinaryFunctions());
const auto Changed = (NewDynoStats != PrevDynoStats);
const DynoStats NewDynoStats = getDynoStats(BC.getBinaryFunctions());
const bool Changed = (NewDynoStats != PrevDynoStats);
outs() << "BOLT-INFO: program-wide dynostats "
<< Title << (Changed ? "" : " (no change)") << ":\n\n"
<< PrevDynoStats;

16
bolt/src/Passes/CallGraph.cpp
View File

@ -72,7 +72,7 @@ int64_t CallGraph::Arc::Hash::operator()(const Arc &Arc) const {
}
CallGraph::NodeId CallGraph::addNode(uint32_t Size, uint64_t Samples) {
auto Id = Nodes.size();
NodeId Id = Nodes.size();
Nodes.emplace_back(Size, Samples);
return Id;
}
@ -81,7 +81,7 @@ const CallGraph::Arc &CallGraph::incArcWeight(NodeId Src, NodeId Dst, double W,
double Offset) {
assert(Offset <= size(Src) && "Call offset exceeds function size");
auto Res = Arcs.emplace(Src, Dst, W);
std::pair<ArcIterator, bool> Res = Arcs.emplace(Src, Dst, W);
if (!Res.second) {
Res.first->Weight += W;
Res.first->AvgCallOffset += Offset * W;
@ -95,9 +95,9 @@ const CallGraph::Arc &CallGraph::incArcWeight(NodeId Src, NodeId Dst, double W,
void CallGraph::normalizeArcWeights() {
for (NodeId FuncId = 0; FuncId < numNodes(); ++FuncId) {
auto& Func = getNode(FuncId);
for (auto Caller : Func.predecessors()) {
auto Arc = findArc(Caller, FuncId);
const Node &Func = getNode(FuncId);
for (NodeId Caller : Func.predecessors()) {
ArcIterator Arc = findArc(Caller, FuncId);
Arc->NormalizedWeight = Arc->weight() / Func.samples();
if (Arc->weight() > 0)
Arc->AvgCallOffset /= Arc->weight();
@ -109,10 +109,10 @@ void CallGraph::normalizeArcWeights() {
void CallGraph::adjustArcWeights() {
for (NodeId FuncId = 0; FuncId < numNodes(); ++FuncId) {
auto& Func = getNode(FuncId);
const Node &Func = getNode(FuncId);
uint64_t InWeight = 0;
for (auto Caller : Func.predecessors()) {
auto Arc = findArc(Caller, FuncId);
for (NodeId Caller : Func.predecessors()) {
ArcIterator Arc = findArc(Caller, FuncId);
InWeight += (uint64_t)Arc->weight();
}
if (Func.samples() < InWeight)

4
bolt/src/Passes/CallGraph.h
View File

@ -192,8 +192,8 @@ void CallGraph::printDot(char* FileName, L GetLabel) const {
}
for (NodeId F = 0; F < Nodes.size(); F++) {
if (Nodes[F].samples() == 0) continue;
for (auto Dst : Nodes[F].successors()) {
auto Arc = findArc(F, Dst);
for (NodeId Dst : Nodes[F].successors()) {
ArcConstIterator Arc = findArc(F, Dst);
fprintf(
File,
"f%lu -> f%u [label=\"normWgt=%.3lf,weight=%.0lf,callOffset=%.1lf\"];"

8
bolt/src/Passes/CallGraphWalker.cpp
View File

@ -25,24 +25,24 @@ void CallGraphWalker::traverseCG() {
std::queue<BinaryFunction *> Queue;
std::set<BinaryFunction *> InQueue;
for (auto *Func : TopologicalCGOrder) {
for (BinaryFunction *Func : TopologicalCGOrder) {
Queue.push(Func);
InQueue.insert(Func);
}
while (!Queue.empty()) {
auto *Func = Queue.front();
BinaryFunction *Func = Queue.front();
Queue.pop();
InQueue.erase(Func);
bool Changed{false};
for (auto Visitor : Visitors) {
for (CallbackTy Visitor : Visitors) {
bool CurVisit = Visitor(Func);
Changed = Changed || CurVisit;
}
if (Changed) {
for (auto CallerID : CG.predecessors(CG.getNodeId(Func))) {
for (CallGraph::NodeId CallerID : CG.predecessors(CG.getNodeId(Func))) {
BinaryFunction *CallerFunc = CG.nodeIdToFunc(CallerID);
if (InQueue.count(CallerFunc))
continue;

24
bolt/src/Passes/DataflowAnalysis.cpp
View File

@ -17,19 +17,19 @@ namespace llvm {
raw_ostream &operator<<(raw_ostream &OS, const BitVector &State) {
LLVM_DEBUG({
OS << "BitVector(";
auto Sep = "";
const char *Sep = "";
if (State.count() > (State.size() >> 1)) {
OS << "all, except: ";
auto BV = State;
BitVector BV = State;
BV.flip();
for (auto I = BV.find_first(); I != -1; I = BV.find_next(I)) {
for (int I = BV.find_first(); I != -1; I = BV.find_next(I)) {
OS << Sep << I;
Sep = " ";
}
OS << ")";
return OS;
}
for (auto I = State.find_first(); I != -1; I = State.find_next(I)) {
for (int I = State.find_first(); I != -1; I = State.find_next(I)) {
OS << Sep << I;
Sep = " ";
}
@ -44,17 +44,17 @@ namespace bolt {
void doForAllPreds(const BinaryContext &BC, const BinaryBasicBlock &BB,
std::function<void(ProgramPoint)> Task) {
for (auto Pred : BB.predecessors()) {
for (BinaryBasicBlock *Pred : BB.predecessors()) {
if (Pred->isValid())
Task(ProgramPoint::getLastPointAt(*Pred));
}
if (!BB.isLandingPad())
return;
for (auto Thrower : BB.throwers()) {
for (auto &Inst : *Thrower) {
for (BinaryBasicBlock *Thrower : BB.throwers()) {
for (MCInst &Inst : *Thrower) {
if (!BC.MIB->isInvoke(Inst))
continue;
const auto EHInfo = BC.MIB->getEHInfo(Inst);
const Optional<MCPlus::MCLandingPad> EHInfo = BC.MIB->getEHInfo(Inst);
if (!EHInfo || EHInfo->first != BB.getLabel())
continue;
Task(ProgramPoint(&Inst));
@ -65,7 +65,7 @@ void doForAllPreds(const BinaryContext &BC, const BinaryBasicBlock &BB,
/// Operates on all successors of a basic block.
void doForAllSuccs(const BinaryBasicBlock &BB,
std::function<void(ProgramPoint)> Task) {
for (auto Succ : BB.successors()) {
for (BinaryBasicBlock *Succ : BB.successors()) {
if (Succ->isValid())
Task(ProgramPoint::getFirstPointAt(*Succ));
}
@ -78,14 +78,14 @@ void RegStatePrinter::print(raw_ostream &OS, const BitVector &State) const {
}
if (State.count() > (State.size() >> 1)) {
OS << "all, except: ";
auto BV = State;
BitVector BV = State;
BV.flip();
for (auto I = BV.find_first(); I != -1; I = BV.find_next(I)) {
for (int I = BV.find_first(); I != -1; I = BV.find_next(I)) {
OS << BC.MRI->getName(I) << " ";
}
return;
}
for (auto I = State.find_first(); I != -1; I = State.find_next(I)) {
for (int I = State.find_first(); I != -1; I = State.find_next(I)) {
OS << BC.MRI->getName(I) << " ";
}
}

32
bolt/src/Passes/DataflowAnalysis.h
View File

@ -322,8 +322,8 @@ public:
/// Remove any state annotations left by this analysis
void cleanAnnotations() {
for (auto &BB : Func) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : Func) {
for (MCInst &Inst : BB) {
BC.MIB->removeAnnotation(Inst, derived().getAnnotationIndex());
}
}
@ -335,11 +335,11 @@ public:
derived().preflight();
// Initialize state for all points of the function
for (auto &BB : Func) {
auto &St = getOrCreateStateAt(BB);
for (BinaryBasicBlock &BB : Func) {
StateTy &St = getOrCreateStateAt(BB);
St = derived().getStartingStateAtBB(BB);
for (auto &Inst : BB) {
auto &St = getOrCreateStateAt(Inst);
for (MCInst &Inst : BB) {
StateTy &St = getOrCreateStateAt(Inst);
St = derived().getStartingStateAtPoint(Inst);
}
}
@ -349,10 +349,10 @@ public:
// TODO: Pushing this in a DFS ordering will greatly speed up the dataflow
// performance.
if (!Backward) {
for (auto &BB : Func) {
for (BinaryBasicBlock &BB : Func) {
Worklist.push(&BB);
MCInst *Prev = nullptr;
for (auto &Inst : BB) {
for (MCInst &Inst : BB) {
PrevPoint[&Inst] = Prev ? ProgramPoint(Prev) : ProgramPoint(&BB);
Prev = &Inst;
}
@ -362,7 +362,7 @@ public:
Worklist.push(&*I);
MCInst *Prev = nullptr;
for (auto J = (*I).rbegin(), E2 = (*I).rend(); J != E2; ++J) {
auto &Inst = *J;
MCInst &Inst = *J;
PrevPoint[&Inst] = Prev ? ProgramPoint(Prev) : ProgramPoint(&*I);
Prev = &Inst;
}
@ -371,7 +371,7 @@ public:
// Main dataflow loop
while (!Worklist.empty()) {
auto *BB = Worklist.front();
BinaryBasicBlock *BB = Worklist.front();
Worklist.pop();
// Calculate state at the entry of first instruction in BB
@ -409,7 +409,7 @@ public:
StateTy CurState = derived().computeNext(Inst, *PrevState);
if (Backward && BC.MIB->isInvoke(Inst)) {
auto *LBB = Func.getLandingPadBBFor(BB, Inst);
BinaryBasicBlock *LBB = Func.getLandingPadBBFor(BB, Inst);
if (LBB) {
auto First = LBB->begin();
if (First != LBB->end()) {
@ -432,7 +432,7 @@ public:
};
if (!Backward) {
for (auto &Inst : *BB) {
for (MCInst &Inst : *BB) {
doNext(Inst, *BB);
}
} else {
@ -443,17 +443,17 @@ public:
if (Changed) {
if (!Backward) {
for (auto Succ : BB->successors()) {
for (BinaryBasicBlock *Succ : BB->successors()) {
Worklist.push(Succ);
}
for (auto LandingPad : BB->landing_pads()) {
for (BinaryBasicBlock *LandingPad : BB->landing_pads()) {
Worklist.push(LandingPad);
}
} else {
for (auto Pred : BB->predecessors()) {
for (BinaryBasicBlock *Pred : BB->predecessors()) {
Worklist.push(Pred);
}
for (auto Thrower : BB->throwers()) {
for (BinaryBasicBlock *Thrower : BB->throwers()) {
Worklist.push(Thrower);
}
}

4
bolt/src/Passes/DataflowInfoManager.cpp
View File

@ -145,8 +145,8 @@ DataflowInfoManager::getInsnToBBMap() {
if (InsnToBB)
return *InsnToBB;
InsnToBB.reset(new std::unordered_map<const MCInst *, BinaryBasicBlock *>());
for (auto &BB : BF) {
for (auto &Inst : BB)
for (BinaryBasicBlock &BB : BF) {
for (MCInst &Inst : BB)
(*InsnToBB)[&Inst] = &BB;
}
return *InsnToBB;

8
bolt/src/Passes/DominatorAnalysis.h
View File

@ -43,9 +43,9 @@ public:
SmallSetVector<ProgramPoint, 4> getDominanceFrontierFor(const MCInst &Dom) {
SmallSetVector<ProgramPoint, 4> Result;
auto DomIdx = this->ExprToIdx[&Dom];
uint64_t DomIdx = this->ExprToIdx[&Dom];
assert(!Backward && "Post-dom frontier not implemented");
for (auto &BB : this->Func) {
for (BinaryBasicBlock &BB : this->Func) {
bool HasDominatedPred = false;
bool HasNonDominatedPred = false;
SmallSetVector<ProgramPoint, 4> Candidates;
@ -111,8 +111,8 @@ private:
void preflight() {
// Populate our universe of tracked expressions with all instructions
// except pseudos
for (auto &BB : this->Func) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : this->Func) {
for (MCInst &Inst : BB) {
this->Expressions.push_back(&Inst);
this->ExprToIdx[&Inst] = this->NumInstrs++;
}

117
bolt/src/Passes/ExtTSPReorderAlgorithm.cpp
View File

@ -109,7 +109,7 @@ double extTSPScore(uint64_t SrcAddr,
}
// Forward
if (SrcAddr + SrcSize < DstAddr) {
const auto Dist = DstAddr - (SrcAddr + SrcSize);
const uint64_t Dist = DstAddr - (SrcAddr + SrcSize);
if (Dist <= opts::ForwardDistance) {
double Prob = 1.0 - static_cast<double>(Dist) / opts::ForwardDistance;
return opts::ForwardWeight * Prob * Count;
@ -117,7 +117,7 @@ double extTSPScore(uint64_t SrcAddr,
return 0;
}
// Backward
const auto Dist = SrcAddr + SrcSize - DstAddr;
const uint64_t Dist = SrcAddr + SrcSize - DstAddr;
if (Dist <= opts::BackwardDistance) {
double Prob = 1.0 - static_cast<double>(Dist) / opts::BackwardDistance;
return opts::BackwardWeight * Prob * Count;
@ -215,7 +215,7 @@ public:
}
bool hasOutJump(const Block *Other) const {
for (auto Jump : OutJumps) {
for (std::pair<Block *, uint64_t> Jump : OutJumps) {
if (Jump.first == Other)
return true;
}
@ -223,7 +223,7 @@ public:
}
bool hasInJump(const Block *Other) const {
for (auto Jump : InJumps) {
for (std::pair<Block *, uint64_t> Jump : InJumps) {
if (Jump.first == Other)
return true;
}
@ -284,7 +284,7 @@ public:
}
Edge *getEdge(Chain *Other) const {
for (auto It : Edges) {
for (std::pair<Chain *, Edge *> It : Edges) {
if (It.first == Other)
return It.second;
}
@ -416,12 +416,12 @@ void Chain::mergeEdges(Chain *Other) {
// Update edges adjacent to chain Other
for (auto EdgeIt : Other->Edges) {
const auto DstChain = EdgeIt.first;
const auto DstEdge = EdgeIt.second;
const auto TargetChain = DstChain == Other ? this : DstChain;
Chain *const DstChain = EdgeIt.first;
Edge *const DstEdge = EdgeIt.second;
Chain *const TargetChain = DstChain == Other ? this : DstChain;
// Find the corresponding edge in the current chain
auto curEdge = getEdge(TargetChain);
Edge *curEdge = getEdge(TargetChain);
if (curEdge == nullptr) {
DstEdge->changeEndpoint(Other, this);
this->addEdge(TargetChain, DstEdge);
@ -492,8 +492,8 @@ private:
/// Deterministically compare pairs of chains
bool compareChainPairs(const Chain *A1, const Chain *B1,
const Chain *A2, const Chain *B2) {
const auto Samples1 = A1->executionCount() + B1->executionCount();
const auto Samples2 = A2->executionCount() + B2->executionCount();
const uint64_t Samples1 = A1->executionCount() + B1->executionCount();
const uint64_t Samples2 = A2->executionCount() + B2->executionCount();
if (Samples1 != Samples2)
return Samples1 < Samples2;
@ -535,22 +535,23 @@ private:
// Initialize CFG nodes
AllBlocks.reserve(BF.layout_size());
size_t LayoutIndex = 0;
for (auto BB : BF.layout()) {
for (BinaryBasicBlock *BB : BF.layout()) {
BB->setLayoutIndex(LayoutIndex++);
auto Size = std::max<uint64_t>(BB->estimateSize(Emitter.MCE.get()), 1);
uint64_t Size =
std::max<uint64_t>(BB->estimateSize(Emitter.MCE.get()), 1);
AllBlocks.emplace_back(BB, Size);
}
// Initialize edges for the blocks and compute their total in/out weights
size_t NumEdges = 0;
for (auto &Block : AllBlocks) {
for (Block &Block : AllBlocks) {
auto BI = Block.BB->branch_info_begin();
for (auto SuccBB : Block.BB->successors()) {
for (BinaryBasicBlock *SuccBB : Block.BB->successors()) {
assert(BI->Count != BinaryBasicBlock::COUNT_NO_PROFILE &&
"missing profile for a jump");
if (SuccBB != Block.BB && BI->Count > 0) {
auto &SuccBlock = AllBlocks[SuccBB->getLayoutIndex()];
auto Count = BI->Count;
class Block &SuccBlock = AllBlocks[SuccBB->getLayoutIndex()];
uint64_t Count = BI->Count;
SuccBlock.InWeight += Count;
SuccBlock.InJumps.push_back(std::make_pair(&Block, Count));
Block.OutWeight += Count;
@ -564,7 +565,7 @@ private:
// Initialize execution count for every basic block, which is the
// maximum over the sums of all in and out edge weights.
// Also execution count of the entry point is set to at least 1
for (auto &Block : AllBlocks) {
for (Block &Block : AllBlocks) {
size_t Index = Block.Index;
Block.ExecutionCount = std::max(Block.ExecutionCount, Block.InWeight);
Block.ExecutionCount = std::max(Block.ExecutionCount, Block.OutWeight);
@ -575,7 +576,7 @@ private:
// Initialize chains
AllChains.reserve(BF.layout_size());
HotChains.reserve(BF.layout_size());
for (auto &Block : AllBlocks) {
for (Block &Block : AllBlocks) {
AllChains.emplace_back(Block.Index, &Block);
Block.CurChain = &AllChains.back();
if (Block.ExecutionCount > 0) {
@ -585,10 +586,10 @@ private:
// Initialize edges
AllEdges.reserve(NumEdges);
for (auto &Block : AllBlocks) {
for (auto &Jump : Block.OutJumps) {
const auto SuccBlock = Jump.first;
auto CurEdge = Block.CurChain->getEdge(SuccBlock->CurChain);
for (Block &Block : AllBlocks) {
for (std::pair<class Block *, uint64_t> &Jump : Block.OutJumps) {
class Block *const SuccBlock = Jump.first;
Edge *CurEdge = Block.CurChain->getEdge(SuccBlock->CurChain);
// this edge is already present in the graph
if (CurEdge != nullptr) {
assert(SuccBlock->CurChain->getEdge(Block.CurChain) != nullptr);
@ -610,7 +611,7 @@ private:
/// the method finds and merges such pairs of blocks
void mergeFallthroughs() {
// Find fallthroughs based on edge weights
for (auto &Block : AllBlocks) {
for (Block &Block : AllBlocks) {
if (Block.BB->succ_size() == 1 &&
Block.BB->getSuccessor()->pred_size() == 1 &&
Block.BB->getSuccessor()->getLayoutIndex() != 0) {
@ -622,8 +623,8 @@ private:
if (Block.OutWeight == 0)
continue;
for (auto &Edge : Block.OutJumps) {
const auto SuccBlock = Edge.first;
for (std::pair<class Block *, uint64_t> &Edge : Block.OutJumps) {
class Block *const SuccBlock = Edge.first;
// Successor cannot be the first BB, which is pinned
if (Block.OutWeight == Edge.second &&
SuccBlock->InWeight == Edge.second &&
@ -638,11 +639,11 @@ private:
// There might be 'cycles' in the fallthrough dependencies (since profile
// data isn't 100% accurate).
// Break the cycles by choosing the block with smallest index as the tail
for (auto &Block : AllBlocks) {
for (Block &Block : AllBlocks) {
if (Block.FallthroughSucc == nullptr || Block.FallthroughPred == nullptr)
continue;
auto SuccBlock = Block.FallthroughSucc;
class Block *SuccBlock = Block.FallthroughSucc;
while (SuccBlock != nullptr && SuccBlock != &Block) {
SuccBlock = SuccBlock->FallthroughSucc;
}
@ -654,12 +655,12 @@ private:
}
// Merge blocks with their fallthrough successors
for (auto &Block : AllBlocks) {
for (Block &Block : AllBlocks) {
if (Block.FallthroughPred == nullptr &&
Block.FallthroughSucc != nullptr) {
auto CurBlock = &Block;
class Block *CurBlock = &Block;
while (CurBlock->FallthroughSucc != nullptr) {
const auto NextBlock = CurBlock->FallthroughSucc;
class Block *const NextBlock = CurBlock->FallthroughSucc;
mergeChains(Block.CurChain, NextBlock->CurChain, 0, MergeTypeTy::X_Y);
CurBlock = NextBlock;
}
@ -674,17 +675,17 @@ private:
Chain *BestChainSucc = nullptr;
auto BestGain = MergeGainTy();
// Iterate over all pairs of chains
for (auto ChainPred : HotChains) {
for (Chain *ChainPred : HotChains) {
// Get candidates for merging with the current chain
for (auto EdgeIter : ChainPred->edges()) {
auto ChainSucc = EdgeIter.first;
auto ChainEdge = EdgeIter.second;
Chain *ChainSucc = EdgeIter.first;
Edge *ChainEdge = EdgeIter.second;
// Ignore loop edges
if (ChainPred == ChainSucc)
continue;
// Compute the gain of merging the two chains
auto CurGain = mergeGain(ChainPred, ChainSucc, ChainEdge);
MergeGainTy CurGain = mergeGain(ChainPred, ChainSucc, ChainEdge);
if (CurGain.score() <= EPS)
continue;
@ -715,15 +716,15 @@ private:
/// Merge cold blocks to reduce code size
void mergeColdChains() {
for (auto SrcBB : BF.layout()) {
for (BinaryBasicBlock *SrcBB : BF.layout()) {
// Iterating in reverse order to make sure original fallthrough jumps are
// merged first
for (auto Itr = SrcBB->succ_rbegin(); Itr != SrcBB->succ_rend(); ++Itr) {
BinaryBasicBlock *DstBB = *Itr;
size_t SrcIndex = SrcBB->getLayoutIndex();
size_t DstIndex = DstBB->getLayoutIndex();
auto SrcChain = AllBlocks[SrcIndex].CurChain;
auto DstChain = AllBlocks[DstIndex].CurChain;
Chain *SrcChain = AllBlocks[SrcIndex].CurChain;
Chain *DstChain = AllBlocks[DstIndex].CurChain;
if (SrcChain != DstChain && !DstChain->isEntryPoint() &&
SrcChain->blocks().back()->Index == SrcIndex &&
DstChain->blocks().front()->Index == DstIndex) {
@ -746,9 +747,9 @@ private:
);
double Score = 0;
for (auto &Jump : Jumps) {
const auto SrcBlock = Jump.first.first;
const auto DstBlock = Jump.first.second;
for (const std::pair<std::pair<Block *, Block *>, uint64_t> &Jump : Jumps) {
const Block *SrcBlock = Jump.first.first;
const Block *DstBlock = Jump.first.second;
Score += extTSPScore(SrcBlock->EstimatedAddr,
SrcBlock->Size,
DstBlock->EstimatedAddr,
@ -769,8 +770,8 @@ private:
}
// Precompute jumps between ChainPred and ChainSucc
auto Jumps = Edge->jumps();
auto EdgePP = ChainPred->getEdge(ChainPred);
JumpList Jumps = Edge->jumps();
class Edge *EdgePP = ChainPred->getEdge(ChainPred);
if (EdgePP != nullptr)
Jumps.insert(Jumps.end(), EdgePP->jumps().begin(), EdgePP->jumps().end());
assert(Jumps.size() > 0 && "trying to merge chains w/o jumps");
@ -783,8 +784,8 @@ private:
// Try to break ChainPred in various ways and concatenate with ChainSucc
if (ChainPred->blocks().size() <= opts::ChainSplitThreshold) {
for (size_t Offset = 1; Offset < ChainPred->blocks().size(); Offset++) {
auto BB1 = ChainPred->blocks()[Offset - 1];
auto BB2 = ChainPred->blocks()[Offset];
Block *BB1 = ChainPred->blocks()[Offset - 1];
Block *BB2 = ChainPred->blocks()[Offset];
// Does the splitting break FT successors?
if (BB1->FallthroughSucc != nullptr) {
assert(BB1->FallthroughSucc == BB2 && "Fallthrough not preserved");
@ -811,10 +812,8 @@ private:
const JumpList &Jumps,
size_t MergeOffset,
MergeTypeTy MergeType) const {
auto MergedBlocks = mergeBlocks(ChainPred->blocks(),
ChainSucc->blocks(),
MergeOffset,
MergeType);
MergedChain MergedBlocks = mergeBlocks(
ChainPred->blocks(), ChainSucc->blocks(), MergeOffset, MergeType);
// Do not allow a merge that does not preserve the original entry block
if ((ChainPred->isEntryPoint() || ChainSucc->isEntryPoint()) &&
@ -822,7 +821,7 @@ private:
return CurGain;
// The gain for the new chain
const auto NewScore = score(MergedBlocks, Jumps) - ChainPred->score();
const double NewScore = score(MergedBlocks, Jumps) - ChainPred->score();
auto NewGain = MergeGainTy(NewScore, MergeOffset, MergeType);
return CurGain < NewGain ? NewGain : CurGain;
}
@ -868,16 +867,14 @@ private:
assert(Into != From && "a chain cannot be merged with itself");
// Merge the blocks
auto MergedBlocks = mergeBlocks(Into->blocks(),
From->blocks(),
MergeOffset,
MergeType);
MergedChain MergedBlocks =
mergeBlocks(Into->blocks(), From->blocks(), MergeOffset, MergeType);
Into->merge(From, MergedBlocks.getBlocks());
Into->mergeEdges(From);
From->clear();
// Update cached ext-tsp score for the new chain
auto SelfEdge = Into->getEdge(Into);
Edge *SelfEdge = Into->getEdge(Into);
if (SelfEdge != nullptr) {
MergedBlocks = MergedChain(Into->blocks().begin(), Into->blocks().end());
Into->setScore(score(MergedBlocks, SelfEdge->jumps()));
@ -888,7 +885,7 @@ private:
HotChains.erase(Iter, HotChains.end());
// Invalidate caches
for (auto EdgeIter : Into->edges()) {
for (std::pair<Chain *, Edge *> EdgeIter : Into->edges()) {
EdgeIter.second->invalidateCache();
}
}
@ -897,7 +894,7 @@ private:
void concatChains(std::vector<BinaryBasicBlock *> &Order) {
// Collect chains
std::vector<Chain *> SortedChains;
for (auto &Chain : AllChains) {
for (Chain &Chain : AllChains) {
if (Chain.blocks().size() > 0) {
SortedChains.push_back(&Chain);
}
@ -927,8 +924,8 @@ private:
// Collect the basic blocks in the order specified by their chains
Order.reserve(BF.layout_size());
for (auto Chain : SortedChains) {
for (auto Block : Chain->blocks()) {
for (Chain *Chain : SortedChains) {
for (Block *Block : Chain->blocks()) {
Order.push_back(Block->BB);
}
}
@ -958,7 +955,7 @@ void ExtTSPReorderAlgorithm::reorderBasicBlocks(
// Do not change layout of functions w/o profile information
if (!BF.hasValidProfile() || BF.layout_size() <= 2) {
for (auto BB : BF.layout()) {
for (BinaryBasicBlock *BB : BF.layout()) {
Order.push_back(BB);
}
return;

56
bolt/src/Passes/FrameAnalysis.cpp
View File

@ -55,7 +55,7 @@ bool shouldFrameOptimize(const llvm::bolt::BinaryFunction &Function) {
bool IsValid = true;
if (!FrameOptFunctionNames.empty()) {
IsValid = false;
for (auto &Name : FrameOptFunctionNames) {
for (std::string &Name : FrameOptFunctionNames) {
if (Function.hasName(Name)) {
IsValid = true;
break;
@ -182,7 +182,7 @@ public:
// Use CFI information to keep track of which register is being used to
// access the frame
if (BC.MIB->isCFI(Inst)) {
const auto *CFI = BF.getCFIFor(Inst);
const MCCFIInstruction *CFI = BF.getCFIFor(Inst);
switch (CFI->getOperation()) {
case MCCFIInstruction::OpDefCfa:
CfaOffset = CFI->getOffset();
@ -201,7 +201,7 @@ public:
dbgs() << "Assertion is about to fail: " << BF.getPrintName() << "\n";
}
assert(!CFIStack.empty() && "Corrupt CFI stack");
auto &Elem = CFIStack.top();
std::pair<int64_t, uint16_t> &Elem = CFIStack.top();
CFIStack.pop();
CfaOffset = Elem.first;
CfaReg = Elem.second;
@ -231,7 +231,7 @@ public:
} // end anonymous namespace
void FrameAnalysis::addArgAccessesFor(MCInst &Inst, ArgAccesses &&AA) {
if (auto OldAA = getArgAccessesFor(Inst)) {
if (ErrorOr<ArgAccesses &> OldAA = getArgAccessesFor(Inst)) {
if (OldAA->AssumeEverything)
return;
*OldAA = std::move(AA);
@ -249,13 +249,13 @@ void FrameAnalysis::addArgAccessesFor(MCInst &Inst, ArgAccesses &&AA) {
void FrameAnalysis::addArgInStackAccessFor(MCInst &Inst,
const ArgInStackAccess &Arg) {
auto AA = getArgAccessesFor(Inst);
ErrorOr<ArgAccesses &> AA = getArgAccessesFor(Inst);
if (!AA) {
addArgAccessesFor(Inst, ArgAccesses(false));
AA = getArgAccessesFor(Inst);
assert(AA && "Object setup failed");
}
auto &Set = AA->Set;
std::set<ArgInStackAccess> &Set = AA->Set;
assert(!AA->AssumeEverything && "Adding arg to AssumeEverything set");
Set.emplace(Arg);
}
@ -302,19 +302,20 @@ void FrameAnalysis::traverseCG(BinaryFunctionCallGraph &CG) {
CGWalker.walk();
DEBUG_WITH_TYPE("ra",
DEBUG_WITH_TYPE("ra", {
for (auto &MapEntry : ArgsTouchedMap) {
const auto *Func = MapEntry.first;
const BinaryFunction *Func = MapEntry.first;
const auto &Set = MapEntry.second;
dbgs() << "Args accessed for " << Func->getPrintName() << ": ";
if (!Set.empty() && Set.count(std::make_pair(-1, 0))) {
dbgs() << "assume everything";
} else {
for (auto &Entry : Set) {
for (const std::pair<int64_t, uint8_t> &Entry : Set) {
dbgs() << "[" << Entry.first << ", " << (int)Entry.second << "] ";
}
}
dbgs() << "\n";
}
});
}
@ -324,7 +325,7 @@ bool FrameAnalysis::updateArgsTouchedFor(const BinaryFunction &BF, MCInst &Inst,
return false;
std::set<int64_t> Res;
const auto *TargetSymbol = BC.MIB->getTargetSymbol(Inst);
const MCSymbol *TargetSymbol = BC.MIB->getTargetSymbol(Inst);
// If indirect call, we conservatively assume it accesses all stack positions
if (TargetSymbol == nullptr) {
addArgAccessesFor(Inst, ArgAccesses(/*AssumeEverything=*/true));
@ -335,7 +336,7 @@ bool FrameAnalysis::updateArgsTouchedFor(const BinaryFunction &BF, MCInst &Inst,
return false;
}
const auto *Function = BC.getFunctionForSymbol(TargetSymbol);
const BinaryFunction *Function = BC.getFunctionForSymbol(TargetSymbol);
// Call to a function without a BinaryFunction object. Conservatively assume
// it accesses all stack positions
if (Function == nullptr) {
@ -354,7 +355,7 @@ bool FrameAnalysis::updateArgsTouchedFor(const BinaryFunction &BF, MCInst &Inst,
// Ignore checking CurOffset because we can't always reliably determine the
// offset specially after an epilogue, where tailcalls happen. It should be
// -8.
for (auto Elem : Iter->second) {
for (std::pair<int64_t, uint8_t> Elem : Iter->second) {
if (ArgsTouchedMap[&BF].find(Elem) == ArgsTouchedMap[&BF].end()) {
ArgsTouchedMap[&BF].emplace(Elem);
Changed = true;
@ -375,7 +376,7 @@ bool FrameAnalysis::updateArgsTouchedFor(const BinaryFunction &BF, MCInst &Inst,
return Changed;
}
for (auto Elem : Iter->second) {
for (std::pair<int64_t, uint8_t> Elem : Iter->second) {
if (Elem.first == -1) {
addArgAccessesFor(Inst, ArgAccesses(/*AssumeEverything=*/true));
break;
@ -407,10 +408,10 @@ bool FrameAnalysis::computeArgsAccessed(BinaryFunction &BF) {
bool NoInfo = false;
FrameAccessAnalysis FAA(BC, BF, getSPT(BF));
for (auto BB : BF.layout()) {
for (BinaryBasicBlock *BB : BF.layout()) {
FAA.enterNewBB();
for (auto &Inst : *BB) {
for (MCInst &Inst : *BB) {
if (!FAA.doNext(*BB, Inst)) {
ArgsTouchedMap[&BF].emplace(std::make_pair(-1, 0));
NoInfo = true;
@ -452,8 +453,8 @@ bool FrameAnalysis::computeArgsAccessed(BinaryFunction &BF) {
return true;
}
for (auto &BB : BF) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : BF) {
for (MCInst &Inst : BB) {
if (BC.MIB->requiresAlignedAddress(Inst)) {
FunctionsRequireAlignment.insert(&BF);
return true;
@ -468,11 +469,11 @@ bool FrameAnalysis::restoreFrameIndex(BinaryFunction &BF) {
LLVM_DEBUG(dbgs() << "Restoring frame indices for \"" << BF.getPrintName()
<< "\"\n");
for (auto BB : BF.layout()) {
for (BinaryBasicBlock *BB : BF.layout()) {
LLVM_DEBUG(dbgs() << "\tNow at BB " << BB->getName() << "\n");
FAA.enterNewBB();
for (auto &Inst : *BB) {
for (MCInst &Inst : *BB) {
if (!FAA.doNext(*BB, Inst))
return false;
LLVM_DEBUG({
@ -501,8 +502,8 @@ void FrameAnalysis::cleanAnnotations() {
"FA breakdown", opts::TimeFA);
ParallelUtilities::WorkFuncTy CleanFunction = [&](BinaryFunction &BF) {
for (auto &BB : BF) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : BF) {
for (MCInst &Inst : BB) {
BC.MIB->removeAnnotation(Inst, "ArgAccessEntry");
BC.MIB->removeAnnotation(Inst, "FrameAccessEntry");
}
@ -533,7 +534,7 @@ FrameAnalysis::FrameAnalysis(BinaryContext &BC, BinaryFunctionCallGraph &CG)
}
for (auto &I : BC.getBinaryFunctions()) {
auto Count = I.second.getExecutionCount();
uint64_t Count = I.second.getExecutionCount();
if (Count != BinaryFunction::COUNT_NO_PROFILE)
CountDenominator += Count;
@ -551,7 +552,7 @@ FrameAnalysis::FrameAnalysis(BinaryContext &BC, BinaryFunctionCallGraph &CG)
"FA breakdown", opts::TimeFA);
if (!restoreFrameIndex(I.second)) {
++NumFunctionsFailedRestoreFI;
auto Count = I.second.getExecutionCount();
uint64_t Count = I.second.getExecutionCount();
if (Count != BinaryFunction::COUNT_NO_PROFILE)
CountFunctionsFailedRestoreFI += Count;
continue;
@ -567,7 +568,7 @@ FrameAnalysis::FrameAnalysis(BinaryContext &BC, BinaryFunctionCallGraph &CG)
// Clean up memory allocated for annotation values
if (!opts::NoThreads) {
for (auto Id : SPTAllocatorsId)
for (MCPlusBuilder::AllocatorIdTy Id : SPTAllocatorsId)
BC.MIB->freeValuesAllocator(Id);
}
}
@ -593,7 +594,7 @@ void FrameAnalysis::clearSPTMap() {
}
ParallelUtilities::WorkFuncTy ClearFunctionSPT = [&](BinaryFunction &BF) {
auto &SPTPtr = SPTMap.find(&BF)->second;
std::unique_ptr<StackPointerTracking> &SPTPtr = SPTMap.find(&BF)->second;
SPTPtr.reset();
};
@ -614,7 +615,7 @@ void FrameAnalysis::preComputeSPT() {
// Create map entries to allow lock-free parallel execution
for (auto &BFI : BC.getBinaryFunctions()) {
auto &BF = BFI.second;
BinaryFunction &BF = BFI.second;
if (!BF.isSimple() || !BF.hasCFG())
continue;
SPTMap.emplace(&BF, std::unique_ptr<StackPointerTracking>());
@ -627,7 +628,8 @@ void FrameAnalysis::preComputeSPT() {
// Run SPT in parallel
ParallelUtilities::WorkFuncWithAllocTy ProcessFunction =
[&](BinaryFunction &BF, MCPlusBuilder::AllocatorIdTy AllocId) {
auto &SPTPtr = SPTMap.find(&BF)->second;
std::unique_ptr<StackPointerTracking> &SPTPtr =
SPTMap.find(&BF)->second;
SPTPtr = std::make_unique<StackPointerTracking>(BC, BF, AllocId);
SPTPtr->run();
};

18
bolt/src/Passes/FrameOptimizer.cpp
View File

@ -63,10 +63,10 @@ void FrameOptimizerPass::removeUnnecessaryLoads(const RegAnalysis &RA,
std::deque<std::pair<BinaryBasicBlock *, MCInst *>> ToErase;
bool Changed = false;
const auto ExprEnd = SAE.expr_end();
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
LLVM_DEBUG(dbgs() <<"\tNow at BB " << BB.getName() << "\n");
const MCInst *Prev = nullptr;
for (auto &Inst : BB) {
for (MCInst &Inst : BB) {
LLVM_DEBUG({
dbgs() << "\t\tNow at ";
Inst.dump();
@ -79,7 +79,7 @@ void FrameOptimizerPass::removeUnnecessaryLoads(const RegAnalysis &RA,
// if Inst is a load from stack and the current available expressions show
// this value is available in a register or immediate, replace this load
// with move from register or from immediate.
auto FIEX = FA.getFIEFor(Inst);
ErrorOr<const FrameIndexEntry &> FIEX = FA.getFIEFor(Inst);
if (!FIEX) {
Prev = &Inst;
continue;
@ -96,7 +96,7 @@ void FrameOptimizerPass::removeUnnecessaryLoads(const RegAnalysis &RA,
for (auto I = Prev ? SAE.expr_begin(*Prev) : SAE.expr_begin(BB);
I != ExprEnd; ++I) {
const MCInst *AvailableInst = *I;
auto FIEY = FA.getFIEFor(*AvailableInst);
ErrorOr<const FrameIndexEntry &> FIEY = FA.getFIEFor(*AvailableInst);
if (!FIEY)
continue;
assert(FIEY->IsStore && FIEY->IsSimple);
@ -152,7 +152,7 @@ void FrameOptimizerPass::removeUnnecessaryLoads(const RegAnalysis &RA,
}
// TODO: Implement an interface of eraseInstruction that works out the
// complete list of elements to remove.
for (auto I : ToErase) {
for (std::pair<BinaryBasicBlock *, MCInst *> I : ToErase) {
I.first->eraseInstruction(I.first->findInstruction(I.second));
}
}
@ -166,11 +166,11 @@ void FrameOptimizerPass::removeUnusedStores(const FrameAnalysis &FA,
LLVM_DEBUG(dbgs() << "Performing unused stores removal\n");
std::vector<std::pair<BinaryBasicBlock *, MCInst *>> ToErase;
bool Changed = false;
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
LLVM_DEBUG(dbgs() <<"\tNow at BB " << BB.getName() << "\n");
const MCInst *Prev = nullptr;
for (auto I = BB.rbegin(), E = BB.rend(); I != E; ++I) {
auto &Inst = *I;
MCInst &Inst = *I;
LLVM_DEBUG({
dbgs() << "\t\tNow at ";
Inst.dump();
@ -180,7 +180,7 @@ void FrameOptimizerPass::removeUnusedStores(const FrameAnalysis &FA,
(*I)->dump();
}
});
auto FIEX = FA.getFIEFor(Inst);
ErrorOr<const FrameIndexEntry &> FIEX = FA.getFIEFor(Inst);
if (!FIEX) {
Prev = &Inst;
continue;
@ -212,7 +212,7 @@ void FrameOptimizerPass::removeUnusedStores(const FrameAnalysis &FA,
}
}
for (auto I : ToErase) {
for (std::pair<BinaryBasicBlock *, MCInst *> I : ToErase) {
I.first->eraseInstruction(I.first->findInstruction(I.second));
}
if (Changed) {

32
bolt/src/Passes/HFSort.cpp
View File

@ -76,7 +76,7 @@ Cluster::Cluster(NodeId Id, const Node &Func)
Cluster::Cluster(const std::vector<NodeId> &Nodes, const CallGraph &Cg) {
Samples = 0;
Size = 0;
for (auto TargetId : Nodes) {
for (NodeId TargetId : Nodes) {
Targets.push_back(TargetId);
Samples += Cg.samples(TargetId);
Size += Cg.size(TargetId);
@ -89,7 +89,7 @@ std::string Cluster::toString() const {
raw_string_ostream CS(Str);
bool PrintComma = false;
CS << "funcs = [";
for (auto &Target : Targets) {
for (const NodeId &Target : Targets) {
if (PrintComma) CS << ", ";
CS << Target;
PrintComma = true;
@ -103,13 +103,13 @@ namespace {
void freezeClusters(const CallGraph &Cg, std::vector<Cluster> &Clusters) {
uint32_t TotalSize = 0;
std::sort(Clusters.begin(), Clusters.end(), compareClustersDensity);
for (auto &C : Clusters) {
for (Cluster &C : Clusters) {
uint32_t NewSize = TotalSize + C.size();
if (NewSize > FrozenPages * HugePageSize) break;
C.freeze();
TotalSize = NewSize;
LLVM_DEBUG(
auto Fid = C.target(0);
NodeId Fid = C.target(0);
dbgs() <<
format("freezing cluster for func %d, size = %u, samples = %lu)\n",
Fid, Cg.size(Fid), Cg.samples(Fid)););
@ -165,7 +165,7 @@ std::vector<Cluster> clusterize(const CallGraph &Cg) {
// The size and order of Clusters is fixed until we reshuffle it immediately
// before returning.
for (auto &Cluster : Clusters) {
for (Cluster &Cluster : Clusters) {
FuncCluster[Cluster.targets().front()] = &Cluster;
}
@ -173,8 +173,8 @@ std::vector<Cluster> clusterize(const CallGraph &Cg) {
SortedFuncs.begin(),
SortedFuncs.end(),
[&] (const NodeId F1, const NodeId F2) {
const auto &Func1 = Cg.getNode(F1);
const auto &Func2 = Cg.getNode(F2);
const CallGraph::Node &Func1 = Cg.getNode(F1);
const CallGraph::Node &Func2 = Cg.getNode(F2);
return
Func1.samples() * Func2.size() > // TODO: is this correct?
Func2.samples() * Func1.size();
@ -183,16 +183,16 @@ std::vector<Cluster> clusterize(const CallGraph &Cg) {
// Process each function, and consider merging its cluster with the
// one containing its most likely predecessor.
for (const auto Fid : SortedFuncs) {
auto Cluster = FuncCluster[Fid];
for (const NodeId Fid : SortedFuncs) {
Cluster *Cluster = FuncCluster[Fid];
if (Cluster->frozen()) continue;
// Find best predecessor.
NodeId BestPred = CallGraph::InvalidId;
double BestProb = 0;
for (const auto Src : Cg.predecessors(Fid)) {
const auto &Arc = *Cg.findArc(Src, Fid);
for (const NodeId Src : Cg.predecessors(Fid)) {
const Arc &Arc = *Cg.findArc(Src, Fid);
if (BestPred == CallGraph::InvalidId || Arc.normalizedWeight() > BestProb) {
BestPred = Arc.src();
BestProb = Arc.normalizedWeight();
@ -206,7 +206,7 @@ std::vector<Cluster> clusterize(const CallGraph &Cg) {
assert(BestPred != CallGraph::InvalidId);
auto PredCluster = FuncCluster[BestPred];
class Cluster *PredCluster = FuncCluster[BestPred];
// Skip if no predCluster (predecessor w/ no samples), or if same
// as cluster, of it's frozen.
@ -236,7 +236,7 @@ std::vector<Cluster> clusterize(const CallGraph &Cg) {
Cg.samples(Fid));
});
for (auto F : Cluster->targets()) {
for (NodeId F : Cluster->targets()) {
FuncCluster[F] = PredCluster;
}
@ -248,8 +248,8 @@ std::vector<Cluster> clusterize(const CallGraph &Cg) {
// didn't get merged (so their first func is its original func).
std::vector<Cluster> SortedClusters;
std::unordered_set<Cluster *> Visited;
for (const auto Func : SortedFuncs) {
auto Cluster = FuncCluster[Func];
for (const NodeId Func : SortedFuncs) {
Cluster *Cluster = FuncCluster[Func];
if (!Cluster ||
Visited.count(Cluster) == 1 ||
Cluster->target(0) != Func) {
@ -300,7 +300,7 @@ std::vector<Cluster> randomClusters(const CallGraph &Cg) {
size_t Idx = 0;
while (Idx < Clusters.size()) {
auto MergeIdx = pickMergeCluster(Idx);
size_t MergeIdx = pickMergeCluster(Idx);
if (MergeIdx == Clusters.size()) {
++Idx;
} else {

64
bolt/src/Passes/HFSortPlus.cpp
View File

@ -106,7 +106,7 @@ public:
double density() const { return static_cast<double>(Samples) / Size; }
Edge *getEdge(Chain *Other) const {
for (auto It : Edges) {
for (std::pair<Chain *, Edge *> It : Edges) {
if (It.first == Other)
return It.second;
}
@ -223,12 +223,12 @@ public:
void Chain::mergeEdges(Chain *Other) {
// Update edges adjacent to chain other
for (auto EdgeIt : Other->Edges) {
const auto DstChain = EdgeIt.first;
const auto DstEdge = EdgeIt.second;
const auto TargetChain = DstChain == Other ? this : DstChain;
Chain *const DstChain = EdgeIt.first;
Edge *const DstEdge = EdgeIt.second;
Chain *const TargetChain = DstChain == Other ? this : DstChain;
// Find the corresponding edge in the current chain
auto CurEdge = getEdge(TargetChain);
Edge *CurEdge = getEdge(TargetChain);
if (CurEdge == nullptr) {
DstEdge->changeEndpoint(Other, this);
this->addEdge(TargetChain, DstEdge);
@ -274,7 +274,7 @@ public:
// didn't get merged (so their first func is its original func)
std::vector<Cluster> Clusters;
Clusters.reserve(HotChains.size());
for (auto Chain : HotChains) {
for (Chain *Chain : HotChains) {
Clusters.emplace_back(Cluster(Chain->Nodes, Cg));
}
return Clusters;
@ -297,10 +297,10 @@ private:
HotChains.push_back(&AllChains.back());
NodeChain[F] = &AllChains.back();
TotalSamples += Cg.samples(F);
for (auto Succ : Cg.successors(F)) {
for (NodeId Succ : Cg.successors(F)) {
if (F == Succ)
continue;
const auto &Arc = *Cg.findArc(F, Succ);
const Arc &Arc = *Cg.findArc(F, Succ);
OutWeight[F] += Arc.weight();
InWeight[Succ] += Arc.weight();
}
@ -308,16 +308,16 @@ private:
AllEdges.reserve(Cg.numArcs());
for (NodeId F = 0; F < Cg.numNodes(); ++F) {
for (auto Succ : Cg.successors(F)) {
for (NodeId Succ : Cg.successors(F)) {
if (F == Succ)
continue;
const auto &Arc = *Cg.findArc(F, Succ);
const Arc &Arc = *Cg.findArc(F, Succ);
if (Arc.weight() == 0.0 ||
Arc.weight() / TotalSamples < opts::ArcThreshold) {
continue;
}
auto CurEdge = NodeChain[F]->getEdge(NodeChain[Succ]);
Edge *CurEdge = NodeChain[F]->getEdge(NodeChain[Succ]);
if (CurEdge != nullptr) {
// This edge is already present in the graph
assert(NodeChain[Succ]->getEdge(NodeChain[F]) != nullptr);
@ -331,7 +331,7 @@ private:
}
}
for (auto &Chain : HotChains) {
for (Chain *&Chain : HotChains) {
Chain->ShortCalls = shortCalls(Chain);
Chain->Score = score(Chain);
}
@ -373,12 +373,12 @@ private:
/// The expected number of calls within a given chain with both endpoints on
/// the same cache page
double shortCalls(Chain *Chain) const {
auto Edge = Chain->getEdge(Chain);
Edge *Edge = Chain->getEdge(Chain);
if (Edge == nullptr)
return 0;
double Calls = 0;
for (auto Arc : Edge->Arcs) {
for (const Arc *Arc : Edge->Arcs) {
uint64_t SrcAddr = Addr[Arc->src()] + uint64_t(Arc->avgCallOffset());
uint64_t DstAddr = Addr[Arc->dst()];
Calls += expectedCalls(SrcAddr, DstAddr, Arc->weight());
@ -390,8 +390,8 @@ private:
/// the same i-TLB page, assuming that a given pair of chains gets merged
double shortCalls(Chain *ChainPred, Chain *ChainSucc, Edge *Edge) const {
double Calls = 0;
for (auto Arc : Edge->Arcs) {
auto SrcChain = NodeChain[Arc->src()];
for (const Arc *Arc : Edge->Arcs) {
Chain *SrcChain = NodeChain[Arc->src()];
uint64_t SrcAddr;
uint64_t DstAddr;
if (SrcChain == ChainPred) {
@ -449,13 +449,13 @@ private:
void runPassOne() {
// Find candidate pairs of chains for merging
std::vector<const Arc *> ArcsToMerge;
for (auto ChainPred : HotChains) {
auto F = ChainPred->Nodes.back();
for (auto Succ : Cg.successors(F)) {
for (Chain *ChainPred : HotChains) {
NodeId F = ChainPred->Nodes.back();
for (NodeId Succ : Cg.successors(F)) {
if (F == Succ)
continue;
const auto &Arc = *Cg.findArc(F, Succ);
const Arc &Arc = *Cg.findArc(F, Succ);
if (Arc.weight() == 0.0 ||
Arc.weight() / TotalSamples < opts::ArcThreshold) {
continue;
@ -486,9 +486,9 @@ private:
[](const Arc *L, const Arc *R) { return L->weight() > R->weight(); });
// Merge the pairs of chains
for (auto Arc : ArcsToMerge) {
auto ChainPred = NodeChain[Arc->src()];
auto ChainSucc = NodeChain[Arc->dst()];
for (const Arc *Arc : ArcsToMerge) {
Chain *ChainPred = NodeChain[Arc->src()];
Chain *ChainSucc = NodeChain[Arc->dst()];
if (ChainPred == ChainSucc)
continue;
if (ChainPred->Nodes.back() == Arc->src() &&
@ -516,10 +516,10 @@ private:
std::set<Edge *, decltype(GainComparator)> Queue(GainComparator);
// Inserting the edges Into the queue
for (auto ChainPred : HotChains) {
for (Chain *ChainPred : HotChains) {
for (auto EdgeIt : ChainPred->Edges) {
auto ChainSucc = EdgeIt.first;
auto ChainEdge = EdgeIt.second;
Chain *ChainSucc = EdgeIt.first;
Edge *ChainEdge = EdgeIt.second;
// Ignore loop edges
if (ChainPred == ChainSucc)
continue;
@ -540,7 +540,7 @@ private:
// Merge the chains while the gain of merging is positive
while (!Queue.empty()) {
// Extract the best (top) edge for merging
auto It = *Queue.begin();
Edge *It = *Queue.begin();
Queue.erase(Queue.begin());
Edge *BestEdge = It;
Chain *BestChainPred = BestEdge->predChain();
@ -549,10 +549,10 @@ private:
continue;
// Remove outdated edges
for (auto EdgeIt : BestChainPred->Edges) {
for (std::pair<Chain *, Edge *> EdgeIt : BestChainPred->Edges) {
Queue.erase(EdgeIt.second);
}
for (auto EdgeIt : BestChainSucc->Edges) {
for (std::pair<Chain *, Edge *> EdgeIt : BestChainSucc->Edges) {
Queue.erase(EdgeIt.second);
}
@ -561,8 +561,8 @@ private:
// Insert newly created edges Into the queue
for (auto EdgeIt : BestChainPred->Edges) {
auto ChainSucc = EdgeIt.first;
auto ChainEdge = EdgeIt.second;
Chain *ChainSucc = EdgeIt.first;
Edge *ChainEdge = EdgeIt.second;
// Ignore loop edges
if (BestChainPred == ChainSucc)
continue;
@ -585,7 +585,7 @@ private:
// Update the chains and addresses for functions merged from From
size_t CurAddr = 0;
for (auto F : Into->Nodes) {
for (NodeId F : Into->Nodes) {
NodeChain[F] = Into;
Addr[F] = CurAddr;
CurAddr += Cg.size(F);

80
bolt/src/Passes/IdenticalCodeFolding.cpp
View File

@ -60,11 +60,11 @@ bool equalJumpTables(const JumpTable &JumpTableA,
return false;
for (uint64_t Index = 0; Index < JumpTableA.Entries.size(); ++Index) {
const auto *LabelA = JumpTableA.Entries[Index];
const auto *LabelB = JumpTableB.Entries[Index];
const MCSymbol *LabelA = JumpTableA.Entries[Index];
const MCSymbol *LabelB = JumpTableB.Entries[Index];
const auto *TargetA = FunctionA.getBasicBlockForLabel(LabelA);
const auto *TargetB = FunctionB.getBasicBlockForLabel(LabelB);
const BinaryBasicBlock *TargetA = FunctionA.getBasicBlockForLabel(LabelA);
const BinaryBasicBlock *TargetB = FunctionB.getBasicBlockForLabel(LabelB);
if (!TargetA || !TargetB) {
assert((TargetA || LabelA == FunctionA.getFunctionEndLabel()) &&
@ -98,7 +98,7 @@ bool isInstrEquivalentWith(const MCInst &InstA, const BinaryBasicBlock &BBA,
return false;
}
const auto &BC = BBA.getFunction()->getBinaryContext();
const BinaryContext &BC = BBA.getFunction()->getBinaryContext();
// In this function we check for special conditions:
//
@ -110,8 +110,8 @@ bool isInstrEquivalentWith(const MCInst &InstA, const BinaryBasicBlock &BBA,
// NB: there's no need to compare jump table indirect jump instructions
// separately as jump tables are handled by comparing corresponding
// symbols.
const auto EHInfoA = BC.MIB->getEHInfo(InstA);
const auto EHInfoB = BC.MIB->getEHInfo(InstB);
const Optional<MCPlus::MCLandingPad> EHInfoA = BC.MIB->getEHInfo(InstA);
const Optional<MCPlus::MCLandingPad> EHInfoB = BC.MIB->getEHInfo(InstB);
if (EHInfoA || EHInfoB) {
if (!EHInfoA && (EHInfoB->first || EHInfoB->second))
@ -129,8 +129,8 @@ bool isInstrEquivalentWith(const MCInst &InstA, const BinaryBasicBlock &BBA,
return false;
if (EHInfoA->first && EHInfoB->first) {
const auto *LPA = BBA.getLandingPad(EHInfoA->first);
const auto *LPB = BBB.getLandingPad(EHInfoB->first);
const BinaryBasicBlock *LPA = BBA.getLandingPad(EHInfoA->first);
const BinaryBasicBlock *LPB = BBB.getLandingPad(EHInfoB->first);
assert(LPA && LPB && "cannot locate landing pad(s)");
if (LPA->getLayoutIndex() != LPB->getLayoutIndex())
@ -166,14 +166,16 @@ bool isIdenticalWith(const BinaryFunction &A, const BinaryFunction &B,
return false;
// Process both functions in either DFS or existing order.
const auto &OrderA = opts::UseDFS ? A.dfs() : A.getLayout();
const auto &OrderB = opts::UseDFS ? B.dfs() : B.getLayout();
const std::vector<BinaryBasicBlock *> &OrderA =
opts::UseDFS ? A.dfs() : A.getLayout();
const std::vector<BinaryBasicBlock *> &OrderB =
opts::UseDFS ? B.dfs() : B.getLayout();
const auto &BC = A.getBinaryContext();
const BinaryContext &BC = A.getBinaryContext();
auto BBI = OrderB.begin();
for (const auto *BB : OrderA) {
const auto *OtherBB = *BBI;
for (const BinaryBasicBlock *BB : OrderA) {
const BinaryBasicBlock *OtherBB = *BBI;
if (BB->getLayoutIndex() != OtherBB->getLayoutIndex())
return false;
@ -184,8 +186,8 @@ bool isIdenticalWith(const BinaryFunction &A, const BinaryFunction &B,
return false;
auto SuccBBI = OtherBB->succ_begin();
for (const auto *SuccBB : BB->successors()) {
const auto *SuccOtherBB = *SuccBBI;
for (const BinaryBasicBlock *SuccBB : BB->successors()) {
const BinaryBasicBlock *SuccOtherBB = *SuccBBI;
if (SuccBB->getLayoutIndex() != SuccOtherBB->getLayoutIndex())
return false;
++SuccBBI;
@ -212,8 +214,10 @@ bool isIdenticalWith(const BinaryFunction &A, const BinaryFunction &B,
// Compare symbols as functions.
uint64_t EntryIDA{0};
uint64_t EntryIDB{0};
const auto *FunctionA = BC.getFunctionForSymbol(SymbolA, &EntryIDA);
const auto *FunctionB = BC.getFunctionForSymbol(SymbolB, &EntryIDB);
const BinaryFunction *FunctionA =
BC.getFunctionForSymbol(SymbolA, &EntryIDA);
const BinaryFunction *FunctionB =
BC.getFunctionForSymbol(SymbolB, &EntryIDB);
if (FunctionA && EntryIDA)
FunctionA = nullptr;
if (FunctionB && EntryIDB)
@ -237,23 +241,23 @@ bool isIdenticalWith(const BinaryFunction &A, const BinaryFunction &B,
}
// Check if symbols are jump tables.
auto *SIA = BC.getBinaryDataByName(SymbolA->getName());
const BinaryData *SIA = BC.getBinaryDataByName(SymbolA->getName());
if (!SIA)
return false;
auto *SIB = BC.getBinaryDataByName(SymbolB->getName());
const BinaryData *SIB = BC.getBinaryDataByName(SymbolB->getName());
if (!SIB)
return false;
assert((SIA->getAddress() != SIB->getAddress()) &&
"different symbols should not have the same value");
const auto *JumpTableA =
A.getJumpTableContainingAddress(SIA->getAddress());
const JumpTable *JumpTableA =
A.getJumpTableContainingAddress(SIA->getAddress());
if (!JumpTableA)
return false;
const auto *JumpTableB =
B.getJumpTableContainingAddress(SIB->getAddress());
const JumpTable *JumpTableB =
B.getJumpTableContainingAddress(SIB->getAddress());
if (!JumpTableB)
return false;
@ -274,8 +278,8 @@ bool isIdenticalWith(const BinaryFunction &A, const BinaryFunction &B,
// One of the identical blocks may have a trailing unconditional jump that
// is ignored for CFG purposes.
auto *TrailingInstr = (I != E ? &(*I)
: (OtherI != OtherE ? &(*OtherI) : 0));
const MCInst *TrailingInstr =
(I != E ? &(*I) : (OtherI != OtherE ? &(*OtherI) : 0));
if (TrailingInstr && !BC.MIB->isUnconditionalBranch(*TrailingInstr)) {
return false;
}
@ -353,7 +357,7 @@ std::string hashSymbol(BinaryContext &BC, const MCSymbol &Symbol) {
if (BC.getFunctionForSymbol(&Symbol))
return HashString;
auto ErrorOrValue = BC.getSymbolValue(Symbol);
llvm::ErrorOr<uint64_t> ErrorOrValue = BC.getSymbolValue(Symbol);
if (!ErrorOrValue)
return HashString;
@ -406,7 +410,7 @@ namespace llvm {
namespace bolt {
void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) {
const auto OriginalFunctionCount = BC.getBinaryFunctions().size();
const size_t OriginalFunctionCount = BC.getBinaryFunctions().size();
uint64_t NumFunctionsFolded{0};
std::atomic<uint64_t> NumJTFunctionsFolded{0};
std::atomic<uint64_t> BytesSavedEstimate{0};
@ -446,7 +450,7 @@ void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) {
"congruent buckets", "ICF breakdown",
"ICF breakdown", opts::TimeICF);
for (auto &BFI : BC.getBinaryFunctions()) {
auto &BF = BFI.second;
BinaryFunction &BF = BFI.second;
if (!this->shouldOptimize(BF))
continue;
CongruentBuckets[&BF].emplace(&BF);
@ -473,13 +477,13 @@ void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) {
// Identical functions go into the same bucket.
IdenticalBucketsMap IdenticalBuckets;
for (auto *BF : Candidates) {
for (BinaryFunction *BF : Candidates) {
IdenticalBuckets[BF].emplace_back(BF);
}
for (auto &IBI : IdenticalBuckets) {
// Functions identified as identical.
auto &Twins = IBI.second;
std::vector<BinaryFunction *> &Twins = IBI.second;
if (Twins.size() < 2)
continue;
@ -492,8 +496,8 @@ void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) {
});
BinaryFunction *ParentBF = Twins[0];
for (unsigned i = 1; i < Twins.size(); ++i) {
auto *ChildBF = Twins[i];
for (unsigned I = 1; I < Twins.size(); ++I) {
BinaryFunction *ChildBF = Twins[I];
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: folding " << *ChildBF << " into "
<< *ParentBF << '\n');
@ -521,7 +525,7 @@ void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) {
// Create a task for each congruent bucket
for (auto &Entry : CongruentBuckets) {
auto &Bucket = Entry.second;
std::set<BinaryFunction *> &Bucket = Entry.second;
if (Bucket.size() < 2)
continue;
@ -553,16 +557,16 @@ void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) {
} while (NumFoldedLastIteration > 0);
LLVM_DEBUG(
LLVM_DEBUG({
// Print functions that are congruent but not identical.
for (auto &CBI : CongruentBuckets) {
auto &Candidates = CBI.second;
std::set<BinaryFunction *> &Candidates = CBI.second;
if (Candidates.size() < 2)
continue;
dbgs() << "BOLT-DEBUG: the following " << Candidates.size()
<< " functions (each of size " << (*Candidates.begin())->getSize()
<< " bytes) are congruent but not identical:\n";
for (auto *BF : Candidates) {
for (BinaryFunction *BF : Candidates) {
dbgs() << " " << *BF;
if (BF->getKnownExecutionCount()) {
dbgs() << " (executed " << BF->getKnownExecutionCount() << " times)";
@ -570,7 +574,7 @@ void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) {
dbgs() << '\n';
}
}
);
});
if (NumFunctionsFolded) {
outs() << "BOLT-INFO: ICF folded " << NumFunctionsFolded

227
bolt/src/Passes/IndirectCallPromotion.cpp
View File

@ -176,11 +176,11 @@ namespace {
bool verifyProfile(std::map<uint64_t, BinaryFunction> &BFs) {
bool IsValid = true;
for (auto &BFI : BFs) {
auto &BF = BFI.second;
BinaryFunction &BF = BFI.second;
if (!BF.isSimple()) continue;
for (auto BB : BF.layout()) {
for (BinaryBasicBlock *BB : BF.layout()) {
auto BI = BB->branch_info_begin();
for (auto SuccBB : BB->successors()) {
for (BinaryBasicBlock *SuccBB : BB->successors()) {
if (BI->Count != BinaryBasicBlock::COUNT_NO_PROFILE && BI->Count > 0) {
if (BB->getKnownExecutionCount() == 0 ||
SuccBB->getKnownExecutionCount() == 0) {
@ -215,7 +215,7 @@ void IndirectCallPromotion::printDecision(
std::vector<IndirectCallPromotion::Callsite> &Targets, unsigned N) const {
uint64_t TotalCount = 0;
uint64_t TotalMispreds = 0;
for (const auto &S : Targets) {
for (const Callsite &S : Targets) {
TotalCount += S.Branches;
TotalMispreds += S.Mispreds;
}
@ -229,7 +229,7 @@ void IndirectCallPromotion::printDecision(
<< "\n";
size_t I = 0;
for (const auto &S : Targets) {
for (const Callsite &S : Targets) {
OS << "Count = " << S.Branches << ", "
<< format("%.1f", (100.0 * S.Branches) / TotalCount) << ", "
<< "Mispreds = " << S.Mispreds << ", "
@ -238,7 +238,7 @@ void IndirectCallPromotion::printDecision(
OS << " * to be optimized *";
if (!S.JTIndices.empty()) {
OS << " Indices:";
for (const auto Idx : S.JTIndices)
for (const uint64_t Idx : S.JTIndices)
OS << " " << Idx;
}
OS << "\n";
@ -253,24 +253,26 @@ IndirectCallPromotion::getCallTargets(
BinaryBasicBlock &BB,
const MCInst &Inst
) const {
auto &BF = *BB.getFunction();
auto &BC = BF.getBinaryContext();
BinaryFunction &BF = *BB.getFunction();
BinaryContext &BC = BF.getBinaryContext();
std::vector<Callsite> Targets;
if (const auto *JT = BF.getJumpTable(Inst)) {
if (const JumpTable *JT = BF.getJumpTable(Inst)) {
// Don't support PIC jump tables for now
if (!opts::ICPJumpTablesByTarget && JT->Type == JumpTable::JTT_PIC)
return Targets;
const Location From(BF.getSymbol());
const auto Range = JT->getEntriesForAddress(BC.MIB->getJumpTable(Inst));
const std::pair<size_t, size_t> Range =
JT->getEntriesForAddress(BC.MIB->getJumpTable(Inst));
assert(JT->Counts.empty() || JT->Counts.size() >= Range.second);
JumpTable::JumpInfo DefaultJI;
const auto *JI = JT->Counts.empty() ? &DefaultJI : &JT->Counts[Range.first];
const JumpTable::JumpInfo *JI =
JT->Counts.empty() ? &DefaultJI : &JT->Counts[Range.first];
const size_t JIAdj = JT->Counts.empty() ? 0 : 1;
assert(JT->Type == JumpTable::JTT_PIC ||
JT->EntrySize == BC.AsmInfo->getCodePointerSize());
for (size_t I = Range.first; I < Range.second; ++I, JI += JIAdj) {
auto *Entry = JT->Entries[I];
MCSymbol *Entry = JT->Entries[I];
assert(BF.getBasicBlockForLabel(Entry) ||
Entry == BF.getFunctionEndLabel() ||
Entry == BF.getFunctionColdEndLabel());
@ -278,7 +280,7 @@ IndirectCallPromotion::getCallTargets(
Entry == BF.getFunctionColdEndLabel())
continue;
const Location To(Entry);
const auto &BI = BB.getBranchInfo(Entry);
const BinaryBasicBlock::BinaryBranchInfo &BI = BB.getBranchInfo(Entry);
Targets.emplace_back(
From, To, BI.MispredictedCount, BI.Count, I - Range.first);
}
@ -304,8 +306,8 @@ IndirectCallPromotion::getCallTargets(
auto Last = Targets.end();
auto Result = First;
while (++First != Last) {
auto &A = *Result;
const auto &B = *First;
Callsite &A = *Result;
const Callsite &B = *First;
if (A.To.Sym && B.To.Sym && A.To.Sym == B.To.Sym) {
A.JTIndices.insert(A.JTIndices.end(), B.JTIndices.begin(),
B.JTIndices.end());
@ -330,7 +332,7 @@ IndirectCallPromotion::getCallTargets(
auto ICSP =
BC.MIB->tryGetAnnotationAs<IndirectCallSiteProfile>(Inst, "CallProfile");
if (ICSP) {
for (const auto &CSP : ICSP.get()) {
for (const IndirectCallProfile &CSP : ICSP.get()) {
Callsite Site(BF, CSP);
if (Site.isValid())
Targets.emplace_back(std::move(Site));
@ -363,7 +365,7 @@ IndirectCallPromotion::getCallTargets(
if (BF.getJumpTable(Inst)) {
uint64_t TotalCount = 0;
uint64_t TotalMispreds = 0;
for (const auto &S : Targets) {
for (const Callsite &S : Targets) {
TotalCount += S.Branches;
TotalMispreds += S.Mispreds;
}
@ -375,7 +377,7 @@ IndirectCallPromotion::getCallTargets(
<< ", Mispreds = " << TotalMispreds << "\n";
size_t I = 0;
for (const auto &S : Targets) {
for (const Callsite &S : Targets) {
dbgs () << "Count[" << I << "] = " << S.Branches << ", "
<< format("%.1f", (100.0*S.Branches)/TotalCount) << ", "
<< "Mispreds[" << I << "] = " << S.Mispreds << ", "
@ -409,7 +411,7 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(
int64_t DispValue;
const MCExpr *DispExpr;
MutableArrayRef<MCInst> Insts(&BB->front(), &CallInst);
const auto Type = BC.MIB->analyzeIndirectBranch(
const IndirectBranchType Type = BC.MIB->analyzeIndirectBranch(
CallInst, Insts.begin(), Insts.end(), BC.AsmInfo->getCodePointerSize(),
MemLocInstr, BaseReg, IndexReg, DispValue, DispExpr, PCRelBaseOut);
@ -448,11 +450,11 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(
DEBUG_VERBOSE(1, dbgs() << "BOLT-INFO: ICP no memory profiling data found\n");
return JumpTableInfoType();
}
auto &MemAccessProfile = ErrorOrMemAccesssProfile.get();
MemoryAccessProfile &MemAccessProfile = ErrorOrMemAccesssProfile.get();
uint64_t ArrayStart;
if (DispExpr) {
auto DispValueOrError =
ErrorOr<uint64_t> DispValueOrError =
BC.getSymbolValue(*BC.MIB->getTargetSymbol(DispExpr));
assert(DispValueOrError && "global symbol needs a value");
ArrayStart = *DispValueOrError;
@ -468,9 +470,9 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(
// into the jump table since there may be multiple addresses that all have the
// same entry.
std::map<MCSymbol *, std::pair<uint64_t, uint64_t>> HotTargetMap;
const auto Range = JT->getEntriesForAddress(ArrayStart);
const std::pair<size_t, size_t> Range = JT->getEntriesForAddress(ArrayStart);
for (const auto &AccessInfo : MemAccessProfile.AddressAccessInfo) {
for (const AddressAccess &AccessInfo : MemAccessProfile.AddressAccessInfo) {
size_t Index;
// Mem data occasionally includes nullprs, ignore them.
if (!AccessInfo.MemoryObject && !AccessInfo.Offset)
@ -507,7 +509,8 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(
<< " in jump table:\n";
JT->print(dbgs());
dbgs() << "HotTargetMap:\n";
for (auto &HT : HotTargetMap) {
for (std::pair<MCSymbol *const, std::pair<uint64_t, uint64_t>> &HT :
HotTargetMap) {
dbgs() << "BOLT-INFO: " << HT.first->getName()
<< " = (count=" << HT.first << ", index=" << HT.second
<< ")\n";
@ -516,7 +519,8 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(
return JumpTableInfoType();
}
auto &HotTarget = HotTargetMap[JT->Entries[Index + Range.first]];
std::pair<uint64_t, uint64_t> &HotTarget =
HotTargetMap[JT->Entries[Index + Range.first]];
HotTarget.first += AccessInfo.Count;
HotTarget.second = Index;
}
@ -534,7 +538,7 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(
LLVM_DEBUG({
dbgs() << "BOLT-INFO: ICP jump table hot targets:\n";
for (const auto &Target : HotTargets) {
for (const std::pair<uint64_t, uint64_t> &Target : HotTargets) {
dbgs() << "BOLT-INFO: Idx = " << Target.second << ", "
<< "Count = " << Target.first << "\n";
}
@ -557,21 +561,17 @@ IndirectCallPromotion::findCallTargetSymbols(
MCInst &CallInst,
MCInst *&TargetFetchInst
) const {
const auto *JT = Function.getJumpTable(CallInst);
const JumpTable *JT = Function.getJumpTable(CallInst);
SymTargetsType SymTargets;
if (JT) {
auto HotTargets = maybeGetHotJumpTableTargets(BC,
Function,
BB,
CallInst,
TargetFetchInst,
JT);
JumpTableInfoType HotTargets = maybeGetHotJumpTableTargets(
BC, Function, BB, CallInst, TargetFetchInst, JT);
if (!HotTargets.empty()) {
auto findTargetsIndex = [&](uint64_t JTIndex) {
for (size_t I = 0; I < Targets.size(); ++I) {
auto &JTIs = Targets[I].JTIndices;
std::vector<uint64_t> &JTIs = Targets[I].JTIndices;
if (std::find(JTIs.begin(), JTIs.end(), JTIndex) != JTIs.end())
return I;
}
@ -598,7 +598,7 @@ IndirectCallPromotion::findCallTargetSymbols(
std::map<const MCSymbol *, uint32_t> IndicesPerTarget;
uint64_t TotalMemAccesses = 0;
for (size_t I = 0; I < HotTargets.size(); ++I) {
const auto TargetIndex = findTargetsIndex(HotTargets[I].second);
const uint64_t TargetIndex = findTargetsIndex(HotTargets[I].second);
++IndicesPerTarget[Targets[TargetIndex].To.Sym];
TotalMemAccesses += HotTargets[I].first;
}
@ -608,7 +608,7 @@ IndirectCallPromotion::findCallTargetSymbols(
: opts::IndirectCallPromotionTopN;
size_t I{0};
for (; I < HotTargets.size(); ++I) {
const auto MemAccesses = HotTargets[I].first;
const uint64_t MemAccesses = HotTargets[I].first;
if (100 * MemAccesses <
TotalMemAccesses * opts::ICPJTTotalPercentThreshold)
break;
@ -619,8 +619,8 @@ IndirectCallPromotion::findCallTargetSymbols(
break;
RemainingMemAccesses -= MemAccesses;
const auto JTIndex = HotTargets[I].second;
auto &Target = Targets[findTargetsIndex(JTIndex)];
const uint64_t JTIndex = HotTargets[I].second;
Callsite &Target = Targets[findTargetsIndex(JTIndex)];
NewTargets.push_back(Target);
std::vector<uint64_t>({JTIndex}).swap(NewTargets.back().JTIndices);
@ -653,10 +653,10 @@ IndirectCallPromotion::findCallTargetSymbols(
}
for (size_t I = 0, TgtIdx = 0; I < N; ++TgtIdx) {
auto &Target = Targets[TgtIdx];
Callsite &Target = Targets[TgtIdx];
assert(Target.To.Sym && "All ICP targets must be to known symbols");
assert(!Target.JTIndices.empty() && "Jump tables must have indices");
for (auto Idx : Target.JTIndices) {
for (uint64_t Idx : Target.JTIndices) {
SymTargets.push_back(std::make_pair(Target.To.Sym, Idx));
++I;
}
@ -709,17 +709,17 @@ IndirectCallPromotion::maybeGetVtableSyms(
++TotalMethodLoadEliminationCandidates;
DEBUG_VERBOSE(1,
DEBUG_VERBOSE(1, {
dbgs() << "BOLT-INFO: ICP found virtual method call in "
<< Function << " at @ " << (&Inst - &BB->front()) << "\n";
dbgs() << "BOLT-INFO: ICP method fetch instructions:\n";
for (auto *Inst : MethodFetchInsns) {
for (MCInst *Inst : MethodFetchInsns) {
BC.printInstruction(dbgs(), *Inst, 0, &Function);
}
if (MethodFetchInsns.back() != &Inst) {
BC.printInstruction(dbgs(), Inst, 0, &Function);
}
);
});
// Try to get value profiling data for the method load instruction.
auto ErrorOrMemAccesssProfile =
@ -730,12 +730,12 @@ IndirectCallPromotion::maybeGetVtableSyms(
dbgs() << "BOLT-INFO: ICP no memory profiling data found\n");
return MethodInfoType();
}
auto &MemAccessProfile = ErrorOrMemAccesssProfile.get();
MemoryAccessProfile &MemAccessProfile = ErrorOrMemAccesssProfile.get();
// Find the vtable that each method belongs to.
std::map<const MCSymbol *, uint64_t> MethodToVtable;
for (const auto &AccessInfo : MemAccessProfile.AddressAccessInfo) {
for (const AddressAccess &AccessInfo : MemAccessProfile.AddressAccessInfo) {
uint64_t Address = AccessInfo.Offset;
if (AccessInfo.MemoryObject) {
Address += AccessInfo.MemoryObject->getAddress();
@ -745,25 +745,25 @@ IndirectCallPromotion::maybeGetVtableSyms(
if (!Address)
continue;
const auto VtableBase = Address - MethodOffset;
const uint64_t VtableBase = Address - MethodOffset;
DEBUG_VERBOSE(1, dbgs() << "BOLT-INFO: ICP vtable = "
<< Twine::utohexstr(VtableBase)
<< "+" << MethodOffset << "/" << AccessInfo.Count
<< "\n");
if (auto MethodAddr = BC.getPointerAtAddress(Address)) {
auto *MethodBD = BC.getBinaryDataAtAddress(MethodAddr.get());
if (ErrorOr<uint64_t> MethodAddr = BC.getPointerAtAddress(Address)) {
BinaryData *MethodBD = BC.getBinaryDataAtAddress(MethodAddr.get());
if (!MethodBD) // skip unknown methods
continue;
auto *MethodSym = MethodBD->getSymbol();
MCSymbol *MethodSym = MethodBD->getSymbol();
MethodToVtable[MethodSym] = VtableBase;
DEBUG_VERBOSE(1,
const auto *Method = BC.getFunctionForSymbol(MethodSym);
DEBUG_VERBOSE(1, {
const BinaryFunction *Method = BC.getFunctionForSymbol(MethodSym);
dbgs() << "BOLT-INFO: ICP found method = "
<< Twine::utohexstr(MethodAddr.get()) << "/"
<< (Method ? Method->getPrintName() : "") << "\n";
);
});
}
}
@ -771,7 +771,7 @@ IndirectCallPromotion::maybeGetVtableSyms(
for (size_t I = 0; I < SymTargets.size(); ++I) {
auto Itr = MethodToVtable.find(SymTargets[I].first);
if (Itr != MethodToVtable.end()) {
if (auto *BD = BC.getBinaryDataContainingAddress(Itr->second)) {
if (BinaryData *BD = BC.getBinaryDataContainingAddress(Itr->second)) {
const uint64_t Addend = Itr->second - BD->getAddress();
VtableSyms.push_back(std::make_pair(BD->getSymbol(), Addend));
continue;
@ -785,8 +785,9 @@ IndirectCallPromotion::maybeGetVtableSyms(
// Make sure the vtable reg is not clobbered by the argument passing code
if (VtableReg != MethodReg) {
for (auto *CurInst = MethodFetchInsns.front(); CurInst < &Inst; ++CurInst) {
const auto &InstrInfo = BC.MII->get(CurInst->getOpcode());
for (MCInst *CurInst = MethodFetchInsns.front(); CurInst < &Inst;
++CurInst) {
const MCInstrDesc &InstrInfo = BC.MII->get(CurInst->getOpcode());
if (InstrInfo.hasDefOfPhysReg(*CurInst, VtableReg, *BC.MRI)) {
return MethodInfoType();
}
@ -816,7 +817,7 @@ IndirectCallPromotion::rewriteCall(
// Remember any pseudo instructions following a tail call. These
// must be preserved and moved to the original block.
std::vector<MCInst> TailInsts;
const auto *TailInst = &CallInst;
const MCInst *TailInst = &CallInst;
if (IsTailCallOrJT) {
while (TailInst + 1 < &(*IndCallBlock->end()) &&
BC.MII->get((TailInst + 1)->getOpcode()).isPseudo()) {
@ -824,11 +825,11 @@ IndirectCallPromotion::rewriteCall(
}
}
auto MovedInst = IndCallBlock->splitInstructions(&CallInst);
std::vector<MCInst> MovedInst = IndCallBlock->splitInstructions(&CallInst);
// Link new BBs to the original input offset of the BB where the indirect
// call site is, so we can map samples recorded in new BBs back to the
// original BB seen in the input binary (if using BAT)
const auto OrigOffset = IndCallBlock->getInputOffset();
const uint32_t OrigOffset = IndCallBlock->getInputOffset();
IndCallBlock->eraseInstructions(MethodFetchInsns.begin(),
MethodFetchInsns.end());
@ -843,11 +844,12 @@ IndirectCallPromotion::rewriteCall(
IndCallBlock->addInstructions(TailInsts.begin(), TailInsts.end());
for (auto Itr = ICPcode.begin() + 1; Itr != ICPcode.end(); ++Itr) {
auto &Sym = Itr->first;
auto &Insts = Itr->second;
MCSymbol *&Sym = Itr->first;
std::vector<MCInst> &Insts = Itr->second;
assert(Sym);
auto TBB = Function.createBasicBlock(OrigOffset, Sym);
for (auto &Inst : Insts) { // sanitize new instructions.
std::unique_ptr<BinaryBasicBlock> TBB =
Function.createBasicBlock(OrigOffset, Sym);
for (MCInst &Inst : Insts) { // sanitize new instructions.
if (BC.MIB->isCall(Inst))
BC.MIB->removeAnnotation(Inst, "CallProfile");
}
@ -878,17 +880,17 @@ BinaryBasicBlock *IndirectCallPromotion::fixCFG(
// Scale indirect call counts to the execution count of the original
// basic block containing the indirect call.
auto TotalCount = IndCallBlock->getKnownExecutionCount();
uint64_t TotalCount = IndCallBlock->getKnownExecutionCount();
uint64_t TotalIndirectBranches = 0;
for (const auto &Target : Targets) {
for (const Callsite &Target : Targets) {
TotalIndirectBranches += Target.Branches;
}
if (TotalIndirectBranches == 0)
TotalIndirectBranches = 1;
std::vector<BinaryBranchInfo> BBI;
std::vector<BinaryBranchInfo> ScaledBBI;
for (const auto &Target : Targets) {
const auto NumEntries = std::max(1UL, Target.JTIndices.size());
for (const Callsite &Target : Targets) {
const size_t NumEntries = std::max(1UL, Target.JTIndices.size());
for (size_t I = 0; I < NumEntries; ++I) {
BBI.push_back(
BinaryBranchInfo{(Target.Branches + NumEntries - 1) / NumEntries,
@ -902,12 +904,12 @@ BinaryBasicBlock *IndirectCallPromotion::fixCFG(
}
if (IsJumpTable) {
auto *NewIndCallBlock = NewBBs.back().get();
BinaryBasicBlock *NewIndCallBlock = NewBBs.back().get();
IndCallBlock->moveAllSuccessorsTo(NewIndCallBlock);
std::vector<MCSymbol*> SymTargets;
for (const auto &Target : Targets) {
const auto NumEntries = std::max(1UL, Target.JTIndices.size());
for (const Callsite &Target : Targets) {
const size_t NumEntries = std::max(1UL, Target.JTIndices.size());
for (size_t I = 0; I < NumEntries; ++I) {
SymTargets.push_back(Target.To.Sym);
}
@ -919,14 +921,16 @@ BinaryBasicBlock *IndirectCallPromotion::fixCFG(
BinaryBasicBlock *SourceBB = I ? NewBBs[I - 1].get() : IndCallBlock;
SourceBB->setExecutionCount(TotalCount);
auto *TargetBB = Function.getBasicBlockForLabel(SymTargets[I]);
BinaryBasicBlock *TargetBB =
Function.getBasicBlockForLabel(SymTargets[I]);
SourceBB->addSuccessor(TargetBB, ScaledBBI[I]); // taken
TotalCount -= ScaledBBI[I].Count;
SourceBB->addSuccessor(NewBBs[I].get(), TotalCount); // fall-through
// Update branch info for the indirect jump.
auto &BranchInfo = NewIndCallBlock->getBranchInfo(*TargetBB);
BinaryBasicBlock::BinaryBranchInfo &BranchInfo =
NewIndCallBlock->getBranchInfo(*TargetBB);
if (BranchInfo.Count > BBI[I].Count)
BranchInfo.Count -= BBI[I].Count;
else
@ -963,7 +967,7 @@ BinaryBasicBlock *IndirectCallPromotion::fixCFG(
for (size_t I = 0; I < NewBBs.size() - Adj; ++I) {
assert(TotalCount <= IndCallBlock->getExecutionCount() ||
TotalCount <= uint64_t(TotalIndirectBranches));
auto ExecCount = ScaledBBI[(I+1)/2].Count;
uint64_t ExecCount = ScaledBBI[(I + 1) / 2].Count;
if (I % 2 == 0) {
if (MergeBlock) {
NewBBs[I]->addSuccessor(MergeBlock, ScaledBBI[(I+1)/2].Count);
@ -1038,10 +1042,10 @@ IndirectCallPromotion::canPromoteCallsite(const BinaryBasicBlock *BB,
} else if (opts::IndirectCallPromotionCallsTopN != 0) {
TopN = opts::IndirectCallPromotionCallsTopN;
}
const auto TrialN = TopN ? std::min(TopN, Targets.size()) : Targets.size();
const size_t TrialN = TopN ? std::min(TopN, Targets.size()) : Targets.size();
if (opts::ICPTopCallsites > 0) {
auto &BC = BB->getFunction()->getBinaryContext();
BinaryContext &BC = BB->getFunction()->getBinaryContext();
if (!BC.MIB->hasAnnotation(Inst, "DoICP"))
return 0;
}
@ -1133,7 +1137,7 @@ IndirectCallPromotion::canPromoteCallsite(const BinaryBasicBlock *BB,
// Filter functions that can have ICP applied (for debugging)
if (!opts::ICPFuncsList.empty()) {
for (auto &Name : opts::ICPFuncsList) {
for (std::string &Name : opts::ICPFuncsList) {
if (BB->getFunction()->hasName(Name))
return N;
}
@ -1149,7 +1153,7 @@ IndirectCallPromotion::printCallsiteInfo(const BinaryBasicBlock *BB,
const std::vector<Callsite> &Targets,
const size_t N,
uint64_t NumCalls) const {
auto &BC = BB->getFunction()->getBinaryContext();
BinaryContext &BC = BB->getFunction()->getBinaryContext();
const bool IsTailCall = BC.MIB->isTailCall(Inst);
const bool IsJumpTable = BB->getFunction()->getJumpTable(Inst);
const auto InstIdx = &Inst - &(*BB->begin());
@ -1161,8 +1165,8 @@ IndirectCallPromotion::printCallsiteInfo(const BinaryBasicBlock *BB,
<< (IsTailCall ? " (tail)" : (IsJumpTable ? " (jump table)" : ""))
<< "\n";
for (size_t I = 0; I < N; I++) {
const auto Frequency = 100.0 * Targets[I].Branches / NumCalls;
const auto MisFrequency = 100.0 * Targets[I].Mispreds / NumCalls;
const double Frequency = 100.0 * Targets[I].Branches / NumCalls;
const double MisFrequency = 100.0 * Targets[I].Mispreds / NumCalls;
outs() << "BOLT-INFO: ";
if (Targets[I].To.Sym)
outs() << Targets[I].To.Sym->getName();
@ -1173,7 +1177,7 @@ IndirectCallPromotion::printCallsiteInfo(const BinaryBasicBlock *BB,
<< ", taken freq = " << format("%.1f", Frequency) << "%"
<< ", mis. freq = " << format("%.1f", MisFrequency) << "%";
bool First = true;
for (auto JTIndex : Targets[I].JTIndices) {
for (uint64_t JTIndex : Targets[I].JTIndices) {
outs() << (First ? ", indices = " : ", ") << JTIndex;
First = false;
}
@ -1221,7 +1225,7 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
// Find all the indirect callsites.
for (auto &BFIt : BFs) {
auto &Function = BFIt.second;
BinaryFunction &Function = BFIt.second;
if (!Function.isSimple() || Function.isIgnored() ||
!Function.hasProfile())
@ -1229,11 +1233,11 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
const bool HasLayout = !Function.layout_empty();
for (auto &BB : Function) {
for (BinaryBasicBlock &BB : Function) {
if (HasLayout && Function.isSplit() && BB.isCold())
continue;
for (auto &Inst : BB) {
for (MCInst &Inst : BB) {
const bool IsJumpTable = Function.getJumpTable(Inst);
const bool HasIndirectCallProfile =
BC.MIB->hasAnnotation(Inst, "CallProfile");
@ -1244,7 +1248,7 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
((HasIndirectCallProfile && !IsJumpTable && OptimizeCalls) ||
(IsJumpTable && OptimizeJumpTables))) {
uint64_t NumCalls = 0;
for (const auto &BInfo : getCallTargets(BB, Inst)) {
for (const Callsite &BInfo : getCallTargets(BB, Inst)) {
NumCalls += BInfo.Branches;
}
IndirectCalls.push_back(
@ -1264,7 +1268,7 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
int64_t MaxCalls = TotalIndirectCalls * TopPerc;
uint64_t LastFreq = std::numeric_limits<uint64_t>::max();
size_t Num = 0;
for (const auto &IC : IndirectCalls) {
for (const IndirectCallsite &IC : IndirectCalls) {
const uint64_t CurFreq = std::get<0>(IC);
// Once we decide to stop, include at least all branches that share the
// same frequency of the last one to avoid non-deterministic behavior
@ -1282,8 +1286,8 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
<< "% of all indirect calls\n";
}
for (auto *FuncPtr : Functions) {
auto &Function = *FuncPtr;
for (BinaryFunction *FuncPtr : Functions) {
BinaryFunction &Function = *FuncPtr;
if (!Function.isSimple() || Function.isIgnored() || !Function.hasProfile())
continue;
@ -1296,7 +1300,7 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
uint64_t FuncTotalIndirectJmps = 0;
std::vector<BinaryBasicBlock *> BBs;
for (auto &BB : Function) {
for (BinaryBasicBlock &BB : Function) {
// Skip indirect calls in cold blocks.
if (!HasLayout || !Function.isSplit() || !BB.isCold()) {
BBs.push_back(&BB);
@ -1307,11 +1311,11 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
DataflowInfoManager Info(BC, Function, RA.get(), nullptr);
while (!BBs.empty()) {
auto *BB = BBs.back();
BinaryBasicBlock *BB = BBs.back();
BBs.pop_back();
for (unsigned Idx = 0; Idx < BB->size(); ++Idx) {
auto &Inst = BB->getInstructionAtIndex(Idx);
MCInst &Inst = BB->getInstructionAtIndex(Idx);
const auto InstIdx = &Inst - &(*BB->begin());
const bool IsTailCall = BC.MIB->isTailCall(Inst);
const bool HasIndirectCallProfile =
@ -1340,11 +1344,11 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
else
++TotalIndirectCallsites;
auto Targets = getCallTargets(*BB, Inst);
std::vector<Callsite> Targets = getCallTargets(*BB, Inst);
// Compute the total number of calls from this particular callsite.
uint64_t NumCalls = 0;
for (const auto &BInfo : Targets) {
for (const Callsite &BInfo : Targets) {
NumCalls += BInfo.Branches;
}
if (!IsJumpTable)
@ -1355,7 +1359,8 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
// If FLAGS regs is alive after this jmp site, do not try
// promoting because we will clobber FLAGS.
if (IsJumpTable) {
auto State = Info.getLivenessAnalysis().getStateBefore(Inst);
ErrorOr<const BitVector &> State =
Info.getLivenessAnalysis().getStateBefore(Inst);
if (!State || (State && (*State)[BC.MIB->getFlagsReg()])) {
if (opts::Verbosity >= 1) {
outs() << "BOLT-INFO: ICP failed in " << Function << " @ "
@ -1385,13 +1390,8 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
// Find MCSymbols or absolute addresses for each call target.
MCInst *TargetFetchInst = nullptr;
const auto SymTargets = findCallTargetSymbols(BC,
Targets,
N,
Function,
BB,
Inst,
TargetFetchInst);
const SymTargetsType SymTargets = findCallTargetSymbols(
BC, Targets, N, Function, BB, Inst, TargetFetchInst);
// findCallTargetSymbols may have changed N if mem profile is available
// for jump tables
@ -1403,7 +1403,7 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
// If we can't resolve any of the target symbols, punt on this callsite.
// TODO: can this ever happen?
if (SymTargets.size() < N) {
const auto LastTarget = SymTargets.size();
const size_t LastTarget = SymTargets.size();
if (opts::Verbosity >= 1) {
outs() << "BOLT-INFO: ICP failed in " << Function << " @ "
<< InstIdx << " in " << BB->getName()
@ -1433,12 +1433,10 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
}
// Generate new promoted call code for this callsite.
auto ICPcode =
MCPlusBuilder::BlocksVectorTy ICPcode =
(IsJumpTable && !opts::ICPJumpTablesByTarget)
? BC.MIB->jumpTablePromotion(Inst,
SymTargets,
MethodInfo.second,
BC.Ctx.get())
? BC.MIB->jumpTablePromotion(Inst, SymTargets,
MethodInfo.second, BC.Ctx.get())
: BC.MIB->indirectCallPromotion(
Inst, SymTargets, MethodInfo.first, MethodInfo.second,
opts::ICPOldCodeSequence, BC.Ctx.get());
@ -1454,11 +1452,11 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
}
LLVM_DEBUG({
auto Offset = Targets[0].From.Addr;
uint64_t Offset = Targets[0].From.Addr;
dbgs() << "BOLT-INFO: ICP indirect call code:\n";
for (const auto &entry : ICPcode) {
const auto &Sym = entry.first;
const auto &Insts = entry.second;
const MCSymbol *const &Sym = entry.first;
const std::vector<MCInst> &Insts = entry.second;
if (Sym) dbgs() << Sym->getName() << ":\n";
Offset = BC.printInstructions(dbgs(),
Insts.begin(),
@ -1469,16 +1467,13 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
});
// Rewrite the CFG with the newly generated ICP code.
auto NewBBs = rewriteCall(BC,
Function,
BB,
Inst,
std::move(ICPcode),
MethodInfo.second);
std::vector<std::unique_ptr<BinaryBasicBlock>> NewBBs = rewriteCall(
BC, Function, BB, Inst, std::move(ICPcode), MethodInfo.second);
// Fix the CFG after inserting the new basic blocks.
auto MergeBlock = fixCFG(BC, Function, BB, IsTailCall, IsJumpTable,
std::move(NewBBs), Targets);
BinaryBasicBlock *MergeBlock =
fixCFG(BC, Function, BB, IsTailCall, IsJumpTable, std::move(NewBBs),
Targets);
// Since the tail of the original block was split off and it may contain
// additional indirect calls, we must add the merge block to the set of

65
bolt/src/Passes/Inliner.cpp
View File

@ -123,7 +123,7 @@ bool inliningEnabled() {
}
bool mustConsider(const llvm::bolt::BinaryFunction &Function) {
for (auto &Name : opts::ForceInlineFunctions) {
for (std::string &Name : opts::ForceInlineFunctions) {
if (Function.hasName(Name))
return true;
}
@ -172,7 +172,7 @@ Inliner::InliningInfo Inliner::getInliningInfo(const BinaryFunction &BF) const {
if (!shouldOptimize(BF))
return INL_NONE;
auto &BC = BF.getBinaryContext();
const BinaryContext &BC = BF.getBinaryContext();
bool DirectSP = false;
bool HasCFI = false;
bool IsLeaf = true;
@ -191,9 +191,9 @@ Inliner::InliningInfo Inliner::getInliningInfo(const BinaryFunction &BF) const {
if (BF.hasJumpTables())
return INL_NONE;
const auto SPReg = BC.MIB->getStackPointer();
for (const auto *BB : BF.layout()) {
for (auto &Inst : *BB) {
const MCPhysReg SPReg = BC.MIB->getStackPointer();
for (const BinaryBasicBlock *BB : BF.layout()) {
for (const MCInst &Inst : *BB) {
// Tail calls are marked as implicitly using the stack pointer and they
// could be inlined.
if (BC.MIB->isTailCall(Inst))
@ -227,7 +227,7 @@ Inliner::InliningInfo Inliner::getInliningInfo(const BinaryFunction &BF) const {
InliningInfo Info(DirectSP ? INL_TAILCALL : INL_ANY);
auto Size = BF.estimateSize();
size_t Size = BF.estimateSize();
Info.SizeAfterInlining = Size;
Info.SizeAfterTailCallInlining = Size;
@ -236,11 +236,11 @@ Inliner::InliningInfo Inliner::getInliningInfo(const BinaryFunction &BF) const {
if (BF.size() == 1) {
// For a regular call the last return instruction could be removed
// (or converted to a branch).
const auto *LastInst = BF.back().getLastNonPseudoInstr();
const MCInst *LastInst = BF.back().getLastNonPseudoInstr();
if (LastInst &&
BC.MIB->isReturn(*LastInst) &&
!BC.MIB->isTailCall(*LastInst)) {
const auto RetInstSize = BC.computeInstructionSize(*LastInst);
const uint64_t RetInstSize = BC.computeInstructionSize(*LastInst);
assert(Size >= RetInstSize);
Info.SizeAfterInlining -= RetInstSize;
}
@ -252,8 +252,8 @@ Inliner::InliningInfo Inliner::getInliningInfo(const BinaryFunction &BF) const {
void
Inliner::findInliningCandidates(BinaryContext &BC) {
for (const auto &BFI : BC.getBinaryFunctions()) {
const auto &Function = BFI.second;
const auto InlInfo = getInliningInfo(Function);
const BinaryFunction &Function = BFI.second;
const InliningInfo InlInfo = getInliningInfo(Function);
if (InlInfo.Type != INL_NONE)
InliningCandidates[&Function] = InlInfo;
}
@ -263,8 +263,8 @@ std::pair<BinaryBasicBlock *, BinaryBasicBlock::iterator>
Inliner::inlineCall(BinaryBasicBlock &CallerBB,
BinaryBasicBlock::iterator CallInst,
const BinaryFunction &Callee) {
auto &CallerFunction = *CallerBB.getFunction();
auto &BC = CallerFunction.getBinaryContext();
BinaryFunction &CallerFunction = *CallerBB.getFunction();
BinaryContext &BC = CallerFunction.getBinaryContext();
auto &MIB = *BC.MIB;
assert(MIB.isCall(*CallInst) && "can only inline a call or a tail call");
@ -274,10 +274,10 @@ Inliner::inlineCall(BinaryBasicBlock &CallerBB,
"cannot inline function with jump table(s)");
// Get information about the call site.
const auto CSIsInvoke = BC.MIB->isInvoke(*CallInst);
const auto CSIsTailCall = BC.MIB->isTailCall(*CallInst);
const auto CSGNUArgsSize = BC.MIB->getGnuArgsSize(*CallInst);
const auto CSEHInfo = BC.MIB->getEHInfo(*CallInst);
const bool CSIsInvoke = BC.MIB->isInvoke(*CallInst);
const bool CSIsTailCall = BC.MIB->isTailCall(*CallInst);
const int64_t CSGNUArgsSize = BC.MIB->getGnuArgsSize(*CallInst);
const Optional<MCPlus::MCLandingPad> CSEHInfo = BC.MIB->getEHInfo(*CallInst);
// Split basic block at the call site if there will be more incoming edges
// coming from the callee.
@ -305,20 +305,20 @@ Inliner::inlineCall(BinaryBasicBlock &CallerBB,
auto InsertII = FirstInlinedBB->eraseInstruction(CallInst);
double ProfileRatio = 0;
if (auto CalleeExecCount = Callee.getKnownExecutionCount()) {
if (uint64_t CalleeExecCount = Callee.getKnownExecutionCount()) {
ProfileRatio =
(double) FirstInlinedBB->getKnownExecutionCount() / CalleeExecCount;
}
// Save execution count of the first block as we don't want it to change
// later due to profile adjustment rounding errors.
const auto FirstInlinedBBCount = FirstInlinedBB->getKnownExecutionCount();
const uint64_t FirstInlinedBBCount = FirstInlinedBB->getKnownExecutionCount();
// Copy basic blocks and maintain a map from their origin.
std::unordered_map<const BinaryBasicBlock *, BinaryBasicBlock *> InlinedBBMap;
InlinedBBMap[&Callee.front()] = FirstInlinedBB;
for (auto BBI = std::next(Callee.begin()); BBI != Callee.end(); ++BBI) {
auto *InlinedBB = CallerFunction.addBasicBlock(0);
BinaryBasicBlock *InlinedBB = CallerFunction.addBasicBlock(0);
InlinedBBMap[&*BBI] = InlinedBB;
InlinedBB->setCFIState(FirstInlinedBB->getCFIState());
if (Callee.hasValidProfile()) {
@ -329,14 +329,14 @@ Inliner::inlineCall(BinaryBasicBlock &CallerBB,
}
// Copy over instructions and edges.
for (const auto &BB : Callee) {
auto *InlinedBB = InlinedBBMap[&BB];
for (const BinaryBasicBlock &BB : Callee) {
BinaryBasicBlock *InlinedBB = InlinedBBMap[&BB];
if (InlinedBB != FirstInlinedBB)
InsertII = InlinedBB->begin();
// Copy over instructions making any necessary mods.
for (auto Inst : BB) {
for (MCInst Inst : BB) {
if (MIB.isPseudo(Inst))
continue;
@ -349,7 +349,7 @@ Inliner::inlineCall(BinaryBasicBlock &CallerBB,
if (MIB.isBranch(Inst)) {
assert(!MIB.isIndirectBranch(Inst) &&
"unexpected indirect branch in callee");
const auto *TargetBB =
const BinaryBasicBlock *TargetBB =
Callee.getBasicBlockForLabel(MIB.getTargetSymbol(Inst));
assert(TargetBB && "cannot find target block in callee");
MIB.replaceBranchTarget(Inst, InlinedBBMap[TargetBB]->getLabel(),
@ -434,7 +434,7 @@ Inliner::inlineCall(BinaryBasicBlock &CallerBB,
}
bool Inliner::inlineCallsInFunction(BinaryFunction &Function) {
auto &BC = Function.getBinaryContext();
BinaryContext &BC = Function.getBinaryContext();
std::vector<BinaryBasicBlock *> Blocks(Function.layout().begin(),
Function.layout().end());
std::sort(Blocks.begin(), Blocks.end(),
@ -443,21 +443,22 @@ bool Inliner::inlineCallsInFunction(BinaryFunction &Function) {
});
bool DidInlining = false;
for (auto *BB : Blocks) {
for (BinaryBasicBlock *BB : Blocks) {
for (auto InstIt = BB->begin(); InstIt != BB->end(); ) {
auto &Inst = *InstIt;
MCInst &Inst = *InstIt;
if (!BC.MIB->isCall(Inst) || MCPlus::getNumPrimeOperands(Inst) != 1 ||
!Inst.getOperand(0).isExpr()) {
++InstIt;
continue;
}
const auto *TargetSymbol = BC.MIB->getTargetSymbol(Inst);
const MCSymbol *TargetSymbol = BC.MIB->getTargetSymbol(Inst);
assert(TargetSymbol && "target symbol expected for direct call");
// Don't inline calls to a secondary entry point in a target function.
uint64_t EntryID{0};
auto *TargetFunction = BC.getFunctionForSymbol(TargetSymbol, &EntryID);
BinaryFunction *TargetFunction =
BC.getFunctionForSymbol(TargetSymbol, &EntryID);
if (!TargetFunction || EntryID != 0) {
++InstIt;
continue;
@ -475,7 +476,7 @@ bool Inliner::inlineCallsInFunction(BinaryFunction &Function) {
continue;
}
const auto IsTailCall = BC.MIB->isTailCall(Inst);
const bool IsTailCall = BC.MIB->isTailCall(Inst);
if (!IsTailCall && IInfo->second.Type == INL_TAILCALL) {
++InstIt;
continue;
@ -560,7 +561,7 @@ void Inliner::runOnFunctions(BinaryContext &BC) {
std::vector<BinaryFunction *> ConsideredFunctions;
for (auto &BFI : BC.getBinaryFunctions()) {
auto &Function = BFI.second;
BinaryFunction &Function = BFI.second;
if (!shouldOptimize(Function))
continue;
ConsideredFunctions.push_back(&Function);
@ -569,11 +570,11 @@ void Inliner::runOnFunctions(BinaryContext &BC) {
[](const BinaryFunction *A, const BinaryFunction *B) {
return B->getKnownExecutionCount() < A->getKnownExecutionCount();
});
for (auto *Function : ConsideredFunctions) {
for (BinaryFunction *Function : ConsideredFunctions) {
if (opts::InlineLimit && NumInlinedCallSites >= opts::InlineLimit)
break;
const auto DidInline = inlineCallsInFunction(*Function);
const bool DidInline = inlineCallsInFunction(*Function);
if (DidInline)
Modified.insert(Function);

36
bolt/src/Passes/Instrumentation.cpp
View File

@ -82,7 +82,7 @@ uint32_t Instrumentation::getFunctionNameIndex(const BinaryFunction &Function) {
auto Iter = FuncToStringIdx.find(&Function);
if (Iter != FuncToStringIdx.end())
return Iter->second;
auto Idx = Summary->StringTable.size();
size_t Idx = Summary->StringTable.size();
FuncToStringIdx.emplace(std::make_pair(&Function, Idx));
Summary->StringTable.append(std::string(Function.getOneName()));
Summary->StringTable.append(1, '\0');
@ -193,7 +193,7 @@ BinaryBasicBlock::iterator
insertInstructions(std::vector<MCInst>& Instrs,
BinaryBasicBlock &BB,
BinaryBasicBlock::iterator Iter) {
for (auto &NewInst : Instrs) {
for (MCInst &NewInst : Instrs) {
Iter = BB.insertInstruction(Iter, NewInst);
++Iter;
}
@ -218,7 +218,7 @@ void Instrumentation::instrumentIndirectTarget(BinaryBasicBlock &BB,
BinaryFunction &FromFunction,
uint32_t From) {
auto L = FromFunction.getBinaryContext().scopeLock();
const auto IndCallSiteID = Summary->IndCallDescriptions.size();
const size_t IndCallSiteID = Summary->IndCallDescriptions.size();
createIndCallDescription(FromFunction, From);
BinaryContext &BC = FromFunction.getBinaryContext();
@ -357,7 +357,7 @@ void Instrumentation::instrumentFunction(BinaryContext &BC,
if (Pred)
STOutSet[Pred].insert(BB);
for (auto *SuccBB : BB->successors())
for (BinaryBasicBlock *SuccBB : BB->successors())
Stack.push(std::make_pair(BB, SuccBB));
}
}
@ -378,13 +378,13 @@ void Instrumentation::instrumentFunction(BinaryContext &BC,
}
for (auto BBI = Function.begin(), BBE = Function.end(); BBI != BBE; ++BBI) {
auto &BB{*BBI};
BinaryBasicBlock &BB{*BBI};
bool HasUnconditionalBranch{false};
bool HasJumpTable{false};
bool IsInvokeBlock = InvokeBlocks.count(&BB) > 0;
for (auto I = BB.begin(); I != BB.end(); ++I) {
const auto &Inst = *I;
const MCInst &Inst = *I;
if (!BC.MIB->hasAnnotation(Inst, "Offset"))
continue;
@ -405,7 +405,8 @@ void Instrumentation::instrumentFunction(BinaryContext &BC,
TargetBB ? &Function : BC.getFunctionForSymbol(Target);
if (TargetFunc && BC.MIB->isCall(Inst)) {
if (opts::InstrumentCalls) {
const auto *ForeignBB = TargetFunc->getBasicBlockForLabel(Target);
const BinaryBasicBlock *ForeignBB =
TargetFunc->getBasicBlockForLabel(Target);
if (ForeignBB)
ToOffset = ForeignBB->getInputOffset();
instrumentOneTarget(SplitWorklist, SplitInstrs, I, Function, BB,
@ -432,7 +433,7 @@ void Instrumentation::instrumentFunction(BinaryContext &BC,
}
if (IsJumpTable) {
for (auto &Succ : BB.successors()) {
for (BinaryBasicBlock *&Succ : BB.successors()) {
// Do not instrument edges in the spanning tree
if (STOutSet[&BB].find(&*Succ) != STOutSet[&BB].end()) {
auto L = BC.scopeLock();
@ -460,7 +461,7 @@ void Instrumentation::instrumentFunction(BinaryContext &BC,
// Instrument fallthroughs (when the direct jump instruction is missing)
if (!HasUnconditionalBranch && !HasJumpTable && BB.succ_size() > 0 &&
BB.size() > 0) {
auto *FTBB = BB.getFallthrough();
BinaryBasicBlock *FTBB = BB.getFallthrough();
assert(FTBB && "expected valid fall-through basic block");
auto I = BB.begin();
auto LastInstr = BB.end();
@ -494,7 +495,7 @@ void Instrumentation::instrumentFunction(BinaryContext &BC,
// Instrument spanning tree leaves
if (!opts::ConservativeInstrumentation) {
for (auto BBI = Function.begin(), BBE = Function.end(); BBI != BBE; ++BBI) {
auto &BB{*BBI};
BinaryBasicBlock &BB{*BBI};
if (STOutSet[&BB].size() == 0)
instrumentLeafNode(BC, BB, BB.begin(), IsLeafFunction, *FuncDesc,
BBToID[&BB]);
@ -504,8 +505,9 @@ void Instrumentation::instrumentFunction(BinaryContext &BC,
// Consume list of critical edges: split them and add instrumentation to the
// newly created BBs
auto Iter = SplitInstrs.begin();
for (auto &BBPair : SplitWorklist) {
auto *NewBB = Function.splitEdge(BBPair.first, BBPair.second);
for (std::pair<BinaryBasicBlock *, BinaryBasicBlock *> &BBPair :
SplitWorklist) {
BinaryBasicBlock *NewBB = Function.splitEdge(BBPair.first, BBPair.second);
NewBB->addInstructions(Iter->begin(), Iter->end());
++Iter;
}
@ -518,9 +520,9 @@ void Instrumentation::runOnFunctions(BinaryContext &BC) {
if (!BC.isX86())
return;
const auto Flags = BinarySection::getFlags(/*IsReadOnly=*/false,
/*IsText=*/false,
/*IsAllocatable=*/true);
const unsigned Flags = BinarySection::getFlags(/*IsReadOnly=*/false,
/*IsText=*/false,
/*IsAllocatable=*/true);
BC.registerOrUpdateSection(".bolt.instr.counters", ELF::SHT_PROGBITS, Flags,
nullptr, 0, 1);
@ -633,7 +635,7 @@ void Instrumentation::createAuxiliaryFunctions(BinaryContext &BC) {
}
void Instrumentation::setupRuntimeLibrary(BinaryContext &BC) {
auto FuncDescSize = Summary->getFDSize();
uint32_t FuncDescSize = Summary->getFDSize();
outs() << "BOLT-INSTRUMENTER: Number of indirect call site descriptors: "
<< Summary->IndCallDescriptions.size() << "\n";
@ -662,7 +664,7 @@ void Instrumentation::setupRuntimeLibrary(BinaryContext &BC) {
outs() << "BOLT-INSTRUMENTER: Profile will be saved to file "
<< opts::InstrumentationFilename << "\n";
auto *RtLibrary =
InstrumentationRuntimeLibrary *RtLibrary =
static_cast<InstrumentationRuntimeLibrary *>(BC.getRuntimeLibrary());
assert(RtLibrary && "instrumentation runtime library object must be set");
RtLibrary->setSummary(std::move(Summary));

2
bolt/src/Passes/InstrumentationSummary.h
View File

@ -130,7 +130,7 @@ struct InstrumentationSummary {
uint32_t getFDSize() const {
uint32_t FuncDescSize = 0;
for (const auto &Func : FunctionDescriptions) {
for (const FunctionDescription &Func : FunctionDescriptions) {
// A function description consists of containers of different
// descriptions. We use vectors to store them and when serializing them,
// we first output a uint32_t-sized field for the number of elements of

65
bolt/src/Passes/JTFootprintReduction.cpp
View File

@ -42,9 +42,9 @@ void JTFootprintReduction::checkOpportunities(BinaryContext &BC,
DataflowInfoManager &Info) {
std::map<JumpTable *, uint64_t> AllJTs;
for (auto &BB : Function) {
for (auto &Inst : BB) {
auto *JumpTable = Function.getJumpTable(Inst);
for (BinaryBasicBlock &BB : Function) {
for (MCInst &Inst : BB) {
JumpTable *JumpTable = Function.getJumpTable(Inst);
if (!JumpTable)
continue;
@ -58,9 +58,10 @@ void JTFootprintReduction::checkOpportunities(BinaryContext &BC,
uint64_t Scale;
// Try a standard indirect jump matcher
auto IndJmpMatcher = BC.MIB->matchIndJmp(
BC.MIB->matchAnyOperand(), BC.MIB->matchImm(Scale),
BC.MIB->matchReg(), BC.MIB->matchAnyOperand());
std::unique_ptr<MCPlusBuilder::MCInstMatcher> IndJmpMatcher =
BC.MIB->matchIndJmp(BC.MIB->matchAnyOperand(),
BC.MIB->matchImm(Scale), BC.MIB->matchReg(),
BC.MIB->matchAnyOperand());
if (!opts::JTFootprintOnlyPIC &&
IndJmpMatcher->match(*BC.MRI, *BC.MIB,
MutableArrayRef<MCInst>(&*BB.begin(), &Inst + 1),
@ -84,13 +85,16 @@ void JTFootprintReduction::checkOpportunities(BinaryContext &BC,
MCPhysReg BaseReg1;
MCPhysReg BaseReg2;
uint64_t Offset;
auto PICIndJmpMatcher = BC.MIB->matchIndJmp(BC.MIB->matchAdd(
BC.MIB->matchReg(BaseReg1),
BC.MIB->matchLoad(BC.MIB->matchReg(BaseReg2), BC.MIB->matchImm(Scale),
BC.MIB->matchReg(), BC.MIB->matchImm(Offset))));
auto PICBaseAddrMatcher = BC.MIB->matchIndJmp(
BC.MIB->matchAdd(BC.MIB->matchLoadAddr(BC.MIB->matchSymbol()),
BC.MIB->matchAnyOperand()));
std::unique_ptr<MCPlusBuilder::MCInstMatcher> PICIndJmpMatcher =
BC.MIB->matchIndJmp(BC.MIB->matchAdd(
BC.MIB->matchReg(BaseReg1),
BC.MIB->matchLoad(BC.MIB->matchReg(BaseReg2),
BC.MIB->matchImm(Scale), BC.MIB->matchReg(),
BC.MIB->matchImm(Offset))));
std::unique_ptr<MCPlusBuilder::MCInstMatcher> PICBaseAddrMatcher =
BC.MIB->matchIndJmp(
BC.MIB->matchAdd(BC.MIB->matchLoadAddr(BC.MIB->matchSymbol()),
BC.MIB->matchAnyOperand()));
if (!PICIndJmpMatcher->match(
*BC.MRI, *BC.MIB,
MutableArrayRef<MCInst>(&*BB.begin(), &Inst + 1), -1) ||
@ -108,7 +112,7 @@ void JTFootprintReduction::checkOpportunities(BinaryContext &BC,
// Statistics only
for (const auto &JTFreq : AllJTs) {
auto *JT = JTFreq.first;
JumpTable *JT = JTFreq.first;
uint64_t CurScore = JTFreq.second;
TotalJTScore += CurScore;
if (!BlacklistedJTs.count(JT)) {
@ -132,9 +136,10 @@ bool JTFootprintReduction::tryOptimizeNonPIC(
uint64_t Scale;
MCPhysReg Index;
MCOperand Offset;
auto IndJmpMatcher = BC.MIB->matchIndJmp(
BC.MIB->matchAnyOperand(Base), BC.MIB->matchImm(Scale),
BC.MIB->matchReg(Index), BC.MIB->matchAnyOperand(Offset));
std::unique_ptr<MCPlusBuilder::MCInstMatcher> IndJmpMatcher =
BC.MIB->matchIndJmp(BC.MIB->matchAnyOperand(Base),
BC.MIB->matchImm(Scale), BC.MIB->matchReg(Index),
BC.MIB->matchAnyOperand(Offset));
if (!IndJmpMatcher->match(*BC.MRI, *BC.MIB,
MutableArrayRef<MCInst>(&*BB.begin(), &*Inst + 1),
-1)) {
@ -146,10 +151,10 @@ bool JTFootprintReduction::tryOptimizeNonPIC(
Scale = 4;
IndJmpMatcher->annotate(*BC.MIB, "DeleteMe");
auto &LA = Info.getLivenessAnalysis();
LivenessAnalysis &LA = Info.getLivenessAnalysis();
MCPhysReg Reg = LA.scavengeRegAfter(&*Inst);
assert(Reg != 0 && "Register scavenger failed!");
auto RegOp = MCOperand::createReg(Reg);
MCOperand RegOp = MCOperand::createReg(Reg);
SmallVector<MCInst, 4> NewFrag;
BC.MIB->createIJmp32Frag(NewFrag, Base, MCOperand::createImm(Scale),
@ -171,10 +176,12 @@ bool JTFootprintReduction::tryOptimizePIC(
MCPhysReg Index;
MCOperand Offset;
MCOperand JumpTableRef;
auto PICIndJmpMatcher = BC.MIB->matchIndJmp(BC.MIB->matchAdd(
BC.MIB->matchLoadAddr(BC.MIB->matchAnyOperand(JumpTableRef)),
BC.MIB->matchLoad(BC.MIB->matchReg(BaseReg), BC.MIB->matchImm(Scale),
BC.MIB->matchReg(Index), BC.MIB->matchAnyOperand())));
std::unique_ptr<MCPlusBuilder::MCInstMatcher> PICIndJmpMatcher =
BC.MIB->matchIndJmp(BC.MIB->matchAdd(
BC.MIB->matchLoadAddr(BC.MIB->matchAnyOperand(JumpTableRef)),
BC.MIB->matchLoad(BC.MIB->matchReg(BaseReg), BC.MIB->matchImm(Scale),
BC.MIB->matchReg(Index),
BC.MIB->matchAnyOperand())));
if (!PICIndJmpMatcher->match(*BC.MRI, *BC.MIB,
MutableArrayRef<MCInst>(&*BB.begin(), &*Inst + 1),
-1)) {
@ -185,7 +192,7 @@ bool JTFootprintReduction::tryOptimizePIC(
PICIndJmpMatcher->annotate(*BC.MIB, "DeleteMe");
auto RegOp = MCOperand::createReg(BaseReg);
MCOperand RegOp = MCOperand::createReg(BaseReg);
SmallVector<MCInst, 4> NewFrag;
BC.MIB->createIJmp32Frag(NewFrag, MCOperand::createReg(0),
@ -204,18 +211,18 @@ bool JTFootprintReduction::tryOptimizePIC(
void JTFootprintReduction::optimizeFunction(BinaryContext &BC,
BinaryFunction &Function,
DataflowInfoManager &Info) {
for (auto &BB : Function) {
for (BinaryBasicBlock &BB : Function) {
if (!BB.getNumNonPseudos())
continue;
auto IndJmpRI = BB.getLastNonPseudo();
auto IndJmp = std::prev(IndJmpRI.base());
const auto JTAddr = BC.MIB->getJumpTable(*IndJmp);
const uint64_t JTAddr = BC.MIB->getJumpTable(*IndJmp);
if (!JTAddr)
continue;
auto *JumpTable = Function.getJumpTable(*IndJmp);
JumpTable *JumpTable = Function.getJumpTable(*IndJmp);
if (BlacklistedJTs.count(JumpTable))
continue;
@ -231,7 +238,7 @@ void JTFootprintReduction::optimizeFunction(BinaryContext &BC,
if (!Modified.count(&Function))
return;
for (auto &BB : Function) {
for (BinaryBasicBlock &BB : Function) {
for (auto I = BB.begin(); I != BB.end(); ) {
if (BC.MIB->hasAnnotation(*I, "DeleteMe"))
I = BB.eraseInstruction(I);
@ -252,7 +259,7 @@ void JTFootprintReduction::runOnFunctions(BinaryContext &BC) {
RA.reset(new RegAnalysis(BC, &BC.getBinaryFunctions(), &*CG));
}
for (auto &BFIt : BC.getBinaryFunctions()) {
auto &Function = BFIt.second;
BinaryFunction &Function = BFIt.second;
if (!Function.isSimple() || Function.isIgnored())
continue;

10
bolt/src/Passes/LivenessAnalysis.h
View File

@ -61,7 +61,7 @@ public:
this->BC.MIB->getGPRegs(GPRegs, /*IncludeAlias=*/false);
// Ignore the register used for frame pointer even if it is not alive (it
// may be used by CFI which is not represented in our dataflow).
auto FP = BC.MIB->getAliases(BC.MIB->getFramePointer());
BitVector FP = BC.MIB->getAliases(BC.MIB->getFramePointer());
FP.flip();
BV &= GPRegs;
BV &= FP;
@ -105,7 +105,7 @@ protected:
BitVector Next = Cur;
bool IsCall = this->BC.MIB->isCall(Point);
// Kill
auto Written = BitVector(NumRegs, false);
BitVector Written = BitVector(NumRegs, false);
if (!IsCall) {
this->BC.MIB->getWrittenRegs(Point, Written);
} else {
@ -120,7 +120,7 @@ protected:
// If ABI is respected, everything except CSRs should be dead after a
// call
if (opts::AssumeABI) {
auto CSR = BitVector(NumRegs, false);
BitVector CSR = BitVector(NumRegs, false);
BC.MIB->getCalleeSavedRegs(CSR);
CSR.flip();
Written |= CSR;
@ -134,7 +134,7 @@ protected:
if (BC.MIB->isCleanRegXOR(Point))
return Next;
auto Used = BitVector(NumRegs, false);
BitVector Used = BitVector(NumRegs, false);
if (IsCall) {
RA.getInstUsedRegsList(Point, Used, /*GetClobbers*/true);
if (RA.isConservative(Used)) {
@ -142,7 +142,7 @@ protected:
BC.MIB->getDefaultLiveOut(Used);
}
}
const auto InstInfo = BC.MII->get(Point.getOpcode());
const MCInstrDesc InstInfo = BC.MII->get(Point.getOpcode());
for (unsigned I = 0, E = Point.getNumOperands(); I != E; ++I) {
if (!Point.getOperand(I).isReg() || I < InstInfo.getNumDefs())
continue;

85
bolt/src/Passes/LongJmp.cpp
View File

@ -79,8 +79,8 @@ LongJmpPass::createNewStub(BinaryBasicBlock &SourceBB, const MCSymbol *TgtSym,
BinaryFunction &Func = *SourceBB.getFunction();
const BinaryContext &BC = Func.getBinaryContext();
const bool IsCold = SourceBB.isCold();
auto *StubSym = BC.Ctx->createNamedTempSymbol("Stub");
auto StubBB = Func.createBasicBlock(0, StubSym);
MCSymbol *StubSym = BC.Ctx->createNamedTempSymbol("Stub");
std::unique_ptr<BinaryBasicBlock> StubBB = Func.createBasicBlock(0, StubSym);
MCInst Inst;
BC.MIB->createUncondBranch(Inst, TgtSym, BC.Ctx.get());
if (TgtIsFunc)
@ -90,7 +90,7 @@ LongJmpPass::createNewStub(BinaryBasicBlock &SourceBB, const MCSymbol *TgtSym,
// Register this in stubs maps
auto registerInMap = [&](StubGroupsTy &Map) {
auto &StubGroup = Map[TgtSym];
StubGroupTy &StubGroup = Map[TgtSym];
StubGroup.insert(
std::lower_bound(
StubGroup.begin(), StubGroup.end(),
@ -126,23 +126,23 @@ BinaryBasicBlock *LongJmpPass::lookupStubFromGroup(
auto CandidatesIter = StubGroups.find(TgtSym);
if (CandidatesIter == StubGroups.end())
return nullptr;
auto &Candidates = CandidatesIter->second;
const StubGroupTy &Candidates = CandidatesIter->second;
if (Candidates.empty())
return nullptr;
auto Cand = std::lower_bound(
const StubTy *Cand = std::lower_bound(
Candidates.begin(), Candidates.end(), std::make_pair(DotAddress, nullptr),
[&](const std::pair<uint64_t, BinaryBasicBlock *> &LHS,
const std::pair<uint64_t, BinaryBasicBlock *> &RHS) {
return LHS.first < RHS.first;
});
if (Cand != Candidates.begin()) {
auto LeftCand = Cand;
const StubTy *LeftCand = Cand;
--LeftCand;
if (Cand->first - DotAddress >
DotAddress - LeftCand->first)
Cand = LeftCand;
}
auto BitsAvail = BC.MIB->getPCRelEncodingSize(Inst) - 1;
int BitsAvail = BC.MIB->getPCRelEncodingSize(Inst) - 1;
uint64_t Mask = ~((1ULL << BitsAvail) - 1);
uint64_t PCRelTgtAddress = Cand->first;
PCRelTgtAddress = DotAddress > PCRelTgtAddress ? DotAddress - PCRelTgtAddress
@ -188,11 +188,11 @@ LongJmpPass::replaceTargetWithStub(BinaryBasicBlock &BB, MCInst &Inst,
const BinaryFunction &Func = *BB.getFunction();
const BinaryContext &BC = Func.getBinaryContext();
std::unique_ptr<BinaryBasicBlock> NewBB;
auto TgtSym = BC.MIB->getTargetSymbol(Inst);
const MCSymbol *TgtSym = BC.MIB->getTargetSymbol(Inst);
assert (TgtSym && "getTargetSymbol failed");
BinaryBasicBlock::BinaryBranchInfo BI{0, 0};
auto *TgtBB = BB.getSuccessor(TgtSym, BI);
BinaryBasicBlock *TgtBB = BB.getSuccessor(TgtSym, BI);
auto LocalStubsIter = Stubs.find(&Func);
// If already using stub and the stub is from another function, create a local
@ -258,7 +258,7 @@ LongJmpPass::replaceTargetWithStub(BinaryBasicBlock &BB, MCInst &Inst,
void LongJmpPass::updateStubGroups() {
auto update = [&](StubGroupsTy &StubGroups) {
for (auto &KeyVal : StubGroups) {
for (auto &Elem : KeyVal.second) {
for (StubTy &Elem : KeyVal.second) {
Elem.first = BBAddresses[Elem.second];
}
std::sort(KeyVal.second.begin(), KeyVal.second.end(),
@ -282,7 +282,7 @@ void LongJmpPass::tentativeBBLayout(const BinaryFunction &Func) {
uint64_t HotDot = HotAddresses[&Func];
uint64_t ColdDot = ColdAddresses[&Func];
bool Cold{false};
for (auto *BB : Func.layout()) {
for (BinaryBasicBlock *BB : Func.layout()) {
if (Cold || BB->isCold()) {
Cold = true;
BBAddresses[BB] = ColdDot;
@ -297,11 +297,12 @@ void LongJmpPass::tentativeBBLayout(const BinaryFunction &Func) {
uint64_t LongJmpPass::tentativeLayoutRelocColdPart(
const BinaryContext &BC, std::vector<BinaryFunction *> &SortedFunctions,
uint64_t DotAddress) {
for (auto Func : SortedFunctions) {
for (BinaryFunction *Func : SortedFunctions) {
if (!Func->isSplit())
continue;
DotAddress = alignTo(DotAddress, BinaryFunction::MinAlign);
auto Pad = offsetToAlignment(DotAddress, llvm::Align(opts::AlignFunctions));
uint64_t Pad =
offsetToAlignment(DotAddress, llvm::Align(opts::AlignFunctions));
if (Pad <= opts::AlignFunctionsMaxBytes)
DotAddress += Pad;
ColdAddresses[Func] = DotAddress;
@ -321,14 +322,14 @@ uint64_t LongJmpPass::tentativeLayoutRelocMode(
uint32_t LastHotIndex = -1u;
uint32_t CurrentIndex = 0;
if (opts::HotFunctionsAtEnd) {
for (auto *BF : SortedFunctions) {
for (BinaryFunction *BF : SortedFunctions) {
if (BF->hasValidIndex() && LastHotIndex == -1u) {
LastHotIndex = CurrentIndex;
}
++CurrentIndex;
}
} else {
for (auto *BF : SortedFunctions) {
for (BinaryFunction *BF : SortedFunctions) {
if (!BF->hasValidIndex() && LastHotIndex == -1u) {
LastHotIndex = CurrentIndex;
}
@ -339,7 +340,7 @@ uint64_t LongJmpPass::tentativeLayoutRelocMode(
// Hot
CurrentIndex = 0;
bool ColdLayoutDone = false;
for (auto Func : SortedFunctions) {
for (BinaryFunction *Func : SortedFunctions) {
if (!ColdLayoutDone && CurrentIndex >= LastHotIndex) {
DotAddress =
tentativeLayoutRelocColdPart(BC, SortedFunctions, DotAddress);
@ -349,7 +350,8 @@ uint64_t LongJmpPass::tentativeLayoutRelocMode(
}
DotAddress = alignTo(DotAddress, BinaryFunction::MinAlign);
auto Pad = offsetToAlignment(DotAddress, llvm::Align(opts::AlignFunctions));
uint64_t Pad =
offsetToAlignment(DotAddress, llvm::Align(opts::AlignFunctions));
if (Pad <= opts::AlignFunctionsMaxBytes)
DotAddress += Pad;
HotAddresses[Func] = DotAddress;
@ -363,7 +365,7 @@ uint64_t LongJmpPass::tentativeLayoutRelocMode(
++CurrentIndex;
}
// BBs
for (auto Func : SortedFunctions)
for (BinaryFunction *Func : SortedFunctions)
tentativeBBLayout(*Func);
return DotAddress;
@ -375,7 +377,7 @@ void LongJmpPass::tentativeLayout(
uint64_t DotAddress = BC.LayoutStartAddress;
if (!BC.HasRelocations) {
for (auto Func : SortedFunctions) {
for (BinaryFunction *Func : SortedFunctions) {
HotAddresses[Func] = Func->getAddress();
DotAddress = alignTo(DotAddress, ColdFragAlign);
ColdAddresses[Func] = DotAddress;
@ -388,12 +390,12 @@ void LongJmpPass::tentativeLayout(
}
// Relocation mode
auto EstimatedTextSize = tentativeLayoutRelocMode(BC, SortedFunctions, 0);
uint64_t EstimatedTextSize = tentativeLayoutRelocMode(BC, SortedFunctions, 0);
// Initial padding
if (opts::UseOldText && EstimatedTextSize <= BC.OldTextSectionSize) {
DotAddress = BC.OldTextSectionAddress;
auto Pad = offsetToAlignment(DotAddress, llvm::Align(BC.PageAlign));
uint64_t Pad = offsetToAlignment(DotAddress, llvm::Align(BC.PageAlign));
if (Pad + EstimatedTextSize <= BC.OldTextSectionSize) {
DotAddress += Pad;
}
@ -406,8 +408,8 @@ void LongJmpPass::tentativeLayout(
bool LongJmpPass::usesStub(const BinaryFunction &Func,
const MCInst &Inst) const {
auto TgtSym = Func.getBinaryContext().MIB->getTargetSymbol(Inst);
auto *TgtBB = Func.getBasicBlockForLabel(TgtSym);
const MCSymbol *TgtSym = Func.getBinaryContext().MIB->getTargetSymbol(Inst);
const BinaryBasicBlock *TgtBB = Func.getBasicBlockForLabel(TgtSym);
auto Iter = Stubs.find(&Func);
if (Iter != Stubs.end())
return Iter->second.count(TgtBB);
@ -423,12 +425,12 @@ uint64_t LongJmpPass::getSymbolAddress(const BinaryContext &BC,
return Iter->second;
}
uint64_t EntryID{0};
auto *TargetFunc = BC.getFunctionForSymbol(Target, &EntryID);
const BinaryFunction *TargetFunc = BC.getFunctionForSymbol(Target, &EntryID);
auto Iter = HotAddresses.find(TargetFunc);
if (Iter == HotAddresses.end() || (TargetFunc && EntryID)) {
// Look at BinaryContext's resolution for this symbol - this is a symbol not
// mapped to a BinaryFunction
auto ValueOrError = BC.getSymbolValue(*Target);
ErrorOr<uint64_t> ValueOrError = BC.getSymbolValue(*Target);
assert(ValueOrError && "Unrecognized symbol");
return *ValueOrError;
}
@ -438,19 +440,19 @@ uint64_t LongJmpPass::getSymbolAddress(const BinaryContext &BC,
bool LongJmpPass::relaxStub(BinaryBasicBlock &StubBB) {
const BinaryFunction &Func = *StubBB.getFunction();
const BinaryContext &BC = Func.getBinaryContext();
const auto Bits = StubBits[&StubBB];
const int Bits = StubBits[&StubBB];
// Already working with the largest range?
if (Bits == static_cast<int>(BC.AsmInfo->getCodePointerSize() * 8))
return false;
const static auto RangeShortJmp = BC.MIB->getShortJmpEncodingSize();
const static auto RangeSingleInstr = BC.MIB->getUncondBranchEncodingSize();
const static int RangeShortJmp = BC.MIB->getShortJmpEncodingSize();
const static int RangeSingleInstr = BC.MIB->getUncondBranchEncodingSize();
const static uint64_t ShortJmpMask = ~((1ULL << RangeShortJmp) - 1);
const static uint64_t SingleInstrMask =
~((1ULL << (RangeSingleInstr - 1)) - 1);
auto *RealTargetSym = BC.MIB->getTargetSymbol(*StubBB.begin());
auto *TgtBB = Func.getBasicBlockForLabel(RealTargetSym);
const MCSymbol *RealTargetSym = BC.MIB->getTargetSymbol(*StubBB.begin());
const BinaryBasicBlock *TgtBB = Func.getBasicBlockForLabel(RealTargetSym);
uint64_t TgtAddress = getSymbolAddress(BC, RealTargetSym, TgtBB);
uint64_t DotAddress = BBAddresses[&StubBB];
uint64_t PCRelTgtAddress = DotAddress > TgtAddress ? DotAddress - TgtAddress
@ -489,10 +491,10 @@ bool LongJmpPass::needsStub(const BinaryBasicBlock &BB, const MCInst &Inst,
uint64_t DotAddress) const {
const BinaryFunction &Func = *BB.getFunction();
const BinaryContext &BC = Func.getBinaryContext();
auto TgtSym = BC.MIB->getTargetSymbol(Inst);
const MCSymbol *TgtSym = BC.MIB->getTargetSymbol(Inst);
assert (TgtSym && "getTargetSymbol failed");
auto *TgtBB = Func.getBasicBlockForLabel(TgtSym);
const BinaryBasicBlock *TgtBB = Func.getBasicBlockForLabel(TgtSym);
// Check for shared stubs from foreign functions
if (!TgtBB) {
auto SSIter = SharedStubs.find(TgtSym);
@ -501,7 +503,7 @@ bool LongJmpPass::needsStub(const BinaryBasicBlock &BB, const MCInst &Inst,
}
}
auto BitsAvail = BC.MIB->getPCRelEncodingSize(Inst) - 1;
int BitsAvail = BC.MIB->getPCRelEncodingSize(Inst) - 1;
uint64_t Mask = ~((1ULL << BitsAvail) - 1);
uint64_t PCRelTgtAddress = getSymbolAddress(BC, TgtSym, TgtBB);
@ -516,7 +518,7 @@ bool LongJmpPass::relax(BinaryFunction &Func) {
bool Modified{false};
assert(BC.isAArch64() && "Unsupported arch");
constexpr auto InsnSize = 4; // AArch64
constexpr int InsnSize = 4; // AArch64
std::vector<std::pair<BinaryBasicBlock *, std::unique_ptr<BinaryBasicBlock>>>
Insertions;
@ -527,13 +529,13 @@ bool LongJmpPass::relax(BinaryFunction &Func) {
}
// Add necessary stubs for branch targets we know we can't fit in the
// instruction
for (auto &BB : Func) {
for (BinaryBasicBlock &BB : Func) {
uint64_t DotAddress = BBAddresses[&BB];
// Stubs themselves are relaxed on the next loop
if (Stubs[&Func].count(&BB))
continue;
for (auto &Inst : BB) {
for (MCInst &Inst : BB) {
if (BC.MII->get(Inst.getOpcode()).isPseudo())
continue;
@ -555,7 +557,7 @@ bool LongJmpPass::relax(BinaryFunction &Func) {
BinaryBasicBlock *InsertionPoint = &BB;
if (Func.isSimple() && !BC.MIB->isCall(Inst) && FrontierAddress &&
!BB.isCold()) {
auto BitsAvail = BC.MIB->getPCRelEncodingSize(Inst) - 1;
int BitsAvail = BC.MIB->getPCRelEncodingSize(Inst) - 1;
uint64_t Mask = ~((1ULL << BitsAvail) - 1);
assert(FrontierAddress > DotAddress &&
"Hot code should be before the frontier");
@ -579,14 +581,15 @@ bool LongJmpPass::relax(BinaryFunction &Func) {
}
// Relax stubs if necessary
for (auto &BB : Func) {
for (BinaryBasicBlock &BB : Func) {
if (!Stubs[&Func].count(&BB) || !BB.isValid())
continue;
Modified |= relaxStub(BB);
}
for (auto &Elmt : Insertions) {
for (std::pair<BinaryBasicBlock *, std::unique_ptr<BinaryBasicBlock>> &Elmt :
Insertions) {
if (!Elmt.second)
continue;
std::vector<std::unique_ptr<BinaryBasicBlock>> NewBBs;
@ -599,7 +602,7 @@ bool LongJmpPass::relax(BinaryFunction &Func) {
void LongJmpPass::runOnFunctions(BinaryContext &BC) {
outs() << "BOLT-INFO: Starting stub-insertion pass\n";
auto Sorted = BC.getSortedFunctions();
std::vector<BinaryFunction *> Sorted = BC.getSortedFunctions();
bool Modified;
uint32_t Iterations{0};
do {
@ -607,7 +610,7 @@ void LongJmpPass::runOnFunctions(BinaryContext &BC) {
Modified = false;
tentativeLayout(BC, Sorted);
updateStubGroups();
for (auto Func : Sorted) {
for (BinaryFunction *Func : Sorted) {
if (relax(*Func)) {
// Don't ruin non-simple functions, they can't afford to have the layout
// changed.

6
bolt/src/Passes/LongJmp.h
View File

@ -33,9 +33,9 @@ namespace bolt {
class LongJmpPass : public BinaryFunctionPass {
/// Used to implement stub grouping (re-using a stub from one function into
/// another)
using StubGroupsTy =
DenseMap<const MCSymbol *,
SmallVector<std::pair<uint64_t, BinaryBasicBlock *>, 4>>;
using StubTy = std::pair<uint64_t, BinaryBasicBlock *>;
using StubGroupTy = SmallVector<StubTy, 4>;
using StubGroupsTy = DenseMap<const MCSymbol *, StubGroupTy>;
StubGroupsTy HotStubGroups;
StubGroupsTy ColdStubGroups;
DenseMap<const MCSymbol *, BinaryBasicBlock *> SharedStubs;

65
bolt/src/Passes/MCF.cpp
View File

@ -184,7 +184,7 @@ void computeEdgeWeights(BinaryBasicBlock *BB, EdgeWeightMap &EdgeWeights) {
template<class NodeT>
void computeEdgeWeights(BinaryFunction &BF, EdgeWeightMap &EdgeWeights) {
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
computeEdgeWeights<NodeT>(&BB, EdgeWeights);
}
}
@ -192,9 +192,9 @@ void computeEdgeWeights(BinaryFunction &BF, EdgeWeightMap &EdgeWeights) {
/// Make BB count match the sum of all incoming edges. If AllEdges is true,
/// make it match max(SumPredEdges, SumSuccEdges).
void recalculateBBCounts(BinaryFunction &BF, bool AllEdges) {
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
uint64_t TotalPredsEWeight{0};
for (auto Pred : BB.predecessors()) {
for (BinaryBasicBlock *Pred : BB.predecessors()) {
TotalPredsEWeight += Pred->getBranchInfo(BB).Count;
}
@ -206,7 +206,7 @@ void recalculateBBCounts(BinaryFunction &BF, bool AllEdges) {
continue;
uint64_t TotalSuccsEWeight{0};
for (auto &BI : BB.branch_info()) {
for (BinaryBasicBlock::BinaryBranchInfo &BI : BB.branch_info()) {
TotalSuccsEWeight += BI.Count;
}
@ -226,11 +226,11 @@ void recalculateBBCounts(BinaryFunction &BF, bool AllEdges) {
void guessEdgeByRelHotness(BinaryFunction &BF, bool UseSucc,
EdgeWeightMap &PredEdgeWeights,
EdgeWeightMap &SuccEdgeWeights) {
for (auto &BB : BF) {
for (auto Pred : BB.predecessors()) {
for (BinaryBasicBlock &BB : BF) {
for (BinaryBasicBlock *Pred : BB.predecessors()) {
double RelativeExec = PredEdgeWeights[std::make_pair(Pred, &BB)];
RelativeExec *= BB.getExecutionCount();
auto &BI = Pred->getBranchInfo(BB);
BinaryBasicBlock::BinaryBranchInfo &BI = Pred->getBranchInfo(BB);
if (static_cast<uint64_t>(RelativeExec) > BI.Count)
BI.Count = static_cast<uint64_t>(RelativeExec);
}
@ -239,7 +239,7 @@ void guessEdgeByRelHotness(BinaryFunction &BF, bool UseSucc,
continue;
auto BI = BB.branch_info_begin();
for (auto Succ : BB.successors()) {
for (BinaryBasicBlock *Succ : BB.successors()) {
double RelativeExec = SuccEdgeWeights[std::make_pair(&BB, Succ)];
RelativeExec *= BB.getExecutionCount();
if (static_cast<uint64_t>(RelativeExec) > BI->Count)
@ -262,7 +262,7 @@ bool guessPredEdgeCounts(BinaryBasicBlock *BB, ArcSet &GuessedArcs) {
uint64_t TotalPredCount{0};
unsigned NumGuessedEdges{0};
for (auto Pred : BB->predecessors()) {
for (BinaryBasicBlock *Pred : BB->predecessors()) {
if (GuessedArcs.count(std::make_pair(Pred, BB)))
++NumGuessedEdges;
TotalPredCount += Pred->getBranchInfo(*BB).Count;
@ -276,7 +276,7 @@ bool guessPredEdgeCounts(BinaryBasicBlock *BB, ArcSet &GuessedArcs) {
if (Guessed < 0)
Guessed = 0;
for (auto Pred : BB->predecessors()) {
for (BinaryBasicBlock *Pred : BB->predecessors()) {
if (GuessedArcs.count(std::make_pair(Pred, BB)))
continue;
@ -297,7 +297,7 @@ bool guessSuccEdgeCounts(BinaryBasicBlock *BB, ArcSet &GuessedArcs) {
uint64_t TotalSuccCount{0};
unsigned NumGuessedEdges{0};
auto BI = BB->branch_info_begin();
for (auto Succ : BB->successors()) {
for (BinaryBasicBlock *Succ : BB->successors()) {
if (GuessedArcs.count(std::make_pair(BB, Succ)))
++NumGuessedEdges;
TotalSuccCount += BI->Count;
@ -313,7 +313,7 @@ bool guessSuccEdgeCounts(BinaryBasicBlock *BB, ArcSet &GuessedArcs) {
Guessed = 0;
BI = BB->branch_info_begin();
for (auto Succ : BB->successors()) {
for (BinaryBasicBlock *Succ : BB->successors()) {
if (GuessedArcs.count(std::make_pair(BB, Succ))) {
++BI;
continue;
@ -336,24 +336,24 @@ void guessEdgeByIterativeApproach(BinaryFunction &BF) {
do {
Changed = false;
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
if (guessPredEdgeCounts(&BB, KnownArcs)) Changed = true;
if (guessSuccEdgeCounts(&BB, KnownArcs)) Changed = true;
}
} while (Changed);
// Guess count for non-inferred edges
for (auto &BB : BF) {
for (auto Pred : BB.predecessors()) {
for (BinaryBasicBlock &BB : BF) {
for (BinaryBasicBlock *Pred : BB.predecessors()) {
if (KnownArcs.count(std::make_pair(Pred, &BB)))
continue;
auto &BI = Pred->getBranchInfo(BB);
BinaryBasicBlock::BinaryBranchInfo &BI = Pred->getBranchInfo(BB);
BI.Count =
std::min(Pred->getExecutionCount(), BB.getExecutionCount()) / 2;
KnownArcs.insert(std::make_pair(Pred, &BB));
}
auto BI = BB.branch_info_begin();
for (auto Succ : BB.successors()) {
for (BinaryBasicBlock *Succ : BB.successors()) {
if (KnownArcs.count(std::make_pair(&BB, Succ))) {
++BI;
continue;
@ -371,9 +371,9 @@ void guessEdgeByIterativeApproach(BinaryFunction &BF) {
DenseMap<const BinaryBasicBlock *, const BinaryLoop*>
createLoopNestLevelMap(BinaryFunction &BF) {
DenseMap<const BinaryBasicBlock *, const BinaryLoop*> LoopNestLevel;
auto &BLI = BF.getLoopInfo();
const BinaryLoopInfo &BLI = BF.getLoopInfo();
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
LoopNestLevel[&BB] = BLI[&BB];
}
@ -386,8 +386,8 @@ createLoopNestLevelMap(BinaryFunction &BF) {
/// same loop and same loop nesting level.
void equalizeBBCounts(BinaryFunction &BF) {
auto Info = DataflowInfoManager(BF.getBinaryContext(), BF, nullptr, nullptr);
auto &DA = Info.getDominatorAnalysis();
auto &PDA = Info.getPostDominatorAnalysis();
DominatorAnalysis<false> &DA = Info.getDominatorAnalysis();
DominatorAnalysis<true> &PDA = Info.getPostDominatorAnalysis();
auto &InsnToBB = Info.getInsnToBBMap();
// These analyses work at the instruction granularity, but we really only need
// basic block granularity here. So we'll use a set of visited edges to avoid
@ -400,19 +400,20 @@ void equalizeBBCounts(BinaryFunction &BF) {
std::vector<std::vector<BinaryBasicBlock *>> Classes;
BF.calculateLoopInfo();
auto LoopNestLevel = createLoopNestLevelMap(BF);
DenseMap<const BinaryBasicBlock *, const BinaryLoop *> LoopNestLevel =
createLoopNestLevelMap(BF);
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
BBsToEC[&BB] = -1;
}
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
auto I = BB.begin();
if (I == BB.end())
continue;
DA.doForAllDominators(*I, [&](const MCInst &DomInst) {
auto *DomBB = InsnToBB[&DomInst];
BinaryBasicBlock *DomBB = InsnToBB[&DomInst];
if (Visited[DomBB].count(&BB))
return;
Visited[DomBB].insert(&BB);
@ -438,10 +439,10 @@ void equalizeBBCounts(BinaryFunction &BF) {
Classes[BBsToEC[DomBB]].push_back(&BB);
return;
}
auto BBECNum = BBsToEC[&BB];
auto DomEC = Classes[BBsToEC[DomBB]];
auto BBEC = Classes[BBECNum];
for (auto *Block : DomEC) {
signed BBECNum = BBsToEC[&BB];
std::vector<BinaryBasicBlock *> DomEC = Classes[BBsToEC[DomBB]];
std::vector<BinaryBasicBlock *> BBEC = Classes[BBECNum];
for (BinaryBasicBlock *Block : DomEC) {
BBsToEC[Block] = BBECNum;
BBEC.push_back(Block);
}
@ -449,12 +450,12 @@ void equalizeBBCounts(BinaryFunction &BF) {
});
}
for (auto &Class : Classes) {
for (std::vector<BinaryBasicBlock *> &Class : Classes) {
uint64_t Max{0ULL};
for (auto *BB : Class) {
for (BinaryBasicBlock *BB : Class) {
Max = std::max(Max, BB->getExecutionCount());
}
for (auto *BB : Class) {
for (BinaryBasicBlock *BB : Class) {
BB->setExecutionCount(Max);
}
}

10
bolt/src/Passes/PLTCall.cpp
View File

@ -48,7 +48,7 @@ void PLTCall::runOnFunctions(BinaryContext &BC) {
uint64_t NumCallsOptimized = 0;
for (auto &It : BC.getBinaryFunctions()) {
auto &Function = It.second;
BinaryFunction &Function = It.second;
if (!shouldOptimize(Function))
continue;
@ -56,17 +56,17 @@ void PLTCall::runOnFunctions(BinaryContext &BC) {
Function.getExecutionCount() == BinaryFunction::COUNT_NO_PROFILE)
continue;
for (auto *BB : Function.layout()) {
for (BinaryBasicBlock *BB : Function.layout()) {
if (opts::PLT == OT_HOT && !BB->getKnownExecutionCount())
continue;
for (auto &Instr : *BB) {
for (MCInst &Instr : *BB) {
if (!BC.MIB->isCall(Instr))
continue;
const auto *CallSymbol = BC.MIB->getTargetSymbol(Instr);
const MCSymbol *CallSymbol = BC.MIB->getTargetSymbol(Instr);
if (!CallSymbol)
continue;
const auto *CalleeBF = BC.getFunctionForSymbol(CallSymbol);
const BinaryFunction *CalleeBF = BC.getFunctionForSymbol(CallSymbol);
if (!CalleeBF || !CalleeBF->isPLTFunction())
continue;
BC.MIB->convertCallToIndirectCall(Instr,

4
bolt/src/Passes/PatchEntries.cpp
View File

@ -64,7 +64,7 @@ void PatchEntries::runOnFunctions(BinaryContext &BC) {
}
for (auto &BFI : BC.getBinaryFunctions()) {
auto &Function = BFI.second;
BinaryFunction &Function = BFI.second;
// Patch original code only for functions that will be emitted.
if (!BC.shouldEmit(Function))
@ -116,7 +116,7 @@ void PatchEntries::runOnFunctions(BinaryContext &BC) {
continue;
}
for (auto &Patch : PendingPatches) {
for (Patch &Patch : PendingPatches) {
BinaryFunction *PatchFunction =
BC.createInjectedBinaryFunction(
NameResolver::append(Patch.Symbol->getName(), ".org.0"));

34
bolt/src/Passes/PettisAndHansen.cpp
View File

@ -54,17 +54,17 @@ public:
using ClusterArcSet = std::unordered_set<ClusterArc, ClusterArcHash>;
void orderFuncs(const CallGraph &Cg, Cluster *C1, Cluster *C2) {
auto C1head = C1->targets().front();
auto C1tail = C1->targets().back();
auto C2head = C2->targets().front();
auto C2tail = C2->targets().back();
NodeId C1head = C1->targets().front();
NodeId C1tail = C1->targets().back();
NodeId C2head = C2->targets().front();
NodeId C2tail = C2->targets().back();
double C1headC2head = 0;
double C1headC2tail = 0;
double C1tailC2head = 0;
double C1tailC2tail = 0;
for (const auto &Arc : Cg.arcs()) {
for (const Arc &Arc : Cg.arcs()) {
if ((Arc.src() == C1head && Arc.dst() == C2head) ||
(Arc.dst() == C1head && Arc.src() == C2head)) {
C1headC2head += Arc.weight();
@ -123,11 +123,11 @@ std::vector<Cluster> pettisAndHansen(const CallGraph &Cg) {
// Create a std::vector of cluster arcs
for (auto &Arc : Cg.arcs()) {
for (const Arc &Arc : Cg.arcs()) {
if (Arc.weight() == 0) continue;
auto const S = FuncCluster[Arc.src()];
auto const D = FuncCluster[Arc.dst()];
Cluster *const S = FuncCluster[Arc.src()];
Cluster *const D = FuncCluster[Arc.dst()];
// ignore if s or d is nullptr
@ -151,11 +151,11 @@ std::vector<Cluster> pettisAndHansen(const CallGraph &Cg) {
}
);
auto Max = *Maxpos;
ClusterArc Max = *Maxpos;
Carcs.erase(Maxpos);
auto const C1 = Max.C1;
auto const C2 = Max.C2;
Cluster *const C1 = Max.C1;
Cluster *const C2 = Max.C2;
if (C1->size() + C2->size() > MaxClusterSize) continue;
@ -172,14 +172,14 @@ std::vector<Cluster> pettisAndHansen(const CallGraph &Cg) {
// update carcs: merge C1arcs to C2arcs
std::unordered_map<ClusterArc, Cluster *, ClusterArcHash> C2arcs;
for (auto &Carc : Carcs) {
for (const ClusterArc &Carc : Carcs) {
if (Carc.C1 == C2) C2arcs.emplace(Carc, Carc.C2);
if (Carc.C2 == C2) C2arcs.emplace(Carc, Carc.C1);
}
for (auto It : C2arcs) {
auto const C = It.second;
auto const C2arc = It.first;
Cluster *const C = It.second;
ClusterArc const C2arc = It.first;
insertOrInc(C, C1, C2arc.Weight);
Carcs.erase(C2arc);
@ -187,7 +187,7 @@ std::vector<Cluster> pettisAndHansen(const CallGraph &Cg) {
// update FuncCluster
for (auto F : C2->targets()) {
for (NodeId F : C2->targets()) {
FuncCluster[F] = C1;
}
C1->merge(*C2, Max.Weight);
@ -200,10 +200,10 @@ std::vector<Cluster> pettisAndHansen(const CallGraph &Cg) {
std::set<Cluster*> LiveClusters;
std::vector<Cluster> OutClusters;
for (auto Fid : Funcs) {
for (NodeId Fid : Funcs) {
LiveClusters.insert(FuncCluster[Fid]);
}
for (auto C : LiveClusters) {
for (Cluster *C : LiveClusters) {
OutClusters.push_back(std::move(*C));
}

12
bolt/src/Passes/ReachingDefOrUse.h
View File

@ -43,7 +43,7 @@ public:
bool isReachedBy(MCPhysReg Reg, ExprIterator Candidates) {
for (auto I = Candidates; I != this->expr_end(); ++I) {
auto BV = BitVector(this->BC.MRI->getNumRegs(), false);
BitVector BV = BitVector(this->BC.MRI->getNumRegs(), false);
if (Def) {
RA.getInstClobberList(**I, BV);
} else {
@ -74,8 +74,8 @@ protected:
void preflight() {
// Populate our universe of tracked expressions with all instructions
// except pseudos
for (auto &BB : this->Func) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : this->Func) {
for (MCInst &Inst : BB) {
this->Expressions.push_back(&Inst);
this->ExprToIdx[&Inst] = this->NumInstrs++;
}
@ -98,8 +98,8 @@ protected:
/// tracked expression, will be considered to be dead after executing X.
bool doesXKillsY(const MCInst *X, const MCInst *Y) {
// getClobberedRegs for X and Y. If they intersect, return true
auto XClobbers = BitVector(this->BC.MRI->getNumRegs(), false);
auto YClobbers = BitVector(this->BC.MRI->getNumRegs(), false);
BitVector XClobbers = BitVector(this->BC.MRI->getNumRegs(), false);
BitVector YClobbers = BitVector(this->BC.MRI->getNumRegs(), false);
RA.getInstClobberList(*X, XClobbers);
// In defs, write after write -> kills first write
// In uses, write after access (read or write) -> kills access
@ -135,7 +135,7 @@ protected:
}
else {
// Track only instructions relevant to TrackingReg
auto Regs = BitVector(this->BC.MRI->getNumRegs(), false);
BitVector Regs = BitVector(this->BC.MRI->getNumRegs(), false);
if (Def)
RA.getInstClobberList(Point, Regs);
else

4
bolt/src/Passes/ReachingInsns.h
View File

@ -53,8 +53,8 @@ protected:
std::unordered_map<const MCInst *, BinaryBasicBlock *> InsnToBB;
void preflight() {
for (auto &BB : this->Func) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : this->Func) {
for (MCInst &Inst : BB) {
this->Expressions.push_back(&Inst);
this->ExprToIdx[&Inst] = this->NumInstrs++;
InsnToBB[&Inst] = &BB;

16
bolt/src/Passes/RegAnalysis.cpp
View File

@ -75,12 +75,12 @@ RegAnalysis::RegAnalysis(BinaryContext &BC,
// This loop is for computing statistics only
for (auto &MapEntry : *BFs) {
auto *Func = &MapEntry.second;
BinaryFunction *Func = &MapEntry.second;
auto Iter = RegsKilledMap.find(Func);
assert(Iter != RegsKilledMap.end() &&
"Failed to compute all clobbers list");
if (Iter->second.all()) {
auto Count = Func->getExecutionCount();
uint64_t Count = Func->getExecutionCount();
if (Count != BinaryFunction::COUNT_NO_PROFILE)
CountFunctionsAllClobber += Count;
++NumFunctionsAllClobber;
@ -155,14 +155,14 @@ void RegAnalysis::getInstUsedRegsList(const MCInst &Inst, BitVector &RegSet,
return;
}
const auto *TargetSymbol = BC.MIB->getTargetSymbol(Inst);
const MCSymbol *TargetSymbol = BC.MIB->getTargetSymbol(Inst);
// If indirect call, we know nothing
if (TargetSymbol == nullptr) {
beConservative(RegSet);
return;
}
const auto *Function = BC.getFunctionForSymbol(TargetSymbol);
const BinaryFunction *Function = BC.getFunctionForSymbol(TargetSymbol);
if (Function == nullptr) {
// Call to a function without a BinaryFunction object.
// This should be a call to a PLT entry, and since it is a trampoline to
@ -200,8 +200,8 @@ BitVector RegAnalysis::getFunctionUsedRegsList(const BinaryFunction *Func) {
return UsedRegs;
}
for (const auto &BB : *Func) {
for (const auto &Inst : BB) {
for (const BinaryBasicBlock &BB : *Func) {
for (const MCInst &Inst : BB) {
getInstUsedRegsList(Inst, UsedRegs, /*GetClobbers*/false);
if (UsedRegs.all())
return UsedRegs;
@ -219,8 +219,8 @@ BitVector RegAnalysis::getFunctionClobberList(const BinaryFunction *Func) {
return RegsKilled;
}
for (const auto &BB : *Func) {
for (const auto &Inst : BB) {
for (const BinaryBasicBlock &BB : *Func) {
for (const MCInst &Inst : BB) {
getInstClobberList(Inst, RegsKilled);
if (RegsKilled.all())
return RegsKilled;

38
bolt/src/Passes/RegReAssign.cpp
View File

@ -42,14 +42,14 @@ void RegReAssign::swap(BinaryContext &BC, BinaryFunction &Function, MCPhysReg A,
const BitVector &AliasB = BC.MIB->getAliases(B, false);
// Regular instructions
for (auto &BB : Function) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : Function) {
for (MCInst &Inst : BB) {
for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) {
auto &Operand = Inst.getOperand(I);
MCOperand &Operand = Inst.getOperand(I);
if (!Operand.isReg())
continue;
auto Reg = Operand.getReg();
unsigned Reg = Operand.getReg();
if (AliasA.test(Reg)) {
Operand.setReg(BC.MIB->getAliasSized(B, BC.MIB->getRegSize(Reg)));
--StaticBytesSaved;
@ -67,8 +67,8 @@ void RegReAssign::swap(BinaryContext &BC, BinaryFunction &Function, MCPhysReg A,
// CFI
DenseSet<const MCCFIInstruction *> Changed;
for (auto &BB : Function) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : Function) {
for (MCInst &Inst : BB) {
if (!BC.MIB->isCFI(Inst))
continue;
const MCCFIInstruction *CFI = Function.getCFIFor(Inst);
@ -78,7 +78,7 @@ void RegReAssign::swap(BinaryContext &BC, BinaryFunction &Function, MCPhysReg A,
switch (CFI->getOperation()) {
case MCCFIInstruction::OpRegister: {
const auto CFIReg2 = CFI->getRegister2();
const unsigned CFIReg2 = CFI->getRegister2();
const MCPhysReg Reg2 = *BC.MRI->getLLVMRegNum(CFIReg2, /*isEH=*/false);
if (AliasA.test(Reg2)) {
Function.setCFIFor(
@ -141,13 +141,13 @@ void RegReAssign::rankRegisters(BinaryContext &BC, BinaryFunction &Function) {
std::fill(RegScore.begin(), RegScore.end(), 0);
std::fill(RankedRegs.begin(), RankedRegs.end(), 0);
for (auto &BB : Function) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : Function) {
for (MCInst &Inst : BB) {
const bool CannotUseREX = BC.MIB->cannotUseREX(Inst);
const auto &Desc = BC.MII->get(Inst.getOpcode());
const MCInstrDesc &Desc = BC.MII->get(Inst.getOpcode());
// Disallow substituitions involving regs in implicit uses lists
const auto *ImplicitUses = Desc.getImplicitUses();
const MCPhysReg *ImplicitUses = Desc.getImplicitUses();
while (ImplicitUses && *ImplicitUses) {
const size_t RegEC =
BC.MIB->getAliases(*ImplicitUses, false).find_first();
@ -157,7 +157,7 @@ void RegReAssign::rankRegisters(BinaryContext &BC, BinaryFunction &Function) {
}
// Disallow substituitions involving regs in implicit defs lists
const auto *ImplicitDefs = Desc.getImplicitDefs();
const MCPhysReg *ImplicitDefs = Desc.getImplicitDefs();
while (ImplicitDefs && *ImplicitDefs) {
const size_t RegEC =
BC.MIB->getAliases(*ImplicitDefs, false).find_first();
@ -167,14 +167,14 @@ void RegReAssign::rankRegisters(BinaryContext &BC, BinaryFunction &Function) {
}
for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) {
const auto &Operand = Inst.getOperand(I);
const MCOperand &Operand = Inst.getOperand(I);
if (!Operand.isReg())
continue;
if (Desc.getOperandConstraint(I, MCOI::TIED_TO) != -1)
continue;
auto Reg = Operand.getReg();
unsigned Reg = Operand.getReg();
size_t RegEC = BC.MIB->getAliases(Reg, false).find_first();
if (RegEC == 0)
continue;
@ -202,7 +202,7 @@ void RegReAssign::rankRegisters(BinaryContext &BC, BinaryFunction &Function) {
[&](size_t A, size_t B) { return RegScore[A] > RegScore[B]; });
LLVM_DEBUG({
for (auto Reg : RankedRegs) {
for (size_t Reg : RankedRegs) {
if (RegScore[Reg] == 0)
continue;
dbgs() << Reg << " ";
@ -251,9 +251,9 @@ void RegReAssign::aggressivePassOverFunction(BinaryContext &BC,
// -- expensive pass -- determine all regs alive during func start
DataflowInfoManager Info(BC, Function, RA.get(), nullptr);
auto AliveAtStart = *Info.getLivenessAnalysis().getStateAt(
BitVector AliveAtStart = *Info.getLivenessAnalysis().getStateAt(
ProgramPoint::getFirstPointAt(*Function.begin()));
for (auto &BB : Function) {
for (BinaryBasicBlock &BB : Function) {
if (BB.pred_size() == 0)
AliveAtStart |= *Info.getLivenessAnalysis().getStateAt(
ProgramPoint::getFirstPointAt(BB));
@ -337,7 +337,7 @@ bool RegReAssign::conservativePassOverFunction(BinaryContext &BC,
// score / utilization rate
MCPhysReg RBX = 0;
for (int I = ClassicCSR.find_first(); I != -1; I = ClassicCSR.find_next(I)) {
auto ScoreRBX = RegScore[I];
int64_t ScoreRBX = RegScore[I];
if (ScoreRBX <= 0)
continue;
@ -410,7 +410,7 @@ void RegReAssign::runOnFunctions(BinaryContext &BC) {
setupConservativePass(BC, BC.getBinaryFunctions());
for (auto &I : BC.getBinaryFunctions()) {
auto &Function = I.second;
BinaryFunction &Function = I.second;
if (!Function.isSimple() || Function.isIgnored())
continue;

73
bolt/src/Passes/ReorderAlgorithm.cpp
View File

@ -86,7 +86,7 @@ void ClusterAlgorithm::computeClusterAverageFrequency(const BinaryContext &BC) {
for (uint32_t I = 0, E = Clusters.size(); I < E; ++I) {
double Freq = 0.0;
uint64_t ClusterSize = 0;
for (auto BB : Clusters[I]) {
for (BinaryBasicBlock *BB : Clusters[I]) {
if (BB->getNumNonPseudos() > 0) {
Freq += BB->getExecutionCount();
// Estimate the size of a block in bytes at run time
@ -104,8 +104,8 @@ void ClusterAlgorithm::printClusters() const {
if (AvgFreq.size() == Clusters.size())
errs() << " (frequency: " << AvgFreq[I] << ")";
errs() << " : ";
auto Sep = "";
for (auto BB : Clusters[I]) {
const char *Sep = "";
for (BinaryBasicBlock *BB : Clusters[I]) {
errs() << Sep << BB->getName();
Sep = ", ";
}
@ -150,16 +150,16 @@ void GreedyClusterAlgorithm::clusterBasicBlocks(const BinaryFunction &BF,
ClusterEdges.resize(BF.layout_size());
// Initialize clusters and edge queue.
for (auto BB : BF.layout()) {
for (BinaryBasicBlock *BB : BF.layout()) {
// Create a cluster for this BB.
uint32_t I = Clusters.size();
Clusters.emplace_back();
auto &Cluster = Clusters.back();
std::vector<BinaryBasicBlock *> &Cluster = Clusters.back();
Cluster.push_back(BB);
BBToClusterMap[BB] = I;
// Populate priority queue with edges.
auto BI = BB->branch_info_begin();
for (auto &I : BB->successors()) {
for (BinaryBasicBlock *&I : BB->successors()) {
assert(BI->Count != BinaryBasicBlock::COUNT_NO_PROFILE &&
"attempted reordering blocks of function with no profile data");
Queue.emplace_back(EdgeTy(BB, I, BI->Count));
@ -171,11 +171,11 @@ void GreedyClusterAlgorithm::clusterBasicBlocks(const BinaryFunction &BF,
// Grow clusters in a greedy fashion.
while (!Queue.empty()) {
auto E = Queue.back();
EdgeTy E = Queue.back();
Queue.pop_back();
const auto *SrcBB = E.Src;
const auto *DstBB = E.Dst;
const BinaryBasicBlock *SrcBB = E.Src;
const BinaryBasicBlock *DstBB = E.Dst;
LLVM_DEBUG(dbgs() << "Popped edge "; E.print(dbgs()); dbgs() << "\n");
@ -197,12 +197,12 @@ void GreedyClusterAlgorithm::clusterBasicBlocks(const BinaryFunction &BF,
continue;
}
auto &ClusterA = Clusters[I];
auto &ClusterB = Clusters[J];
std::vector<BinaryBasicBlock *> &ClusterA = Clusters[I];
std::vector<BinaryBasicBlock *> &ClusterB = Clusters[J];
if (areClustersCompatible(ClusterA, ClusterB, E)) {
// Case 3: SrcBB is at the end of a cluster and DstBB is at the start,
// allowing us to merge two clusters.
for (auto BB : ClusterB)
for (BinaryBasicBlock *BB : ClusterB)
BBToClusterMap[BB] = I;
ClusterA.insert(ClusterA.end(), ClusterB.begin(), ClusterB.end());
ClusterB.clear();
@ -246,7 +246,7 @@ void PHGreedyClusterAlgorithm::initQueue(
// source/destination. This helps to keep original block order for blocks
// when optimal order cannot be deducted from a profile.
if (A.Count == B.Count) {
const auto SrcOrder = BF.getOriginalLayoutRelativeOrder(A.Src, B.Src);
const signed SrcOrder = BF.getOriginalLayoutRelativeOrder(A.Src, B.Src);
return (SrcOrder != 0)
? SrcOrder > 0
: BF.getOriginalLayoutRelativeOrder(A.Dst, B.Dst) > 0;
@ -333,7 +333,7 @@ void MinBranchGreedyClusterAlgorithm::adjustQueue(
// source/destination. This helps to keep original block order for blocks
// when optimal order cannot be deduced from a profile.
if (Weight[A] == Weight[B]) {
const auto SrcOrder = BF.getOriginalLayoutRelativeOrder(A.Src, B.Src);
const signed SrcOrder = BF.getOriginalLayoutRelativeOrder(A.Src, B.Src);
return (SrcOrder != 0)
? SrcOrder > 0
: BF.getOriginalLayoutRelativeOrder(A.Dst, B.Dst) > 0;
@ -345,8 +345,8 @@ void MinBranchGreedyClusterAlgorithm::adjustQueue(
// source and destination in the same cluster.
std::vector<EdgeTy> NewQueue;
for (const EdgeTy &E : Queue) {
const auto *SrcBB = E.Src;
const auto *DstBB = E.Dst;
const BinaryBasicBlock *SrcBB = E.Src;
const BinaryBasicBlock *DstBB = E.Dst;
// Case 1: SrcBB and DstBB are the same or DstBB is the entry block. Ignore
// this edge.
@ -358,8 +358,8 @@ void MinBranchGreedyClusterAlgorithm::adjustQueue(
int I = BBToClusterMap[SrcBB];
int J = BBToClusterMap[DstBB];
auto &ClusterA = Clusters[I];
auto &ClusterB = Clusters[J];
std::vector<BinaryBasicBlock *> &ClusterA = Clusters[I];
std::vector<BinaryBasicBlock *> &ClusterB = Clusters[J];
// Case 2: They are already allocated at the same cluster or incompatible
// clusters. Adjust the weights of edges with the same source or
@ -371,14 +371,14 @@ void MinBranchGreedyClusterAlgorithm::adjustQueue(
LLVM_DEBUG(dbgs() << "\tAdjustment: Ignored edge "; E.print(dbgs());
dbgs() << " (src, dst belong to same cluster or incompatible "
"clusters)\n");
for (const auto *SuccBB : SrcBB->successors()) {
for (const BinaryBasicBlock *SuccBB : SrcBB->successors()) {
if (SuccBB == DstBB)
continue;
auto WI = Weight.find(EdgeTy(SrcBB, SuccBB, 0));
assert(WI != Weight.end() && "CFG edge not found in Weight map");
WI->second += (int64_t)E.Count;
}
for (const auto *PredBB : DstBB->predecessors()) {
for (const BinaryBasicBlock *PredBB : DstBB->predecessors()) {
if (PredBB == SrcBB)
continue;
auto WI = Weight.find(EdgeTy(PredBB, DstBB, 0));
@ -413,20 +413,20 @@ void TSPReorderAlgorithm::reorderBasicBlocks(
std::vector<std::vector<uint64_t>> Weight;
std::vector<BinaryBasicBlock *> IndexToBB;
const auto N = BF.layout_size();
const size_t N = BF.layout_size();
assert(N <= std::numeric_limits<uint64_t>::digits &&
"cannot use TSP solution for sizes larger than bits in uint64_t");
// Populating weight map and index map
for (auto *BB : BF.layout()) {
for (BinaryBasicBlock *BB : BF.layout()) {
BB->setLayoutIndex(IndexToBB.size());
IndexToBB.push_back(BB);
}
Weight.resize(N);
for (auto *BB : BF.layout()) {
for (BinaryBasicBlock *BB : BF.layout()) {
auto BI = BB->branch_info_begin();
Weight[BB->getLayoutIndex()].resize(N);
for (auto *SuccBB : BB->successors()) {
for (BinaryBasicBlock *SuccBB : BB->successors()) {
if (BI->Count != BinaryBasicBlock::COUNT_NO_PROFILE)
Weight[BB->getLayoutIndex()][SuccBB->getLayoutIndex()] = BI->Count;
++BI;
@ -435,7 +435,7 @@ void TSPReorderAlgorithm::reorderBasicBlocks(
std::vector<std::vector<int64_t>> DP;
DP.resize(1 << N);
for (auto &Elmt : DP) {
for (std::vector<long> &Elmt : DP) {
Elmt.resize(N, -1);
}
// Start with the entry basic block being allocated with cost zero
@ -505,7 +505,7 @@ void TSPReorderAlgorithm::reorderBasicBlocks(
// Finalize layout with BBs that weren't assigned to the layout using the
// input layout.
for (auto *BB : BF.layout()) {
for (BinaryBasicBlock *BB : BF.layout()) {
if (Visited[BB->getLayoutIndex()] == false)
Order.push_back(BB);
}
@ -535,7 +535,8 @@ void OptimizeBranchReorderAlgorithm::reorderBasicBlocks(
// Cluster basic blocks.
CAlgo->clusterBasicBlocks(BF, /* ComputeEdges = */true);
std::vector<ClusterAlgorithm::ClusterTy> &Clusters = CAlgo->Clusters;
auto &ClusterEdges = CAlgo->ClusterEdges;
std::vector<std::unordered_map<uint32_t, uint64_t>> &ClusterEdges =
CAlgo->ClusterEdges;
// Compute clusters' average frequencies.
CAlgo->computeClusterAverageFrequency(BF.getBinaryContext());
@ -568,7 +569,7 @@ void OptimizeBranchReorderAlgorithm::reorderBasicBlocks(
};
std::priority_queue<uint32_t, std::vector<uint32_t>,
decltype(ClusterComp)> SuccQueue(ClusterComp);
for (auto &Target: ClusterEdges[I]) {
for (std::pair<const uint32_t, uint64_t> &Target : ClusterEdges[I]) {
if (Target.second > 0 && !(Status[Target.first] & STACKED) &&
!Clusters[Target.first].empty()) {
Parent[Target.first] = I;
@ -615,8 +616,8 @@ void OptimizeBranchReorderAlgorithm::reorderBasicBlocks(
if (opts::PrintClusters) {
errs() << "New cluster order: ";
auto Sep = "";
for (auto O : ClusterOrder) {
const char *Sep = "";
for (uint32_t O : ClusterOrder) {
errs() << Sep << O;
Sep = ", ";
}
@ -665,8 +666,8 @@ void OptimizeCacheReorderAlgorithm::reorderBasicBlocks(
if (opts::PrintClusters) {
errs() << "New cluster order: ";
auto Sep = "";
for (auto O : ClusterOrder) {
const char *Sep = "";
for (uint32_t O : ClusterOrder) {
errs() << Sep << O;
Sep = ", ";
}
@ -680,7 +681,7 @@ void OptimizeCacheReorderAlgorithm::reorderBasicBlocks(
// Force zero execution count on clusters that do not meet the cut off
// specified by --cold-threshold.
if (AvgFreq[ClusterIndex] < static_cast<double>(ColdThreshold)) {
for (auto BBPtr : Cluster) {
for (BinaryBasicBlock *BBPtr : Cluster) {
BBPtr->setExecutionCount(0);
}
}
@ -692,7 +693,7 @@ void ReverseReorderAlgorithm::reorderBasicBlocks(
if (BF.layout_empty())
return;
auto FirstBB = *BF.layout_begin();
BinaryBasicBlock *FirstBB = *BF.layout_begin();
Order.push_back(FirstBB);
for (auto RLI = BF.layout_rbegin(); *RLI != FirstBB; ++RLI)
Order.push_back(*RLI);
@ -723,8 +724,8 @@ void RandomClusterReorderAlgorithm::reorderBasicBlocks(
if (opts::PrintClusters) {
errs() << "New cluster order: ";
auto Sep = "";
for (auto O : ClusterOrder) {
const char *Sep = "";
for (uint32_t O : ClusterOrder) {
errs() << Sep << O;
Sep = ", ";
}

93
bolt/src/Passes/ReorderData.cpp
View File

@ -118,7 +118,7 @@ bool filterSymbol(const BinaryData *BD) {
if (!opts::ReorderSymbols.empty()) {
IsValid = false;
for (auto &Name : opts::ReorderSymbols) {
for (const std::string &Name : opts::ReorderSymbols) {
if (BD->hasName(Name)) {
IsValid = true;
break;
@ -130,7 +130,7 @@ bool filterSymbol(const BinaryData *BD) {
return false;
if (!opts::SkipSymbols.empty()) {
for (auto &Name : opts::SkipSymbols) {
for (const std::string &Name : opts::SkipSymbols) {
if (BD->hasName(Name)) {
IsValid = false;
break;
@ -151,7 +151,7 @@ void ReorderData::printOrder(const BinarySection &Section,
uint64_t TotalSize = 0;
bool PrintHeader = false;
while (Begin != End) {
const auto *BD = Begin->first;
const BinaryData *BD = Begin->first;
if (!PrintHeader) {
outs() << "BOLT-INFO: Hot global symbols for "
@ -173,7 +173,7 @@ DataOrder ReorderData::baseOrder(BinaryContext &BC,
const BinarySection &Section) const {
DataOrder Order;
for (auto &Entry : BC.getBinaryDataForSection(Section)) {
auto *BD = Entry.second;
BinaryData *BD = Entry.second;
if (!BD->isAtomic()) // skip sub-symbols
continue;
auto BDCI = BinaryDataCounts.find(BD);
@ -189,21 +189,23 @@ void ReorderData::assignMemData(BinaryContext &BC) {
StringMap<uint64_t> JumpTableCounts;
uint64_t TotalCount{0};
for (auto &BFI : BC.getBinaryFunctions()) {
const auto &BF = BFI.second;
const BinaryFunction &BF = BFI.second;
if (!BF.hasMemoryProfile())
continue;
for (const auto &BB : BF) {
for (const auto &Inst : BB) {
for (const BinaryBasicBlock &BB : BF) {
for (const MCInst &Inst : BB) {
auto ErrorOrMemAccesssProfile =
BC.MIB->tryGetAnnotationAs<MemoryAccessProfile>(
Inst, "MemoryAccessProfile");
if (!ErrorOrMemAccesssProfile)
continue;
const auto &MemAccessProfile = ErrorOrMemAccesssProfile.get();
for (const auto &AccessInfo : MemAccessProfile.AddressAccessInfo) {
if (auto *BD = AccessInfo.MemoryObject) {
const MemoryAccessProfile &MemAccessProfile =
ErrorOrMemAccesssProfile.get();
for (const AddressAccess &AccessInfo :
MemAccessProfile.AddressAccessInfo) {
if (BinaryData *BD = AccessInfo.MemoryObject) {
BinaryDataCounts[BD->getAtomicRoot()] += AccessInfo.Count;
Counts[BD->getSectionName()] += AccessInfo.Count;
if (BD->getAtomicRoot()->isJumpTable()) {
@ -220,13 +222,13 @@ void ReorderData::assignMemData(BinaryContext &BC) {
if (!Counts.empty()) {
outs() << "BOLT-INFO: Memory stats breakdown:\n";
for (auto &Entry : Counts) {
for (StringMapEntry<uint64_t> &Entry : Counts) {
StringRef Section = Entry.first();
const auto Count = Entry.second;
const uint64_t Count = Entry.second;
outs() << "BOLT-INFO: " << Section << " = " << Count
<< format(" (%.1f%%)\n", 100.0*Count/TotalCount);
if (JumpTableCounts.count(Section) != 0) {
const auto JTCount = JumpTableCounts[Section];
const uint64_t JTCount = JumpTableCounts[Section];
outs() << "BOLT-INFO: jump tables = " << JTCount
<< format(" (%.1f%%)\n", 100.0*JTCount/Count);
}
@ -247,19 +249,21 @@ std::pair<DataOrder, unsigned> ReorderData::sortedByFunc(
auto dataUses = [&BC](const BinaryFunction &BF, bool OnlyHot) {
std::set<BinaryData *> Uses;
for (const auto &BB : BF) {
for (const BinaryBasicBlock &BB : BF) {
if (OnlyHot && BB.isCold())
continue;
for (const auto &Inst : BB) {
for (const MCInst &Inst : BB) {
auto ErrorOrMemAccesssProfile =
BC.MIB->tryGetAnnotationAs<MemoryAccessProfile>(
Inst, "MemoryAccessProfile");
if (!ErrorOrMemAccesssProfile)
continue;
const auto &MemAccessProfile = ErrorOrMemAccesssProfile.get();
for (const auto &AccessInfo : MemAccessProfile.AddressAccessInfo) {
const MemoryAccessProfile &MemAccessProfile =
ErrorOrMemAccesssProfile.get();
for (const AddressAccess &AccessInfo :
MemAccessProfile.AddressAccessInfo) {
if (AccessInfo.MemoryObject)
Uses.insert(AccessInfo.MemoryObject);
}
@ -269,9 +273,9 @@ std::pair<DataOrder, unsigned> ReorderData::sortedByFunc(
};
for (auto &Entry : BFs) {
auto &BF = Entry.second;
BinaryFunction &BF = Entry.second;
if (BF.hasValidProfile()) {
for (auto *BD : dataUses(BF, true)) {
for (BinaryData *BD : dataUses(BF, true)) {
if (!BC.getFunctionForSymbol(BD->getSymbol())) {
BDtoFunc[BD->getAtomicRoot()].insert(&BF);
BDtoFuncCount[BD->getAtomicRoot()] += BF.getKnownExecutionCount();
@ -287,11 +291,11 @@ std::pair<DataOrder, unsigned> ReorderData::sortedByFunc(
[&](const DataOrder::value_type &A,
const DataOrder::value_type &B) {
// Total execution counts of functions referencing BD.
const auto ACount = BDtoFuncCount[A.first];
const auto BCount = BDtoFuncCount[B.first];
const uint64_t ACount = BDtoFuncCount[A.first];
const uint64_t BCount = BDtoFuncCount[B.first];
// Weight by number of loads/data size.
const auto AWeight = double(A.second) / A.first->getSize();
const auto BWeight = double(B.second) / B.first->getSize();
const double AWeight = double(A.second) / A.first->getSize();
const double BWeight = double(B.second) / B.first->getSize();
return (ACount > BCount ||
(ACount == BCount &&
(AWeight > BWeight ||
@ -313,15 +317,15 @@ std::pair<DataOrder, unsigned> ReorderData::sortedByCount(
BinaryContext &BC,
const BinarySection &Section
) const {
auto Order = baseOrder(BC, Section);
DataOrder Order = baseOrder(BC, Section);
unsigned SplitPoint = Order.size();
std::sort(Order.begin(), Order.end(),
[](const DataOrder::value_type &A,
const DataOrder::value_type &B) {
// Weight by number of loads/data size.
const auto AWeight = double(A.second) / A.first->getSize();
const auto BWeight = double(B.second) / B.first->getSize();
const double AWeight = double(A.second) / A.first->getSize();
const double BWeight = double(B.second) / B.first->getSize();
return (AWeight > BWeight ||
(AWeight == BWeight &&
(A.first->getSize() < B.first->getSize() ||
@ -361,7 +365,7 @@ void ReorderData::setSectionOrder(BinaryContext &BC,
<< OutputSection.getName() << "\n");
for (; Begin != End; ++Begin) {
auto *BD = Begin->first;
BinaryData *BD = Begin->first;
// we can't move certain symbols because they are screwy, see T25076484.
if (!filterSymbol(BD))
@ -376,7 +380,7 @@ void ReorderData::setSectionOrder(BinaryContext &BC,
break;
}
auto Alignment = std::max(BD->getAlignment(), MinAlignment);
uint16_t Alignment = std::max(BD->getAlignment(), MinAlignment);
Offset = alignTo(Offset, Alignment);
if ((Offset + BD->getSize()) > opts::ReorderDataMaxBytes) {
@ -393,9 +397,11 @@ void ReorderData::setSectionOrder(BinaryContext &BC,
BD->setOutputLocation(OutputSection, Offset);
// reorder sub-symbols
for (auto &SubBD : BC.getSubBinaryData(BD)) {
for (std::pair<const uint64_t, BinaryData *> &SubBD :
BC.getSubBinaryData(BD)) {
if (!SubBD.second->isJumpTable()) {
auto SubOffset = Offset + SubBD.second->getAddress() - BD->getAddress();
uint64_t SubOffset =
Offset + SubBD.second->getAddress() - BD->getAddress();
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: SubBD " << SubBD.second->getName()
<< " @ " << SubOffset << "\n");
SubBD.second->setOutputLocation(OutputSection, SubOffset);
@ -428,15 +434,16 @@ bool ReorderData::markUnmoveableSymbols(BinaryContext &BC,
bool FoundUnmoveable = false;
for (auto Itr = Range.begin(); Itr != Range.end(); ++Itr) {
if (Itr->second->getName().startswith("PG.")) {
auto *Prev = Itr != Range.begin() ? std::prev(Itr)->second : nullptr;
auto *Next = Itr != Range.end() ? std::next(Itr)->second : nullptr;
auto PrevIsPrivate = Prev && isPrivate(Prev);
auto NextIsPrivate = Next && isPrivate(Next);
BinaryData *Prev =
Itr != Range.begin() ? std::prev(Itr)->second : nullptr;
BinaryData *Next = Itr != Range.end() ? std::next(Itr)->second : nullptr;
bool PrevIsPrivate = Prev && isPrivate(Prev);
bool NextIsPrivate = Next && isPrivate(Next);
if (isPrivate(Itr->second) && (PrevIsPrivate || NextIsPrivate))
Itr->second->setIsMoveable(false);
} else {
// check for overlapping symbols.
auto *Next = Itr != Range.end() ? std::next(Itr)->second : nullptr;
BinaryData *Next = Itr != Range.end() ? std::next(Itr)->second : nullptr;
if (Next &&
Itr->second->getEndAddress() != Next->getAddress() &&
Next->containsAddress(Itr->second->getEndAddress())) {
@ -471,16 +478,17 @@ void ReorderData::runOnFunctions(BinaryContext &BC) {
std::vector<BinarySection *> Sections;
for (auto &SectionName : opts::ReorderData) {
for (const std::string &SectionName : opts::ReorderData) {
if (SectionName == "default") {
for (unsigned I = 0; DefaultSections[I]; ++I) {
if (auto Section = BC.getUniqueSectionByName(DefaultSections[I]))
if (ErrorOr<BinarySection &> Section =
BC.getUniqueSectionByName(DefaultSections[I]))
Sections.push_back(&*Section);
}
continue;
}
auto Section = BC.getUniqueSectionByName(SectionName);
ErrorOr<BinarySection &> Section = BC.getUniqueSectionByName(SectionName);
if (!Section) {
outs() << "BOLT-WARNING: Section " << SectionName
<< " not found, skipping.\n";
@ -495,7 +503,7 @@ void ReorderData::runOnFunctions(BinaryContext &BC) {
Sections.push_back(&*Section);
}
for (auto *Section : Sections) {
for (BinarySection *Section : Sections) {
const bool FoundUnmoveable = markUnmoveableSymbols(BC, *Section);
DataOrder Order;
@ -523,14 +531,15 @@ void ReorderData::runOnFunctions(BinaryContext &BC) {
}
// Copy original section to <section name>.cold.
auto &Cold = BC.registerSection(std::string(Section->getName()) + ".cold",
*Section);
BinarySection &Cold = BC.registerSection(
std::string(Section->getName()) + ".cold", *Section);
// Reorder contents of original section.
setSectionOrder(BC, *Section, Order.begin(), SplitPoint);
// This keeps the original data from thinking it has been moved.
for (auto &Entry : BC.getBinaryDataForSection(*Section)) {
for (std::pair<const uint64_t, BinaryData *> &Entry :
BC.getBinaryDataForSection(*Section)) {
if (!Entry.second->isMoved()) {
Entry.second->setSection(Cold);
Entry.second->setOutputSection(Cold);

45
bolt/src/Passes/ReorderFunctions.cpp
View File

@ -124,8 +124,8 @@ void ReorderFunctions::reorder(std::vector<Cluster> &&Clusters,
uint32_t Index = 0;
// Set order of hot functions based on clusters.
for (const auto& Cluster : Clusters) {
for (const auto FuncId : Cluster.targets()) {
for (const Cluster &Cluster : Clusters) {
for (const NodeId FuncId : Cluster.targets()) {
Cg.nodeIdToFunc(FuncId)->setIndex(Index++);
FuncAddr[FuncId] = TotalSize;
TotalSize += Cg.size(FuncId);
@ -161,14 +161,14 @@ void ReorderFunctions::reorder(std::vector<Cluster> &&Clusters,
outs() << "BOLT-INFO: Function reordering page layout\n"
<< "BOLT-INFO: ============== page 0 ==============\n";
}
for (auto& Cluster : Clusters) {
for (Cluster &Cluster : Clusters) {
if (PrintDetailed) {
outs() <<
format("BOLT-INFO: -------- density = %.3lf (%u / %u) --------\n",
Cluster.density(), Cluster.samples(), Cluster.size());
}
for (auto FuncId : Cluster.targets()) {
for (NodeId FuncId : Cluster.targets()) {
if (Cg.samples(FuncId) > 0) {
Hotfuncs++;
@ -179,13 +179,13 @@ void ReorderFunctions::reorder(std::vector<Cluster> &&Clusters,
uint64_t Dist = 0;
uint64_t Calls = 0;
for (auto Dst : Cg.successors(FuncId)) {
for (NodeId Dst : Cg.successors(FuncId)) {
if (FuncId == Dst) // ignore recursive calls in stats
continue;
const auto& Arc = *Cg.findArc(FuncId, Dst);
const Arc &Arc = *Cg.findArc(FuncId, Dst);
const auto D = std::abs(FuncAddr[Arc.dst()] -
(FuncAddr[FuncId] + Arc.avgCallOffset()));
const auto W = Arc.weight();
(FuncAddr[FuncId] + Arc.avgCallOffset()));
const double W = Arc.weight();
if (D < 64 && PrintDetailed && opts::Verbosity > 2) {
outs() << "BOLT-INFO: short (" << D << "B) call:\n"
<< "BOLT-INFO: Src: " << *Cg.nodeIdToFunc(FuncId) << "\n"
@ -218,7 +218,7 @@ void ReorderFunctions::reorder(std::vector<Cluster> &&Clusters,
TotalSize,
Calls ? Dist / Calls : 0)
<< Cg.nodeIdToFunc(FuncId)->getPrintName() << '\n';
const auto NewPage = TotalSize / HugePageSize;
const uint64_t NewPage = TotalSize / HugePageSize;
if (NewPage != CurPage) {
CurPage = NewPage;
outs() <<
@ -308,8 +308,8 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) {
[&](const BinaryFunction *A, const BinaryFunction *B) {
if (A->isIgnored())
return false;
const auto PadA = opts::padFunction(*A);
const auto PadB = opts::padFunction(*B);
const size_t PadA = opts::padFunction(*A);
const size_t PadB = opts::padFunction(*B);
if (!PadA || !PadB) {
if (PadA)
return true;
@ -320,7 +320,7 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) {
(B->hasProfile() ||
(A->getExecutionCount() > B->getExecutionCount()));
});
for (auto *BF : SortedFunctions) {
for (BinaryFunction *BF : SortedFunctions) {
if (BF->hasProfile())
BF->setIndex(Index++);
}
@ -342,15 +342,16 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) {
case RT_USER:
{
uint32_t Index = 0;
for (const auto &Function : readFunctionOrderFile()) {
for (const std::string &Function : readFunctionOrderFile()) {
std::vector<uint64_t> FuncAddrs;
auto *BD = BC.getBinaryDataByName(Function);
BinaryData *BD = BC.getBinaryDataByName(Function);
if (!BD) {
uint32_t LocalID = 1;
while(1) {
// If we can't find the main symbol name, look for alternates.
const auto FuncName = Function + "/" + std::to_string(LocalID);
const std::string FuncName =
Function + "/" + std::to_string(LocalID);
BD = BC.getBinaryDataByName(FuncName);
if (BD)
FuncAddrs.push_back(BD->getAddress());
@ -368,11 +369,11 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) {
continue;
}
for (const auto FuncAddr : FuncAddrs) {
const auto *FuncBD = BC.getBinaryDataAtAddress(FuncAddr);
for (const uint64_t FuncAddr : FuncAddrs) {
const BinaryData *FuncBD = BC.getBinaryDataAtAddress(FuncAddr);
assert(FuncBD);
auto *BF = BC.getFunctionForSymbol(FuncBD->getSymbol());
BinaryFunction *BF = BC.getFunctionForSymbol(FuncBD->getSymbol());
if (!BF) {
errs() << "BOLT-WARNING: Reorder functions: can't find function for "
<< Function << ".\n";
@ -441,7 +442,7 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) {
}
});
for (const auto *Func : SortedFunctions) {
for (const BinaryFunction *Func : SortedFunctions) {
if (!Func->hasValidIndex())
break;
if (Func->isPLTFunction())
@ -452,10 +453,10 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) {
if (LinkSectionsFile) {
const char *Indent = "";
auto AllNames = Func->getNames();
std::vector<StringRef> AllNames = Func->getNames();
std::sort(AllNames.begin(), AllNames.end());
for (auto Name : AllNames) {
const auto SlashPos = Name.find('/');
for (StringRef Name : AllNames) {
const size_t SlashPos = Name.find('/');
if (SlashPos != std::string::npos) {
// Avoid duplicates for local functions.
if (Name.find('/', SlashPos + 1) != std::string::npos)

20
bolt/src/Passes/ReorderUtils.h
View File

@ -33,7 +33,7 @@ public:
}
template <typename F> void forAllAdjacent(const Cluster *C, F Func) {
for (auto I = Bits[C->id()].find_first(); I != -1;
for (int I = Bits[C->id()].find_first(); I != -1;
I = Bits[C->id()].find_next(I)) {
Func(Clusters[I]);
}
@ -46,7 +46,7 @@ public:
Bits[A->id()][A->id()] = false;
Bits[A->id()][B->id()] = false;
Bits[B->id()][A->id()] = false;
for (auto I = Bits[B->id()].find_first(); I != -1;
for (int I = Bits[B->id()].find_first(); I != -1;
I = Bits[B->id()].find_next(I)) {
Bits[I][A->id()] = true;
Bits[I][B->id()] = false;
@ -83,15 +83,15 @@ public:
}
void set(const Cluster *First, const Cluster *Second, ValueType Value) {
const auto Index = index(First, Second);
const size_t Index = index(First, Second);
Cache[Index] = Value;
Valid[Index] = true;
}
void invalidate(const Cluster *C) {
Valid.reset(C->id() * Size, (C->id() + 1) * Size);
for (size_t id = 0; id < Size; id++) {
Valid.reset((id * Size) + C->id());
for (size_t Id = 0; Id < Size; Id++) {
Valid.reset((Id * Size) + C->id());
}
}
@ -125,17 +125,17 @@ public:
}
void set(const Cluster *First, const Cluster *Second, ValueType Value) {
const auto Index = index(First, Second);
const size_t Index = index(First, Second);
Cache[Index] = Value;
Valid[Index] = true;
}
void invalidate(const Cluster *C) {
for (size_t idx = C->id() * Size; idx < (C->id() + 1) * Size; idx++)
Valid[idx] = false;
for (size_t Idx = C->id() * Size; Idx < (C->id() + 1) * Size; Idx++)
Valid[Idx] = false;
for (size_t id = 0; id < Size; id++)
Valid[(id * Size) + C->id()] = false;
for (size_t Id = 0; Id < Size; Id++)
Valid[(Id * Size) + C->id()] = false;
}
private:

31
bolt/src/Passes/RetpolineInsertion.cpp
View File

@ -83,23 +83,24 @@ BinaryFunction *createNewRetpoline(BinaryContext &BC,
const IndirectBranchInfo &BrInfo,
bool R11Available) {
auto &MIB = *BC.MIB;
auto &Ctx = *BC.Ctx.get();
MCContext &Ctx = *BC.Ctx.get();
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Creating a new retpoline function["
<< RetpolineTag << "]\n");
auto *NewRetpoline = BC.createInjectedBinaryFunction(RetpolineTag, true);
BinaryFunction *NewRetpoline =
BC.createInjectedBinaryFunction(RetpolineTag, true);
std::vector<std::unique_ptr<BinaryBasicBlock>> NewBlocks(3);
for (int I = 0; I < 3; I++) {
auto Symbol =
MCSymbol *Symbol =
Ctx.createNamedTempSymbol(Twine(RetpolineTag + "_BB" + to_string(I)));
NewBlocks[I] = NewRetpoline->createBasicBlock(
BinaryBasicBlock::INVALID_OFFSET, Symbol);
NewBlocks[I].get()->setCFIState(0);
}
auto &BB0 = *NewBlocks[0].get();
auto &BB1 = *NewBlocks[1].get();
auto &BB2 = *NewBlocks[2].get();
BinaryBasicBlock &BB0 = *NewBlocks[0].get();
BinaryBasicBlock &BB1 = *NewBlocks[1].get();
BinaryBasicBlock &BB2 = *NewBlocks[2].get();
BB0.addSuccessor(&BB2, 0, 0);
BB1.addSuccessor(&BB1, 0, 0);
@ -137,7 +138,7 @@ BinaryFunction *createNewRetpoline(BinaryContext &BC,
BB2.addInstruction(PushR11);
MCInst LoadCalleeAddrs;
const auto &MemRef = BrInfo.Memory;
const IndirectBranchInfo::MemOpInfo &MemRef = BrInfo.Memory;
MIB.createLoad(LoadCalleeAddrs, MemRef.BaseRegNum, MemRef.ScaleValue,
MemRef.IndexRegNum, MemRef.DispValue, MemRef.DispExpr,
MemRef.SegRegNum, MIB.getX86R11(), 8);
@ -186,7 +187,7 @@ std::string createRetpolineFunctionTag(BinaryContext &BC,
std::string Tag = "__retpoline_mem_";
const auto &MemRef = BrInfo.Memory;
const IndirectBranchInfo::MemOpInfo &MemRef = BrInfo.Memory;
std::string DispExprStr;
if (MemRef.DispExpr) {
@ -216,7 +217,7 @@ std::string createRetpolineFunctionTag(BinaryContext &BC,
BinaryFunction *RetpolineInsertion::getOrCreateRetpoline(
BinaryContext &BC, const IndirectBranchInfo &BrInfo, bool R11Available) {
const auto RetpolineTag =
const std::string RetpolineTag =
createRetpolineFunctionTag(BC, BrInfo, R11Available);
if (CreatedRetpolines.count(RetpolineTag))
@ -234,7 +235,7 @@ void createBranchReplacement(BinaryContext &BC,
auto &MIB = *BC.MIB;
// Load the branch address in r11 if available
if (BrInfo.isMem() && R11Available) {
const auto &MemRef = BrInfo.Memory;
const IndirectBranchInfo::MemOpInfo &MemRef = BrInfo.Memory;
MCInst LoadCalleeAddrs;
MIB.createLoad(LoadCalleeAddrs, MemRef.BaseRegNum, MemRef.ScaleValue,
MemRef.IndexRegNum, MemRef.DispValue, MemRef.DispExpr,
@ -284,10 +285,10 @@ void RetpolineInsertion::runOnFunctions(BinaryContext &BC) {
auto &MIB = *BC.MIB;
uint32_t RetpolinedBranches = 0;
for (auto &It : BC.getBinaryFunctions()) {
auto &Function = It.second;
for (auto &BB : Function) {
BinaryFunction &Function = It.second;
for (BinaryBasicBlock &BB : Function) {
for (auto It = BB.begin(); It != BB.end(); ++It) {
auto &Inst = *It;
MCInst &Inst = *It;
if (!MIB.isIndirectCall(Inst) && !MIB.isIndirectBranch(Inst))
continue;
@ -310,8 +311,8 @@ void RetpolineInsertion::runOnFunctions(BinaryContext &BC) {
// If the instruction addressing pattern uses rsp and the retpoline
// loads the callee address then displacement needs to be updated
if (BrInfo.isMem() && !R11Available) {
auto &MemRef = BrInfo.Memory;
auto Addend = (BrInfo.isJump() || BrInfo.isTailCall()) ? 8 : 16;
IndirectBranchInfo::MemOpInfo &MemRef = BrInfo.Memory;
int Addend = (BrInfo.isJump() || BrInfo.isTailCall()) ? 8 : 16;
if (MemRef.BaseRegNum == MIB.getStackPointer()) {
MemRef.DispValue += Addend;
}

281
bolt/src/Passes/ShrinkWrapping.cpp
View File

@ -40,11 +40,11 @@ void CalleeSavedAnalysis::analyzeSaves() {
BitVector BlacklistedRegs(BC.MRI->getNumRegs(), false);
LLVM_DEBUG(dbgs() << "Checking spill locations\n");
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
LLVM_DEBUG(dbgs() << "\tNow at BB " << BB.getName() << "\n");
const MCInst *Prev = nullptr;
for (auto &Inst : BB) {
if (auto FIE = FA.getFIEFor(Inst)) {
for (MCInst &Inst : BB) {
if (ErrorOr<const FrameIndexEntry &> FIE = FA.getFIEFor(Inst)) {
// Blacklist weird stores we don't understand
if ((!FIE->IsSimple || FIE->StackOffset >= 0) && FIE->IsStore &&
FIE->IsStoreFromReg) {
@ -116,11 +116,11 @@ void CalleeSavedAnalysis::analyzeRestores() {
ReachingDefOrUse</*Def=*/false> &RU = Info.getReachingUses();
// Now compute all restores of these callee-saved regs
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
const MCInst *Prev = nullptr;
for (auto I = BB.rbegin(), E = BB.rend(); I != E; ++I) {
auto &Inst = *I;
if (auto FIE = FA.getFIEFor(Inst)) {
MCInst &Inst = *I;
if (ErrorOr<const FrameIndexEntry &> FIE = FA.getFIEFor(Inst)) {
if (!FIE->IsLoad || !CalleeSaved[FIE->RegOrImm]) {
Prev = &Inst;
continue;
@ -164,8 +164,8 @@ void CalleeSavedAnalysis::analyzeRestores() {
std::vector<MCInst *> CalleeSavedAnalysis::getSavesByReg(uint16_t Reg) {
std::vector<MCInst *> Results;
for (auto &BB : BF) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : BF) {
for (MCInst &Inst : BB) {
if (getSavedReg(Inst) == Reg)
Results.push_back(&Inst);
}
@ -175,8 +175,8 @@ std::vector<MCInst *> CalleeSavedAnalysis::getSavesByReg(uint16_t Reg) {
std::vector<MCInst *> CalleeSavedAnalysis::getRestoresByReg(uint16_t Reg) {
std::vector<MCInst *> Results;
for (auto &BB : BF) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : BF) {
for (MCInst &Inst : BB) {
if (getRestoredReg(Inst) == Reg)
Results.push_back(&Inst);
}
@ -185,8 +185,8 @@ std::vector<MCInst *> CalleeSavedAnalysis::getRestoresByReg(uint16_t Reg) {
}
CalleeSavedAnalysis::~CalleeSavedAnalysis() {
for (auto &BB : BF) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : BF) {
for (MCInst &Inst : BB) {
BC.MIB->removeAnnotation(Inst, getSaveTag());
BC.MIB->removeAnnotation(Inst, getRestoreTag());
}
@ -200,7 +200,7 @@ void StackLayoutModifier::blacklistRegion(int64_t Offset, int64_t Size) {
}
bool StackLayoutModifier::isRegionBlacklisted(int64_t Offset, int64_t Size) {
for (auto Elem : BlacklistedRegions) {
for (std::pair<const int64_t, int64_t> Elem : BlacklistedRegions) {
if (Offset + Size > Elem.first && Offset < Elem.first + Elem.second)
return true;
}
@ -211,7 +211,7 @@ bool StackLayoutModifier::blacklistAllInConflictWith(int64_t Offset,
int64_t Size) {
bool HasConflict = false;
for (auto Iter = AvailableRegions.begin(); Iter != AvailableRegions.end();) {
auto &Elem = *Iter;
std::pair<const int64_t, int64_t> &Elem = *Iter;
if (Offset + Size > Elem.first && Offset < Elem.first + Elem.second &&
(Offset != Elem.first || Size != Elem.second)) {
Iter = AvailableRegions.erase(Iter);
@ -228,7 +228,7 @@ bool StackLayoutModifier::blacklistAllInConflictWith(int64_t Offset,
}
void StackLayoutModifier::checkFramePointerInitialization(MCInst &Point) {
auto &SPT = Info.getStackPointerTracking();
StackPointerTracking &SPT = Info.getStackPointerTracking();
if (!BC.MII->get(Point.getOpcode())
.hasDefOfPhysReg(Point, BC.MIB->getFramePointer(), *BC.MRI))
return;
@ -258,18 +258,18 @@ void StackLayoutModifier::checkFramePointerInitialization(MCInst &Point) {
}
void StackLayoutModifier::checkStackPointerRestore(MCInst &Point) {
auto &SPT = Info.getStackPointerTracking();
StackPointerTracking &SPT = Info.getStackPointerTracking();
if (!BC.MII->get(Point.getOpcode())
.hasDefOfPhysReg(Point, BC.MIB->getStackPointer(), *BC.MRI))
return;
// Check if the definition of SP comes from FP -- in this case, this
// value may need to be updated depending on our stack layout changes
const auto InstInfo = BC.MII->get(Point.getOpcode());
auto NumDefs = InstInfo.getNumDefs();
const MCInstrDesc InstInfo = BC.MII->get(Point.getOpcode());
unsigned NumDefs = InstInfo.getNumDefs();
bool UsesFP{false};
for (unsigned I = NumDefs, E = MCPlus::getNumPrimeOperands(Point);
I < E; ++I) {
auto &Operand = Point.getOperand(I);
MCOperand &Operand = Point.getOperand(I);
if (!Operand.isReg())
continue;
if (Operand.getReg() == BC.MIB->getFramePointer()) {
@ -317,15 +317,15 @@ void StackLayoutModifier::checkStackPointerRestore(MCInst &Point) {
void StackLayoutModifier::classifyStackAccesses() {
// Understand when stack slots are being used non-locally
auto &SRU = Info.getStackReachingUses();
StackReachingUses &SRU = Info.getStackReachingUses();
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
const MCInst *Prev = nullptr;
for (auto I = BB.rbegin(), E = BB.rend(); I != E; ++I) {
auto &Inst = *I;
MCInst &Inst = *I;
checkFramePointerInitialization(Inst);
checkStackPointerRestore(Inst);
auto FIEX = FA.getFIEFor(Inst);
ErrorOr<const FrameIndexEntry &> FIEX = FA.getFIEFor(Inst);
if (!FIEX) {
Prev = &Inst;
continue;
@ -380,8 +380,8 @@ void StackLayoutModifier::classifyCFIs() {
}
};
for (auto &BB : BF.layout()) {
for (auto &Inst : *BB) {
for (BinaryBasicBlock *&BB : BF.layout()) {
for (MCInst &Inst : *BB) {
if (!BC.MIB->isCFI(Inst))
continue;
const MCCFIInstruction *CFI = BF.getCFIFor(Inst);
@ -415,7 +415,7 @@ void StackLayoutModifier::classifyCFIs() {
break;
case MCCFIInstruction::OpRestoreState: {
assert(!CFIStack.empty() && "Corrupt CFI stack");
auto &Elem = CFIStack.top();
std::pair<int64_t, uint16_t> &Elem = CFIStack.top();
CFIStack.pop();
CfaOffset = Elem.first;
CfaReg = Elem.second;
@ -443,7 +443,7 @@ bool StackLayoutModifier::canCollapseRegion(MCInst *DeletedPush) {
if (!IsSimple || !BC.MIB->isPush(*DeletedPush))
return false;
auto FIE = FA.getFIEFor(*DeletedPush);
ErrorOr<const FrameIndexEntry &> FIE = FA.getFIEFor(*DeletedPush);
if (!FIE)
return false;
@ -467,7 +467,7 @@ bool StackLayoutModifier::canCollapseRegion(int64_t RegionAddr) {
}
bool StackLayoutModifier::collapseRegion(MCInst *DeletedPush) {
auto FIE = FA.getFIEFor(*DeletedPush);
ErrorOr<const FrameIndexEntry &> FIE = FA.getFIEFor(*DeletedPush);
if (!FIE)
return false;
int64_t RegionAddr = FIE->StackOffset;
@ -481,10 +481,10 @@ bool StackLayoutModifier::collapseRegion(MCInst *Alloc, int64_t RegionAddr,
return false;
assert(IsInitialized);
auto &SAA = Info.getStackAllocationAnalysis();
StackAllocationAnalysis &SAA = Info.getStackAllocationAnalysis();
for (auto &BB : BF) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : BF) {
for (MCInst &Inst : BB) {
if (!BC.MIB->hasAnnotation(Inst, getSlotTag()))
continue;
auto Slot =
@ -502,7 +502,7 @@ bool StackLayoutModifier::collapseRegion(MCInst *Alloc, int64_t RegionAddr,
scheduleChange(Inst, WorklistItem(WorklistItem::AdjustCFI, RegionSz));
continue;
}
auto FIE = FA.getFIEFor(Inst);
ErrorOr<const FrameIndexEntry &> FIE = FA.getFIEFor(Inst);
if (!FIE) {
if (Slot > RegionAddr)
continue;
@ -537,8 +537,8 @@ bool StackLayoutModifier::collapseRegion(MCInst *Alloc, int64_t RegionAddr,
}
void StackLayoutModifier::setOffsetForCollapsedAccesses(int64_t NewOffset) {
for (auto &BB : BF) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : BF) {
for (MCInst &Inst : BB) {
if (!BC.MIB->hasAnnotation(Inst, "AccessesDeletedPos"))
continue;
BC.MIB->removeAnnotation(Inst, "AccessesDeletedPos");
@ -554,7 +554,7 @@ bool StackLayoutModifier::canInsertRegion(ProgramPoint P) {
if (!IsSimple)
return false;
auto &SPT = Info.getStackPointerTracking();
StackPointerTracking &SPT = Info.getStackPointerTracking();
int64_t RegionAddr = SPT.getStateBefore(P)->first;
if (RegionAddr == SPT.SUPERPOSITION || RegionAddr == SPT.EMPTY)
return false;
@ -575,17 +575,17 @@ bool StackLayoutModifier::insertRegion(ProgramPoint P, int64_t RegionSz) {
return false;
assert(IsInitialized);
auto &SPT = Info.getStackPointerTracking();
StackPointerTracking &SPT = Info.getStackPointerTracking();
// This RegionAddr is slightly different from the one seen in collapseRegion
// This is the value of SP before the allocation the user wants to make.
int64_t RegionAddr = SPT.getStateBefore(P)->first;
if (RegionAddr == SPT.SUPERPOSITION || RegionAddr == SPT.EMPTY)
return false;
auto &DA = Info.getDominatorAnalysis();
DominatorAnalysis<false> &DA = Info.getDominatorAnalysis();
for (auto &BB : BF) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : BF) {
for (MCInst &Inst : BB) {
if (!BC.MIB->hasAnnotation(Inst, getSlotTag()))
continue;
auto Slot =
@ -603,7 +603,7 @@ bool StackLayoutModifier::insertRegion(ProgramPoint P, int64_t RegionSz) {
scheduleChange(Inst, WorklistItem(WorklistItem::AdjustCFI, -RegionSz));
continue;
}
auto FIE = FA.getFIEFor(Inst);
ErrorOr<const FrameIndexEntry &> FIE = FA.getFIEFor(Inst);
if (!FIE) {
if (Slot >= RegionAddr)
continue;
@ -629,9 +629,9 @@ bool StackLayoutModifier::insertRegion(ProgramPoint P, int64_t RegionSz) {
void StackLayoutModifier::performChanges() {
std::set<uint32_t> ModifiedCFIIndices;
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
for (auto I = BB.rbegin(), E = BB.rend(); I != E; ++I) {
auto &Inst = *I;
MCInst &Inst = *I;
if (BC.MIB->hasAnnotation(Inst, "AccessesDeletedPos")) {
assert(BC.MIB->isPop(Inst) || BC.MIB->isPush(Inst));
BC.MIB->removeAnnotation(Inst, "AccessesDeletedPos");
@ -642,7 +642,7 @@ void StackLayoutModifier::performChanges() {
Inst, getTodoTag());
int64_t Adjustment = 0;
WorklistItem::ActionType AdjustmentType = WorklistItem::None;
for (auto &WI : WList) {
for (WorklistItem &WI : WList) {
if (WI.Action == WorklistItem::None)
continue;
assert(WI.Action == WorklistItem::AdjustLoadStoreOffset ||
@ -722,18 +722,18 @@ uint64_t ShrinkWrapping::SpillsMovedPushPopMode = 0;
using BBIterTy = BinaryBasicBlock::iterator;
void ShrinkWrapping::classifyCSRUses() {
auto &DA = Info.getDominatorAnalysis();
auto &SPT = Info.getStackPointerTracking();
DominatorAnalysis<false> &DA = Info.getDominatorAnalysis();
StackPointerTracking &SPT = Info.getStackPointerTracking();
UsesByReg = std::vector<BitVector>(BC.MRI->getNumRegs(),
BitVector(DA.NumInstrs, false));
const BitVector &FPAliases =
BC.MIB->getAliases(BC.MIB->getFramePointer());
for (auto &BB : BF) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : BF) {
for (MCInst &Inst : BB) {
if (BC.MIB->isCFI(Inst))
continue;
auto BV = BitVector(BC.MRI->getNumRegs(), false);
BitVector BV = BitVector(BC.MRI->getNumRegs(), false);
BC.MIB->getTouchedRegs(Inst, BV);
BV &= CSA.CalleeSaved;
for (int I = BV.find_first(); I != -1; I = BV.find_next(I)) {
@ -782,12 +782,12 @@ void ShrinkWrapping::pruneUnwantedCSRs() {
void ShrinkWrapping::computeSaveLocations() {
SavePos = std::vector<SmallSetVector<MCInst *, 4>>(BC.MRI->getNumRegs());
auto &RI = Info.getReachingInsnsBackwards();
auto &DA = Info.getDominatorAnalysis();
auto &SPT = Info.getStackPointerTracking();
ReachingInsns<true> &RI = Info.getReachingInsnsBackwards();
DominatorAnalysis<false> &DA = Info.getDominatorAnalysis();
StackPointerTracking &SPT = Info.getStackPointerTracking();
LLVM_DEBUG(dbgs() << "Checking save/restore possibilities\n");
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
LLVM_DEBUG(dbgs() << "\tNow at BB " << BB.getName() << "\n");
MCInst *First = BB.begin() != BB.end() ? &*BB.begin() : nullptr;
@ -799,7 +799,7 @@ void ShrinkWrapping::computeSaveLocations() {
if (RI.isInLoop(BB))
continue;
const auto SPFP = *SPT.getStateBefore(*First);
const std::pair<int, int> SPFP = *SPT.getStateBefore(*First);
// If we don't know stack state at this point, bail
if ((SPFP.first == SPT.SUPERPOSITION || SPFP.first == SPT.EMPTY) &&
(SPFP.second == SPT.SUPERPOSITION || SPFP.second == SPT.EMPTY))
@ -809,8 +809,8 @@ void ShrinkWrapping::computeSaveLocations() {
if (!CSA.CalleeSaved[I])
continue;
auto BBDominatedUses = BitVector(DA.NumInstrs, false);
for (auto J = UsesByReg[I].find_first(); J > 0;
BitVector BBDominatedUses = BitVector(DA.NumInstrs, false);
for (int J = UsesByReg[I].find_first(); J > 0;
J = UsesByReg[I].find_next(J)) {
if (DA.doesADominateB(*First, J))
BBDominatedUses.set(J);
@ -826,7 +826,7 @@ void ShrinkWrapping::computeSaveLocations() {
SavePos[I].insert(First);
LLVM_DEBUG({
dbgs() << "Dominated uses are:\n";
for (auto J = UsesByReg[I].find_first(); J > 0;
for (int J = UsesByReg[I].find_first(); J > 0;
J = UsesByReg[I].find_next(J)) {
dbgs() << "Idx " << J << ": ";
DA.Expressions[J]->dump();
@ -844,8 +844,8 @@ void ShrinkWrapping::computeSaveLocations() {
if (!CSA.CalleeSaved[I])
continue;
for (auto *Pos : SavePos[I]) {
auto *BB = InsnToBB[Pos];
for (MCInst *Pos : SavePos[I]) {
BinaryBasicBlock *BB = InsnToBB[Pos];
uint64_t Count = BB->getExecutionCount();
if (Count != BinaryBasicBlock::COUNT_NO_PROFILE &&
Count < BestSaveCount[I]) {
@ -862,24 +862,26 @@ void ShrinkWrapping::computeDomOrder() {
Order.push_back(I);
}
auto &DA = Info.getDominatorAnalysis();
DominatorAnalysis<false> &DA = Info.getDominatorAnalysis();
auto &InsnToBB = Info.getInsnToBBMap();
std::sort(Order.begin(), Order.end(), [&](const MCPhysReg &A,
const MCPhysReg &B) {
auto *BBA = BestSavePos[A] ? InsnToBB[BestSavePos[A]] : nullptr;
auto *BBB = BestSavePos[B] ? InsnToBB[BestSavePos[B]] : nullptr;
if (BBA == BBB)
return A < B;
if (!BBA && BBB)
return false;
if (BBA && !BBB)
return true;
if (DA.doesADominateB(*BestSavePos[A], *BestSavePos[B]))
return true;
if (DA.doesADominateB(*BestSavePos[B], *BestSavePos[A]))
return false;
return A < B;
});
std::sort(Order.begin(), Order.end(),
[&](const MCPhysReg &A, const MCPhysReg &B) {
BinaryBasicBlock *BBA =
BestSavePos[A] ? InsnToBB[BestSavePos[A]] : nullptr;
BinaryBasicBlock *BBB =
BestSavePos[B] ? InsnToBB[BestSavePos[B]] : nullptr;
if (BBA == BBB)
return A < B;
if (!BBA && BBB)
return false;
if (BBA && !BBB)
return true;
if (DA.doesADominateB(*BestSavePos[A], *BestSavePos[B]))
return true;
if (DA.doesADominateB(*BestSavePos[B], *BestSavePos[A]))
return false;
return A < B;
});
for (MCPhysReg I = 0, E = BC.MRI->getNumRegs(); I != E; ++I) {
DomOrder[Order[I]] = I;
@ -940,14 +942,14 @@ void ShrinkWrapping::splitFrontierCritEdges(
continue;
if (To[I].empty())
continue;
auto FromBB = From[I];
BinaryBasicBlock *FromBB = From[I];
LLVM_DEBUG(dbgs() << " - Now handling FrontierBB " << FromBB->getName()
<< "\n");
// Split edge for every DestinationBBs
for (size_t DI = 0, DIE = To[I].size(); DI < DIE; ++DI) {
auto DestinationBB = To[I][DI];
BinaryBasicBlock *DestinationBB = To[I][DI];
LLVM_DEBUG(dbgs() << " - Dest : " << DestinationBB->getName() << "\n");
auto *NewBB = Func->splitEdge(FromBB, DestinationBB);
BinaryBasicBlock *NewBB = Func->splitEdge(FromBB, DestinationBB);
// Insert dummy instruction so this BB is never empty (we need this for
// PredictiveStackPointerTracking to work, since it annotates instructions
// and not BBs).
@ -959,7 +961,7 @@ void ShrinkWrapping::splitFrontierCritEdges(
}
// Update frontier
auto NewFrontierPP = ProgramPoint::getLastPointAt(*NewBB);
ProgramPoint NewFrontierPP = ProgramPoint::getLastPointAt(*NewBB);
if (DI == 0) {
// Update frontier inplace
Frontier[I] = NewFrontierPP;
@ -981,7 +983,7 @@ ShrinkWrapping::doRestorePlacement(MCInst *BestPosSave, unsigned CSR,
SmallVector<ProgramPoint, 4> Frontier;
SmallVector<bool, 4> IsCritEdge;
bool CannotPlace{false};
auto &DA = Info.getDominatorAnalysis();
DominatorAnalysis<false> &DA = Info.getDominatorAnalysis();
SmallVector<BinaryBasicBlock *, 4> CritEdgesFrom;
SmallVector<SmallVector<BinaryBasicBlock *, 4>, 4> CritEdgesTo;
@ -992,7 +994,7 @@ ShrinkWrapping::doRestorePlacement(MCInst *BestPosSave, unsigned CSR,
LLVM_DEBUG({
dbgs() << "Dumping dominance frontier for ";
BC.printInstruction(dbgs(), *BestPosSave);
for (auto &PP : Frontier) {
for (ProgramPoint &PP : Frontier) {
if (PP.isInst()) {
BC.printInstruction(dbgs(), *PP.getInst());
} else {
@ -1000,7 +1002,7 @@ ShrinkWrapping::doRestorePlacement(MCInst *BestPosSave, unsigned CSR,
}
}
});
for (auto &PP : Frontier) {
for (ProgramPoint &PP : Frontier) {
bool HasCritEdges{false};
if (PP.isInst() && BC.MIB->isTerminator(*PP.getInst()) &&
doesInstUsesCSR(*PP.getInst(), CSR)) {
@ -1009,7 +1011,7 @@ ShrinkWrapping::doRestorePlacement(MCInst *BestPosSave, unsigned CSR,
BinaryBasicBlock *FrontierBB = Info.getParentBB(PP);
CritEdgesFrom.emplace_back(FrontierBB);
CritEdgesTo.emplace_back(0);
auto &Dests = CritEdgesTo.back();
SmallVector<BinaryBasicBlock *, 4> &Dests = CritEdgesTo.back();
// Check for invoke instructions at the dominance frontier, which indicates
// the landing pad is not dominated.
if (PP.isInst() && BC.MIB->isInvoke(*PP.getInst())) {
@ -1032,7 +1034,7 @@ ShrinkWrapping::doRestorePlacement(MCInst *BestPosSave, unsigned CSR,
// (PredictiveStackPointerTracking). Detect now for empty BBs and add a
// dummy nop that is scheduled to be removed later.
bool InvalidateRequired = false;
for (auto &BB : BF.layout()) {
for (BinaryBasicBlock *&BB : BF.layout()) {
if (BB->size() != 0)
continue;
MCInst NewInst;
@ -1044,7 +1046,7 @@ ShrinkWrapping::doRestorePlacement(MCInst *BestPosSave, unsigned CSR,
if (std::accumulate(IsCritEdge.begin(), IsCritEdge.end(), 0)) {
LLVM_DEBUG({
dbgs() << "Now detected critical edges in the following frontier:\n";
for (auto &PP : Frontier) {
for (ProgramPoint &PP : Frontier) {
if (PP.isBB())
dbgs() << " BB: " << PP.getBB()->getName() << "\n";
else {
@ -1099,7 +1101,7 @@ bool ShrinkWrapping::validatePushPopsMode(unsigned CSR, MCInst *BestPosSave,
}
}
auto &SPT = Info.getStackPointerTracking();
StackPointerTracking &SPT = Info.getStackPointerTracking();
// Abort if we are inserting a push into an entry BB (offset -8) and this
// func sets up a frame pointer.
if (!SLM.canInsertRegion(BestPosSave) ||
@ -1119,11 +1121,11 @@ SmallVector<ProgramPoint, 4> ShrinkWrapping::fixPopsPlacements(
unsigned CSR) {
SmallVector<ProgramPoint, 4> FixedRestorePoints = RestorePoints;
// Moving pop locations to the correct sp offset
auto &RI = Info.getReachingInsnsBackwards();
auto &SPT = Info.getStackPointerTracking();
for (auto &PP : FixedRestorePoints) {
auto *BB = Info.getParentBB(PP);
auto Found = false;
ReachingInsns<true> &RI = Info.getReachingInsnsBackwards();
StackPointerTracking &SPT = Info.getStackPointerTracking();
for (ProgramPoint &PP : FixedRestorePoints) {
BinaryBasicBlock *BB = Info.getParentBB(PP);
bool Found = false;
if (SPT.getStateAt(ProgramPoint::getLastPointAt(*BB))->first ==
SaveOffset) {
BitVector BV = *RI.getStateAt(ProgramPoint::getLastPointAt(*BB));
@ -1160,10 +1162,10 @@ SmallVector<ProgramPoint, 4> ShrinkWrapping::fixPopsPlacements(
void ShrinkWrapping::scheduleOldSaveRestoresRemoval(unsigned CSR,
bool UsePushPops) {
for (auto &BB : BF.layout()) {
for (BinaryBasicBlock *&BB : BF.layout()) {
std::vector<MCInst *> CFIs;
for (auto I = BB->rbegin(), E = BB->rend(); I != E; ++I) {
auto &Inst = *I;
MCInst &Inst = *I;
if (BC.MIB->isCFI(Inst)) {
// Delete all offset CFIs related to this CSR
if (SLM.getOffsetCFIReg(Inst) == CSR) {
@ -1175,8 +1177,8 @@ void ShrinkWrapping::scheduleOldSaveRestoresRemoval(unsigned CSR,
continue;
}
auto SavedReg = CSA.getSavedReg(Inst);
auto RestoredReg = CSA.getRestoredReg(Inst);
uint16_t SavedReg = CSA.getSavedReg(Inst);
uint16_t RestoredReg = CSA.getRestoredReg(Inst);
if (SavedReg != CSR && RestoredReg != CSR) {
CFIs.clear();
continue;
@ -1230,8 +1232,8 @@ void ShrinkWrapping::scheduleSaveRestoreInsertions(
unsigned CSR, MCInst *BestPosSave,
SmallVector<ProgramPoint, 4> &RestorePoints, bool UsePushPops) {
auto &InsnToBB = Info.getInsnToBBMap();
auto FIESave = CSA.SaveFIEByReg[CSR];
auto FIELoad = CSA.LoadFIEByReg[CSR];
const FrameIndexEntry *FIESave = CSA.SaveFIEByReg[CSR];
const FrameIndexEntry *FIELoad = CSA.LoadFIEByReg[CSR];
assert(FIESave && FIELoad && "Invalid CSR");
LLVM_DEBUG({
@ -1243,7 +1245,7 @@ void ShrinkWrapping::scheduleSaveRestoreInsertions(
: WorklistItem::InsertLoadOrStore,
*FIESave, CSR);
for (auto &PP : RestorePoints) {
for (ProgramPoint &PP : RestorePoints) {
BinaryBasicBlock *FrontierBB = Info.getParentBB(PP);
LLVM_DEBUG({
dbgs() << "Scheduling restore insertion at: ";
@ -1298,13 +1300,13 @@ void ShrinkWrapping::moveSaveRestores() {
if (RestorePoints.empty())
continue;
auto FIESave = CSA.SaveFIEByReg[I];
auto FIELoad = CSA.LoadFIEByReg[I];
const FrameIndexEntry *FIESave = CSA.SaveFIEByReg[I];
const FrameIndexEntry *FIELoad = CSA.LoadFIEByReg[I];
assert(FIESave && FIELoad);
auto &SPT = Info.getStackPointerTracking();
const auto SPFP = *SPT.getStateBefore(*BestPosSave);
auto SaveOffset = SPFP.first;
auto SaveSize = FIESave->Size;
StackPointerTracking &SPT = Info.getStackPointerTracking();
const std::pair<int, int> SPFP = *SPT.getStateBefore(*BestPosSave);
int SaveOffset = SPFP.first;
uint8_t SaveSize = FIESave->Size;
// If we don't know stack state at this point, bail
if ((SPFP.first == SPT.SUPERPOSITION || SPFP.first == SPT.EMPTY) &&
@ -1315,7 +1317,8 @@ void ShrinkWrapping::moveSaveRestores() {
bool UsePushPops = validatePushPopsMode(I, BestPosSave, SaveOffset);
if (UsePushPops) {
auto FixedRestorePoints = fixPopsPlacements(RestorePoints, SaveOffset, I);
SmallVector<ProgramPoint, 4> FixedRestorePoints =
fixPopsPlacements(RestorePoints, SaveOffset, I);
if (FixedRestorePoints.empty())
UsePushPops = false;
else
@ -1336,23 +1339,23 @@ void ShrinkWrapping::moveSaveRestores() {
// Revert push-pop mode if it failed for a single CSR
if (DisablePushPopMode && UsedPushPopMode) {
UsedPushPopMode = false;
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
auto WRI = Todo.find(&BB);
if (WRI != Todo.end()) {
auto &TodoList = WRI->second;
for (auto &Item : TodoList) {
std::vector<WorklistItem> &TodoList = WRI->second;
for (WorklistItem &Item : TodoList) {
if (Item.Action == WorklistItem::InsertPushOrPop)
Item.Action = WorklistItem::InsertLoadOrStore;
}
}
for (auto I = BB.rbegin(), E = BB.rend(); I != E; ++I) {
auto &Inst = *I;
MCInst &Inst = *I;
auto TodoList = BC.MIB->tryGetAnnotationAs<std::vector<WorklistItem>>(
Inst, getAnnotationIndex());
if (!TodoList)
continue;
bool isCFI = BC.MIB->isCFI(Inst);
for (auto &Item : *TodoList) {
for (WorklistItem &Item : *TodoList) {
if (Item.Action == WorklistItem::InsertPushOrPop)
Item.Action = WorklistItem::InsertLoadOrStore;
if (!isCFI && Item.Action == WorklistItem::Erase)
@ -1371,7 +1374,7 @@ void ShrinkWrapping::moveSaveRestores() {
// Schedule modifications to stack-accessing instructions via
// StackLayoutModifier.
SpillsMovedPushPopMode += MovedRegs.size();
for (auto &I : MovedRegs) {
for (std::tuple<unsigned, MCInst *, size_t> &I : MovedRegs) {
unsigned RegNdx;
MCInst *SavePos;
size_t SaveSize;
@ -1462,7 +1465,7 @@ protected:
void compNextAux(const MCInst &Point,
const std::vector<ShrinkWrapping::WorklistItem> &TodoItems,
std::pair<int, int> &Res) {
for (const auto &Item : TodoItems) {
for (const ShrinkWrapping::WorklistItem &Item : TodoItems) {
if (Item.Action == ShrinkWrapping::WorklistItem::Erase &&
BC.MIB->isPush(Point)) {
Res.first += BC.MIB->getPushSize(Point);
@ -1532,7 +1535,7 @@ public:
void ShrinkWrapping::insertUpdatedCFI(unsigned CSR, int SPValPush,
int SPValPop) {
MCInst *SavePoint{nullptr};
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
for (auto InstIter = BB.rbegin(), EndIter = BB.rend(); InstIter != EndIter;
++InstIter) {
int32_t SrcImm{0};
@ -1564,8 +1567,8 @@ void ShrinkWrapping::insertUpdatedCFI(unsigned CSR, int SPValPush,
});
bool PrevAffectedZone{false};
BinaryBasicBlock *PrevBB{nullptr};
auto &DA = Info.getDominatorAnalysis();
for (auto BB : BF.layout()) {
DominatorAnalysis<false> &DA = Info.getDominatorAnalysis();
for (BinaryBasicBlock *BB : BF.layout()) {
if (BB->size() == 0)
continue;
const bool InAffectedZoneAtEnd = DA.count(*BB->rbegin(), *SavePoint);
@ -1609,8 +1612,8 @@ void ShrinkWrapping::insertUpdatedCFI(unsigned CSR, int SPValPush,
}
void ShrinkWrapping::rebuildCFIForSP() {
for (auto &BB : BF) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : BF) {
for (MCInst &Inst : BB) {
if (!BC.MIB->isCFI(Inst))
continue;
const MCCFIInstruction *CFI = BF.getCFIFor(Inst);
@ -1621,8 +1624,8 @@ void ShrinkWrapping::rebuildCFIForSP() {
int PrevSPVal{-8};
BinaryBasicBlock *PrevBB{nullptr};
auto &SPT = Info.getStackPointerTracking();
for (auto BB : BF.layout()) {
StackPointerTracking &SPT = Info.getStackPointerTracking();
for (BinaryBasicBlock *BB : BF.layout()) {
if (BB->size() == 0)
continue;
const int SPValAtEnd = SPT.getStateAt(*BB->rbegin())->first;
@ -1654,7 +1657,7 @@ void ShrinkWrapping::rebuildCFIForSP() {
PrevBB = BB;
}
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
for (auto I = BB.begin(); I != BB.end(); ) {
if (BC.MIB->hasAnnotation(*I, "DeleteMe"))
I = BB.eraseInstruction(I);
@ -1809,7 +1812,7 @@ BBIterTy ShrinkWrapping::processInsertionsList(
BBIterTy InsertionPoint, BinaryBasicBlock *CurBB,
std::vector<WorklistItem> &TodoList, int64_t SPVal, int64_t FPVal) {
bool HasInsertions{false};
for (auto &Item : TodoList) {
for (WorklistItem &Item : TodoList) {
if (Item.Action == WorklistItem::Erase ||
Item.Action == WorklistItem::ChangeToAdjustment)
continue;
@ -1829,7 +1832,7 @@ BBIterTy ShrinkWrapping::processInsertionsList(
// Revert the effect of PSPT for this location, we want SP Value before
// insertions
if (InsertionPoint == CurBB->end()) {
for (auto &Item : TodoList) {
for (WorklistItem &Item : TodoList) {
if (Item.Action != WorklistItem::InsertPushOrPop)
continue;
if (Item.FIEToInsert.IsStore)
@ -1854,7 +1857,7 @@ BBIterTy ShrinkWrapping::processInsertionsList(
});
// Process insertions
for (auto &Item : TodoList) {
for (WorklistItem &Item : TodoList) {
if (Item.Action == WorklistItem::Erase ||
Item.Action == WorklistItem::ChangeToAdjustment)
continue;
@ -1878,30 +1881,30 @@ bool ShrinkWrapping::processInsertions() {
PSPT.run();
bool Changes{false};
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
// Process insertions before some inst.
for (auto I = BB.begin(); I != BB.end(); ++I) {
auto &Inst = *I;
MCInst &Inst = *I;
auto TodoList = BC.MIB->tryGetAnnotationAs<std::vector<WorklistItem>>(
Inst, getAnnotationIndex());
if (!TodoList)
continue;
Changes = true;
auto List = *TodoList;
std::vector<WorklistItem> List = *TodoList;
LLVM_DEBUG({
dbgs() << "Now processing insertions in " << BB.getName()
<< " before inst: ";
Inst.dump();
});
auto Iter = I;
auto SPTState =
std::pair<int, int> SPTState =
*PSPT.getStateAt(Iter == BB.begin() ? (ProgramPoint)&BB : &*(--Iter));
I = processInsertionsList(I, &BB, List, SPTState.first, SPTState.second);
}
// Process insertions at the end of bb
auto WRI = Todo.find(&BB);
if (WRI != Todo.end()) {
auto SPTState = *PSPT.getStateAt(*BB.rbegin());
std::pair<int, int> SPTState = *PSPT.getStateAt(*BB.rbegin());
processInsertionsList(BB.end(), &BB, WRI->second, SPTState.first,
SPTState.second);
Changes = true;
@ -1911,16 +1914,16 @@ bool ShrinkWrapping::processInsertions() {
}
void ShrinkWrapping::processDeletions() {
auto &LA = Info.getLivenessAnalysis();
for (auto &BB : BF) {
LivenessAnalysis &LA = Info.getLivenessAnalysis();
for (BinaryBasicBlock &BB : BF) {
for (auto II = BB.begin(); II != BB.end(); ++II) {
auto &Inst = *II;
MCInst &Inst = *II;
auto TodoList = BC.MIB->tryGetAnnotationAs<std::vector<WorklistItem>>(
Inst, getAnnotationIndex());
if (!TodoList)
continue;
// Process all deletions
for (auto &Item : *TodoList) {
for (WorklistItem &Item : *TodoList) {
if (Item.Action != WorklistItem::Erase &&
Item.Action != WorklistItem::ChangeToAdjustment)
continue;
@ -1928,11 +1931,11 @@ void ShrinkWrapping::processDeletions() {
if (Item.Action == WorklistItem::ChangeToAdjustment) {
// Is flag reg alive across this func?
bool DontClobberFlags = LA.isAlive(&Inst, BC.MIB->getFlagsReg());
if (auto Sz = BC.MIB->getPushSize(Inst)) {
if (int Sz = BC.MIB->getPushSize(Inst)) {
BC.MIB->createStackPointerIncrement(Inst, Sz, DontClobberFlags);
continue;
}
if (auto Sz = BC.MIB->getPopSize(Inst)) {
if (int Sz = BC.MIB->getPopSize(Inst)) {
BC.MIB->createStackPointerDecrement(Inst, Sz, DontClobberFlags);
continue;
}
@ -1998,7 +2001,7 @@ bool ShrinkWrapping::perform() {
return false;
processDeletions();
if (foldIdenticalSplitEdges()) {
const auto Stats = BF.eraseInvalidBBs();
const std::pair<unsigned, uint64_t> Stats = BF.eraseInvalidBBs();
LLVM_DEBUG(dbgs() << "Deleted " << Stats.first
<< " redundant split edge BBs (" << Stats.second
<< " bytes) for " << BF.getPrintName() << "\n");
@ -2023,8 +2026,8 @@ void ShrinkWrapping::printStats() {
raw_ostream &operator<<(raw_ostream &OS,
const std::vector<ShrinkWrapping::WorklistItem> &Vec) {
OS << "SWTodo[";
auto Sep = "";
for (const auto &Item : Vec) {
const char *Sep = "";
for (const ShrinkWrapping::WorklistItem &Item : Vec) {
OS << Sep;
switch (Item.Action) {
case ShrinkWrapping::WorklistItem::Erase:
@ -2050,8 +2053,8 @@ raw_ostream &
operator<<(raw_ostream &OS,
const std::vector<StackLayoutModifier::WorklistItem> &Vec) {
OS << "SLMTodo[";
auto Sep = "";
for (const auto &Item : Vec) {
const char *Sep = "";
for (const StackLayoutModifier::WorklistItem &Item : Vec) {
OS << Sep;
switch (Item.Action) {
case StackLayoutModifier::WorklistItem::None:

10
bolt/src/Passes/ShrinkWrapping.h
View File

@ -228,8 +228,8 @@ public:
: FA(FA), BC(BC), BF(BF), Info(Info), AllocatorId(AllocId) {}
~StackLayoutModifier() {
for (auto &BB : BF) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : BF) {
for (MCInst &Inst : BB) {
BC.MIB->removeAnnotation(Inst, getTodoTag());
BC.MIB->removeAnnotation(Inst, getSlotTag());
BC.MIB->removeAnnotation(Inst, getOffsetCFIRegTag());
@ -487,7 +487,7 @@ private:
/// Insert any CFI that should be attached to a register spill save/restore.
BBIterTy insertCFIsForPushOrPop(BinaryBasicBlock &BB, BBIterTy Pos,
unsigned Reg, bool isPush, int Sz,
unsigned Reg, bool IsPush, int Sz,
int64_t NewOffset);
/// Auxiliary function to processInsertionsList, adding a new instruction
@ -524,8 +524,8 @@ public:
SLM(FA, BC, BF, Info, AllocId), CSA(FA, BC, BF, Info, AllocId) {}
~ShrinkWrapping() {
for (auto &BB : BF) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : BF) {
for (MCInst &Inst : BB) {
BC.MIB->removeAnnotation(Inst, getAnnotationIndex());
}
}

14
bolt/src/Passes/SplitFunctions.cpp
View File

@ -123,8 +123,8 @@ void SplitFunctions::splitFunction(BinaryFunction &BF) {
return;
bool AllCold = true;
for (auto *BB : BF.layout()) {
auto ExecCount = BB->getExecutionCount();
for (BinaryBasicBlock *BB : BF.layout()) {
uint64_t ExecCount = BB->getExecutionCount();
if (ExecCount == BinaryBasicBlock::COUNT_NO_PROFILE)
return;
if (ExecCount != 0)
@ -134,9 +134,9 @@ void SplitFunctions::splitFunction(BinaryFunction &BF) {
if (AllCold)
return;
auto PreSplitLayout = BF.getLayout();
std::vector<BinaryBasicBlock *> PreSplitLayout = BF.getLayout();
auto &BC = BF.getBinaryContext();
BinaryContext &BC = BF.getBinaryContext();
size_t OriginalHotSize;
size_t HotSize;
size_t ColdSize;
@ -156,7 +156,7 @@ void SplitFunctions::splitFunction(BinaryFunction &BF) {
// Never outline the first basic block.
BF.layout_front()->setCanOutline(false);
for (auto *BB : BF.layout()) {
for (BinaryBasicBlock *BB : BF.layout()) {
if (!BB->canOutline())
continue;
if (BB->getExecutionCount() != 0) {
@ -181,7 +181,7 @@ void SplitFunctions::splitFunction(BinaryFunction &BF) {
// runtime cannot deal with split functions. However, if we can guarantee
// that the block never throws, it is safe to move the block to
// decrease the size of the function.
for (auto &Instr : *BB) {
for (MCInst &Instr : *BB) {
if (BF.getBinaryContext().MIB->isInvoke(Instr)) {
BB->setCanOutline(false);
break;
@ -235,7 +235,7 @@ void SplitFunctions::splitFunction(BinaryFunction &BF) {
<< Twine::utohexstr(OriginalHotSize) << '\n');
BF.updateBasicBlockLayout(PreSplitLayout);
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
BB.setIsCold(false);
}
} else {

6
bolt/src/Passes/StackAllocationAnalysis.cpp
View File

@ -20,8 +20,8 @@ void StackAllocationAnalysis::preflight() {
LLVM_DEBUG(dbgs() << "Starting StackAllocationAnalysis on \""
<< Func.getPrintName() << "\"\n");
for (auto &BB : this->Func) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : this->Func) {
for (MCInst &Inst : BB) {
MCPhysReg From, To;
if (!BC.MIB->isPush(Inst) && (!BC.MIB->isRegToRegMove(Inst, From, To) ||
To != BC.MIB->getStackPointer() ||
@ -82,7 +82,7 @@ void StackAllocationAnalysis::doConfluenceWithLP(BitVector &StateOut,
const BitVector &StateIn,
const MCInst &Invoke) {
BitVector NewIn = StateIn;
const auto GnuArgsSize = BC.MIB->getGnuArgsSize(Invoke);
const int64_t GnuArgsSize = BC.MIB->getGnuArgsSize(Invoke);
if (GnuArgsSize >= 0)
NewIn = doKill(Invoke, NewIn, GnuArgsSize);
StateOut |= NewIn;

12
bolt/src/Passes/StackAvailableExpressions.cpp
View File

@ -29,9 +29,9 @@ void StackAvailableExpressions::preflight() {
// Populate our universe of tracked expressions. We are interested in
// tracking available stores to frame position at any given point of the
// program.
for (auto &BB : Func) {
for (auto &Inst : BB) {
auto FIE = FA.getFIEFor(Inst);
for (BinaryBasicBlock &BB : Func) {
for (MCInst &Inst : BB) {
ErrorOr<const FrameIndexEntry &> FIE = FA.getFIEFor(Inst);
if (!FIE)
continue;
if (FIE->IsStore == true && FIE->IsSimple == true) {
@ -80,8 +80,8 @@ bool isLoadRedundant(const FrameIndexEntry &LoadFIE,
bool StackAvailableExpressions::doesXKillsY(const MCInst *X, const MCInst *Y) {
// if both are stores, and both store to the same stack location, return
// true
auto FIEX = FA.getFIEFor(*X);
auto FIEY = FA.getFIEFor(*Y);
ErrorOr<const FrameIndexEntry &> FIEX = FA.getFIEFor(*X);
ErrorOr<const FrameIndexEntry &> FIEY = FA.getFIEFor(*Y);
if (FIEX && FIEY) {
if (isLoadRedundant(*FIEX, *FIEY))
return false;
@ -121,7 +121,7 @@ BitVector StackAvailableExpressions::computeNext(const MCInst &Point,
}
}
// Gen
if (auto FIE = FA.getFIEFor(Point)) {
if (ErrorOr<const FrameIndexEntry &> FIE = FA.getFIEFor(Point)) {
if (FIE->IsStore == true && FIE->IsSimple == true)
Next.set(ExprToIdx[&Point]);
}

2
bolt/src/Passes/StackPointerTracking.h
View File

@ -69,7 +69,7 @@ protected:
const MCInst &Invoke) {
int SPVal = StateIn.first;
if (SPVal != EMPTY && SPVal != SUPERPOSITION) {
const auto GnuArgsSize = this->BC.MIB->getGnuArgsSize(Invoke);
const int64_t GnuArgsSize = this->BC.MIB->getGnuArgsSize(Invoke);
if (GnuArgsSize > 0)
SPVal += GnuArgsSize;
}

24
bolt/src/Passes/StackReachingUses.cpp
View File

@ -19,7 +19,7 @@ bool StackReachingUses::isLoadedInDifferentReg(const FrameIndexEntry &StoreFIE,
ExprIterator Candidates) const {
for (auto I = Candidates; I != expr_end(); ++I) {
const MCInst *ReachingInst = *I;
if (auto FIEY = FA.getFIEFor(*ReachingInst)) {
if (ErrorOr<const FrameIndexEntry &> FIEY = FA.getFIEFor(*ReachingInst)) {
assert(FIEY->IsLoad == 1);
if (StoreFIE.StackOffset + StoreFIE.Size > FIEY->StackOffset &&
StoreFIE.StackOffset < FIEY->StackOffset + FIEY->Size &&
@ -37,7 +37,7 @@ bool StackReachingUses::isStoreUsed(const FrameIndexEntry &StoreFIE,
for (auto I = Candidates; I != expr_end(); ++I) {
const MCInst *ReachingInst = *I;
if (IncludeLocalAccesses) {
if (auto FIEY = FA.getFIEFor(*ReachingInst)) {
if (ErrorOr<const FrameIndexEntry &> FIEY = FA.getFIEFor(*ReachingInst)) {
assert(FIEY->IsLoad == 1);
if (StoreFIE.StackOffset + StoreFIE.Size > FIEY->StackOffset &&
StoreFIE.StackOffset < FIEY->StackOffset + FIEY->Size) {
@ -45,13 +45,13 @@ bool StackReachingUses::isStoreUsed(const FrameIndexEntry &StoreFIE,
}
}
}
auto Args = FA.getArgAccessesFor(*ReachingInst);
ErrorOr<const ArgAccesses &> Args = FA.getArgAccessesFor(*ReachingInst);
if (!Args)
continue;
if (Args->AssumeEverything) {
return true;
}
for (auto FIEY : Args->Set) {
for (ArgInStackAccess FIEY : Args->Set) {
if (StoreFIE.StackOffset + StoreFIE.Size > FIEY.StackOffset &&
StoreFIE.StackOffset < FIEY.StackOffset + FIEY.Size) {
return true;
@ -68,16 +68,16 @@ void StackReachingUses::preflight() {
// Populate our universe of tracked expressions. We are interested in
// tracking reaching loads from frame position at any given point of the
// program.
for (auto &BB : Func) {
for (auto &Inst : BB) {
if (auto FIE = FA.getFIEFor(Inst)) {
for (BinaryBasicBlock &BB : Func) {
for (MCInst &Inst : BB) {
if (ErrorOr<const FrameIndexEntry &> FIE = FA.getFIEFor(Inst)) {
if (FIE->IsLoad == true) {
Expressions.push_back(&Inst);
ExprToIdx[&Inst] = NumInstrs++;
continue;
}
}
auto AA = FA.getArgAccessesFor(Inst);
ErrorOr<const ArgAccesses &> AA = FA.getArgAccessesFor(Inst);
if (AA && (!AA->Set.empty() || AA->AssumeEverything)) {
Expressions.push_back(&Inst);
ExprToIdx[&Inst] = NumInstrs++;
@ -89,8 +89,8 @@ void StackReachingUses::preflight() {
bool StackReachingUses::doesXKillsY(const MCInst *X, const MCInst *Y) {
// if X is a store to the same stack location and the bytes fetched is a
// superset of those bytes affected by the load in Y, return true
auto FIEX = FA.getFIEFor(*X);
auto FIEY = FA.getFIEFor(*Y);
ErrorOr<const FrameIndexEntry &> FIEX = FA.getFIEFor(*X);
ErrorOr<const FrameIndexEntry &> FIEY = FA.getFIEFor(*Y);
if (FIEX && FIEY) {
if (FIEX->IsSimple == true && FIEY->IsSimple == true &&
FIEX->IsStore == true && FIEY->IsLoad == true &&
@ -114,11 +114,11 @@ BitVector StackReachingUses::computeNext(const MCInst &Point,
}
};
// Gen
if (auto FIE = FA.getFIEFor(Point)) {
if (ErrorOr<const FrameIndexEntry &> FIE = FA.getFIEFor(Point)) {
if (FIE->IsLoad == true)
Next.set(ExprToIdx[&Point]);
}
auto AA = FA.getArgAccessesFor(Point);
ErrorOr<const ArgAccesses &> AA = FA.getArgAccessesFor(Point);
if (AA && (!AA->Set.empty() || AA->AssumeEverything))
Next.set(ExprToIdx[&Point]);
return Next;

23
bolt/src/Passes/StokeInfo.cpp
View File

@ -48,12 +48,12 @@ void StokeInfo::checkInstr(const BinaryContext &BC, const BinaryFunction &BF,
StokeFuncInfo &FuncInfo) {
BitVector RegV(NumRegs, false);
for (auto *BB : BF.layout()) {
for (BinaryBasicBlock *BB : BF.layout()) {
if (BB->empty()) {
continue;
}
for (auto &It : *BB) {
auto &InstDesc = BC.MII->get(It.getOpcode());
for (MCInst &It : *BB) {
const MCInstrDesc &InstDesc = BC.MII->get(It.getOpcode());
if (InstDesc.isPseudo()) {
continue;
}
@ -65,7 +65,7 @@ void StokeInfo::checkInstr(const BinaryContext &BC, const BinaryFunction &BF,
// check if this function contains call instruction
if (BC.MIB->isCall(It)) {
FuncInfo.HasCall = true;
const auto *TargetSymbol = BC.MIB->getTargetSymbol(It);
const MCSymbol *TargetSymbol = BC.MIB->getTargetSymbol(It);
// if it is an indirect call, skip
if (TargetSymbol == nullptr) {
FuncInfo.Omitted = true;
@ -74,8 +74,8 @@ void StokeInfo::checkInstr(const BinaryContext &BC, const BinaryFunction &BF,
}
// check if this function modify stack or heap
// TODO: more accurate analysis
auto IsPush = BC.MIB->isPush(It);
auto IsRipAddr = BC.MIB->hasPCRelOperand(It);
bool IsPush = BC.MIB->isPush(It);
bool IsRipAddr = BC.MIB->hasPCRelOperand(It);
if (IsPush) {
FuncInfo.StackOut = true;
}
@ -113,7 +113,7 @@ bool StokeInfo::checkFunction(const BinaryContext &BC, BinaryFunction &BF,
FuncInfo.IsLoopFree = BF.isLoopFree();
if (!FuncInfo.IsLoopFree) {
auto &BLI = BF.getLoopInfo();
const BinaryLoopInfo &BLI = BF.getLoopInfo();
FuncInfo.NumLoops = BLI.OuterLoops;
FuncInfo.MaxLoopDepth = BLI.MaximumDepth;
}
@ -128,18 +128,19 @@ bool StokeInfo::checkFunction(const BinaryContext &BC, BinaryFunction &BF,
BinaryBasicBlock &EntryBB = BF.front();
assert(EntryBB.isEntryPoint() && "Weird, this should be the entry block!");
auto *FirstNonPseudo = EntryBB.getFirstNonPseudoInstr();
MCInst *FirstNonPseudo = EntryBB.getFirstNonPseudoInstr();
if (!FirstNonPseudo) {
return false;
}
LLVM_DEBUG(dbgs() << "\t [DefIn]\n\t ");
auto LiveInBV = *(DInfo.getLivenessAnalysis().getStateAt(FirstNonPseudo));
BitVector LiveInBV =
*(DInfo.getLivenessAnalysis().getStateAt(FirstNonPseudo));
LiveInBV &= DefaultDefInMask;
getRegNameFromBitVec(BC, LiveInBV, &FuncInfo.DefIn);
LLVM_DEBUG(dbgs() << "\t [LiveOut]\n\t ");
auto LiveOutBV = RA.getFunctionClobberList(&BF);
BitVector LiveOutBV = RA.getFunctionClobberList(&BF);
LiveOutBV &= DefaultLiveOutMask;
getRegNameFromBitVec(BC, LiveOutBV, &FuncInfo.LiveOut);
@ -163,7 +164,7 @@ void StokeInfo::runOnFunctions(BinaryContext &BC) {
LLVM_DEBUG(dbgs() << "\tTripleName " << BC.TripleName << "\n");
LLVM_DEBUG(dbgs() << "\tgetNumRegs " << BC.MRI->getNumRegs() << "\n");
auto CG = buildCallGraph(BC);
BinaryFunctionCallGraph CG = buildCallGraph(BC);
RegAnalysis RA(BC, &BC.getBinaryFunctions(), &CG);
NumRegs = BC.MRI->getNumRegs();

8
bolt/src/Passes/StokeInfo.h
View File

@ -97,12 +97,12 @@ struct StokeFuncInfo {
<< HotSize << "," << TotalSize << ","
<< Score << ","
<< HasCall << ",\"{ ";
for (auto s : DefIn) {
Outfile << "%" << s << " ";
for (std::string S : DefIn) {
Outfile << "%" << S << " ";
}
Outfile << "}\",\"{ ";
for (auto s : LiveOut) {
Outfile << "%" << s << " ";
for (std::string S : LiveOut) {
Outfile << "%" << S << " ";
}
Outfile << "}\"," << HeapOut << "," << StackOut << ","
<< HasRipAddr << ","

34
bolt/src/Passes/ValidateInternalCalls.cpp
View File

@ -91,15 +91,15 @@ public:
bool ValidateInternalCalls::fixCFGForPIC(BinaryFunction &Function) const {
const BinaryContext &BC = Function.getBinaryContext();
for (auto &BB : Function) {
for (BinaryBasicBlock &BB : Function) {
for (auto II = BB.begin(); II != BB.end(); ++II) {
auto &Inst = *II;
auto *Target = getInternalCallTarget(Function, Inst);
MCInst &Inst = *II;
BinaryBasicBlock *Target = getInternalCallTarget(Function, Inst);
if (!Target || BC.MIB->hasAnnotation(Inst, getProcessedICTag()))
continue;
BC.MIB->addAnnotation(Inst, getProcessedICTag(), 0U);
auto MovedInsts = BB.splitInstructions(&Inst);
std::vector<MCInst> MovedInsts = BB.splitInstructions(&Inst);
if (!MovedInsts.empty()) {
// Split this block at the call instruction. Create an unreachable
// block.
@ -170,8 +170,8 @@ bool ValidateInternalCalls::fixCFGForIC(BinaryFunction &Function) const {
// connecting RETs to potentially unrelated internal calls. This is safe
// and if this causes a problem to recover the stack offsets properly, we
// will fail later.
for (auto &BB : Function) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : Function) {
for (MCInst &Inst : BB) {
if (!BC.MIB->isReturn(Inst))
continue;
@ -183,8 +183,8 @@ bool ValidateInternalCalls::fixCFGForIC(BinaryFunction &Function) const {
bool ValidateInternalCalls::hasTailCallsInRange(BinaryFunction &Function) const {
const BinaryContext &BC = Function.getBinaryContext();
for (auto &BB : Function) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : Function) {
for (MCInst &Inst : BB) {
if (BC.MIB->isTailCall(Inst))
return true;
}
@ -202,9 +202,9 @@ bool ValidateInternalCalls::analyzeFunction(BinaryFunction &Function) const {
RA.setConservativeStrategy(RegAnalysis::ConservativeStrategy::CLOBBERS_NONE);
bool HasTailCalls = hasTailCallsInRange(Function);
for (auto &BB : Function) {
for (auto &Inst : BB) {
auto *Target = getInternalCallTarget(Function, Inst);
for (BinaryBasicBlock &BB : Function) {
for (MCInst &Inst : BB) {
BinaryBasicBlock *Target = getInternalCallTarget(Function, Inst);
if (!Target || BC.MIB->hasAnnotation(Inst, getProcessedICTag()))
continue;
@ -219,7 +219,7 @@ bool ValidateInternalCalls::analyzeFunction(BinaryFunction &Function) const {
MCPhysReg Reg{0};
int64_t StackOffset{0};
bool IsIndexed{false};
auto *TargetInst = ProgramPoint::getFirstPointAt(*Target).getInst();
MCInst *TargetInst = ProgramPoint::getFirstPointAt(*Target).getInst();
if (!BC.MIB->isStackAccess(*TargetInst, FIE.IsLoad, FIE.IsStore,
FIE.IsStoreFromReg, Reg, SrcImm,
FIE.StackPtrReg, StackOffset, FIE.Size,
@ -251,7 +251,7 @@ bool ValidateInternalCalls::analyzeFunction(BinaryFunction &Function) const {
E = RU.expr_end();
I != E; ++I) {
MCInst &Use = **I;
auto UsedRegs = BitVector(BC.MRI->getNumRegs(), false);
BitVector UsedRegs = BitVector(BC.MRI->getNumRegs(), false);
BC.MIB->getTouchedRegs(Use, UsedRegs);
if (!UsedRegs[Reg])
continue;
@ -299,8 +299,8 @@ void ValidateInternalCalls::runOnFunctions(BinaryContext &BC) {
std::set<BinaryFunction *> NeedsValidation;
for (auto &BFI : BC.getBinaryFunctions()) {
BinaryFunction &Function = BFI.second;
for (auto &BB : Function) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : Function) {
for (MCInst &Inst : BB) {
if (getInternalCallTarget(Function, Inst)) {
NeedsValidation.insert(&Function);
Function.setSimple(false);
@ -320,7 +320,7 @@ void ValidateInternalCalls::runOnFunctions(BinaryContext &BC) {
// value, so it is not really a call, but a jump. If we find that it's not the
// case, we mark this function as non-simple and stop processing it.
std::set<BinaryFunction *> Invalid;
for (auto *Function : NeedsValidation) {
for (BinaryFunction *Function : NeedsValidation) {
LLVM_DEBUG(dbgs() << "Validating " << *Function << "\n");
if (!analyzeFunction(*Function)) {
Invalid.insert(Function);
@ -331,7 +331,7 @@ void ValidateInternalCalls::runOnFunctions(BinaryContext &BC) {
if (!Invalid.empty()) {
errs() << "BOLT-WARNING: will skip the following function(s) as unsupported"
" internal calls were detected:\n";
for (auto *Function : Invalid) {
for (BinaryFunction *Function : Invalid) {
errs() << " " << *Function << "\n";
Function->setIgnored();
}

4
bolt/src/Passes/ValidateInternalCalls.h
View File

@ -89,8 +89,8 @@ private:
void clearAnnotations(BinaryFunction &Function) const {
const BinaryContext &BC = Function.getBinaryContext();
for (auto &BB : Function) {
for (auto &Inst : BB) {
for (BinaryBasicBlock &BB : Function) {
for (MCInst &Inst : BB) {
BC.MIB->removeAnnotation(Inst, getProcessedICTag());
}
}

14
bolt/src/Passes/VeneerElimination.cpp
View File

@ -46,13 +46,13 @@ void VeneerElimination::runOnFunctions(BinaryContext &BC) {
VeneersCount++;
BinaryFunction &VeneerFunction = CurrentIt->second;
auto &FirstInstruction = *(VeneerFunction.begin()->begin());
MCInst &FirstInstruction = *(VeneerFunction.begin()->begin());
const MCSymbol *VeneerTargetSymbol =
BC.MIB->getTargetSymbol(FirstInstruction, 1);
// Functions can have multiple symbols
for (auto Name : VeneerFunction.getNames()) {
auto *Symbol = BC.Ctx->lookupSymbol(Name);
for (StringRef Name : VeneerFunction.getNames()) {
MCSymbol *Symbol = BC.Ctx->lookupSymbol(Name);
VeneerDestinations[Symbol] = VeneerTargetSymbol;
BC.SymbolToFunctionMap.erase(Symbol);
}
@ -77,13 +77,13 @@ void VeneerElimination::runOnFunctions(BinaryContext &BC) {
uint64_t VeneerCallers = 0;
for (auto &It : BFs) {
auto &Function = It.second;
for (auto &BB : Function) {
for (auto &Instr : BB) {
BinaryFunction &Function = It.second;
for (BinaryBasicBlock &BB : Function) {
for (MCInst &Instr : BB) {
if (!BC.MIB->isCall(Instr) || BC.MIB->isIndirectCall(Instr))
continue;
auto *TargetSymbol = BC.MIB->getTargetSymbol(Instr, 0);
const MCSymbol *TargetSymbol = BC.MIB->getTargetSymbol(Instr, 0);
if (VeneerDestinations.find(TargetSymbol) == VeneerDestinations.end())
continue;

10
bolt/src/Relocation.cpp
View File

@ -153,8 +153,8 @@ uint64_t extractValueAArch64(uint64_t Type, uint64_t Contents, uint64_t PC) {
// Bits 32:12 of Symbol address goes in bits 30:29 + 23:5 of ADRP
// instruction
Contents &= ~0xffffffff9f00001fUll;
auto LowBits = (Contents >> 29) & 0x3;
auto HighBits = (Contents >> 5) & 0x7ffff;
uint64_t LowBits = (Contents >> 29) & 0x3;
uint64_t HighBits = (Contents >> 5) & 0x7ffff;
Contents = LowBits | (HighBits << 2);
Contents = static_cast<int64_t>(PC) + SignExtend64<32>(Contents << 12);
Contents &= ~0xfffUll;
@ -380,10 +380,10 @@ uint64_t Relocation::getPC64() {
}
size_t Relocation::emit(MCStreamer *Streamer) const {
const auto Size = getSizeForType(Type);
auto &Ctx = Streamer->getContext();
const size_t Size = getSizeForType(Type);
MCContext &Ctx = Streamer->getContext();
if (isPCRelative(Type)) {
auto *TempLabel = Ctx.createNamedTempSymbol();
MCSymbol *TempLabel = Ctx.createNamedTempSymbol();
Streamer->emitLabel(TempLabel);
const MCExpr *Value{nullptr};
if (Symbol) {

640
bolt/src/RewriteInstance.cpp
View File

File diff suppressed because it is too large Load Diff

2
bolt/src/RewriteInstance.h
View File

@ -92,6 +92,8 @@ public:
}
private:
using ELF64LEPhdrTy = ELF64LEFile::Elf_Phdr;
/// Populate array of binary functions and other objects of interest
/// from meta data in the file.
void discoverFileObjects();

6
bolt/src/RuntimeLibs/HugifyRuntimeLibrary.cpp
View File

@ -56,7 +56,7 @@ void HugifyRuntimeLibrary::adjustCommandLineOptions(
}
void HugifyRuntimeLibrary::emitBinary(BinaryContext &BC, MCStreamer &Streamer) {
const auto *StartFunction =
const BinaryFunction *StartFunction =
BC.getBinaryFunctionAtAddress(*(BC.StartFunctionAddress));
assert(!StartFunction->isFragment() && "expected main function fragment");
if (!StartFunction) {
@ -67,7 +67,7 @@ void HugifyRuntimeLibrary::emitBinary(BinaryContext &BC, MCStreamer &Streamer) {
const auto Flags = BinarySection::getFlags(/*IsReadOnly=*/false,
/*IsText=*/false,
/*IsAllocatable=*/true);
auto *Section =
MCSectionELF *Section =
BC.Ctx->getELFSection(".bolt.hugify.entries", ELF::SHT_PROGBITS, Flags);
// __bolt_hugify_init_ptr stores the poiter the hugify library needs to
@ -87,7 +87,7 @@ void HugifyRuntimeLibrary::emitBinary(BinaryContext &BC, MCStreamer &Streamer) {
void HugifyRuntimeLibrary::link(BinaryContext &BC, StringRef ToolPath,
RuntimeDyld &RTDyld,
std::function<void(RuntimeDyld &)> OnLoad) {
auto LibPath = getLibPath(ToolPath, opts::RuntimeHugifyLib);
std::string LibPath = getLibPath(ToolPath, opts::RuntimeHugifyLib);
loadLibrary(LibPath, RTDyld);
OnLoad(RTDyld);
RTDyld.finalizeWithMemoryManagerLocking();

41
bolt/src/RuntimeLibs/InstrumentationRuntimeLibrary.cpp
View File

@ -68,7 +68,7 @@ void InstrumentationRuntimeLibrary::adjustCommandLineOptions(
void InstrumentationRuntimeLibrary::emitBinary(BinaryContext &BC,
MCStreamer &Streamer) {
const auto *StartFunction =
const BinaryFunction *StartFunction =
BC.getBinaryFunctionAtAddress(*BC.StartFunctionAddress);
assert(!StartFunction->isFragment() && "expected main function fragment");
if (!StartFunction) {
@ -76,7 +76,7 @@ void InstrumentationRuntimeLibrary::emitBinary(BinaryContext &BC,
exit(1);
}
const auto *FiniFunction =
const BinaryFunction *FiniFunction =
BC.FiniFunctionAddress
? BC.getBinaryFunctionAtAddress(*BC.FiniFunctionAddress)
: nullptr;
@ -136,7 +136,7 @@ void InstrumentationRuntimeLibrary::emitBinary(BinaryContext &BC,
// Label marking start of the memory region containing instrumentation
// counters, total vector size is Counters.size() 8-byte counters
EmitLabelByName("__bolt_instr_locations");
for (const auto &Label : Summary->Counters) {
for (MCSymbol *const &Label : Summary->Counters) {
EmitLabel(Label, /*IsGlobal*/ false);
Streamer.emitFill(8, 0);
}
@ -185,7 +185,7 @@ void InstrumentationRuntimeLibrary::emitBinary(BinaryContext &BC,
void InstrumentationRuntimeLibrary::link(
BinaryContext &BC, StringRef ToolPath, RuntimeDyld &RTDyld,
std::function<void(RuntimeDyld &)> OnLoad) {
auto LibPath = getLibPath(ToolPath, opts::RuntimeInstrumentationLib);
std::string LibPath = getLibPath(ToolPath, opts::RuntimeInstrumentationLib);
loadLibrary(LibPath, RTDyld);
OnLoad(RTDyld);
RTDyld.finalizeWithMemoryManagerLocking();
@ -245,18 +245,19 @@ std::string InstrumentationRuntimeLibrary::buildTables(BinaryContext &BC) {
// Start of the vector with descriptions (one CounterDescription for each
// counter), vector size is Counters.size() CounterDescription-sized elmts
const auto IDSize =
const size_t IDSize =
Summary->IndCallDescriptions.size() * sizeof(IndCallDescription);
OS.write(reinterpret_cast<const char *>(&IDSize), 4);
for (const auto &Desc : Summary->IndCallDescriptions) {
for (const IndCallDescription &Desc : Summary->IndCallDescriptions) {
OS.write(reinterpret_cast<const char *>(&Desc.FromLoc.FuncString), 4);
OS.write(reinterpret_cast<const char *>(&Desc.FromLoc.Offset), 4);
}
const auto ITDSize = Summary->IndCallTargetDescriptions.size() *
sizeof(IndCallTargetDescription);
const size_t ITDSize = Summary->IndCallTargetDescriptions.size() *
sizeof(IndCallTargetDescription);
OS.write(reinterpret_cast<const char *>(&ITDSize), 4);
for (const auto &Desc : Summary->IndCallTargetDescriptions) {
for (const IndCallTargetDescription &Desc :
Summary->IndCallTargetDescriptions) {
OS.write(reinterpret_cast<const char *>(&Desc.ToLoc.FuncString), 4);
OS.write(reinterpret_cast<const char *>(&Desc.ToLoc.Offset), 4);
uint64_t TargetFuncAddress =
@ -264,18 +265,18 @@ std::string InstrumentationRuntimeLibrary::buildTables(BinaryContext &BC) {
OS.write(reinterpret_cast<const char *>(&TargetFuncAddress), 8);
}
auto FuncDescSize = Summary->getFDSize();
uint32_t FuncDescSize = Summary->getFDSize();
OS.write(reinterpret_cast<const char *>(&FuncDescSize), 4);
for (const auto &Desc : Summary->FunctionDescriptions) {
const auto LeafNum = Desc.LeafNodes.size();
for (const FunctionDescription &Desc : Summary->FunctionDescriptions) {
const size_t LeafNum = Desc.LeafNodes.size();
OS.write(reinterpret_cast<const char *>(&LeafNum), 4);
for (const auto &LeafNode : Desc.LeafNodes) {
for (const InstrumentedNode &LeafNode : Desc.LeafNodes) {
OS.write(reinterpret_cast<const char *>(&LeafNode.Node), 4);
OS.write(reinterpret_cast<const char *>(&LeafNode.Counter), 4);
}
const auto EdgesNum = Desc.Edges.size();
const size_t EdgesNum = Desc.Edges.size();
OS.write(reinterpret_cast<const char *>(&EdgesNum), 4);
for (const auto &Edge : Desc.Edges) {
for (const EdgeDescription &Edge : Desc.Edges) {
OS.write(reinterpret_cast<const char *>(&Edge.FromLoc.FuncString), 4);
OS.write(reinterpret_cast<const char *>(&Edge.FromLoc.Offset), 4);
OS.write(reinterpret_cast<const char *>(&Edge.FromNode), 4);
@ -284,9 +285,9 @@ std::string InstrumentationRuntimeLibrary::buildTables(BinaryContext &BC) {
OS.write(reinterpret_cast<const char *>(&Edge.ToNode), 4);
OS.write(reinterpret_cast<const char *>(&Edge.Counter), 4);
}
const auto CallsNum = Desc.Calls.size();
const size_t CallsNum = Desc.Calls.size();
OS.write(reinterpret_cast<const char *>(&CallsNum), 4);
for (const auto &Call : Desc.Calls) {
for (const CallDescription &Call : Desc.Calls) {
OS.write(reinterpret_cast<const char *>(&Call.FromLoc.FuncString), 4);
OS.write(reinterpret_cast<const char *>(&Call.FromLoc.Offset), 4);
OS.write(reinterpret_cast<const char *>(&Call.FromNode), 4);
@ -297,9 +298,9 @@ std::string InstrumentationRuntimeLibrary::buildTables(BinaryContext &BC) {
getOutputAddress(*Call.Target, Call.ToLoc.Offset);
OS.write(reinterpret_cast<const char *>(&TargetFuncAddress), 8);
}
const auto EntryNum = Desc.EntryNodes.size();
const size_t EntryNum = Desc.EntryNodes.size();
OS.write(reinterpret_cast<const char *>(&EntryNum), 4);
for (const auto &EntryNode : Desc.EntryNodes) {
for (const EntryNode &EntryNode : Desc.EntryNodes) {
OS.write(reinterpret_cast<const char *>(&EntryNode.Node), 8);
uint64_t TargetFuncAddress =
getOutputAddress(*Desc.Function, EntryNode.Address);
@ -315,7 +316,7 @@ std::string InstrumentationRuntimeLibrary::buildTables(BinaryContext &BC) {
void InstrumentationRuntimeLibrary::emitTablesAsELFNote(BinaryContext &BC) {
std::string TablesStr = buildTables(BC);
const auto BoltInfo = BinarySection::encodeELFNote(
const std::string BoltInfo = BinarySection::encodeELFNote(
"BOLT", TablesStr, BinarySection::NT_BOLT_INSTRUMENTATION_TABLES);
BC.registerOrUpdateNoteSection(".bolt.instr.tables", copyByteArray(BoltInfo),
BoltInfo.size(),

6
bolt/src/RuntimeLibs/RuntimeLibrary.cpp
View File

@ -25,7 +25,7 @@ void RuntimeLibrary::anchor() {}
std::string RuntimeLibrary::getLibPath(StringRef ToolPath,
StringRef LibFileName) {
auto Dir = llvm::sys::path::parent_path(ToolPath);
StringRef Dir = llvm::sys::path::parent_path(ToolPath);
SmallString<128> LibPath = llvm::sys::path::parent_path(Dir);
llvm::sys::path::append(LibPath, "lib");
if (!llvm::sys::fs::exists(LibPath)) {
@ -53,9 +53,9 @@ void RuntimeLibrary::loadLibrary(StringRef LibPath, RuntimeDyld &RTDyld) {
if (Magic == file_magic::archive) {
Error Err = Error::success();
object::Archive Archive(B.get()->getMemBufferRef(), Err);
for (auto &C : Archive.children(Err)) {
for (const object::Archive::Child &C : Archive.children(Err)) {
std::unique_ptr<object::Binary> Bin = cantFail(C.getAsBinary());
if (auto *Obj = dyn_cast<object::ObjectFile>(&*Bin)) {
if (object::ObjectFile *Obj = dyn_cast<object::ObjectFile>(&*Bin)) {
RTDyld.loadObject(*Obj);
}
}

52
bolt/src/Target/AArch64/AArch64MCPlusBuilder.cpp
View File

@ -179,13 +179,13 @@ public:
bool isLoadFromStack(const MCInst &Inst) const {
if (!isLoad(Inst))
return false;
const auto InstInfo = Info->get(Inst.getOpcode());
auto NumDefs = InstInfo.getNumDefs();
const MCInstrDesc InstInfo = Info->get(Inst.getOpcode());
unsigned NumDefs = InstInfo.getNumDefs();
for (unsigned I = NumDefs, E = InstInfo.getNumOperands(); I < E; ++I) {
auto &Operand = Inst.getOperand(I);
const MCOperand &Operand = Inst.getOperand(I);
if (!Operand.isReg())
continue;
auto Reg = Operand.getReg();
unsigned Reg = Operand.getReg();
if (Reg == AArch64::SP || Reg == AArch64::WSP ||
Reg == AArch64::FP || Reg == AArch64::W29)
return true;
@ -225,7 +225,7 @@ public:
return true;
// Look for literal addressing mode (see C1-143 ARM DDI 0487B.a)
const auto MCII = Info->get(Inst.getOpcode());
const MCInstrDesc MCII = Info->get(Inst.getOpcode());
for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) {
if (MCII.OpInfo[I].OperandType == MCOI::OPERAND_PCREL)
return true;
@ -237,7 +237,7 @@ public:
const MCExpr **DispExpr) const {
assert((isADR(Inst) || isADRP(Inst)) && "Not an ADR instruction");
auto &Label = Inst.getOperand(1);
const MCOperand &Label = Inst.getOperand(1);
if (!Label.isImm()) {
assert (Label.isExpr() && "Unexpected ADR operand");
assert (DispExpr && "DispExpr must be set");
@ -261,7 +261,7 @@ public:
return evaluateADR(Inst, DispImm, DispExpr);
// Literal addressing mode
const auto MCII = Info->get(Inst.getOpcode());
const MCInstrDesc MCII = Info->get(Inst.getOpcode());
for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) {
if (MCII.OpInfo[I].OperandType != MCOI::OPERAND_PCREL)
continue;
@ -306,7 +306,7 @@ public:
"Unexpected number of operands");
++OI;
} else {
const auto MCII = Info->get(Inst.getOpcode());
const MCInstrDesc MCII = Info->get(Inst.getOpcode());
for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) {
if (MCII.OpInfo[I].OperandType == MCOI::OPERAND_PCREL) {
break;
@ -375,7 +375,7 @@ public:
OpNum = 1;
}
auto &Op = Inst.getOperand(OpNum);
const MCOperand &Op = Inst.getOperand(OpNum);
if (!Op.isExpr())
return nullptr;
@ -457,11 +457,11 @@ public:
assert(Inst.getOpcode() == AArch64::BR && "Unexpected opcode");
// Match the indirect branch pattern for aarch64
auto &UsesRoot = UDChain[&Inst];
SmallVector<MCInst *, 4> &UsesRoot = UDChain[&Inst];
if (UsesRoot.size() == 0 || UsesRoot[0] == nullptr) {
return false;
}
const auto *DefAdd = UsesRoot[0];
const MCInst *DefAdd = UsesRoot[0];
// Now we match an ADD
if (!isADD(*DefAdd)) {
@ -497,9 +497,10 @@ public:
"Failed to match indirect branch!");
// Validate ADD operands
auto OperandExtension = DefAdd->getOperand(3).getImm();
auto ShiftVal = AArch64_AM::getArithShiftValue(OperandExtension);
auto ExtendType = AArch64_AM::getArithExtendType(OperandExtension);
int64_t OperandExtension = DefAdd->getOperand(3).getImm();
unsigned ShiftVal = AArch64_AM::getArithShiftValue(OperandExtension);
AArch64_AM::ShiftExtendType ExtendType =
AArch64_AM::getArithExtendType(OperandExtension);
if (ShiftVal != 2) {
llvm_unreachable("Failed to match indirect branch! (fragment 2)");
}
@ -514,20 +515,20 @@ public:
}
// Match an ADR to load base address to be used when addressing JT targets
auto &UsesAdd = UDChain[DefAdd];
SmallVector<MCInst *, 4> &UsesAdd = UDChain[DefAdd];
if (UsesAdd.size() <= 1 || UsesAdd[1] == nullptr || UsesAdd[2] == nullptr) {
// This happens when we don't have enough context about this jump table
// because the jumping code sequence was split in multiple basic blocks.
// This was observed in the wild in HHVM code (dispatchImpl).
return false;
}
auto *DefBaseAddr = UsesAdd[1];
MCInst *DefBaseAddr = UsesAdd[1];
assert(DefBaseAddr->getOpcode() == AArch64::ADR &&
"Failed to match indirect branch pattern! (fragment 3)");
PCRelBase = DefBaseAddr;
// Match LOAD to load the jump table (relative) target
const auto *DefLoad = UsesAdd[2];
const MCInst *DefLoad = UsesAdd[2];
assert(isLoad(*DefLoad) &&
"Failed to match indirect branch load pattern! (1)");
assert((ScaleValue != 1LL || isLDRB(*DefLoad)) &&
@ -536,8 +537,8 @@ public:
"Failed to match indirect branch load pattern! (3)");
// Match ADD that calculates the JumpTable Base Address (not the offset)
auto &UsesLoad = UDChain[DefLoad];
const auto *DefJTBaseAdd = UsesLoad[1];
SmallVector<MCInst *, 4> &UsesLoad = UDChain[DefLoad];
const MCInst *DefJTBaseAdd = UsesLoad[1];
MCPhysReg From, To;
if (DefJTBaseAdd == nullptr || isLoadFromStack(*DefJTBaseAdd) ||
isRegToRegMove(*DefJTBaseAdd, From, To)) {
@ -552,8 +553,8 @@ public:
if (DefJTBaseAdd->getOperand(2).isImm())
Offset = DefJTBaseAdd->getOperand(2).getImm();
auto &UsesJTBaseAdd = UDChain[DefJTBaseAdd];
const auto *DefJTBasePage = UsesJTBaseAdd[1];
SmallVector<MCInst *, 4> &UsesJTBaseAdd = UDChain[DefJTBaseAdd];
const MCInst *DefJTBasePage = UsesJTBaseAdd[1];
if (DefJTBasePage == nullptr || isLoadFromStack(*DefJTBasePage)) {
JumpTable = nullptr;
return true;
@ -593,7 +594,7 @@ public:
LLVM_DEBUG(dbgs() << "computeLocalUDChain\n");
bool TerminatorSeen = false;
for (auto II = Begin; II != End; ++II) {
auto &Instr = *II;
MCInst &Instr = *II;
// Ignore nops and CFIs
if (Info->get(Instr.getOpcode()).isPseudo() || isNoop(Instr))
continue;
@ -656,7 +657,8 @@ public:
int64_t ScaleValue, DispValue;
const MCExpr *DispExpr;
auto UDChain = computeLocalUDChain(&Instruction, Begin, End);
DenseMap<const MCInst *, SmallVector<llvm::MCInst *, 4>> UDChain =
computeLocalUDChain(&Instruction, Begin, End);
MCInst *PCRelBase;
if (!analyzeIndirectBranchFragment(Instruction, UDChain, DispExpr,
DispValue, ScaleValue, PCRelBase)) {
@ -828,7 +830,7 @@ public:
// unreachable code. Ignore them.
CondBranch = nullptr;
UncondBranch = &*I;
const auto *Sym = getTargetSymbol(*I);
const MCSymbol *Sym = getTargetSymbol(*I);
assert(Sym != nullptr &&
"Couldn't extract BB symbol from jump operand");
TBB = Sym;
@ -841,7 +843,7 @@ public:
}
if (CondBranch == nullptr) {
const auto *TargetBB = getTargetSymbol(*I);
const MCSymbol *TargetBB = getTargetSymbol(*I);
if (TargetBB == nullptr) {
// Unrecognized branch target
return false;

198
bolt/src/Target/X86/X86MCPlusBuilder.cpp
View File

@ -621,8 +621,8 @@ public:
if (isPop(Inst))
return true;
auto MemOpNo = getMemoryOperandNo(Inst);
const auto MCII = Info->get(Inst.getOpcode());
int MemOpNo = getMemoryOperandNo(Inst);
const MCInstrDesc MCII = Info->get(Inst.getOpcode());
if (MemOpNo == -1)
return false;
@ -634,8 +634,8 @@ public:
if (isPush(Inst))
return true;
auto MemOpNo = getMemoryOperandNo(Inst);
const auto MCII = Info->get(Inst.getOpcode());
int MemOpNo = getMemoryOperandNo(Inst);
const MCInstrDesc MCII = Info->get(Inst.getOpcode());
if (MemOpNo == -1)
return false;
@ -679,7 +679,7 @@ public:
if (CurInst->getOpcode() != X86::JMP64m)
return false;
auto MemOpNo = MIB.getMemoryOperandNo(*CurInst);
int MemOpNo = MIB.getMemoryOperandNo(*CurInst);
if (MemOpNo == -1)
return false;
@ -766,7 +766,7 @@ public:
CurInst->getOpcode() != X86::MOVSX64rm32)
return false;
auto MemOpNo = MIB.getMemoryOperandNo(*CurInst);
int MemOpNo = MIB.getMemoryOperandNo(*CurInst);
if (MemOpNo == -1)
return false;
@ -886,7 +886,7 @@ public:
}
bool hasPCRelOperand(const MCInst &Inst) const override {
for (const auto &Operand : Inst) {
for (const MCOperand &Operand : Inst) {
if (Operand.isReg() && Operand.getReg() == X86::RIP)
return true;
}
@ -894,35 +894,35 @@ public:
}
int getMemoryOperandNo(const MCInst &Inst) const override {
auto Opcode = Inst.getOpcode();
auto const &Desc = Info->get(Opcode);
auto MemOpNo = X86II::getMemoryOperandNo(Desc.TSFlags);
unsigned Opcode = Inst.getOpcode();
const MCInstrDesc &Desc = Info->get(Opcode);
int MemOpNo = X86II::getMemoryOperandNo(Desc.TSFlags);
if (MemOpNo >= 0)
MemOpNo += X86II::getOperandBias(Desc);
return MemOpNo;
}
bool hasEVEXEncoding(const MCInst &Inst) const override {
auto const &Desc = Info->get(Inst.getOpcode());
const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
return (Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX;
}
bool isMacroOpFusionPair(ArrayRef<MCInst> Insts) const override {
// FIXME: the macro-op fusion is triggered under different conditions
// on different cores. This implementation is for sandy-bridge+.
auto I = Insts.begin();
const auto *I = Insts.begin();
while (I != Insts.end() && isPrefix(*I))
++I;
if (I == Insts.end())
return false;
const auto &FirstInst = *I;
const MCInst &FirstInst = *I;
++I;
while (I != Insts.end() && isPrefix(*I))
++I;
if (I == Insts.end())
return false;
const auto &SecondInst = *I;
const MCInst &SecondInst = *I;
if (!isConditionalBranch(SecondInst))
return false;
@ -935,8 +935,8 @@ public:
return false;
// Cannot fuse if first instruction operands are MEM-IMM.
auto const &Desc = Info->get(FirstInst.getOpcode());
auto MemOpNo = X86II::getMemoryOperandNo(Desc.TSFlags);
const MCInstrDesc &Desc = Info->get(FirstInst.getOpcode());
int MemOpNo = X86II::getMemoryOperandNo(Desc.TSFlags);
if (MemOpNo != -1 && X86II::hasImm(Desc.TSFlags))
return false;
@ -958,7 +958,8 @@ public:
if (FirstInstGroup == 0)
return false;
const auto CondCode = getCanonicalBranchCondCode(getCondCode(SecondInst));
const unsigned CondCode =
getCanonicalBranchCondCode(getCondCode(SecondInst));
switch (CondCode) {
default:
llvm_unreachable("unexpected conditional code");
@ -991,7 +992,7 @@ public:
const override {
assert(BaseRegNum && ScaleImm && IndexRegNum && SegmentRegNum &&
"one of the input pointers is null");
auto MemOpNo = getMemoryOperandNo(Inst);
int MemOpNo = getMemoryOperandNo(Inst);
if (MemOpNo < 0)
return false;
unsigned MemOpOffset = static_cast<unsigned>(MemOpNo);
@ -999,11 +1000,12 @@ public:
if (MemOpOffset + X86::AddrSegmentReg >= MCPlus::getNumPrimeOperands(Inst))
return false;
auto &Base = Inst.getOperand(MemOpOffset + X86::AddrBaseReg);
auto &Scale = Inst.getOperand(MemOpOffset + X86::AddrScaleAmt);
auto &Index = Inst.getOperand(MemOpOffset + X86::AddrIndexReg);
auto &Disp = Inst.getOperand(MemOpOffset + X86::AddrDisp);
auto &Segment = Inst.getOperand(MemOpOffset + X86::AddrSegmentReg);
const MCOperand &Base = Inst.getOperand(MemOpOffset + X86::AddrBaseReg);
const MCOperand &Scale = Inst.getOperand(MemOpOffset + X86::AddrScaleAmt);
const MCOperand &Index = Inst.getOperand(MemOpOffset + X86::AddrIndexReg);
const MCOperand &Disp = Inst.getOperand(MemOpOffset + X86::AddrDisp);
const MCOperand &Segment =
Inst.getOperand(MemOpOffset + X86::AddrSegmentReg);
// Make sure it is a well-formed memory operand.
if (!Base.isReg() || !Scale.isImm() || !Index.isReg() ||
@ -1061,14 +1063,14 @@ public:
}
MCInst::iterator getMemOperandDisp(MCInst &Inst) const override {
auto MemOpNo = getMemoryOperandNo(Inst);
int MemOpNo = getMemoryOperandNo(Inst);
if (MemOpNo < 0)
return Inst.end();
return Inst.begin() + (MemOpNo + X86::AddrDisp);
}
bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override {
auto OI = getMemOperandDisp(Inst);
MCOperand *OI = getMemOperandDisp(Inst);
if (OI == Inst.end())
return false;
*OI = Operand;
@ -1293,7 +1295,7 @@ public:
case X86::MOVZX32rr8:
case X86::TEST8ri:
for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) {
const auto &Operand = Inst.getOperand(I);
const MCOperand &Operand = Inst.getOperand(I);
if (!Operand.isReg())
continue;
if (isUpper8BitReg(Operand.getReg()))
@ -1311,7 +1313,7 @@ public:
uint8_t &Size, bool &IsSimple,
bool &IsIndexed) const override {
// Detect simple push/pop cases first
if (auto Sz = getPushSize(Inst)) {
if (int Sz = getPushSize(Inst)) {
IsLoad = false;
IsStore = true;
IsStoreFromReg = true;
@ -1328,7 +1330,7 @@ public:
}
return true;
}
if (auto Sz = getPopSize(Inst)) {
if (int Sz = getPopSize(Inst)) {
assert(Inst.getOperand(0).isReg() &&
"Expected register operand for push");
IsLoad = true;
@ -1351,8 +1353,8 @@ public:
};
InstInfo I;
auto MemOpNo = getMemoryOperandNo(Inst);
const auto MCII = Info->get(Inst.getOpcode());
int MemOpNo = getMemoryOperandNo(Inst);
const MCInstrDesc MCII = Info->get(Inst.getOpcode());
// If it is not dealing with a memory operand, we discard it
if (MemOpNo == -1 || MCII.isCall())
return false;
@ -1423,11 +1425,12 @@ public:
// Retrieve related register in simple MOV from/to stack operations.
unsigned MemOpOffset = static_cast<unsigned>(MemOpNo);
if (I.IsLoad) {
auto RegOpnd = Inst.getOperand(0);
MCOperand RegOpnd = Inst.getOperand(0);
assert(RegOpnd.isReg() && "unexpected destination operand");
Reg = RegOpnd.getReg();
} else if (I.IsStore) {
auto SrcOpnd = Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1);
MCOperand SrcOpnd =
Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1);
if (I.StoreFromReg) {
assert(SrcOpnd.isReg() && "unexpected source operand");
Reg = SrcOpnd.getReg();
@ -1493,12 +1496,13 @@ public:
default:
assert(false);
}
auto RegOpndNum = Inst.getOperand(0).getReg();
unsigned RegOpndNum = Inst.getOperand(0).getReg();
Inst.clear();
Inst.setOpcode(NewOpcode);
Inst.addOperand(MCOperand::createReg(RegOpndNum));
} else {
auto SrcOpnd = Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1);
MCOperand SrcOpnd =
Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1);
if (I.StoreFromReg) {
switch (I.DataSize) {
case 2: NewOpcode = X86::PUSH16r; break;
@ -1508,7 +1512,7 @@ public:
assert(false);
}
assert(SrcOpnd.isReg() && "unexpected source operand");
auto RegOpndNum = SrcOpnd.getReg();
unsigned RegOpndNum = SrcOpnd.getReg();
Inst.clear();
Inst.setOpcode(NewOpcode);
Inst.addOperand(MCOperand::createReg(RegOpndNum));
@ -1521,7 +1525,7 @@ public:
assert(false);
}
assert(SrcOpnd.isImm() && "unexpected source operand");
auto SrcImm = SrcOpnd.getImm();
int64_t SrcImm = SrcOpnd.getImm();
Inst.clear();
Inst.setOpcode(NewOpcode);
Inst.addOperand(MCOperand::createImm(SrcImm));
@ -1541,9 +1545,9 @@ public:
break;
}
const auto MCII = Info->get(Inst.getOpcode());
const MCInstrDesc MCII = Info->get(Inst.getOpcode());
for (int I = 0, E = MCII.getNumDefs(); I != E; ++I) {
const auto &Operand = Inst.getOperand(I);
const MCOperand &Operand = Inst.getOperand(I);
if (Operand.isReg() && Operand.getReg() == X86::RSP) {
return true;
}
@ -1555,7 +1559,7 @@ public:
std::pair<MCPhysReg, int64_t> Input1,
std::pair<MCPhysReg, int64_t> Input2) const override {
auto getOperandVal = [&] (MCPhysReg Reg) -> ErrorOr<int64_t> {
auto getOperandVal = [&](MCPhysReg Reg) -> ErrorOr<int64_t> {
if (Reg == Input1.first)
return Input1.second;
if (Reg == Input2.first)
@ -1571,7 +1575,8 @@ public:
case X86::AND64ri8:
if (!Inst.getOperand(2).isImm())
return false;
if (auto InputVal = getOperandVal(Inst.getOperand(1).getReg())) {
if (ErrorOr<int64_t> InputVal =
getOperandVal(Inst.getOperand(1).getReg())) {
Output = *InputVal & Inst.getOperand(2).getImm();
} else {
return false;
@ -1581,7 +1586,8 @@ public:
case X86::SUB64ri8:
if (!Inst.getOperand(2).isImm())
return false;
if (auto InputVal = getOperandVal(Inst.getOperand(1).getReg())) {
if (ErrorOr<int64_t> InputVal =
getOperandVal(Inst.getOperand(1).getReg())) {
Output = *InputVal - Inst.getOperand(2).getImm();
} else {
return false;
@ -1591,7 +1597,8 @@ public:
case X86::ADD64ri8:
if (!Inst.getOperand(2).isImm())
return false;
if (auto InputVal = getOperandVal(Inst.getOperand(1).getReg())) {
if (ErrorOr<int64_t> InputVal =
getOperandVal(Inst.getOperand(1).getReg())) {
Output = *InputVal + Inst.getOperand(2).getImm();
} else {
return false;
@ -1600,7 +1607,7 @@ public:
case X86::ADD64i32:
if (!Inst.getOperand(0).isImm())
return false;
if (auto InputVal = getOperandVal(X86::RAX)) {
if (ErrorOr<int64_t> InputVal = getOperandVal(X86::RAX)) {
Output = *InputVal + Inst.getOperand(0).getImm();
} else {
return false;
@ -1625,7 +1632,7 @@ public:
return false;
}
if (auto InputVal = getOperandVal(BaseRegNum)) {
if (ErrorOr<int64_t> InputVal = getOperandVal(BaseRegNum)) {
Output = *InputVal + DispValue;
} else {
return false;
@ -1659,9 +1666,9 @@ public:
MCPhysReg getFlagsReg() const override { return X86::EFLAGS; }
bool escapesVariable(const MCInst &Inst, bool HasFramePointer) const override {
auto MemOpNo = getMemoryOperandNo(Inst);
const auto MCII = Info->get(Inst.getOpcode());
const auto NumDefs = MCII.getNumDefs();
int MemOpNo = getMemoryOperandNo(Inst);
const MCInstrDesc MCII = Info->get(Inst.getOpcode());
const unsigned NumDefs = MCII.getNumDefs();
static BitVector SPBPAliases(BitVector(getAliases(X86::RSP)) |=
getAliases(X86::RBP));
static BitVector SPAliases(getAliases(X86::RSP));
@ -1686,7 +1693,7 @@ public:
if (I < static_cast<int>(NumDefs))
continue;
const auto &Operand = Inst.getOperand(I);
const MCOperand &Operand = Inst.getOperand(I);
if (HasFramePointer && Operand.isReg() && SPBPAliases[Operand.getReg()]) {
DoesLeak = true;
break;
@ -1700,7 +1707,7 @@ public:
// If potential leak, check if it is not just writing to itself/sp/bp
if (DoesLeak) {
for (int I = 0, E = NumDefs; I != E; ++I) {
const auto &Operand = Inst.getOperand(I);
const MCOperand &Operand = Inst.getOperand(I);
if (HasFramePointer && Operand.isReg() &&
SPBPAliases[Operand.getReg()]) {
DoesLeak = false;
@ -1718,7 +1725,7 @@ public:
bool addToImm(MCInst &Inst, int64_t &Amt, MCContext *Ctx) const override {
unsigned ImmOpNo = -1U;
auto MemOpNo = getMemoryOperandNo(Inst);
int MemOpNo = getMemoryOperandNo(Inst);
if (MemOpNo != -1) {
ImmOpNo = MemOpNo + X86::AddrDisp;
} else {
@ -1883,17 +1890,17 @@ public:
// Compute the new opcode.
unsigned NewOpcode = 0;
for (const auto &Check : I.Checks) {
for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) {
NewOpcode = Check.second;
if (Check.first == NOCHECK)
break;
else if (Check.first == CHECK8 &&
ImmVal >= std::numeric_limits<int8_t>::min() &&
ImmVal <= std::numeric_limits<int8_t>::max())
if (Check.first == CHECK8 &&
ImmVal >= std::numeric_limits<int8_t>::min() &&
ImmVal <= std::numeric_limits<int8_t>::max())
break;
else if (Check.first == CHECK32 &&
ImmVal >= std::numeric_limits<int32_t>::min() &&
ImmVal <= std::numeric_limits<int32_t>::max())
if (Check.first == CHECK32 &&
ImmVal >= std::numeric_limits<int32_t>::min() &&
ImmVal <= std::numeric_limits<int32_t>::max())
break;
}
if (NewOpcode == Inst.getOpcode())
@ -1988,9 +1995,9 @@ public:
}
bool requiresAlignedAddress(const MCInst &Inst) const override {
auto const &Desc = Info->get(Inst.getOpcode());
const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
for (unsigned int I = 0; I < Desc.getNumOperands(); ++I) {
const auto &Op = Desc.OpInfo[I];
const MCOperandInfo &Op = Desc.OpInfo[I];
if (Op.OperandType != MCOI::OPERAND_REGISTER)
continue;
if (Op.RegClass == X86::VR128RegClassID)
@ -2131,9 +2138,9 @@ public:
// Check and remove EIZ/RIZ. These cases represent ambiguous cases where SIB
// byte is present, but no index is used and modrm alone shoud have been
// enough. Converting to NoRegister effectively removes the SIB byte.
auto MemOpNo = getMemoryOperandNo(Inst);
int MemOpNo = getMemoryOperandNo(Inst);
if (MemOpNo >= 0) {
auto &IndexOp =
MCOperand &IndexOp =
Inst.getOperand(static_cast<unsigned>(MemOpNo) + X86::AddrIndexReg);
if (IndexOp.getReg() == X86::EIZ ||
IndexOp.getReg() == X86::RIZ) {
@ -2145,18 +2152,18 @@ public:
NewOpcode = getShortBranchOpcode(OldOpcode);
} else if (OldOpcode == X86::MOV64ri) {
if (Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) {
const auto Imm =
Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm();
const int64_t Imm =
Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm();
if (int64_t(Imm) == int64_t(int32_t(Imm))) {
NewOpcode = X86::MOV64ri32;
}
}
} else {
// If it's arithmetic instruction check if signed operand fits in 1 byte.
const auto ShortOpcode = getShortArithOpcode(OldOpcode);
const unsigned ShortOpcode = getShortArithOpcode(OldOpcode);
if (ShortOpcode != OldOpcode &&
Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) {
auto Imm =
int64_t Imm =
Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm();
if (int64_t(Imm) == int64_t(int8_t(Imm))) {
NewOpcode = ShortOpcode;
@ -2185,7 +2192,7 @@ public:
if (OpNum >= MCPlus::getNumPrimeOperands(Inst))
return nullptr;
auto &Op = Inst.getOperand(OpNum);
const MCOperand &Op = Inst.getOperand(OpNum);
if (!Op.isExpr())
return nullptr;
@ -2234,7 +2241,7 @@ public:
// unreachable code. Ignore them.
CondBranch = nullptr;
UncondBranch = &*I;
const auto *Sym = getTargetSymbol(*I);
const MCSymbol *Sym = getTargetSymbol(*I);
assert(Sym != nullptr &&
"Couldn't extract BB symbol from jump operand");
TBB = Sym;
@ -2249,7 +2256,7 @@ public:
}
if (CondBranch == nullptr) {
const auto *TargetBB = getTargetSymbol(*I);
const MCSymbol *TargetBB = getTargetSymbol(*I);
if (TargetBB == nullptr) {
// Unrecognized branch target
return false;
@ -2308,8 +2315,8 @@ public:
MCInst *MemLocInstr = nullptr;
const MCInst *MovInstr = nullptr;
while (++II != IE) {
auto &Instr = *II;
const auto &InstrDesc = Info->get(Instr.getOpcode());
MCInst &Instr = *II;
const MCInstrDesc &InstrDesc = Info->get(Instr.getOpcode());
if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo) &&
!InstrDesc.hasDefOfPhysReg(Instr, R2, *RegInfo)) {
// Ignore instructions that don't affect R1, R2 registers.
@ -2324,7 +2331,7 @@ public:
// Check if it's setting %r1 or %r2. In canonical form it sets %r2.
// If it sets %r1 - rename the registers so we have to only check
// a single form.
auto MovDestReg = Instr.getOperand(0).getReg();
unsigned MovDestReg = Instr.getOperand(0).getReg();
if (MovDestReg != R2)
std::swap(R1, R2);
if (MovDestReg != R2) {
@ -2443,11 +2450,11 @@ public:
// If the indirect jump is on register - try to detect if the
// register value is loaded from a memory location.
assert(Instruction.getOperand(0).isReg() && "register operand expected");
const auto R1 = Instruction.getOperand(0).getReg();
const unsigned R1 = Instruction.getOperand(0).getReg();
// Check if one of the previous instructions defines the jump-on register.
for (auto PrevII = II; PrevII != IE; ++PrevII) {
auto &PrevInstr = *PrevII;
const auto &PrevInstrDesc = Info->get(PrevInstr.getOpcode());
MCInst &PrevInstr = *PrevII;
const MCInstrDesc &PrevInstrDesc = Info->get(PrevInstr.getOpcode());
if (!PrevInstrDesc.hasDefOfPhysReg(PrevInstr, R1, *RegInfo))
continue;
@ -2456,7 +2463,7 @@ public:
MemLocInstr = &PrevInstr;
break;
} else if (isADD64rr(PrevInstr)) {
auto R2 = PrevInstr.getOperand(2).getReg();
unsigned R2 = PrevInstr.getOperand(2).getReg();
if (R1 == R2)
return IndirectBranchType::UNKNOWN;
std::tie(Type, MemLocInstr) = analyzePICJumpTable(PrevII, IE, R1, R2);
@ -2477,7 +2484,7 @@ public:
MemLocInstr = &Instruction;
}
const auto RIPRegister = RegInfo->getProgramCounter();
const MCRegister RIPRegister = RegInfo->getProgramCounter();
// Analyze the memory location.
unsigned BaseRegNum, IndexRegNum, SegRegNum;
@ -2552,7 +2559,7 @@ public:
std::reverse_iterator<InstructionIterator> Itr(ForwardEnd);
std::reverse_iterator<InstructionIterator> End(ForwardBegin);
auto &CallInst = *Itr++;
MCInst &CallInst = *Itr++;
assert(isIndirectBranch(CallInst) || isCall(CallInst));
unsigned BaseReg, IndexReg, SegmentReg;
@ -2596,8 +2603,8 @@ public:
// find load from vtable, this may or may not include the method offset
while (Itr != End) {
auto &CurInst = *Itr++;
const auto &Desc = Info->get(CurInst.getOpcode());
MCInst &CurInst = *Itr++;
const MCInstrDesc &Desc = Info->get(CurInst.getOpcode());
if (Desc.hasDefOfPhysReg(CurInst, MethodRegNum, *RegInfo)) {
if (isLoad(CurInst) &&
evaluateX86MemoryOperand(CurInst,
@ -2632,8 +2639,8 @@ public:
// look for any adds affecting the method register.
while (Itr != End) {
auto &CurInst = *Itr++;
const auto &Desc = Info->get(CurInst.getOpcode());
MCInst &CurInst = *Itr++;
const MCInstrDesc &Desc = Info->get(CurInst.getOpcode());
if (Desc.hasDefOfPhysReg(CurInst, VtableRegNum, *RegInfo)) {
if (isADDri(CurInst)) {
assert(!MethodOffset);
@ -3094,7 +3101,7 @@ public:
}
bool isBranchOnMem(const MCInst &Inst) const override {
auto OpCode = Inst.getOpcode();
unsigned OpCode = Inst.getOpcode();
if (OpCode == X86::CALL64m || (OpCode == X86::JMP32m && isTailCall(Inst)) ||
OpCode == X86::JMP64m)
return true;
@ -3103,7 +3110,7 @@ public:
}
bool isBranchOnReg(const MCInst &Inst) const override {
auto OpCode = Inst.getOpcode();
unsigned OpCode = Inst.getOpcode();
if (OpCode == X86::CALL64r || (OpCode == X86::JMP32r && isTailCall(Inst)) ||
OpCode == X86::JMP64r)
return true;
@ -3204,7 +3211,7 @@ public:
// Skip defs.
for (unsigned I = Info->get(CallInst.getOpcode()).getNumDefs(),
E = MCPlus::getNumPrimeOperands(CallInst); I != E; ++I) {
const auto &Operand = CallInst.getOperand(I);
const MCOperand &Operand = CallInst.getOperand(I);
if (Operand.isReg() && SPAliases[Operand.getReg()]) {
UsesSP = true;
break;
@ -3348,8 +3355,8 @@ public:
if (MinimizeCodeSize && !LoadElim) {
std::set<unsigned> UsedRegs;
for (unsigned int i = 0; i < MCPlus::getNumPrimeOperands(CallInst); ++i) {
const auto &Op = CallInst.getOperand(i);
for (unsigned int I = 0; I < MCPlus::getNumPrimeOperands(CallInst); ++I) {
const MCOperand &Op = CallInst.getOperand(I);
if (Op.isReg()) {
UsedRegs.insert(Op.getReg());
}
@ -3390,7 +3397,7 @@ public:
MCSymbolRefExpr::VK_None,
*Ctx)));
} else {
const auto Addr = Targets[i].second;
const uint64_t Addr = Targets[i].second;
// Immediate address is out of sign extended 32 bit range.
if (int64_t(Addr) != int64_t(int32_t(Addr))) {
return BlocksVectorTy();
@ -3443,8 +3450,9 @@ public:
// Target address.
if (Targets[i].first || LoadElim) {
const auto *Sym = LoadElim ? VtableSyms[i].first : Targets[i].first;
const auto Addend = LoadElim ? VtableSyms[i].second : 0;
const MCSymbol *Sym =
LoadElim ? VtableSyms[i].first : Targets[i].first;
const uint64_t Addend = LoadElim ? VtableSyms[i].second : 0;
const MCExpr *Expr = MCSymbolRefExpr::create(Sym, *Ctx);
@ -3456,7 +3464,7 @@ public:
Compare.addOperand(MCOperand::createExpr(Expr));
} else {
const auto Addr = Targets[i].second;
const uint64_t Addr = Targets[i].second;
// Immediate address is out of sign extended 32 bit range.
if (int64_t(Addr) != int64_t(int32_t(Addr))) {
return BlocksVectorTy();
@ -3524,10 +3532,10 @@ public:
if (isInvoke(CallInst) && !isInvoke(CallOrJmp)) {
// Copy over any EH or GNU args size information from the original
// call.
auto EHInfo = getEHInfo(CallInst);
Optional<MCPlus::MCLandingPad> EHInfo = getEHInfo(CallInst);
if (EHInfo)
addEHInfo(CallOrJmp, *EHInfo);
auto GnuArgsSize = getGnuArgsSize(CallInst);
int64_t GnuArgsSize = getGnuArgsSize(CallInst);
if (GnuArgsSize >= 0)
addGnuArgsSize(CallOrJmp, GnuArgsSize);
}
@ -3549,7 +3557,7 @@ public:
// Cold call block.
Results.push_back(std::make_pair(NextTarget, std::vector<MCInst>()));
std::vector<MCInst> &NewCall = Results.back().second;
for (auto *Inst : MethodFetchInsns) {
for (const MCInst *Inst : MethodFetchInsns) {
if (Inst != &CallInst)
NewCall.push_back(*Inst);
}
@ -3593,7 +3601,7 @@ public:
CompareInst.setOpcode(X86::CMP64ri32);
CompareInst.addOperand(MCOperand::createReg(IndexReg));
const auto CaseIdx = Targets[i].second;
const uint64_t CaseIdx = Targets[i].second;
// Immediate address is out of sign extended 32 bit range.
if (int64_t(CaseIdx) != int64_t(int32_t(CaseIdx))) {
return BlocksVectorTy();
@ -3619,7 +3627,7 @@ public:
// Cold call block.
Results.push_back(std::make_pair(NextTarget, std::vector<MCInst>()));
std::vector<MCInst> &CurBB = Results.back().second;
for (auto *Inst : TargetFetchInsns) {
for (const MCInst *Inst : TargetFetchInsns) {
if (Inst != &IJmpInst)
CurBB.push_back(*Inst);
}

86
bolt/src/YAMLProfileReader.cpp
View File

@ -37,10 +37,11 @@ namespace llvm {
namespace bolt {
bool YAMLProfileReader::isYAML(const StringRef Filename) {
auto MB = MemoryBuffer::getFileOrSTDIN(Filename);
ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
MemoryBuffer::getFileOrSTDIN(Filename);
if (std::error_code EC = MB.getError())
report_error(Filename, EC);
auto Buffer = MB.get()->getBuffer();
StringRef Buffer = MB.get()->getBuffer();
if (Buffer.startswith("---\n"))
return true;
return false;
@ -48,20 +49,20 @@ bool YAMLProfileReader::isYAML(const StringRef Filename) {
void YAMLProfileReader::buildNameMaps(
std::map<uint64_t, BinaryFunction> &Functions) {
for (auto &YamlBF : YamlBP.Functions) {
for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) {
StringRef Name = YamlBF.Name;
const auto Pos = Name.find("(*");
const size_t Pos = Name.find("(*");
if (Pos != StringRef::npos)
Name = Name.substr(0, Pos);
ProfileNameToProfile[Name] = &YamlBF;
if (const auto CommonName = getLTOCommonName(Name)) {
if (const Optional<StringRef> CommonName = getLTOCommonName(Name)) {
LTOCommonNameMap[*CommonName].push_back(&YamlBF);
}
}
for (auto &BFI : Functions) {
const auto &Function = BFI.second;
for (auto Name : Function.getNames()) {
if (const auto CommonName = getLTOCommonName(Name)) {
const BinaryFunction &Function = BFI.second;
for (StringRef Name : Function.getNames()) {
if (const Optional<StringRef> CommonName = getLTOCommonName(Name)) {
LTOCommonNameFunctionMap[*CommonName].insert(&Function);
}
}
@ -69,8 +70,9 @@ void YAMLProfileReader::buildNameMaps(
}
bool YAMLProfileReader::hasLocalsWithFileName() const {
for (const auto &KV : ProfileNameToProfile) {
const auto &FuncName = KV.getKey();
for (const StringMapEntry<yaml::bolt::BinaryFunctionProfile *> &KV :
ProfileNameToProfile) {
const StringRef &FuncName = KV.getKey();
if (FuncName.count('/') == 2 && FuncName[0] != '/')
return true;
}
@ -80,7 +82,7 @@ bool YAMLProfileReader::hasLocalsWithFileName() const {
bool YAMLProfileReader::parseFunctionProfile(
BinaryFunction &BF,
const yaml::bolt::BinaryFunctionProfile &YamlBF) {
auto &BC = BF.getBinaryContext();
BinaryContext &BC = BF.getBinaryContext();
bool ProfileMatched = true;
uint64_t MismatchedBlocks = 0;
@ -103,9 +105,9 @@ bool YAMLProfileReader::parseFunctionProfile(
ProfileMatched = false;
}
auto DFSOrder = BF.dfs();
std::vector<BinaryBasicBlock *> DFSOrder = BF.dfs();
for (const auto &YamlBB : YamlBF.Blocks) {
for (const yaml::bolt::BinaryBasicBlockProfile &YamlBB : YamlBF.Blocks) {
if (YamlBB.Index >= DFSOrder.size()) {
if (opts::Verbosity >= 2)
errs() << "BOLT-WARNING: index " << YamlBB.Index
@ -114,7 +116,7 @@ bool YAMLProfileReader::parseFunctionProfile(
continue;
}
auto &BB = *DFSOrder[YamlBB.Index];
BinaryBasicBlock &BB = *DFSOrder[YamlBB.Index];
// Basic samples profile (without LBR) does not have branches information
// and needs a special processing.
@ -123,7 +125,7 @@ bool YAMLProfileReader::parseFunctionProfile(
BB.setExecutionCount(0);
continue;
}
auto NumSamples = YamlBB.EventCount * 1000;
uint64_t NumSamples = YamlBB.EventCount * 1000;
if (NormalizeByInsnCount && BB.getNumNonPseudos()) {
NumSamples /= BB.getNumNonPseudos();
} else if (NormalizeByCalls) {
@ -137,9 +139,10 @@ bool YAMLProfileReader::parseFunctionProfile(
BB.setExecutionCount(YamlBB.ExecCount);
for (const auto &YamlCSI: YamlBB.CallSites) {
auto *Callee = YamlCSI.DestId < YamlProfileToFunction.size() ?
YamlProfileToFunction[YamlCSI.DestId] : nullptr;
for (const yaml::bolt::CallSiteInfo &YamlCSI : YamlBB.CallSites) {
BinaryFunction *Callee = YamlCSI.DestId < YamlProfileToFunction.size()
? YamlProfileToFunction[YamlCSI.DestId]
: nullptr;
bool IsFunction = Callee ? true : false;
MCSymbol *CalleeSymbol = nullptr;
if (IsFunction) {
@ -156,8 +159,8 @@ bool YAMLProfileReader::parseFunctionProfile(
continue;
}
auto *Instr =
BF.getInstructionAtOffset(BB.getInputOffset() + YamlCSI.Offset);
MCInst *Instr =
BF.getInstructionAtOffset(BB.getInputOffset() + YamlCSI.Offset);
if (!Instr) {
if (opts::Verbosity >= 2)
errs() << "BOLT-WARNING: no instruction at offset " << YamlCSI.Offset
@ -185,9 +188,8 @@ bool YAMLProfileReader::parseFunctionProfile(
};
if (BC.MIB->isIndirectCall(*Instr) || BC.MIB->isIndirectBranch(*Instr)) {
IndirectCallSiteProfile &CSP =
BC.MIB->getOrCreateAnnotationAs<IndirectCallSiteProfile>(
*Instr, "CallProfile");
auto &CSP = BC.MIB->getOrCreateAnnotationAs<IndirectCallSiteProfile>(
*Instr, "CallProfile");
CSP.emplace_back(CalleeSymbol, YamlCSI.Count, YamlCSI.Mispreds);
} else if (BC.MIB->getConditionalTailCall(*Instr)) {
setAnnotation("CTCTakenCount", YamlCSI.Count);
@ -197,7 +199,7 @@ bool YAMLProfileReader::parseFunctionProfile(
}
}
for (const auto &YamlSI : YamlBB.Successors) {
for (const yaml::bolt::SuccessorInfo &YamlSI : YamlBB.Successors) {
if (YamlSI.Index >= DFSOrder.size()) {
if (opts::Verbosity >= 1)
errs() << "BOLT-WARNING: index out of bounds for profiled block\n";
@ -205,7 +207,7 @@ bool YAMLProfileReader::parseFunctionProfile(
continue;
}
auto &SuccessorBB = *DFSOrder[YamlSI.Index];
BinaryBasicBlock &SuccessorBB = *DFSOrder[YamlSI.Index];
if (!BB.getSuccessor(SuccessorBB.getLabel())) {
if (opts::Verbosity >= 1)
errs() << "BOLT-WARNING: no successor for block " << BB.getName()
@ -215,14 +217,14 @@ bool YAMLProfileReader::parseFunctionProfile(
continue;
}
auto &BI = BB.getBranchInfo(SuccessorBB);
BinaryBasicBlock::BinaryBranchInfo &BI = BB.getBranchInfo(SuccessorBB);
BI.Count += YamlSI.Count;
BI.MispredictedCount += YamlSI.Mispreds;
}
}
// If basic block profile wasn't read it should be 0.
for (auto &BB : BF) {
for (BinaryBasicBlock &BB : BF) {
if (BB.getExecutionCount() == BinaryBasicBlock::COUNT_NO_PROFILE)
BB.setExecutionCount(0);
}
@ -247,7 +249,8 @@ bool YAMLProfileReader::parseFunctionProfile(
}
Error YAMLProfileReader::preprocessProfile(BinaryContext &BC) {
auto MB = MemoryBuffer::getFileOrSTDIN(Filename);
ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
MemoryBuffer::getFileOrSTDIN(Filename);
if (std::error_code EC = MB.getError()) {
errs() << "ERROR: cannot open " << Filename << ": " << EC.message() << "\n";
return errorCodeToError(EC);
@ -297,7 +300,7 @@ bool YAMLProfileReader::mayHaveProfileData(const BinaryFunction &BF) {
for (StringRef Name : BF.getNames()) {
if (ProfileNameToProfile.find(Name) != ProfileNameToProfile.end())
return true;
if (const auto CommonName = getLTOCommonName(Name)) {
if (const Optional<StringRef> CommonName = getLTOCommonName(Name)) {
if (LTOCommonNameMap.find(*CommonName) != LTOCommonNameMap.end()) {
return true;
}
@ -324,7 +327,7 @@ Error YAMLProfileReader::readProfile(BinaryContext &BC) {
// names. The first pass assigns profiles that match 100% by name and
// by hash. The second pass allows name ambiguity for LTO private functions.
for (auto &BFI : BC.getBinaryFunctions()) {
auto &Function = BFI.second;
BinaryFunction &Function = BFI.second;
// Clear function call count that may have been set while pre-processing
// the profile.
@ -334,33 +337,34 @@ Error YAMLProfileReader::readProfile(BinaryContext &BC) {
if (!opts::IgnoreHash)
Function.computeHash(/*UseDFS=*/true);
for (auto FunctionName : Function.getNames()) {
for (StringRef FunctionName : Function.getNames()) {
auto PI = ProfileNameToProfile.find(FunctionName);
if (PI == ProfileNameToProfile.end()) {
continue;
}
auto &YamlBF = *PI->getValue();
yaml::bolt::BinaryFunctionProfile &YamlBF = *PI->getValue();
if (profileMatches(YamlBF, Function))
matchProfileToFunction(YamlBF, Function);
}
}
for (auto &BFI : BC.getBinaryFunctions()) {
auto &Function = BFI.second;
BinaryFunction &Function = BFI.second;
if (ProfiledFunctions.count(&Function))
continue;
for (auto FunctionName : Function.getNames()) {
const auto CommonName = getLTOCommonName(FunctionName);
for (StringRef FunctionName : Function.getNames()) {
const Optional<StringRef> CommonName = getLTOCommonName(FunctionName);
if (CommonName) {
auto I = LTOCommonNameMap.find(*CommonName);
if (I == LTOCommonNameMap.end())
continue;
bool ProfileMatched{false};
auto &LTOProfiles = I->getValue();
for (auto *YamlBF : LTOProfiles) {
std::vector<yaml::bolt::BinaryFunctionProfile *> &LTOProfiles =
I->getValue();
for (yaml::bolt::BinaryFunctionProfile *YamlBF : LTOProfiles) {
if (YamlBF->Used)
continue;
if ((ProfileMatched = profileMatches(*YamlBF, Function))) {
@ -384,7 +388,7 @@ Error YAMLProfileReader::readProfile(BinaryContext &BC) {
if (PI == ProfileNameToProfile.end())
continue;
auto &YamlBF = *PI->getValue();
yaml::bolt::BinaryFunctionProfile &YamlBF = *PI->getValue();
if (!YamlBF.Used) {
matchProfileToFunction(YamlBF, Function);
break;
@ -393,7 +397,7 @@ Error YAMLProfileReader::readProfile(BinaryContext &BC) {
}
}
for (auto &YamlBF : YamlBP.Functions) {
for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) {
if (!YamlBF.Used && opts::Verbosity >= 1) {
errs() << "BOLT-WARNING: profile ignored for function " << YamlBF.Name
<< '\n';
@ -405,13 +409,13 @@ Error YAMLProfileReader::readProfile(BinaryContext &BC) {
NormalizeByCalls = usesEvent("branches");
uint64_t NumUnused{0};
for (auto &YamlBF : YamlBP.Functions) {
for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) {
if (YamlBF.Id >= YamlProfileToFunction.size()) {
// Such profile was ignored.
++NumUnused;
continue;
}
if (auto *BF = YamlProfileToFunction[YamlBF.Id]) {
if (BinaryFunction *BF = YamlProfileToFunction[YamlBF.Id]) {
parseFunctionProfile(*BF, YamlBF);
} else {
++NumUnused;

34
bolt/src/YAMLProfileWriter.cpp
View File

@ -29,9 +29,9 @@ namespace bolt {
namespace {
void
convert(const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF) {
auto &BC = BF.getBinaryContext();
const BinaryContext &BC = BF.getBinaryContext();
const auto LBRProfile = BF.getProfileFlags() & BinaryFunction::PF_LBR;
const uint16_t LBRProfile = BF.getProfileFlags() & BinaryFunction::PF_LBR;
YamlBF.Name = BF.getPrintName();
YamlBF.Id = BF.getFunctionNumber();
@ -39,7 +39,7 @@ convert(const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF) {
YamlBF.NumBasicBlocks = BF.size();
YamlBF.ExecCount = BF.getKnownExecutionCount();
for (const auto *BB : BF.dfs()) {
for (const BinaryBasicBlock *BB : BF.dfs()) {
yaml::bolt::BinaryBasicBlockProfile YamlBB;
YamlBB.Index = BB->getLayoutIndex();
YamlBB.NumInstructions = BB->getNumNonPseudos();
@ -53,7 +53,7 @@ convert(const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF) {
YamlBB.ExecCount = BB->getKnownExecutionCount();
for (const auto &Instr : *BB) {
for (const MCInst &Instr : *BB) {
if (!BC.MIB->isCall(Instr) && !BC.MIB->isIndirectBranch(Instr))
continue;
@ -69,11 +69,11 @@ convert(const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF) {
"CallProfile");
if (!ICSP)
continue;
for (auto &CSP : ICSP.get()) {
for (const IndirectCallProfile &CSP : ICSP.get()) {
CSI.DestId = 0; // designated for unknown functions
CSI.EntryDiscriminator = 0;
if (CSP.Symbol) {
const auto *Callee = BC.getFunctionForSymbol(CSP.Symbol);
const BinaryFunction *Callee = BC.getFunctionForSymbol(CSP.Symbol);
if (Callee) {
CSI.DestId = Callee->getFunctionNumber();
}
@ -84,8 +84,9 @@ convert(const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF) {
}
} else { // direct call or a tail call
uint64_t EntryID{0};
const auto *CalleeSymbol = BC.MIB->getTargetSymbol(Instr);
const auto Callee = BC.getFunctionForSymbol(CalleeSymbol, &EntryID);
const MCSymbol *CalleeSymbol = BC.MIB->getTargetSymbol(Instr);
const BinaryFunction *const Callee =
BC.getFunctionForSymbol(CalleeSymbol, &EntryID);
if (Callee) {
CSI.DestId = Callee->getFunctionNumber();;
CSI.EntryDiscriminator = EntryID;
@ -120,7 +121,8 @@ convert(const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF) {
!BB->isEntryPoint() &&
!(BB->isLandingPad() && BB->getKnownExecutionCount() != 0)) {
uint64_t SuccessorExecCount = 0;
for (auto &BranchInfo : BB->branch_info()) {
for (const BinaryBasicBlock::BinaryBranchInfo &BranchInfo :
BB->branch_info()) {
SuccessorExecCount += BranchInfo.Count;
}
if (!SuccessorExecCount)
@ -128,7 +130,7 @@ convert(const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF) {
}
auto BranchInfo = BB->branch_info_begin();
for (const auto *Successor : BB->successors()) {
for (const BinaryBasicBlock *Successor : BB->successors()) {
yaml::bolt::SuccessorInfo YamlSI;
YamlSI.Index = Successor->getLayoutIndex();
YamlSI.Count = BranchInfo->Count;
@ -146,7 +148,7 @@ convert(const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF) {
std::error_code
YAMLProfileWriter::writeProfile(const RewriteInstance &RI) {
auto &BC = RI.getBinaryContext();
const BinaryContext &BC = RI.getBinaryContext();
const auto &Functions = BC.getBinaryFunctions();
std::error_code EC;
@ -162,14 +164,14 @@ YAMLProfileWriter::writeProfile(const RewriteInstance &RI) {
// Fill out the header info.
BP.Header.Version = 1;
BP.Header.FileName = std::string(BC.getFilename());
auto BuildID = BC.getFileBuildID();
Optional<StringRef> BuildID = BC.getFileBuildID();
BP.Header.Id = BuildID ? std::string(*BuildID) : "<unknown>";
BP.Header.Origin = std::string(RI.getProfileReader()->getReaderName());
auto EventNames = RI.getProfileReader()->getEventNames();
StringSet<> EventNames = RI.getProfileReader()->getEventNames();
if (!EventNames.empty()) {
std::string Sep = "";
for (const auto &EventEntry : EventNames) {
for (const StringMapEntry<NoneType> &EventEntry : EventNames) {
BP.Header.EventNames += Sep + EventEntry.first().str();
Sep = ",";
}
@ -178,7 +180,7 @@ YAMLProfileWriter::writeProfile(const RewriteInstance &RI) {
// Make sure the profile is consistent across all functions.
uint16_t ProfileFlags = BinaryFunction::PF_NONE;
for (const auto &BFI : Functions) {
const auto &BF = BFI.second;
const BinaryFunction &BF = BFI.second;
if (BF.hasProfile() && !BF.empty()) {
assert(BF.getProfileFlags() != BinaryFunction::PF_NONE);
if (ProfileFlags == BinaryFunction::PF_NONE) {
@ -192,7 +194,7 @@ YAMLProfileWriter::writeProfile(const RewriteInstance &RI) {
// Add all function objects.
for (const auto &BFI : Functions) {
const auto &BF = BFI.second;
const BinaryFunction &BF = BFI.second;
if (BF.hasProfile()) {
if (!BF.hasValidProfile() && !RI.getProfileReader()->isTrustedSource())
continue;

16
bolt/src/llvm-bolt.cpp
View File

@ -295,7 +295,7 @@ int main(int argc, char **argv) {
if (!opts::DiffOnly) {
Expected<OwningBinary<Binary>> BinaryOrErr =
createBinary(opts::InputFilename);
if (auto E = BinaryOrErr.takeError())
if (Error E = BinaryOrErr.takeError())
report_error(opts::InputFilename, std::move(E));
Binary &Binary = *BinaryOrErr.get().getBinary();
@ -307,11 +307,11 @@ int main(int argc, char **argv) {
<< ": WARNING: reading perf data directly is unsupported, please use "
"-aggregate-only or perf2bolt.\n!!! Proceed on your own risk. !!!\n";
}
if (auto E = RI.setProfile(opts::PerfData))
if (Error E = RI.setProfile(opts::PerfData))
report_error(opts::PerfData, std::move(E));
}
if (!opts::InputDataFilename.empty()) {
if (auto E = RI.setProfile(opts::InputDataFilename))
if (Error E = RI.setProfile(opts::InputDataFilename))
report_error(opts::InputDataFilename, std::move(E));
}
if (opts::AggregateOnly && opts::PerfData.empty()) {
@ -324,7 +324,7 @@ int main(int argc, char **argv) {
MachORewriteInstance MachORI(O, ToolPath);
if (!opts::InputDataFilename.empty())
if (auto E = MachORI.setProfile(opts::InputDataFilename))
if (Error E = MachORI.setProfile(opts::InputDataFilename))
report_error(opts::InputDataFilename, std::move(E));
MachORI.run();
@ -340,19 +340,19 @@ int main(int argc, char **argv) {
createBinary(opts::InputFilename);
Expected<OwningBinary<Binary>> BinaryOrErr2 =
createBinary(opts::InputFilename2);
if (auto E = BinaryOrErr1.takeError())
if (Error E = BinaryOrErr1.takeError())
report_error(opts::InputFilename, std::move(E));
if (auto E = BinaryOrErr2.takeError())
if (Error E = BinaryOrErr2.takeError())
report_error(opts::InputFilename2, std::move(E));
Binary &Binary1 = *BinaryOrErr1.get().getBinary();
Binary &Binary2 = *BinaryOrErr2.get().getBinary();
if (auto *ELFObj1 = dyn_cast<ELFObjectFileBase>(&Binary1)) {
if (auto *ELFObj2 = dyn_cast<ELFObjectFileBase>(&Binary2)) {
RewriteInstance RI1(ELFObj1, argc, argv, ToolPath);
if (auto E = RI1.setProfile(opts::InputDataFilename))
if (Error E = RI1.setProfile(opts::InputDataFilename))
report_error(opts::InputDataFilename, std::move(E));
RewriteInstance RI2(ELFObj2, argc, argv, ToolPath);
if (auto E = RI2.setProfile(opts::InputDataFilename2))
if (Error E = RI2.setProfile(opts::InputDataFilename2))
report_error(opts::InputDataFilename2, std::move(E));
outs() << "BOLT-DIFF: *** Analyzing binary 1: " << opts::InputFilename
<< "\n";

51
bolt/src/merge-fdata/merge-fdata.cpp
View File

@ -146,14 +146,14 @@ void mergeBasicBlockProfile(BinaryBasicBlockProfile &MergedBB,
// Merge calls sites.
std::unordered_map<uint32_t, CallSiteInfo *> CSByOffset;
for (auto &CS : BB.CallSites)
for (CallSiteInfo &CS : BB.CallSites)
CSByOffset.emplace(std::make_pair(CS.Offset, &CS));
for (auto &MergedCS : MergedBB.CallSites) {
for (CallSiteInfo &MergedCS : MergedBB.CallSites) {
auto CSI = CSByOffset.find(MergedCS.Offset);
if (CSI == CSByOffset.end())
continue;
auto &CS = *CSI->second;
yaml::bolt::CallSiteInfo &CS = *CSI->second;
if (CS != MergedCS)
continue;
@ -164,28 +164,28 @@ void mergeBasicBlockProfile(BinaryBasicBlockProfile &MergedBB,
}
// Append the rest of call sites.
for (auto CSI : CSByOffset) {
for (std::pair<const uint32_t, CallSiteInfo *> CSI : CSByOffset) {
MergedBB.CallSites.emplace_back(std::move(*CSI.second));
}
// Merge successor info.
std::vector<SuccessorInfo *> SIByIndex(BF.NumBasicBlocks);
for (auto &SI : BB.Successors) {
for (SuccessorInfo &SI : BB.Successors) {
if (SI.Index >= BF.NumBasicBlocks)
report_error(BF.Name, "bad successor index");
SIByIndex[SI.Index] = &SI;
}
for (auto &MergedSI : MergedBB.Successors) {
for (SuccessorInfo &MergedSI : MergedBB.Successors) {
if (!SIByIndex[MergedSI.Index])
continue;
auto &SI = *SIByIndex[MergedSI.Index];
SuccessorInfo &SI = *SIByIndex[MergedSI.Index];
MergedSI.Count += SI.Count;
MergedSI.Mispreds += SI.Mispreds;
SIByIndex[MergedSI.Index] = nullptr;
}
for (auto *SI : SIByIndex) {
for (SuccessorInfo *SI : SIByIndex) {
if (SI) {
MergedBB.Successors.emplace_back(std::move(*SI));
}
@ -207,16 +207,16 @@ void mergeFunctionProfile(BinaryFunctionProfile &MergedBF,
// Merge basic blocks profile.
std::vector<BinaryBasicBlockProfile *> BlockByIndex(BF.NumBasicBlocks);
for (auto &BB : BF.Blocks) {
for (BinaryBasicBlockProfile &BB : BF.Blocks) {
if (BB.Index >= BF.NumBasicBlocks)
report_error(BF.Name + " : BB #" + Twine(BB.Index),
"bad basic block index");
BlockByIndex[BB.Index] = &BB;
}
for (auto &MergedBB : MergedBF.Blocks) {
for (BinaryBasicBlockProfile &MergedBB : MergedBF.Blocks) {
if (!BlockByIndex[MergedBB.Index])
continue;
auto &BB = *BlockByIndex[MergedBB.Index];
BinaryBasicBlockProfile &BB = *BlockByIndex[MergedBB.Index];
mergeBasicBlockProfile(MergedBB, std::move(BB), MergedBF);
@ -225,7 +225,7 @@ void mergeFunctionProfile(BinaryFunctionProfile &MergedBF,
}
// Append blocks unique to BF (i.e. those that are not in MergedBF).
for (auto *BB : BlockByIndex) {
for (BinaryBasicBlockProfile *BB : BlockByIndex) {
if (BB) {
MergedBF.Blocks.emplace_back(std::move(*BB));
}
@ -233,10 +233,11 @@ void mergeFunctionProfile(BinaryFunctionProfile &MergedBF,
}
bool isYAML(const StringRef Filename) {
auto MB = MemoryBuffer::getFileOrSTDIN(Filename);
ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
MemoryBuffer::getFileOrSTDIN(Filename);
if (std::error_code EC = MB.getError())
report_error(Filename, EC);
auto Buffer = MB.get()->getBuffer();
StringRef Buffer = MB.get()->getBuffer();
if (Buffer.startswith("---\n"))
return true;
return false;
@ -246,15 +247,16 @@ void mergeLegacyProfiles(const cl::list<std::string> &Filenames) {
errs() << "Using legacy profile format.\n";
bool BoltedCollection{false};
bool First{true};
for (auto &Filename : Filenames) {
for (const std::string &Filename : Filenames) {
if (isYAML(Filename))
report_error(Filename, "cannot mix YAML and legacy formats");
auto MB = MemoryBuffer::getFileOrSTDIN(Filename);
ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
MemoryBuffer::getFileOrSTDIN(Filename);
if (std::error_code EC = MB.getError())
report_error(Filename, EC);
errs() << "Merging data from " << Filename << "...\n";
auto Buf = MB.get()->getBuffer();
StringRef Buf = MB.get()->getBuffer();
// Check if the string "boltedcollection" is in the first line
if (Buf.startswith("boltedcollection\n")) {
if (!First && !BoltedCollection) {
@ -307,8 +309,9 @@ int main(int argc, char **argv) {
// Merged information for all functions.
StringMap<BinaryFunctionProfile> MergedBFs;
for (auto &InputDataFilename : opts::InputDataFilenames) {
auto MB = MemoryBuffer::getFileOrSTDIN(InputDataFilename);
for (std::string &InputDataFilename : opts::InputDataFilenames) {
ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
MemoryBuffer::getFileOrSTDIN(InputDataFilename);
if (std::error_code EC = MB.getError())
report_error(InputDataFilename, EC);
yaml::Input YamlInput(MB.get()->getBuffer());
@ -331,13 +334,13 @@ int main(int argc, char **argv) {
mergeProfileHeaders(MergedHeader, BP.Header);
// Do the function merge.
for (auto &BF : BP.Functions) {
for (BinaryFunctionProfile &BF : BP.Functions) {
if (!MergedBFs.count(BF.Name)) {
MergedBFs.insert(std::make_pair(BF.Name, BF));
continue;
}
auto &MergedBF = MergedBFs.find(BF.Name)->second;
BinaryFunctionProfile &MergedBF = MergedBFs.find(BF.Name)->second;
mergeFunctionProfile(MergedBF, std::move(BF));
}
}
@ -383,8 +386,8 @@ int main(int argc, char **argv) {
[](const StringMapEntry<BinaryFunctionProfile> &V) {
// Return total branch count.
uint64_t BranchCount = 0;
for (const auto &BI : V.second.Blocks) {
for (const auto &SI : BI.Successors) {
for (const BinaryBasicBlockProfile &BI : V.second.Blocks) {
for (const SuccessorInfo &SI : BI.Successors) {
BranchCount += SI.Count;
}
}
@ -405,7 +408,7 @@ int main(int argc, char **argv) {
? "execution"
: "total branch")
<< " count:\n";
for (auto &FI : FunctionList) {
for (std::pair<uint64_t, StringRef> &FI : FunctionList) {
errs() << FI.second << " : " << FI.first << '\n';
}
}