forked from OSchip/llvm-project
[BOLT][NFC] Fix braces usage in Passes
Summary: Refactor bolt/*/Passes to follow the braces rule for if/else/loop from [LLVM Coding Standards](https://llvm.org/docs/CodingStandards.html). (cherry picked from FBD33344642)
This commit is contained in:
parent
cd7a630585
commit
f92ab6af35
|
@ -393,13 +393,12 @@ public:
|
|||
BinaryBasicBlock *LBB = Func.getLandingPadBBFor(BB, Inst);
|
||||
if (LBB) {
|
||||
auto First = LBB->begin();
|
||||
if (First != LBB->end()) {
|
||||
if (First != LBB->end())
|
||||
derived().doConfluenceWithLP(CurState,
|
||||
getOrCreateStateAt(&*First), Inst);
|
||||
} else {
|
||||
else
|
||||
derived().doConfluenceWithLP(CurState, getOrCreateStateAt(LBB),
|
||||
Inst);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -412,31 +411,24 @@ public:
|
|||
PrevState = &St;
|
||||
};
|
||||
|
||||
if (!Backward) {
|
||||
for (MCInst &Inst : *BB) {
|
||||
if (!Backward)
|
||||
for (MCInst &Inst : *BB)
|
||||
doNext(Inst, *BB);
|
||||
}
|
||||
} else {
|
||||
for (auto I = BB->rbegin(), E = BB->rend(); I != E; ++I) {
|
||||
else
|
||||
for (auto I = BB->rbegin(), E = BB->rend(); I != E; ++I)
|
||||
doNext(*I, *BB);
|
||||
}
|
||||
}
|
||||
|
||||
if (Changed) {
|
||||
if (!Backward) {
|
||||
for (BinaryBasicBlock *Succ : BB->successors()) {
|
||||
for (BinaryBasicBlock *Succ : BB->successors())
|
||||
Worklist.push(Succ);
|
||||
}
|
||||
for (BinaryBasicBlock *LandingPad : BB->landing_pads()) {
|
||||
for (BinaryBasicBlock *LandingPad : BB->landing_pads())
|
||||
Worklist.push(LandingPad);
|
||||
}
|
||||
} else {
|
||||
for (BinaryBasicBlock *Pred : BB->predecessors()) {
|
||||
for (BinaryBasicBlock *Pred : BB->predecessors())
|
||||
Worklist.push(Pred);
|
||||
}
|
||||
for (BinaryBasicBlock *Thrower : BB->throwers()) {
|
||||
for (BinaryBasicBlock *Thrower : BB->throwers())
|
||||
Worklist.push(Thrower);
|
||||
}
|
||||
}
|
||||
}
|
||||
} // end while (!Worklist.empty())
|
||||
|
|
|
@ -95,9 +95,8 @@ public:
|
|||
|
||||
void doForAllDominators(const MCInst &Inst,
|
||||
std::function<void(const MCInst &)> Task) {
|
||||
for (auto I = this->expr_begin(Inst), E = this->expr_end(); I != E; ++I) {
|
||||
for (auto I = this->expr_begin(Inst), E = this->expr_end(); I != E; ++I)
|
||||
Task(**I);
|
||||
}
|
||||
}
|
||||
|
||||
void run() {
|
||||
|
@ -137,9 +136,8 @@ private:
|
|||
BitVector computeNext(const MCInst &Point, const BitVector &Cur) {
|
||||
BitVector Next = Cur;
|
||||
// Gen
|
||||
if (!this->BC.MIB->isCFI(Point)) {
|
||||
if (!this->BC.MIB->isCFI(Point))
|
||||
Next.set(this->ExprToIdx[&Point]);
|
||||
}
|
||||
return Next;
|
||||
}
|
||||
|
||||
|
|
|
@ -143,9 +143,8 @@ protected:
|
|||
}
|
||||
for (auto I = InstInfo.getImplicitUses(),
|
||||
E = InstInfo.getImplicitUses() + InstInfo.getNumImplicitUses();
|
||||
I != E; ++I) {
|
||||
I != E; ++I)
|
||||
Used |= BC.MIB->getAliases(*I, false);
|
||||
}
|
||||
if (IsCall &&
|
||||
(!BC.MIB->isTailCall(Point) || !BC.MIB->isConditionalBranch(Point))) {
|
||||
// Never gen FLAGS from a non-conditional call... this is overly
|
||||
|
|
|
@ -42,11 +42,10 @@ public:
|
|||
bool isReachedBy(MCPhysReg Reg, ExprIterator Candidates) {
|
||||
for (auto I = Candidates; I != this->expr_end(); ++I) {
|
||||
BitVector BV = BitVector(this->BC.MRI->getNumRegs(), false);
|
||||
if (Def) {
|
||||
if (Def)
|
||||
RA.getInstClobberList(**I, BV);
|
||||
} else {
|
||||
else
|
||||
this->BC.MIB->getTouchedRegs(**I, BV);
|
||||
}
|
||||
if (BV[Reg])
|
||||
return true;
|
||||
}
|
||||
|
|
|
@ -74,9 +74,8 @@ protected:
|
|||
BitVector computeNext(const MCInst &Point, const BitVector &Cur) {
|
||||
BitVector Next = Cur;
|
||||
// Gen
|
||||
if (!this->BC.MIB->isCFI(Point)) {
|
||||
if (!this->BC.MIB->isCFI(Point))
|
||||
Next.set(this->ExprToIdx[&Point]);
|
||||
}
|
||||
return Next;
|
||||
}
|
||||
|
||||
|
|
|
@ -37,9 +37,8 @@ public:
|
|||
|
||||
template <typename F> void forAllAdjacent(const Cluster *C, F Func) {
|
||||
for (int I = Bits[C->id()].find_first(); I != -1;
|
||||
I = Bits[C->id()].find_next(I)) {
|
||||
I = Bits[C->id()].find_next(I))
|
||||
Func(Clusters[I]);
|
||||
}
|
||||
}
|
||||
|
||||
/// Merge adjacency info from cluster B into cluster A. Info for cluster B is
|
||||
|
@ -93,9 +92,8 @@ public:
|
|||
|
||||
void invalidate(const Cluster *C) {
|
||||
Valid.reset(C->id() * Size, (C->id() + 1) * Size);
|
||||
for (size_t Id = 0; Id < Size; Id++) {
|
||||
for (size_t Id = 0; Id < Size; Id++)
|
||||
Valid.reset((Id * Size) + C->id());
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
|
|
|
@ -509,11 +509,9 @@ public:
|
|||
CSA(FA, BF, Info, AllocId) {}
|
||||
|
||||
~ShrinkWrapping() {
|
||||
for (BinaryBasicBlock &BB : BF) {
|
||||
for (MCInst &Inst : BB) {
|
||||
for (BinaryBasicBlock &BB : BF)
|
||||
for (MCInst &Inst : BB)
|
||||
BC.MIB->removeAnnotation(Inst, getAnnotationIndex());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool perform();
|
||||
|
|
|
@ -99,8 +99,7 @@ protected:
|
|||
|
||||
if (MIB->isLeave(Point))
|
||||
return FPVal + 8;
|
||||
else
|
||||
return FPVal;
|
||||
return FPVal;
|
||||
}
|
||||
|
||||
if (this->BC.MII->get(Point.getOpcode())
|
||||
|
@ -157,11 +156,8 @@ protected:
|
|||
return SUPERPOSITION;
|
||||
}
|
||||
|
||||
if (!HasFramePointer) {
|
||||
if (MIB->escapesVariable(Point, false)) {
|
||||
HasFramePointer = true;
|
||||
}
|
||||
}
|
||||
if (!HasFramePointer && MIB->escapesVariable(Point, false))
|
||||
HasFramePointer = true;
|
||||
return static_cast<int>(Output);
|
||||
}
|
||||
|
||||
|
|
|
@ -70,7 +70,7 @@ struct StokeFuncInfo {
|
|||
}
|
||||
|
||||
void printCsvHeader(std::ofstream &Outfile) {
|
||||
if (Outfile.is_open()) {
|
||||
if (Outfile.is_open())
|
||||
Outfile << "FuncName,Offset,Size,NumInstrs,NumBlocks,"
|
||||
<< "IsLoopFree,NumLoops,MaxLoopDepth,"
|
||||
<< "HotSize,TotalSize,"
|
||||
|
@ -79,7 +79,6 @@ struct StokeFuncInfo {
|
|||
<< "DefIn,LiveOut,HeapOut,StackOut,"
|
||||
<< "HasRipAddr,"
|
||||
<< "Omitted\n";
|
||||
}
|
||||
}
|
||||
|
||||
void printData(std::ofstream &Outfile) {
|
||||
|
@ -88,13 +87,11 @@ struct StokeFuncInfo {
|
|||
<< "," << NumBlocks << "," << IsLoopFree << "," << NumLoops << ","
|
||||
<< MaxLoopDepth << "," << HotSize << "," << TotalSize << ","
|
||||
<< Score << "," << HasCall << ",\"{ ";
|
||||
for (std::string S : DefIn) {
|
||||
for (std::string S : DefIn)
|
||||
Outfile << "%" << S << " ";
|
||||
}
|
||||
Outfile << "}\",\"{ ";
|
||||
for (std::string S : LiveOut) {
|
||||
for (std::string S : LiveOut)
|
||||
Outfile << "%" << S << " ";
|
||||
}
|
||||
Outfile << "}\"," << HeapOut << "," << StackOut << "," << HasRipAddr
|
||||
<< "," << Omitted << "\n";
|
||||
}
|
||||
|
|
|
@ -84,11 +84,9 @@ private:
|
|||
|
||||
void clearAnnotations(BinaryFunction &Function) const {
|
||||
const BinaryContext &BC = Function.getBinaryContext();
|
||||
for (BinaryBasicBlock &BB : Function) {
|
||||
for (MCInst &Inst : BB) {
|
||||
for (BinaryBasicBlock &BB : Function)
|
||||
for (MCInst &Inst : BB)
|
||||
BC.MIB->removeAnnotation(Inst, getProcessedICTag());
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
@ -18,9 +18,7 @@ class VeneerElimination : public BinaryFunctionPass {
|
|||
public:
|
||||
/// BinaryPass public interface
|
||||
explicit VeneerElimination(const cl::opt<bool> &PrintPass)
|
||||
: BinaryFunctionPass(PrintPass) {
|
||||
;
|
||||
}
|
||||
: BinaryFunctionPass(PrintPass) {}
|
||||
|
||||
const char *getName() const override { return "veneer-elimination"; }
|
||||
|
||||
|
|
|
@ -95,12 +95,11 @@ void alignCompact(BinaryFunction &Function, const MCCodeEmitter *Emitter) {
|
|||
size_t HotSize = 0;
|
||||
size_t ColdSize = 0;
|
||||
|
||||
for (const BinaryBasicBlock *BB : Function.layout()) {
|
||||
for (const BinaryBasicBlock *BB : Function.layout())
|
||||
if (BB->isCold())
|
||||
ColdSize += BC.computeCodeSize(BB->begin(), BB->end(), Emitter);
|
||||
else
|
||||
HotSize += BC.computeCodeSize(BB->begin(), BB->end(), Emitter);
|
||||
}
|
||||
|
||||
Function.setAlignment(opts::AlignFunctions);
|
||||
if (HotSize > 0)
|
||||
|
@ -135,9 +134,9 @@ void AlignerPass::alignBlocks(BinaryFunction &Function,
|
|||
}
|
||||
|
||||
uint64_t FTCount = 0;
|
||||
if (PrevBB && PrevBB->getFallthrough() == BB) {
|
||||
if (PrevBB && PrevBB->getFallthrough() == BB)
|
||||
FTCount = PrevBB->getBranchInfo(*BB).Count;
|
||||
}
|
||||
|
||||
PrevBB = BB;
|
||||
|
||||
if (Count < FTCount * 2)
|
||||
|
@ -189,9 +188,9 @@ void AlignerPass::runOnFunctions(BinaryContext &BC) {
|
|||
|
||||
LLVM_DEBUG(
|
||||
dbgs() << "BOLT-DEBUG: max bytes per basic block alignment distribution:\n";
|
||||
for (unsigned I = 1; I < AlignHistogram.size(); ++I) {
|
||||
for (unsigned I = 1; I < AlignHistogram.size(); ++I)
|
||||
dbgs() << " " << I << " : " << AlignHistogram[I] << '\n';
|
||||
}
|
||||
|
||||
dbgs() << "BOLT-DEBUG: total execution count of aligned blocks: "
|
||||
<< AlignedBlocksCount << '\n';
|
||||
);
|
||||
|
|
|
@ -46,9 +46,8 @@ bool isIndifferentToSP(const MCInst &Inst, const BinaryContext &BC) {
|
|||
|
||||
for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) {
|
||||
const MCOperand &Operand = Inst.getOperand(I);
|
||||
if (Operand.isReg() && Operand.getReg() == BC.MIB->getStackPointer()) {
|
||||
if (Operand.isReg() && Operand.getReg() == BC.MIB->getStackPointer())
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
|
|
@ -42,11 +42,11 @@ void dumpCFI(const BinaryFunction &BF, const MCInst &Instr, AsmPrinter &MAP) {
|
|||
// Skip unsupported CFI instructions.
|
||||
case MCCFIInstruction::OpRememberState:
|
||||
case MCCFIInstruction::OpRestoreState:
|
||||
if (opts::Verbosity >= 2) {
|
||||
if (opts::Verbosity >= 2)
|
||||
errs()
|
||||
<< "BOLT-WARNING: AsmDump: skipping unsupported CFI instruction in "
|
||||
<< BF << ".\n";
|
||||
}
|
||||
|
||||
return;
|
||||
|
||||
default:
|
||||
|
|
|
@ -114,9 +114,8 @@ buildCallGraph(BinaryContext &BC, CgFilterFunction Filter, bool CgFromPerfData,
|
|||
// samples. Results from perfomance testing seem to favor the zero
|
||||
// count though, so I'm leaving it this way for now.
|
||||
return Cg.addNode(Function, Size, Function->getKnownExecutionCount());
|
||||
} else {
|
||||
return Id;
|
||||
}
|
||||
return Id;
|
||||
};
|
||||
|
||||
// Add call graph edges.
|
||||
|
@ -128,9 +127,8 @@ buildCallGraph(BinaryContext &BC, CgFilterFunction Filter, bool CgFromPerfData,
|
|||
for (auto &It : BC.getBinaryFunctions()) {
|
||||
BinaryFunction *Function = &It.second;
|
||||
|
||||
if (Filter(*Function)) {
|
||||
if (Filter(*Function))
|
||||
continue;
|
||||
}
|
||||
|
||||
const CallGraph::NodeId SrcId = lookupNode(Function);
|
||||
// Offset of the current basic block from the beginning of the function
|
||||
|
@ -146,9 +144,9 @@ buildCallGraph(BinaryContext &BC, CgFilterFunction Filter, bool CgFromPerfData,
|
|||
if (IgnoreRecursiveCalls)
|
||||
return false;
|
||||
}
|
||||
if (Filter(*DstFunc)) {
|
||||
if (Filter(*DstFunc))
|
||||
return false;
|
||||
}
|
||||
|
||||
const CallGraph::NodeId DstId = lookupNode(DstFunc);
|
||||
const bool IsValidCount = Count != COUNT_NO_PROFILE;
|
||||
const uint64_t AdjCount = UseEdgeCounts && IsValidCount ? Count : 1;
|
||||
|
@ -180,10 +178,9 @@ buildCallGraph(BinaryContext &BC, CgFilterFunction Filter, bool CgFromPerfData,
|
|||
if (!DstSym && BC.MIB->hasAnnotation(Inst, "CallProfile")) {
|
||||
const auto &ICSP = BC.MIB->getAnnotationAs<IndirectCallSiteProfile>(
|
||||
Inst, "CallProfile");
|
||||
for (const IndirectCallProfile &CSI : ICSP) {
|
||||
for (const IndirectCallProfile &CSI : ICSP)
|
||||
if (CSI.Symbol)
|
||||
Counts.emplace_back(CSI.Symbol, CSI.Count);
|
||||
}
|
||||
} else {
|
||||
const uint64_t Count = BB->getExecutionCount();
|
||||
Counts.emplace_back(DstSym, Count);
|
||||
|
@ -214,9 +211,8 @@ buildCallGraph(BinaryContext &BC, CgFilterFunction Filter, bool CgFromPerfData,
|
|||
if (Offset > Size)
|
||||
Offset = Size;
|
||||
|
||||
if (!recordCall(CSI.Symbol, CSI.Count)) {
|
||||
if (!recordCall(CSI.Symbol, CSI.Count))
|
||||
++NotProcessed;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (BinaryBasicBlock *BB : Function->layout()) {
|
||||
|
@ -253,9 +249,8 @@ buildCallGraph(BinaryContext &BC, CgFilterFunction Filter, bool CgFromPerfData,
|
|||
}
|
||||
}
|
||||
// Increase Offset if needed
|
||||
if (BBIncludedInFunctionSize) {
|
||||
if (BBIncludedInFunctionSize)
|
||||
Offset += BC.computeCodeSize(&Inst, &Inst + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -266,7 +261,7 @@ buildCallGraph(BinaryContext &BC, CgFilterFunction Filter, bool CgFromPerfData,
|
|||
#else
|
||||
bool PrintInfo = false;
|
||||
#endif
|
||||
if (PrintInfo || opts::Verbosity > 0) {
|
||||
if (PrintInfo || opts::Verbosity > 0)
|
||||
outs() << format("BOLT-INFO: buildCallGraph: %u nodes, %u callsites "
|
||||
"(%u recursive), density = %.6lf, %u callsites not "
|
||||
"processed, %u callsites with invalid profile, "
|
||||
|
@ -274,7 +269,6 @@ buildCallGraph(BinaryContext &BC, CgFilterFunction Filter, bool CgFromPerfData,
|
|||
Cg.numNodes(), TotalCallsites, RecursiveCallsites,
|
||||
Cg.density(), NotProcessed, NoProfileCallsites,
|
||||
NumFallbacks);
|
||||
}
|
||||
|
||||
return Cg;
|
||||
}
|
||||
|
|
|
@ -374,11 +374,10 @@ void EliminateUnreachableBlocks::runOnFunction(BinaryFunction &Function) {
|
|||
DeletedBytes += Bytes;
|
||||
if (Count) {
|
||||
Modified.insert(&Function);
|
||||
if (opts::Verbosity > 0) {
|
||||
if (opts::Verbosity > 0)
|
||||
outs() << "BOLT-INFO: Removed " << Count
|
||||
<< " dead basic block(s) accounting for " << Bytes
|
||||
<< " bytes in function " << Function << '\n';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -386,9 +385,8 @@ void EliminateUnreachableBlocks::runOnFunction(BinaryFunction &Function) {
|
|||
void EliminateUnreachableBlocks::runOnFunctions(BinaryContext &BC) {
|
||||
for (auto &It : BC.getBinaryFunctions()) {
|
||||
BinaryFunction &Function = It.second;
|
||||
if (shouldOptimize(Function)) {
|
||||
if (shouldOptimize(Function))
|
||||
runOnFunction(Function);
|
||||
}
|
||||
}
|
||||
|
||||
outs() << "BOLT-INFO: UCE removed " << DeletedBlocks << " blocks and "
|
||||
|
@ -416,9 +414,8 @@ void ReorderBasicBlocks::runOnFunctions(BinaryContext &BC) {
|
|||
|
||||
ParallelUtilities::WorkFuncTy WorkFun = [&](BinaryFunction &BF) {
|
||||
modifyFunctionLayout(BF, opts::ReorderBlocks, opts::MinBranchClusters);
|
||||
if (BF.hasLayoutChanged()) {
|
||||
if (BF.hasLayoutChanged())
|
||||
++ModifiedFuncCount;
|
||||
}
|
||||
};
|
||||
|
||||
ParallelUtilities::PredicateTy SkipFunc = [&](const BinaryFunction &BF) {
|
||||
|
@ -439,10 +436,9 @@ void ReorderBasicBlocks::runOnFunctions(BinaryContext &BC) {
|
|||
// Copy all the values into vector in order to sort them
|
||||
std::map<uint64_t, BinaryFunction &> ScoreMap;
|
||||
auto &BFs = BC.getBinaryFunctions();
|
||||
for (auto It = BFs.begin(); It != BFs.end(); ++It) {
|
||||
for (auto It = BFs.begin(); It != BFs.end(); ++It)
|
||||
ScoreMap.insert(std::pair<uint64_t, BinaryFunction &>(
|
||||
It->second.getFunctionScore(), It->second));
|
||||
}
|
||||
|
||||
OS << "\nBOLT-INFO: Printing Function Statistics:\n\n";
|
||||
OS << " There are " << BFs.size() << " functions in total. \n";
|
||||
|
@ -634,21 +630,17 @@ void LowerAnnotations::runOnFunctions(BinaryContext &BC) {
|
|||
// Now record preserved annotations separately and then strip annotations.
|
||||
for (auto II = BB->begin(); II != BB->end(); ++II) {
|
||||
if (BF.requiresAddressTranslation() &&
|
||||
BC.MIB->hasAnnotation(*II, "Offset")) {
|
||||
BC.MIB->hasAnnotation(*II, "Offset"))
|
||||
PreservedOffsetAnnotations.emplace_back(
|
||||
&(*II), BC.MIB->getAnnotationAs<uint32_t>(*II, "Offset"));
|
||||
}
|
||||
BC.MIB->stripAnnotations(*II);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (BinaryFunction *BF : BC.getInjectedBinaryFunctions()) {
|
||||
for (BinaryBasicBlock &BB : *BF) {
|
||||
for (MCInst &Instruction : BB) {
|
||||
for (BinaryFunction *BF : BC.getInjectedBinaryFunctions())
|
||||
for (BinaryBasicBlock &BB : *BF)
|
||||
for (MCInst &Instruction : BB)
|
||||
BC.MIB->stripAnnotations(Instruction);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Release all memory taken by annotations
|
||||
BC.MIB->freeAnnotations();
|
||||
|
@ -762,12 +754,10 @@ uint64_t fixDoubleJumps(BinaryFunction &Function, bool MarkInvalid) {
|
|||
|
||||
if (Pred->getSuccessor() == &BB ||
|
||||
(Pred->getConditionalSuccessor(true) == &BB && !IsTailCall) ||
|
||||
Pred->getConditionalSuccessor(false) == &BB) {
|
||||
if (checkAndPatch(Pred, Succ, SuccSym) && MarkInvalid) {
|
||||
Pred->getConditionalSuccessor(false) == &BB)
|
||||
if (checkAndPatch(Pred, Succ, SuccSym) && MarkInvalid)
|
||||
BB.markValid(BB.pred_size() != 0 || BB.isLandingPad() ||
|
||||
BB.isEntryPoint());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -920,11 +910,10 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryFunction &BF) {
|
|||
// if there are no other users.
|
||||
PredBB->removeSuccessor(BB);
|
||||
// Update BB execution count
|
||||
if (CTCTakenFreq && CTCTakenFreq <= BB->getKnownExecutionCount()) {
|
||||
if (CTCTakenFreq && CTCTakenFreq <= BB->getKnownExecutionCount())
|
||||
BB->setExecutionCount(BB->getExecutionCount() - CTCTakenFreq);
|
||||
} else if (CTCTakenFreq > BB->getKnownExecutionCount()) {
|
||||
else if (CTCTakenFreq > BB->getKnownExecutionCount())
|
||||
BB->setExecutionCount(0);
|
||||
}
|
||||
|
||||
++NumLocalCTCs;
|
||||
LocalCTCTakenCount += CTCTakenFreq;
|
||||
|
@ -950,9 +939,8 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryFunction &BF) {
|
|||
// Find the next valid block. Invalid blocks will be deleted
|
||||
// so they shouldn't be considered fallthrough targets.
|
||||
const BinaryBasicBlock *NextBlock = BF.getBasicBlockAfter(PredBB, false);
|
||||
while (NextBlock && !isValid(NextBlock)) {
|
||||
while (NextBlock && !isValid(NextBlock))
|
||||
NextBlock = BF.getBasicBlockAfter(NextBlock, false);
|
||||
}
|
||||
|
||||
// Get the unconditional successor to this block.
|
||||
const BinaryBasicBlock *PredSucc = PredBB->getSuccessor();
|
||||
|
@ -1216,9 +1204,8 @@ bool SimplifyRODataLoads::simplifyRODataLoads(BinaryFunction &BF) {
|
|||
void SimplifyRODataLoads::runOnFunctions(BinaryContext &BC) {
|
||||
for (auto &It : BC.getBinaryFunctions()) {
|
||||
BinaryFunction &Function = It.second;
|
||||
if (shouldOptimize(Function) && simplifyRODataLoads(Function)) {
|
||||
if (shouldOptimize(Function) && simplifyRODataLoads(Function))
|
||||
Modified.insert(&Function);
|
||||
}
|
||||
}
|
||||
|
||||
outs() << "BOLT-INFO: simplified " << NumLoadsSimplified << " out of "
|
||||
|
@ -1250,11 +1237,10 @@ void AssignSections::runOnFunctions(BinaryContext &BC) {
|
|||
}
|
||||
|
||||
if (!UseColdSection || Function.hasValidIndex() ||
|
||||
Function.hasValidProfile()) {
|
||||
Function.hasValidProfile())
|
||||
Function.setCodeSectionName(BC.getMainCodeSectionName());
|
||||
} else {
|
||||
else
|
||||
Function.setCodeSectionName(BC.getColdCodeSectionName());
|
||||
}
|
||||
|
||||
if (Function.isSplit())
|
||||
Function.setColdCodeSectionName(BC.getColdCodeSectionName());
|
||||
|
@ -1373,18 +1359,17 @@ void PrintProgramStats::runOnFunctions(BinaryContext &BC) {
|
|||
++NumRegularFunctions;
|
||||
|
||||
if (!Function.isSimple()) {
|
||||
if (Function.hasProfile()) {
|
||||
if (Function.hasProfile())
|
||||
++NumNonSimpleProfiledFunctions;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Function.hasUnknownControlFlow()) {
|
||||
if (opts::PrintUnknownCFG) {
|
||||
if (opts::PrintUnknownCFG)
|
||||
Function.dump();
|
||||
} else if (opts::PrintUnknown) {
|
||||
else if (opts::PrintUnknown)
|
||||
errs() << "function with unknown control flow: " << Function << '\n';
|
||||
}
|
||||
|
||||
++NumUnknownControlFlowFunctions;
|
||||
}
|
||||
|
||||
|
@ -1424,11 +1409,10 @@ void PrintProgramStats::runOnFunctions(BinaryContext &BC) {
|
|||
const float PctStale =
|
||||
NumStaleProfileFunctions / (float)NumAllProfiledFunctions * 100.0f;
|
||||
auto printErrorOrWarning = [&]() {
|
||||
if (PctStale > opts::StaleThreshold) {
|
||||
if (PctStale > opts::StaleThreshold)
|
||||
errs() << "BOLT-ERROR: ";
|
||||
} else {
|
||||
else
|
||||
errs() << "BOLT-WARNING: ";
|
||||
}
|
||||
};
|
||||
printErrorOrWarning();
|
||||
errs() << NumStaleProfileFunctions
|
||||
|
@ -1466,9 +1450,8 @@ void PrintProgramStats::runOnFunctions(BinaryContext &BC) {
|
|||
auto SFI = ProfiledFunctions.begin();
|
||||
auto SFIend = ProfiledFunctions.end();
|
||||
for (unsigned I = 0u; I < opts::TopCalledLimit && SFI != SFIend;
|
||||
++SFI, ++I) {
|
||||
++SFI, ++I)
|
||||
outs() << " " << **SFI << " : " << (*SFI)->getExecutionCount() << '\n';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1616,30 +1599,25 @@ void PrintProgramStats::runOnFunctions(BinaryContext &BC) {
|
|||
for (unsigned I = 0;
|
||||
I < std::min(static_cast<size_t>(opts::ReportBadLayout),
|
||||
SuboptimalFuncs.size());
|
||||
++I) {
|
||||
++I)
|
||||
SuboptimalFuncs[I]->print(outs());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (NumUnknownControlFlowFunctions) {
|
||||
outs() << "BOLT-INFO: " << NumUnknownControlFlowFunctions
|
||||
<< " functions have instructions with unknown control flow";
|
||||
if (!opts::PrintUnknown) {
|
||||
if (!opts::PrintUnknown)
|
||||
outs() << ". Use -print-unknown to see the list.";
|
||||
}
|
||||
outs() << '\n';
|
||||
}
|
||||
}
|
||||
|
||||
void InstructionLowering::runOnFunctions(BinaryContext &BC) {
|
||||
for (auto &BFI : BC.getBinaryFunctions()) {
|
||||
for (BinaryBasicBlock &BB : BFI.second) {
|
||||
for (MCInst &Instruction : BB) {
|
||||
for (auto &BFI : BC.getBinaryFunctions())
|
||||
for (BinaryBasicBlock &BB : BFI.second)
|
||||
for (MCInst &Instruction : BB)
|
||||
BC.MIB->lowerTailCall(Instruction);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void StripRepRet::runOnFunctions(BinaryContext &BC) {
|
||||
|
@ -1657,12 +1635,11 @@ void StripRepRet::runOnFunctions(BinaryContext &BC) {
|
|||
}
|
||||
}
|
||||
|
||||
if (NumBytesSaved) {
|
||||
if (NumBytesSaved)
|
||||
outs() << "BOLT-INFO: removed " << NumBytesSaved
|
||||
<< " 'repz' prefixes"
|
||||
" with estimated execution count of "
|
||||
<< NumPrefixesRemoved << " times.\n";
|
||||
}
|
||||
}
|
||||
|
||||
void InlineMemcpy::runOnFunctions(BinaryContext &BC) {
|
||||
|
|
|
@ -95,10 +95,9 @@ double calcExtTSPScore(
|
|||
for (BinaryBasicBlock *SrcBB : BF->layout()) {
|
||||
auto BI = SrcBB->branch_info_begin();
|
||||
for (BinaryBasicBlock *DstBB : SrcBB->successors()) {
|
||||
if (DstBB != SrcBB) {
|
||||
if (DstBB != SrcBB)
|
||||
Score += CacheMetrics::extTSPScore(BBAddr.at(SrcBB), BBSize.at(SrcBB),
|
||||
BBAddr.at(DstBB), BI->Count);
|
||||
}
|
||||
++BI;
|
||||
}
|
||||
}
|
||||
|
@ -169,9 +168,8 @@ double expectedCacheHitRatio(
|
|||
std::unordered_map<BinaryFunction *, double> FunctionSamples;
|
||||
for (BinaryFunction *BF : BinaryFunctions) {
|
||||
double Samples = 0;
|
||||
for (std::pair<BinaryFunction *, uint64_t> Pair : Calls[BF]) {
|
||||
for (std::pair<BinaryFunction *, uint64_t> Pair : Calls[BF])
|
||||
Samples += Pair.second;
|
||||
}
|
||||
Samples = std::max(Samples, (double)BF->getKnownExecutionCount());
|
||||
FunctionSamples[BF] = Samples;
|
||||
TotalSamples += Samples;
|
||||
|
|
|
@ -68,10 +68,9 @@ void doForAllPreds(const BinaryBasicBlock &BB,
|
|||
/// Operates on all successors of a basic block.
|
||||
void doForAllSuccs(const BinaryBasicBlock &BB,
|
||||
std::function<void(ProgramPoint)> Task) {
|
||||
for (BinaryBasicBlock *Succ : BB.successors()) {
|
||||
for (BinaryBasicBlock *Succ : BB.successors())
|
||||
if (Succ->isValid())
|
||||
Task(ProgramPoint::getFirstPointAt(*Succ));
|
||||
}
|
||||
}
|
||||
|
||||
void RegStatePrinter::print(raw_ostream &OS, const BitVector &State) const {
|
||||
|
@ -83,14 +82,12 @@ void RegStatePrinter::print(raw_ostream &OS, const BitVector &State) const {
|
|||
OS << "all, except: ";
|
||||
BitVector BV = State;
|
||||
BV.flip();
|
||||
for (int I = BV.find_first(); I != -1; I = BV.find_next(I)) {
|
||||
for (int I = BV.find_first(); I != -1; I = BV.find_next(I))
|
||||
OS << BC.MRI->getName(I) << " ";
|
||||
}
|
||||
return;
|
||||
}
|
||||
for (int I = State.find_first(); I != -1; I = State.find_next(I)) {
|
||||
for (int I = State.find_first(); I != -1; I = State.find_next(I))
|
||||
OS << BC.MRI->getName(I) << " ";
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace bolt
|
||||
|
|
|
@ -254,10 +254,9 @@ public:
|
|||
const std::vector<std::pair<Chain *, Edge *>> &edges() const { return Edges; }
|
||||
|
||||
Edge *getEdge(Chain *Other) const {
|
||||
for (std::pair<Chain *, Edge *> It : Edges) {
|
||||
for (std::pair<Chain *, Edge *> It : Edges)
|
||||
if (It.first == Other)
|
||||
return It.second;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
|
@ -390,16 +389,14 @@ void Chain::mergeEdges(Chain *Other) {
|
|||
if (curEdge == nullptr) {
|
||||
DstEdge->changeEndpoint(Other, this);
|
||||
this->addEdge(TargetChain, DstEdge);
|
||||
if (DstChain != this && DstChain != Other) {
|
||||
if (DstChain != this && DstChain != Other)
|
||||
DstChain->addEdge(this, DstEdge);
|
||||
}
|
||||
} else {
|
||||
curEdge->moveJumps(DstEdge);
|
||||
}
|
||||
// Cleanup leftover edge
|
||||
if (DstChain != Other) {
|
||||
if (DstChain != Other)
|
||||
DstChain->removeEdge(Other);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -480,9 +477,8 @@ private:
|
|||
void initialize() {
|
||||
// Create a separate MCCodeEmitter to allow lock-free execution
|
||||
BinaryContext::IndependentCodeEmitter Emitter;
|
||||
if (!opts::NoThreads) {
|
||||
if (!opts::NoThreads)
|
||||
Emitter = BF.getBinaryContext().createIndependentMCCodeEmitter();
|
||||
}
|
||||
|
||||
// Initialize CFG nodes
|
||||
AllBlocks.reserve(BF.layout_size());
|
||||
|
@ -531,9 +527,8 @@ private:
|
|||
for (Block &Block : AllBlocks) {
|
||||
AllChains.emplace_back(Block.Index, &Block);
|
||||
Block.CurChain = &AllChains.back();
|
||||
if (Block.ExecutionCount > 0) {
|
||||
if (Block.ExecutionCount > 0)
|
||||
HotChains.push_back(&AllChains.back());
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize edges
|
||||
|
@ -595,9 +590,9 @@ private:
|
|||
continue;
|
||||
|
||||
class Block *SuccBlock = Block.FallthroughSucc;
|
||||
while (SuccBlock != nullptr && SuccBlock != &Block) {
|
||||
while (SuccBlock != nullptr && SuccBlock != &Block)
|
||||
SuccBlock = SuccBlock->FallthroughSucc;
|
||||
}
|
||||
|
||||
if (SuccBlock == nullptr)
|
||||
continue;
|
||||
// break the cycle
|
||||
|
@ -674,9 +669,8 @@ private:
|
|||
Chain *DstChain = AllBlocks[DstIndex].CurChain;
|
||||
if (SrcChain != DstChain && !DstChain->isEntryPoint() &&
|
||||
SrcChain->blocks().back()->Index == SrcIndex &&
|
||||
DstChain->blocks().front()->Index == DstIndex) {
|
||||
DstChain->blocks().front()->Index == DstIndex)
|
||||
mergeChains(SrcChain, DstChain, 0, MergeTypeTy::X_Y);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -710,9 +704,8 @@ private:
|
|||
/// result is a pair with the first element being the gain and the second
|
||||
/// element being the corresponding merging type.
|
||||
MergeGainTy mergeGain(Chain *ChainPred, Chain *ChainSucc, Edge *Edge) const {
|
||||
if (Edge->hasCachedMergeGain(ChainPred, ChainSucc)) {
|
||||
if (Edge->hasCachedMergeGain(ChainPred, ChainSucc))
|
||||
return Edge->getCachedMergeGain(ChainPred, ChainSucc);
|
||||
}
|
||||
|
||||
// Precompute jumps between ChainPred and ChainSucc
|
||||
JumpList Jumps = Edge->jumps();
|
||||
|
@ -826,20 +819,17 @@ private:
|
|||
HotChains.erase(Iter, HotChains.end());
|
||||
|
||||
// Invalidate caches
|
||||
for (std::pair<Chain *, Edge *> EdgeIter : Into->edges()) {
|
||||
for (std::pair<Chain *, Edge *> EdgeIter : Into->edges())
|
||||
EdgeIter.second->invalidateCache();
|
||||
}
|
||||
}
|
||||
|
||||
/// Concatenate all chains into a final order
|
||||
void concatChains(BinaryFunction::BasicBlockOrderType &Order) {
|
||||
// Collect chains
|
||||
std::vector<Chain *> SortedChains;
|
||||
for (Chain &Chain : AllChains) {
|
||||
if (Chain.blocks().size() > 0) {
|
||||
for (Chain &Chain : AllChains)
|
||||
if (Chain.blocks().size() > 0)
|
||||
SortedChains.push_back(&Chain);
|
||||
}
|
||||
}
|
||||
|
||||
// Sorting chains by density in decreasing order
|
||||
std::stable_sort(
|
||||
|
@ -865,11 +855,9 @@ private:
|
|||
|
||||
// Collect the basic blocks in the order specified by their chains
|
||||
Order.reserve(BF.layout_size());
|
||||
for (Chain *Chain : SortedChains) {
|
||||
for (Block *Block : Chain->blocks()) {
|
||||
for (Chain *Chain : SortedChains)
|
||||
for (Block *Block : Chain->blocks())
|
||||
Order.push_back(Block->BB);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
|
@ -896,9 +884,8 @@ void ExtTSPReorderAlgorithm::reorderBasicBlocks(const BinaryFunction &BF,
|
|||
|
||||
// Do not change layout of functions w/o profile information
|
||||
if (!BF.hasValidProfile() || BF.layout_size() <= 2) {
|
||||
for (BinaryBasicBlock *BB : BF.layout()) {
|
||||
for (BinaryBasicBlock *BB : BF.layout())
|
||||
Order.push_back(BB);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
|
@ -49,9 +49,8 @@ bool shouldFrameOptimize(const llvm::bolt::BinaryFunction &Function) {
|
|||
assert(!FrameOptFunctionNamesFile.empty() && "unexpected empty file name");
|
||||
std::ifstream FuncsFile(FrameOptFunctionNamesFile, std::ios::in);
|
||||
std::string FuncName;
|
||||
while (std::getline(FuncsFile, FuncName)) {
|
||||
while (std::getline(FuncsFile, FuncName))
|
||||
FrameOptFunctionNames.push_back(FuncName);
|
||||
}
|
||||
FrameOptFunctionNamesFile = "";
|
||||
}
|
||||
|
||||
|
@ -199,9 +198,8 @@ public:
|
|||
CFIStack.push(std::make_pair(CfaOffset, CfaReg));
|
||||
break;
|
||||
case MCCFIInstruction::OpRestoreState: {
|
||||
if (CFIStack.empty()) {
|
||||
if (CFIStack.empty())
|
||||
dbgs() << "Assertion is about to fail: " << BF.getPrintName() << "\n";
|
||||
}
|
||||
assert(!CFIStack.empty() && "Corrupt CFI stack");
|
||||
std::pair<int64_t, uint16_t> &Elem = CFIStack.top();
|
||||
CFIStack.pop();
|
||||
|
@ -307,13 +305,11 @@ void FrameAnalysis::traverseCG(BinaryFunctionCallGraph &CG) {
|
|||
const BinaryFunction *Func = MapEntry.first;
|
||||
const auto &Set = MapEntry.second;
|
||||
dbgs() << "Args accessed for " << Func->getPrintName() << ": ";
|
||||
if (!Set.empty() && Set.count(std::make_pair(-1, 0))) {
|
||||
if (!Set.empty() && Set.count(std::make_pair(-1, 0)))
|
||||
dbgs() << "assume everything";
|
||||
} else {
|
||||
for (const std::pair<int64_t, uint8_t> &Entry : Set) {
|
||||
else
|
||||
for (const std::pair<int64_t, uint8_t> &Entry : Set)
|
||||
dbgs() << "[" << Entry.first << ", " << (int)Entry.second << "] ";
|
||||
}
|
||||
}
|
||||
dbgs() << "\n";
|
||||
}
|
||||
});
|
||||
|
@ -567,10 +563,9 @@ FrameAnalysis::FrameAnalysis(BinaryContext &BC, BinaryFunctionCallGraph &CG)
|
|||
clearSPTMap();
|
||||
|
||||
// Clean up memory allocated for annotation values
|
||||
if (!opts::NoThreads) {
|
||||
if (!opts::NoThreads)
|
||||
for (MCPlusBuilder::AllocatorIdTy Id : SPTAllocatorsId)
|
||||
BC.MIB->freeValuesAllocator(Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -151,14 +151,13 @@ void FrameOptimizerPass::removeUnnecessaryLoads(const RegAnalysis &RA,
|
|||
Prev = &Inst;
|
||||
}
|
||||
}
|
||||
if (Changed) {
|
||||
if (Changed)
|
||||
LLVM_DEBUG(dbgs() << "FOP modified \"" << BF.getPrintName() << "\"\n");
|
||||
}
|
||||
|
||||
// TODO: Implement an interface of eraseInstruction that works out the
|
||||
// complete list of elements to remove.
|
||||
for (std::pair<BinaryBasicBlock *, MCInst *> I : ToErase) {
|
||||
for (std::pair<BinaryBasicBlock *, MCInst *> I : ToErase)
|
||||
I.first->eraseInstruction(I.first->findInstruction(I.second));
|
||||
}
|
||||
}
|
||||
|
||||
void FrameOptimizerPass::removeUnusedStores(const FrameAnalysis &FA,
|
||||
|
@ -215,12 +214,11 @@ void FrameOptimizerPass::removeUnusedStores(const FrameAnalysis &FA,
|
|||
}
|
||||
}
|
||||
|
||||
for (std::pair<BinaryBasicBlock *, MCInst *> I : ToErase) {
|
||||
for (std::pair<BinaryBasicBlock *, MCInst *> I : ToErase)
|
||||
I.first->eraseInstruction(I.first->findInstruction(I.second));
|
||||
}
|
||||
if (Changed) {
|
||||
|
||||
if (Changed)
|
||||
LLVM_DEBUG(dbgs() << "FOP modified \"" << BF.getPrintName() << "\"\n");
|
||||
}
|
||||
}
|
||||
|
||||
void FrameOptimizerPass::runOnFunctions(BinaryContext &BC) {
|
||||
|
|
|
@ -147,9 +147,8 @@ std::vector<Cluster> clusterize(const CallGraph &Cg) {
|
|||
|
||||
// The size and order of Clusters is fixed until we reshuffle it immediately
|
||||
// before returning.
|
||||
for (Cluster &Cluster : Clusters) {
|
||||
for (Cluster &Cluster : Clusters)
|
||||
FuncCluster[Cluster.targets().front()] = &Cluster;
|
||||
}
|
||||
|
||||
std::sort(SortedFuncs.begin(), SortedFuncs.end(),
|
||||
[&](const NodeId F1, const NodeId F2) {
|
||||
|
@ -192,9 +191,8 @@ std::vector<Cluster> clusterize(const CallGraph &Cg) {
|
|||
// Skip if no predCluster (predecessor w/ no samples), or if same
|
||||
// as cluster, of it's frozen.
|
||||
if (PredCluster == nullptr || PredCluster == Cluster ||
|
||||
PredCluster->frozen()) {
|
||||
PredCluster->frozen())
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip if merged cluster would be bigger than the threshold.
|
||||
if (Cluster->size() + PredCluster->size() > MaxClusterSize)
|
||||
|
@ -216,9 +214,8 @@ std::vector<Cluster> clusterize(const CallGraph &Cg) {
|
|||
Cluster->toString().c_str(), Cg.samples(Fid));
|
||||
});
|
||||
|
||||
for (NodeId F : Cluster->targets()) {
|
||||
for (NodeId F : Cluster->targets())
|
||||
FuncCluster[F] = PredCluster;
|
||||
}
|
||||
|
||||
PredCluster->merge(*Cluster);
|
||||
Cluster->clear();
|
||||
|
@ -230,9 +227,9 @@ std::vector<Cluster> clusterize(const CallGraph &Cg) {
|
|||
std::unordered_set<Cluster *> Visited;
|
||||
for (const NodeId Func : SortedFuncs) {
|
||||
Cluster *Cluster = FuncCluster[Func];
|
||||
if (!Cluster || Visited.count(Cluster) == 1 || Cluster->target(0) != Func) {
|
||||
if (!Cluster || Visited.count(Cluster) == 1 || Cluster->target(0) != Func)
|
||||
continue;
|
||||
}
|
||||
|
||||
SortedClusters.emplace_back(std::move(*Cluster));
|
||||
Visited.insert(Cluster);
|
||||
}
|
||||
|
@ -262,9 +259,8 @@ std::vector<Cluster> randomClusters(const CallGraph &Cg) {
|
|||
size_t MaxIdx = Idx + 1;
|
||||
|
||||
while (MaxIdx < Clusters.size() &&
|
||||
Clusters[Idx].size() + Clusters[MaxIdx].size() <= MaxClusterSize) {
|
||||
Clusters[Idx].size() + Clusters[MaxIdx].size() <= MaxClusterSize)
|
||||
++MaxIdx;
|
||||
}
|
||||
|
||||
if (MaxIdx - Idx > 1) {
|
||||
size_t MergeIdx = (std::rand() % (MaxIdx - Idx - 1)) + Idx + 1;
|
||||
|
|
|
@ -104,10 +104,9 @@ public:
|
|||
double density() const { return static_cast<double>(Samples) / Size; }
|
||||
|
||||
Edge *getEdge(Chain *Other) const {
|
||||
for (std::pair<Chain *, Edge *> It : Edges) {
|
||||
for (std::pair<Chain *, Edge *> It : Edges)
|
||||
if (It.first == Other)
|
||||
return It.second;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
|
@ -228,16 +227,14 @@ void Chain::mergeEdges(Chain *Other) {
|
|||
if (CurEdge == nullptr) {
|
||||
DstEdge->changeEndpoint(Other, this);
|
||||
this->addEdge(TargetChain, DstEdge);
|
||||
if (DstChain != this && DstChain != Other) {
|
||||
if (DstChain != this && DstChain != Other)
|
||||
DstChain->addEdge(this, DstEdge);
|
||||
}
|
||||
} else {
|
||||
CurEdge->moveArcs(DstEdge);
|
||||
}
|
||||
// Cleanup leftover edge
|
||||
if (DstChain != Other) {
|
||||
if (DstChain != Other)
|
||||
DstChain->removeEdge(Other);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -258,9 +255,8 @@ public:
|
|||
|
||||
// Sorting chains by density in decreasing order
|
||||
auto DensityComparator = [](const Chain *L, const Chain *R) {
|
||||
if (L->density() != R->density()) {
|
||||
if (L->density() != R->density())
|
||||
return L->density() > R->density();
|
||||
}
|
||||
// Making sure the comparison is deterministic
|
||||
return L->Id < R->Id;
|
||||
};
|
||||
|
@ -270,9 +266,8 @@ public:
|
|||
// didn't get merged (so their first func is its original func)
|
||||
std::vector<Cluster> Clusters;
|
||||
Clusters.reserve(HotChains.size());
|
||||
for (Chain *Chain : HotChains) {
|
||||
for (Chain *Chain : HotChains)
|
||||
Clusters.emplace_back(Cluster(Chain->Nodes, Cg));
|
||||
}
|
||||
return Clusters;
|
||||
}
|
||||
|
||||
|
@ -344,9 +339,8 @@ private:
|
|||
double missProbability(double ChainDensity) const {
|
||||
double PageSamples = ChainDensity * opts::ITLBDensity;
|
||||
|
||||
if (PageSamples >= TotalSamples) {
|
||||
if (PageSamples >= TotalSamples)
|
||||
return 0;
|
||||
}
|
||||
|
||||
double P = PageSamples / TotalSamples;
|
||||
return pow(1.0 - P, double(opts::ITLBEntries));
|
||||
|
@ -357,9 +351,8 @@ private:
|
|||
double expectedCalls(uint64_t SrcAddr, uint64_t DstAddr,
|
||||
double Weight) const {
|
||||
uint64_t Dist = SrcAddr >= DstAddr ? SrcAddr - DstAddr : DstAddr - SrcAddr;
|
||||
if (Dist >= opts::ITLBPageSize) {
|
||||
if (Dist >= opts::ITLBPageSize)
|
||||
return 0;
|
||||
}
|
||||
|
||||
double D = double(Dist) / double(opts::ITLBPageSize);
|
||||
// Increasing the importance of shorter calls
|
||||
|
@ -453,9 +446,8 @@ private:
|
|||
|
||||
const Arc &Arc = *Cg.findArc(F, Succ);
|
||||
if (Arc.weight() == 0.0 ||
|
||||
Arc.weight() / TotalSamples < opts::ArcThreshold) {
|
||||
Arc.weight() / TotalSamples < opts::ArcThreshold)
|
||||
continue;
|
||||
}
|
||||
|
||||
const double CallsFromPred = OutWeight[F];
|
||||
const double CallsToSucc = InWeight[Succ];
|
||||
|
@ -470,9 +462,8 @@ private:
|
|||
const double ProbIn = CallsToSucc > 0 ? CallsPredSucc / CallsToSucc : 0;
|
||||
assert(0.0 <= ProbIn && ProbIn <= 1.0 && "incorrect in-probability");
|
||||
|
||||
if (std::min(ProbOut, ProbIn) >= opts::MergeProbability) {
|
||||
if (std::min(ProbOut, ProbIn) >= opts::MergeProbability)
|
||||
ArcsToMerge.push_back(&Arc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -488,9 +479,8 @@ private:
|
|||
if (ChainPred == ChainSucc)
|
||||
continue;
|
||||
if (ChainPred->Nodes.back() == Arc->src() &&
|
||||
ChainSucc->Nodes.front() == Arc->dst()) {
|
||||
ChainSucc->Nodes.front() == Arc->dst())
|
||||
mergeChains(ChainPred, ChainSucc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -500,13 +490,13 @@ private:
|
|||
void runPassTwo() {
|
||||
// Creating a priority queue containing all edges ordered by the merge gain
|
||||
auto GainComparator = [](Edge *L, Edge *R) {
|
||||
if (std::abs(L->gain() - R->gain()) > 1e-8) {
|
||||
if (std::abs(L->gain() - R->gain()) > 1e-8)
|
||||
return L->gain() > R->gain();
|
||||
}
|
||||
|
||||
// Making sure the comparison is deterministic
|
||||
if (L->predChain()->Id != R->predChain()->Id) {
|
||||
if (L->predChain()->Id != R->predChain()->Id)
|
||||
return L->predChain()->Id < R->predChain()->Id;
|
||||
}
|
||||
|
||||
return L->succChain()->Id < R->succChain()->Id;
|
||||
};
|
||||
std::set<Edge *, decltype(GainComparator)> Queue(GainComparator);
|
||||
|
@ -527,9 +517,8 @@ private:
|
|||
auto ForwardGain = mergeGain(ChainPred, ChainSucc, ChainEdge);
|
||||
auto BackwardGain = mergeGain(ChainSucc, ChainPred, ChainEdge);
|
||||
ChainEdge->setMergeGain(ChainPred, ForwardGain, BackwardGain);
|
||||
if (ChainEdge->gain() > 0.0) {
|
||||
if (ChainEdge->gain() > 0.0)
|
||||
Queue.insert(ChainEdge);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -545,12 +534,10 @@ private:
|
|||
continue;
|
||||
|
||||
// Remove outdated edges
|
||||
for (std::pair<Chain *, Edge *> EdgeIt : BestChainPred->Edges) {
|
||||
for (std::pair<Chain *, Edge *> EdgeIt : BestChainPred->Edges)
|
||||
Queue.erase(EdgeIt.second);
|
||||
}
|
||||
for (std::pair<Chain *, Edge *> EdgeIt : BestChainSucc->Edges) {
|
||||
for (std::pair<Chain *, Edge *> EdgeIt : BestChainSucc->Edges)
|
||||
Queue.erase(EdgeIt.second);
|
||||
}
|
||||
|
||||
// Merge the best pair of chains
|
||||
mergeChains(BestChainPred, BestChainSucc);
|
||||
|
@ -567,9 +554,8 @@ private:
|
|||
auto ForwardGain = mergeGain(BestChainPred, ChainSucc, ChainEdge);
|
||||
auto BackwardGain = mergeGain(ChainSucc, BestChainPred, ChainEdge);
|
||||
ChainEdge->setMergeGain(BestChainPred, ForwardGain, BackwardGain);
|
||||
if (ChainEdge->gain() > 0.0) {
|
||||
if (ChainEdge->gain() > 0.0)
|
||||
Queue.insert(ChainEdge);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -96,9 +96,8 @@ template <class Compare>
|
|||
bool isInstrEquivalentWith(const MCInst &InstA, const BinaryBasicBlock &BBA,
|
||||
const MCInst &InstB, const BinaryBasicBlock &BBB,
|
||||
Compare Comp) {
|
||||
if (InstA.getOpcode() != InstB.getOpcode()) {
|
||||
if (InstA.getOpcode() != InstB.getOpcode())
|
||||
return false;
|
||||
}
|
||||
|
||||
const BinaryContext &BC = BBA.getFunction()->getBinaryContext();
|
||||
|
||||
|
@ -237,9 +236,8 @@ bool isIdenticalWith(const BinaryFunction &A, const BinaryFunction &B,
|
|||
}
|
||||
|
||||
// One of the symbols represents a function, the other one does not.
|
||||
if (FunctionA != FunctionB) {
|
||||
if (FunctionA != FunctionB)
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check if symbols are jump tables.
|
||||
const BinaryData *SIA = BC.getBinaryDataByName(SymbolA->getName());
|
||||
|
@ -270,9 +268,8 @@ bool isIdenticalWith(const BinaryFunction &A, const BinaryFunction &B,
|
|||
};
|
||||
|
||||
if (!isInstrEquivalentWith(*I, *BB, *OtherI, *OtherBB,
|
||||
AreSymbolsIdentical)) {
|
||||
AreSymbolsIdentical))
|
||||
return false;
|
||||
}
|
||||
|
||||
++I;
|
||||
++OtherI;
|
||||
|
@ -282,9 +279,8 @@ bool isIdenticalWith(const BinaryFunction &A, const BinaryFunction &B,
|
|||
// is ignored for CFG purposes.
|
||||
const MCInst *TrailingInstr =
|
||||
(I != E ? &(*I) : (OtherI != OtherE ? &(*OtherI) : 0));
|
||||
if (TrailingInstr && !BC.MIB->isUnconditionalBranch(*TrailingInstr)) {
|
||||
if (TrailingInstr && !BC.MIB->isUnconditionalBranch(*TrailingInstr))
|
||||
return false;
|
||||
}
|
||||
|
||||
++BBI;
|
||||
}
|
||||
|
@ -292,9 +288,8 @@ bool isIdenticalWith(const BinaryFunction &A, const BinaryFunction &B,
|
|||
// Compare exceptions action tables.
|
||||
if (A.getLSDAActionTable() != B.getLSDAActionTable() ||
|
||||
A.getLSDATypeTable() != B.getLSDATypeTable() ||
|
||||
A.getLSDATypeIndexTable() != B.getLSDATypeIndexTable()) {
|
||||
A.getLSDATypeIndexTable() != B.getLSDATypeIndexTable())
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
@ -338,9 +333,9 @@ typedef std::unordered_map<BinaryFunction *, std::vector<BinaryFunction *>,
|
|||
|
||||
std::string hashInteger(uint64_t Value) {
|
||||
std::string HashString;
|
||||
if (Value == 0) {
|
||||
if (Value == 0)
|
||||
HashString.push_back(0);
|
||||
}
|
||||
|
||||
while (Value) {
|
||||
uint8_t LSB = Value & 0xff;
|
||||
HashString.push_back(LSB);
|
||||
|
@ -393,13 +388,12 @@ std::string hashExpr(BinaryContext &BC, const MCExpr &Expr) {
|
|||
}
|
||||
|
||||
std::string hashInstOperand(BinaryContext &BC, const MCOperand &Operand) {
|
||||
if (Operand.isImm()) {
|
||||
if (Operand.isImm())
|
||||
return hashInteger(Operand.getImm());
|
||||
} else if (Operand.isReg()) {
|
||||
if (Operand.isReg())
|
||||
return hashInteger(Operand.getReg());
|
||||
} else if (Operand.isExpr()) {
|
||||
if (Operand.isExpr())
|
||||
return hashExpr(BC, *Operand.getExpr());
|
||||
}
|
||||
|
||||
return std::string();
|
||||
}
|
||||
|
@ -567,15 +561,14 @@ void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) {
|
|||
<< " bytes) are congruent but not identical:\n";
|
||||
for (BinaryFunction *BF : Candidates) {
|
||||
dbgs() << " " << *BF;
|
||||
if (BF->getKnownExecutionCount()) {
|
||||
if (BF->getKnownExecutionCount())
|
||||
dbgs() << " (executed " << BF->getKnownExecutionCount() << " times)";
|
||||
}
|
||||
dbgs() << '\n';
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (NumFunctionsFolded) {
|
||||
if (NumFunctionsFolded)
|
||||
outs() << "BOLT-INFO: ICF folded " << NumFunctionsFolded << " out of "
|
||||
<< OriginalFunctionCount << " functions in " << Iteration
|
||||
<< " passes. " << NumJTFunctionsFolded
|
||||
|
@ -584,7 +577,6 @@ void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) {
|
|||
<< format("%.2lf", (double)BytesSavedEstimate / 1024)
|
||||
<< " KB of code space. Folded functions were called "
|
||||
<< CallsSavedEstimate << " times based on profile.\n";
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace bolt
|
||||
|
|
|
@ -261,12 +261,11 @@ IndirectCallPromotion::getCallTargets(BinaryBasicBlock &BB,
|
|||
while (++First != Last) {
|
||||
Callsite &A = *Result;
|
||||
const Callsite &B = *First;
|
||||
if (A.To.Sym && B.To.Sym && A.To.Sym == B.To.Sym) {
|
||||
if (A.To.Sym && B.To.Sym && A.To.Sym == B.To.Sym)
|
||||
A.JTIndices.insert(A.JTIndices.end(), B.JTIndices.begin(),
|
||||
B.JTIndices.end());
|
||||
} else {
|
||||
else
|
||||
*(++Result) = *First;
|
||||
}
|
||||
}
|
||||
++Result;
|
||||
|
||||
|
@ -279,9 +278,9 @@ IndirectCallPromotion::getCallTargets(BinaryBasicBlock &BB,
|
|||
} else {
|
||||
// Don't try to optimize PC relative indirect calls.
|
||||
if (Inst.getOperand(0).isReg() &&
|
||||
Inst.getOperand(0).getReg() == BC.MRI->getProgramCounter()) {
|
||||
Inst.getOperand(0).getReg() == BC.MRI->getProgramCounter())
|
||||
return Targets;
|
||||
}
|
||||
|
||||
auto ICSP = BC.MIB->tryGetAnnotationAs<IndirectCallSiteProfile>(
|
||||
Inst, "CallProfile");
|
||||
if (ICSP) {
|
||||
|
@ -374,9 +373,8 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(BinaryBasicBlock &BB,
|
|||
<< (&CallInst - &BB.front()) << "\n"
|
||||
<< "BOLT-INFO: ICP target fetch instructions:\n";
|
||||
BC.printInstruction(dbgs(), *MemLocInstr, 0, &Function);
|
||||
if (MemLocInstr != &CallInst) {
|
||||
if (MemLocInstr != &CallInst)
|
||||
BC.printInstruction(dbgs(), CallInst, 0, &Function);
|
||||
}
|
||||
});
|
||||
|
||||
DEBUG_VERBOSE(1, {
|
||||
|
@ -412,9 +410,8 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(BinaryBasicBlock &BB,
|
|||
ArrayStart = static_cast<uint64_t>(DispValue);
|
||||
}
|
||||
|
||||
if (BaseReg == BC.MRI->getProgramCounter()) {
|
||||
if (BaseReg == BC.MRI->getProgramCounter())
|
||||
ArrayStart += Function.getAddress() + MemAccessProfile.NextInstrOffset;
|
||||
}
|
||||
|
||||
// This is a map of [symbol] -> [count, index] and is used to combine indices
|
||||
// into the jump table since there may be multiple addresses that all have the
|
||||
|
@ -460,11 +457,10 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(BinaryBasicBlock &BB,
|
|||
JT->print(dbgs());
|
||||
dbgs() << "HotTargetMap:\n";
|
||||
for (std::pair<MCSymbol *const, std::pair<uint64_t, uint64_t>> &HT :
|
||||
HotTargetMap) {
|
||||
HotTargetMap)
|
||||
dbgs() << "BOLT-INFO: " << HT.first->getName()
|
||||
<< " = (count=" << HT.first << ", index=" << HT.second
|
||||
<< ")\n";
|
||||
}
|
||||
});
|
||||
return JumpTableInfoType();
|
||||
}
|
||||
|
@ -486,10 +482,9 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(BinaryBasicBlock &BB,
|
|||
|
||||
LLVM_DEBUG({
|
||||
dbgs() << "BOLT-INFO: ICP jump table hot targets:\n";
|
||||
for (const std::pair<uint64_t, uint64_t> &Target : HotTargets) {
|
||||
for (const std::pair<uint64_t, uint64_t> &Target : HotTargets)
|
||||
dbgs() << "BOLT-INFO: Idx = " << Target.second << ", "
|
||||
<< "Count = " << Target.first << "\n";
|
||||
}
|
||||
});
|
||||
|
||||
BC.MIB->getOrCreateAnnotationAs<uint16_t>(CallInst, "JTIndexReg") = IndexReg;
|
||||
|
@ -525,13 +520,11 @@ IndirectCallPromotion::findCallTargetSymbols(std::vector<Callsite> &Targets,
|
|||
"callsite");
|
||||
};
|
||||
|
||||
if (opts::Verbosity >= 1) {
|
||||
for (size_t I = 0; I < HotTargets.size(); ++I) {
|
||||
if (opts::Verbosity >= 1)
|
||||
for (size_t I = 0; I < HotTargets.size(); ++I)
|
||||
outs() << "BOLT-INFO: HotTarget[" << I << "] = ("
|
||||
<< HotTargets[I].first << ", " << HotTargets[I].second
|
||||
<< ")\n";
|
||||
}
|
||||
}
|
||||
|
||||
// Recompute hottest targets, now discriminating which index is hot
|
||||
// NOTE: This is a tradeoff. On one hand, we get index information. On the
|
||||
|
@ -649,12 +642,11 @@ IndirectCallPromotion::MethodInfoType IndirectCallPromotion::maybeGetVtableSyms(
|
|||
dbgs() << "BOLT-INFO: ICP found virtual method call in " << Function
|
||||
<< " at @ " << (&Inst - &BB.front()) << "\n";
|
||||
dbgs() << "BOLT-INFO: ICP method fetch instructions:\n";
|
||||
for (MCInst *Inst : MethodFetchInsns) {
|
||||
for (MCInst *Inst : MethodFetchInsns)
|
||||
BC.printInstruction(dbgs(), *Inst, 0, &Function);
|
||||
}
|
||||
if (MethodFetchInsns.back() != &Inst) {
|
||||
|
||||
if (MethodFetchInsns.back() != &Inst)
|
||||
BC.printInstruction(dbgs(), Inst, 0, &Function);
|
||||
}
|
||||
});
|
||||
|
||||
// Try to get value profiling data for the method load instruction.
|
||||
|
@ -673,9 +665,8 @@ IndirectCallPromotion::MethodInfoType IndirectCallPromotion::maybeGetVtableSyms(
|
|||
|
||||
for (const AddressAccess &AccessInfo : MemAccessProfile.AddressAccessInfo) {
|
||||
uint64_t Address = AccessInfo.Offset;
|
||||
if (AccessInfo.MemoryObject) {
|
||||
if (AccessInfo.MemoryObject)
|
||||
Address += AccessInfo.MemoryObject->getAddress();
|
||||
}
|
||||
|
||||
// Ignore bogus data.
|
||||
if (!Address)
|
||||
|
@ -723,9 +714,8 @@ IndirectCallPromotion::MethodInfoType IndirectCallPromotion::maybeGetVtableSyms(
|
|||
for (MCInst *CurInst = MethodFetchInsns.front(); CurInst < &Inst;
|
||||
++CurInst) {
|
||||
const MCInstrDesc &InstrInfo = BC.MII->get(CurInst->getOpcode());
|
||||
if (InstrInfo.hasDefOfPhysReg(*CurInst, VtableReg, *BC.MRI)) {
|
||||
if (InstrInfo.hasDefOfPhysReg(*CurInst, VtableReg, *BC.MRI))
|
||||
return MethodInfoType();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -752,12 +742,10 @@ IndirectCallPromotion::rewriteCall(
|
|||
// must be preserved and moved to the original block.
|
||||
InstructionListType TailInsts;
|
||||
const MCInst *TailInst = &CallInst;
|
||||
if (IsTailCallOrJT) {
|
||||
if (IsTailCallOrJT)
|
||||
while (TailInst + 1 < &(*IndCallBlock.end()) &&
|
||||
MIB->isPseudo(*(TailInst + 1))) {
|
||||
MIB->isPseudo(*(TailInst + 1)))
|
||||
TailInsts.push_back(*++TailInst);
|
||||
}
|
||||
}
|
||||
|
||||
InstructionListType MovedInst = IndCallBlock.splitInstructions(&CallInst);
|
||||
// Link new BBs to the original input offset of the BB where the indirect
|
||||
|
@ -768,13 +756,12 @@ IndirectCallPromotion::rewriteCall(
|
|||
IndCallBlock.eraseInstructions(MethodFetchInsns.begin(),
|
||||
MethodFetchInsns.end());
|
||||
if (IndCallBlock.empty() ||
|
||||
(!MethodFetchInsns.empty() && MethodFetchInsns.back() == &CallInst)) {
|
||||
(!MethodFetchInsns.empty() && MethodFetchInsns.back() == &CallInst))
|
||||
IndCallBlock.addInstructions(ICPcode.front().second.begin(),
|
||||
ICPcode.front().second.end());
|
||||
} else {
|
||||
else
|
||||
IndCallBlock.replaceInstruction(std::prev(IndCallBlock.end()),
|
||||
ICPcode.front().second);
|
||||
}
|
||||
IndCallBlock.addInstructions(TailInsts.begin(), TailInsts.end());
|
||||
|
||||
for (auto Itr = ICPcode.begin() + 1; Itr != ICPcode.end(); ++Itr) {
|
||||
|
@ -783,19 +770,17 @@ IndirectCallPromotion::rewriteCall(
|
|||
assert(Sym);
|
||||
std::unique_ptr<BinaryBasicBlock> TBB =
|
||||
Function.createBasicBlock(OrigOffset, Sym);
|
||||
for (MCInst &Inst : Insts) { // sanitize new instructions.
|
||||
for (MCInst &Inst : Insts) // sanitize new instructions.
|
||||
if (MIB->isCall(Inst))
|
||||
MIB->removeAnnotation(Inst, "CallProfile");
|
||||
}
|
||||
TBB->addInstructions(Insts.begin(), Insts.end());
|
||||
NewBBs.emplace_back(std::move(TBB));
|
||||
}
|
||||
|
||||
// Move tail of instructions from after the original call to
|
||||
// the merge block.
|
||||
if (!IsTailCallOrJT) {
|
||||
if (!IsTailCallOrJT)
|
||||
NewBBs.back()->addInstructions(MovedInst.begin(), MovedInst.end());
|
||||
}
|
||||
|
||||
return NewBBs;
|
||||
}
|
||||
|
@ -813,9 +798,8 @@ IndirectCallPromotion::fixCFG(BinaryBasicBlock &IndCallBlock,
|
|||
// basic block containing the indirect call.
|
||||
uint64_t TotalCount = IndCallBlock.getKnownExecutionCount();
|
||||
uint64_t TotalIndirectBranches = 0;
|
||||
for (const Callsite &Target : Targets) {
|
||||
for (const Callsite &Target : Targets)
|
||||
TotalIndirectBranches += Target.Branches;
|
||||
}
|
||||
if (TotalIndirectBranches == 0)
|
||||
TotalIndirectBranches = 1;
|
||||
BinaryBasicBlock::BranchInfoType BBI;
|
||||
|
@ -843,9 +827,8 @@ IndirectCallPromotion::fixCFG(BinaryBasicBlock &IndCallBlock,
|
|||
for (const Callsite &Target : Targets) {
|
||||
const size_t NumEntries =
|
||||
std::max(static_cast<std::size_t>(1UL), Target.JTIndices.size());
|
||||
for (size_t I = 0; I < NumEntries; ++I) {
|
||||
for (size_t I = 0; I < NumEntries; ++I)
|
||||
SymTargets.push_back(Target.To.Sym);
|
||||
}
|
||||
}
|
||||
assert(SymTargets.size() > NewBBs.size() - 1 &&
|
||||
"There must be a target symbol associated with each new BB.");
|
||||
|
@ -902,9 +885,8 @@ IndirectCallPromotion::fixCFG(BinaryBasicBlock &IndCallBlock,
|
|||
TotalCount <= uint64_t(TotalIndirectBranches));
|
||||
uint64_t ExecCount = ScaledBBI[(I + 1) / 2].Count;
|
||||
if (I % 2 == 0) {
|
||||
if (MergeBlock) {
|
||||
if (MergeBlock)
|
||||
NewBBs[I]->addSuccessor(MergeBlock, ScaledBBI[(I + 1) / 2].Count);
|
||||
}
|
||||
} else {
|
||||
assert(I + 2 < NewBBs.size());
|
||||
updateCurrentBranchInfo();
|
||||
|
@ -946,12 +928,11 @@ size_t IndirectCallPromotion::canPromoteCallsite(
|
|||
const bool IsJumpTable = BB.getFunction()->getJumpTable(Inst);
|
||||
|
||||
auto computeStats = [&](size_t N) {
|
||||
for (size_t I = 0; I < N; ++I) {
|
||||
for (size_t I = 0; I < N; ++I)
|
||||
if (!IsJumpTable)
|
||||
TotalNumFrequentCalls += Targets[I].Branches;
|
||||
else
|
||||
TotalNumFrequentJmps += Targets[I].Branches;
|
||||
}
|
||||
};
|
||||
|
||||
// If we have no targets (or no calls), skip this callsite.
|
||||
|
@ -1067,10 +1048,9 @@ size_t IndirectCallPromotion::canPromoteCallsite(
|
|||
|
||||
// Filter functions that can have ICP applied (for debugging)
|
||||
if (!opts::ICPFuncsList.empty()) {
|
||||
for (std::string &Name : opts::ICPFuncsList) {
|
||||
for (std::string &Name : opts::ICPFuncsList)
|
||||
if (BB.getFunction()->hasName(Name))
|
||||
return N;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
@ -1141,9 +1121,8 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
|
|||
// total.
|
||||
SetVector<BinaryFunction *> Functions;
|
||||
if (opts::ICPTopCallsites == 0) {
|
||||
for (auto &KV : BFs) {
|
||||
for (auto &KV : BFs)
|
||||
Functions.insert(&KV.second);
|
||||
}
|
||||
} else {
|
||||
using IndirectCallsite = std::tuple<uint64_t, MCInst *, BinaryFunction *>;
|
||||
std::vector<IndirectCallsite> IndirectCalls;
|
||||
|
@ -1174,9 +1153,8 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
|
|||
((HasIndirectCallProfile && !IsJumpTable && OptimizeCalls) ||
|
||||
(IsJumpTable && OptimizeJumpTables))) {
|
||||
uint64_t NumCalls = 0;
|
||||
for (const Callsite &BInfo : getCallTargets(BB, Inst)) {
|
||||
for (const Callsite &BInfo : getCallTargets(BB, Inst))
|
||||
NumCalls += BInfo.Branches;
|
||||
}
|
||||
IndirectCalls.push_back(
|
||||
std::make_tuple(NumCalls, &Inst, &Function));
|
||||
TotalIndirectCalls += NumCalls;
|
||||
|
@ -1228,9 +1206,8 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
|
|||
std::vector<BinaryBasicBlock *> BBs;
|
||||
for (BinaryBasicBlock &BB : Function) {
|
||||
// Skip indirect calls in cold blocks.
|
||||
if (!HasLayout || !Function.isSplit() || !BB.isCold()) {
|
||||
if (!HasLayout || !Function.isSplit() || !BB.isCold())
|
||||
BBs.push_back(&BB);
|
||||
}
|
||||
}
|
||||
if (BBs.empty())
|
||||
continue;
|
||||
|
@ -1248,9 +1225,8 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
|
|||
BC.MIB->hasAnnotation(Inst, "CallProfile");
|
||||
const bool IsJumpTable = Function.getJumpTable(Inst);
|
||||
|
||||
if (BC.MIB->isCall(Inst)) {
|
||||
if (BC.MIB->isCall(Inst))
|
||||
TotalCalls += BB->getKnownExecutionCount();
|
||||
}
|
||||
|
||||
if (IsJumpTable && !OptimizeJumpTables)
|
||||
continue;
|
||||
|
@ -1274,9 +1250,8 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
|
|||
|
||||
// Compute the total number of calls from this particular callsite.
|
||||
uint64_t NumCalls = 0;
|
||||
for (const Callsite &BInfo : Targets) {
|
||||
for (const Callsite &BInfo : Targets)
|
||||
NumCalls += BInfo.Branches;
|
||||
}
|
||||
if (!IsJumpTable)
|
||||
FuncTotalIndirectCalls += NumCalls;
|
||||
else
|
||||
|
@ -1288,13 +1263,12 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
|
|||
ErrorOr<const BitVector &> State =
|
||||
Info.getLivenessAnalysis().getStateBefore(Inst);
|
||||
if (!State || (State && (*State)[BC.MIB->getFlagsReg()])) {
|
||||
if (opts::Verbosity >= 1) {
|
||||
if (opts::Verbosity >= 1)
|
||||
outs() << "BOLT-INFO: ICP failed in " << Function << " @ "
|
||||
<< InstIdx << " in " << BB->getName()
|
||||
<< ", calls = " << NumCalls
|
||||
<< (State ? ", cannot clobber flags reg.\n"
|
||||
: ", no liveness data available.\n");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
@ -1310,9 +1284,8 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
|
|||
if (!N && !IsJumpTable)
|
||||
continue;
|
||||
|
||||
if (opts::Verbosity >= 1) {
|
||||
if (opts::Verbosity >= 1)
|
||||
printCallsiteInfo(*BB, Inst, Targets, N, NumCalls);
|
||||
}
|
||||
|
||||
// Find MCSymbols or absolute addresses for each call target.
|
||||
MCInst *TargetFetchInst = nullptr;
|
||||
|
@ -1330,13 +1303,12 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
|
|||
// TODO: can this ever happen?
|
||||
if (SymTargets.size() < N) {
|
||||
const size_t LastTarget = SymTargets.size();
|
||||
if (opts::Verbosity >= 1) {
|
||||
if (opts::Verbosity >= 1)
|
||||
outs() << "BOLT-INFO: ICP failed in " << Function << " @ "
|
||||
<< InstIdx << " in " << BB->getName()
|
||||
<< ", calls = " << NumCalls
|
||||
<< ", ICP failed to find target symbol for "
|
||||
<< Targets[LastTarget].To.Sym->getName() << "\n";
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -1364,12 +1336,11 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
|
|||
opts::ICPOldCodeSequence, BC.Ctx.get());
|
||||
|
||||
if (ICPcode.empty()) {
|
||||
if (opts::Verbosity >= 1) {
|
||||
if (opts::Verbosity >= 1)
|
||||
outs() << "BOLT-INFO: ICP failed in " << Function << " @ "
|
||||
<< InstIdx << " in " << BB->getName()
|
||||
<< ", calls = " << NumCalls
|
||||
<< ", unable to generate promoted call code.\n";
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -1398,15 +1369,13 @@ void IndirectCallPromotion::runOnFunctions(BinaryContext &BC) {
|
|||
// Since the tail of the original block was split off and it may contain
|
||||
// additional indirect calls, we must add the merge block to the set of
|
||||
// blocks to process.
|
||||
if (MergeBlock) {
|
||||
if (MergeBlock)
|
||||
BBs.push_back(MergeBlock);
|
||||
}
|
||||
|
||||
if (opts::Verbosity >= 1) {
|
||||
if (opts::Verbosity >= 1)
|
||||
outs() << "BOLT-INFO: ICP succeeded in " << Function << " @ "
|
||||
<< InstIdx << " in " << BB->getName()
|
||||
<< " -> calls = " << NumCalls << "\n";
|
||||
}
|
||||
|
||||
if (IsJumpTable)
|
||||
++TotalOptimizedJumpTableCallsites;
|
||||
|
|
|
@ -123,10 +123,9 @@ bool inliningEnabled() {
|
|||
}
|
||||
|
||||
bool mustConsider(const llvm::bolt::BinaryFunction &Function) {
|
||||
for (std::string &Name : opts::ForceInlineFunctions) {
|
||||
for (std::string &Name : opts::ForceInlineFunctions)
|
||||
if (Function.hasName(Name))
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
@ -290,11 +289,10 @@ Inliner::inlineCall(BinaryBasicBlock &CallerBB,
|
|||
// for return instructions in the callee to redirect to.
|
||||
BinaryBasicBlock *NextBB = nullptr;
|
||||
if (Callee.size() > 1) {
|
||||
if (std::next(CallInst) != FirstInlinedBB->end()) {
|
||||
if (std::next(CallInst) != FirstInlinedBB->end())
|
||||
NextBB = FirstInlinedBB->splitAt(std::next(CallInst));
|
||||
} else {
|
||||
else
|
||||
NextBB = FirstInlinedBB->getSuccessor();
|
||||
}
|
||||
}
|
||||
if (NextBB)
|
||||
FirstInlinedBB->removeSuccessor(NextBB);
|
||||
|
@ -303,10 +301,9 @@ Inliner::inlineCall(BinaryBasicBlock &CallerBB,
|
|||
auto InsertII = FirstInlinedBB->eraseInstruction(CallInst);
|
||||
|
||||
double ProfileRatio = 0;
|
||||
if (uint64_t CalleeExecCount = Callee.getKnownExecutionCount()) {
|
||||
if (uint64_t CalleeExecCount = Callee.getKnownExecutionCount())
|
||||
ProfileRatio =
|
||||
(double)FirstInlinedBB->getKnownExecutionCount() / CalleeExecCount;
|
||||
}
|
||||
|
||||
// Save execution count of the first block as we don't want it to change
|
||||
// later due to profile adjustment rounding errors.
|
||||
|
@ -319,11 +316,10 @@ Inliner::inlineCall(BinaryBasicBlock &CallerBB,
|
|||
BinaryBasicBlock *InlinedBB = CallerFunction.addBasicBlock(0);
|
||||
InlinedBBMap[&*BBI] = InlinedBB;
|
||||
InlinedBB->setCFIState(FirstInlinedBB->getCFIState());
|
||||
if (Callee.hasValidProfile()) {
|
||||
if (Callee.hasValidProfile())
|
||||
InlinedBB->setExecutionCount(BBI->getKnownExecutionCount());
|
||||
} else {
|
||||
else
|
||||
InlinedBB->setExecutionCount(FirstInlinedBBCount);
|
||||
}
|
||||
}
|
||||
|
||||
// Copy over instructions and edges.
|
||||
|
@ -386,12 +382,11 @@ Inliner::inlineCall(BinaryBasicBlock &CallerBB,
|
|||
return InlinedBBMap.at(BB);
|
||||
});
|
||||
|
||||
if (CallerFunction.hasValidProfile() && Callee.hasValidProfile()) {
|
||||
if (CallerFunction.hasValidProfile() && Callee.hasValidProfile())
|
||||
InlinedBB->addSuccessors(Successors.begin(), Successors.end(),
|
||||
BB.branch_info_begin(), BB.branch_info_end());
|
||||
} else {
|
||||
else
|
||||
InlinedBB->addSuccessors(Successors.begin(), Successors.end());
|
||||
}
|
||||
|
||||
if (!CSIsTailCall && BB.succ_size() == 0 && NextBB) {
|
||||
// Either it's a return block or the last instruction never returns.
|
||||
|
@ -400,12 +395,11 @@ Inliner::inlineCall(BinaryBasicBlock &CallerBB,
|
|||
|
||||
// Scale profiling info for blocks and edges after inlining.
|
||||
if (CallerFunction.hasValidProfile() && Callee.size() > 1) {
|
||||
if (opts::AdjustProfile) {
|
||||
if (opts::AdjustProfile)
|
||||
InlinedBB->adjustExecutionCount(ProfileRatio);
|
||||
} else {
|
||||
else
|
||||
InlinedBB->setExecutionCount(InlinedBB->getKnownExecutionCount() *
|
||||
ProfileRatio);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -474,13 +468,12 @@ bool Inliner::inlineCallsInFunction(BinaryFunction &Function) {
|
|||
}
|
||||
|
||||
int64_t SizeAfterInlining;
|
||||
if (IsTailCall) {
|
||||
if (IsTailCall)
|
||||
SizeAfterInlining =
|
||||
IInfo->second.SizeAfterTailCallInlining - getSizeOfTailCallInst(BC);
|
||||
} else {
|
||||
else
|
||||
SizeAfterInlining =
|
||||
IInfo->second.SizeAfterInlining - getSizeOfCallInst(BC);
|
||||
}
|
||||
|
||||
if (!opts::InlineAll && !opts::mustConsider(*TargetFunction)) {
|
||||
if (!opts::InlineSmallFunctions ||
|
||||
|
@ -505,9 +498,8 @@ bool Inliner::inlineCallsInFunction(BinaryFunction &Function) {
|
|||
NumInlinedDynamicCalls += BB->getExecutionCount();
|
||||
|
||||
// Subtract basic block execution count from the callee execution count.
|
||||
if (opts::AdjustProfile) {
|
||||
if (opts::AdjustProfile)
|
||||
TargetFunction->adjustExecutionCount(BB->getKnownExecutionCount());
|
||||
}
|
||||
|
||||
// Check if the caller inlining status has to be adjusted.
|
||||
if (IInfo->second.Type == INL_TAILCALL) {
|
||||
|
@ -520,9 +512,8 @@ bool Inliner::inlineCallsInFunction(BinaryFunction &Function) {
|
|||
}
|
||||
}
|
||||
|
||||
if (NumInlinedCallSites == opts::InlineLimit) {
|
||||
if (NumInlinedCallSites == opts::InlineLimit)
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -577,12 +568,11 @@ void Inliner::runOnFunctions(BinaryContext &BC) {
|
|||
++NumIters;
|
||||
} while (InlinedOnce && NumIters < opts::InlineMaxIters);
|
||||
|
||||
if (NumInlinedCallSites) {
|
||||
if (NumInlinedCallSites)
|
||||
outs() << "BOLT-INFO: inlined " << NumInlinedDynamicCalls << " calls at "
|
||||
<< NumInlinedCallSites << " call sites in " << NumIters
|
||||
<< " iteration(s). Change in binary size: " << TotalInlinedBytes
|
||||
<< " bytes.\n";
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace bolt
|
||||
|
|
|
@ -362,9 +362,8 @@ void Instrumentation::instrumentFunction(BinaryFunction &Function,
|
|||
for (auto BBI = Function.begin(), BBE = Function.end(); BBI != BBE; ++BBI) {
|
||||
for (auto I = BBI->begin(), E = BBI->end(); I != E; ++I) {
|
||||
if (BC.MIB->isCall(*I)) {
|
||||
if (BC.MIB->isInvoke(*I)) {
|
||||
if (BC.MIB->isInvoke(*I))
|
||||
InvokeBlocks.insert(&*BBI);
|
||||
}
|
||||
IsLeafFunction = false;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -143,9 +143,8 @@ bool JTFootprintReduction::tryOptimizeNonPIC(
|
|||
BC.MIB->matchAnyOperand(Offset));
|
||||
if (!IndJmpMatcher->match(*BC.MRI, *BC.MIB,
|
||||
MutableArrayRef<MCInst>(&*BB.begin(), &*Inst + 1),
|
||||
-1)) {
|
||||
-1))
|
||||
return false;
|
||||
}
|
||||
|
||||
assert(Scale == 8 && "Wrong scale");
|
||||
|
||||
|
@ -186,9 +185,8 @@ bool JTFootprintReduction::tryOptimizePIC(BinaryContext &BC,
|
|||
BC.MIB->matchAnyOperand())));
|
||||
if (!PICIndJmpMatcher->match(
|
||||
*BC.MRI, *BC.MIB, MutableArrayRef<MCInst>(&*BB.begin(), &*Inst + 1),
|
||||
-1)) {
|
||||
-1))
|
||||
return false;
|
||||
}
|
||||
|
||||
assert(Scale == 4 && "Wrong scale");
|
||||
|
||||
|
@ -240,14 +238,12 @@ void JTFootprintReduction::optimizeFunction(BinaryFunction &Function,
|
|||
if (!Modified.count(&Function))
|
||||
return;
|
||||
|
||||
for (BinaryBasicBlock &BB : Function) {
|
||||
for (auto I = BB.begin(); I != BB.end();) {
|
||||
for (BinaryBasicBlock &BB : Function)
|
||||
for (auto I = BB.begin(); I != BB.end();)
|
||||
if (BC.MIB->hasAnnotation(*I, "DeleteMe"))
|
||||
I = BB.eraseInstruction(I);
|
||||
else
|
||||
++I;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void JTFootprintReduction::runOnFunctions(BinaryContext &BC) {
|
||||
|
@ -281,10 +277,9 @@ void JTFootprintReduction::runOnFunctions(BinaryContext &BC) {
|
|||
}
|
||||
|
||||
outs() << "BOLT-INFO: JT Footprint reduction stats (simple funcs only):\n";
|
||||
if (OptimizedScore) {
|
||||
if (OptimizedScore)
|
||||
outs() << format("\t %.2lf%%", (OptimizedScore * 100.0 / TotalJTScore))
|
||||
<< " of dynamic JT entries were reduced.\n";
|
||||
}
|
||||
outs() << "\t " << TotalJTs - TotalJTsDenied << " of " << TotalJTs
|
||||
<< " jump tables affected.\n";
|
||||
outs() << "\t " << IndJmps - IndJmpsDenied << " of " << IndJmps
|
||||
|
|
|
@ -259,9 +259,8 @@ LongJmpPass::replaceTargetWithStub(BinaryBasicBlock &BB, MCInst &Inst,
|
|||
void LongJmpPass::updateStubGroups() {
|
||||
auto update = [&](StubGroupsTy &StubGroups) {
|
||||
for (auto &KeyVal : StubGroups) {
|
||||
for (StubTy &Elem : KeyVal.second) {
|
||||
for (StubTy &Elem : KeyVal.second)
|
||||
Elem.first = BBAddresses[Elem.second];
|
||||
}
|
||||
std::sort(KeyVal.second.begin(), KeyVal.second.end(),
|
||||
[&](const std::pair<uint64_t, BinaryBasicBlock *> &LHS,
|
||||
const std::pair<uint64_t, BinaryBasicBlock *> &RHS) {
|
||||
|
@ -324,16 +323,14 @@ uint64_t LongJmpPass::tentativeLayoutRelocMode(
|
|||
uint32_t CurrentIndex = 0;
|
||||
if (opts::HotFunctionsAtEnd) {
|
||||
for (BinaryFunction *BF : SortedFunctions) {
|
||||
if (BF->hasValidIndex() && LastHotIndex == -1u) {
|
||||
if (BF->hasValidIndex() && LastHotIndex == -1u)
|
||||
LastHotIndex = CurrentIndex;
|
||||
}
|
||||
++CurrentIndex;
|
||||
}
|
||||
} else {
|
||||
for (BinaryFunction *BF : SortedFunctions) {
|
||||
if (!BF->hasValidIndex() && LastHotIndex == -1u) {
|
||||
if (!BF->hasValidIndex() && LastHotIndex == -1u)
|
||||
LastHotIndex = CurrentIndex;
|
||||
}
|
||||
++CurrentIndex;
|
||||
}
|
||||
}
|
||||
|
@ -396,9 +393,8 @@ void LongJmpPass::tentativeLayout(
|
|||
if (opts::UseOldText && EstimatedTextSize <= BC.OldTextSectionSize) {
|
||||
DotAddress = BC.OldTextSectionAddress;
|
||||
uint64_t Pad = offsetToAlignment(DotAddress, llvm::Align(BC.PageAlign));
|
||||
if (Pad + EstimatedTextSize <= BC.OldTextSectionSize) {
|
||||
if (Pad + EstimatedTextSize <= BC.OldTextSectionSize)
|
||||
DotAddress += Pad;
|
||||
}
|
||||
} else {
|
||||
DotAddress = alignTo(BC.LayoutStartAddress, BC.PageAlign);
|
||||
}
|
||||
|
@ -501,9 +497,8 @@ bool LongJmpPass::needsStub(const BinaryBasicBlock &BB, const MCInst &Inst,
|
|||
// Check for shared stubs from foreign functions
|
||||
if (!TgtBB) {
|
||||
auto SSIter = SharedStubs.find(TgtSym);
|
||||
if (SSIter != SharedStubs.end()) {
|
||||
if (SSIter != SharedStubs.end())
|
||||
TgtBB = SSIter->second;
|
||||
}
|
||||
}
|
||||
|
||||
int BitsAvail = BC.MIB->getPCRelEncodingSize(Inst) - 1;
|
||||
|
@ -527,9 +522,9 @@ bool LongJmpPass::relax(BinaryFunction &Func) {
|
|||
|
||||
BinaryBasicBlock *Frontier = getBBAtHotColdSplitPoint(Func);
|
||||
uint64_t FrontierAddress = Frontier ? BBAddresses[Frontier] : 0;
|
||||
if (FrontierAddress) {
|
||||
if (FrontierAddress)
|
||||
FrontierAddress += Frontier->getNumNonPseudos() * InsnSize;
|
||||
}
|
||||
|
||||
// Add necessary stubs for branch targets we know we can't fit in the
|
||||
// instruction
|
||||
for (BinaryBasicBlock &BB : Func) {
|
||||
|
|
|
@ -142,9 +142,8 @@ void computeEdgeWeights(BinaryBasicBlock *BB, EdgeWeightMap &EdgeWeights) {
|
|||
II != IE; ++II) {
|
||||
typename GraphT::NodeRef N = *II;
|
||||
Denominator += N->getExecutionCount();
|
||||
if (N != BB) {
|
||||
if (N != BB)
|
||||
continue;
|
||||
}
|
||||
CritWeight = N->getExecutionCount();
|
||||
}
|
||||
if (Denominator)
|
||||
|
@ -187,9 +186,8 @@ void computeEdgeWeights(BinaryBasicBlock *BB, EdgeWeightMap &EdgeWeights) {
|
|||
|
||||
template <class NodeT>
|
||||
void computeEdgeWeights(BinaryFunction &BF, EdgeWeightMap &EdgeWeights) {
|
||||
for (BinaryBasicBlock &BB : BF) {
|
||||
for (BinaryBasicBlock &BB : BF)
|
||||
computeEdgeWeights<NodeT>(&BB, EdgeWeights);
|
||||
}
|
||||
}
|
||||
|
||||
/// Make BB count match the sum of all incoming edges. If AllEdges is true,
|
||||
|
@ -197,25 +195,21 @@ void computeEdgeWeights(BinaryFunction &BF, EdgeWeightMap &EdgeWeights) {
|
|||
void recalculateBBCounts(BinaryFunction &BF, bool AllEdges) {
|
||||
for (BinaryBasicBlock &BB : BF) {
|
||||
uint64_t TotalPredsEWeight = 0;
|
||||
for (BinaryBasicBlock *Pred : BB.predecessors()) {
|
||||
for (BinaryBasicBlock *Pred : BB.predecessors())
|
||||
TotalPredsEWeight += Pred->getBranchInfo(BB).Count;
|
||||
}
|
||||
|
||||
if (TotalPredsEWeight > BB.getExecutionCount()) {
|
||||
if (TotalPredsEWeight > BB.getExecutionCount())
|
||||
BB.setExecutionCount(TotalPredsEWeight);
|
||||
}
|
||||
|
||||
if (!AllEdges)
|
||||
continue;
|
||||
|
||||
uint64_t TotalSuccsEWeight = 0;
|
||||
for (BinaryBasicBlock::BinaryBranchInfo &BI : BB.branch_info()) {
|
||||
for (BinaryBasicBlock::BinaryBranchInfo &BI : BB.branch_info())
|
||||
TotalSuccsEWeight += BI.Count;
|
||||
}
|
||||
|
||||
if (TotalSuccsEWeight > BB.getExecutionCount()) {
|
||||
if (TotalSuccsEWeight > BB.getExecutionCount())
|
||||
BB.setExecutionCount(TotalSuccsEWeight);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -378,9 +372,8 @@ createLoopNestLevelMap(BinaryFunction &BF) {
|
|||
DenseMap<const BinaryBasicBlock *, const BinaryLoop *> LoopNestLevel;
|
||||
const BinaryLoopInfo &BLI = BF.getLoopInfo();
|
||||
|
||||
for (BinaryBasicBlock &BB : BF) {
|
||||
for (BinaryBasicBlock &BB : BF)
|
||||
LoopNestLevel[&BB] = BLI[&BB];
|
||||
}
|
||||
|
||||
return LoopNestLevel;
|
||||
}
|
||||
|
@ -408,9 +401,8 @@ void equalizeBBCounts(BinaryFunction &BF) {
|
|||
DenseMap<const BinaryBasicBlock *, const BinaryLoop *> LoopNestLevel =
|
||||
createLoopNestLevelMap(BF);
|
||||
|
||||
for (BinaryBasicBlock &BB : BF) {
|
||||
for (BinaryBasicBlock &BB : BF)
|
||||
BBsToEC[&BB] = -1;
|
||||
}
|
||||
|
||||
for (BinaryBasicBlock &BB : BF) {
|
||||
auto I = BB.begin();
|
||||
|
@ -457,12 +449,10 @@ void equalizeBBCounts(BinaryFunction &BF) {
|
|||
|
||||
for (std::vector<BinaryBasicBlock *> &Class : Classes) {
|
||||
uint64_t Max = 0ULL;
|
||||
for (BinaryBasicBlock *BB : Class) {
|
||||
for (BinaryBasicBlock *BB : Class)
|
||||
Max = std::max(Max, BB->getExecutionCount());
|
||||
}
|
||||
for (BinaryBasicBlock *BB : Class) {
|
||||
for (BinaryBasicBlock *BB : Class)
|
||||
BB->setExecutionCount(Max);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -64,18 +64,17 @@ void orderFuncs(const CallGraph &Cg, Cluster *C1, Cluster *C2) {
|
|||
|
||||
for (const Arc &Arc : Cg.arcs()) {
|
||||
if ((Arc.src() == C1head && Arc.dst() == C2head) ||
|
||||
(Arc.dst() == C1head && Arc.src() == C2head)) {
|
||||
(Arc.dst() == C1head && Arc.src() == C2head))
|
||||
C1headC2head += Arc.weight();
|
||||
} else if ((Arc.src() == C1head && Arc.dst() == C2tail) ||
|
||||
(Arc.dst() == C1head && Arc.src() == C2tail)) {
|
||||
else if ((Arc.src() == C1head && Arc.dst() == C2tail) ||
|
||||
(Arc.dst() == C1head && Arc.src() == C2tail))
|
||||
C1headC2tail += Arc.weight();
|
||||
} else if ((Arc.src() == C1tail && Arc.dst() == C2head) ||
|
||||
(Arc.dst() == C1tail && Arc.src() == C2head)) {
|
||||
else if ((Arc.src() == C1tail && Arc.dst() == C2head) ||
|
||||
(Arc.dst() == C1tail && Arc.src() == C2head))
|
||||
C1tailC2head += Arc.weight();
|
||||
} else if ((Arc.src() == C1tail && Arc.dst() == C2tail) ||
|
||||
(Arc.dst() == C1tail && Arc.src() == C2tail)) {
|
||||
else if ((Arc.src() == C1tail && Arc.dst() == C2tail) ||
|
||||
(Arc.dst() == C1tail && Arc.src() == C2tail))
|
||||
C1tailC2tail += Arc.weight();
|
||||
}
|
||||
}
|
||||
|
||||
const double Max = std::max(std::max(C1headC2head, C1headC2tail),
|
||||
|
@ -115,9 +114,8 @@ std::vector<Cluster> pettisAndHansen(const CallGraph &Cg) {
|
|||
|
||||
auto insertOrInc = [&](Cluster *C1, Cluster *C2, double Weight) {
|
||||
auto Res = Carcs.emplace(C1, C2, Weight);
|
||||
if (!Res.second) {
|
||||
if (!Res.second)
|
||||
Res.first->Weight += Weight;
|
||||
}
|
||||
};
|
||||
|
||||
// Create a std::vector of cluster arcs
|
||||
|
@ -191,9 +189,9 @@ std::vector<Cluster> pettisAndHansen(const CallGraph &Cg) {
|
|||
|
||||
// update FuncCluster
|
||||
|
||||
for (NodeId F : C2->targets()) {
|
||||
for (NodeId F : C2->targets())
|
||||
FuncCluster[F] = C1;
|
||||
}
|
||||
|
||||
C1->merge(*C2, Max.Weight);
|
||||
C2->clear();
|
||||
}
|
||||
|
@ -204,12 +202,10 @@ std::vector<Cluster> pettisAndHansen(const CallGraph &Cg) {
|
|||
std::set<Cluster *> LiveClusters;
|
||||
std::vector<Cluster> OutClusters;
|
||||
|
||||
for (NodeId Fid : Funcs) {
|
||||
for (NodeId Fid : Funcs)
|
||||
LiveClusters.insert(FuncCluster[Fid]);
|
||||
}
|
||||
for (Cluster *C : LiveClusters) {
|
||||
for (Cluster *C : LiveClusters)
|
||||
OutClusters.push_back(std::move(*C));
|
||||
}
|
||||
|
||||
std::sort(OutClusters.begin(), OutClusters.end(), compareClustersDensity);
|
||||
|
||||
|
|
|
@ -120,17 +120,16 @@ void RegReAssign::swap(BinaryFunction &Function, MCPhysReg A, MCPhysReg B) {
|
|||
CFIReg = *Reg;
|
||||
}
|
||||
const MCPhysReg Reg = *BC.MRI->getLLVMRegNum(CFIReg, /*isEH=*/false);
|
||||
if (AliasA.test(Reg)) {
|
||||
if (AliasA.test(Reg))
|
||||
Function.mutateCFIRegisterFor(
|
||||
Inst,
|
||||
BC.MRI->getDwarfRegNum(
|
||||
BC.MIB->getAliasSized(B, BC.MIB->getRegSize(Reg)), false));
|
||||
} else if (AliasB.test(Reg)) {
|
||||
else if (AliasB.test(Reg))
|
||||
Function.mutateCFIRegisterFor(
|
||||
Inst,
|
||||
BC.MRI->getDwarfRegNum(
|
||||
BC.MIB->getAliasSized(A, BC.MIB->getRegSize(Reg)), false));
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
|
@ -257,11 +256,11 @@ void RegReAssign::aggressivePassOverFunction(BinaryFunction &Function) {
|
|||
DataflowInfoManager Info(Function, RA.get(), nullptr);
|
||||
BitVector AliveAtStart = *Info.getLivenessAnalysis().getStateAt(
|
||||
ProgramPoint::getFirstPointAt(*Function.begin()));
|
||||
for (BinaryBasicBlock &BB : Function) {
|
||||
for (BinaryBasicBlock &BB : Function)
|
||||
if (BB.pred_size() == 0)
|
||||
AliveAtStart |= *Info.getLivenessAnalysis().getStateAt(
|
||||
ProgramPoint::getFirstPointAt(BB));
|
||||
}
|
||||
|
||||
// Mark frame pointer alive because of CFI
|
||||
AliveAtStart |= BC.MIB->getAliases(BC.MIB->getFramePointer(), false);
|
||||
// Never touch return registers
|
||||
|
@ -329,10 +328,9 @@ bool RegReAssign::conservativePassOverFunction(BinaryFunction &Function) {
|
|||
// regs except RBP)
|
||||
MCPhysReg Candidate = 0;
|
||||
for (int J = ExtendedCSR.find_first(); J != -1;
|
||||
J = ExtendedCSR.find_next(J)) {
|
||||
J = ExtendedCSR.find_next(J))
|
||||
if (RegScore[J] > RegScore[Candidate])
|
||||
Candidate = J;
|
||||
}
|
||||
|
||||
if (!Candidate || RegScore[Candidate] < 0)
|
||||
return false;
|
||||
|
@ -345,9 +343,8 @@ bool RegReAssign::conservativePassOverFunction(BinaryFunction &Function) {
|
|||
if (ScoreRBX <= 0)
|
||||
continue;
|
||||
|
||||
if (RegScore[Candidate] > (ScoreRBX + 10)) {
|
||||
if (RegScore[Candidate] > (ScoreRBX + 10))
|
||||
RBX = I;
|
||||
}
|
||||
}
|
||||
|
||||
if (!RBX)
|
||||
|
@ -424,10 +421,9 @@ void RegReAssign::runOnFunctions(BinaryContext &BC) {
|
|||
if (!conservativePassOverFunction(Function) && opts::AggressiveReAssign) {
|
||||
aggressivePassOverFunction(Function);
|
||||
LLVM_DEBUG({
|
||||
if (FuncsChanged.count(&Function)) {
|
||||
if (FuncsChanged.count(&Function))
|
||||
dbgs() << "Aggressive pass successful on " << Function.getPrintName()
|
||||
<< "\n";
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
@ -436,12 +432,11 @@ void RegReAssign::runOnFunctions(BinaryContext &BC) {
|
|||
outs() << "BOLT-INFO: Reg Reassignment Pass: no changes were made.\n";
|
||||
return;
|
||||
}
|
||||
if (opts::UpdateDebugSections) {
|
||||
if (opts::UpdateDebugSections)
|
||||
outs() << "BOLT-WARNING: You used -reg-reassign and -update-debug-sections."
|
||||
<< " Some registers were changed but associated AT_LOCATION for "
|
||||
<< "impacted variables were NOT updated! This operation is "
|
||||
<< "currently unsupported by BOLT.\n";
|
||||
}
|
||||
outs() << "BOLT-INFO: Reg Reassignment Pass Stats:\n";
|
||||
outs() << "\t " << FuncsChanged.size() << " functions affected.\n";
|
||||
outs() << "\t " << StaticBytesSaved << " static bytes saved.\n";
|
||||
|
|
|
@ -76,9 +76,8 @@ template <typename A, typename B> struct HashPair {
|
|||
void ClusterAlgorithm::computeClusterAverageFrequency(const BinaryContext &BC) {
|
||||
// Create a separate MCCodeEmitter to allow lock-free execution
|
||||
BinaryContext::IndependentCodeEmitter Emitter;
|
||||
if (!opts::NoThreads) {
|
||||
if (!opts::NoThreads)
|
||||
Emitter = BC.createIndependentMCCodeEmitter();
|
||||
}
|
||||
|
||||
AvgFreq.resize(Clusters.size(), 0.0);
|
||||
for (uint32_t I = 0, E = Clusters.size(); I < E; ++I) {
|
||||
|
@ -434,9 +433,9 @@ void TSPReorderAlgorithm::reorderBasicBlocks(const BinaryFunction &BF,
|
|||
|
||||
std::vector<std::vector<int64_t>> DP;
|
||||
DP.resize(1 << N);
|
||||
for (std::vector<int64_t> &Elmt : DP) {
|
||||
for (std::vector<int64_t> &Elmt : DP)
|
||||
Elmt.resize(N, -1);
|
||||
}
|
||||
|
||||
// Start with the entry basic block being allocated with cost zero
|
||||
DP[1][0] = 0;
|
||||
// Walk through TSP solutions using a bitmask to represent state (current set
|
||||
|
@ -504,10 +503,9 @@ void TSPReorderAlgorithm::reorderBasicBlocks(const BinaryFunction &BF,
|
|||
|
||||
// Finalize layout with BBs that weren't assigned to the layout using the
|
||||
// input layout.
|
||||
for (BinaryBasicBlock *BB : BF.layout()) {
|
||||
for (BinaryBasicBlock *BB : BF.layout())
|
||||
if (Visited[BB->getLayoutIndex()] == false)
|
||||
Order.push_back(BB);
|
||||
}
|
||||
}
|
||||
|
||||
void OptimizeReorderAlgorithm::reorderBasicBlocks(
|
||||
|
@ -678,11 +676,9 @@ void OptimizeCacheReorderAlgorithm::reorderBasicBlocks(
|
|||
Order.insert(Order.end(), Cluster.begin(), Cluster.end());
|
||||
// Force zero execution count on clusters that do not meet the cut off
|
||||
// specified by --cold-threshold.
|
||||
if (AvgFreq[ClusterIndex] < static_cast<double>(ColdThreshold)) {
|
||||
for (BinaryBasicBlock *BBPtr : Cluster) {
|
||||
if (AvgFreq[ClusterIndex] < static_cast<double>(ColdThreshold))
|
||||
for (BinaryBasicBlock *BBPtr : Cluster)
|
||||
BBPtr->setExecutionCount(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -208,9 +208,8 @@ void ReorderData::assignMemData(BinaryContext &BC) {
|
|||
if (BinaryData *BD = AccessInfo.MemoryObject) {
|
||||
BinaryDataCounts[BD->getAtomicRoot()] += AccessInfo.Count;
|
||||
Counts[BD->getSectionName()] += AccessInfo.Count;
|
||||
if (BD->getAtomicRoot()->isJumpTable()) {
|
||||
if (BD->getAtomicRoot()->isJumpTable())
|
||||
JumpTableCounts[BD->getSectionName()] += AccessInfo.Count;
|
||||
}
|
||||
} else {
|
||||
Counts["Heap/stack"] += AccessInfo.Count;
|
||||
}
|
||||
|
@ -353,9 +352,8 @@ void ReorderData::setSectionOrder(BinaryContext &BC,
|
|||
|
||||
// Get the total count just for stats
|
||||
uint64_t TotalCount = 0;
|
||||
for (auto Itr = Begin; Itr != End; ++Itr) {
|
||||
for (auto Itr = Begin; Itr != End; ++Itr)
|
||||
TotalCount += Itr->second;
|
||||
}
|
||||
|
||||
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: setSectionOrder for "
|
||||
<< OutputSection.getName() << "\n");
|
||||
|
@ -369,10 +367,9 @@ void ReorderData::setSectionOrder(BinaryContext &BC,
|
|||
|
||||
++NumReordered;
|
||||
if (NumReordered > opts::ReorderDataMaxSymbols) {
|
||||
if (!NewOrder.empty()) {
|
||||
if (!NewOrder.empty())
|
||||
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: processing ending on symbol "
|
||||
<< *NewOrder.back() << "\n");
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
|
@ -380,10 +377,9 @@ void ReorderData::setSectionOrder(BinaryContext &BC,
|
|||
Offset = alignTo(Offset, Alignment);
|
||||
|
||||
if ((Offset + BD->getSize()) > opts::ReorderDataMaxBytes) {
|
||||
if (!NewOrder.empty()) {
|
||||
if (!NewOrder.empty())
|
||||
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: processing ending on symbol "
|
||||
<< *NewOrder.back() << "\n");
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
|
@ -470,11 +466,10 @@ void ReorderData::runOnFunctions(BinaryContext &BC) {
|
|||
|
||||
for (const std::string &SectionName : opts::ReorderData) {
|
||||
if (SectionName == "default") {
|
||||
for (unsigned I = 0; DefaultSections[I]; ++I) {
|
||||
for (unsigned I = 0; DefaultSections[I]; ++I)
|
||||
if (ErrorOr<BinarySection &> Section =
|
||||
BC.getUniqueSectionByName(DefaultSections[I]))
|
||||
Sections.push_back(&*Section);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -509,16 +504,14 @@ void ReorderData::runOnFunctions(BinaryContext &BC) {
|
|||
}
|
||||
auto SplitPoint = Order.begin() + SplitPointIdx;
|
||||
|
||||
if (opts::PrintReorderedData) {
|
||||
if (opts::PrintReorderedData)
|
||||
printOrder(*Section, Order.begin(), SplitPoint);
|
||||
}
|
||||
|
||||
if (!opts::ReorderInplace || FoundUnmoveable) {
|
||||
if (opts::ReorderInplace && FoundUnmoveable) {
|
||||
if (opts::ReorderInplace && FoundUnmoveable)
|
||||
outs() << "BOLT-INFO: Found unmoveable symbols in "
|
||||
<< Section->getName() << " falling back to splitting "
|
||||
<< "instead of in-place reordering.\n";
|
||||
}
|
||||
|
||||
// Copy original section to <section name>.cold.
|
||||
BinarySection &Cold = BC.registerSection(
|
||||
|
|
|
@ -159,25 +159,22 @@ void ReorderFunctions::reorder(std::vector<Cluster> &&Clusters,
|
|||
double TotalCalls64B = 0;
|
||||
double TotalCalls4KB = 0;
|
||||
double TotalCalls2MB = 0;
|
||||
if (PrintDetailed) {
|
||||
if (PrintDetailed)
|
||||
outs() << "BOLT-INFO: Function reordering page layout\n"
|
||||
<< "BOLT-INFO: ============== page 0 ==============\n";
|
||||
}
|
||||
for (Cluster &Cluster : Clusters) {
|
||||
if (PrintDetailed) {
|
||||
if (PrintDetailed)
|
||||
outs() << format(
|
||||
"BOLT-INFO: -------- density = %.3lf (%u / %u) --------\n",
|
||||
Cluster.density(), Cluster.samples(), Cluster.size());
|
||||
}
|
||||
|
||||
for (NodeId FuncId : Cluster.targets()) {
|
||||
if (Cg.samples(FuncId) > 0) {
|
||||
Hotfuncs++;
|
||||
|
||||
if (PrintDetailed) {
|
||||
if (PrintDetailed)
|
||||
outs() << "BOLT-INFO: hot func " << *Cg.nodeIdToFunc(FuncId) << " ("
|
||||
<< Cg.size(FuncId) << ")\n";
|
||||
}
|
||||
|
||||
uint64_t Dist = 0;
|
||||
uint64_t Calls = 0;
|
||||
|
@ -188,19 +185,18 @@ void ReorderFunctions::reorder(std::vector<Cluster> &&Clusters,
|
|||
const auto D = std::abs(FuncAddr[Arc.dst()] -
|
||||
(FuncAddr[FuncId] + Arc.avgCallOffset()));
|
||||
const double W = Arc.weight();
|
||||
if (D < 64 && PrintDetailed && opts::Verbosity > 2) {
|
||||
if (D < 64 && PrintDetailed && opts::Verbosity > 2)
|
||||
outs() << "BOLT-INFO: short (" << D << "B) call:\n"
|
||||
<< "BOLT-INFO: Src: " << *Cg.nodeIdToFunc(FuncId) << "\n"
|
||||
<< "BOLT-INFO: Dst: " << *Cg.nodeIdToFunc(Dst) << "\n"
|
||||
<< "BOLT-INFO: Weight = " << W << "\n"
|
||||
<< "BOLT-INFO: AvgOffset = " << Arc.avgCallOffset() << "\n";
|
||||
}
|
||||
Calls += W;
|
||||
if (D < 64) TotalCalls64B += W;
|
||||
if (D < 4096) TotalCalls4KB += W;
|
||||
if (D < (2 << 20)) TotalCalls2MB += W;
|
||||
Dist += Arc.weight() * D;
|
||||
if (PrintDetailed) {
|
||||
if (PrintDetailed)
|
||||
outs() << format("BOLT-INFO: arc: %u [@%lu+%.1lf] -> %u [@%lu]: "
|
||||
"weight = %.0lf, callDist = %f\n",
|
||||
Arc.src(),
|
||||
|
@ -209,7 +205,6 @@ void ReorderFunctions::reorder(std::vector<Cluster> &&Clusters,
|
|||
Arc.dst(),
|
||||
FuncAddr[Arc.dst()],
|
||||
Arc.weight(), D);
|
||||
}
|
||||
}
|
||||
TotalCalls += Calls;
|
||||
TotalDistance += Dist;
|
||||
|
@ -238,14 +233,13 @@ void ReorderFunctions::reorder(std::vector<Cluster> &&Clusters,
|
|||
TotalCalls ? TotalDistance / TotalCalls : 0, TotalDistance,
|
||||
TotalCalls)
|
||||
<< format("BOLT-INFO: Total Calls = %.0lf\n", TotalCalls);
|
||||
if (TotalCalls) {
|
||||
if (TotalCalls)
|
||||
outs() << format("BOLT-INFO: Total Calls within 64B = %.0lf (%.2lf%%)\n",
|
||||
TotalCalls64B, 100 * TotalCalls64B / TotalCalls)
|
||||
<< format("BOLT-INFO: Total Calls within 4KB = %.0lf (%.2lf%%)\n",
|
||||
TotalCalls4KB, 100 * TotalCalls4KB / TotalCalls)
|
||||
<< format("BOLT-INFO: Total Calls within 2MB = %.0lf (%.2lf%%)\n",
|
||||
TotalCalls2MB, 100 * TotalCalls2MB / TotalCalls);
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
@ -259,9 +253,8 @@ std::vector<std::string> readFunctionOrderFile() {
|
|||
exit(1);
|
||||
}
|
||||
std::string FuncName;
|
||||
while (std::getline(FuncsFile, FuncName)) {
|
||||
while (std::getline(FuncsFile, FuncName))
|
||||
FunctionNames.push_back(FuncName);
|
||||
}
|
||||
return FunctionNames;
|
||||
}
|
||||
|
||||
|
@ -320,10 +313,9 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) {
|
|||
(B->hasProfile() ||
|
||||
(A->getExecutionCount() > B->getExecutionCount()));
|
||||
});
|
||||
for (BinaryFunction *BF : SortedFunctions) {
|
||||
for (BinaryFunction *BF : SortedFunctions)
|
||||
if (BF->hasProfile())
|
||||
BF->setIndex(Index++);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case RT_HFSORT:
|
||||
|
@ -379,12 +371,11 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) {
|
|||
<< Function << ".\n";
|
||||
break;
|
||||
}
|
||||
if (!BF->hasValidIndex()) {
|
||||
if (!BF->hasValidIndex())
|
||||
BF->setIndex(Index++);
|
||||
} else if (opts::Verbosity > 0) {
|
||||
else if (opts::Verbosity > 0)
|
||||
errs() << "BOLT-WARNING: Duplicate reorder entry for " << Function
|
||||
<< ".\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -427,15 +418,13 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) {
|
|||
SortedFunctions.begin(),
|
||||
SortedFunctions.end(),
|
||||
[](const BinaryFunction *A, const BinaryFunction *B) {
|
||||
if (A->hasValidIndex() && B->hasValidIndex()) {
|
||||
if (A->hasValidIndex() && B->hasValidIndex())
|
||||
return A->getIndex() < B->getIndex();
|
||||
} else if (A->hasValidIndex() && !B->hasValidIndex()) {
|
||||
if (A->hasValidIndex() && !B->hasValidIndex())
|
||||
return true;
|
||||
} else if (!A->hasValidIndex() && B->hasValidIndex()) {
|
||||
if (!A->hasValidIndex() && B->hasValidIndex())
|
||||
return false;
|
||||
} else {
|
||||
return A->getAddress() < B->getAddress();
|
||||
}
|
||||
return A->getAddress() < B->getAddress();
|
||||
});
|
||||
|
||||
for (const BinaryFunction *Func : SortedFunctions) {
|
||||
|
|
|
@ -261,9 +261,8 @@ IndirectBranchInfo::IndirectBranchInfo(MCInst &Inst, MCPlusBuilder &MIB) {
|
|||
if (!MIB.evaluateX86MemoryOperand(Inst, &Memory.BaseRegNum,
|
||||
&Memory.ScaleValue,
|
||||
&Memory.IndexRegNum, &Memory.DispValue,
|
||||
&Memory.SegRegNum, &Memory.DispExpr)) {
|
||||
&Memory.SegRegNum, &Memory.DispExpr))
|
||||
llvm_unreachable("not expected");
|
||||
}
|
||||
} else if (MIB.isBranchOnReg(Inst)) {
|
||||
assert(MCPlus::getNumPrimeOperands(Inst) == 1 && "expect 1 operand");
|
||||
BranchReg = Inst.getOperand(0).getReg();
|
||||
|
@ -312,9 +311,8 @@ void RetpolineInsertion::runOnFunctions(BinaryContext &BC) {
|
|||
if (BrInfo.isMem() && !R11Available) {
|
||||
IndirectBranchInfo::MemOpInfo &MemRef = BrInfo.Memory;
|
||||
int Addend = (BrInfo.isJump() || BrInfo.isTailCall()) ? 8 : 16;
|
||||
if (MemRef.BaseRegNum == MIB.getStackPointer()) {
|
||||
if (MemRef.BaseRegNum == MIB.getStackPointer())
|
||||
MemRef.DispValue += Addend;
|
||||
}
|
||||
if (MemRef.IndexRegNum == MIB.getStackPointer())
|
||||
MemRef.DispValue += Addend * MemRef.ScaleValue;
|
||||
}
|
||||
|
|
|
@ -167,23 +167,19 @@ void CalleeSavedAnalysis::analyzeRestores() {
|
|||
|
||||
std::vector<MCInst *> CalleeSavedAnalysis::getSavesByReg(uint16_t Reg) {
|
||||
std::vector<MCInst *> Results;
|
||||
for (BinaryBasicBlock &BB : BF) {
|
||||
for (MCInst &Inst : BB) {
|
||||
for (BinaryBasicBlock &BB : BF)
|
||||
for (MCInst &Inst : BB)
|
||||
if (getSavedReg(Inst) == Reg)
|
||||
Results.push_back(&Inst);
|
||||
}
|
||||
}
|
||||
return Results;
|
||||
}
|
||||
|
||||
std::vector<MCInst *> CalleeSavedAnalysis::getRestoresByReg(uint16_t Reg) {
|
||||
std::vector<MCInst *> Results;
|
||||
for (BinaryBasicBlock &BB : BF) {
|
||||
for (MCInst &Inst : BB) {
|
||||
for (BinaryBasicBlock &BB : BF)
|
||||
for (MCInst &Inst : BB)
|
||||
if (getRestoredReg(Inst) == Reg)
|
||||
Results.push_back(&Inst);
|
||||
}
|
||||
}
|
||||
return Results;
|
||||
}
|
||||
|
||||
|
@ -197,16 +193,14 @@ CalleeSavedAnalysis::~CalleeSavedAnalysis() {
|
|||
}
|
||||
|
||||
void StackLayoutModifier::blacklistRegion(int64_t Offset, int64_t Size) {
|
||||
if (BlacklistedRegions[Offset] < Size) {
|
||||
if (BlacklistedRegions[Offset] < Size)
|
||||
BlacklistedRegions[Offset] = Size;
|
||||
}
|
||||
}
|
||||
|
||||
bool StackLayoutModifier::isRegionBlacklisted(int64_t Offset, int64_t Size) {
|
||||
for (std::pair<const int64_t, int64_t> Elem : BlacklistedRegions) {
|
||||
for (std::pair<const int64_t, int64_t> Elem : BlacklistedRegions)
|
||||
if (Offset + Size > Elem.first && Offset < Elem.first + Elem.second)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
@ -519,9 +513,8 @@ bool StackLayoutModifier::collapseRegion(MCInst *Alloc, int64_t RegionAddr,
|
|||
BC.MIB->addAnnotation(Inst, "AccessesDeletedPos", 0U, AllocatorId);
|
||||
continue;
|
||||
}
|
||||
if (BC.MIB->isPush(Inst) || BC.MIB->isPop(Inst)) {
|
||||
if (BC.MIB->isPush(Inst) || BC.MIB->isPop(Inst))
|
||||
continue;
|
||||
}
|
||||
|
||||
if (FIE->StackPtrReg == BC.MIB->getStackPointer() && Slot < RegionAddr)
|
||||
continue;
|
||||
|
@ -666,9 +659,8 @@ void StackLayoutModifier::performChanges() {
|
|||
const MCCFIInstruction *CFI = BF.getCFIFor(Inst);
|
||||
const MCCFIInstruction::OpType Operation = CFI->getOperation();
|
||||
if (Operation == MCCFIInstruction::OpDefCfa ||
|
||||
Operation == MCCFIInstruction::OpDefCfaOffset) {
|
||||
Operation == MCCFIInstruction::OpDefCfaOffset)
|
||||
Adjustment = 0 - Adjustment;
|
||||
}
|
||||
LLVM_DEBUG(dbgs() << "Changing CFI offset from " << CFI->getOffset()
|
||||
<< " to " << (CFI->getOffset() + Adjustment) << "\n");
|
||||
BF.mutateCFIOffsetFor(Inst, CFI->getOffset() + Adjustment);
|
||||
|
@ -747,9 +739,8 @@ void ShrinkWrapping::classifyCSRUses() {
|
|||
continue;
|
||||
BV = CSA.CalleeSaved;
|
||||
BV &= FPAliases;
|
||||
for (int I = BV.find_first(); I > 0; I = BV.find_next(I)) {
|
||||
for (int I = BV.find_first(); I > 0; I = BV.find_next(I))
|
||||
UsesByReg[I].set(DA.ExprToIdx[&Inst]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -812,10 +803,9 @@ void ShrinkWrapping::computeSaveLocations() {
|
|||
|
||||
BitVector BBDominatedUses = BitVector(DA.NumInstrs, false);
|
||||
for (int J = UsesByReg[I].find_first(); J > 0;
|
||||
J = UsesByReg[I].find_next(J)) {
|
||||
J = UsesByReg[I].find_next(J))
|
||||
if (DA.doesADominateB(*First, J))
|
||||
BBDominatedUses.set(J);
|
||||
}
|
||||
LLVM_DEBUG(dbgs() << "\t\tBB " << BB.getName() << " dominates "
|
||||
<< BBDominatedUses.count() << " uses for reg " << I
|
||||
<< ". Total uses for reg is " << UsesByReg[I].count()
|
||||
|
@ -884,9 +874,8 @@ void ShrinkWrapping::computeDomOrder() {
|
|||
return A < B;
|
||||
});
|
||||
|
||||
for (MCPhysReg I = 0, E = BC.MRI->getNumRegs(); I != E; ++I) {
|
||||
for (MCPhysReg I = 0, E = BC.MRI->getNumRegs(); I != E; ++I)
|
||||
DomOrder[Order[I]] = I;
|
||||
}
|
||||
}
|
||||
|
||||
bool ShrinkWrapping::isBestSavePosCold(unsigned CSR, MCInst *&BestPosSave,
|
||||
|
@ -995,20 +984,17 @@ ShrinkWrapping::doRestorePlacement(MCInst *BestPosSave, unsigned CSR,
|
|||
LLVM_DEBUG({
|
||||
dbgs() << "Dumping dominance frontier for ";
|
||||
BC.printInstruction(dbgs(), *BestPosSave);
|
||||
for (ProgramPoint &PP : Frontier) {
|
||||
if (PP.isInst()) {
|
||||
for (ProgramPoint &PP : Frontier)
|
||||
if (PP.isInst())
|
||||
BC.printInstruction(dbgs(), *PP.getInst());
|
||||
} else {
|
||||
else
|
||||
dbgs() << PP.getBB()->getName() << "\n";
|
||||
}
|
||||
}
|
||||
});
|
||||
for (ProgramPoint &PP : Frontier) {
|
||||
bool HasCritEdges = false;
|
||||
if (PP.isInst() && BC.MIB->isTerminator(*PP.getInst()) &&
|
||||
doesInstUsesCSR(*PP.getInst(), CSR)) {
|
||||
doesInstUsesCSR(*PP.getInst(), CSR))
|
||||
CannotPlace = true;
|
||||
}
|
||||
BinaryBasicBlock *FrontierBB = Info.getParentBB(PP);
|
||||
CritEdgesFrom.emplace_back(FrontierBB);
|
||||
CritEdgesTo.emplace_back(0);
|
||||
|
@ -1048,9 +1034,9 @@ ShrinkWrapping::doRestorePlacement(MCInst *BestPosSave, unsigned CSR,
|
|||
LLVM_DEBUG({
|
||||
dbgs() << "Now detected critical edges in the following frontier:\n";
|
||||
for (ProgramPoint &PP : Frontier) {
|
||||
if (PP.isBB())
|
||||
if (PP.isBB()) {
|
||||
dbgs() << " BB: " << PP.getBB()->getName() << "\n";
|
||||
else {
|
||||
} else {
|
||||
dbgs() << " Inst: ";
|
||||
PP.getInst()->dump();
|
||||
}
|
||||
|
@ -1254,9 +1240,8 @@ void ShrinkWrapping::scheduleSaveRestoreInsertions(
|
|||
dbgs() << "Scheduling restore insertion at: ";
|
||||
if (PP.isInst())
|
||||
PP.getInst()->dump();
|
||||
else {
|
||||
else
|
||||
dbgs() << PP.getBB()->getName() << "\n";
|
||||
}
|
||||
});
|
||||
MCInst *Term =
|
||||
FrontierBB->getTerminatorBefore(PP.isInst() ? PP.getInst() : nullptr);
|
||||
|
@ -1348,10 +1333,9 @@ void ShrinkWrapping::moveSaveRestores() {
|
|||
auto WRI = Todo.find(&BB);
|
||||
if (WRI != Todo.end()) {
|
||||
std::vector<WorklistItem> &TodoList = WRI->second;
|
||||
for (WorklistItem &Item : TodoList) {
|
||||
for (WorklistItem &Item : TodoList)
|
||||
if (Item.Action == WorklistItem::InsertPushOrPop)
|
||||
Item.Action = WorklistItem::InsertLoadOrStore;
|
||||
}
|
||||
}
|
||||
for (auto I = BB.rbegin(), E = BB.rend(); I != E; ++I) {
|
||||
MCInst &Inst = *I;
|
||||
|
@ -1384,9 +1368,8 @@ void ShrinkWrapping::moveSaveRestores() {
|
|||
MCInst *SavePos;
|
||||
size_t SaveSize;
|
||||
std::tie(RegNdx, SavePos, SaveSize) = I;
|
||||
for (MCInst *Save : CSA.getSavesByReg(RegNdx)) {
|
||||
for (MCInst *Save : CSA.getSavesByReg(RegNdx))
|
||||
SLM.collapseRegion(Save);
|
||||
}
|
||||
SLM.insertRegion(SavePos, SaveSize);
|
||||
}
|
||||
}
|
||||
|
@ -1596,19 +1579,13 @@ void ShrinkWrapping::insertUpdatedCFI(unsigned CSR, int SPValPush,
|
|||
}
|
||||
// Are we at the first basic block or hot-cold split point?
|
||||
if (!PrevBB || (BF.isSplit() && BB->isCold() != PrevBB->isCold())) {
|
||||
if (InAffectedZoneAtBegin) {
|
||||
if (InAffectedZoneAtBegin)
|
||||
insertCFIsForPushOrPop(*BB, BB->begin(), CSR, true, 0, SPValPush);
|
||||
}
|
||||
} else {
|
||||
if (InAffectedZoneAtBegin != PrevAffectedZone) {
|
||||
if (InAffectedZoneAtBegin) {
|
||||
insertCFIsForPushOrPop(*PrevBB, PrevBB->end(), CSR, true, 0,
|
||||
SPValPush);
|
||||
} else {
|
||||
insertCFIsForPushOrPop(*PrevBB, PrevBB->end(), CSR, false, 0,
|
||||
SPValPop);
|
||||
}
|
||||
}
|
||||
} else if (InAffectedZoneAtBegin != PrevAffectedZone) {
|
||||
if (InAffectedZoneAtBegin)
|
||||
insertCFIsForPushOrPop(*PrevBB, PrevBB->end(), CSR, true, 0, SPValPush);
|
||||
else
|
||||
insertCFIsForPushOrPop(*PrevBB, PrevBB->end(), CSR, false, 0, SPValPop);
|
||||
}
|
||||
PrevAffectedZone = InAffectedZoneAtEnd;
|
||||
PrevBB = BB;
|
||||
|
@ -1646,29 +1623,24 @@ void ShrinkWrapping::rebuildCFIForSP() {
|
|||
SPVal = CurVal;
|
||||
}
|
||||
}
|
||||
if (BF.isSplit() && PrevBB && BB->isCold() != PrevBB->isCold()) {
|
||||
if (BF.isSplit() && PrevBB && BB->isCold() != PrevBB->isCold())
|
||||
BF.addCFIInstruction(
|
||||
BB, BB->begin(),
|
||||
MCCFIInstruction::cfiDefCfaOffset(nullptr, -SPValAtBegin));
|
||||
} else {
|
||||
if (SPValAtBegin != PrevSPVal) {
|
||||
BF.addCFIInstruction(
|
||||
PrevBB, PrevBB->end(),
|
||||
MCCFIInstruction::cfiDefCfaOffset(nullptr, -SPValAtBegin));
|
||||
}
|
||||
}
|
||||
else if (SPValAtBegin != PrevSPVal)
|
||||
BF.addCFIInstruction(
|
||||
PrevBB, PrevBB->end(),
|
||||
MCCFIInstruction::cfiDefCfaOffset(nullptr, -SPValAtBegin));
|
||||
PrevSPVal = SPValAtEnd;
|
||||
PrevBB = BB;
|
||||
}
|
||||
|
||||
for (BinaryBasicBlock &BB : BF) {
|
||||
for (auto I = BB.begin(); I != BB.end();) {
|
||||
for (BinaryBasicBlock &BB : BF)
|
||||
for (auto I = BB.begin(); I != BB.end();)
|
||||
if (BC.MIB->hasAnnotation(*I, "DeleteMe"))
|
||||
I = BB.eraseInstruction(I);
|
||||
else
|
||||
++I;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
MCInst ShrinkWrapping::createStackAccess(int SPVal, int FPVal,
|
||||
|
@ -1876,13 +1848,11 @@ BBIterTy ShrinkWrapping::processInsertionsList(
|
|||
InsertionPoint =
|
||||
processInsertion(InsertionPoint, CurBB, Item, SPVal, FPVal);
|
||||
if (Item.Action == WorklistItem::InsertPushOrPop &&
|
||||
Item.FIEToInsert.IsStore) {
|
||||
Item.FIEToInsert.IsStore)
|
||||
SPVal -= Item.FIEToInsert.Size;
|
||||
}
|
||||
if (Item.Action == WorklistItem::InsertPushOrPop &&
|
||||
Item.FIEToInsert.IsLoad) {
|
||||
Item.FIEToInsert.IsLoad)
|
||||
SPVal += Item.FIEToInsert.Size;
|
||||
}
|
||||
}
|
||||
return InsertionPoint;
|
||||
}
|
||||
|
|
|
@ -108,12 +108,11 @@ void SplitFunctions::runOnFunctions(BinaryContext &BC) {
|
|||
BC, ParallelUtilities::SchedulingPolicy::SP_BB_LINEAR, WorkFun, SkipFunc,
|
||||
"SplitFunctions");
|
||||
|
||||
if (SplitBytesHot + SplitBytesCold > 0) {
|
||||
if (SplitBytesHot + SplitBytesCold > 0)
|
||||
outs() << "BOLT-INFO: splitting separates " << SplitBytesHot
|
||||
<< " hot bytes from " << SplitBytesCold << " cold bytes "
|
||||
<< format("(%.2lf%% of split functions is hot).\n",
|
||||
100.0 * SplitBytesHot / (SplitBytesHot + SplitBytesCold));
|
||||
}
|
||||
}
|
||||
|
||||
void SplitFunctions::splitFunction(BinaryFunction &BF) {
|
||||
|
@ -235,9 +234,8 @@ void SplitFunctions::splitFunction(BinaryFunction &BF) {
|
|||
<< Twine::utohexstr(OriginalHotSize) << '\n');
|
||||
|
||||
BF.updateBasicBlockLayout(PreSplitLayout);
|
||||
for (BinaryBasicBlock &BB : BF) {
|
||||
for (BinaryBasicBlock &BB : BF)
|
||||
BB.setIsCold(false);
|
||||
}
|
||||
} else {
|
||||
SplitBytesHot += HotSize;
|
||||
SplitBytesCold += ColdSize;
|
||||
|
|
|
@ -67,13 +67,11 @@ namespace {
|
|||
|
||||
bool isLoadRedundant(const FrameIndexEntry &LoadFIE,
|
||||
const FrameIndexEntry &StoreFIE) {
|
||||
if (LoadFIE.IsLoad == false || LoadFIE.IsSimple == false) {
|
||||
if (LoadFIE.IsLoad == false || LoadFIE.IsSimple == false)
|
||||
return false;
|
||||
}
|
||||
if (LoadFIE.StackOffset == StoreFIE.StackOffset &&
|
||||
LoadFIE.Size == StoreFIE.Size) {
|
||||
LoadFIE.Size == StoreFIE.Size)
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
@ -99,11 +97,11 @@ bool StackAvailableExpressions::doesXKillsY(const MCInst *X, const MCInst *Y) {
|
|||
// If Y is a store to stack, its clobber list is its source reg. This is
|
||||
// different than the rest because we want to check if the store source
|
||||
// reaches its corresponding load untouched.
|
||||
if (FIEY && FIEY->IsStore == true && FIEY->IsStoreFromReg) {
|
||||
if (FIEY && FIEY->IsStore == true && FIEY->IsStoreFromReg)
|
||||
YClobbers.set(FIEY->RegOrImm);
|
||||
} else {
|
||||
else
|
||||
RA.getInstClobberList(*Y, YClobbers);
|
||||
}
|
||||
|
||||
XClobbers &= YClobbers;
|
||||
return XClobbers.any();
|
||||
}
|
||||
|
|
|
@ -26,9 +26,8 @@ bool StackReachingUses::isLoadedInDifferentReg(const FrameIndexEntry &StoreFIE,
|
|||
assert(FIEY->IsLoad == 1);
|
||||
if (StoreFIE.StackOffset + StoreFIE.Size > FIEY->StackOffset &&
|
||||
StoreFIE.StackOffset < FIEY->StackOffset + FIEY->Size &&
|
||||
StoreFIE.RegOrImm != FIEY->RegOrImm) {
|
||||
StoreFIE.RegOrImm != FIEY->RegOrImm)
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
|
@ -43,23 +42,20 @@ bool StackReachingUses::isStoreUsed(const FrameIndexEntry &StoreFIE,
|
|||
if (ErrorOr<const FrameIndexEntry &> FIEY = FA.getFIEFor(*ReachingInst)) {
|
||||
assert(FIEY->IsLoad == 1);
|
||||
if (StoreFIE.StackOffset + StoreFIE.Size > FIEY->StackOffset &&
|
||||
StoreFIE.StackOffset < FIEY->StackOffset + FIEY->Size) {
|
||||
StoreFIE.StackOffset < FIEY->StackOffset + FIEY->Size)
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
ErrorOr<const ArgAccesses &> Args = FA.getArgAccessesFor(*ReachingInst);
|
||||
if (!Args)
|
||||
continue;
|
||||
if (Args->AssumeEverything) {
|
||||
if (Args->AssumeEverything)
|
||||
return true;
|
||||
}
|
||||
for (ArgInStackAccess FIEY : Args->Set) {
|
||||
|
||||
for (ArgInStackAccess FIEY : Args->Set)
|
||||
if (StoreFIE.StackOffset + StoreFIE.Size > FIEY.StackOffset &&
|
||||
StoreFIE.StackOffset < FIEY.StackOffset + FIEY.Size) {
|
||||
StoreFIE.StackOffset < FIEY.StackOffset + FIEY.Size)
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
|
|
@ -38,9 +38,8 @@ void getRegNameFromBitVec(const BinaryContext &BC, const BitVector &RegV,
|
|||
int RegIdx = RegV.find_first();
|
||||
while (RegIdx != -1) {
|
||||
LLVM_DEBUG(dbgs() << BC.MRI->getName(RegIdx) << " ");
|
||||
if (NameVec) {
|
||||
if (NameVec)
|
||||
NameVec->insert(std::string(BC.MRI->getName(RegIdx)));
|
||||
}
|
||||
RegIdx = RegV.find_next(RegIdx);
|
||||
}
|
||||
LLVM_DEBUG(dbgs() << "\n");
|
||||
|
@ -50,9 +49,9 @@ void StokeInfo::checkInstr(const BinaryFunction &BF, StokeFuncInfo &FuncInfo) {
|
|||
MCPlusBuilder *MIB = BF.getBinaryContext().MIB.get();
|
||||
BitVector RegV(NumRegs, false);
|
||||
for (BinaryBasicBlock *BB : BF.layout()) {
|
||||
if (BB->empty()) {
|
||||
if (BB->empty())
|
||||
continue;
|
||||
}
|
||||
|
||||
for (MCInst &It : *BB) {
|
||||
if (MIB->isPseudo(It))
|
||||
continue;
|
||||
|
@ -75,16 +74,14 @@ void StokeInfo::checkInstr(const BinaryFunction &BF, StokeFuncInfo &FuncInfo) {
|
|||
// TODO: more accurate analysis
|
||||
bool IsPush = MIB->isPush(It);
|
||||
bool IsRipAddr = MIB->hasPCRelOperand(It);
|
||||
if (IsPush) {
|
||||
if (IsPush)
|
||||
FuncInfo.StackOut = true;
|
||||
}
|
||||
if (MIB->isStore(It) && !IsPush && !IsRipAddr) {
|
||||
FuncInfo.HeapOut = true;
|
||||
}
|
||||
if (IsRipAddr) {
|
||||
FuncInfo.HasRipAddr = true;
|
||||
}
|
||||
|
||||
if (MIB->isStore(It) && !IsPush && !IsRipAddr)
|
||||
FuncInfo.HeapOut = true;
|
||||
|
||||
if (IsRipAddr)
|
||||
FuncInfo.HasRipAddr = true;
|
||||
} // end of for (auto &It : ...)
|
||||
} // end of for (auto *BB : ...)
|
||||
}
|
||||
|
@ -94,9 +91,8 @@ bool StokeInfo::checkFunction(BinaryFunction &BF, DataflowInfoManager &DInfo,
|
|||
|
||||
std::string Name = BF.getSymbol()->getName().str();
|
||||
|
||||
if (!BF.isSimple() || BF.isMultiEntry() || BF.empty()) {
|
||||
if (!BF.isSimple() || BF.isMultiEntry() || BF.empty())
|
||||
return false;
|
||||
}
|
||||
outs() << " STOKE-INFO: analyzing function " << Name << "\n";
|
||||
|
||||
FuncInfo.FuncName = Name;
|
||||
|
@ -105,9 +101,8 @@ bool StokeInfo::checkFunction(BinaryFunction &BF, DataflowInfoManager &DInfo,
|
|||
FuncInfo.NumInstrs = BF.getNumNonPseudos();
|
||||
FuncInfo.NumBlocks = BF.size();
|
||||
// early stop for large functions
|
||||
if (FuncInfo.NumInstrs > 500) {
|
||||
if (FuncInfo.NumInstrs > 500)
|
||||
return false;
|
||||
}
|
||||
|
||||
FuncInfo.IsLoopFree = BF.isLoopFree();
|
||||
if (!FuncInfo.IsLoopFree) {
|
||||
|
@ -127,9 +122,8 @@ bool StokeInfo::checkFunction(BinaryFunction &BF, DataflowInfoManager &DInfo,
|
|||
assert(EntryBB.isEntryPoint() && "Weird, this should be the entry block!");
|
||||
|
||||
MCInst *FirstNonPseudo = EntryBB.getFirstNonPseudoInstr();
|
||||
if (!FirstNonPseudo) {
|
||||
if (!FirstNonPseudo)
|
||||
return false;
|
||||
}
|
||||
|
||||
LLVM_DEBUG(dbgs() << "\t [DefIn]\n\t ");
|
||||
BitVector LiveInBV =
|
||||
|
@ -183,9 +177,8 @@ void StokeInfo::runOnFunctions(BinaryContext &BC) {
|
|||
for (auto &BF : BC.getBinaryFunctions()) {
|
||||
DataflowInfoManager DInfo(BF.second, &RA, nullptr);
|
||||
FuncInfo.reset();
|
||||
if (checkFunction(BF.second, DInfo, RA, FuncInfo)) {
|
||||
if (checkFunction(BF.second, DInfo, RA, FuncInfo))
|
||||
FuncInfo.printData(Outfile);
|
||||
}
|
||||
}
|
||||
|
||||
outs() << "STOKE-INFO: end of stoke pass\n";
|
||||
|
|
|
@ -174,13 +174,12 @@ void TailDuplication::constantAndCopyPropagate(
|
|||
if (RegsActive) {
|
||||
// Set Replaced and so ReplacedEverwhere to false if it cannot be
|
||||
// replaced (no replacing that opcode, Register is src and dest)
|
||||
if (ConstantProp) {
|
||||
if (ConstantProp)
|
||||
Replaced = BC.MIB->replaceRegWithImm(
|
||||
PropagateInst, Reg, OriginalInst.getOperand(1).getImm());
|
||||
} else {
|
||||
else
|
||||
Replaced = BC.MIB->replaceRegWithReg(
|
||||
PropagateInst, Reg, OriginalInst.getOperand(1).getReg());
|
||||
}
|
||||
}
|
||||
ReplacedEverywhere = ReplacedEverywhere && Replaced;
|
||||
}
|
||||
|
@ -188,14 +187,14 @@ void TailDuplication::constantAndCopyPropagate(
|
|||
// register to replace with is overwritten
|
||||
if (!ConstantProp &&
|
||||
regIsPossiblyOverwritten(PropagateInst,
|
||||
OriginalInst.getOperand(1).getReg(), BC)) {
|
||||
OriginalInst.getOperand(1).getReg(), BC))
|
||||
RegsActive = false;
|
||||
}
|
||||
|
||||
// Make sure no propagation happens after the register to replace is
|
||||
// overwritten
|
||||
if (regIsPossiblyOverwritten(PropagateInst, Reg, BC)) {
|
||||
if (regIsPossiblyOverwritten(PropagateInst, Reg, BC))
|
||||
RegsActive = false;
|
||||
}
|
||||
|
||||
// Record if the register to replace is overwritten
|
||||
if (regIsDefinitelyOverwritten(PropagateInst, Reg, BC)) {
|
||||
Overwritten = true;
|
||||
|
|
|
@ -22,12 +22,10 @@ namespace bolt {
|
|||
bool ThreeWayBranch::shouldRunOnFunction(BinaryFunction &Function) {
|
||||
BinaryContext &BC = Function.getBinaryContext();
|
||||
BinaryFunction::BasicBlockOrderType BlockLayout = Function.getLayout();
|
||||
for (BinaryBasicBlock *BB : BlockLayout) {
|
||||
for (MCInst &Inst : *BB) {
|
||||
for (BinaryBasicBlock *BB : BlockLayout)
|
||||
for (MCInst &Inst : *BB)
|
||||
if (BC.MIB->isPacked(Inst))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
@ -136,14 +134,13 @@ void ThreeWayBranch::runOnFunction(BinaryFunction &Function) {
|
|||
SecondBranch->setExecutionCount(NewSecondBranchCount);
|
||||
|
||||
// Replace the branch condition to fallthrough for the most common block
|
||||
if (SecondBranchBigger) {
|
||||
if (SecondBranchBigger)
|
||||
BC.MIB->replaceBranchCondition(*FirstJump, Blocks[2].first->getLabel(),
|
||||
Ctx, Blocks[2].second);
|
||||
} else {
|
||||
else
|
||||
BC.MIB->replaceBranchCondition(
|
||||
*FirstJump, SecondBranch->getLabel(), Ctx,
|
||||
BC.MIB->getInvertedCondCode(Blocks[2].second));
|
||||
}
|
||||
|
||||
// Replace the branch condition to fallthrough for the second most common
|
||||
// block
|
||||
|
|
|
@ -185,12 +185,10 @@ bool ValidateInternalCalls::fixCFGForIC(BinaryFunction &Function) const {
|
|||
bool ValidateInternalCalls::hasTailCallsInRange(
|
||||
BinaryFunction &Function) const {
|
||||
const BinaryContext &BC = Function.getBinaryContext();
|
||||
for (BinaryBasicBlock &BB : Function) {
|
||||
for (MCInst &Inst : BB) {
|
||||
for (BinaryBasicBlock &BB : Function)
|
||||
for (MCInst &Inst : BB)
|
||||
if (BC.MIB->isTailCall(Inst))
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
@ -326,9 +324,8 @@ void ValidateInternalCalls::runOnFunctions(BinaryContext &BC) {
|
|||
std::set<BinaryFunction *> Invalid;
|
||||
for (BinaryFunction *Function : NeedsValidation) {
|
||||
LLVM_DEBUG(dbgs() << "Validating " << *Function << "\n");
|
||||
if (!analyzeFunction(*Function)) {
|
||||
if (!analyzeFunction(*Function))
|
||||
Invalid.insert(Function);
|
||||
}
|
||||
clearAnnotations(*Function);
|
||||
}
|
||||
|
||||
|
|
|
@ -90,9 +90,8 @@ void VeneerElimination::runOnFunctions(BinaryContext &BC) {
|
|||
|
||||
VeneerCallers++;
|
||||
if (!BC.MIB->replaceBranchTarget(
|
||||
Instr, VeneerDestinations[TargetSymbol], BC.Ctx.get())) {
|
||||
Instr, VeneerDestinations[TargetSymbol], BC.Ctx.get()))
|
||||
assert(false && "updating veneer call destination failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue