maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[BOLT] Improve annotations format and processing 

Summary:
Change the way annotations are stored and processed.

Embed annotation type/index into immediate value stored as an operand.
This limits the effective range of values that could be stored as
annotations to 56 bits, which is still plenty for most integer types
that we use and for pointers on real systems. High 8 bits are reserved
for storing annotation type/index.

Expand the interface for general annotations to include reference to
annotations by index. The main purpose of this interface is to improve
performance of annotations that are used by heavy (>O(N)) algorithms,
such as data flow analysis.

For -frame-opt pass, new memory usage and processing times are slightly
better compared to those before refactoring.

(cherry picked from FBD7492017)
This commit is contained in:
Maksim Panchenko 2018-03-29 18:42:06 -07:00
parent d8cf08b243
commit 489e514530
9 changed files with 328 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
bolt/BinaryFunction.cpp
View File

@ -329,11 +329,10 @@ std::pair<unsigned, uint64_t> BinaryFunction::eraseInvalidBBs() {
unsigned Count = 0;
uint64_t Bytes = 0;
for (auto *BB : layout()) {
assert((!BB->isEntryPoint() || BB->isValid()) &&
"all entry blocks must be valid");
if (BB->isValid()) {
NewLayout.push_back(BB);
} else {
assert(!BB->isEntryPoint() && "all entry blocks must be valid");
++Count;
Bytes += BC.computeCodeSize(BB->begin(), BB->end());
}

16
bolt/MCPlus.h
View File

@ -26,7 +26,7 @@ namespace MCPlus {
/// pad and the uint64_t represents the action.
using MCLandingPad = std::pair<const MCSymbol *, uint64_t>;
/// An extension to MCInst is provided via an extra operand of type Inst with
/// An extension to MCInst is provided via an extra operand of type MCInst with
/// ANNOTATION_LABEL opcode (i.e. we are tying an annotation instruction to an
/// existing one). The annotation instruction contains a list of Immediate
/// operands. Each operand either contains a value, or is a pointer to
@ -39,11 +39,17 @@ using MCLandingPad = std::pair<const MCSymbol *, uint64_t>;
/// correctness of the program. Debugging information, and profile information
/// belong to the second group.
///
/// For the first group, we use a reserved operand number/index. Operands in
/// the first groups store a value of an annotation.
/// Note: some optimization/transformation passes could use generic annotations
/// inside the pass and remove these annotations after the pass. In this
/// case, the internal state saved with annotations could affect the
/// correctness.
///
/// Annotations in the second group are addressed by name, and their respective
/// operands store a pointer to an instance of MCAnnotation class.
/// For the first group, we use a reserved annotation index. Operands in
/// the first groups store a value of an annotation in the immediate field
/// of their corresponding operand.
///
/// Annotations in the second group could be addressed either by name, or by
/// by and index which could be queried by providing a name.
class MCAnnotation {
public:
enum Kind {

95
bolt/MCPlusBuilder.cpp
View File

@ -27,7 +27,6 @@ using namespace llvm;
using namespace bolt;
using namespace MCPlus;
Optional<MCLandingPad> MCPlusBuilder::getEHInfo(const MCInst &Inst) const {
if (!isCall(Inst))
return NoneType();
@ -104,65 +103,39 @@ MCPlusBuilder::setConditionalTailCall(MCInst &Inst, uint64_t Dest) {
bool MCPlusBuilder::unsetConditionalTailCall(MCInst &Inst) {
if (!getConditionalTailCall(Inst))
return false;
setAnnotationOpValue(Inst, MCAnnotation::kConditionalTailCall, INVALID_VALUE);
removeAnnotation(Inst, MCAnnotation::kConditionalTailCall);
return true;
}
bool MCPlusBuilder::hasAnnotation(const MCInst &Inst, StringRef Name) const {
bool MCPlusBuilder::hasAnnotation(const MCInst &Inst, unsigned Index) const {
const auto *AnnotationInst = getAnnotationInst(Inst);
if (!AnnotationInst)
return false;
auto AI = AnnotationNameIndexMap.find(Name);
if (AI == AnnotationNameIndexMap.end())
return false;
if (AI->second + 1 > AnnotationInst->getNumOperands())
return false;
const auto Value = AnnotationInst->getOperand(AI->second).getImm();
if (Value == INVALID_VALUE)
return false;
return true;
return (bool)getAnnotationOpValue(Inst, Index);
}
const MCAnnotation *
MCPlusBuilder::getAnnotation(const MCInst &Inst, StringRef Name) const {
const auto Idx = getAnnotationIndex(Name);
if (!Idx)
return nullptr;
auto Value = getAnnotationOpValue(Inst, *Idx);
if (!Value)
return nullptr;
return reinterpret_cast<MCAnnotation *>(*Value);
}
bool MCPlusBuilder::removeAnnotation(MCInst &Inst, StringRef Name) {
bool MCPlusBuilder::removeAnnotation(MCInst &Inst, unsigned Index) {
auto *AnnotationInst = getAnnotationInst(Inst);
if (!AnnotationInst)
return false;
const auto Idx = getAnnotationIndex(Name);
if (!Idx || *Idx >= AnnotationInst->getNumOperands())
return false;
auto &Op = AnnotationInst->getOperand(*Idx);
assert(Op.isImm());
if (Op.getImm() == INVALID_VALUE)
return false;
auto *Annotation = reinterpret_cast<MCAnnotation *>(Op.getImm());
auto Itr = AnnotationPool.find(Annotation);
if (Itr != AnnotationPool.end()) {
AnnotationPool.erase(Itr);
Annotation->~MCAnnotation();
for (int I = AnnotationInst->getNumOperands() - 1; I >= 0; --I) {
auto ImmValue = AnnotationInst->getOperand(I).getImm();
if (extractAnnotationIndex(ImmValue) == Index) {
AnnotationInst->erase(AnnotationInst->begin() + I);
auto *Annotation =
reinterpret_cast<MCAnnotation *>(extractAnnotationValue(ImmValue));
auto Itr = AnnotationPool.find(Annotation);
if (Itr != AnnotationPool.end()) {
AnnotationPool.erase(Itr);
Annotation->~MCAnnotation();
}
return true;
}
}
AnnotationInst->getOperand(*Idx).setImm(INVALID_VALUE);
return true;
return false;
}
void MCPlusBuilder::removeAllAnnotations(MCInst &Inst) {
@ -170,13 +143,11 @@ void MCPlusBuilder::removeAllAnnotations(MCInst &Inst) {
if (!AnnotationInst)
return;
for (unsigned I = AnnotationInst->getNumOperands() - 1;
I >= MCAnnotation::kGeneric; --I) {
const auto &Op = AnnotationInst->getOperand(I);
assert(Op.isImm());
if (Op.getImm() == INVALID_VALUE)
continue;
auto *Annotation = reinterpret_cast<MCAnnotation *>(Op.getImm());
for (int I = AnnotationInst->getNumOperands() - 1; I >= 0; --I) {
auto ImmValue = AnnotationInst->getOperand(I).getImm();
AnnotationInst->erase(std::prev(AnnotationInst->end()));
auto *Annotation =
reinterpret_cast<MCAnnotation *>(extractAnnotationValue(ImmValue));
auto Itr = AnnotationPool.find(Annotation);
if (Itr != AnnotationPool.end()) {
AnnotationPool.erase(Itr);
@ -194,17 +165,17 @@ MCPlusBuilder::printAnnotations(const MCInst &Inst, raw_ostream &OS) const {
if (!AnnotationInst)
return;
for (unsigned I = MCAnnotation::kGeneric;
I < AnnotationInst->getNumOperands(); ++I) {
const auto &Op = AnnotationInst->getOperand(I);
assert(Op.isImm());
if (Op.getImm() == INVALID_VALUE)
continue;
for (unsigned I = 0; I < AnnotationInst->getNumOperands(); ++I) {
const auto Imm = AnnotationInst->getOperand(I).getImm();
const auto Index = extractAnnotationIndex(Imm);
const auto Value = extractAnnotationValue(Imm);
const auto *Annotation =
reinterpret_cast<const MCAnnotation *>(Op.getImm());
OS << " # " << AnnotationNames[I - MCAnnotation::kGeneric]
<< ": ";
Annotation->print(OS);
reinterpret_cast<const MCAnnotation *>(Value);
if (Index >= MCAnnotation::kGeneric) {
OS << " # " << AnnotationNames[Index - MCAnnotation::kGeneric]
<< ": ";
Annotation->print(OS);
}
}
}

253
bolt/MCPlusBuilder.h
View File

@ -18,6 +18,7 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCInst.h"
@ -36,7 +37,6 @@
#include <map>
#include <set>
#include <system_error>
#include <unordered_map>
#include <unordered_set>
namespace llvm {
@ -53,19 +53,37 @@ enum class IndirectBranchType : char {
class MCPlusBuilder {
private:
/// Annotation instruction allocator.
SpecificBumpPtrAllocator<MCInst> MCInstAllocator;
/// Annotation value allocator.
BumpPtrAllocator Allocator;
/// Annotation instruction allocator.
SpecificBumpPtrAllocator<MCInst> MCInstAllocator;
/// Record all the annotations with non-trivial type. To prevent leaks, these
/// will need destructors called when the annotation is removed or when all
/// annotations are destroyed.
std::unordered_set<MCPlus::MCAnnotation*> AnnotationPool;
static constexpr int64_t INVALID_VALUE = -1LL;
/// We encode Index and Value into a 64-bit immediate operand value.
static int64_t encodeAnnotationImm(unsigned Index, int64_t Value) {
assert(Index < 256 && "annotation index max value exceeded");
assert((Value == (Value << 8) >> 8) && "annotation value out of range");
Value &= 0xffffffffffffff;
Value |= (int64_t)Index << 56;
return Value;
}
/// Extract annotation index from immediate operand value.
static unsigned extractAnnotationIndex(int64_t ImmValue) {
return ImmValue >> 56;
}
/// Extract annotation value from immediate operand value.
static int64_t extractAnnotationValue(int64_t ImmValue) {
return (ImmValue << 8) >> 8;
}
MCInst *getAnnotationInst(const MCInst &Inst) const {
if (Inst.getNumOperands() == 0)
@ -89,11 +107,16 @@ private:
Inst.addOperand(MCOperand::createInst(AnnotationInst));
}
// Make sure the instruction has enough operands.
for (auto OpNum = AnnotationInst->getNumOperands(); OpNum <= Index; ++OpNum)
AnnotationInst->addOperand(MCOperand::createImm(INVALID_VALUE));
const auto AnnotationValue = encodeAnnotationImm(Index, Value);
for (int I = AnnotationInst->getNumOperands() - 1; I >= 0; --I) {
auto ImmValue = AnnotationInst->getOperand(I).getImm();
if (extractAnnotationIndex(ImmValue) == Index) {
AnnotationInst->getOperand(I).setImm(AnnotationValue);
return;
}
}
AnnotationInst->getOperand(Index).setImm(Value);
AnnotationInst->addOperand(MCOperand::createImm(AnnotationValue));
}
Optional<int64_t>
@ -102,14 +125,14 @@ private:
if (!AnnotationInst)
return NoneType();
if (Index + 1 > AnnotationInst->getNumOperands())
return NoneType();
for (int I = AnnotationInst->getNumOperands() - 1; I >= 0; --I) {
auto ImmValue = AnnotationInst->getOperand(I).getImm();
if (extractAnnotationIndex(ImmValue) == Index) {
return extractAnnotationValue(ImmValue);
}
}
const auto Value = AnnotationInst->getOperand(Index).getImm();
if (Value == INVALID_VALUE)
return NoneType();
return Value;
return NoneType();
}
protected:
@ -125,48 +148,12 @@ protected:
}
};
/// Map annotation name into an operand index.
std::unordered_map<std::string, uint64_t> AnnotationNameIndexMap;
/// Map annotation name into an annotation index.
StringMap<uint64_t> AnnotationNameIndexMap;
/// Names of non-standard annotations.
SmallVector<std::string, 8> AnnotationNames;
private:
Optional<unsigned> getAnnotationIndex(StringRef Name) const {
auto AI = AnnotationNameIndexMap.find(Name);
if (AI != AnnotationNameIndexMap.end())
return AI->second;
return NoneType();
}
unsigned getOrCreateAnnotationIndex(StringRef Name) {
auto AI = AnnotationNameIndexMap.find(Name);
if (AI != AnnotationNameIndexMap.end())
return AI->second;
auto Index = AnnotationNameIndexMap.size() + MCPlus::MCAnnotation::kGeneric;
AnnotationNameIndexMap.insert(std::make_pair(Name, Index));
AnnotationNames.push_back(Name);
return Index;
}
std::string getAnnotationName(unsigned Index) {
if (Index < MCPlus::MCAnnotation::kGeneric)
return "standard annotation";
if (Index > MCPlus::MCAnnotation::kGeneric + AnnotationNames.size() - 1)
return "invalid annotation";
return AnnotationNames[Index - MCPlus::MCAnnotation::kGeneric];
}
/// Get an annotation. Assumes that the annotation exists.
/// Use hasAnnotation() if the annotation may not exist.
const MCPlus::MCAnnotation *
getAnnotation(const MCInst &Inst, StringRef Name) const;
public:
class InstructionIterator
: public std::iterator<std::bidirectional_iterator_tag, MCInst> {
@ -302,8 +289,8 @@ public:
Annotation->~MCAnnotation();
}
AnnotationPool.clear();
Allocator.Reset();
MCInstAllocator.DestroyAll();
Allocator.Reset();
}
virtual bool isBranch(const MCInst &Inst) const {
@ -1292,20 +1279,64 @@ public:
return true;
}
/// Return annotation index matching the \p Name.
Optional<unsigned> getAnnotationIndex(StringRef Name) const {
auto AI = AnnotationNameIndexMap.find(Name);
if (AI != AnnotationNameIndexMap.end())
return AI->second;
return NoneType();
}
/// Return annotation index matching the \p Name. Create a new index if the
/// \p Name wasn't registered previously.
unsigned getOrCreateAnnotationIndex(StringRef Name) {
auto AI = AnnotationNameIndexMap.find(Name);
if (AI != AnnotationNameIndexMap.end())
return AI->second;
const auto Index =
AnnotationNameIndexMap.size() + MCPlus::MCAnnotation::kGeneric;
AnnotationNameIndexMap.insert(std::make_pair(Name, Index));
AnnotationNames.push_back(Name);
return Index;
}
/// Store an annotation value on an MCInst. This assumes the annotation
/// is not already present.
template <typename ValueType>
const ValueType &addAnnotation(MCInst &Inst,
unsigned Index,
const ValueType &Val) {
assert(!hasAnnotation(Inst, Index));
auto *A = new (Allocator) MCPlus::MCSimpleAnnotation<ValueType>(Val);
if (!std::is_trivial<ValueType>::value) {
AnnotationPool.insert(A);
}
setAnnotationOpValue(Inst, Index, reinterpret_cast<int64_t>(A));
return A->getValue();
}
/// Store an annotation value on an MCInst. This assumes the annotation
/// is not already present.
template <typename ValueType>
const ValueType &addAnnotation(MCInst &Inst,
StringRef Name,
const ValueType &Val) {
assert(!hasAnnotation(Inst, Name));
auto *A = new (Allocator) MCPlus::MCSimpleAnnotation<ValueType>(Val);
if (!std::is_trivial<ValueType>::value) {
AnnotationPool.insert(A);
}
setAnnotationOpValue(Inst, getOrCreateAnnotationIndex(Name),
reinterpret_cast<int64_t>(A));
return A->getValue();
return addAnnotation(Inst, getOrCreateAnnotationIndex(Name), Val);
}
/// Get an annotation as a specific value, but if the annotation does not
/// exist, create a new annotation with the default constructor for that type.
/// Return a non-const ref so caller can freely modify its contents
/// afterwards.
template <typename ValueType>
ValueType& getOrCreateAnnotationAs(MCInst &Inst, unsigned Index) {
auto Val =
tryGetAnnotationAs<ValueType>(const_cast<const MCInst &>(Inst), Index);
if (!Val)
Val = addAnnotation(Inst, Index, ValueType());
return const_cast<ValueType&>(*Val);
}
/// Get an annotation as a specific value, but if the annotation does not
@ -1314,21 +1345,37 @@ public:
/// afterwards.
template <typename ValueType>
ValueType& getOrCreateAnnotationAs(MCInst &Inst, StringRef Name) {
auto Val = tryGetAnnotationAs<ValueType>((const MCInst &)Inst, Name);
if (!Val) {
Val = addAnnotation(Inst, Name, ValueType());
}
return const_cast<ValueType&>(*Val);
const auto Index = getOrCreateAnnotationIndex(Name);
return getOrCreateAnnotationAs<ValueType>(Inst, Index);
}
/// Get an annotation as a specific value. Assumes that the annotation exist.
/// Get an annotation as a specific value. Assumes that the annotation exists.
/// Use hasAnnotation() if the annotation may not exist.
template <typename ValueType>
const ValueType &getAnnotationAs(const MCInst &Inst, unsigned Index) const {
auto Value = getAnnotationOpValue(Inst, Index);
assert(Value && "annotation should exist");
return reinterpret_cast<const MCPlus::MCSimpleAnnotation<ValueType> *>
(*Value)->getValue();
}
/// Get an annotation as a specific value. Assumes that the annotation exists.
/// Use hasAnnotation() if the annotation may not exist.
template <typename ValueType>
const ValueType &getAnnotationAs(const MCInst &Inst, StringRef Name) const {
const auto *Annotation = getAnnotation(Inst, Name);
assert(Annotation);
return static_cast<const MCPlus::MCSimpleAnnotation<ValueType> *>
(Annotation)->getValue();
const auto Index = getAnnotationIndex(Name);
assert(Index && "annotation should exist");
return getAnnotationAs<ValueType>(Inst, *Index);
}
/// Get an annotation as a specific value. If the annotation does not exist,
/// return the \p DefaultValue.
template <typename ValueType> const ValueType &
getAnnotationWithDefault(const MCInst &Inst, unsigned Index,
const ValueType &DefaultValue = ValueType()) {
if (!hasAnnotation(Inst, Index))
return DefaultValue;
return getAnnotationAs<ValueType>(Inst, Index);
}
/// Get an annotation as a specific value. If the annotation does not exist,
@ -1336,40 +1383,76 @@ public:
template <typename ValueType> const ValueType &
getAnnotationWithDefault(const MCInst &Inst, StringRef Name,
const ValueType &DefaultValue = ValueType()) {
if (!hasAnnotation(Inst, Name))
return DefaultValue;
return getAnnotationAs<ValueType>(Inst, Name);
const auto Index = getOrCreateAnnotationIndex(Name);
return getAnnotationWithDefault<ValueType>(Inst, Index, DefaultValue);
}
/// Check if the specified annotation exists on this instruction.
bool hasAnnotation(const MCInst &Inst, StringRef Name) const;
bool hasAnnotation(const MCInst &Inst, unsigned Index) const;
/// Check if an annotation with a specified \p Name exists on \p Inst.
bool hasAnnotation(const MCInst &Inst, StringRef Name) const {
const auto Index = getAnnotationIndex(Name);
if (!Index)
return false;
return hasAnnotation(Inst, *Index);
}
/// Get an annotation as a specific value, but if the annotation does not
/// exist, return errc::result_out_of_range.
template <typename ValueType>
ErrorOr<const ValueType &> tryGetAnnotationAs(const MCInst &Inst,
unsigned Index) const {
if (!hasAnnotation(Inst, Index))
return make_error_code(std::errc::result_out_of_range);
return getAnnotationAs<ValueType>(Inst, Index);
}
/// Get an annotation as a specific value, but if the annotation does not
/// exist, return errc::result_out_of_range.
template <typename ValueType>
ErrorOr<const ValueType &> tryGetAnnotationAs(const MCInst &Inst,
StringRef Name) const {
if (!hasAnnotation(Inst, Name))
const auto Index = getAnnotationIndex(Name);
if (!Index)
return make_error_code(std::errc::result_out_of_range);
return getAnnotationAs<ValueType>(Inst, Name);
return tryGetAnnotationAs<ValueType>(Inst, *Index);
}
template <typename ValueType>
ErrorOr<ValueType &> tryGetAnnotationAs(MCInst &Inst,
StringRef Name) const {
if (!hasAnnotation(Inst, Name))
ErrorOr<ValueType &> tryGetAnnotationAs(MCInst &Inst, unsigned Index) const {
if (!hasAnnotation(Inst, Index))
return make_error_code(std::errc::result_out_of_range);
return const_cast<ValueType &>(getAnnotationAs<ValueType>(Inst, Name));
return const_cast<ValueType &>(getAnnotationAs<ValueType>(Inst, Index));
}
template <typename ValueType>
ErrorOr<ValueType &> tryGetAnnotationAs(MCInst &Inst, StringRef Name) const {
const auto Index = getAnnotationIndex(Name);
if (!Index)
return make_error_code(std::errc::result_out_of_range);
return tryGetAnnotationAs<ValueType>(Inst, *Index);
}
/// Print each annotation attached to \p Inst.
void printAnnotations(const MCInst &Inst, raw_ostream &OS) const;
/// Remove annotation with a given \p Index.
///
/// Return true if the annotation was removed, false if the annotation
/// was not present.
bool removeAnnotation(MCInst &Inst, unsigned Index);
/// Remove annotation associated with \p Name.
///
/// Return true if the annotation was removed, false if the the annotation
/// Return true if the annotation was removed, false if the annotation
/// was not present.
bool removeAnnotation(MCInst &Inst, StringRef Name);
bool removeAnnotation(MCInst &Inst, StringRef Name) {
const auto Index = getAnnotationIndex(Name);
if (!Index)
return false;
return removeAnnotation(Inst, *Index);
}
/// Remove all meta-data annotations from Inst.
void removeAllAnnotations(MCInst &Inst);

2
bolt/Passes/BinaryPasses.cpp
View File

@ -1279,8 +1279,6 @@ void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC,
BF.hash(/*Recompute=*/true, /*UseDFS=*/UseDFS);
CongruentBuckets[&BF].emplace(&BF);
dbgs() << BF.getPrintName() << " : " << BF.getKnownExecutionCount() << "\n";
}
// We repeat the pass until no new modifications happen.

16
bolt/Passes/DataflowAnalysis.h
View File

@ -165,6 +165,8 @@ class DataflowAnalysis {
return *static_cast<const Derived*>(this);
}
mutable Optional<unsigned> AnnotationIndex;
protected:
const BinaryContext &BC;
/// Reference to the function being analysed
@ -227,6 +229,14 @@ protected:
return StringRef("");
}
unsigned getAnnotationIndex() const {
if (AnnotationIndex)
return *AnnotationIndex;
AnnotationIndex =
BC.MIB->getOrCreateAnnotationIndex(const_derived().getAnnotationName());
return *AnnotationIndex;
}
/// Private getter methods accessing state in a read-write fashion
StateTy &getOrCreateStateAt(const BinaryBasicBlock &BB) {
return StateAtBBEntry[&BB];
@ -234,7 +244,7 @@ protected:
StateTy &getOrCreateStateAt(MCInst &Point) {
return BC.MIB->getOrCreateAnnotationAs<StateTy>(
Point, derived().getAnnotationName());
Point, derived().getAnnotationIndex());
}
StateTy &getOrCreateStateAt(ProgramPoint Point) {
@ -276,7 +286,7 @@ public:
/// the direction of the dataflow is forward (backward).
ErrorOr<const StateTy &> getStateAt(const MCInst &Point) const {
return BC.MIB->tryGetAnnotationAs<StateTy>(
Point, const_derived().getAnnotationName());
Point, const_derived().getAnnotationIndex());
}
/// Return the out set (in set) of a given program point if the direction of
@ -304,7 +314,7 @@ public:
void cleanAnnotations() {
for (auto &BB : Func) {
for (auto &Inst : BB) {
BC.MIB->removeAnnotation(Inst, derived().getAnnotationName());
BC.MIB->removeAnnotation(Inst, derived().getAnnotationIndex());
}
}
}

40
bolt/Passes/ShrinkWrapping.cpp
View File

@ -311,7 +311,7 @@ void StackLayoutModifier::checkStackPointerRestore(MCInst &Point) {
// We are restoring SP to an old value based on FP. Mark it as a stack
// access to be fixed later.
BC.MIB->addAnnotation(Point, getSlotTagName(), Output);
BC.MIB->addAnnotation(Point, getSlotTag(), Output);
}
void StackLayoutModifier::classifyStackAccesses() {
@ -354,7 +354,7 @@ void StackLayoutModifier::classifyStackAccesses() {
// We are free to go. Add it as available stack slot which we know how
// to move it.
AvailableRegions[FIEX->StackOffset] = FIEX->Size;
BC.MIB->addAnnotation(Inst, getSlotTagName(), FIEX->StackOffset);
BC.MIB->addAnnotation(Inst, getSlotTag(), FIEX->StackOffset);
RegionToRegMap[FIEX->StackOffset].insert(FIEX->RegOrImm);
RegToRegionMap[FIEX->RegOrImm].insert(FIEX->StackOffset);
DEBUG(dbgs() << "Adding region " << FIEX->StackOffset << " size "
@ -371,7 +371,7 @@ void StackLayoutModifier::classifyCFIs() {
auto recordAccess = [&](MCInst *Inst, int64_t Offset) {
const uint16_t Reg = BC.MRI->getLLVMRegNum(CfaReg, /*isEH=*/false);
if (Reg == BC.MIB->getStackPointer() || Reg == BC.MIB->getFramePointer()) {
BC.MIB->addAnnotation(*Inst, getSlotTagName(), Offset);
BC.MIB->addAnnotation(*Inst, getSlotTag(), Offset);
DEBUG(dbgs() << "Recording CFI " << Offset << "\n");
} else {
IsSimple = false;
@ -398,12 +398,12 @@ void StackLayoutModifier::classifyCFIs() {
break;
case MCCFIInstruction::OpOffset:
recordAccess(&Inst, CFI->getOffset());
BC.MIB->addAnnotation(Inst, getOffsetCFIRegTagName(),
BC.MIB->addAnnotation(Inst, getOffsetCFIRegTag(),
BC.MRI->getLLVMRegNum(CFI->getRegister(),
/*isEH=*/false));
break;
case MCCFIInstruction::OpSameValue:
BC.MIB->addAnnotation(Inst, getOffsetCFIRegTagName(),
BC.MIB->addAnnotation(Inst, getOffsetCFIRegTag(),
BC.MRI->getLLVMRegNum(CFI->getRegister(),
/*isEH=*/false));
break;
@ -432,7 +432,7 @@ void StackLayoutModifier::classifyCFIs() {
void StackLayoutModifier::scheduleChange(
MCInst &Inst, StackLayoutModifier::WorklistItem Item) {
auto &WList = BC.MIB->getOrCreateAnnotationAs<std::vector<WorklistItem>>(
Inst, getTodoTagName());
Inst, getTodoTag());
WList.push_back(Item);
}
@ -482,11 +482,11 @@ bool StackLayoutModifier::collapseRegion(MCInst *Alloc, int64_t RegionAddr,
for (auto &BB : BF) {
for (auto &Inst : BB) {
if (!BC.MIB->hasAnnotation(Inst, getSlotTagName()))
if (!BC.MIB->hasAnnotation(Inst, getSlotTag()))
continue;
auto Slot =
BC.MIB->getAnnotationAs<decltype(FrameIndexEntry::StackOffset)>(
Inst, getSlotTagName());
Inst, getSlotTag());
if (!AvailableRegions.count(Slot))
continue;
// We need to ensure this access is affected by the deleted push
@ -583,11 +583,11 @@ bool StackLayoutModifier::insertRegion(ProgramPoint P, int64_t RegionSz) {
for (auto &BB : BF) {
for (auto &Inst : BB) {
if (!BC.MIB->hasAnnotation(Inst, getSlotTagName()))
if (!BC.MIB->hasAnnotation(Inst, getSlotTag()))
continue;
auto Slot =
BC.MIB->getAnnotationAs<decltype(FrameIndexEntry::StackOffset)>(
Inst, getSlotTagName());
Inst, getSlotTag());
if (!AvailableRegions.count(Slot))
continue;
@ -633,10 +633,10 @@ void StackLayoutModifier::performChanges() {
assert(BC.MIB->isPop(Inst) || BC.MIB->isPush(Inst));
BC.MIB->removeAnnotation(Inst, "AccessesDeletedPos");
}
if (!BC.MIB->hasAnnotation(Inst, getTodoTagName()))
if (!BC.MIB->hasAnnotation(Inst, getTodoTag()))
continue;
auto &WList = BC.MIB->getAnnotationAs<std::vector<WorklistItem>>(
Inst, getTodoTagName());
Inst, getTodoTag());
int64_t Adjustment = 0;
WorklistItem::ActionType AdjustmentType = WorklistItem::None;
for (auto &WI : WList) {
@ -1275,7 +1275,7 @@ void ShrinkWrapping::moveSaveRestores() {
for (auto I = BB.rbegin(), E = BB.rend(); I != E; ++I) {
auto &Inst = *I;
auto TodoList = BC.MIB->tryGetAnnotationAs<std::vector<WorklistItem>>(
Inst, getAnnotationName());
Inst, getAnnotationIndex());
if (!TodoList)
continue;
bool isCFI = BC.MIB->isCFI(Inst);
@ -1382,6 +1382,8 @@ class PredictiveStackPointerTracking
decltype(ShrinkWrapping::Todo) &TodoMap;
DataflowInfoManager &Info;
Optional<unsigned> AnnotationIndex;
protected:
void compNextAux(const MCInst &Point,
const std::vector<ShrinkWrapping::WorklistItem> &TodoItems,
@ -1803,7 +1805,7 @@ bool ShrinkWrapping::processInsertions() {
for (auto I = BB.begin(); I != BB.end(); ++I) {
auto &Inst = *I;
auto TodoList = BC.MIB->tryGetAnnotationAs<std::vector<WorklistItem>>(
Inst, getAnnotationName());
Inst, getAnnotationIndex());
if (!TodoList)
continue;
Changes = true;
@ -1833,10 +1835,10 @@ bool ShrinkWrapping::processInsertions() {
void ShrinkWrapping::processDeletions() {
auto &LA = Info.getLivenessAnalysis();
for (auto &BB : BF) {
for (auto I = BB.rbegin(), E = BB.rend(); I != E; ++I) {
auto &Inst = *I;
for (auto II = BB.begin(); II != BB.end(); ++II) {
auto &Inst = *II;
auto TodoList = BC.MIB->tryGetAnnotationAs<std::vector<WorklistItem>>(
Inst, getAnnotationName());
Inst, getAnnotationIndex());
if (!TodoList)
continue;
// Process all deletions
@ -1860,9 +1862,9 @@ void ShrinkWrapping::processDeletions() {
DEBUG({
dbgs() << "Erasing: ";
Inst.dump();
BC.printInstruction(dbgs(), Inst);
});
BB.eraseInstruction(&Inst);
II = std::prev(BB.eraseInstruction(II));
break;
}
}

72
bolt/Passes/ShrinkWrapping.h
View File

@ -27,6 +27,8 @@ class CalleeSavedAnalysis {
const BinaryContext &BC;
BinaryFunction &BF;
DataflowInfoManager &Info;
Optional<unsigned> SaveTagIndex;
Optional<unsigned> RestoreTagIndex;
/// Compute all stores of callee-saved regs. Those are the ones that stores a
/// register whose definition is not local.
@ -39,13 +41,28 @@ class CalleeSavedAnalysis {
/// Returns the identifying string used to annotate instructions with metadata
/// for this analysis. These are deleted in the destructor.
static StringRef getSaveTag() {
static StringRef getSaveTagName() {
return StringRef("CSA-SavedReg");
}
static StringRef getRestoreTag() {
unsigned getSaveTag() {
if (SaveTagIndex)
return *SaveTagIndex;
SaveTagIndex = BC.MIB->getOrCreateAnnotationIndex(getSaveTagName());
return *SaveTagIndex;
}
static StringRef getRestoreTagName() {
return StringRef("CSA-RestoredReg");
}
unsigned getRestoreTag() {
if (RestoreTagIndex)
return *RestoreTagIndex;
RestoreTagIndex = BC.MIB->getOrCreateAnnotationIndex(getRestoreTagName());
return *RestoreTagIndex;
}
public:
BitVector CalleeSaved;
std::vector<int64_t> OffsetsByReg;
@ -125,6 +142,10 @@ class StackLayoutModifier {
bool IsInitialized{false};
Optional<unsigned> TodoTagIndex;
Optional<unsigned> SlotTagIndex;
Optional<unsigned> OffsetCFIRegTagIndex;
public:
// Keep a worklist of operations to perform on the function to perform
// the requested layout modifications via collapseRegion()/insertRegion().
@ -173,6 +194,29 @@ private:
/// Used to keep track of modifications to the function that will later be
/// performed by performChanges();
void scheduleChange(MCInst &Inst, WorklistItem Item);
unsigned getTodoTag() {
if (TodoTagIndex)
return *TodoTagIndex;
TodoTagIndex = BC.MIB->getOrCreateAnnotationIndex(getTodoTagName());
return *TodoTagIndex;
}
unsigned getSlotTag() {
if (SlotTagIndex)
return *SlotTagIndex;
SlotTagIndex = BC.MIB->getOrCreateAnnotationIndex(getSlotTagName());
return *SlotTagIndex;
}
unsigned getOffsetCFIRegTag() {
if (OffsetCFIRegTagIndex)
return *OffsetCFIRegTagIndex;
OffsetCFIRegTagIndex =
BC.MIB->getOrCreateAnnotationIndex(getOffsetCFIRegTagName());
return *OffsetCFIRegTagIndex;
}
static StringRef getTodoTagName() {
return StringRef("SLM-TodoTag");
}
@ -191,9 +235,9 @@ public:
~StackLayoutModifier() {
for (auto &BB : BF) {
for (auto &Inst : BB) {
BC.MIB->removeAnnotation(Inst, getTodoTagName());
BC.MIB->removeAnnotation(Inst, getSlotTagName());
BC.MIB->removeAnnotation(Inst, getOffsetCFIRegTagName());
BC.MIB->removeAnnotation(Inst, getTodoTag());
BC.MIB->removeAnnotation(Inst, getSlotTag());
BC.MIB->removeAnnotation(Inst, getOffsetCFIRegTag());
}
}
}
@ -202,7 +246,7 @@ public:
/// instruction or 0 if this is not a CSR restore instruction.
uint16_t getOffsetCFIReg(const MCInst &Inst) {
auto Val =
BC.MIB->tryGetAnnotationAs<uint16_t>(Inst, getOffsetCFIRegTagName());
BC.MIB->tryGetAnnotationAs<uint16_t>(Inst, getOffsetCFIRegTag());
if (Val)
return *Val;
return 0;
@ -270,6 +314,8 @@ class ShrinkWrapping {
static uint64_t SpillsMovedRegularMode;
static uint64_t SpillsMovedPushPopMode;
Optional<unsigned> AnnotationIndex;
/// Allow our custom worklist-sensitive analysis
/// PredictiveStackPointerTracking to access WorklistItem
public:
@ -301,6 +347,14 @@ public:
static StringRef getAnnotationName() {
return StringRef("ShrinkWrap-Todo");
}
unsigned getAnnotationIndex() {
if (AnnotationIndex)
return *AnnotationIndex;
AnnotationIndex = BC.MIB->getOrCreateAnnotationIndex(getAnnotationName());
return *AnnotationIndex;
}
private:
using BBIterTy = BinaryBasicBlock::iterator;
@ -327,7 +381,7 @@ private:
void scheduleChange(ProgramPoint PP, T&& ...Item) {
if (PP.isInst()) {
auto &WList = BC.MIB->getOrCreateAnnotationAs<std::vector<WorklistItem>>(
*PP.getInst(), getAnnotationName());
*PP.getInst(), getAnnotationIndex());
WList.emplace_back(std::forward<T>(Item)...);
return;
}
@ -344,7 +398,7 @@ private:
BB = *BB->succ_begin();
}
auto &WList = BC.MIB->getOrCreateAnnotationAs<std::vector<WorklistItem>>(
*BB->begin(), getAnnotationName());
*BB->begin(), getAnnotationIndex());
WList.emplace_back(std::forward<T>(Item)...);
}
@ -470,7 +524,7 @@ public:
~ShrinkWrapping() {
for (auto &BB : BF) {
for (auto &Inst : BB) {
BC.MIB->removeAnnotation(Inst, getAnnotationName());
BC.MIB->removeAnnotation(Inst, getAnnotationIndex());
}
}
}

37
bolt/Target/X86/X86MCPlusBuilder.cpp
View File

@ -986,10 +986,10 @@ public:
case X86::MOV8mr_NOREX:
case X86::MOV8rm_NOREX:
case X86::MOV8rr_NOREX:
case X86::MOVSX32_NOREXrm8:
case X86::MOVSX32_NOREXrr8:
case X86::MOVZX32_NOREXrm8:
case X86::MOVZX32_NOREXrr8:
case X86::MOVSX32rm8_NOREX:
case X86::MOVSX32rr8_NOREX:
case X86::MOVZX32rm8_NOREX:
case X86::MOVZX32rr8_NOREX:
case X86::MOV8mr:
case X86::MOV8rm:
case X86::MOV8rr:
@ -2548,7 +2548,7 @@ public:
ICPdata indirectCallPromotion(
const MCInst &CallInst,
const std::vector<std::pair<MCSymbol *,uint64_t>>& Targets,
const std::vector<std::pair<MCSymbol *,uint64_t>> &Targets,
const std::vector<uint64_t> &VtableAddrs,
const std::vector<MCInst *> &MethodFetchInsns,
const bool MinimizeCodeSize,
@ -2590,10 +2590,10 @@ public:
return Results;
}
const auto jumpToMergeBlock = [&](std::vector<MCInst> *NewCall) {
const auto jumpToMergeBlock = [&](std::vector<MCInst> &NewCall) {
assert(MergeBlock);
NewCall->push_back(CallInst);
MCInst& Merge = NewCall->back();
NewCall.push_back(CallInst);
MCInst &Merge = NewCall.back();
Merge.clear();
createUncondBranch(Merge, MergeBlock, Ctx);
};
@ -2605,7 +2605,7 @@ public:
if (MinimizeCodeSize && !LoadElim) {
// Load the call target into FuncAddrReg.
NewCall->push_back(CallInst); // Copy CallInst in order to get SMLoc
MCInst& Target = NewCall->back();
MCInst &Target = NewCall->back();
Target.clear();
Target.setOpcode(X86::MOV64ri32);
Target.addOperand(MCOperand::createReg(FuncAddrReg));
@ -2627,7 +2627,7 @@ public:
// Compare current call target to a specific address.
NewCall->push_back(CallInst);
MCInst& Compare = NewCall->back();
MCInst &Compare = NewCall->back();
Compare.clear();
if (CallInst.getOpcode() == X86::CALL64r ||
CallInst.getOpcode() == X86::JMP64r ||
@ -2650,7 +2650,7 @@ public:
} else {
// Compare current call target to a specific address.
NewCall->push_back(CallInst);
MCInst& Compare = NewCall->back();
MCInst &Compare = NewCall->back();
Compare.clear();
if (CallInst.getOpcode() == X86::CALL64r ||
CallInst.getOpcode() == X86::JMP64r ||
@ -2690,7 +2690,7 @@ public:
NewCall->push_back(CallInst);
if (IsJumpTable) {
MCInst& Je = NewCall->back();
MCInst &Je = NewCall->back();
// Jump to next compare if target addresses don't match.
Je.clear();
@ -2705,7 +2705,7 @@ public:
}
assert(!isInvoke(CallInst));
} else {
MCInst& Jne = NewCall->back();
MCInst &Jne = NewCall->back();
// Jump to next compare if target addresses don't match.
Jne.clear();
@ -2755,7 +2755,7 @@ public:
MergeBlock = Ctx->createTempSymbol();
} else {
// Insert jump to the merge block if we are not doing a fallthrough.
jumpToMergeBlock(NewCall);
jumpToMergeBlock(*NewCall);
}
}
}
@ -2763,7 +2763,7 @@ public:
// Cold call block.
Results.push_back(std::make_pair(NextTarget, std::vector<MCInst>()));
std::vector<MCInst>& NewCall = Results.back().second;
std::vector<MCInst> &NewCall = Results.back().second;
for (auto *Inst : MethodFetchInsns) {
if (Inst != &CallInst)
NewCall.push_back(*Inst);
@ -2772,7 +2772,7 @@ public:
// Jump to merge block from cold call block
if (!IsTailCall && !IsJumpTable) {
jumpToMergeBlock(&NewCall);
jumpToMergeBlock(NewCall);
// Record merge block
Results.push_back(std::make_pair(MergeBlock, std::vector<MCInst>()));
@ -2783,7 +2783,7 @@ public:
ICPdata jumpTablePromotion(
const MCInst &IJmpInst,
const std::vector<std::pair<MCSymbol *,uint64_t>>& Targets,
const std::vector<std::pair<MCSymbol *,uint64_t>> &Targets,
const std::vector<MCInst *> &TargetFetchInsns,
MCContext *Ctx
) const override {
@ -2831,7 +2831,7 @@ public:
// Cold call block.
Results.push_back(std::make_pair(NextTarget, std::vector<MCInst>()));
std::vector<MCInst>& CurBB = Results.back().second;
std::vector<MCInst> &CurBB = Results.back().second;
for (auto *Inst : TargetFetchInsns) {
if (Inst != &IJmpInst)
CurBB.push_back(*Inst);
@ -2850,4 +2850,3 @@ MCPlusBuilder *createX86MCPlusBuilder(const MCInstrAnalysis *Analysis,
const MCRegisterInfo *RegInfo) {
return new X86MCPlusBuilder(Analysis, Info, RegInfo);
}