[Attributor][NFC] Encode IRPositions in the bits of a single pointer

This reduces memory consumption for IRPositions by eliminating the
vtable pointer and the `KindOrArgNo` integer. Since each abstract
attribute has an associated IRPosition, the 12-16 bytes we save add up
quickly.

No functional change is intended.

---

Single run of the Attributor module and then CGSCC pass (oldPM)
for SPASS/clause.c (~10k LLVM-IR loc):

Before:
```
calls to allocation functions: 469545 (260135/s)
temporary memory allocations: 77137 (42735/s)
peak heap memory consumption: 30.50MB
peak RSS (including heaptrack overhead): 119.50MB
total memory leaked: 269.07KB
```

After:
```
calls to allocation functions: 468999 (274108/s)
temporary memory allocations: 77002 (45004/s)
peak heap memory consumption: 28.83MB
peak RSS (including heaptrack overhead): 118.05MB
total memory leaked: 269.07KB
```

Difference:
```
calls to allocation functions: -546 (5808/s)
temporary memory allocations: -135 (1436/s)
peak heap memory consumption: -1.67MB
peak RSS (including heaptrack overhead): 0B
total memory leaked: 0B
```

---

CTMark 15 runs

Metric: compile_time

Program                                        lhs    rhs    diff
 test-suite...:: CTMark/sqlite3/sqlite3.test    25.07  24.09 -3.9%
 test-suite...Mark/mafft/pairlocalalign.test    14.58  14.14 -3.0%
 test-suite...-typeset/consumer-typeset.test    21.78  21.58 -0.9%
 test-suite :: CTMark/SPASS/SPASS.test          21.95  22.03  0.4%
 test-suite :: CTMark/lencod/lencod.test        25.43  25.50  0.3%
 test-suite...ark/tramp3d-v4/tramp3d-v4.test    23.88  23.83 -0.2%
 test-suite...TMark/7zip/7zip-benchmark.test    60.24  60.11 -0.2%
 test-suite :: CTMark/kimwitu++/kc.test         15.69  15.69 -0.0%
 test-suite...:: CTMark/ClamAV/clamscan.test    25.43  25.42 -0.0%
 test-suite :: CTMark/Bullet/bullet.test        37.63  37.62 -0.0%
 Geomean difference                                          -0.8%

---

Reviewed By: lebedev.ri

Differential Revision: https://reviews.llvm.org/D78722
This commit is contained in:
Johannes Doerfert 2020-04-17 18:03:04 -05:00
parent 6782635e90
commit d254b50b2b
3 changed files with 197 additions and 106 deletions

View File

@ -153,29 +153,24 @@ enum class DepClassTy {
/// are floating values that do not have a corresponding attribute list
/// position.
struct IRPosition {
virtual ~IRPosition() {}
/// The positions we distinguish in the IR.
///
/// The values are chosen such that the KindOrArgNo member has a value >= 0
/// if it is an argument or call site argument while a value < 0 indicates the
/// respective kind of that value.
enum Kind : int {
IRP_INVALID = -6, ///< An invalid position.
IRP_FLOAT = -5, ///< A position that is not associated with a spot suitable
///< for attributes. This could be any value or instruction.
IRP_RETURNED = -4, ///< An attribute for the function return value.
IRP_CALL_SITE_RETURNED = -3, ///< An attribute for a call site return value.
IRP_FUNCTION = -2, ///< An attribute for a function (scope).
IRP_CALL_SITE = -1, ///< An attribute for a call site (function scope).
IRP_ARGUMENT = 0, ///< An attribute for a function argument.
IRP_CALL_SITE_ARGUMENT = 1, ///< An attribute for a call site argument.
enum Kind : char {
IRP_INVALID, ///< An invalid position.
IRP_FLOAT, ///< A position that is not associated with a spot suitable
///< for attributes. This could be any value or instruction.
IRP_RETURNED, ///< An attribute for the function return value.
IRP_CALL_SITE_RETURNED, ///< An attribute for a call site return value.
IRP_FUNCTION, ///< An attribute for a function (scope).
IRP_CALL_SITE, ///< An attribute for a call site (function scope).
IRP_ARGUMENT, ///< An attribute for a function argument.
IRP_CALL_SITE_ARGUMENT, ///< An attribute for a call site argument.
};
/// Default constructor available to create invalid positions implicitly. All
/// other positions need to be created explicitly through the appropriate
/// static member function.
IRPosition() : AnchorVal(nullptr), KindOrArgNo(IRP_INVALID) { verify(); }
IRPosition() : Enc(nullptr, ENC_VALUE) { verify(); }
/// Create a position describing the value of \p V.
static const IRPosition value(const Value &V) {
@ -198,7 +193,7 @@ struct IRPosition {
/// Create a position describing the argument \p Arg.
static const IRPosition argument(const Argument &Arg) {
return IRPosition(const_cast<Argument &>(Arg), Kind(Arg.getArgNo()));
return IRPosition(const_cast<Argument &>(Arg), IRP_ARGUMENT);
}
/// Create a position describing the function scope of \p CB.
@ -214,7 +209,8 @@ struct IRPosition {
/// Create a position describing the argument of \p CB at position \p ArgNo.
static const IRPosition callsite_argument(const CallBase &CB,
unsigned ArgNo) {
return IRPosition(const_cast<CallBase &>(CB), Kind(ArgNo));
return IRPosition(const_cast<Use &>(CB.getArgOperandUse(ArgNo)),
IRP_CALL_SITE_ARGUMENT);
}
/// Create a position describing the argument of \p ACS at position \p ArgNo.
@ -242,9 +238,7 @@ struct IRPosition {
return IRPosition::function(*IRP.getAssociatedFunction());
}
bool operator==(const IRPosition &RHS) const {
return (AnchorVal == RHS.AnchorVal) && (KindOrArgNo == RHS.KindOrArgNo);
}
bool operator==(const IRPosition &RHS) const { return Enc == RHS.Enc; }
bool operator!=(const IRPosition &RHS) const { return !(*this == RHS); }
/// Return the value this abstract attribute is anchored with.
@ -254,16 +248,21 @@ struct IRPosition {
/// far, only the case for call site arguments as the value is not sufficient
/// to pinpoint them. Instead, we can use the call site as an anchor.
Value &getAnchorValue() const {
assert(KindOrArgNo != IRP_INVALID &&
"Invalid position does not have an anchor value!");
return *AnchorVal;
switch (getEncodingBits()) {
case ENC_VALUE:
case ENC_RETURNED_VALUE:
case ENC_FLOATING_FUNCTION:
return *getAsValuePtr();
case ENC_CALL_SITE_ARGUMENT_USE:
return *(getAsUsePtr()->getUser());
default:
llvm_unreachable("Unkown encoding!");
};
}
/// Return the associated function, if any.
Function *getAssociatedFunction() const {
assert(KindOrArgNo != IRP_INVALID &&
"Invalid position does not have an anchor scope!");
if (auto *CB = dyn_cast<CallBase>(AnchorVal))
if (auto *CB = dyn_cast<CallBase>(&getAnchorValue()))
return CB->getCalledFunction();
return getAnchorScope();
}
@ -312,18 +311,14 @@ struct IRPosition {
/// Return the value this abstract attribute is associated with.
Value &getAssociatedValue() const {
assert(KindOrArgNo != IRP_INVALID &&
"Invalid position does not have an associated value!");
if (getArgNo() < 0 || isa<Argument>(AnchorVal))
return *AnchorVal;
assert(isa<CallBase>(AnchorVal) && "Expected a call base!");
return *cast<CallBase>(AnchorVal)->getArgOperand(getArgNo());
if (getArgNo() < 0 || isa<Argument>(&getAnchorValue()))
return getAnchorValue();
assert(isa<CallBase>(&getAnchorValue()) && "Expected a call base!");
return *cast<CallBase>(&getAnchorValue())->getArgOperand(getArgNo());
}
/// Return the type this abstract attribute is associated with.
Type *getAssociatedType() const {
assert(KindOrArgNo != IRP_INVALID &&
"Invalid position does not have an associated type!");
if (getPositionKind() == IRPosition::IRP_RETURNED)
return getAssociatedFunction()->getReturnType();
return getAssociatedValue().getType();
@ -331,7 +326,18 @@ struct IRPosition {
/// Return the argument number of the associated value if it is an argument or
/// call site argument, otherwise a negative value.
int getArgNo() const { return KindOrArgNo; }
int getArgNo() const {
switch (getPositionKind()) {
case IRPosition::IRP_ARGUMENT:
return cast<Argument>(getAsValuePtr())->getArgNo();
case IRPosition::IRP_CALL_SITE_ARGUMENT: {
Use &U = *getAsUsePtr();
return cast<CallBase>(U.getUser())->getArgOperandNo(&U);
}
default:
return -1;
}
}
/// Return the index in the attribute list for this position.
unsigned getAttrIdx() const {
@ -347,7 +353,7 @@ struct IRPosition {
return AttributeList::ReturnIndex;
case IRPosition::IRP_ARGUMENT:
case IRPosition::IRP_CALL_SITE_ARGUMENT:
return KindOrArgNo + AttributeList::FirstArgIndex;
return getArgNo() + AttributeList::FirstArgIndex;
}
llvm_unreachable(
"There is no attribute index for a floating or invalid position!");
@ -355,19 +361,23 @@ struct IRPosition {
/// Return the associated position kind.
Kind getPositionKind() const {
if (getArgNo() >= 0) {
assert(((isa<Argument>(getAnchorValue()) &&
isa<Argument>(getAssociatedValue())) ||
isa<CallBase>(getAnchorValue())) &&
"Expected argument or call base due to argument number!");
if (isa<CallBase>(getAnchorValue()))
return IRP_CALL_SITE_ARGUMENT;
return IRP_ARGUMENT;
}
char EncodingBits = getEncodingBits();
if (EncodingBits == ENC_CALL_SITE_ARGUMENT_USE)
return IRP_CALL_SITE_ARGUMENT;
if (EncodingBits == ENC_FLOATING_FUNCTION)
return IRP_FLOAT;
assert(KindOrArgNo < 0 &&
"Expected (call site) arguments to never reach this point!");
return Kind(KindOrArgNo);
Value *V = getAsValuePtr();
if (!V)
return IRP_INVALID;
if (isa<Argument>(V))
return IRP_ARGUMENT;
if (isa<Function>(V))
return isReturnPosition(EncodingBits) ? IRP_RETURNED : IRP_FUNCTION;
if (isa<CallBase>(V))
return isReturnPosition(EncodingBits) ? IRP_CALL_SITE_RETURNED
: IRP_CALL_SITE;
return IRP_FLOAT;
}
/// TODO: Figure out if the attribute related helper functions should live
@ -435,14 +445,52 @@ struct IRPosition {
static const IRPosition TombstoneKey;
///}
/// Conversion into a void * to allow reuse of pointer hashing.
operator void *() const { return Enc.getOpaqueValue(); }
private:
/// Private constructor for special values only!
explicit IRPosition(int KindOrArgNo)
: AnchorVal(0), KindOrArgNo(KindOrArgNo) {}
explicit IRPosition(void *Ptr) { Enc.setFromOpaqueValue(Ptr); }
/// IRPosition anchored at \p AnchorVal with kind/argument numbet \p PK.
explicit IRPosition(Value &AnchorVal, Kind PK)
: AnchorVal(&AnchorVal), KindOrArgNo(PK) {
explicit IRPosition(Value &AnchorVal, Kind PK) {
switch (PK) {
case IRPosition::IRP_INVALID:
llvm_unreachable("Cannot create invalid IRP with an anchor value!");
break;
case IRPosition::IRP_FLOAT:
// Special case for floating functions.
if (isa<Function>(AnchorVal))
Enc = {&AnchorVal, ENC_FLOATING_FUNCTION};
else
Enc = {&AnchorVal, ENC_VALUE};
break;
case IRPosition::IRP_FUNCTION:
case IRPosition::IRP_CALL_SITE:
Enc = {&AnchorVal, ENC_VALUE};
break;
case IRPosition::IRP_RETURNED:
case IRPosition::IRP_CALL_SITE_RETURNED:
Enc = {&AnchorVal, ENC_RETURNED_VALUE};
break;
case IRPosition::IRP_ARGUMENT:
Enc = {&AnchorVal, ENC_VALUE};
break;
case IRPosition::IRP_CALL_SITE_ARGUMENT:
llvm_unreachable(
"Cannot create call site argument IRP with an anchor value!");
break;
}
verify();
}
/// IRPosition for the use \p U. The position kind \p PK needs to be
/// IRP_CALL_SITE_ARGUMENT, the anchor value is the user, the associated value
/// the used value.
explicit IRPosition(Use &U, Kind PK) {
assert(PK == IRP_CALL_SITE_ARGUMENT &&
"Use constructor is for call site arguments only!");
Enc = {&U, ENC_CALL_SITE_ARGUMENT_USE};
verify();
}
@ -459,30 +507,65 @@ private:
SmallVectorImpl<Attribute> &Attrs,
Attributor &A) const;
protected:
/// The value this position is anchored at.
Value *AnchorVal;
/// Return the underlying pointer as Value *, valid for all positions but
/// IRP_CALL_SITE_ARGUMENT.
Value *getAsValuePtr() const {
assert(getEncodingBits() != ENC_CALL_SITE_ARGUMENT_USE &&
"Not a value pointer!");
return reinterpret_cast<Value *>(Enc.getPointer());
}
/// If AnchorVal is Argument or CallBase then this number should be
/// non-negative and it denotes the argument or call site argument index
/// respectively. Otherwise, it denotes the kind of this IRPosition according
/// to Kind above.
int KindOrArgNo;
/// Return the underlying pointer as Use *, valid only for
/// IRP_CALL_SITE_ARGUMENT positions.
Use *getAsUsePtr() const {
assert(getEncodingBits() == ENC_CALL_SITE_ARGUMENT_USE &&
"Not a value pointer!");
return reinterpret_cast<Use *>(Enc.getPointer());
}
/// Return true if \p EncodingBits describe a returned or call site returned
/// position.
static bool isReturnPosition(char EncodingBits) {
return EncodingBits == ENC_RETURNED_VALUE;
}
/// Return true if the encoding bits describe a returned or call site returned
/// position.
bool isReturnPosition() const { return isReturnPosition(getEncodingBits()); }
/// The encoding of the IRPosition is a combination of a pointer and two
/// encoding bits. The values of the encoding bits are defined in the enum
/// below. The pointer is either a Value* (for the first three encoding bit
/// combinations) or Use* (for ENC_CALL_SITE_ARGUMENT_USE).
///
///{
enum {
ENC_VALUE = 0b00,
ENC_RETURNED_VALUE = 0b01,
ENC_FLOATING_FUNCTION = 0b10,
ENC_CALL_SITE_ARGUMENT_USE = 0b11,
};
// Reserve the maximal amount of bits so there is no need to mask out the
// remaining ones. We will not encode anything else in the pointer anyway.
static constexpr int NumEncodingBits =
PointerLikeTypeTraits<void *>::NumLowBitsAvailable;
static_assert(NumEncodingBits >= 2, "At least two bits are required!");
/// The pointer with the encoding bits.
PointerIntPair<void *, NumEncodingBits, char> Enc;
///}
/// Return the encoding bits.
char getEncodingBits() const { return Enc.getInt(); }
};
/// Helper that allows IRPosition as a key in a DenseMap.
template <> struct DenseMapInfo<IRPosition> {
template <> struct DenseMapInfo<IRPosition> : DenseMapInfo<void *> {
static inline IRPosition getEmptyKey() { return IRPosition::EmptyKey; }
static inline IRPosition getTombstoneKey() {
return IRPosition::TombstoneKey;
}
static unsigned getHashValue(const IRPosition &IRP) {
return (DenseMapInfo<Value *>::getHashValue(&IRP.getAnchorValue()) << 4) ^
(unsigned(IRP.getArgNo()));
}
static bool isEqual(const IRPosition &LHS, const IRPosition &RHS) {
return LHS == RHS;
}
};
/// A visitor class for IR positions.

View File

@ -258,8 +258,9 @@ IRAttributeManifest::manifestAttrs(Attributor &A, const IRPosition &IRP,
return HasChanged;
}
const IRPosition IRPosition::EmptyKey(255);
const IRPosition IRPosition::TombstoneKey(256);
const IRPosition IRPosition::EmptyKey(DenseMapInfo<void *>::getEmptyKey());
const IRPosition
IRPosition::TombstoneKey(DenseMapInfo<void *>::getTombstoneKey());
SubsumingPositionIterator::SubsumingPositionIterator(const IRPosition &IRP) {
IRPositions.emplace_back(IRP);
@ -401,52 +402,60 @@ bool IRPosition::getAttrsFromAssumes(Attribute::AttrKind AK,
void IRPosition::verify() {
#ifdef EXPENSIVE_CHECKS
switch (KindOrArgNo) {
default:
assert(KindOrArgNo >= 0 && "Expected argument or call site argument!");
assert((isa<CallBase>(AnchorVal) || isa<Argument>(AnchorVal)) &&
"Expected call base or argument for positive attribute index!");
if (isa<Argument>(AnchorVal)) {
assert(cast<Argument>(AnchorVal)->getArgNo() == unsigned(getArgNo()) &&
"Argument number mismatch!");
assert(cast<Argument>(AnchorVal) == &getAssociatedValue() &&
"Associated value mismatch!");
} else {
assert(cast<CallBase>(*AnchorVal).arg_size() > unsigned(getArgNo()) &&
"Call site argument number mismatch!");
assert(cast<CallBase>(*AnchorVal).getArgOperand(getArgNo()) ==
&getAssociatedValue() &&
"Associated value mismatch!");
}
break;
switch (getPositionKind()) {
case IRP_INVALID:
assert(!AnchorVal && "Expected no value for an invalid position!");
break;
assert(!Enc.getOpaqueValue() &&
"Expected a nullptr for an invalid position!");
return;
case IRP_FLOAT:
assert((!isa<CallBase>(&getAssociatedValue()) &&
!isa<Argument>(&getAssociatedValue())) &&
"Expected specialized kind for call base and argument values!");
break;
return;
case IRP_RETURNED:
assert(isa<Function>(AnchorVal) &&
assert(isa<Function>(getAsValuePtr()) &&
"Expected function for a 'returned' position!");
assert(AnchorVal == &getAssociatedValue() && "Associated value mismatch!");
break;
assert(getAsValuePtr() == &getAssociatedValue() &&
"Associated value mismatch!");
return;
case IRP_CALL_SITE_RETURNED:
assert((isa<CallBase>(AnchorVal)) &&
assert((isa<CallBase>(getAsValuePtr())) &&
"Expected call base for 'call site returned' position!");
assert(AnchorVal == &getAssociatedValue() && "Associated value mismatch!");
break;
assert(getAsValuePtr() == &getAssociatedValue() &&
"Associated value mismatch!");
return;
case IRP_CALL_SITE:
assert((isa<CallBase>(AnchorVal)) &&
assert((isa<CallBase>(getAsValuePtr())) &&
"Expected call base for 'call site function' position!");
assert(AnchorVal == &getAssociatedValue() && "Associated value mismatch!");
break;
assert(getAsValuePtr() == &getAssociatedValue() &&
"Associated value mismatch!");
return;
case IRP_FUNCTION:
assert(isa<Function>(AnchorVal) &&
assert(isa<Function>(getAsValuePtr()) &&
"Expected function for a 'function' position!");
assert(AnchorVal == &getAssociatedValue() && "Associated value mismatch!");
break;
assert(getAsValuePtr() == &getAssociatedValue() &&
"Associated value mismatch!");
return;
case IRP_ARGUMENT:
assert(isa<Argument>(getAsValuePtr()) &&
"Expected argument for a 'argument' position!");
assert(getAsValuePtr() == &getAssociatedValue() &&
"Associated value mismatch!");
return;
case IRP_CALL_SITE_ARGUMENT: {
Use *U = getAsUsePtr();
assert(U && "Expected use for a 'call site argument' position!");
assert(isa<CallBase>(U->getUser()) &&
"Expected call base user for a 'call site argument' position!");
assert(cast<CallBase>(U->getUser())->isArgOperand(U) &&
"Expected call base argument operand for a 'call site argument' "
"position");
assert(cast<CallBase>(U->getUser())->getArgOperandNo(U) ==
unsigned(getArgNo()) &&
"Argument number mismatch!");
assert(U->get() == &getAssociatedValue() && "Associated value mismatch!");
return;
}
}
#endif
}

View File

@ -906,8 +906,7 @@ ChangeStatus AAReturnedValuesImpl::manifest(Attributor &A) {
// If the assumed unique return value is an argument, annotate it.
if (auto *UniqueRVArg = dyn_cast<Argument>(UniqueRV.getValue())) {
// TODO: This should be handled differently!
this->AnchorVal = UniqueRVArg;
this->KindOrArgNo = UniqueRVArg->getArgNo();
getIRPosition() = IRPosition::argument(*UniqueRVArg);
Changed = IRAttribute::manifest(A);
} else if (auto *RVC = dyn_cast<Constant>(UniqueRV.getValue())) {
// We can replace the returned value with the unique returned constant.