maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[DAG] Avoid needing to walk out legalization tables. NFCI. 

To avoid redundant work, during DAG legalization we keep tables
mapping pre-legalized SDValues to post-legalized SDValues and a
SDValue-to-SDValue map to enable fast node replacements. However, as
the keys are nodes which may be reused it is possible that an entry in
a table refers to a now deleted node N (that should have been renamed
by the value replacement map) while a new node N' exists. If N' is
then replaced that entry would be wrong. Previously we avoided this by
when potentially violating this property, walking every table and
updating all node pointers. This is very expensive but hopefully rare
occurance.

This patch assigns each instance of a SDValue used in legalization a
unique id and uses these ids in the legalization tables. This avoids
any such aliasing issue, avoiding the full table search and allowing
more aggressive incremental table pruning.

In some cases this is a 1000x speedup to compilation.

Reviewers: jyknight, echristo, bogner, tra

Reviewed By: bogner

Subscribers: dberris, grandinj, hiraditya, llvm-commits

Differential Revision: https://reviews.llvm.org/D47959

llvm-svn: 334729
This commit is contained in:
Nirav Dave 2018-06-14 15:46:23 +00:00
parent 3ffeb41f6b
commit a1ee983a95
2 changed files with 155 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

246
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
View File

@ -84,9 +84,10 @@ void DAGTypeLegalizer::PerformExpensiveChecks() {
SDValue Res(&Node, i); SDValue Res(&Node, i);
EVT VT = Res.getValueType(); EVT VT = Res.getValueType();
bool Failed = false; bool Failed = false;
auto ResId = getTableId(Res);
unsigned Mapped = 0; unsigned Mapped = 0;
if (ReplacedValues.find(Res) != ReplacedValues.end()) { if (ReplacedValues.find(ResId) != ReplacedValues.end()) {
Mapped |= 1; Mapped |= 1;
// Check that remapped values are only used by nodes marked NewNode. // Check that remapped values are only used by nodes marked NewNode.
for (SDNode::use_iterator UI = Node.use_begin(), UE = Node.use_end(); for (SDNode::use_iterator UI = Node.use_begin(), UE = Node.use_end();
@ -97,30 +98,31 @@ void DAGTypeLegalizer::PerformExpensiveChecks() {
// Check that the final result of applying ReplacedValues is not // Check that the final result of applying ReplacedValues is not
// marked NewNode. // marked NewNode.
SDValue NewVal = ReplacedValues[Res]; auto NewValId = ReplacedValues[ResId];
DenseMap<SDValue, SDValue>::iterator I = ReplacedValues.find(NewVal); auto I = ReplacedValues.find(NewValId);
while (I != ReplacedValues.end()) { while (I != ReplacedValues.end()) {
NewVal = I->second; NewValId = I->second;
I = ReplacedValues.find(NewVal); I = ReplacedValues.find(NewValId);
} }
SDValue NewVal = getSDValue(NewValId);
assert(NewVal.getNode()->getNodeId() != NewNode && assert(NewVal.getNode()->getNodeId() != NewNode &&
"ReplacedValues maps to a new node!"); "ReplacedValues maps to a new node!");
} }
if (PromotedIntegers.find(Res) != PromotedIntegers.end()) if (PromotedIntegers.find(ResId) != PromotedIntegers.end())
Mapped |= 2; Mapped |= 2;
if (SoftenedFloats.find(Res) != SoftenedFloats.end()) if (SoftenedFloats.find(ResId) != SoftenedFloats.end())
Mapped |= 4; Mapped |= 4;
if (ScalarizedVectors.find(Res) != ScalarizedVectors.end()) if (ScalarizedVectors.find(ResId) != ScalarizedVectors.end())
Mapped |= 8; Mapped |= 8;
if (ExpandedIntegers.find(Res) != ExpandedIntegers.end()) if (ExpandedIntegers.find(ResId) != ExpandedIntegers.end())
Mapped |= 16; Mapped |= 16;
if (ExpandedFloats.find(Res) != ExpandedFloats.end()) if (ExpandedFloats.find(ResId) != ExpandedFloats.end())
Mapped |= 32; Mapped |= 32;
if (SplitVectors.find(Res) != SplitVectors.end()) if (SplitVectors.find(ResId) != SplitVectors.end())
Mapped |= 64; Mapped |= 64;
if (WidenedVectors.find(Res) != WidenedVectors.end()) if (WidenedVectors.find(ResId) != WidenedVectors.end())
Mapped |= 128; Mapped |= 128;
if (PromotedFloats.find(Res) != PromotedFloats.end()) if (PromotedFloats.find(ResId) != PromotedFloats.end())
Mapped |= 256; Mapped |= 256;
if (Node.getNodeId() != Processed) { if (Node.getNodeId() != Processed) {
@ -491,9 +493,6 @@ SDNode *DAGTypeLegalizer::AnalyzeNewNode(SDNode *N) {
if (N->getNodeId() != NewNode && N->getNodeId() != Unanalyzed) if (N->getNodeId() != NewNode && N->getNodeId() != Unanalyzed)
return N; return N;
// Remove any stale map entries.
ExpungeNode(N);
// Okay, we know that this node is new. Recursively walk all of its operands // Okay, we know that this node is new. Recursively walk all of its operands
// to see if they are new also. The depth of this walk is bounded by the size // to see if they are new also. The depth of this walk is bounded by the size
// of the new tree that was constructed (usually 2-3 nodes), so we don't worry // of the new tree that was constructed (usually 2-3 nodes), so we don't worry
@ -544,7 +543,6 @@ SDNode *DAGTypeLegalizer::AnalyzeNewNode(SDNode *N) {
// to remap the operands, since they are the same as the operands we // to remap the operands, since they are the same as the operands we
// remapped above. // remapped above.
N = M; N = M;
ExpungeNode(N);
} }
} }
@ -565,106 +563,24 @@ void DAGTypeLegalizer::AnalyzeNewValue(SDValue &Val) {
RemapValue(Val); RemapValue(Val);
} }
/// If N has a bogus mapping in ReplacedValues, eliminate it.
/// This can occur when a node is deleted then reallocated as a new node -
/// the mapping in ReplacedValues applies to the deleted node, not the new
/// one.
/// The only map that can have a deleted node as a source is ReplacedValues.
/// Other maps can have deleted nodes as targets, but since their looked-up
/// values are always immediately remapped using RemapValue, resulting in a
/// not-deleted node, this is harmless as long as ReplacedValues/RemapValue
/// always performs correct mappings. In order to keep the mapping correct,
/// ExpungeNode should be called on any new nodes *before* adding them as
/// either source or target to ReplacedValues (which typically means calling
/// Expunge when a new node is first seen, since it may no longer be marked
/// NewNode by the time it is added to ReplacedValues).
void DAGTypeLegalizer::ExpungeNode(SDNode *N) {
if (N->getNodeId() != NewNode)
return;
// If N is not remapped by ReplacedValues then there is nothing to do.
unsigned i, e;
for (i = 0, e = N->getNumValues(); i != e; ++i)
if (ReplacedValues.find(SDValue(N, i)) != ReplacedValues.end())
break;
if (i == e)
return;
// Remove N from all maps - this is expensive but rare.
for (DenseMap<SDValue, SDValue>::iterator I = PromotedIntegers.begin(),
E = PromotedIntegers.end(); I != E; ++I) {
assert(I->first.getNode() != N);
RemapValue(I->second);
}
for (DenseMap<SDValue, SDValue>::iterator I = PromotedFloats.begin(),
E = PromotedFloats.end(); I != E; ++I) {
assert(I->first.getNode() != N);
RemapValue(I->second);
}
for (DenseMap<SDValue, SDValue>::iterator I = SoftenedFloats.begin(),
E = SoftenedFloats.end(); I != E; ++I) {
assert(I->first.getNode() != N);
RemapValue(I->second);
}
for (DenseMap<SDValue, SDValue>::iterator I = ScalarizedVectors.begin(),
E = ScalarizedVectors.end(); I != E; ++I) {
assert(I->first.getNode() != N);
RemapValue(I->second);
}
for (DenseMap<SDValue, SDValue>::iterator I = WidenedVectors.begin(),
E = WidenedVectors.end(); I != E; ++I) {
assert(I->first.getNode() != N);
RemapValue(I->second);
}
for (DenseMap<SDValue, std::pair<SDValue, SDValue> >::iterator
I = ExpandedIntegers.begin(), E = ExpandedIntegers.end(); I != E; ++I){
assert(I->first.getNode() != N);
RemapValue(I->second.first);
RemapValue(I->second.second);
}
for (DenseMap<SDValue, std::pair<SDValue, SDValue> >::iterator
I = ExpandedFloats.begin(), E = ExpandedFloats.end(); I != E; ++I) {
assert(I->first.getNode() != N);
RemapValue(I->second.first);
RemapValue(I->second.second);
}
for (DenseMap<SDValue, std::pair<SDValue, SDValue> >::iterator
I = SplitVectors.begin(), E = SplitVectors.end(); I != E; ++I) {
assert(I->first.getNode() != N);
RemapValue(I->second.first);
RemapValue(I->second.second);
}
for (DenseMap<SDValue, SDValue>::iterator I = ReplacedValues.begin(),
E = ReplacedValues.end(); I != E; ++I)
RemapValue(I->second);
for (unsigned i = 0, e = N->getNumValues(); i != e; ++i)
ReplacedValues.erase(SDValue(N, i));
}
/// If the specified value was already legalized to another value, /// If the specified value was already legalized to another value,
/// replace it by that value. /// replace it by that value.
void DAGTypeLegalizer::RemapValue(SDValue &N) { void DAGTypeLegalizer::RemapValue(SDValue &V) {
DenseMap<SDValue, SDValue>::iterator I = ReplacedValues.find(N); auto Id = getTableId(V);
V = getSDValue(Id);
}
void DAGTypeLegalizer::RemapId(TableId &Id) {
auto I = ReplacedValues.find(Id);
if (I != ReplacedValues.end()) { if (I != ReplacedValues.end()) {
// Use path compression to speed up future lookups if values get multiply // Use path compression to speed up future lookups if values get multiply
// replaced with other values. // replaced with other values.
RemapValue(I->second); RemapId(I->second);
N = I->second; Id = I->second;
// Note that it is possible to have N.getNode()->getNodeId() == NewNode at // Note that N = IdToValueMap[Id] it is possible to have
// this point because it is possible for a node to be put in the map before // N.getNode()->getNodeId() == NewNode at this point because it is possible
// being processed. // for a node to be put in the map before being processed.
} }
} }
@ -721,19 +637,21 @@ void DAGTypeLegalizer::ReplaceValueWith(SDValue From, SDValue To) {
assert(From.getNode() != To.getNode() && "Potential legalization loop!"); assert(From.getNode() != To.getNode() && "Potential legalization loop!");
// If expansion produced new nodes, make sure they are properly marked. // If expansion produced new nodes, make sure they are properly marked.
ExpungeNode(From.getNode()); AnalyzeNewValue(To);
AnalyzeNewValue(To); // Expunges To.
// Anything that used the old node should now use the new one. Note that this // Anything that used the old node should now use the new one. Note that this
// can potentially cause recursive merging. // can potentially cause recursive merging.
SmallSetVector<SDNode*, 16> NodesToAnalyze; SmallSetVector<SDNode*, 16> NodesToAnalyze;
NodeUpdateListener NUL(*this, NodesToAnalyze); NodeUpdateListener NUL(*this, NodesToAnalyze);
do { do {
DAG.ReplaceAllUsesOfValueWith(From, To);
// The old node may still be present in a map like ExpandedIntegers or // The old node may be present in a map like ExpandedIntegers or
// PromotedIntegers. Inform maps about the replacement. // PromotedIntegers. Inform maps about the replacement.
ReplacedValues[From] = To; auto FromId = getTableId(From);
auto ToId = getTableId(To);
ReplacedValues[FromId] = ToId;
DAG.ReplaceAllUsesOfValueWith(From, To);
// Process the list of nodes that need to be reanalyzed. // Process the list of nodes that need to be reanalyzed.
while (!NodesToAnalyze.empty()) { while (!NodesToAnalyze.empty()) {
@ -758,12 +676,14 @@ void DAGTypeLegalizer::ReplaceValueWith(SDValue From, SDValue To) {
SDValue NewVal(M, i); SDValue NewVal(M, i);
if (M->getNodeId() == Processed) if (M->getNodeId() == Processed)
RemapValue(NewVal); RemapValue(NewVal);
DAG.ReplaceAllUsesOfValueWith(OldVal, NewVal);
// OldVal may be a target of the ReplacedValues map which was marked // OldVal may be a target of the ReplacedValues map which was marked
// NewNode to force reanalysis because it was updated. Ensure that // NewNode to force reanalysis because it was updated. Ensure that
// anything that ReplacedValues mapped to OldVal will now be mapped // anything that ReplacedValues mapped to OldVal will now be mapped
// all the way to NewVal. // all the way to NewVal.
ReplacedValues[OldVal] = NewVal; auto OldValId = getTableId(OldVal);
auto NewValId = getTableId(NewVal);
DAG.ReplaceAllUsesOfValueWith(OldVal, NewVal);
ReplacedValues[OldValId] = NewValId;
} }
// The original node continues to exist in the DAG, marked NewNode. // The original node continues to exist in the DAG, marked NewNode.
} }
@ -780,9 +700,9 @@ void DAGTypeLegalizer::SetPromotedInteger(SDValue Op, SDValue Result) {
"Invalid type for promoted integer"); "Invalid type for promoted integer");
AnalyzeNewValue(Result); AnalyzeNewValue(Result);
SDValue &OpEntry = PromotedIntegers[Op]; auto &OpIdEntry = PromotedIntegers[getTableId(Op)];
assert(!OpEntry.getNode() && "Node is already promoted!"); assert((OpIdEntry == 0) && "Node is already promoted!");
OpEntry = Result; OpIdEntry = getTableId(Result);
DAG.transferDbgValues(Op, Result); DAG.transferDbgValues(Op, Result);
} }
@ -797,15 +717,15 @@ void DAGTypeLegalizer::SetSoftenedFloat(SDValue Op, SDValue Result) {
"Invalid type for softened float"); "Invalid type for softened float");
AnalyzeNewValue(Result); AnalyzeNewValue(Result);
SDValue &OpEntry = SoftenedFloats[Op]; auto &OpIdEntry = SoftenedFloats[getTableId(Op)];
// Allow repeated calls to save f128 type nodes // Allow repeated calls to save f128 type nodes
// or any node with type that transforms to itself. // or any node with type that transforms to itself.
// Many operations on these types are not softened. // Many operations on these types are not softened.
assert((!OpEntry.getNode()|| assert(((OpIdEntry == 0) ||
Op.getValueType() == Op.getValueType() ==
TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType())) && TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType())) &&
"Node is already converted to integer!"); "Node is already converted to integer!");
OpEntry = Result; OpIdEntry = getTableId(Result);
} }
void DAGTypeLegalizer::SetPromotedFloat(SDValue Op, SDValue Result) { void DAGTypeLegalizer::SetPromotedFloat(SDValue Op, SDValue Result) {
@ -814,9 +734,9 @@ void DAGTypeLegalizer::SetPromotedFloat(SDValue Op, SDValue Result) {
"Invalid type for promoted float"); "Invalid type for promoted float");
AnalyzeNewValue(Result); AnalyzeNewValue(Result);
SDValue &OpEntry = PromotedFloats[Op]; auto &OpIdEntry = PromotedFloats[getTableId(Op)];
assert(!OpEntry.getNode() && "Node is already promoted!"); assert((OpIdEntry == 0) && "Node is already promoted!");
OpEntry = Result; OpIdEntry = getTableId(Result);
} }
void DAGTypeLegalizer::SetScalarizedVector(SDValue Op, SDValue Result) { void DAGTypeLegalizer::SetScalarizedVector(SDValue Op, SDValue Result) {
@ -827,19 +747,17 @@ void DAGTypeLegalizer::SetScalarizedVector(SDValue Op, SDValue Result) {
"Invalid type for scalarized vector"); "Invalid type for scalarized vector");
AnalyzeNewValue(Result); AnalyzeNewValue(Result);
SDValue &OpEntry = ScalarizedVectors[Op]; auto &OpIdEntry = ScalarizedVectors[getTableId(Op)];
assert(!OpEntry.getNode() && "Node is already scalarized!"); assert((OpIdEntry == 0) && "Node is already scalarized!");
OpEntry = Result; OpIdEntry = getTableId(Result);
} }
void DAGTypeLegalizer::GetExpandedInteger(SDValue Op, SDValue &Lo, void DAGTypeLegalizer::GetExpandedInteger(SDValue Op, SDValue &Lo,
SDValue &Hi) { SDValue &Hi) {
std::pair<SDValue, SDValue> &Entry = ExpandedIntegers[Op]; std::pair<TableId, TableId> &Entry = ExpandedIntegers[getTableId(Op)];
RemapValue(Entry.first); assert((Entry.first != 0) && "Operand isn't expanded");
RemapValue(Entry.second); Lo = getSDValue(Entry.first);
assert(Entry.first.getNode() && "Operand isn't expanded"); Hi = getSDValue(Entry.second);
Lo = Entry.first;
Hi = Entry.second;
} }
void DAGTypeLegalizer::SetExpandedInteger(SDValue Op, SDValue Lo, void DAGTypeLegalizer::SetExpandedInteger(SDValue Op, SDValue Lo,
@ -865,20 +783,18 @@ void DAGTypeLegalizer::SetExpandedInteger(SDValue Op, SDValue Lo,
} }
// Remember that this is the result of the node. // Remember that this is the result of the node.
std::pair<SDValue, SDValue> &Entry = ExpandedIntegers[Op]; std::pair<TableId, TableId> &Entry = ExpandedIntegers[getTableId(Op)];
assert(!Entry.first.getNode() && "Node already expanded"); assert((Entry.first == 0) && "Node already expanded");
Entry.first = Lo; Entry.first = getTableId(Lo);
Entry.second = Hi; Entry.second = getTableId(Hi);
} }
void DAGTypeLegalizer::GetExpandedFloat(SDValue Op, SDValue &Lo, void DAGTypeLegalizer::GetExpandedFloat(SDValue Op, SDValue &Lo,
SDValue &Hi) { SDValue &Hi) {
std::pair<SDValue, SDValue> &Entry = ExpandedFloats[Op]; std::pair<TableId, TableId> &Entry = ExpandedFloats[getTableId(Op)];
RemapValue(Entry.first); assert((Entry.first != 0) && "Operand isn't expanded");
RemapValue(Entry.second); Lo = getSDValue(Entry.first);
assert(Entry.first.getNode() && "Operand isn't expanded"); Hi = getSDValue(Entry.second);
Lo = Entry.first;
Hi = Entry.second;
} }
void DAGTypeLegalizer::SetExpandedFloat(SDValue Op, SDValue Lo, void DAGTypeLegalizer::SetExpandedFloat(SDValue Op, SDValue Lo,
@ -891,21 +807,19 @@ void DAGTypeLegalizer::SetExpandedFloat(SDValue Op, SDValue Lo,
AnalyzeNewValue(Lo); AnalyzeNewValue(Lo);
AnalyzeNewValue(Hi); AnalyzeNewValue(Hi);
// Remember that this is the result of the node. std::pair<TableId, TableId> &Entry = ExpandedFloats[getTableId(Op)];
std::pair<SDValue, SDValue> &Entry = ExpandedFloats[Op]; assert((Entry.first == 0) && "Node already expanded");
assert(!Entry.first.getNode() && "Node already expanded"); Entry.first = getTableId(Lo);
Entry.first = Lo; Entry.second = getTableId(Hi);
Entry.second = Hi;
} }
void DAGTypeLegalizer::GetSplitVector(SDValue Op, SDValue &Lo, void DAGTypeLegalizer::GetSplitVector(SDValue Op, SDValue &Lo,
SDValue &Hi) { SDValue &Hi) {
std::pair<SDValue, SDValue> &Entry = SplitVectors[Op]; std::pair<TableId, TableId> &Entry = SplitVectors[getTableId(Op)];
RemapValue(Entry.first); Lo = getSDValue(Entry.first);
RemapValue(Entry.second); Hi = getSDValue(Entry.second);
assert(Entry.first.getNode() && "Operand isn't split"); assert(Lo.getNode() && "Operand isn't split");
Lo = Entry.first; ;
Hi = Entry.second;
} }
void DAGTypeLegalizer::SetSplitVector(SDValue Op, SDValue Lo, void DAGTypeLegalizer::SetSplitVector(SDValue Op, SDValue Lo,
@ -921,10 +835,10 @@ void DAGTypeLegalizer::SetSplitVector(SDValue Op, SDValue Lo,
AnalyzeNewValue(Hi); AnalyzeNewValue(Hi);
// Remember that this is the result of the node. // Remember that this is the result of the node.
std::pair<SDValue, SDValue> &Entry = SplitVectors[Op]; std::pair<TableId, TableId> &Entry = SplitVectors[getTableId(Op)];
assert(!Entry.first.getNode() && "Node already split"); assert((Entry.first == 0) && "Node already split");
Entry.first = Lo; Entry.first = getTableId(Lo);
Entry.second = Hi; Entry.second = getTableId(Hi);
} }
void DAGTypeLegalizer::SetWidenedVector(SDValue Op, SDValue Result) { void DAGTypeLegalizer::SetWidenedVector(SDValue Op, SDValue Result) {
@ -933,9 +847,9 @@ void DAGTypeLegalizer::SetWidenedVector(SDValue Op, SDValue Result) {
"Invalid type for widened vector"); "Invalid type for widened vector");
AnalyzeNewValue(Result); AnalyzeNewValue(Result);
SDValue &OpEntry = WidenedVectors[Op]; auto &OpIdEntry = WidenedVectors[getTableId(Op)];
assert(!OpEntry.getNode() && "Node already widened!"); assert((OpIdEntry == 0) && "Node already widened!");
OpEntry = Result; OpIdEntry = getTableId(Result);
} }

102
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
View File

@ -93,46 +93,81 @@ private:
N->getOpcode() == ISD::Register; N->getOpcode() == ISD::Register;
} }
// Bijection from SDValue to unique id. As each created node gets a
// new id we do not need to worry about reuse expunging. Should we
// run out of ids, we can do a one time expensive compactifcation.
typedef unsigned TableId;
TableId NextValueId = 1;
SmallDenseMap<SDValue, TableId, 8> ValueToIdMap;
SmallDenseMap<TableId, SDValue, 8> IdToValueMap;
/// For integer nodes that are below legal width, this map indicates what /// For integer nodes that are below legal width, this map indicates what
/// promoted value to use. /// promoted value to use.
SmallDenseMap<SDValue, SDValue, 8> PromotedIntegers; SmallDenseMap<TableId, TableId, 8> PromotedIntegers;
/// For integer nodes that need to be expanded this map indicates which /// For integer nodes that need to be expanded this map indicates which
/// operands are the expanded version of the input. /// operands are the expanded version of the input.
SmallDenseMap<SDValue, std::pair<SDValue, SDValue>, 8> ExpandedIntegers; SmallDenseMap<TableId, std::pair<TableId, TableId>, 8> ExpandedIntegers;
/// For floating-point nodes converted to integers of the same size, this map /// For floating-point nodes converted to integers of the same size, this map
/// indicates the converted value to use. /// indicates the converted value to use.
SmallDenseMap<SDValue, SDValue, 8> SoftenedFloats; SmallDenseMap<TableId, TableId, 8> SoftenedFloats;
/// For floating-point nodes that have a smaller precision than the smallest /// For floating-point nodes that have a smaller precision than the smallest
/// supported precision, this map indicates what promoted value to use. /// supported precision, this map indicates what promoted value to use.
SmallDenseMap<SDValue, SDValue, 8> PromotedFloats; SmallDenseMap<TableId, TableId, 8> PromotedFloats;
/// For float nodes that need to be expanded this map indicates which operands /// For float nodes that need to be expanded this map indicates which operands
/// are the expanded version of the input. /// are the expanded version of the input.
SmallDenseMap<SDValue, std::pair<SDValue, SDValue>, 8> ExpandedFloats; SmallDenseMap<TableId, std::pair<TableId, TableId>, 8> ExpandedFloats;
/// For nodes that are <1 x ty>, this map indicates the scalar value of type /// For nodes that are <1 x ty>, this map indicates the scalar value of type
/// 'ty' to use. /// 'ty' to use.
SmallDenseMap<SDValue, SDValue, 8> ScalarizedVectors; SmallDenseMap<TableId, TableId, 8> ScalarizedVectors;
/// For nodes that need to be split this map indicates which operands are the /// For nodes that need to be split this map indicates which operands are the
/// expanded version of the input. /// expanded version of the input.
SmallDenseMap<SDValue, std::pair<SDValue, SDValue>, 8> SplitVectors; SmallDenseMap<TableId, std::pair<TableId, TableId>, 8> SplitVectors;
/// For vector nodes that need to be widened, indicates the widened value to /// For vector nodes that need to be widened, indicates the widened value to
/// use. /// use.
SmallDenseMap<SDValue, SDValue, 8> WidenedVectors; SmallDenseMap<TableId, TableId, 8> WidenedVectors;
/// For values that have been replaced with another, indicates the replacement /// For values that have been replaced with another, indicates the replacement
/// value to use. /// value to use.
SmallDenseMap<SDValue, SDValue, 8> ReplacedValues; SmallDenseMap<TableId, TableId, 8> ReplacedValues;
/// This defines a worklist of nodes to process. In order to be pushed onto /// This defines a worklist of nodes to process. In order to be pushed onto
/// this worklist, all operands of a node must have already been processed. /// this worklist, all operands of a node must have already been processed.
SmallVector<SDNode*, 128> Worklist; SmallVector<SDNode*, 128> Worklist;
TableId getTableId(SDValue V) {
assert(V.getNode() && "Getting TableId on SDValue()");
auto I = ValueToIdMap.find(V);
if (I != ValueToIdMap.end()) {
// replace if there's been a shift.
RemapId(I->second);
assert(I->second && "All Ids should be nonzero");
return I->second;
}
// Add if it's not there.
ValueToIdMap.insert(std::make_pair(V, NextValueId));
IdToValueMap.insert(std::make_pair(NextValueId, V));
++NextValueId;
assert(NextValueId != 0 &&
"Ran out of Ids. Increase id type size or add compactification");
return NextValueId - 1;
}
const SDValue &getSDValue(TableId &Id) {
RemapId(Id);
assert(Id && "TableId should be non-zero");
return IdToValueMap[Id];
}
public: public:
explicit DAGTypeLegalizer(SelectionDAG &dag) explicit DAGTypeLegalizer(SelectionDAG &dag)
: TLI(dag.getTargetLoweringInfo()), DAG(dag), : TLI(dag.getTargetLoweringInfo()), DAG(dag),
@ -147,10 +182,24 @@ public:
bool run(); bool run();
void NoteDeletion(SDNode *Old, SDNode *New) { void NoteDeletion(SDNode *Old, SDNode *New) {
ExpungeNode(Old); for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
ExpungeNode(New); TableId NewId = getTableId(SDValue(New, i));
for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) TableId OldId = getTableId(SDValue(Old, i));
ReplacedValues[SDValue(Old, i)] = SDValue(New, i);
ReplacedValues[OldId] = NewId;
// Delete Node from tables.
ValueToIdMap.erase(SDValue(Old, i));
IdToValueMap.erase(OldId);
PromotedIntegers.erase(OldId);
ExpandedIntegers.erase(OldId);
SoftenedFloats.erase(OldId);
PromotedFloats.erase(OldId);
ExpandedFloats.erase(OldId);
ScalarizedVectors.erase(OldId);
SplitVectors.erase(OldId);
WidenedVectors.erase(OldId);
}
} }
SelectionDAG &getDAG() const { return DAG; } SelectionDAG &getDAG() const { return DAG; }
@ -158,9 +207,9 @@ public:
private: private:
SDNode *AnalyzeNewNode(SDNode *N); SDNode *AnalyzeNewNode(SDNode *N);
void AnalyzeNewValue(SDValue &Val); void AnalyzeNewValue(SDValue &Val);
void ExpungeNode(SDNode *N);
void PerformExpensiveChecks(); void PerformExpensiveChecks();
void RemapValue(SDValue &N); void RemapId(TableId &N);
void RemapValue(SDValue &V);
// Common routines. // Common routines.
SDValue BitConvertToInteger(SDValue Op); SDValue BitConvertToInteger(SDValue Op);
@ -207,8 +256,8 @@ private:
/// returns an i32, the lower 16 bits of which coincide with Op, and the upper /// returns an i32, the lower 16 bits of which coincide with Op, and the upper
/// 16 bits of which contain rubbish. /// 16 bits of which contain rubbish.
SDValue GetPromotedInteger(SDValue Op) { SDValue GetPromotedInteger(SDValue Op) {
SDValue &PromotedOp = PromotedIntegers[Op]; TableId &PromotedId = PromotedIntegers[getTableId(Op)];
RemapValue(PromotedOp); SDValue PromotedOp = getSDValue(PromotedId);
assert(PromotedOp.getNode() && "Operand wasn't promoted?"); assert(PromotedOp.getNode() && "Operand wasn't promoted?");
return PromotedOp; return PromotedOp;
} }
@ -402,16 +451,15 @@ private:
/// stay in a register, the Op is not converted to an integer. /// stay in a register, the Op is not converted to an integer.
/// In that case, the given op is returned. /// In that case, the given op is returned.
SDValue GetSoftenedFloat(SDValue Op) { SDValue GetSoftenedFloat(SDValue Op) {
auto Iter = SoftenedFloats.find(Op); TableId Id = getTableId(Op);
auto Iter = SoftenedFloats.find(Id);
if (Iter == SoftenedFloats.end()) { if (Iter == SoftenedFloats.end()) {
assert(isSimpleLegalType(Op.getValueType()) && assert(isSimpleLegalType(Op.getValueType()) &&
"Operand wasn't converted to integer?"); "Operand wasn't converted to integer?");
return Op; return Op;
} }
SDValue SoftenedOp = getSDValue(Iter->second);
SDValue &SoftenedOp = Iter->second;
assert(SoftenedOp.getNode() && "Unconverted op in SoftenedFloats?"); assert(SoftenedOp.getNode() && "Unconverted op in SoftenedFloats?");
RemapValue(SoftenedOp);
return SoftenedOp; return SoftenedOp;
} }
void SetSoftenedFloat(SDValue Op, SDValue Result); void SetSoftenedFloat(SDValue Op, SDValue Result);
@ -548,8 +596,8 @@ private:
//===--------------------------------------------------------------------===// //===--------------------------------------------------------------------===//
SDValue GetPromotedFloat(SDValue Op) { SDValue GetPromotedFloat(SDValue Op) {
SDValue &PromotedOp = PromotedFloats[Op]; TableId &PromotedId = PromotedFloats[getTableId(Op)];
RemapValue(PromotedOp); SDValue PromotedOp = getSDValue(PromotedId);
assert(PromotedOp.getNode() && "Operand wasn't promoted?"); assert(PromotedOp.getNode() && "Operand wasn't promoted?");
return PromotedOp; return PromotedOp;
} }
@ -588,8 +636,8 @@ private:
/// element type, this returns the element. For example, if Op is a v1i32, /// element type, this returns the element. For example, if Op is a v1i32,
/// Op = < i32 val >, this method returns val, an i32. /// Op = < i32 val >, this method returns val, an i32.
SDValue GetScalarizedVector(SDValue Op) { SDValue GetScalarizedVector(SDValue Op) {
SDValue &ScalarizedOp = ScalarizedVectors[Op]; TableId &ScalarizedId = ScalarizedVectors[getTableId(Op)];
RemapValue(ScalarizedOp); SDValue ScalarizedOp = getSDValue(ScalarizedId);
assert(ScalarizedOp.getNode() && "Operand wasn't scalarized?"); assert(ScalarizedOp.getNode() && "Operand wasn't scalarized?");
return ScalarizedOp; return ScalarizedOp;
} }
@ -700,8 +748,8 @@ private:
/// method returns a v4i32 for which the first two elements are the same as /// method returns a v4i32 for which the first two elements are the same as
/// those of Op, while the last two elements contain rubbish. /// those of Op, while the last two elements contain rubbish.
SDValue GetWidenedVector(SDValue Op) { SDValue GetWidenedVector(SDValue Op) {
SDValue &WidenedOp = WidenedVectors[Op]; TableId &WidenedId = WidenedVectors[getTableId(Op)];
RemapValue(WidenedOp); SDValue WidenedOp = getSDValue(WidenedId);
assert(WidenedOp.getNode() && "Operand wasn't widened?"); assert(WidenedOp.getNode() && "Operand wasn't widened?");
return WidenedOp; return WidenedOp;
} }