maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[DAG] Rework store merge to loop on load candidates. NFCI. 

Continue to consider remaining candidate merges until all possible
merges have been considered.

llvm-svn: 303560
This commit is contained in:
Nirav Dave 2017-05-22 15:33:47 +00:00
parent 365cb47818
commit e00da22ef3
1 changed files with 238 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
View File

@ -12544,6 +12544,8 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
// mergeable cases. To prevent this, we prune such stores from the
// front of StoreNodes here.
bool RV = false;
while (StoreNodes.size() > 1) {
unsigned StartIdx = 0;
while ((StartIdx + 1 < StoreNodes.size()) &&
StoreNodes[StartIdx].OffsetFromBase + ElementSizeBytes !=
@ -12552,16 +12554,15 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
// Bail if we don't have enough candidates to merge.
if (StartIdx + 1 >= StoreNodes.size())
return false;
return RV;
if (StartIdx)
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + StartIdx);
// Scan the memory operations on the chain and find the first non-consecutive
// store memory address.
unsigned NumConsecutiveStores = 0;
// Scan the memory operations on the chain and find the first
// non-consecutive store memory address.
unsigned NumConsecutiveStores = 1;
int64_t StartAddress = StoreNodes[0].OffsetFromBase;
// Check that the addresses are consecutive starting from the second
// element in the list of stores.
for (unsigned i = 1, e = StoreNodes.size(); i < e; ++i) {
@ -12571,13 +12572,19 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
NumConsecutiveStores = i + 1;
}
if (NumConsecutiveStores < 2)
return false;
if (NumConsecutiveStores < 2) {
StoreNodes.erase(StoreNodes.begin(),
StoreNodes.begin() + NumConsecutiveStores);
continue;
}
// Check that we can merge these candidates without causing a cycle
if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumConsecutiveStores))
return false;
if (!checkMergeStoreCandidatesForDependencies(StoreNodes,
NumConsecutiveStores)) {
StoreNodes.erase(StoreNodes.begin(),
StoreNodes.begin() + NumConsecutiveStores);
continue;
}
// The node with the lowest store address.
LLVMContext &Context = *DAG.getContext();
@ -12585,8 +12592,6 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
// Store the constants into memory as one consecutive store.
if (IsConstantSrc) {
bool RV = false;
while (NumConsecutiveStores > 1) {
LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
unsigned FirstStoreAS = FirstInChain->getAddressSpace();
unsigned FirstStoreAlign = FirstInChain->getAlignment();
@ -12646,33 +12651,33 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
}
// Check if we found a legal integer type that creates a meaningful merge.
if (LastLegalType < 2 && LastLegalVectorType < 2)
break;
if (LastLegalType < 2 && LastLegalVectorType < 2) {
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + 1);
continue;
}
bool UseVector = (LastLegalVectorType > LastLegalType) && !NoVectors;
unsigned NumElem = (UseVector) ? LastLegalVectorType : LastLegalType;
bool Merged = MergeStoresOfConstantsOrVecElts(StoreNodes, MemVT, NumElem,
true, UseVector);
if (!Merged)
break;
// Remove merged stores for next iteration.
if (!Merged) {
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
RV = true;
NumConsecutiveStores -= NumElem;
continue;
}
return RV;
// Remove merged stores for next iteration.
RV = true;
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
continue;
}
// When extracting multiple vector elements, try to store them
// in one vector store rather than a sequence of scalar stores.
if (IsExtractVecSrc) {
bool RV = false;
while (StoreNodes.size() >= 2) {
LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
unsigned FirstStoreAS = FirstInChain->getAddressSpace();
unsigned FirstStoreAlign = FirstInChain->getAlignment();
unsigned NumStoresToMerge = 0;
unsigned NumStoresToMerge = 1;
bool IsVec = MemVT.isVector();
for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
@ -12684,7 +12689,7 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
// handles consecutive loads).
if (StoreValOpcode != ISD::EXTRACT_VECTOR_ELT &&
StoreValOpcode != ISD::EXTRACT_SUBVECTOR)
return false;
return RV;
// Find a legal type for the vector store.
unsigned Elts = i + 1;
@ -12704,15 +12709,16 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
bool Merged = MergeStoresOfConstantsOrVecElts(
StoreNodes, MemVT, NumStoresToMerge, false, true);
if (!Merged)
break;
if (!Merged) {
StoreNodes.erase(StoreNodes.begin(),
StoreNodes.begin() + NumStoresToMerge);
continue;
}
// Remove merged stores for next iteration.
StoreNodes.erase(StoreNodes.begin(),
StoreNodes.begin() + NumStoresToMerge);
RV = true;
NumConsecutiveStores -= NumStoresToMerge;
}
return RV;
continue;
}
// Below we handle the case of multiple consecutive stores that
@ -12728,7 +12734,8 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
LoadSDNode *Ld = dyn_cast<LoadSDNode>(St->getValue());
if (!Ld) break;
if (!Ld)
break;
// Loads must only have one use.
if (!Ld->hasNUsesOfValue(1, 0))
@ -12761,15 +12768,19 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
LoadNodes.push_back(MemOpLink(Ld, LdPtr.Offset));
}
if (LoadNodes.size() < 2)
return false;
if (LoadNodes.size() < 2) {
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + 1);
continue;
}
// If we have load/store pair instructions and we only have two values,
// don't bother.
unsigned RequiredAlignment;
if (LoadNodes.size() == 2 && TLI.hasPairedLoad(MemVT, RequiredAlignment) &&
St->getAlignment() >= RequiredAlignment)
return false;
St->getAlignment() >= RequiredAlignment) {
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + 2);
continue;
}
LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
unsigned FirstStoreAS = FirstInChain->getAddressSpace();
unsigned FirstStoreAlign = FirstInChain->getAlignment();
@ -12777,9 +12788,9 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
unsigned FirstLoadAS = FirstLoad->getAddressSpace();
unsigned FirstLoadAlign = FirstLoad->getAlignment();
// Scan the memory operations on the chain and find the first non-consecutive
// load memory address. These variables hold the index in the store node
// array.
// Scan the memory operations on the chain and find the first
// non-consecutive load memory address. These variables hold the index in
// the store node array.
unsigned LastConsecutiveLoad = 0;
// This variable refers to the size and not index in the array.
unsigned LastLegalVectorType = 0;
@ -12800,9 +12811,11 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
bool IsFastSt, IsFastLd;
if (TLI.isTypeLegal(StoreTy) &&
TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS,
FirstStoreAlign, &IsFastSt) && IsFastSt &&
FirstStoreAlign, &IsFastSt) &&
IsFastSt &&
TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstLoadAS,
FirstLoadAlign, &IsFastLd) && IsFastLd) {
FirstLoadAlign, &IsFastLd) &&
IsFastLd) {
LastLegalVectorType = i + 1;
}
@ -12811,18 +12824,21 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
StoreTy = EVT::getIntegerVT(Context, SizeInBits);
if (TLI.isTypeLegal(StoreTy) &&
TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS,
FirstStoreAlign, &IsFastSt) && IsFastSt &&
FirstStoreAlign, &IsFastSt) &&
IsFastSt &&
TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstLoadAS,
FirstLoadAlign, &IsFastLd) && IsFastLd)
FirstLoadAlign, &IsFastLd) &&
IsFastLd)
LastLegalIntegerType = i + 1;
// Or check whether a truncstore and extload is legal.
else if (TLI.getTypeAction(Context, StoreTy) ==
TargetLowering::TypePromoteInteger) {
EVT LegalizedStoredValueTy =
TLI.getTypeToTransformTo(Context, StoreTy);
EVT LegalizedStoredValueTy = TLI.getTypeToTransformTo(Context, StoreTy);
if (TLI.isTruncStoreLegal(LegalizedStoredValueTy, StoreTy) &&
TLI.isLoadExtLegal(ISD::ZEXTLOAD, LegalizedStoredValueTy, StoreTy) &&
TLI.isLoadExtLegal(ISD::SEXTLOAD, LegalizedStoredValueTy, StoreTy) &&
TLI.isLoadExtLegal(ISD::ZEXTLOAD, LegalizedStoredValueTy,
StoreTy) &&
TLI.isLoadExtLegal(ISD::SEXTLOAD, LegalizedStoredValueTy,
StoreTy) &&
TLI.isLoadExtLegal(ISD::EXTLOAD, LegalizedStoredValueTy, StoreTy) &&
TLI.allowsMemoryAccess(Context, DL, LegalizedStoredValueTy,
FirstStoreAS, FirstStoreAlign, &IsFastSt) &&
@ -12837,15 +12853,18 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
// Only use vector types if the vector type is larger than the integer type.
// If they are the same, use integers.
bool UseVectorTy = LastLegalVectorType > LastLegalIntegerType && !NoVectors;
unsigned LastLegalType = std::max(LastLegalVectorType, LastLegalIntegerType);
unsigned LastLegalType =
std::max(LastLegalVectorType, LastLegalIntegerType);
// We add +1 here because the LastXXX variables refer to location while
// the NumElem refers to array/index size.
unsigned NumElem = std::min(NumConsecutiveStores, LastConsecutiveLoad + 1);
NumElem = std::min(LastLegalType, NumElem);
if (NumElem < 2)
return false;
if (NumElem < 2) {
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + 1);
continue;
}
// Find if it is better to use vectors or integers to load and store
// to memory.
@ -12870,8 +12889,8 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
AddToWorklist(NewStoreChain.getNode());
SDValue NewStore =
DAG.getStore(NewStoreChain, StoreDL, NewLoad, FirstInChain->getBasePtr(),
SDValue NewStore = DAG.getStore(
NewStoreChain, StoreDL, NewLoad, FirstInChain->getBasePtr(),
FirstInChain->getPointerInfo(), FirstStoreAlign);
// Transfer chain users from old loads to the new load.
@ -12884,7 +12903,11 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
// Replace the all stores with the new store.
for (unsigned i = 0; i < NumElem; ++i)
CombineTo(StoreNodes[i].MemNode, NewStore);
return true;
RV = true;
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
continue;
}
return RV;
}
SDValue DAGCombiner::replaceStoreChain(StoreSDNode *ST, SDValue BetterChain) {