forked from OSchip/llvm-project
[DAG] Rework store merge to loop on load candidates. NFCI.
Continue to consider remaining candidate merges until all possible merges have been considered. llvm-svn: 303560
This commit is contained in:
Nirav Dave
parent
365cb47818
commit
e00da22ef3
|
|
@ -12544,49 +12544,54 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
|
|||
// mergeable cases. To prevent this, we prune such stores from the
|
||||
// front of StoreNodes here.
|
||||
|
||||
unsigned StartIdx = 0;
|
||||
while ((StartIdx + 1 < StoreNodes.size()) &&
|
||||
StoreNodes[StartIdx].OffsetFromBase + ElementSizeBytes !=
|
||||
StoreNodes[StartIdx + 1].OffsetFromBase)
|
||||
++StartIdx;
|
||||
bool RV = false;
|
||||
while (StoreNodes.size() > 1) {
|
||||
unsigned StartIdx = 0;
|
||||
while ((StartIdx + 1 < StoreNodes.size()) &&
|
||||
StoreNodes[StartIdx].OffsetFromBase + ElementSizeBytes !=
|
||||
StoreNodes[StartIdx + 1].OffsetFromBase)
|
||||
++StartIdx;
|
||||
|
||||
// Bail if we don't have enough candidates to merge.
|
||||
if (StartIdx + 1 >= StoreNodes.size())
|
||||
return false;
|
||||
// Bail if we don't have enough candidates to merge.
|
||||
if (StartIdx + 1 >= StoreNodes.size())
|
||||
return RV;
|
||||
|
||||
if (StartIdx)
|
||||
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + StartIdx);
|
||||
if (StartIdx)
|
||||
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + StartIdx);
|
||||
|
||||
// Scan the memory operations on the chain and find the first non-consecutive
|
||||
// store memory address.
|
||||
unsigned NumConsecutiveStores = 0;
|
||||
int64_t StartAddress = StoreNodes[0].OffsetFromBase;
|
||||
// Scan the memory operations on the chain and find the first
|
||||
// non-consecutive store memory address.
|
||||
unsigned NumConsecutiveStores = 1;
|
||||
int64_t StartAddress = StoreNodes[0].OffsetFromBase;
|
||||
// Check that the addresses are consecutive starting from the second
|
||||
// element in the list of stores.
|
||||
for (unsigned i = 1, e = StoreNodes.size(); i < e; ++i) {
|
||||
int64_t CurrAddress = StoreNodes[i].OffsetFromBase;
|
||||
if (CurrAddress - StartAddress != (ElementSizeBytes * i))
|
||||
break;
|
||||
NumConsecutiveStores = i + 1;
|
||||
}
|
||||
|
||||
// Check that the addresses are consecutive starting from the second
|
||||
// element in the list of stores.
|
||||
for (unsigned i = 1, e = StoreNodes.size(); i < e; ++i) {
|
||||
int64_t CurrAddress = StoreNodes[i].OffsetFromBase;
|
||||
if (CurrAddress - StartAddress != (ElementSizeBytes * i))
|
||||
break;
|
||||
NumConsecutiveStores = i + 1;
|
||||
}
|
||||
if (NumConsecutiveStores < 2) {
|
||||
StoreNodes.erase(StoreNodes.begin(),
|
||||
StoreNodes.begin() + NumConsecutiveStores);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (NumConsecutiveStores < 2)
|
||||
return false;
|
||||
// Check that we can merge these candidates without causing a cycle
|
||||
if (!checkMergeStoreCandidatesForDependencies(StoreNodes,
|
||||
NumConsecutiveStores)) {
|
||||
StoreNodes.erase(StoreNodes.begin(),
|
||||
StoreNodes.begin() + NumConsecutiveStores);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check that we can merge these candidates without causing a cycle
|
||||
if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumConsecutiveStores))
|
||||
return false;
|
||||
// The node with the lowest store address.
|
||||
LLVMContext &Context = *DAG.getContext();
|
||||
const DataLayout &DL = DAG.getDataLayout();
|
||||
|
||||
|
||||
// The node with the lowest store address.
|
||||
LLVMContext &Context = *DAG.getContext();
|
||||
const DataLayout &DL = DAG.getDataLayout();
|
||||
|
||||
// Store the constants into memory as one consecutive store.
|
||||
if (IsConstantSrc) {
|
||||
bool RV = false;
|
||||
while (NumConsecutiveStores > 1) {
|
||||
// Store the constants into memory as one consecutive store.
|
||||
if (IsConstantSrc) {
|
||||
LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
|
||||
unsigned FirstStoreAS = FirstInChain->getAddressSpace();
|
||||
unsigned FirstStoreAlign = FirstInChain->getAlignment();
|
||||
|
|
@ -12646,33 +12651,33 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
|
|||
}
|
||||
|
||||
// Check if we found a legal integer type that creates a meaningful merge.
|
||||
if (LastLegalType < 2 && LastLegalVectorType < 2)
|
||||
break;
|
||||
if (LastLegalType < 2 && LastLegalVectorType < 2) {
|
||||
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + 1);
|
||||
continue;
|
||||
}
|
||||
|
||||
bool UseVector = (LastLegalVectorType > LastLegalType) && !NoVectors;
|
||||
unsigned NumElem = (UseVector) ? LastLegalVectorType : LastLegalType;
|
||||
|
||||
bool Merged = MergeStoresOfConstantsOrVecElts(StoreNodes, MemVT, NumElem,
|
||||
true, UseVector);
|
||||
if (!Merged)
|
||||
break;
|
||||
if (!Merged) {
|
||||
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
|
||||
continue;
|
||||
}
|
||||
// Remove merged stores for next iteration.
|
||||
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
|
||||
RV = true;
|
||||
NumConsecutiveStores -= NumElem;
|
||||
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
|
||||
continue;
|
||||
}
|
||||
return RV;
|
||||
}
|
||||
|
||||
// When extracting multiple vector elements, try to store them
|
||||
// in one vector store rather than a sequence of scalar stores.
|
||||
if (IsExtractVecSrc) {
|
||||
bool RV = false;
|
||||
while (StoreNodes.size() >= 2) {
|
||||
// When extracting multiple vector elements, try to store them
|
||||
// in one vector store rather than a sequence of scalar stores.
|
||||
if (IsExtractVecSrc) {
|
||||
LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
|
||||
unsigned FirstStoreAS = FirstInChain->getAddressSpace();
|
||||
unsigned FirstStoreAlign = FirstInChain->getAlignment();
|
||||
unsigned NumStoresToMerge = 0;
|
||||
unsigned NumStoresToMerge = 1;
|
||||
bool IsVec = MemVT.isVector();
|
||||
for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
|
||||
StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
|
||||
|
|
@ -12684,7 +12689,7 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
|
|||
// handles consecutive loads).
|
||||
if (StoreValOpcode != ISD::EXTRACT_VECTOR_ELT &&
|
||||
StoreValOpcode != ISD::EXTRACT_SUBVECTOR)
|
||||
return false;
|
||||
return RV;
|
||||
|
||||
// Find a legal type for the vector store.
|
||||
unsigned Elts = i + 1;
|
||||
|
|
@ -12704,187 +12709,205 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) {
|
|||
|
||||
bool Merged = MergeStoresOfConstantsOrVecElts(
|
||||
StoreNodes, MemVT, NumStoresToMerge, false, true);
|
||||
if (!Merged)
|
||||
break;
|
||||
if (!Merged) {
|
||||
StoreNodes.erase(StoreNodes.begin(),
|
||||
StoreNodes.begin() + NumStoresToMerge);
|
||||
continue;
|
||||
}
|
||||
// Remove merged stores for next iteration.
|
||||
StoreNodes.erase(StoreNodes.begin(),
|
||||
StoreNodes.begin() + NumStoresToMerge);
|
||||
RV = true;
|
||||
NumConsecutiveStores -= NumStoresToMerge;
|
||||
continue;
|
||||
}
|
||||
return RV;
|
||||
}
|
||||
|
||||
// Below we handle the case of multiple consecutive stores that
|
||||
// come from multiple consecutive loads. We merge them into a single
|
||||
// wide load and a single wide store.
|
||||
// Below we handle the case of multiple consecutive stores that
|
||||
// come from multiple consecutive loads. We merge them into a single
|
||||
// wide load and a single wide store.
|
||||
|
||||
// Look for load nodes which are used by the stored values.
|
||||
SmallVector<MemOpLink, 8> LoadNodes;
|
||||
// Look for load nodes which are used by the stored values.
|
||||
SmallVector<MemOpLink, 8> LoadNodes;
|
||||
|
||||
// Find acceptable loads. Loads need to have the same chain (token factor),
|
||||
// must not be zext, volatile, indexed, and they must be consecutive.
|
||||
BaseIndexOffset LdBasePtr;
|
||||
for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
|
||||
StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
|
||||
LoadSDNode *Ld = dyn_cast<LoadSDNode>(St->getValue());
|
||||
if (!Ld) break;
|
||||
|
||||
// Loads must only have one use.
|
||||
if (!Ld->hasNUsesOfValue(1, 0))
|
||||
break;
|
||||
|
||||
// The memory operands must not be volatile.
|
||||
if (Ld->isVolatile() || Ld->isIndexed())
|
||||
break;
|
||||
|
||||
// We do not accept ext loads.
|
||||
if (Ld->getExtensionType() != ISD::NON_EXTLOAD)
|
||||
break;
|
||||
|
||||
// The stored memory type must be the same.
|
||||
if (Ld->getMemoryVT() != MemVT)
|
||||
break;
|
||||
|
||||
BaseIndexOffset LdPtr = BaseIndexOffset::match(Ld->getBasePtr(), DAG);
|
||||
// If this is not the first ptr that we check.
|
||||
if (LdBasePtr.Base.getNode()) {
|
||||
// The base ptr must be the same.
|
||||
if (!LdPtr.equalBaseIndex(LdBasePtr))
|
||||
// Find acceptable loads. Loads need to have the same chain (token factor),
|
||||
// must not be zext, volatile, indexed, and they must be consecutive.
|
||||
BaseIndexOffset LdBasePtr;
|
||||
for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
|
||||
StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
|
||||
LoadSDNode *Ld = dyn_cast<LoadSDNode>(St->getValue());
|
||||
if (!Ld)
|
||||
break;
|
||||
} else {
|
||||
// Check that all other base pointers are the same as this one.
|
||||
LdBasePtr = LdPtr;
|
||||
|
||||
// Loads must only have one use.
|
||||
if (!Ld->hasNUsesOfValue(1, 0))
|
||||
break;
|
||||
|
||||
// The memory operands must not be volatile.
|
||||
if (Ld->isVolatile() || Ld->isIndexed())
|
||||
break;
|
||||
|
||||
// We do not accept ext loads.
|
||||
if (Ld->getExtensionType() != ISD::NON_EXTLOAD)
|
||||
break;
|
||||
|
||||
// The stored memory type must be the same.
|
||||
if (Ld->getMemoryVT() != MemVT)
|
||||
break;
|
||||
|
||||
BaseIndexOffset LdPtr = BaseIndexOffset::match(Ld->getBasePtr(), DAG);
|
||||
// If this is not the first ptr that we check.
|
||||
if (LdBasePtr.Base.getNode()) {
|
||||
// The base ptr must be the same.
|
||||
if (!LdPtr.equalBaseIndex(LdBasePtr))
|
||||
break;
|
||||
} else {
|
||||
// Check that all other base pointers are the same as this one.
|
||||
LdBasePtr = LdPtr;
|
||||
}
|
||||
|
||||
// We found a potential memory operand to merge.
|
||||
LoadNodes.push_back(MemOpLink(Ld, LdPtr.Offset));
|
||||
}
|
||||
|
||||
// We found a potential memory operand to merge.
|
||||
LoadNodes.push_back(MemOpLink(Ld, LdPtr.Offset));
|
||||
}
|
||||
|
||||
if (LoadNodes.size() < 2)
|
||||
return false;
|
||||
|
||||
// If we have load/store pair instructions and we only have two values,
|
||||
// don't bother.
|
||||
unsigned RequiredAlignment;
|
||||
if (LoadNodes.size() == 2 && TLI.hasPairedLoad(MemVT, RequiredAlignment) &&
|
||||
St->getAlignment() >= RequiredAlignment)
|
||||
return false;
|
||||
LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
|
||||
unsigned FirstStoreAS = FirstInChain->getAddressSpace();
|
||||
unsigned FirstStoreAlign = FirstInChain->getAlignment();
|
||||
LoadSDNode *FirstLoad = cast<LoadSDNode>(LoadNodes[0].MemNode);
|
||||
unsigned FirstLoadAS = FirstLoad->getAddressSpace();
|
||||
unsigned FirstLoadAlign = FirstLoad->getAlignment();
|
||||
|
||||
// Scan the memory operations on the chain and find the first non-consecutive
|
||||
// load memory address. These variables hold the index in the store node
|
||||
// array.
|
||||
unsigned LastConsecutiveLoad = 0;
|
||||
// This variable refers to the size and not index in the array.
|
||||
unsigned LastLegalVectorType = 0;
|
||||
unsigned LastLegalIntegerType = 0;
|
||||
StartAddress = LoadNodes[0].OffsetFromBase;
|
||||
SDValue FirstChain = FirstLoad->getChain();
|
||||
for (unsigned i = 1; i < LoadNodes.size(); ++i) {
|
||||
// All loads must share the same chain.
|
||||
if (LoadNodes[i].MemNode->getChain() != FirstChain)
|
||||
break;
|
||||
|
||||
int64_t CurrAddress = LoadNodes[i].OffsetFromBase;
|
||||
if (CurrAddress - StartAddress != (ElementSizeBytes * i))
|
||||
break;
|
||||
LastConsecutiveLoad = i;
|
||||
// Find a legal type for the vector store.
|
||||
EVT StoreTy = EVT::getVectorVT(Context, MemVT, i+1);
|
||||
bool IsFastSt, IsFastLd;
|
||||
if (TLI.isTypeLegal(StoreTy) &&
|
||||
TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS,
|
||||
FirstStoreAlign, &IsFastSt) && IsFastSt &&
|
||||
TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstLoadAS,
|
||||
FirstLoadAlign, &IsFastLd) && IsFastLd) {
|
||||
LastLegalVectorType = i + 1;
|
||||
if (LoadNodes.size() < 2) {
|
||||
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + 1);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Find a legal type for the integer store.
|
||||
unsigned SizeInBits = (i+1) * ElementSizeBytes * 8;
|
||||
StoreTy = EVT::getIntegerVT(Context, SizeInBits);
|
||||
if (TLI.isTypeLegal(StoreTy) &&
|
||||
TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS,
|
||||
FirstStoreAlign, &IsFastSt) && IsFastSt &&
|
||||
TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstLoadAS,
|
||||
FirstLoadAlign, &IsFastLd) && IsFastLd)
|
||||
LastLegalIntegerType = i + 1;
|
||||
// Or check whether a truncstore and extload is legal.
|
||||
else if (TLI.getTypeAction(Context, StoreTy) ==
|
||||
TargetLowering::TypePromoteInteger) {
|
||||
EVT LegalizedStoredValueTy =
|
||||
TLI.getTypeToTransformTo(Context, StoreTy);
|
||||
if (TLI.isTruncStoreLegal(LegalizedStoredValueTy, StoreTy) &&
|
||||
TLI.isLoadExtLegal(ISD::ZEXTLOAD, LegalizedStoredValueTy, StoreTy) &&
|
||||
TLI.isLoadExtLegal(ISD::SEXTLOAD, LegalizedStoredValueTy, StoreTy) &&
|
||||
TLI.isLoadExtLegal(ISD::EXTLOAD, LegalizedStoredValueTy, StoreTy) &&
|
||||
TLI.allowsMemoryAccess(Context, DL, LegalizedStoredValueTy,
|
||||
FirstStoreAS, FirstStoreAlign, &IsFastSt) &&
|
||||
// If we have load/store pair instructions and we only have two values,
|
||||
// don't bother.
|
||||
unsigned RequiredAlignment;
|
||||
if (LoadNodes.size() == 2 && TLI.hasPairedLoad(MemVT, RequiredAlignment) &&
|
||||
St->getAlignment() >= RequiredAlignment) {
|
||||
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + 2);
|
||||
continue;
|
||||
}
|
||||
LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
|
||||
unsigned FirstStoreAS = FirstInChain->getAddressSpace();
|
||||
unsigned FirstStoreAlign = FirstInChain->getAlignment();
|
||||
LoadSDNode *FirstLoad = cast<LoadSDNode>(LoadNodes[0].MemNode);
|
||||
unsigned FirstLoadAS = FirstLoad->getAddressSpace();
|
||||
unsigned FirstLoadAlign = FirstLoad->getAlignment();
|
||||
|
||||
// Scan the memory operations on the chain and find the first
|
||||
// non-consecutive load memory address. These variables hold the index in
|
||||
// the store node array.
|
||||
unsigned LastConsecutiveLoad = 0;
|
||||
// This variable refers to the size and not index in the array.
|
||||
unsigned LastLegalVectorType = 0;
|
||||
unsigned LastLegalIntegerType = 0;
|
||||
StartAddress = LoadNodes[0].OffsetFromBase;
|
||||
SDValue FirstChain = FirstLoad->getChain();
|
||||
for (unsigned i = 1; i < LoadNodes.size(); ++i) {
|
||||
// All loads must share the same chain.
|
||||
if (LoadNodes[i].MemNode->getChain() != FirstChain)
|
||||
break;
|
||||
|
||||
int64_t CurrAddress = LoadNodes[i].OffsetFromBase;
|
||||
if (CurrAddress - StartAddress != (ElementSizeBytes * i))
|
||||
break;
|
||||
LastConsecutiveLoad = i;
|
||||
// Find a legal type for the vector store.
|
||||
EVT StoreTy = EVT::getVectorVT(Context, MemVT, i + 1);
|
||||
bool IsFastSt, IsFastLd;
|
||||
if (TLI.isTypeLegal(StoreTy) &&
|
||||
TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS,
|
||||
FirstStoreAlign, &IsFastSt) &&
|
||||
IsFastSt &&
|
||||
TLI.allowsMemoryAccess(Context, DL, LegalizedStoredValueTy,
|
||||
FirstLoadAS, FirstLoadAlign, &IsFastLd) &&
|
||||
TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstLoadAS,
|
||||
FirstLoadAlign, &IsFastLd) &&
|
||||
IsFastLd) {
|
||||
LastLegalVectorType = i + 1;
|
||||
}
|
||||
|
||||
// Find a legal type for the integer store.
|
||||
unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
|
||||
StoreTy = EVT::getIntegerVT(Context, SizeInBits);
|
||||
if (TLI.isTypeLegal(StoreTy) &&
|
||||
TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS,
|
||||
FirstStoreAlign, &IsFastSt) &&
|
||||
IsFastSt &&
|
||||
TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstLoadAS,
|
||||
FirstLoadAlign, &IsFastLd) &&
|
||||
IsFastLd)
|
||||
LastLegalIntegerType = i+1;
|
||||
LastLegalIntegerType = i + 1;
|
||||
// Or check whether a truncstore and extload is legal.
|
||||
else if (TLI.getTypeAction(Context, StoreTy) ==
|
||||
TargetLowering::TypePromoteInteger) {
|
||||
EVT LegalizedStoredValueTy = TLI.getTypeToTransformTo(Context, StoreTy);
|
||||
if (TLI.isTruncStoreLegal(LegalizedStoredValueTy, StoreTy) &&
|
||||
TLI.isLoadExtLegal(ISD::ZEXTLOAD, LegalizedStoredValueTy,
|
||||
StoreTy) &&
|
||||
TLI.isLoadExtLegal(ISD::SEXTLOAD, LegalizedStoredValueTy,
|
||||
StoreTy) &&
|
||||
TLI.isLoadExtLegal(ISD::EXTLOAD, LegalizedStoredValueTy, StoreTy) &&
|
||||
TLI.allowsMemoryAccess(Context, DL, LegalizedStoredValueTy,
|
||||
FirstStoreAS, FirstStoreAlign, &IsFastSt) &&
|
||||
IsFastSt &&
|
||||
TLI.allowsMemoryAccess(Context, DL, LegalizedStoredValueTy,
|
||||
FirstLoadAS, FirstLoadAlign, &IsFastLd) &&
|
||||
IsFastLd)
|
||||
LastLegalIntegerType = i + 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Only use vector types if the vector type is larger than the integer type.
|
||||
// If they are the same, use integers.
|
||||
bool UseVectorTy = LastLegalVectorType > LastLegalIntegerType && !NoVectors;
|
||||
unsigned LastLegalType =
|
||||
std::max(LastLegalVectorType, LastLegalIntegerType);
|
||||
|
||||
// We add +1 here because the LastXXX variables refer to location while
|
||||
// the NumElem refers to array/index size.
|
||||
unsigned NumElem = std::min(NumConsecutiveStores, LastConsecutiveLoad + 1);
|
||||
NumElem = std::min(LastLegalType, NumElem);
|
||||
|
||||
if (NumElem < 2) {
|
||||
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + 1);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Find if it is better to use vectors or integers to load and store
|
||||
// to memory.
|
||||
EVT JointMemOpVT;
|
||||
if (UseVectorTy) {
|
||||
JointMemOpVT = EVT::getVectorVT(Context, MemVT, NumElem);
|
||||
} else {
|
||||
unsigned SizeInBits = NumElem * ElementSizeBytes * 8;
|
||||
JointMemOpVT = EVT::getIntegerVT(Context, SizeInBits);
|
||||
}
|
||||
|
||||
SDLoc LoadDL(LoadNodes[0].MemNode);
|
||||
SDLoc StoreDL(StoreNodes[0].MemNode);
|
||||
|
||||
// The merged loads are required to have the same incoming chain, so
|
||||
// using the first's chain is acceptable.
|
||||
SDValue NewLoad = DAG.getLoad(JointMemOpVT, LoadDL, FirstLoad->getChain(),
|
||||
FirstLoad->getBasePtr(),
|
||||
FirstLoad->getPointerInfo(), FirstLoadAlign);
|
||||
|
||||
SDValue NewStoreChain = getMergeStoreChains(StoreNodes, NumElem);
|
||||
|
||||
AddToWorklist(NewStoreChain.getNode());
|
||||
|
||||
SDValue NewStore = DAG.getStore(
|
||||
NewStoreChain, StoreDL, NewLoad, FirstInChain->getBasePtr(),
|
||||
FirstInChain->getPointerInfo(), FirstStoreAlign);
|
||||
|
||||
// Transfer chain users from old loads to the new load.
|
||||
for (unsigned i = 0; i < NumElem; ++i) {
|
||||
LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[i].MemNode);
|
||||
DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1),
|
||||
SDValue(NewLoad.getNode(), 1));
|
||||
}
|
||||
|
||||
// Replace the all stores with the new store.
|
||||
for (unsigned i = 0; i < NumElem; ++i)
|
||||
CombineTo(StoreNodes[i].MemNode, NewStore);
|
||||
RV = true;
|
||||
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Only use vector types if the vector type is larger than the integer type.
|
||||
// If they are the same, use integers.
|
||||
bool UseVectorTy = LastLegalVectorType > LastLegalIntegerType && !NoVectors;
|
||||
unsigned LastLegalType = std::max(LastLegalVectorType, LastLegalIntegerType);
|
||||
|
||||
// We add +1 here because the LastXXX variables refer to location while
|
||||
// the NumElem refers to array/index size.
|
||||
unsigned NumElem = std::min(NumConsecutiveStores, LastConsecutiveLoad + 1);
|
||||
NumElem = std::min(LastLegalType, NumElem);
|
||||
|
||||
if (NumElem < 2)
|
||||
return false;
|
||||
|
||||
// Find if it is better to use vectors or integers to load and store
|
||||
// to memory.
|
||||
EVT JointMemOpVT;
|
||||
if (UseVectorTy) {
|
||||
JointMemOpVT = EVT::getVectorVT(Context, MemVT, NumElem);
|
||||
} else {
|
||||
unsigned SizeInBits = NumElem * ElementSizeBytes * 8;
|
||||
JointMemOpVT = EVT::getIntegerVT(Context, SizeInBits);
|
||||
}
|
||||
|
||||
SDLoc LoadDL(LoadNodes[0].MemNode);
|
||||
SDLoc StoreDL(StoreNodes[0].MemNode);
|
||||
|
||||
// The merged loads are required to have the same incoming chain, so
|
||||
// using the first's chain is acceptable.
|
||||
SDValue NewLoad = DAG.getLoad(JointMemOpVT, LoadDL, FirstLoad->getChain(),
|
||||
FirstLoad->getBasePtr(),
|
||||
FirstLoad->getPointerInfo(), FirstLoadAlign);
|
||||
|
||||
SDValue NewStoreChain = getMergeStoreChains(StoreNodes, NumElem);
|
||||
|
||||
AddToWorklist(NewStoreChain.getNode());
|
||||
|
||||
SDValue NewStore =
|
||||
DAG.getStore(NewStoreChain, StoreDL, NewLoad, FirstInChain->getBasePtr(),
|
||||
FirstInChain->getPointerInfo(), FirstStoreAlign);
|
||||
|
||||
// Transfer chain users from old loads to the new load.
|
||||
for (unsigned i = 0; i < NumElem; ++i) {
|
||||
LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[i].MemNode);
|
||||
DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1),
|
||||
SDValue(NewLoad.getNode(), 1));
|
||||
}
|
||||
|
||||
// Replace the all stores with the new store.
|
||||
for (unsigned i = 0; i < NumElem; ++i)
|
||||
CombineTo(StoreNodes[i].MemNode, NewStore);
|
||||
return true;
|
||||
return RV;
|
||||
}
|
||||
|
||||
SDValue DAGCombiner::replaceStoreChain(StoreSDNode *ST, SDValue BetterChain) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue