maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[ExpandMemCmp] Split ExpandMemCmp from CodeGen into its own pass. 

Summary:
This is mostly a noop (most of the test diffs are renamed blocks).
There are a few temporary register renames (eax<->ecx) and a few blocks are
shuffled around.

See the discussion in PR33325 for more details.

Reviewers: spatel

Subscribers: mgorny

Differential Revision: https://reviews.llvm.org/D39456

llvm-svn: 317211
This commit is contained in:
Clement Courbet 2017-11-02 15:02:51 +00:00
parent 5347ebef23
commit 1dc37b9c3b
13 changed files with 1352 additions and 1200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
llvm/include/llvm/InitializePasses.h
View File

@ -128,6 +128,7 @@ void initializeEdgeBundlesPass(PassRegistry&);
void initializeEfficiencySanitizerPass(PassRegistry&);
void initializeEliminateAvailableExternallyLegacyPassPass(PassRegistry&);
void initializeExpandISelPseudosPass(PassRegistry&);
void initializeExpandMemCmpPassPass(PassRegistry&);
void initializeExpandPostRAPass(PassRegistry&);
void initializeExpandReductionsPass(PassRegistry&);
void initializeExternalAAWrapperPassPass(PassRegistry&);

1
llvm/include/llvm/LinkAllPasses.h
View File

@ -180,6 +180,7 @@ namespace {
(void) llvm::createReversePostOrderFunctionAttrsPass();
(void) llvm::createMergeFunctionsPass();
(void) llvm::createMergeICmpsPass();
(void) llvm::createExpandMemCmpPass();
std::string buf;
llvm::raw_string_ostream os(buf);
(void) llvm::createPrintModulePass(os);

8
llvm/include/llvm/Transforms/Scalar.h
View File

@ -422,10 +422,16 @@ Pass *createLowerGuardIntrinsicPass();
//===----------------------------------------------------------------------===//
//
// MergeICmps - Merge integer comparison chains
// MergeICmps - Merge integer comparison chains into a memcmp
//
Pass *createMergeICmpsPass();
//===----------------------------------------------------------------------===//
//
// ExpandMemCmp - Expand memcmp() to load/stores.
//
Pass *createExpandMemCmpPass();
//===----------------------------------------------------------------------===//
//
// ValuePropagation - Propagate CFG-derived value information

710
llvm/lib/CodeGen/CodeGenPrepare.cpp
View File

@ -123,12 +123,6 @@ STATISTIC(NumDbgValueMoved, "Number of debug value instructions moved");
STATISTIC(NumSelectsExpanded, "Number of selects turned into branches");
STATISTIC(NumStoreExtractExposed, "Number of store(extractelement) exposed");
STATISTIC(NumMemCmpCalls, "Number of memcmp calls");
STATISTIC(NumMemCmpNotConstant, "Number of memcmp calls without constant size");
STATISTIC(NumMemCmpGreaterThanMax,
"Number of memcmp calls with size greater than max size");
STATISTIC(NumMemCmpInlined, "Number of inlined memcmp calls");
static cl::opt<bool> DisableBranchOpts(
"disable-cgp-branch-opts", cl::Hidden, cl::init(false),
cl::desc("Disable branch optimizations in CodeGenPrepare"));
@ -189,11 +183,6 @@ EnableTypePromotionMerge("cgp-type-promotion-merge", cl::Hidden,
cl::desc("Enable merging of redundant sexts when one is dominating"
" the other."), cl::init(true));
static cl::opt<unsigned> MemCmpNumLoadsPerBlock(
"memcmp-num-loads-per-block", cl::Hidden, cl::init(1),
cl::desc("The number of loads per basic block for inline expansion of "
"memcmp that is only being compared against zero."));
namespace {
using SetOfInstrs = SmallPtrSet<Instruction *, 16>;
@ -1697,699 +1686,6 @@ static bool despeculateCountZeros(IntrinsicInst *CountZeros,
return true;
}
namespace {
// This class provides helper functions to expand a memcmp library call into an
// inline expansion.
class MemCmpExpansion {
struct ResultBlock {
BasicBlock *BB = nullptr;
PHINode *PhiSrc1 = nullptr;
PHINode *PhiSrc2 = nullptr;
ResultBlock() = default;
};
CallInst *const CI;
ResultBlock ResBlock;
const uint64_t Size;
unsigned MaxLoadSize;
uint64_t NumLoadsNonOneByte;
const uint64_t NumLoadsPerBlock;
std::vector<BasicBlock *> LoadCmpBlocks;
BasicBlock *EndBlock;
PHINode *PhiRes;
const bool IsUsedForZeroCmp;
const DataLayout &DL;
IRBuilder<> Builder;
// Represents the decomposition in blocks of the expansion. For example,
// comparing 33 bytes on X86+sse can be done with 2x16-byte loads and
// 1x1-byte load, which would be represented as [{16, 0}, {16, 16}, {32, 1}.
// TODO(courbet): Involve the target more in this computation. On X86, 7
// bytes can be done more efficiently with two overlaping 4-byte loads than
// covering the interval with [{4, 0},{2, 4},{1, 6}}.
struct LoadEntry {
LoadEntry(unsigned LoadSize, uint64_t Offset)
: LoadSize(LoadSize), Offset(Offset) {
assert(Offset % LoadSize == 0 && "invalid load entry");
}
uint64_t getGEPIndex() const { return Offset / LoadSize; }
// The size of the load for this block, in bytes.
const unsigned LoadSize;
// The offset of this load WRT the base pointer, in bytes.
const uint64_t Offset;
};
SmallVector<LoadEntry, 8> LoadSequence;
void createLoadCmpBlocks();
void createResultBlock();
void setupResultBlockPHINodes();
void setupEndBlockPHINodes();
Value *getCompareLoadPairs(unsigned BlockIndex, unsigned &LoadIndex);
void emitLoadCompareBlock(unsigned BlockIndex);
void emitLoadCompareBlockMultipleLoads(unsigned BlockIndex,
unsigned &LoadIndex);
void emitLoadCompareByteBlock(unsigned BlockIndex, unsigned GEPIndex);
void emitMemCmpResultBlock();
Value *getMemCmpExpansionZeroCase();
Value *getMemCmpEqZeroOneBlock();
Value *getMemCmpOneBlock();
public:
MemCmpExpansion(CallInst *CI, uint64_t Size,
const TargetTransformInfo::MemCmpExpansionOptions &Options,
unsigned MaxNumLoads, const bool IsUsedForZeroCmp,
unsigned NumLoadsPerBlock, const DataLayout &DL);
unsigned getNumBlocks();
uint64_t getNumLoads() const { return LoadSequence.size(); }
Value *getMemCmpExpansion();
};
} // end anonymous namespace
// Initialize the basic block structure required for expansion of memcmp call
// with given maximum load size and memcmp size parameter.
// This structure includes:
// 1. A list of load compare blocks - LoadCmpBlocks.
// 2. An EndBlock, split from original instruction point, which is the block to
// return from.
// 3. ResultBlock, block to branch to for early exit when a
// LoadCmpBlock finds a difference.
MemCmpExpansion::MemCmpExpansion(
CallInst *const CI, uint64_t Size,
const TargetTransformInfo::MemCmpExpansionOptions &Options,
const unsigned MaxNumLoads, const bool IsUsedForZeroCmp,
const unsigned NumLoadsPerBlock, const DataLayout &TheDataLayout)
: CI(CI),
Size(Size),
MaxLoadSize(0),
NumLoadsNonOneByte(0),
NumLoadsPerBlock(NumLoadsPerBlock),
IsUsedForZeroCmp(IsUsedForZeroCmp),
DL(TheDataLayout),
Builder(CI) {
assert(Size > 0 && "zero blocks");
// Scale the max size down if the target can load more bytes than we need.
size_t LoadSizeIndex = 0;
while (LoadSizeIndex < Options.LoadSizes.size() &&
Options.LoadSizes[LoadSizeIndex] > Size) {
++LoadSizeIndex;
}
this->MaxLoadSize = Options.LoadSizes[LoadSizeIndex];
// Compute the decomposition.
uint64_t CurSize = Size;
uint64_t Offset = 0;
while (CurSize && LoadSizeIndex < Options.LoadSizes.size()) {
const unsigned LoadSize = Options.LoadSizes[LoadSizeIndex];
assert(LoadSize > 0 && "zero load size");
const uint64_t NumLoadsForThisSize = CurSize / LoadSize;
if (LoadSequence.size() + NumLoadsForThisSize > MaxNumLoads) {
// Do not expand if the total number of loads is larger than what the
// target allows. Note that it's important that we exit before completing
// the expansion to avoid using a ton of memory to store the expansion for
// large sizes.
LoadSequence.clear();
return;
}
if (NumLoadsForThisSize > 0) {
for (uint64_t I = 0; I < NumLoadsForThisSize; ++I) {
LoadSequence.push_back({LoadSize, Offset});
Offset += LoadSize;
}
if (LoadSize > 1) {
++NumLoadsNonOneByte;
}
CurSize = CurSize % LoadSize;
}
++LoadSizeIndex;
}
assert(LoadSequence.size() <= MaxNumLoads && "broken invariant");
}
unsigned MemCmpExpansion::getNumBlocks() {
if (IsUsedForZeroCmp)
return getNumLoads() / NumLoadsPerBlock +
(getNumLoads() % NumLoadsPerBlock != 0 ? 1 : 0);
return getNumLoads();
}
void MemCmpExpansion::createLoadCmpBlocks() {
for (unsigned i = 0; i < getNumBlocks(); i++) {
BasicBlock *BB = BasicBlock::Create(CI->getContext(), "loadbb",
EndBlock->getParent(), EndBlock);
LoadCmpBlocks.push_back(BB);
}
}
void MemCmpExpansion::createResultBlock() {
ResBlock.BB = BasicBlock::Create(CI->getContext(), "res_block",
EndBlock->getParent(), EndBlock);
}
// This function creates the IR instructions for loading and comparing 1 byte.
// It loads 1 byte from each source of the memcmp parameters with the given
// GEPIndex. It then subtracts the two loaded values and adds this result to the
// final phi node for selecting the memcmp result.
void MemCmpExpansion::emitLoadCompareByteBlock(unsigned BlockIndex,
unsigned GEPIndex) {
Value *Source1 = CI->getArgOperand(0);
Value *Source2 = CI->getArgOperand(1);
Builder.SetInsertPoint(LoadCmpBlocks[BlockIndex]);
Type *LoadSizeType = Type::getInt8Ty(CI->getContext());
// Cast source to LoadSizeType*.
if (Source1->getType() != LoadSizeType)
Source1 = Builder.CreateBitCast(Source1, LoadSizeType->getPointerTo());
if (Source2->getType() != LoadSizeType)
Source2 = Builder.CreateBitCast(Source2, LoadSizeType->getPointerTo());
// Get the base address using the GEPIndex.
if (GEPIndex != 0) {
Source1 = Builder.CreateGEP(LoadSizeType, Source1,
ConstantInt::get(LoadSizeType, GEPIndex));
Source2 = Builder.CreateGEP(LoadSizeType, Source2,
ConstantInt::get(LoadSizeType, GEPIndex));
}
Value *LoadSrc1 = Builder.CreateLoad(LoadSizeType, Source1);
Value *LoadSrc2 = Builder.CreateLoad(LoadSizeType, Source2);
LoadSrc1 = Builder.CreateZExt(LoadSrc1, Type::getInt32Ty(CI->getContext()));
LoadSrc2 = Builder.CreateZExt(LoadSrc2, Type::getInt32Ty(CI->getContext()));
Value *Diff = Builder.CreateSub(LoadSrc1, LoadSrc2);
PhiRes->addIncoming(Diff, LoadCmpBlocks[BlockIndex]);
if (BlockIndex < (LoadCmpBlocks.size() - 1)) {
// Early exit branch if difference found to EndBlock. Otherwise, continue to
// next LoadCmpBlock,
Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_NE, Diff,
ConstantInt::get(Diff->getType(), 0));
BranchInst *CmpBr =
BranchInst::Create(EndBlock, LoadCmpBlocks[BlockIndex + 1], Cmp);
Builder.Insert(CmpBr);
} else {
// The last block has an unconditional branch to EndBlock.
BranchInst *CmpBr = BranchInst::Create(EndBlock);
Builder.Insert(CmpBr);
}
}
/// Generate an equality comparison for one or more pairs of loaded values.
/// This is used in the case where the memcmp() call is compared equal or not
/// equal to zero.
Value *MemCmpExpansion::getCompareLoadPairs(unsigned BlockIndex,
unsigned &LoadIndex) {
assert(LoadIndex < getNumLoads() &&
"getCompareLoadPairs() called with no remaining loads");
std::vector<Value *> XorList, OrList;
Value *Diff;
const unsigned NumLoads =
std::min(getNumLoads() - LoadIndex, NumLoadsPerBlock);
// For a single-block expansion, start inserting before the memcmp call.
if (LoadCmpBlocks.empty())
Builder.SetInsertPoint(CI);
else
Builder.SetInsertPoint(LoadCmpBlocks[BlockIndex]);
Value *Cmp = nullptr;
// If we have multiple loads per block, we need to generate a composite
// comparison using xor+or. The type for the combinations is the largest load
// type.
IntegerType *const MaxLoadType =
NumLoads == 1 ? nullptr
: IntegerType::get(CI->getContext(), MaxLoadSize * 8);
for (unsigned i = 0; i < NumLoads; ++i, ++LoadIndex) {
const LoadEntry &CurLoadEntry = LoadSequence[LoadIndex];
IntegerType *LoadSizeType =
IntegerType::get(CI->getContext(), CurLoadEntry.LoadSize * 8);
Value *Source1 = CI->getArgOperand(0);
Value *Source2 = CI->getArgOperand(1);
// Cast source to LoadSizeType*.
if (Source1->getType() != LoadSizeType)
Source1 = Builder.CreateBitCast(Source1, LoadSizeType->getPointerTo());
if (Source2->getType() != LoadSizeType)
Source2 = Builder.CreateBitCast(Source2, LoadSizeType->getPointerTo());
// Get the base address using a GEP.
if (CurLoadEntry.Offset != 0) {
Source1 = Builder.CreateGEP(
LoadSizeType, Source1,
ConstantInt::get(LoadSizeType, CurLoadEntry.getGEPIndex()));
Source2 = Builder.CreateGEP(
LoadSizeType, Source2,
ConstantInt::get(LoadSizeType, CurLoadEntry.getGEPIndex()));
}
// Get a constant or load a value for each source address.
Value *LoadSrc1 = nullptr;
if (auto *Source1C = dyn_cast<Constant>(Source1))
LoadSrc1 = ConstantFoldLoadFromConstPtr(Source1C, LoadSizeType, DL);
if (!LoadSrc1)
LoadSrc1 = Builder.CreateLoad(LoadSizeType, Source1);
Value *LoadSrc2 = nullptr;
if (auto *Source2C = dyn_cast<Constant>(Source2))
LoadSrc2 = ConstantFoldLoadFromConstPtr(Source2C, LoadSizeType, DL);
if (!LoadSrc2)
LoadSrc2 = Builder.CreateLoad(LoadSizeType, Source2);
if (NumLoads != 1) {
if (LoadSizeType != MaxLoadType) {
LoadSrc1 = Builder.CreateZExt(LoadSrc1, MaxLoadType);
LoadSrc2 = Builder.CreateZExt(LoadSrc2, MaxLoadType);
}
// If we have multiple loads per block, we need to generate a composite
// comparison using xor+or.
Diff = Builder.CreateXor(LoadSrc1, LoadSrc2);
Diff = Builder.CreateZExt(Diff, MaxLoadType);
XorList.push_back(Diff);
} else {
// If there's only one load per block, we just compare the loaded values.
Cmp = Builder.CreateICmpNE(LoadSrc1, LoadSrc2);
}
}
auto pairWiseOr = [&](std::vector<Value *> &InList) -> std::vector<Value *> {
std::vector<Value *> OutList;
for (unsigned i = 0; i < InList.size() - 1; i = i + 2) {
Value *Or = Builder.CreateOr(InList[i], InList[i + 1]);
OutList.push_back(Or);
}
if (InList.size() % 2 != 0)
OutList.push_back(InList.back());
return OutList;
};
if (!Cmp) {
// Pairwise OR the XOR results.
OrList = pairWiseOr(XorList);
// Pairwise OR the OR results until one result left.
while (OrList.size() != 1) {
OrList = pairWiseOr(OrList);
}
Cmp = Builder.CreateICmpNE(OrList[0], ConstantInt::get(Diff->getType(), 0));
}
return Cmp;
}
void MemCmpExpansion::emitLoadCompareBlockMultipleLoads(unsigned BlockIndex,
unsigned &LoadIndex) {
Value *Cmp = getCompareLoadPairs(BlockIndex, LoadIndex);
BasicBlock *NextBB = (BlockIndex == (LoadCmpBlocks.size() - 1))
? EndBlock
: LoadCmpBlocks[BlockIndex + 1];
// Early exit branch if difference found to ResultBlock. Otherwise,
// continue to next LoadCmpBlock or EndBlock.
BranchInst *CmpBr = BranchInst::Create(ResBlock.BB, NextBB, Cmp);
Builder.Insert(CmpBr);
// Add a phi edge for the last LoadCmpBlock to Endblock with a value of 0
// since early exit to ResultBlock was not taken (no difference was found in
// any of the bytes).
if (BlockIndex == LoadCmpBlocks.size() - 1) {
Value *Zero = ConstantInt::get(Type::getInt32Ty(CI->getContext()), 0);
PhiRes->addIncoming(Zero, LoadCmpBlocks[BlockIndex]);
}
}
// This function creates the IR intructions for loading and comparing using the
// given LoadSize. It loads the number of bytes specified by LoadSize from each
// source of the memcmp parameters. It then does a subtract to see if there was
// a difference in the loaded values. If a difference is found, it branches
// with an early exit to the ResultBlock for calculating which source was
// larger. Otherwise, it falls through to the either the next LoadCmpBlock or
// the EndBlock if this is the last LoadCmpBlock. Loading 1 byte is handled with
// a special case through emitLoadCompareByteBlock. The special handling can
// simply subtract the loaded values and add it to the result phi node.
void MemCmpExpansion::emitLoadCompareBlock(unsigned BlockIndex) {
// There is one load per block in this case, BlockIndex == LoadIndex.
const LoadEntry &CurLoadEntry = LoadSequence[BlockIndex];
if (CurLoadEntry.LoadSize == 1) {
MemCmpExpansion::emitLoadCompareByteBlock(BlockIndex,
CurLoadEntry.getGEPIndex());
return;
}
Type *LoadSizeType =
IntegerType::get(CI->getContext(), CurLoadEntry.LoadSize * 8);
Type *MaxLoadType = IntegerType::get(CI->getContext(), MaxLoadSize * 8);
assert(CurLoadEntry.LoadSize <= MaxLoadSize && "Unexpected load type");
Value *Source1 = CI->getArgOperand(0);
Value *Source2 = CI->getArgOperand(1);
Builder.SetInsertPoint(LoadCmpBlocks[BlockIndex]);
// Cast source to LoadSizeType*.
if (Source1->getType() != LoadSizeType)
Source1 = Builder.CreateBitCast(Source1, LoadSizeType->getPointerTo());
if (Source2->getType() != LoadSizeType)
Source2 = Builder.CreateBitCast(Source2, LoadSizeType->getPointerTo());
// Get the base address using a GEP.
if (CurLoadEntry.Offset != 0) {
Source1 = Builder.CreateGEP(
LoadSizeType, Source1,
ConstantInt::get(LoadSizeType, CurLoadEntry.getGEPIndex()));
Source2 = Builder.CreateGEP(
LoadSizeType, Source2,
ConstantInt::get(LoadSizeType, CurLoadEntry.getGEPIndex()));
}
// Load LoadSizeType from the base address.
Value *LoadSrc1 = Builder.CreateLoad(LoadSizeType, Source1);
Value *LoadSrc2 = Builder.CreateLoad(LoadSizeType, Source2);
if (DL.isLittleEndian()) {
Function *Bswap = Intrinsic::getDeclaration(CI->getModule(),
Intrinsic::bswap, LoadSizeType);
LoadSrc1 = Builder.CreateCall(Bswap, LoadSrc1);
LoadSrc2 = Builder.CreateCall(Bswap, LoadSrc2);
}
if (LoadSizeType != MaxLoadType) {
LoadSrc1 = Builder.CreateZExt(LoadSrc1, MaxLoadType);
LoadSrc2 = Builder.CreateZExt(LoadSrc2, MaxLoadType);
}
// Add the loaded values to the phi nodes for calculating memcmp result only
// if result is not used in a zero equality.
if (!IsUsedForZeroCmp) {
ResBlock.PhiSrc1->addIncoming(LoadSrc1, LoadCmpBlocks[BlockIndex]);
ResBlock.PhiSrc2->addIncoming(LoadSrc2, LoadCmpBlocks[BlockIndex]);
}
Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, LoadSrc1, LoadSrc2);
BasicBlock *NextBB = (BlockIndex == (LoadCmpBlocks.size() - 1))
? EndBlock
: LoadCmpBlocks[BlockIndex + 1];
// Early exit branch if difference found to ResultBlock. Otherwise, continue
// to next LoadCmpBlock or EndBlock.
BranchInst *CmpBr = BranchInst::Create(NextBB, ResBlock.BB, Cmp);
Builder.Insert(CmpBr);
// Add a phi edge for the last LoadCmpBlock to Endblock with a value of 0
// since early exit to ResultBlock was not taken (no difference was found in
// any of the bytes).
if (BlockIndex == LoadCmpBlocks.size() - 1) {
Value *Zero = ConstantInt::get(Type::getInt32Ty(CI->getContext()), 0);
PhiRes->addIncoming(Zero, LoadCmpBlocks[BlockIndex]);
}
}
// This function populates the ResultBlock with a sequence to calculate the
// memcmp result. It compares the two loaded source values and returns -1 if
// src1 < src2 and 1 if src1 > src2.
void MemCmpExpansion::emitMemCmpResultBlock() {
// Special case: if memcmp result is used in a zero equality, result does not
// need to be calculated and can simply return 1.
if (IsUsedForZeroCmp) {
BasicBlock::iterator InsertPt = ResBlock.BB->getFirstInsertionPt();
Builder.SetInsertPoint(ResBlock.BB, InsertPt);
Value *Res = ConstantInt::get(Type::getInt32Ty(CI->getContext()), 1);
PhiRes->addIncoming(Res, ResBlock.BB);
BranchInst *NewBr = BranchInst::Create(EndBlock);
Builder.Insert(NewBr);
return;
}
BasicBlock::iterator InsertPt = ResBlock.BB->getFirstInsertionPt();
Builder.SetInsertPoint(ResBlock.BB, InsertPt);
Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_ULT, ResBlock.PhiSrc1,
ResBlock.PhiSrc2);
Value *Res =
Builder.CreateSelect(Cmp, ConstantInt::get(Builder.getInt32Ty(), -1),
ConstantInt::get(Builder.getInt32Ty(), 1));
BranchInst *NewBr = BranchInst::Create(EndBlock);
Builder.Insert(NewBr);
PhiRes->addIncoming(Res, ResBlock.BB);
}
void MemCmpExpansion::setupResultBlockPHINodes() {
Type *MaxLoadType = IntegerType::get(CI->getContext(), MaxLoadSize * 8);
Builder.SetInsertPoint(ResBlock.BB);
// Note: this assumes one load per block.
ResBlock.PhiSrc1 =
Builder.CreatePHI(MaxLoadType, NumLoadsNonOneByte, "phi.src1");
ResBlock.PhiSrc2 =
Builder.CreatePHI(MaxLoadType, NumLoadsNonOneByte, "phi.src2");
}
void MemCmpExpansion::setupEndBlockPHINodes() {
Builder.SetInsertPoint(&EndBlock->front());
PhiRes = Builder.CreatePHI(Type::getInt32Ty(CI->getContext()), 2, "phi.res");
}
Value *MemCmpExpansion::getMemCmpExpansionZeroCase() {
unsigned LoadIndex = 0;
// This loop populates each of the LoadCmpBlocks with the IR sequence to
// handle multiple loads per block.
for (unsigned I = 0; I < getNumBlocks(); ++I) {
emitLoadCompareBlockMultipleLoads(I, LoadIndex);
}
emitMemCmpResultBlock();
return PhiRes;
}
/// A memcmp expansion that compares equality with 0 and only has one block of
/// load and compare can bypass the compare, branch, and phi IR that is required
/// in the general case.
Value *MemCmpExpansion::getMemCmpEqZeroOneBlock() {
unsigned LoadIndex = 0;
Value *Cmp = getCompareLoadPairs(0, LoadIndex);
assert(LoadIndex == getNumLoads() && "some entries were not consumed");
return Builder.CreateZExt(Cmp, Type::getInt32Ty(CI->getContext()));
}
/// A memcmp expansion that only has one block of load and compare can bypass
/// the compare, branch, and phi IR that is required in the general case.
Value *MemCmpExpansion::getMemCmpOneBlock() {
assert(NumLoadsPerBlock == 1 && "Only handles one load pair per block");
Type *LoadSizeType = IntegerType::get(CI->getContext(), Size * 8);
Value *Source1 = CI->getArgOperand(0);
Value *Source2 = CI->getArgOperand(1);
// Cast source to LoadSizeType*.
if (Source1->getType() != LoadSizeType)
Source1 = Builder.CreateBitCast(Source1, LoadSizeType->getPointerTo());
if (Source2->getType() != LoadSizeType)
Source2 = Builder.CreateBitCast(Source2, LoadSizeType->getPointerTo());
// Load LoadSizeType from the base address.
Value *LoadSrc1 = Builder.CreateLoad(LoadSizeType, Source1);
Value *LoadSrc2 = Builder.CreateLoad(LoadSizeType, Source2);
if (DL.isLittleEndian() && Size != 1) {
Function *Bswap = Intrinsic::getDeclaration(CI->getModule(),
Intrinsic::bswap, LoadSizeType);
LoadSrc1 = Builder.CreateCall(Bswap, LoadSrc1);
LoadSrc2 = Builder.CreateCall(Bswap, LoadSrc2);
}
if (Size < 4) {
// The i8 and i16 cases don't need compares. We zext the loaded values and
// subtract them to get the suitable negative, zero, or positive i32 result.
LoadSrc1 = Builder.CreateZExt(LoadSrc1, Builder.getInt32Ty());
LoadSrc2 = Builder.CreateZExt(LoadSrc2, Builder.getInt32Ty());
return Builder.CreateSub(LoadSrc1, LoadSrc2);
}
// The result of memcmp is negative, zero, or positive, so produce that by
// subtracting 2 extended compare bits: sub (ugt, ult).
// If a target prefers to use selects to get -1/0/1, they should be able
// to transform this later. The inverse transform (going from selects to math)
// may not be possible in the DAG because the selects got converted into
// branches before we got there.
Value *CmpUGT = Builder.CreateICmpUGT(LoadSrc1, LoadSrc2);
Value *CmpULT = Builder.CreateICmpULT(LoadSrc1, LoadSrc2);
Value *ZextUGT = Builder.CreateZExt(CmpUGT, Builder.getInt32Ty());
Value *ZextULT = Builder.CreateZExt(CmpULT, Builder.getInt32Ty());
return Builder.CreateSub(ZextUGT, ZextULT);
}
// This function expands the memcmp call into an inline expansion and returns
// the memcmp result.
Value *MemCmpExpansion::getMemCmpExpansion() {
// A memcmp with zero-comparison with only one block of load and compare does
// not need to set up any extra blocks. This case could be handled in the DAG,
// but since we have all of the machinery to flexibly expand any memcpy here,
// we choose to handle this case too to avoid fragmented lowering.
if ((!IsUsedForZeroCmp && NumLoadsPerBlock != 1) || getNumBlocks() != 1) {
BasicBlock *StartBlock = CI->getParent();
EndBlock = StartBlock->splitBasicBlock(CI, "endblock");
setupEndBlockPHINodes();
createResultBlock();
// If return value of memcmp is not used in a zero equality, we need to
// calculate which source was larger. The calculation requires the
// two loaded source values of each load compare block.
// These will be saved in the phi nodes created by setupResultBlockPHINodes.
if (!IsUsedForZeroCmp) setupResultBlockPHINodes();
// Create the number of required load compare basic blocks.
createLoadCmpBlocks();
// Update the terminator added by splitBasicBlock to branch to the first
// LoadCmpBlock.
StartBlock->getTerminator()->setSuccessor(0, LoadCmpBlocks[0]);
}
Builder.SetCurrentDebugLocation(CI->getDebugLoc());
if (IsUsedForZeroCmp)
return getNumBlocks() == 1 ? getMemCmpEqZeroOneBlock()
: getMemCmpExpansionZeroCase();
// TODO: Handle more than one load pair per block in getMemCmpOneBlock().
if (getNumBlocks() == 1 && NumLoadsPerBlock == 1) return getMemCmpOneBlock();
for (unsigned I = 0; I < getNumBlocks(); ++I) {
emitLoadCompareBlock(I);
}
emitMemCmpResultBlock();
return PhiRes;
}
// This function checks to see if an expansion of memcmp can be generated.
// It checks for constant compare size that is less than the max inline size.
// If an expansion cannot occur, returns false to leave as a library call.
// Otherwise, the library call is replaced with a new IR instruction sequence.
/// We want to transform:
/// %call = call signext i32 @memcmp(i8* %0, i8* %1, i64 15)
/// To:
/// loadbb:
/// %0 = bitcast i32* %buffer2 to i8*
/// %1 = bitcast i32* %buffer1 to i8*
/// %2 = bitcast i8* %1 to i64*
/// %3 = bitcast i8* %0 to i64*
/// %4 = load i64, i64* %2
/// %5 = load i64, i64* %3
/// %6 = call i64 @llvm.bswap.i64(i64 %4)
/// %7 = call i64 @llvm.bswap.i64(i64 %5)
/// %8 = sub i64 %6, %7
/// %9 = icmp ne i64 %8, 0
/// br i1 %9, label %res_block, label %loadbb1
/// res_block: ; preds = %loadbb2,
/// %loadbb1, %loadbb
/// %phi.src1 = phi i64 [ %6, %loadbb ], [ %22, %loadbb1 ], [ %36, %loadbb2 ]
/// %phi.src2 = phi i64 [ %7, %loadbb ], [ %23, %loadbb1 ], [ %37, %loadbb2 ]
/// %10 = icmp ult i64 %phi.src1, %phi.src2
/// %11 = select i1 %10, i32 -1, i32 1
/// br label %endblock
/// loadbb1: ; preds = %loadbb
/// %12 = bitcast i32* %buffer2 to i8*
/// %13 = bitcast i32* %buffer1 to i8*
/// %14 = bitcast i8* %13 to i32*
/// %15 = bitcast i8* %12 to i32*
/// %16 = getelementptr i32, i32* %14, i32 2
/// %17 = getelementptr i32, i32* %15, i32 2
/// %18 = load i32, i32* %16
/// %19 = load i32, i32* %17
/// %20 = call i32 @llvm.bswap.i32(i32 %18)
/// %21 = call i32 @llvm.bswap.i32(i32 %19)
/// %22 = zext i32 %20 to i64
/// %23 = zext i32 %21 to i64
/// %24 = sub i64 %22, %23
/// %25 = icmp ne i64 %24, 0
/// br i1 %25, label %res_block, label %loadbb2
/// loadbb2: ; preds = %loadbb1
/// %26 = bitcast i32* %buffer2 to i8*
/// %27 = bitcast i32* %buffer1 to i8*
/// %28 = bitcast i8* %27 to i16*
/// %29 = bitcast i8* %26 to i16*
/// %30 = getelementptr i16, i16* %28, i16 6
/// %31 = getelementptr i16, i16* %29, i16 6
/// %32 = load i16, i16* %30
/// %33 = load i16, i16* %31
/// %34 = call i16 @llvm.bswap.i16(i16 %32)
/// %35 = call i16 @llvm.bswap.i16(i16 %33)
/// %36 = zext i16 %34 to i64
/// %37 = zext i16 %35 to i64
/// %38 = sub i64 %36, %37
/// %39 = icmp ne i64 %38, 0
/// br i1 %39, label %res_block, label %loadbb3
/// loadbb3: ; preds = %loadbb2
/// %40 = bitcast i32* %buffer2 to i8*
/// %41 = bitcast i32* %buffer1 to i8*
/// %42 = getelementptr i8, i8* %41, i8 14
/// %43 = getelementptr i8, i8* %40, i8 14
/// %44 = load i8, i8* %42
/// %45 = load i8, i8* %43
/// %46 = zext i8 %44 to i32
/// %47 = zext i8 %45 to i32
/// %48 = sub i32 %46, %47
/// br label %endblock
/// endblock: ; preds = %res_block,
/// %loadbb3
/// %phi.res = phi i32 [ %48, %loadbb3 ], [ %11, %res_block ]
/// ret i32 %phi.res
static bool expandMemCmp(CallInst *CI, const TargetTransformInfo *TTI,
const TargetLowering *TLI, const DataLayout *DL) {
NumMemCmpCalls++;
// Early exit from expansion if -Oz.
if (CI->getFunction()->optForMinSize())
return false;
// Early exit from expansion if size is not a constant.
ConstantInt *SizeCast = dyn_cast<ConstantInt>(CI->getArgOperand(2));
if (!SizeCast) {
NumMemCmpNotConstant++;
return false;
}
const uint64_t SizeVal = SizeCast->getZExtValue();
if (SizeVal == 0) {
return false;
}
// TTI call to check if target would like to expand memcmp. Also, get the
// available load sizes.
const bool IsUsedForZeroCmp = isOnlyUsedInZeroEqualityComparison(CI);
const auto *const Options = TTI->enableMemCmpExpansion(IsUsedForZeroCmp);
if (!Options) return false;
const unsigned MaxNumLoads =
TLI->getMaxExpandSizeMemcmp(CI->getFunction()->optForSize());
MemCmpExpansion Expansion(CI, SizeVal, *Options, MaxNumLoads,
IsUsedForZeroCmp, MemCmpNumLoadsPerBlock, *DL);
// Don't expand if this will require more loads than desired by the target.
if (Expansion.getNumLoads() == 0) {
NumMemCmpGreaterThanMax++;
return false;
}
NumMemCmpInlined++;
Value *Res = Expansion.getMemCmpExpansion();
// Replace call with result of expansion and erase call.
CI->replaceAllUsesWith(Res);
CI->eraseFromParent();
return true;
}
bool CodeGenPrepare::optimizeCallInst(CallInst *CI, bool &ModifiedDT) {
BasicBlock *BB = CI->getParent();
@ -2542,12 +1838,6 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, bool &ModifiedDT) {
return true;
}
LibFunc Func;
if (TLInfo->getLibFunc(ImmutableCallSite(CI), Func) &&
Func == LibFunc_memcmp && expandMemCmp(CI, TTI, TLI, DL)) {
ModifiedDT = true;
return true;
}
return false;
}

10
llvm/lib/CodeGen/TargetPassConfig.cpp
View File

@ -600,8 +600,14 @@ void TargetPassConfig::addIRPasses() {
addPass(createPrintFunctionPass(dbgs(), "\n\n*** Code after LSR ***\n"));
}
if (getOptLevel() != CodeGenOpt::None && EnableMergeICmps) {
addPass(createMergeICmpsPass());
if (getOptLevel() != CodeGenOpt::None) {
// The MergeICmpsPass tries to create memcmp calls by grouping sequences of
// loads and compares. ExpandMemCmpPass then tries to expand those calls
// into optimally-sized loads and compares. The transforms are enabled by a
// target lowering hook.
if (EnableMergeICmps)
addPass(createMergeICmpsPass());
addPass(createExpandMemCmpPass());
}
// Run GC lowering passes for builtin collectors

1
llvm/lib/Transforms/Scalar/CMakeLists.txt
View File

@ -9,6 +9,7 @@ add_llvm_library(LLVMScalarOpts
DeadStoreElimination.cpp
DivRemPairs.cpp
EarlyCSE.cpp
ExpandMemCmp.cpp
FlattenCFGPass.cpp
Float2Int.cpp
GuardWidening.cpp

828
llvm/lib/Transforms/Scalar/ExpandMemCmp.cpp Normal file
View File

@ -0,0 +1,828 @@
//===--- ExpandMemCmp.cpp - Expand memcmp() to load/stores ----------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This pass tries to partially inline the fast path of well-known library
// functions, such as using square-root instructions for cases where sqrt()
// does not need to set errno.
//
//===----------------------------------------------------------------------===//
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/Target/TargetLowering.h"
#include "llvm/Target/TargetSubtargetInfo.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
using namespace llvm;
#define DEBUG_TYPE "expandmemcmp"
STATISTIC(NumMemCmpCalls, "Number of memcmp calls");
STATISTIC(NumMemCmpNotConstant, "Number of memcmp calls without constant size");
STATISTIC(NumMemCmpGreaterThanMax,
"Number of memcmp calls with size greater than max size");
STATISTIC(NumMemCmpInlined, "Number of inlined memcmp calls");
static cl::opt<unsigned> MemCmpNumLoadsPerBlock(
"memcmp-num-loads-per-block", cl::Hidden, cl::init(1),
cl::desc("The number of loads per basic block for inline expansion of "
"memcmp that is only being compared against zero."));
namespace {
// This class provides helper functions to expand a memcmp library call into an
// inline expansion.
class MemCmpExpansion {
struct ResultBlock {
BasicBlock *BB = nullptr;
PHINode *PhiSrc1 = nullptr;
PHINode *PhiSrc2 = nullptr;
ResultBlock() = default;
};
CallInst *const CI;
ResultBlock ResBlock;
const uint64_t Size;
unsigned MaxLoadSize;
uint64_t NumLoadsNonOneByte;
const uint64_t NumLoadsPerBlock;
std::vector<BasicBlock *> LoadCmpBlocks;
BasicBlock *EndBlock;
PHINode *PhiRes;
const bool IsUsedForZeroCmp;
const DataLayout &DL;
IRBuilder<> Builder;
// Represents the decomposition in blocks of the expansion. For example,
// comparing 33 bytes on X86+sse can be done with 2x16-byte loads and
// 1x1-byte load, which would be represented as [{16, 0}, {16, 16}, {32, 1}.
// TODO(courbet): Involve the target more in this computation. On X86, 7
// bytes can be done more efficiently with two overlaping 4-byte loads than
// covering the interval with [{4, 0},{2, 4},{1, 6}}.
struct LoadEntry {
LoadEntry(unsigned LoadSize, uint64_t Offset)
: LoadSize(LoadSize), Offset(Offset) {
assert(Offset % LoadSize == 0 && "invalid load entry");
}
uint64_t getGEPIndex() const { return Offset / LoadSize; }
// The size of the load for this block, in bytes.
const unsigned LoadSize;
// The offset of this load WRT the base pointer, in bytes.
const uint64_t Offset;
};
SmallVector<LoadEntry, 8> LoadSequence;
void createLoadCmpBlocks();
void createResultBlock();
void setupResultBlockPHINodes();
void setupEndBlockPHINodes();
Value *getCompareLoadPairs(unsigned BlockIndex, unsigned &LoadIndex);
void emitLoadCompareBlock(unsigned BlockIndex);
void emitLoadCompareBlockMultipleLoads(unsigned BlockIndex,
unsigned &LoadIndex);
void emitLoadCompareByteBlock(unsigned BlockIndex, unsigned GEPIndex);
void emitMemCmpResultBlock();
Value *getMemCmpExpansionZeroCase();
Value *getMemCmpEqZeroOneBlock();
Value *getMemCmpOneBlock();
public:
MemCmpExpansion(CallInst *CI, uint64_t Size,
const TargetTransformInfo::MemCmpExpansionOptions &Options,
unsigned MaxNumLoads, const bool IsUsedForZeroCmp,
unsigned NumLoadsPerBlock, const DataLayout &DL);
unsigned getNumBlocks();
uint64_t getNumLoads() const { return LoadSequence.size(); }
Value *getMemCmpExpansion();
};
// Initialize the basic block structure required for expansion of memcmp call
// with given maximum load size and memcmp size parameter.
// This structure includes:
// 1. A list of load compare blocks - LoadCmpBlocks.
// 2. An EndBlock, split from original instruction point, which is the block to
// return from.
// 3. ResultBlock, block to branch to for early exit when a
// LoadCmpBlock finds a difference.
MemCmpExpansion::MemCmpExpansion(
CallInst *const CI, uint64_t Size,
const TargetTransformInfo::MemCmpExpansionOptions &Options,
const unsigned MaxNumLoads, const bool IsUsedForZeroCmp,
const unsigned NumLoadsPerBlock, const DataLayout &TheDataLayout)
: CI(CI),
Size(Size),
MaxLoadSize(0),
NumLoadsNonOneByte(0),
NumLoadsPerBlock(NumLoadsPerBlock),
IsUsedForZeroCmp(IsUsedForZeroCmp),
DL(TheDataLayout),
Builder(CI) {
assert(Size > 0 && "zero blocks");
// Scale the max size down if the target can load more bytes than we need.
size_t LoadSizeIndex = 0;
while (LoadSizeIndex < Options.LoadSizes.size() &&
Options.LoadSizes[LoadSizeIndex] > Size) {
++LoadSizeIndex;
}
this->MaxLoadSize = Options.LoadSizes[LoadSizeIndex];
// Compute the decomposition.
uint64_t CurSize = Size;
uint64_t Offset = 0;
while (CurSize && LoadSizeIndex < Options.LoadSizes.size()) {
const unsigned LoadSize = Options.LoadSizes[LoadSizeIndex];
assert(LoadSize > 0 && "zero load size");
const uint64_t NumLoadsForThisSize = CurSize / LoadSize;
if (LoadSequence.size() + NumLoadsForThisSize > MaxNumLoads) {
// Do not expand if the total number of loads is larger than what the
// target allows. Note that it's important that we exit before completing
// the expansion to avoid using a ton of memory to store the expansion for
// large sizes.
LoadSequence.clear();
return;
}
if (NumLoadsForThisSize > 0) {
for (uint64_t I = 0; I < NumLoadsForThisSize; ++I) {
LoadSequence.push_back({LoadSize, Offset});
Offset += LoadSize;
}
if (LoadSize > 1) {
++NumLoadsNonOneByte;
}
CurSize = CurSize % LoadSize;
}
++LoadSizeIndex;
}
assert(LoadSequence.size() <= MaxNumLoads && "broken invariant");
}
unsigned MemCmpExpansion::getNumBlocks() {
if (IsUsedForZeroCmp)
return getNumLoads() / NumLoadsPerBlock +
(getNumLoads() % NumLoadsPerBlock != 0 ? 1 : 0);
return getNumLoads();
}
void MemCmpExpansion::createLoadCmpBlocks() {
for (unsigned i = 0; i < getNumBlocks(); i++) {
BasicBlock *BB = BasicBlock::Create(CI->getContext(), "loadbb",
EndBlock->getParent(), EndBlock);
LoadCmpBlocks.push_back(BB);
}
}
void MemCmpExpansion::createResultBlock() {
ResBlock.BB = BasicBlock::Create(CI->getContext(), "res_block",
EndBlock->getParent(), EndBlock);
}
// This function creates the IR instructions for loading and comparing 1 byte.
// It loads 1 byte from each source of the memcmp parameters with the given
// GEPIndex. It then subtracts the two loaded values and adds this result to the
// final phi node for selecting the memcmp result.
void MemCmpExpansion::emitLoadCompareByteBlock(unsigned BlockIndex,
unsigned GEPIndex) {
Value *Source1 = CI->getArgOperand(0);
Value *Source2 = CI->getArgOperand(1);
Builder.SetInsertPoint(LoadCmpBlocks[BlockIndex]);
Type *LoadSizeType = Type::getInt8Ty(CI->getContext());
// Cast source to LoadSizeType*.
if (Source1->getType() != LoadSizeType)
Source1 = Builder.CreateBitCast(Source1, LoadSizeType->getPointerTo());
if (Source2->getType() != LoadSizeType)
Source2 = Builder.CreateBitCast(Source2, LoadSizeType->getPointerTo());
// Get the base address using the GEPIndex.
if (GEPIndex != 0) {
Source1 = Builder.CreateGEP(LoadSizeType, Source1,
ConstantInt::get(LoadSizeType, GEPIndex));
Source2 = Builder.CreateGEP(LoadSizeType, Source2,
ConstantInt::get(LoadSizeType, GEPIndex));
}
Value *LoadSrc1 = Builder.CreateLoad(LoadSizeType, Source1);
Value *LoadSrc2 = Builder.CreateLoad(LoadSizeType, Source2);
LoadSrc1 = Builder.CreateZExt(LoadSrc1, Type::getInt32Ty(CI->getContext()));
LoadSrc2 = Builder.CreateZExt(LoadSrc2, Type::getInt32Ty(CI->getContext()));
Value *Diff = Builder.CreateSub(LoadSrc1, LoadSrc2);
PhiRes->addIncoming(Diff, LoadCmpBlocks[BlockIndex]);
if (BlockIndex < (LoadCmpBlocks.size() - 1)) {
// Early exit branch if difference found to EndBlock. Otherwise, continue to
// next LoadCmpBlock,
Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_NE, Diff,
ConstantInt::get(Diff->getType(), 0));
BranchInst *CmpBr =
BranchInst::Create(EndBlock, LoadCmpBlocks[BlockIndex + 1], Cmp);
Builder.Insert(CmpBr);
} else {
// The last block has an unconditional branch to EndBlock.
BranchInst *CmpBr = BranchInst::Create(EndBlock);
Builder.Insert(CmpBr);
}
}
/// Generate an equality comparison for one or more pairs of loaded values.
/// This is used in the case where the memcmp() call is compared equal or not
/// equal to zero.
Value *MemCmpExpansion::getCompareLoadPairs(unsigned BlockIndex,
unsigned &LoadIndex) {
assert(LoadIndex < getNumLoads() &&
"getCompareLoadPairs() called with no remaining loads");
std::vector<Value *> XorList, OrList;
Value *Diff;
const unsigned NumLoads =
std::min(getNumLoads() - LoadIndex, NumLoadsPerBlock);
// For a single-block expansion, start inserting before the memcmp call.
if (LoadCmpBlocks.empty())
Builder.SetInsertPoint(CI);
else
Builder.SetInsertPoint(LoadCmpBlocks[BlockIndex]);
Value *Cmp = nullptr;
// If we have multiple loads per block, we need to generate a composite
// comparison using xor+or. The type for the combinations is the largest load
// type.
IntegerType *const MaxLoadType =
NumLoads == 1 ? nullptr
: IntegerType::get(CI->getContext(), MaxLoadSize * 8);
for (unsigned i = 0; i < NumLoads; ++i, ++LoadIndex) {
const LoadEntry &CurLoadEntry = LoadSequence[LoadIndex];
IntegerType *LoadSizeType =
IntegerType::get(CI->getContext(), CurLoadEntry.LoadSize * 8);
Value *Source1 = CI->getArgOperand(0);
Value *Source2 = CI->getArgOperand(1);
// Cast source to LoadSizeType*.
if (Source1->getType() != LoadSizeType)
Source1 = Builder.CreateBitCast(Source1, LoadSizeType->getPointerTo());
if (Source2->getType() != LoadSizeType)
Source2 = Builder.CreateBitCast(Source2, LoadSizeType->getPointerTo());
// Get the base address using a GEP.
if (CurLoadEntry.Offset != 0) {
Source1 = Builder.CreateGEP(
LoadSizeType, Source1,
ConstantInt::get(LoadSizeType, CurLoadEntry.getGEPIndex()));
Source2 = Builder.CreateGEP(
LoadSizeType, Source2,
ConstantInt::get(LoadSizeType, CurLoadEntry.getGEPIndex()));
}
// Get a constant or load a value for each source address.
Value *LoadSrc1 = nullptr;
if (auto *Source1C = dyn_cast<Constant>(Source1))
LoadSrc1 = ConstantFoldLoadFromConstPtr(Source1C, LoadSizeType, DL);
if (!LoadSrc1)
LoadSrc1 = Builder.CreateLoad(LoadSizeType, Source1);
Value *LoadSrc2 = nullptr;
if (auto *Source2C = dyn_cast<Constant>(Source2))
LoadSrc2 = ConstantFoldLoadFromConstPtr(Source2C, LoadSizeType, DL);
if (!LoadSrc2)
LoadSrc2 = Builder.CreateLoad(LoadSizeType, Source2);
if (NumLoads != 1) {
if (LoadSizeType != MaxLoadType) {
LoadSrc1 = Builder.CreateZExt(LoadSrc1, MaxLoadType);
LoadSrc2 = Builder.CreateZExt(LoadSrc2, MaxLoadType);
}
// If we have multiple loads per block, we need to generate a composite
// comparison using xor+or.
Diff = Builder.CreateXor(LoadSrc1, LoadSrc2);
Diff = Builder.CreateZExt(Diff, MaxLoadType);
XorList.push_back(Diff);
} else {
// If there's only one load per block, we just compare the loaded values.
Cmp = Builder.CreateICmpNE(LoadSrc1, LoadSrc2);
}
}
auto pairWiseOr = [&](std::vector<Value *> &InList) -> std::vector<Value *> {
std::vector<Value *> OutList;
for (unsigned i = 0; i < InList.size() - 1; i = i + 2) {
Value *Or = Builder.CreateOr(InList[i], InList[i + 1]);
OutList.push_back(Or);
}
if (InList.size() % 2 != 0)
OutList.push_back(InList.back());
return OutList;
};
if (!Cmp) {
// Pairwise OR the XOR results.
OrList = pairWiseOr(XorList);
// Pairwise OR the OR results until one result left.
while (OrList.size() != 1) {
OrList = pairWiseOr(OrList);
}
Cmp = Builder.CreateICmpNE(OrList[0], ConstantInt::get(Diff->getType(), 0));
}
return Cmp;
}
void MemCmpExpansion::emitLoadCompareBlockMultipleLoads(unsigned BlockIndex,
unsigned &LoadIndex) {
Value *Cmp = getCompareLoadPairs(BlockIndex, LoadIndex);
BasicBlock *NextBB = (BlockIndex == (LoadCmpBlocks.size() - 1))
? EndBlock
: LoadCmpBlocks[BlockIndex + 1];
// Early exit branch if difference found to ResultBlock. Otherwise,
// continue to next LoadCmpBlock or EndBlock.
BranchInst *CmpBr = BranchInst::Create(ResBlock.BB, NextBB, Cmp);
Builder.Insert(CmpBr);
// Add a phi edge for the last LoadCmpBlock to Endblock with a value of 0
// since early exit to ResultBlock was not taken (no difference was found in
// any of the bytes).
if (BlockIndex == LoadCmpBlocks.size() - 1) {
Value *Zero = ConstantInt::get(Type::getInt32Ty(CI->getContext()), 0);
PhiRes->addIncoming(Zero, LoadCmpBlocks[BlockIndex]);
}
}
// This function creates the IR intructions for loading and comparing using the
// given LoadSize. It loads the number of bytes specified by LoadSize from each
// source of the memcmp parameters. It then does a subtract to see if there was
// a difference in the loaded values. If a difference is found, it branches
// with an early exit to the ResultBlock for calculating which source was
// larger. Otherwise, it falls through to the either the next LoadCmpBlock or
// the EndBlock if this is the last LoadCmpBlock. Loading 1 byte is handled with
// a special case through emitLoadCompareByteBlock. The special handling can
// simply subtract the loaded values and add it to the result phi node.
void MemCmpExpansion::emitLoadCompareBlock(unsigned BlockIndex) {
// There is one load per block in this case, BlockIndex == LoadIndex.
const LoadEntry &CurLoadEntry = LoadSequence[BlockIndex];
if (CurLoadEntry.LoadSize == 1) {
MemCmpExpansion::emitLoadCompareByteBlock(BlockIndex,
CurLoadEntry.getGEPIndex());
return;
}
Type *LoadSizeType =
IntegerType::get(CI->getContext(), CurLoadEntry.LoadSize * 8);
Type *MaxLoadType = IntegerType::get(CI->getContext(), MaxLoadSize * 8);
assert(CurLoadEntry.LoadSize <= MaxLoadSize && "Unexpected load type");
Value *Source1 = CI->getArgOperand(0);
Value *Source2 = CI->getArgOperand(1);
Builder.SetInsertPoint(LoadCmpBlocks[BlockIndex]);
// Cast source to LoadSizeType*.
if (Source1->getType() != LoadSizeType)
Source1 = Builder.CreateBitCast(Source1, LoadSizeType->getPointerTo());
if (Source2->getType() != LoadSizeType)
Source2 = Builder.CreateBitCast(Source2, LoadSizeType->getPointerTo());
// Get the base address using a GEP.
if (CurLoadEntry.Offset != 0) {
Source1 = Builder.CreateGEP(
LoadSizeType, Source1,
ConstantInt::get(LoadSizeType, CurLoadEntry.getGEPIndex()));
Source2 = Builder.CreateGEP(
LoadSizeType, Source2,
ConstantInt::get(LoadSizeType, CurLoadEntry.getGEPIndex()));
}
// Load LoadSizeType from the base address.
Value *LoadSrc1 = Builder.CreateLoad(LoadSizeType, Source1);
Value *LoadSrc2 = Builder.CreateLoad(LoadSizeType, Source2);
if (DL.isLittleEndian()) {
Function *Bswap = Intrinsic::getDeclaration(CI->getModule(),
Intrinsic::bswap, LoadSizeType);
LoadSrc1 = Builder.CreateCall(Bswap, LoadSrc1);
LoadSrc2 = Builder.CreateCall(Bswap, LoadSrc2);
}
if (LoadSizeType != MaxLoadType) {
LoadSrc1 = Builder.CreateZExt(LoadSrc1, MaxLoadType);
LoadSrc2 = Builder.CreateZExt(LoadSrc2, MaxLoadType);
}
// Add the loaded values to the phi nodes for calculating memcmp result only
// if result is not used in a zero equality.
if (!IsUsedForZeroCmp) {
ResBlock.PhiSrc1->addIncoming(LoadSrc1, LoadCmpBlocks[BlockIndex]);
ResBlock.PhiSrc2->addIncoming(LoadSrc2, LoadCmpBlocks[BlockIndex]);
}
Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, LoadSrc1, LoadSrc2);
BasicBlock *NextBB = (BlockIndex == (LoadCmpBlocks.size() - 1))
? EndBlock
: LoadCmpBlocks[BlockIndex + 1];
// Early exit branch if difference found to ResultBlock. Otherwise, continue
// to next LoadCmpBlock or EndBlock.
BranchInst *CmpBr = BranchInst::Create(NextBB, ResBlock.BB, Cmp);
Builder.Insert(CmpBr);
// Add a phi edge for the last LoadCmpBlock to Endblock with a value of 0
// since early exit to ResultBlock was not taken (no difference was found in
// any of the bytes).
if (BlockIndex == LoadCmpBlocks.size() - 1) {
Value *Zero = ConstantInt::get(Type::getInt32Ty(CI->getContext()), 0);
PhiRes->addIncoming(Zero, LoadCmpBlocks[BlockIndex]);
}
}
// This function populates the ResultBlock with a sequence to calculate the
// memcmp result. It compares the two loaded source values and returns -1 if
// src1 < src2 and 1 if src1 > src2.
void MemCmpExpansion::emitMemCmpResultBlock() {
// Special case: if memcmp result is used in a zero equality, result does not
// need to be calculated and can simply return 1.
if (IsUsedForZeroCmp) {
BasicBlock::iterator InsertPt = ResBlock.BB->getFirstInsertionPt();
Builder.SetInsertPoint(ResBlock.BB, InsertPt);
Value *Res = ConstantInt::get(Type::getInt32Ty(CI->getContext()), 1);
PhiRes->addIncoming(Res, ResBlock.BB);
BranchInst *NewBr = BranchInst::Create(EndBlock);
Builder.Insert(NewBr);
return;
}
BasicBlock::iterator InsertPt = ResBlock.BB->getFirstInsertionPt();
Builder.SetInsertPoint(ResBlock.BB, InsertPt);
Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_ULT, ResBlock.PhiSrc1,
ResBlock.PhiSrc2);
Value *Res =
Builder.CreateSelect(Cmp, ConstantInt::get(Builder.getInt32Ty(), -1),
ConstantInt::get(Builder.getInt32Ty(), 1));
BranchInst *NewBr = BranchInst::Create(EndBlock);
Builder.Insert(NewBr);
PhiRes->addIncoming(Res, ResBlock.BB);
}
void MemCmpExpansion::setupResultBlockPHINodes() {
Type *MaxLoadType = IntegerType::get(CI->getContext(), MaxLoadSize * 8);
Builder.SetInsertPoint(ResBlock.BB);
// Note: this assumes one load per block.
ResBlock.PhiSrc1 =
Builder.CreatePHI(MaxLoadType, NumLoadsNonOneByte, "phi.src1");
ResBlock.PhiSrc2 =
Builder.CreatePHI(MaxLoadType, NumLoadsNonOneByte, "phi.src2");
}
void MemCmpExpansion::setupEndBlockPHINodes() {
Builder.SetInsertPoint(&EndBlock->front());
PhiRes = Builder.CreatePHI(Type::getInt32Ty(CI->getContext()), 2, "phi.res");
}
Value *MemCmpExpansion::getMemCmpExpansionZeroCase() {
unsigned LoadIndex = 0;
// This loop populates each of the LoadCmpBlocks with the IR sequence to
// handle multiple loads per block.
for (unsigned I = 0; I < getNumBlocks(); ++I) {
emitLoadCompareBlockMultipleLoads(I, LoadIndex);
}
emitMemCmpResultBlock();
return PhiRes;
}
/// A memcmp expansion that compares equality with 0 and only has one block of
/// load and compare can bypass the compare, branch, and phi IR that is required
/// in the general case.
Value *MemCmpExpansion::getMemCmpEqZeroOneBlock() {
unsigned LoadIndex = 0;
Value *Cmp = getCompareLoadPairs(0, LoadIndex);
assert(LoadIndex == getNumLoads() && "some entries were not consumed");
return Builder.CreateZExt(Cmp, Type::getInt32Ty(CI->getContext()));
}
/// A memcmp expansion that only has one block of load and compare can bypass
/// the compare, branch, and phi IR that is required in the general case.
Value *MemCmpExpansion::getMemCmpOneBlock() {
assert(NumLoadsPerBlock == 1 && "Only handles one load pair per block");
Type *LoadSizeType = IntegerType::get(CI->getContext(), Size * 8);
Value *Source1 = CI->getArgOperand(0);
Value *Source2 = CI->getArgOperand(1);
// Cast source to LoadSizeType*.
if (Source1->getType() != LoadSizeType)
Source1 = Builder.CreateBitCast(Source1, LoadSizeType->getPointerTo());
if (Source2->getType() != LoadSizeType)
Source2 = Builder.CreateBitCast(Source2, LoadSizeType->getPointerTo());
// Load LoadSizeType from the base address.
Value *LoadSrc1 = Builder.CreateLoad(LoadSizeType, Source1);
Value *LoadSrc2 = Builder.CreateLoad(LoadSizeType, Source2);
if (DL.isLittleEndian() && Size != 1) {
Function *Bswap = Intrinsic::getDeclaration(CI->getModule(),
Intrinsic::bswap, LoadSizeType);
LoadSrc1 = Builder.CreateCall(Bswap, LoadSrc1);
LoadSrc2 = Builder.CreateCall(Bswap, LoadSrc2);
}
if (Size < 4) {
// The i8 and i16 cases don't need compares. We zext the loaded values and
// subtract them to get the suitable negative, zero, or positive i32 result.
LoadSrc1 = Builder.CreateZExt(LoadSrc1, Builder.getInt32Ty());
LoadSrc2 = Builder.CreateZExt(LoadSrc2, Builder.getInt32Ty());
return Builder.CreateSub(LoadSrc1, LoadSrc2);
}
// The result of memcmp is negative, zero, or positive, so produce that by
// subtracting 2 extended compare bits: sub (ugt, ult).
// If a target prefers to use selects to get -1/0/1, they should be able
// to transform this later. The inverse transform (going from selects to math)
// may not be possible in the DAG because the selects got converted into
// branches before we got there.
Value *CmpUGT = Builder.CreateICmpUGT(LoadSrc1, LoadSrc2);
Value *CmpULT = Builder.CreateICmpULT(LoadSrc1, LoadSrc2);
Value *ZextUGT = Builder.CreateZExt(CmpUGT, Builder.getInt32Ty());
Value *ZextULT = Builder.CreateZExt(CmpULT, Builder.getInt32Ty());
return Builder.CreateSub(ZextUGT, ZextULT);
}
// This function expands the memcmp call into an inline expansion and returns
// the memcmp result.
Value *MemCmpExpansion::getMemCmpExpansion() {
// A memcmp with zero-comparison with only one block of load and compare does
// not need to set up any extra blocks. This case could be handled in the DAG,
// but since we have all of the machinery to flexibly expand any memcpy here,
// we choose to handle this case too to avoid fragmented lowering.
if ((!IsUsedForZeroCmp && NumLoadsPerBlock != 1) || getNumBlocks() != 1) {
BasicBlock *StartBlock = CI->getParent();
EndBlock = StartBlock->splitBasicBlock(CI, "endblock");
setupEndBlockPHINodes();
createResultBlock();
// If return value of memcmp is not used in a zero equality, we need to
// calculate which source was larger. The calculation requires the
// two loaded source values of each load compare block.
// These will be saved in the phi nodes created by setupResultBlockPHINodes.
if (!IsUsedForZeroCmp) setupResultBlockPHINodes();
// Create the number of required load compare basic blocks.
createLoadCmpBlocks();
// Update the terminator added by splitBasicBlock to branch to the first
// LoadCmpBlock.
StartBlock->getTerminator()->setSuccessor(0, LoadCmpBlocks[0]);
}
Builder.SetCurrentDebugLocation(CI->getDebugLoc());
if (IsUsedForZeroCmp)
return getNumBlocks() == 1 ? getMemCmpEqZeroOneBlock()
: getMemCmpExpansionZeroCase();
// TODO: Handle more than one load pair per block in getMemCmpOneBlock().
if (getNumBlocks() == 1 && NumLoadsPerBlock == 1) return getMemCmpOneBlock();
for (unsigned I = 0; I < getNumBlocks(); ++I) {
emitLoadCompareBlock(I);
}
emitMemCmpResultBlock();
return PhiRes;
}
// This function checks to see if an expansion of memcmp can be generated.
// It checks for constant compare size that is less than the max inline size.
// If an expansion cannot occur, returns false to leave as a library call.
// Otherwise, the library call is replaced with a new IR instruction sequence.
/// We want to transform:
/// %call = call signext i32 @memcmp(i8* %0, i8* %1, i64 15)
/// To:
/// loadbb:
/// %0 = bitcast i32* %buffer2 to i8*
/// %1 = bitcast i32* %buffer1 to i8*
/// %2 = bitcast i8* %1 to i64*
/// %3 = bitcast i8* %0 to i64*
/// %4 = load i64, i64* %2
/// %5 = load i64, i64* %3
/// %6 = call i64 @llvm.bswap.i64(i64 %4)
/// %7 = call i64 @llvm.bswap.i64(i64 %5)
/// %8 = sub i64 %6, %7
/// %9 = icmp ne i64 %8, 0
/// br i1 %9, label %res_block, label %loadbb1
/// res_block: ; preds = %loadbb2,
/// %loadbb1, %loadbb
/// %phi.src1 = phi i64 [ %6, %loadbb ], [ %22, %loadbb1 ], [ %36, %loadbb2 ]
/// %phi.src2 = phi i64 [ %7, %loadbb ], [ %23, %loadbb1 ], [ %37, %loadbb2 ]
/// %10 = icmp ult i64 %phi.src1, %phi.src2
/// %11 = select i1 %10, i32 -1, i32 1
/// br label %endblock
/// loadbb1: ; preds = %loadbb
/// %12 = bitcast i32* %buffer2 to i8*
/// %13 = bitcast i32* %buffer1 to i8*
/// %14 = bitcast i8* %13 to i32*
/// %15 = bitcast i8* %12 to i32*
/// %16 = getelementptr i32, i32* %14, i32 2
/// %17 = getelementptr i32, i32* %15, i32 2
/// %18 = load i32, i32* %16
/// %19 = load i32, i32* %17
/// %20 = call i32 @llvm.bswap.i32(i32 %18)
/// %21 = call i32 @llvm.bswap.i32(i32 %19)
/// %22 = zext i32 %20 to i64
/// %23 = zext i32 %21 to i64
/// %24 = sub i64 %22, %23
/// %25 = icmp ne i64 %24, 0
/// br i1 %25, label %res_block, label %loadbb2
/// loadbb2: ; preds = %loadbb1
/// %26 = bitcast i32* %buffer2 to i8*
/// %27 = bitcast i32* %buffer1 to i8*
/// %28 = bitcast i8* %27 to i16*
/// %29 = bitcast i8* %26 to i16*
/// %30 = getelementptr i16, i16* %28, i16 6
/// %31 = getelementptr i16, i16* %29, i16 6
/// %32 = load i16, i16* %30
/// %33 = load i16, i16* %31
/// %34 = call i16 @llvm.bswap.i16(i16 %32)
/// %35 = call i16 @llvm.bswap.i16(i16 %33)
/// %36 = zext i16 %34 to i64
/// %37 = zext i16 %35 to i64
/// %38 = sub i64 %36, %37
/// %39 = icmp ne i64 %38, 0
/// br i1 %39, label %res_block, label %loadbb3
/// loadbb3: ; preds = %loadbb2
/// %40 = bitcast i32* %buffer2 to i8*
/// %41 = bitcast i32* %buffer1 to i8*
/// %42 = getelementptr i8, i8* %41, i8 14
/// %43 = getelementptr i8, i8* %40, i8 14
/// %44 = load i8, i8* %42
/// %45 = load i8, i8* %43
/// %46 = zext i8 %44 to i32
/// %47 = zext i8 %45 to i32
/// %48 = sub i32 %46, %47
/// br label %endblock
/// endblock: ; preds = %res_block,
/// %loadbb3
/// %phi.res = phi i32 [ %48, %loadbb3 ], [ %11, %res_block ]
/// ret i32 %phi.res
static bool expandMemCmp(CallInst *CI, const TargetTransformInfo *TTI,
const TargetLowering *TLI, const DataLayout *DL) {
NumMemCmpCalls++;
// Early exit from expansion if -Oz.
if (CI->getFunction()->optForMinSize())
return false;
// Early exit from expansion if size is not a constant.
ConstantInt *SizeCast = dyn_cast<ConstantInt>(CI->getArgOperand(2));
if (!SizeCast) {
NumMemCmpNotConstant++;
return false;
}
const uint64_t SizeVal = SizeCast->getZExtValue();
if (SizeVal == 0) {
return false;
}
// TTI call to check if target would like to expand memcmp. Also, get the
// available load sizes.
const bool IsUsedForZeroCmp = isOnlyUsedInZeroEqualityComparison(CI);
const auto *const Options = TTI->enableMemCmpExpansion(IsUsedForZeroCmp);
if (!Options) return false;
const unsigned MaxNumLoads =
TLI->getMaxExpandSizeMemcmp(CI->getFunction()->optForSize());
MemCmpExpansion Expansion(CI, SizeVal, *Options, MaxNumLoads,
IsUsedForZeroCmp, MemCmpNumLoadsPerBlock, *DL);
// Don't expand if this will require more loads than desired by the target.
if (Expansion.getNumLoads() == 0) {
NumMemCmpGreaterThanMax++;
return false;
}
NumMemCmpInlined++;
Value *Res = Expansion.getMemCmpExpansion();
// Replace call with result of expansion and erase call.
CI->replaceAllUsesWith(Res);
CI->eraseFromParent();
return true;
}
class ExpandMemCmpPass : public FunctionPass {
public:
static char ID;
ExpandMemCmpPass() : FunctionPass(ID) {
initializeExpandMemCmpPassPass(*PassRegistry::getPassRegistry());
}
bool runOnFunction(Function &F) override {
if (skipFunction(F)) return false;
auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
if (!TPC) {
return false;
}
const TargetLowering* TL =
TPC->getTM<TargetMachine>().getSubtargetImpl(F)->getTargetLowering();
const TargetLibraryInfo *TLI =
&getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
const TargetTransformInfo *TTI =
&getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
auto PA = runImpl(F, TLI, TTI, TL);
return !PA.areAllPreserved();
}
private:
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<TargetLibraryInfoWrapperPass>();
AU.addRequired<TargetTransformInfoWrapperPass>();
FunctionPass::getAnalysisUsage(AU);
}
PreservedAnalyses runImpl(Function &F, const TargetLibraryInfo *TLI,
const TargetTransformInfo *TTI,
const TargetLowering* TL);
// Returns true if a change was made.
bool runOnBlock(BasicBlock &BB, const TargetLibraryInfo *TLI,
const TargetTransformInfo *TTI, const TargetLowering* TL,
const DataLayout& DL);
};
bool ExpandMemCmpPass::runOnBlock(
BasicBlock &BB, const TargetLibraryInfo *TLI,
const TargetTransformInfo *TTI, const TargetLowering* TL,
const DataLayout& DL) {
for (Instruction& I : BB) {
CallInst *CI = dyn_cast<CallInst>(&I);
if (!CI) {
continue;
}
LibFunc Func;
if (TLI->getLibFunc(ImmutableCallSite(CI), Func) &&
Func == LibFunc_memcmp && expandMemCmp(CI, TTI, TL, &DL)) {
return true;
}
}
return false;
}
PreservedAnalyses ExpandMemCmpPass::runImpl(
Function &F, const TargetLibraryInfo *TLI, const TargetTransformInfo *TTI,
const TargetLowering* TL) {
const DataLayout& DL = F.getParent()->getDataLayout();
bool MadeChanges = false;
for (auto BBIt = F.begin(); BBIt != F.end();) {
if (runOnBlock(*BBIt, TLI, TTI, TL, DL)) {
MadeChanges = true;
// If changes were made, restart the function from the beginning, since
// the structure of the function was changed.
BBIt = F.begin();
} else {
++BBIt;
}
}
return MadeChanges ? PreservedAnalyses::none() : PreservedAnalyses::all();
}
} // namespace
char ExpandMemCmpPass::ID = 0;
INITIALIZE_PASS_BEGIN(ExpandMemCmpPass, "expandmemcmp",
"Expand memcmp() to load/stores", false, false)
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
INITIALIZE_PASS_END(ExpandMemCmpPass, "expandmemcmp",
"Expand memcmp() to load/stores", false, false)
Pass *llvm::createExpandMemCmpPass() {
return new ExpandMemCmpPass();
}

1
llvm/lib/Transforms/Scalar/Scalar.cpp
View File

@ -48,6 +48,7 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
initializeNewGVNLegacyPassPass(Registry);
initializeEarlyCSELegacyPassPass(Registry);
initializeEarlyCSEMemSSALegacyPassPass(Registry);
initializeExpandMemCmpPassPass(Registry);
initializeGVNHoistLegacyPassPass(Registry);
initializeGVNSinkLegacyPassPass(Registry);
initializeFlattenCFGPassPass(Registry);

6
llvm/test/CodeGen/Generic/llc-start-stop.ll
View File

@ -13,15 +13,15 @@
; STOP-BEFORE-NOT: Loop Strength Reduction
; RUN: llc < %s -debug-pass=Structure -start-after=loop-reduce -o /dev/null 2>&1 | FileCheck %s -check-prefix=START-AFTER
; START-AFTER: -machine-branch-prob -gc-lowering
; START-AFTER: -machine-branch-prob -expandmemcmp
; START-AFTER: FunctionPass Manager
; START-AFTER-NEXT: Lower Garbage Collection Instructions
; START-AFTER-NEXT: Expand memcmp() to load/stores
; RUN: llc < %s -debug-pass=Structure -start-before=loop-reduce -o /dev/null 2>&1 | FileCheck %s -check-prefix=START-BEFORE
; START-BEFORE: -machine-branch-prob -domtree
; START-BEFORE: FunctionPass Manager
; START-BEFORE: Loop Strength Reduction
; START-BEFORE-NEXT: Lower Garbage Collection Instructions
; START-BEFORE-NEXT: Expand memcmp() to load/stores
; RUN: not llc < %s -start-before=nonexistent -o /dev/null 2>&1 | FileCheck %s -check-prefix=NONEXISTENT-START-BEFORE
; RUN: not llc < %s -stop-before=nonexistent -o /dev/null 2>&1 | FileCheck %s -check-prefix=NONEXISTENT-STOP-BEFORE

224
llvm/test/CodeGen/X86/memcmp-optsize.ll
View File

@ -156,36 +156,36 @@ define i32 @length3(i8* %X, i8* %Y) nounwind optsize {
define i1 @length3_eq(i8* %X, i8* %Y) nounwind optsize {
; X86-LABEL: length3_eq:
; X86: # BB#0: # %loadbb
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86: # BB#0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzwl (%eax), %edx
; X86-NEXT: cmpw (%ecx), %dx
; X86-NEXT: jne .LBB5_1
; X86-NEXT: # BB#2: # %loadbb1
; X86-NEXT: movb 2(%eax), %dl
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: cmpb 2(%ecx), %dl
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movzwl (%ecx), %edx
; X86-NEXT: cmpw (%eax), %dx
; X86-NEXT: jne .LBB5_2
; X86-NEXT: # BB#1: # %loadbb1
; X86-NEXT: movb 2(%ecx), %dl
; X86-NEXT: xorl %ecx, %ecx
; X86-NEXT: cmpb 2(%eax), %dl
; X86-NEXT: je .LBB5_3
; X86-NEXT: .LBB5_1: # %res_block
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: incl %eax
; X86-NEXT: .LBB5_2: # %res_block
; X86-NEXT: xorl %ecx, %ecx
; X86-NEXT: incl %ecx
; X86-NEXT: .LBB5_3: # %endblock
; X86-NEXT: testl %eax, %eax
; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: setne %al
; X86-NEXT: retl
;
; X64-LABEL: length3_eq:
; X64: # BB#0: # %loadbb
; X64: # BB#0:
; X64-NEXT: movzwl (%rdi), %eax
; X64-NEXT: cmpw (%rsi), %ax
; X64-NEXT: jne .LBB5_1
; X64-NEXT: # BB#2: # %loadbb1
; X64-NEXT: jne .LBB5_2
; X64-NEXT: # BB#1: # %loadbb1
; X64-NEXT: movb 2(%rdi), %cl
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpb 2(%rsi), %cl
; X64-NEXT: je .LBB5_3
; X64-NEXT: .LBB5_1: # %res_block
; X64-NEXT: .LBB5_2: # %res_block
; X64-NEXT: movl $1, %eax
; X64-NEXT: .LBB5_3: # %endblock
; X64-NEXT: testl %eax, %eax
@ -314,36 +314,36 @@ define i32 @length5(i8* %X, i8* %Y) nounwind optsize {
define i1 @length5_eq(i8* %X, i8* %Y) nounwind optsize {
; X86-LABEL: length5_eq:
; X86: # BB#0: # %loadbb
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86: # BB#0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl (%eax), %edx
; X86-NEXT: cmpl (%ecx), %edx
; X86-NEXT: jne .LBB10_1
; X86-NEXT: # BB#2: # %loadbb1
; X86-NEXT: movb 4(%eax), %dl
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: cmpb 4(%ecx), %dl
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl (%ecx), %edx
; X86-NEXT: cmpl (%eax), %edx
; X86-NEXT: jne .LBB10_2
; X86-NEXT: # BB#1: # %loadbb1
; X86-NEXT: movb 4(%ecx), %dl
; X86-NEXT: xorl %ecx, %ecx
; X86-NEXT: cmpb 4(%eax), %dl
; X86-NEXT: je .LBB10_3
; X86-NEXT: .LBB10_1: # %res_block
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: incl %eax
; X86-NEXT: .LBB10_2: # %res_block
; X86-NEXT: xorl %ecx, %ecx
; X86-NEXT: incl %ecx
; X86-NEXT: .LBB10_3: # %endblock
; X86-NEXT: testl %eax, %eax
; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: setne %al
; X86-NEXT: retl
;
; X64-LABEL: length5_eq:
; X64: # BB#0: # %loadbb
; X64: # BB#0:
; X64-NEXT: movl (%rdi), %eax
; X64-NEXT: cmpl (%rsi), %eax
; X64-NEXT: jne .LBB10_1
; X64-NEXT: # BB#2: # %loadbb1
; X64-NEXT: jne .LBB10_2
; X64-NEXT: # BB#1: # %loadbb1
; X64-NEXT: movb 4(%rdi), %cl
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpb 4(%rsi), %cl
; X64-NEXT: je .LBB10_3
; X64-NEXT: .LBB10_1: # %res_block
; X64-NEXT: .LBB10_2: # %res_block
; X64-NEXT: movl $1, %eax
; X64-NEXT: .LBB10_3: # %endblock
; X64-NEXT: testl %eax, %eax
@ -356,7 +356,7 @@ define i1 @length5_eq(i8* %X, i8* %Y) nounwind optsize {
define i32 @length8(i8* %X, i8* %Y) nounwind optsize {
; X86-LABEL: length8:
; X86: # BB#0: # %loadbb
; X86: # BB#0:
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
@ -365,8 +365,8 @@ define i32 @length8(i8* %X, i8* %Y) nounwind optsize {
; X86-NEXT: bswapl %ecx
; X86-NEXT: bswapl %edx
; X86-NEXT: cmpl %edx, %ecx
; X86-NEXT: jne .LBB11_1
; X86-NEXT: # BB#2: # %loadbb1
; X86-NEXT: jne .LBB11_2
; X86-NEXT: # BB#1: # %loadbb1
; X86-NEXT: movl 4(%esi), %ecx
; X86-NEXT: movl 4(%eax), %edx
; X86-NEXT: bswapl %ecx
@ -374,7 +374,7 @@ define i32 @length8(i8* %X, i8* %Y) nounwind optsize {
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: cmpl %edx, %ecx
; X86-NEXT: je .LBB11_3
; X86-NEXT: .LBB11_1: # %res_block
; X86-NEXT: .LBB11_2: # %res_block
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: cmpl %edx, %ecx
; X86-NEXT: setae %al
@ -400,22 +400,22 @@ define i32 @length8(i8* %X, i8* %Y) nounwind optsize {
define i1 @length8_eq(i8* %X, i8* %Y) nounwind optsize {
; X86-LABEL: length8_eq:
; X86: # BB#0: # %loadbb
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86: # BB#0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl (%eax), %edx
; X86-NEXT: cmpl (%ecx), %edx
; X86-NEXT: jne .LBB12_1
; X86-NEXT: # BB#2: # %loadbb1
; X86-NEXT: movl 4(%eax), %edx
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: cmpl 4(%ecx), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl (%ecx), %edx
; X86-NEXT: cmpl (%eax), %edx
; X86-NEXT: jne .LBB12_2
; X86-NEXT: # BB#1: # %loadbb1
; X86-NEXT: movl 4(%ecx), %edx
; X86-NEXT: xorl %ecx, %ecx
; X86-NEXT: cmpl 4(%eax), %edx
; X86-NEXT: je .LBB12_3
; X86-NEXT: .LBB12_1: # %res_block
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: incl %eax
; X86-NEXT: .LBB12_2: # %res_block
; X86-NEXT: xorl %ecx, %ecx
; X86-NEXT: incl %ecx
; X86-NEXT: .LBB12_3: # %endblock
; X86-NEXT: testl %eax, %eax
; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: sete %al
; X86-NEXT: retl
;
@ -432,15 +432,15 @@ define i1 @length8_eq(i8* %X, i8* %Y) nounwind optsize {
define i1 @length8_eq_const(i8* %X) nounwind optsize {
; X86-LABEL: length8_eq_const:
; X86: # BB#0: # %loadbb
; X86: # BB#0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: cmpl $858927408, (%ecx) # imm = 0x33323130
; X86-NEXT: jne .LBB13_1
; X86-NEXT: # BB#2: # %loadbb1
; X86-NEXT: jne .LBB13_2
; X86-NEXT: # BB#1: # %loadbb1
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: cmpl $926299444, 4(%ecx) # imm = 0x37363534
; X86-NEXT: je .LBB13_3
; X86-NEXT: .LBB13_1: # %res_block
; X86-NEXT: .LBB13_2: # %res_block
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: incl %eax
; X86-NEXT: .LBB13_3: # %endblock
@ -473,16 +473,16 @@ define i1 @length12_eq(i8* %X, i8* %Y) nounwind optsize {
; X86-NEXT: retl
;
; X64-LABEL: length12_eq:
; X64: # BB#0: # %loadbb
; X64: # BB#0:
; X64-NEXT: movq (%rdi), %rax
; X64-NEXT: cmpq (%rsi), %rax
; X64-NEXT: jne .LBB14_1
; X64-NEXT: # BB#2: # %loadbb1
; X64-NEXT: jne .LBB14_2
; X64-NEXT: # BB#1: # %loadbb1
; X64-NEXT: movl 8(%rdi), %ecx
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpl 8(%rsi), %ecx
; X64-NEXT: je .LBB14_3
; X64-NEXT: .LBB14_1: # %res_block
; X64-NEXT: .LBB14_2: # %res_block
; X64-NEXT: movl $1, %eax
; X64-NEXT: .LBB14_3: # %endblock
; X64-NEXT: testl %eax, %eax
@ -505,28 +505,27 @@ define i32 @length12(i8* %X, i8* %Y) nounwind optsize {
; X86-NEXT: retl
;
; X64-LABEL: length12:
; X64: # BB#0: # %loadbb
; X64: # BB#0:
; X64-NEXT: movq (%rdi), %rcx
; X64-NEXT: movq (%rsi), %rdx
; X64-NEXT: bswapq %rcx
; X64-NEXT: bswapq %rdx
; X64-NEXT: cmpq %rdx, %rcx
; X64-NEXT: jne .LBB15_1
; X64-NEXT: # BB#2: # %loadbb1
; X64-NEXT: jne .LBB15_2
; X64-NEXT: # BB#1: # %loadbb1
; X64-NEXT: movl 8(%rdi), %ecx
; X64-NEXT: movl 8(%rsi), %edx
; X64-NEXT: bswapl %ecx
; X64-NEXT: bswapl %edx
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpq %rdx, %rcx
; X64-NEXT: jne .LBB15_1
; X64-NEXT: # BB#3: # %endblock
; X64-NEXT: retq
; X64-NEXT: .LBB15_1: # %res_block
; X64-NEXT: je .LBB15_3
; X64-NEXT: .LBB15_2: # %res_block
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpq %rdx, %rcx
; X64-NEXT: setae %al
; X64-NEXT: leal -1(%rax,%rax), %eax
; X64-NEXT: .LBB15_3: # %endblock
; X64-NEXT: retq
%m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 12) nounwind
ret i32 %m
@ -546,28 +545,27 @@ define i32 @length16(i8* %X, i8* %Y) nounwind optsize {
; X86-NEXT: retl
;
; X64-LABEL: length16:
; X64: # BB#0: # %loadbb
; X64: # BB#0:
; X64-NEXT: movq (%rdi), %rcx
; X64-NEXT: movq (%rsi), %rdx
; X64-NEXT: bswapq %rcx
; X64-NEXT: bswapq %rdx
; X64-NEXT: cmpq %rdx, %rcx
; X64-NEXT: jne .LBB16_1
; X64-NEXT: # BB#2: # %loadbb1
; X64-NEXT: jne .LBB16_2
; X64-NEXT: # BB#1: # %loadbb1
; X64-NEXT: movq 8(%rdi), %rcx
; X64-NEXT: movq 8(%rsi), %rdx
; X64-NEXT: bswapq %rcx
; X64-NEXT: bswapq %rdx
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpq %rdx, %rcx
; X64-NEXT: jne .LBB16_1
; X64-NEXT: # BB#3: # %endblock
; X64-NEXT: retq
; X64-NEXT: .LBB16_1: # %res_block
; X64-NEXT: je .LBB16_3
; X64-NEXT: .LBB16_2: # %res_block
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpq %rdx, %rcx
; X64-NEXT: setae %al
; X64-NEXT: leal -1(%rax,%rax), %eax
; X64-NEXT: .LBB16_3: # %endblock
; X64-NEXT: retq
%m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 16) nounwind
ret i32 %m
@ -701,19 +699,19 @@ define i1 @length24_eq(i8* %x, i8* %y) nounwind optsize {
; X86-NEXT: retl
;
; X64-SSE2-LABEL: length24_eq:
; X64-SSE2: # BB#0: # %loadbb
; X64-SSE2: # BB#0:
; X64-SSE2-NEXT: movdqu (%rdi), %xmm0
; X64-SSE2-NEXT: movdqu (%rsi), %xmm1
; X64-SSE2-NEXT: pcmpeqb %xmm0, %xmm1
; X64-SSE2-NEXT: pmovmskb %xmm1, %eax
; X64-SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF
; X64-SSE2-NEXT: jne .LBB20_1
; X64-SSE2-NEXT: # BB#2: # %loadbb1
; X64-SSE2-NEXT: jne .LBB20_2
; X64-SSE2-NEXT: # BB#1: # %loadbb1
; X64-SSE2-NEXT: movq 16(%rdi), %rcx
; X64-SSE2-NEXT: xorl %eax, %eax
; X64-SSE2-NEXT: cmpq 16(%rsi), %rcx
; X64-SSE2-NEXT: je .LBB20_3
; X64-SSE2-NEXT: .LBB20_1: # %res_block
; X64-SSE2-NEXT: .LBB20_2: # %res_block
; X64-SSE2-NEXT: movl $1, %eax
; X64-SSE2-NEXT: .LBB20_3: # %endblock
; X64-SSE2-NEXT: testl %eax, %eax
@ -721,18 +719,18 @@ define i1 @length24_eq(i8* %x, i8* %y) nounwind optsize {
; X64-SSE2-NEXT: retq
;
; X64-AVX2-LABEL: length24_eq:
; X64-AVX2: # BB#0: # %loadbb
; X64-AVX2: # BB#0:
; X64-AVX2-NEXT: vmovdqu (%rdi), %xmm0
; X64-AVX2-NEXT: vpcmpeqb (%rsi), %xmm0, %xmm0
; X64-AVX2-NEXT: vpmovmskb %xmm0, %eax
; X64-AVX2-NEXT: cmpl $65535, %eax # imm = 0xFFFF
; X64-AVX2-NEXT: jne .LBB20_1
; X64-AVX2-NEXT: # BB#2: # %loadbb1
; X64-AVX2-NEXT: jne .LBB20_2
; X64-AVX2-NEXT: # BB#1: # %loadbb1
; X64-AVX2-NEXT: movq 16(%rdi), %rcx
; X64-AVX2-NEXT: xorl %eax, %eax
; X64-AVX2-NEXT: cmpq 16(%rsi), %rcx
; X64-AVX2-NEXT: je .LBB20_3
; X64-AVX2-NEXT: .LBB20_1: # %res_block
; X64-AVX2-NEXT: .LBB20_2: # %res_block
; X64-AVX2-NEXT: movl $1, %eax
; X64-AVX2-NEXT: .LBB20_3: # %endblock
; X64-AVX2-NEXT: testl %eax, %eax
@ -757,18 +755,18 @@ define i1 @length24_eq_const(i8* %X) nounwind optsize {
; X86-NEXT: retl
;
; X64-SSE2-LABEL: length24_eq_const:
; X64-SSE2: # BB#0: # %loadbb
; X64-SSE2: # BB#0:
; X64-SSE2-NEXT: movdqu (%rdi), %xmm0
; X64-SSE2-NEXT: pcmpeqb {{.*}}(%rip), %xmm0
; X64-SSE2-NEXT: pmovmskb %xmm0, %eax
; X64-SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF
; X64-SSE2-NEXT: jne .LBB21_1
; X64-SSE2-NEXT: # BB#2: # %loadbb1
; X64-SSE2-NEXT: jne .LBB21_2
; X64-SSE2-NEXT: # BB#1: # %loadbb1
; X64-SSE2-NEXT: xorl %eax, %eax
; X64-SSE2-NEXT: movabsq $3689065127958034230, %rcx # imm = 0x3332313039383736
; X64-SSE2-NEXT: cmpq %rcx, 16(%rdi)
; X64-SSE2-NEXT: je .LBB21_3
; X64-SSE2-NEXT: .LBB21_1: # %res_block
; X64-SSE2-NEXT: .LBB21_2: # %res_block
; X64-SSE2-NEXT: movl $1, %eax
; X64-SSE2-NEXT: .LBB21_3: # %endblock
; X64-SSE2-NEXT: testl %eax, %eax
@ -776,18 +774,18 @@ define i1 @length24_eq_const(i8* %X) nounwind optsize {
; X64-SSE2-NEXT: retq
;
; X64-AVX2-LABEL: length24_eq_const:
; X64-AVX2: # BB#0: # %loadbb
; X64-AVX2: # BB#0:
; X64-AVX2-NEXT: vmovdqu (%rdi), %xmm0
; X64-AVX2-NEXT: vpcmpeqb {{.*}}(%rip), %xmm0, %xmm0
; X64-AVX2-NEXT: vpmovmskb %xmm0, %eax
; X64-AVX2-NEXT: cmpl $65535, %eax # imm = 0xFFFF
; X64-AVX2-NEXT: jne .LBB21_1
; X64-AVX2-NEXT: # BB#2: # %loadbb1
; X64-AVX2-NEXT: jne .LBB21_2
; X64-AVX2-NEXT: # BB#1: # %loadbb1
; X64-AVX2-NEXT: xorl %eax, %eax
; X64-AVX2-NEXT: movabsq $3689065127958034230, %rcx # imm = 0x3332313039383736
; X64-AVX2-NEXT: cmpq %rcx, 16(%rdi)
; X64-AVX2-NEXT: je .LBB21_3
; X64-AVX2-NEXT: .LBB21_1: # %res_block
; X64-AVX2-NEXT: .LBB21_2: # %res_block
; X64-AVX2-NEXT: movl $1, %eax
; X64-AVX2-NEXT: .LBB21_3: # %endblock
; X64-AVX2-NEXT: testl %eax, %eax
@ -833,7 +831,7 @@ define i1 @length32_eq(i8* %x, i8* %y) nounwind optsize {
; X86-NOSSE-NEXT: retl
;
; X86-SSE2-LABEL: length32_eq:
; X86-SSE2: # BB#0: # %loadbb
; X86-SSE2: # BB#0:
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-SSE2-NEXT: movdqu (%ecx), %xmm0
@ -841,8 +839,8 @@ define i1 @length32_eq(i8* %x, i8* %y) nounwind optsize {
; X86-SSE2-NEXT: pcmpeqb %xmm0, %xmm1
; X86-SSE2-NEXT: pmovmskb %xmm1, %edx
; X86-SSE2-NEXT: cmpl $65535, %edx # imm = 0xFFFF
; X86-SSE2-NEXT: jne .LBB23_1
; X86-SSE2-NEXT: # BB#2: # %loadbb1
; X86-SSE2-NEXT: jne .LBB23_2
; X86-SSE2-NEXT: # BB#1: # %loadbb1
; X86-SSE2-NEXT: movdqu 16(%ecx), %xmm0
; X86-SSE2-NEXT: movdqu 16(%eax), %xmm1
; X86-SSE2-NEXT: pcmpeqb %xmm0, %xmm1
@ -850,7 +848,7 @@ define i1 @length32_eq(i8* %x, i8* %y) nounwind optsize {
; X86-SSE2-NEXT: xorl %eax, %eax
; X86-SSE2-NEXT: cmpl $65535, %ecx # imm = 0xFFFF
; X86-SSE2-NEXT: je .LBB23_3
; X86-SSE2-NEXT: .LBB23_1: # %res_block
; X86-SSE2-NEXT: .LBB23_2: # %res_block
; X86-SSE2-NEXT: xorl %eax, %eax
; X86-SSE2-NEXT: incl %eax
; X86-SSE2-NEXT: .LBB23_3: # %endblock
@ -859,14 +857,14 @@ define i1 @length32_eq(i8* %x, i8* %y) nounwind optsize {
; X86-SSE2-NEXT: retl
;
; X64-SSE2-LABEL: length32_eq:
; X64-SSE2: # BB#0: # %loadbb
; X64-SSE2: # BB#0:
; X64-SSE2-NEXT: movdqu (%rdi), %xmm0
; X64-SSE2-NEXT: movdqu (%rsi), %xmm1
; X64-SSE2-NEXT: pcmpeqb %xmm0, %xmm1
; X64-SSE2-NEXT: pmovmskb %xmm1, %eax
; X64-SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF
; X64-SSE2-NEXT: jne .LBB23_1
; X64-SSE2-NEXT: # BB#2: # %loadbb1
; X64-SSE2-NEXT: jne .LBB23_2
; X64-SSE2-NEXT: # BB#1: # %loadbb1
; X64-SSE2-NEXT: movdqu 16(%rdi), %xmm0
; X64-SSE2-NEXT: movdqu 16(%rsi), %xmm1
; X64-SSE2-NEXT: pcmpeqb %xmm0, %xmm1
@ -874,7 +872,7 @@ define i1 @length32_eq(i8* %x, i8* %y) nounwind optsize {
; X64-SSE2-NEXT: xorl %eax, %eax
; X64-SSE2-NEXT: cmpl $65535, %ecx # imm = 0xFFFF
; X64-SSE2-NEXT: je .LBB23_3
; X64-SSE2-NEXT: .LBB23_1: # %res_block
; X64-SSE2-NEXT: .LBB23_2: # %res_block
; X64-SSE2-NEXT: movl $1, %eax
; X64-SSE2-NEXT: .LBB23_3: # %endblock
; X64-SSE2-NEXT: testl %eax, %eax
@ -909,21 +907,21 @@ define i1 @length32_eq_const(i8* %X) nounwind optsize {
; X86-NOSSE-NEXT: retl
;
; X86-SSE2-LABEL: length32_eq_const:
; X86-SSE2: # BB#0: # %loadbb
; X86-SSE2: # BB#0:
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-SSE2-NEXT: movdqu (%eax), %xmm0
; X86-SSE2-NEXT: pcmpeqb {{\.LCPI.*}}, %xmm0
; X86-SSE2-NEXT: pmovmskb %xmm0, %ecx
; X86-SSE2-NEXT: cmpl $65535, %ecx # imm = 0xFFFF
; X86-SSE2-NEXT: jne .LBB24_1
; X86-SSE2-NEXT: # BB#2: # %loadbb1
; X86-SSE2-NEXT: jne .LBB24_2
; X86-SSE2-NEXT: # BB#1: # %loadbb1
; X86-SSE2-NEXT: movdqu 16(%eax), %xmm0
; X86-SSE2-NEXT: pcmpeqb {{\.LCPI.*}}, %xmm0
; X86-SSE2-NEXT: pmovmskb %xmm0, %ecx
; X86-SSE2-NEXT: xorl %eax, %eax
; X86-SSE2-NEXT: cmpl $65535, %ecx # imm = 0xFFFF
; X86-SSE2-NEXT: je .LBB24_3
; X86-SSE2-NEXT: .LBB24_1: # %res_block
; X86-SSE2-NEXT: .LBB24_2: # %res_block
; X86-SSE2-NEXT: xorl %eax, %eax
; X86-SSE2-NEXT: incl %eax
; X86-SSE2-NEXT: .LBB24_3: # %endblock
@ -932,20 +930,20 @@ define i1 @length32_eq_const(i8* %X) nounwind optsize {
; X86-SSE2-NEXT: retl
;
; X64-SSE2-LABEL: length32_eq_const:
; X64-SSE2: # BB#0: # %loadbb
; X64-SSE2: # BB#0:
; X64-SSE2-NEXT: movdqu (%rdi), %xmm0
; X64-SSE2-NEXT: pcmpeqb {{.*}}(%rip), %xmm0
; X64-SSE2-NEXT: pmovmskb %xmm0, %eax
; X64-SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF
; X64-SSE2-NEXT: jne .LBB24_1
; X64-SSE2-NEXT: # BB#2: # %loadbb1
; X64-SSE2-NEXT: jne .LBB24_2
; X64-SSE2-NEXT: # BB#1: # %loadbb1
; X64-SSE2-NEXT: movdqu 16(%rdi), %xmm0
; X64-SSE2-NEXT: pcmpeqb {{.*}}(%rip), %xmm0
; X64-SSE2-NEXT: pmovmskb %xmm0, %ecx
; X64-SSE2-NEXT: xorl %eax, %eax
; X64-SSE2-NEXT: cmpl $65535, %ecx # imm = 0xFFFF
; X64-SSE2-NEXT: je .LBB24_3
; X64-SSE2-NEXT: .LBB24_1: # %res_block
; X64-SSE2-NEXT: .LBB24_2: # %res_block
; X64-SSE2-NEXT: movl $1, %eax
; X64-SSE2-NEXT: .LBB24_3: # %endblock
; X64-SSE2-NEXT: testl %eax, %eax
@ -1009,20 +1007,20 @@ define i1 @length64_eq(i8* %x, i8* %y) nounwind optsize {
; X64-SSE2-NEXT: retq
;
; X64-AVX2-LABEL: length64_eq:
; X64-AVX2: # BB#0: # %loadbb
; X64-AVX2: # BB#0:
; X64-AVX2-NEXT: vmovdqu (%rdi), %ymm0
; X64-AVX2-NEXT: vpcmpeqb (%rsi), %ymm0, %ymm0
; X64-AVX2-NEXT: vpmovmskb %ymm0, %eax
; X64-AVX2-NEXT: cmpl $-1, %eax
; X64-AVX2-NEXT: jne .LBB26_1
; X64-AVX2-NEXT: # BB#2: # %loadbb1
; X64-AVX2-NEXT: jne .LBB26_2
; X64-AVX2-NEXT: # BB#1: # %loadbb1
; X64-AVX2-NEXT: vmovdqu 32(%rdi), %ymm0
; X64-AVX2-NEXT: vpcmpeqb 32(%rsi), %ymm0, %ymm0
; X64-AVX2-NEXT: vpmovmskb %ymm0, %ecx
; X64-AVX2-NEXT: xorl %eax, %eax
; X64-AVX2-NEXT: cmpl $-1, %ecx
; X64-AVX2-NEXT: je .LBB26_3
; X64-AVX2-NEXT: .LBB26_1: # %res_block
; X64-AVX2-NEXT: .LBB26_2: # %res_block
; X64-AVX2-NEXT: movl $1, %eax
; X64-AVX2-NEXT: .LBB26_3: # %endblock
; X64-AVX2-NEXT: testl %eax, %eax
@ -1059,20 +1057,20 @@ define i1 @length64_eq_const(i8* %X) nounwind optsize {
; X64-SSE2-NEXT: retq
;
; X64-AVX2-LABEL: length64_eq_const:
; X64-AVX2: # BB#0: # %loadbb
; X64-AVX2: # BB#0:
; X64-AVX2-NEXT: vmovdqu (%rdi), %ymm0
; X64-AVX2-NEXT: vpcmpeqb {{.*}}(%rip), %ymm0, %ymm0
; X64-AVX2-NEXT: vpmovmskb %ymm0, %eax
; X64-AVX2-NEXT: cmpl $-1, %eax
; X64-AVX2-NEXT: jne .LBB27_1
; X64-AVX2-NEXT: # BB#2: # %loadbb1
; X64-AVX2-NEXT: jne .LBB27_2
; X64-AVX2-NEXT: # BB#1: # %loadbb1
; X64-AVX2-NEXT: vmovdqu 32(%rdi), %ymm0
; X64-AVX2-NEXT: vpcmpeqb {{.*}}(%rip), %ymm0, %ymm0
; X64-AVX2-NEXT: vpmovmskb %ymm0, %ecx
; X64-AVX2-NEXT: xorl %eax, %eax
; X64-AVX2-NEXT: cmpl $-1, %ecx
; X64-AVX2-NEXT: je .LBB27_3
; X64-AVX2-NEXT: .LBB27_1: # %res_block
; X64-AVX2-NEXT: .LBB27_2: # %res_block
; X64-AVX2-NEXT: movl $1, %eax
; X64-AVX2-NEXT: .LBB27_3: # %endblock
; X64-AVX2-NEXT: testl %eax, %eax

240
llvm/test/CodeGen/X86/memcmp.ll
View File

@ -187,35 +187,35 @@ define i32 @length3(i8* %X, i8* %Y) nounwind {
define i1 @length3_eq(i8* %X, i8* %Y) nounwind {
; X86-LABEL: length3_eq:
; X86: # BB#0: # %loadbb
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86: # BB#0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzwl (%eax), %edx
; X86-NEXT: cmpw (%ecx), %dx
; X86-NEXT: jne .LBB7_1
; X86-NEXT: # BB#2: # %loadbb1
; X86-NEXT: movb 2(%eax), %dl
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: cmpb 2(%ecx), %dl
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movzwl (%ecx), %edx
; X86-NEXT: cmpw (%eax), %dx
; X86-NEXT: jne .LBB7_2
; X86-NEXT: # BB#1: # %loadbb1
; X86-NEXT: movb 2(%ecx), %dl
; X86-NEXT: xorl %ecx, %ecx
; X86-NEXT: cmpb 2(%eax), %dl
; X86-NEXT: je .LBB7_3
; X86-NEXT: .LBB7_1: # %res_block
; X86-NEXT: movl $1, %eax
; X86-NEXT: .LBB7_2: # %res_block
; X86-NEXT: movl $1, %ecx
; X86-NEXT: .LBB7_3: # %endblock
; X86-NEXT: testl %eax, %eax
; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: setne %al
; X86-NEXT: retl
;
; X64-LABEL: length3_eq:
; X64: # BB#0: # %loadbb
; X64: # BB#0:
; X64-NEXT: movzwl (%rdi), %eax
; X64-NEXT: cmpw (%rsi), %ax
; X64-NEXT: jne .LBB7_1
; X64-NEXT: # BB#2: # %loadbb1
; X64-NEXT: jne .LBB7_2
; X64-NEXT: # BB#1: # %loadbb1
; X64-NEXT: movb 2(%rdi), %cl
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpb 2(%rsi), %cl
; X64-NEXT: je .LBB7_3
; X64-NEXT: .LBB7_1: # %res_block
; X64-NEXT: .LBB7_2: # %res_block
; X64-NEXT: movl $1, %eax
; X64-NEXT: .LBB7_3: # %endblock
; X64-NEXT: testl %eax, %eax
@ -344,35 +344,35 @@ define i32 @length5(i8* %X, i8* %Y) nounwind {
define i1 @length5_eq(i8* %X, i8* %Y) nounwind {
; X86-LABEL: length5_eq:
; X86: # BB#0: # %loadbb
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86: # BB#0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl (%eax), %edx
; X86-NEXT: cmpl (%ecx), %edx
; X86-NEXT: jne .LBB12_1
; X86-NEXT: # BB#2: # %loadbb1
; X86-NEXT: movb 4(%eax), %dl
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: cmpb 4(%ecx), %dl
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl (%ecx), %edx
; X86-NEXT: cmpl (%eax), %edx
; X86-NEXT: jne .LBB12_2
; X86-NEXT: # BB#1: # %loadbb1
; X86-NEXT: movb 4(%ecx), %dl
; X86-NEXT: xorl %ecx, %ecx
; X86-NEXT: cmpb 4(%eax), %dl
; X86-NEXT: je .LBB12_3
; X86-NEXT: .LBB12_1: # %res_block
; X86-NEXT: movl $1, %eax
; X86-NEXT: .LBB12_2: # %res_block
; X86-NEXT: movl $1, %ecx
; X86-NEXT: .LBB12_3: # %endblock
; X86-NEXT: testl %eax, %eax
; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: setne %al
; X86-NEXT: retl
;
; X64-LABEL: length5_eq:
; X64: # BB#0: # %loadbb
; X64: # BB#0:
; X64-NEXT: movl (%rdi), %eax
; X64-NEXT: cmpl (%rsi), %eax
; X64-NEXT: jne .LBB12_1
; X64-NEXT: # BB#2: # %loadbb1
; X64-NEXT: jne .LBB12_2
; X64-NEXT: # BB#1: # %loadbb1
; X64-NEXT: movb 4(%rdi), %cl
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpb 4(%rsi), %cl
; X64-NEXT: je .LBB12_3
; X64-NEXT: .LBB12_1: # %res_block
; X64-NEXT: .LBB12_2: # %res_block
; X64-NEXT: movl $1, %eax
; X64-NEXT: .LBB12_3: # %endblock
; X64-NEXT: testl %eax, %eax
@ -385,7 +385,7 @@ define i1 @length5_eq(i8* %X, i8* %Y) nounwind {
define i32 @length8(i8* %X, i8* %Y) nounwind {
; X86-LABEL: length8:
; X86: # BB#0: # %loadbb
; X86: # BB#0:
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
@ -394,23 +394,21 @@ define i32 @length8(i8* %X, i8* %Y) nounwind {
; X86-NEXT: bswapl %ecx
; X86-NEXT: bswapl %edx
; X86-NEXT: cmpl %edx, %ecx
; X86-NEXT: jne .LBB13_1
; X86-NEXT: # BB#2: # %loadbb1
; X86-NEXT: jne .LBB13_2
; X86-NEXT: # BB#1: # %loadbb1
; X86-NEXT: movl 4(%esi), %ecx
; X86-NEXT: movl 4(%eax), %edx
; X86-NEXT: bswapl %ecx
; X86-NEXT: bswapl %edx
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: cmpl %edx, %ecx
; X86-NEXT: jne .LBB13_1
; X86-NEXT: # BB#3: # %endblock
; X86-NEXT: popl %esi
; X86-NEXT: retl
; X86-NEXT: .LBB13_1: # %res_block
; X86-NEXT: je .LBB13_3
; X86-NEXT: .LBB13_2: # %res_block
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: cmpl %edx, %ecx
; X86-NEXT: setae %al
; X86-NEXT: leal -1(%eax,%eax), %eax
; X86-NEXT: .LBB13_3: # %endblock
; X86-NEXT: popl %esi
; X86-NEXT: retl
;
@ -431,21 +429,21 @@ define i32 @length8(i8* %X, i8* %Y) nounwind {
define i1 @length8_eq(i8* %X, i8* %Y) nounwind {
; X86-LABEL: length8_eq:
; X86: # BB#0: # %loadbb
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86: # BB#0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl (%eax), %edx
; X86-NEXT: cmpl (%ecx), %edx
; X86-NEXT: jne .LBB14_1
; X86-NEXT: # BB#2: # %loadbb1
; X86-NEXT: movl 4(%eax), %edx
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: cmpl 4(%ecx), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl (%ecx), %edx
; X86-NEXT: cmpl (%eax), %edx
; X86-NEXT: jne .LBB14_2
; X86-NEXT: # BB#1: # %loadbb1
; X86-NEXT: movl 4(%ecx), %edx
; X86-NEXT: xorl %ecx, %ecx
; X86-NEXT: cmpl 4(%eax), %edx
; X86-NEXT: je .LBB14_3
; X86-NEXT: .LBB14_1: # %res_block
; X86-NEXT: movl $1, %eax
; X86-NEXT: .LBB14_2: # %res_block
; X86-NEXT: movl $1, %ecx
; X86-NEXT: .LBB14_3: # %endblock
; X86-NEXT: testl %eax, %eax
; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: sete %al
; X86-NEXT: retl
;
@ -462,15 +460,15 @@ define i1 @length8_eq(i8* %X, i8* %Y) nounwind {
define i1 @length8_eq_const(i8* %X) nounwind {
; X86-LABEL: length8_eq_const:
; X86: # BB#0: # %loadbb
; X86: # BB#0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: cmpl $858927408, (%ecx) # imm = 0x33323130
; X86-NEXT: jne .LBB15_1
; X86-NEXT: # BB#2: # %loadbb1
; X86-NEXT: jne .LBB15_2
; X86-NEXT: # BB#1: # %loadbb1
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: cmpl $926299444, 4(%ecx) # imm = 0x37363534
; X86-NEXT: je .LBB15_3
; X86-NEXT: .LBB15_1: # %res_block
; X86-NEXT: .LBB15_2: # %res_block
; X86-NEXT: movl $1, %eax
; X86-NEXT: .LBB15_3: # %endblock
; X86-NEXT: testl %eax, %eax
@ -502,16 +500,16 @@ define i1 @length12_eq(i8* %X, i8* %Y) nounwind {
; X86-NEXT: retl
;
; X64-LABEL: length12_eq:
; X64: # BB#0: # %loadbb
; X64: # BB#0:
; X64-NEXT: movq (%rdi), %rax
; X64-NEXT: cmpq (%rsi), %rax
; X64-NEXT: jne .LBB16_1
; X64-NEXT: # BB#2: # %loadbb1
; X64-NEXT: jne .LBB16_2
; X64-NEXT: # BB#1: # %loadbb1
; X64-NEXT: movl 8(%rdi), %ecx
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpl 8(%rsi), %ecx
; X64-NEXT: je .LBB16_3
; X64-NEXT: .LBB16_1: # %res_block
; X64-NEXT: .LBB16_2: # %res_block
; X64-NEXT: movl $1, %eax
; X64-NEXT: .LBB16_3: # %endblock
; X64-NEXT: testl %eax, %eax
@ -534,28 +532,27 @@ define i32 @length12(i8* %X, i8* %Y) nounwind {
; X86-NEXT: retl
;
; X64-LABEL: length12:
; X64: # BB#0: # %loadbb
; X64: # BB#0:
; X64-NEXT: movq (%rdi), %rcx
; X64-NEXT: movq (%rsi), %rdx
; X64-NEXT: bswapq %rcx
; X64-NEXT: bswapq %rdx
; X64-NEXT: cmpq %rdx, %rcx
; X64-NEXT: jne .LBB17_1
; X64-NEXT: # BB#2: # %loadbb1
; X64-NEXT: jne .LBB17_2
; X64-NEXT: # BB#1: # %loadbb1
; X64-NEXT: movl 8(%rdi), %ecx
; X64-NEXT: movl 8(%rsi), %edx
; X64-NEXT: bswapl %ecx
; X64-NEXT: bswapl %edx
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpq %rdx, %rcx
; X64-NEXT: jne .LBB17_1
; X64-NEXT: # BB#3: # %endblock
; X64-NEXT: retq
; X64-NEXT: .LBB17_1: # %res_block
; X64-NEXT: je .LBB17_3
; X64-NEXT: .LBB17_2: # %res_block
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpq %rdx, %rcx
; X64-NEXT: setae %al
; X64-NEXT: leal -1(%rax,%rax), %eax
; X64-NEXT: .LBB17_3: # %endblock
; X64-NEXT: retq
%m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 12) nounwind
ret i32 %m
@ -575,28 +572,27 @@ define i32 @length16(i8* %X, i8* %Y) nounwind {
; X86-NEXT: retl
;
; X64-LABEL: length16:
; X64: # BB#0: # %loadbb
; X64: # BB#0:
; X64-NEXT: movq (%rdi), %rcx
; X64-NEXT: movq (%rsi), %rdx
; X64-NEXT: bswapq %rcx
; X64-NEXT: bswapq %rdx
; X64-NEXT: cmpq %rdx, %rcx
; X64-NEXT: jne .LBB18_1
; X64-NEXT: # BB#2: # %loadbb1
; X64-NEXT: jne .LBB18_2
; X64-NEXT: # BB#1: # %loadbb1
; X64-NEXT: movq 8(%rdi), %rcx
; X64-NEXT: movq 8(%rsi), %rdx
; X64-NEXT: bswapq %rcx
; X64-NEXT: bswapq %rdx
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpq %rdx, %rcx
; X64-NEXT: jne .LBB18_1
; X64-NEXT: # BB#3: # %endblock
; X64-NEXT: retq
; X64-NEXT: .LBB18_1: # %res_block
; X64-NEXT: je .LBB18_3
; X64-NEXT: .LBB18_2: # %res_block
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpq %rdx, %rcx
; X64-NEXT: setae %al
; X64-NEXT: leal -1(%rax,%rax), %eax
; X64-NEXT: .LBB18_3: # %endblock
; X64-NEXT: retq
%m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 16) nounwind
ret i32 %m
@ -754,19 +750,19 @@ define i1 @length24_eq(i8* %x, i8* %y) nounwind {
; X86-NEXT: retl
;
; X64-SSE2-LABEL: length24_eq:
; X64-SSE2: # BB#0: # %loadbb
; X64-SSE2: # BB#0:
; X64-SSE2-NEXT: movdqu (%rdi), %xmm0
; X64-SSE2-NEXT: movdqu (%rsi), %xmm1
; X64-SSE2-NEXT: pcmpeqb %xmm0, %xmm1
; X64-SSE2-NEXT: pmovmskb %xmm1, %eax
; X64-SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF
; X64-SSE2-NEXT: jne .LBB22_1
; X64-SSE2-NEXT: # BB#2: # %loadbb1
; X64-SSE2-NEXT: jne .LBB22_2
; X64-SSE2-NEXT: # BB#1: # %loadbb1
; X64-SSE2-NEXT: movq 16(%rdi), %rcx
; X64-SSE2-NEXT: xorl %eax, %eax
; X64-SSE2-NEXT: cmpq 16(%rsi), %rcx
; X64-SSE2-NEXT: je .LBB22_3
; X64-SSE2-NEXT: .LBB22_1: # %res_block
; X64-SSE2-NEXT: .LBB22_2: # %res_block
; X64-SSE2-NEXT: movl $1, %eax
; X64-SSE2-NEXT: .LBB22_3: # %endblock
; X64-SSE2-NEXT: testl %eax, %eax
@ -774,18 +770,18 @@ define i1 @length24_eq(i8* %x, i8* %y) nounwind {
; X64-SSE2-NEXT: retq
;
; X64-AVX-LABEL: length24_eq:
; X64-AVX: # BB#0: # %loadbb
; X64-AVX: # BB#0:
; X64-AVX-NEXT: vmovdqu (%rdi), %xmm0
; X64-AVX-NEXT: vpcmpeqb (%rsi), %xmm0, %xmm0
; X64-AVX-NEXT: vpmovmskb %xmm0, %eax
; X64-AVX-NEXT: cmpl $65535, %eax # imm = 0xFFFF
; X64-AVX-NEXT: jne .LBB22_1
; X64-AVX-NEXT: # BB#2: # %loadbb1
; X64-AVX-NEXT: jne .LBB22_2
; X64-AVX-NEXT: # BB#1: # %loadbb1
; X64-AVX-NEXT: movq 16(%rdi), %rcx
; X64-AVX-NEXT: xorl %eax, %eax
; X64-AVX-NEXT: cmpq 16(%rsi), %rcx
; X64-AVX-NEXT: je .LBB22_3
; X64-AVX-NEXT: .LBB22_1: # %res_block
; X64-AVX-NEXT: .LBB22_2: # %res_block
; X64-AVX-NEXT: movl $1, %eax
; X64-AVX-NEXT: .LBB22_3: # %endblock
; X64-AVX-NEXT: testl %eax, %eax
@ -810,18 +806,18 @@ define i1 @length24_eq_const(i8* %X) nounwind {
; X86-NEXT: retl
;
; X64-SSE2-LABEL: length24_eq_const:
; X64-SSE2: # BB#0: # %loadbb
; X64-SSE2: # BB#0:
; X64-SSE2-NEXT: movdqu (%rdi), %xmm0
; X64-SSE2-NEXT: pcmpeqb {{.*}}(%rip), %xmm0
; X64-SSE2-NEXT: pmovmskb %xmm0, %eax
; X64-SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF
; X64-SSE2-NEXT: jne .LBB23_1
; X64-SSE2-NEXT: # BB#2: # %loadbb1
; X64-SSE2-NEXT: jne .LBB23_2
; X64-SSE2-NEXT: # BB#1: # %loadbb1
; X64-SSE2-NEXT: xorl %eax, %eax
; X64-SSE2-NEXT: movabsq $3689065127958034230, %rcx # imm = 0x3332313039383736
; X64-SSE2-NEXT: cmpq %rcx, 16(%rdi)
; X64-SSE2-NEXT: je .LBB23_3
; X64-SSE2-NEXT: .LBB23_1: # %res_block
; X64-SSE2-NEXT: .LBB23_2: # %res_block
; X64-SSE2-NEXT: movl $1, %eax
; X64-SSE2-NEXT: .LBB23_3: # %endblock
; X64-SSE2-NEXT: testl %eax, %eax
@ -829,18 +825,18 @@ define i1 @length24_eq_const(i8* %X) nounwind {
; X64-SSE2-NEXT: retq
;
; X64-AVX-LABEL: length24_eq_const:
; X64-AVX: # BB#0: # %loadbb
; X64-AVX: # BB#0:
; X64-AVX-NEXT: vmovdqu (%rdi), %xmm0
; X64-AVX-NEXT: vpcmpeqb {{.*}}(%rip), %xmm0, %xmm0
; X64-AVX-NEXT: vpmovmskb %xmm0, %eax
; X64-AVX-NEXT: cmpl $65535, %eax # imm = 0xFFFF
; X64-AVX-NEXT: jne .LBB23_1
; X64-AVX-NEXT: # BB#2: # %loadbb1
; X64-AVX-NEXT: jne .LBB23_2
; X64-AVX-NEXT: # BB#1: # %loadbb1
; X64-AVX-NEXT: xorl %eax, %eax
; X64-AVX-NEXT: movabsq $3689065127958034230, %rcx # imm = 0x3332313039383736
; X64-AVX-NEXT: cmpq %rcx, 16(%rdi)
; X64-AVX-NEXT: je .LBB23_3
; X64-AVX-NEXT: .LBB23_1: # %res_block
; X64-AVX-NEXT: .LBB23_2: # %res_block
; X64-AVX-NEXT: movl $1, %eax
; X64-AVX-NEXT: .LBB23_3: # %endblock
; X64-AVX-NEXT: testl %eax, %eax
@ -898,7 +894,7 @@ define i1 @length32_eq(i8* %x, i8* %y) nounwind {
; X86-SSE1-NEXT: retl
;
; X86-SSE2-LABEL: length32_eq:
; X86-SSE2: # BB#0: # %loadbb
; X86-SSE2: # BB#0:
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-SSE2-NEXT: movdqu (%ecx), %xmm0
@ -906,8 +902,8 @@ define i1 @length32_eq(i8* %x, i8* %y) nounwind {
; X86-SSE2-NEXT: pcmpeqb %xmm0, %xmm1
; X86-SSE2-NEXT: pmovmskb %xmm1, %edx
; X86-SSE2-NEXT: cmpl $65535, %edx # imm = 0xFFFF
; X86-SSE2-NEXT: jne .LBB25_1
; X86-SSE2-NEXT: # BB#2: # %loadbb1
; X86-SSE2-NEXT: jne .LBB25_2
; X86-SSE2-NEXT: # BB#1: # %loadbb1
; X86-SSE2-NEXT: movdqu 16(%ecx), %xmm0
; X86-SSE2-NEXT: movdqu 16(%eax), %xmm1
; X86-SSE2-NEXT: pcmpeqb %xmm0, %xmm1
@ -915,7 +911,7 @@ define i1 @length32_eq(i8* %x, i8* %y) nounwind {
; X86-SSE2-NEXT: xorl %eax, %eax
; X86-SSE2-NEXT: cmpl $65535, %ecx # imm = 0xFFFF
; X86-SSE2-NEXT: je .LBB25_3
; X86-SSE2-NEXT: .LBB25_1: # %res_block
; X86-SSE2-NEXT: .LBB25_2: # %res_block
; X86-SSE2-NEXT: movl $1, %eax
; X86-SSE2-NEXT: .LBB25_3: # %endblock
; X86-SSE2-NEXT: testl %eax, %eax
@ -923,14 +919,14 @@ define i1 @length32_eq(i8* %x, i8* %y) nounwind {
; X86-SSE2-NEXT: retl
;
; X64-SSE2-LABEL: length32_eq:
; X64-SSE2: # BB#0: # %loadbb
; X64-SSE2: # BB#0:
; X64-SSE2-NEXT: movdqu (%rdi), %xmm0
; X64-SSE2-NEXT: movdqu (%rsi), %xmm1
; X64-SSE2-NEXT: pcmpeqb %xmm0, %xmm1
; X64-SSE2-NEXT: pmovmskb %xmm1, %eax
; X64-SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF
; X64-SSE2-NEXT: jne .LBB25_1
; X64-SSE2-NEXT: # BB#2: # %loadbb1
; X64-SSE2-NEXT: jne .LBB25_2
; X64-SSE2-NEXT: # BB#1: # %loadbb1
; X64-SSE2-NEXT: movdqu 16(%rdi), %xmm0
; X64-SSE2-NEXT: movdqu 16(%rsi), %xmm1
; X64-SSE2-NEXT: pcmpeqb %xmm0, %xmm1
@ -938,7 +934,7 @@ define i1 @length32_eq(i8* %x, i8* %y) nounwind {
; X64-SSE2-NEXT: xorl %eax, %eax
; X64-SSE2-NEXT: cmpl $65535, %ecx # imm = 0xFFFF
; X64-SSE2-NEXT: je .LBB25_3
; X64-SSE2-NEXT: .LBB25_1: # %res_block
; X64-SSE2-NEXT: .LBB25_2: # %res_block
; X64-SSE2-NEXT: movl $1, %eax
; X64-SSE2-NEXT: .LBB25_3: # %endblock
; X64-SSE2-NEXT: testl %eax, %eax
@ -946,20 +942,20 @@ define i1 @length32_eq(i8* %x, i8* %y) nounwind {
; X64-SSE2-NEXT: retq
;
; X64-AVX1-LABEL: length32_eq:
; X64-AVX1: # BB#0: # %loadbb
; X64-AVX1: # BB#0:
; X64-AVX1-NEXT: vmovdqu (%rdi), %xmm0
; X64-AVX1-NEXT: vpcmpeqb (%rsi), %xmm0, %xmm0
; X64-AVX1-NEXT: vpmovmskb %xmm0, %eax
; X64-AVX1-NEXT: cmpl $65535, %eax # imm = 0xFFFF
; X64-AVX1-NEXT: jne .LBB25_1
; X64-AVX1-NEXT: # BB#2: # %loadbb1
; X64-AVX1-NEXT: jne .LBB25_2
; X64-AVX1-NEXT: # BB#1: # %loadbb1
; X64-AVX1-NEXT: vmovdqu 16(%rdi), %xmm0
; X64-AVX1-NEXT: vpcmpeqb 16(%rsi), %xmm0, %xmm0
; X64-AVX1-NEXT: vpmovmskb %xmm0, %ecx
; X64-AVX1-NEXT: xorl %eax, %eax
; X64-AVX1-NEXT: cmpl $65535, %ecx # imm = 0xFFFF
; X64-AVX1-NEXT: je .LBB25_3
; X64-AVX1-NEXT: .LBB25_1: # %res_block
; X64-AVX1-NEXT: .LBB25_2: # %res_block
; X64-AVX1-NEXT: movl $1, %eax
; X64-AVX1-NEXT: .LBB25_3: # %endblock
; X64-AVX1-NEXT: testl %eax, %eax
@ -1006,21 +1002,21 @@ define i1 @length32_eq_const(i8* %X) nounwind {
; X86-SSE1-NEXT: retl
;
; X86-SSE2-LABEL: length32_eq_const:
; X86-SSE2: # BB#0: # %loadbb
; X86-SSE2: # BB#0:
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-SSE2-NEXT: movdqu (%eax), %xmm0
; X86-SSE2-NEXT: pcmpeqb {{\.LCPI.*}}, %xmm0
; X86-SSE2-NEXT: pmovmskb %xmm0, %ecx
; X86-SSE2-NEXT: cmpl $65535, %ecx # imm = 0xFFFF
; X86-SSE2-NEXT: jne .LBB26_1
; X86-SSE2-NEXT: # BB#2: # %loadbb1
; X86-SSE2-NEXT: jne .LBB26_2
; X86-SSE2-NEXT: # BB#1: # %loadbb1
; X86-SSE2-NEXT: movdqu 16(%eax), %xmm0
; X86-SSE2-NEXT: pcmpeqb {{\.LCPI.*}}, %xmm0
; X86-SSE2-NEXT: pmovmskb %xmm0, %ecx
; X86-SSE2-NEXT: xorl %eax, %eax
; X86-SSE2-NEXT: cmpl $65535, %ecx # imm = 0xFFFF
; X86-SSE2-NEXT: je .LBB26_3
; X86-SSE2-NEXT: .LBB26_1: # %res_block
; X86-SSE2-NEXT: .LBB26_2: # %res_block
; X86-SSE2-NEXT: movl $1, %eax
; X86-SSE2-NEXT: .LBB26_3: # %endblock
; X86-SSE2-NEXT: testl %eax, %eax
@ -1028,20 +1024,20 @@ define i1 @length32_eq_const(i8* %X) nounwind {
; X86-SSE2-NEXT: retl
;
; X64-SSE2-LABEL: length32_eq_const:
; X64-SSE2: # BB#0: # %loadbb
; X64-SSE2: # BB#0:
; X64-SSE2-NEXT: movdqu (%rdi), %xmm0
; X64-SSE2-NEXT: pcmpeqb {{.*}}(%rip), %xmm0
; X64-SSE2-NEXT: pmovmskb %xmm0, %eax
; X64-SSE2-NEXT: cmpl $65535, %eax # imm = 0xFFFF
; X64-SSE2-NEXT: jne .LBB26_1
; X64-SSE2-NEXT: # BB#2: # %loadbb1
; X64-SSE2-NEXT: jne .LBB26_2
; X64-SSE2-NEXT: # BB#1: # %loadbb1
; X64-SSE2-NEXT: movdqu 16(%rdi), %xmm0
; X64-SSE2-NEXT: pcmpeqb {{.*}}(%rip), %xmm0
; X64-SSE2-NEXT: pmovmskb %xmm0, %ecx
; X64-SSE2-NEXT: xorl %eax, %eax
; X64-SSE2-NEXT: cmpl $65535, %ecx # imm = 0xFFFF
; X64-SSE2-NEXT: je .LBB26_3
; X64-SSE2-NEXT: .LBB26_1: # %res_block
; X64-SSE2-NEXT: .LBB26_2: # %res_block
; X64-SSE2-NEXT: movl $1, %eax
; X64-SSE2-NEXT: .LBB26_3: # %endblock
; X64-SSE2-NEXT: testl %eax, %eax
@ -1049,20 +1045,20 @@ define i1 @length32_eq_const(i8* %X) nounwind {
; X64-SSE2-NEXT: retq
;
; X64-AVX1-LABEL: length32_eq_const:
; X64-AVX1: # BB#0: # %loadbb
; X64-AVX1: # BB#0:
; X64-AVX1-NEXT: vmovdqu (%rdi), %xmm0
; X64-AVX1-NEXT: vpcmpeqb {{.*}}(%rip), %xmm0, %xmm0
; X64-AVX1-NEXT: vpmovmskb %xmm0, %eax
; X64-AVX1-NEXT: cmpl $65535, %eax # imm = 0xFFFF
; X64-AVX1-NEXT: jne .LBB26_1
; X64-AVX1-NEXT: # BB#2: # %loadbb1
; X64-AVX1-NEXT: jne .LBB26_2
; X64-AVX1-NEXT: # BB#1: # %loadbb1
; X64-AVX1-NEXT: vmovdqu 16(%rdi), %xmm0
; X64-AVX1-NEXT: vpcmpeqb {{.*}}(%rip), %xmm0, %xmm0
; X64-AVX1-NEXT: vpmovmskb %xmm0, %ecx
; X64-AVX1-NEXT: xorl %eax, %eax
; X64-AVX1-NEXT: cmpl $65535, %ecx # imm = 0xFFFF
; X64-AVX1-NEXT: je .LBB26_3
; X64-AVX1-NEXT: .LBB26_1: # %res_block
; X64-AVX1-NEXT: .LBB26_2: # %res_block
; X64-AVX1-NEXT: movl $1, %eax
; X64-AVX1-NEXT: .LBB26_3: # %endblock
; X64-AVX1-NEXT: testl %eax, %eax
@ -1136,20 +1132,20 @@ define i1 @length64_eq(i8* %x, i8* %y) nounwind {
; X64-AVX1-NEXT: retq
;
; X64-AVX2-LABEL: length64_eq:
; X64-AVX2: # BB#0: # %loadbb
; X64-AVX2: # BB#0:
; X64-AVX2-NEXT: vmovdqu (%rdi), %ymm0
; X64-AVX2-NEXT: vpcmpeqb (%rsi), %ymm0, %ymm0
; X64-AVX2-NEXT: vpmovmskb %ymm0, %eax
; X64-AVX2-NEXT: cmpl $-1, %eax
; X64-AVX2-NEXT: jne .LBB28_1
; X64-AVX2-NEXT: # BB#2: # %loadbb1
; X64-AVX2-NEXT: jne .LBB28_2
; X64-AVX2-NEXT: # BB#1: # %loadbb1
; X64-AVX2-NEXT: vmovdqu 32(%rdi), %ymm0
; X64-AVX2-NEXT: vpcmpeqb 32(%rsi), %ymm0, %ymm0
; X64-AVX2-NEXT: vpmovmskb %ymm0, %ecx
; X64-AVX2-NEXT: xorl %eax, %eax
; X64-AVX2-NEXT: cmpl $-1, %ecx
; X64-AVX2-NEXT: je .LBB28_3
; X64-AVX2-NEXT: .LBB28_1: # %res_block
; X64-AVX2-NEXT: .LBB28_2: # %res_block
; X64-AVX2-NEXT: movl $1, %eax
; X64-AVX2-NEXT: .LBB28_3: # %endblock
; X64-AVX2-NEXT: testl %eax, %eax
@ -1197,20 +1193,20 @@ define i1 @length64_eq_const(i8* %X) nounwind {
; X64-AVX1-NEXT: retq
;
; X64-AVX2-LABEL: length64_eq_const:
; X64-AVX2: # BB#0: # %loadbb
; X64-AVX2: # BB#0:
; X64-AVX2-NEXT: vmovdqu (%rdi), %ymm0
; X64-AVX2-NEXT: vpcmpeqb {{.*}}(%rip), %ymm0, %ymm0
; X64-AVX2-NEXT: vpmovmskb %ymm0, %eax
; X64-AVX2-NEXT: cmpl $-1, %eax
; X64-AVX2-NEXT: jne .LBB29_1
; X64-AVX2-NEXT: # BB#2: # %loadbb1
; X64-AVX2-NEXT: jne .LBB29_2
; X64-AVX2-NEXT: # BB#1: # %loadbb1
; X64-AVX2-NEXT: vmovdqu 32(%rdi), %ymm0
; X64-AVX2-NEXT: vpcmpeqb {{.*}}(%rip), %ymm0, %ymm0
; X64-AVX2-NEXT: vpmovmskb %ymm0, %ecx
; X64-AVX2-NEXT: xorl %eax, %eax
; X64-AVX2-NEXT: cmpl $-1, %ecx
; X64-AVX2-NEXT: je .LBB29_3
; X64-AVX2-NEXT: .LBB29_1: # %res_block
; X64-AVX2-NEXT: .LBB29_2: # %res_block
; X64-AVX2-NEXT: movl $1, %eax
; X64-AVX2-NEXT: .LBB29_3: # %endblock
; X64-AVX2-NEXT: testl %eax, %eax

3
llvm/test/Transforms/ExpandMemCmp/X86/lit.local.cfg Normal file
View File

@ -0,0 +1,3 @@
if not 'X86' in config.root.targets:
config.unsupported = True

519
llvm/test/Transforms/CodeGenPrepare/X86/memcmp.ll → llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll
View File

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -S -codegenprepare -mtriple=i686-unknown-unknown -data-layout=e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128 < %s | FileCheck %s --check-prefix=ALL --check-prefix=X32
; RUN: opt -S -codegenprepare -mtriple=x86_64-unknown-unknown -data-layout=e-m:o-i64:64-f80:128-n8:16:32:64-S128 < %s | FileCheck %s --check-prefix=ALL --check-prefix=X64
; RUN: opt -S -expandmemcmp -mtriple=i686-unknown-unknown -data-layout=e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128 < %s | FileCheck %s --check-prefix=ALL --check-prefix=X32
; RUN: opt -S -expandmemcmp -mtriple=x86_64-unknown-unknown -data-layout=e-m:o-i64:64-f80:128-n8:16:32:64-S128 < %s | FileCheck %s --check-prefix=ALL --check-prefix=X64
declare i32 @memcmp(i8* nocapture, i8* nocapture, i64)
@ -23,30 +23,33 @@ define i32 @cmp2(i8* nocapture readonly %x, i8* nocapture readonly %y) {
define i32 @cmp3(i8* nocapture readonly %x, i8* nocapture readonly %y) {
; ALL-LABEL: @cmp3(
; ALL-NEXT: loadbb:
; ALL-NEXT: [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i16*
; ALL-NEXT: [[TMP1:%.*]] = bitcast i8* [[Y:%.*]] to i16*
; ALL-NEXT: [[TMP2:%.*]] = load i16, i16* [[TMP0]]
; ALL-NEXT: [[TMP3:%.*]] = load i16, i16* [[TMP1]]
; ALL-NEXT: [[TMP4:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP2]])
; ALL-NEXT: [[TMP5:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP3]])
; ALL-NEXT: [[TMP6:%.*]] = icmp eq i16 [[TMP4]], [[TMP5]]
; ALL-NEXT: br i1 [[TMP6]], label [[LOADBB1:%.*]], label [[RES_BLOCK:%.*]]
; ALL-NEXT: br label [[LOADBB:%.*]]
; ALL: res_block:
; ALL-NEXT: [[TMP7:%.*]] = icmp ult i16 [[TMP4]], [[TMP5]]
; ALL-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i32 -1, i32 1
; ALL-NEXT: [[PHI_SRC1:%.*]] = phi i16 [ [[TMP7:%.*]], [[LOADBB]] ]
; ALL-NEXT: [[PHI_SRC2:%.*]] = phi i16 [ [[TMP8:%.*]], [[LOADBB]] ]
; ALL-NEXT: [[TMP1:%.*]] = icmp ult i16 [[PHI_SRC1]], [[PHI_SRC2]]
; ALL-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
; ALL-NEXT: br label [[ENDBLOCK:%.*]]
; ALL: loadbb:
; ALL-NEXT: [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i16*
; ALL-NEXT: [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i16*
; ALL-NEXT: [[TMP5:%.*]] = load i16, i16* [[TMP3]]
; ALL-NEXT: [[TMP6:%.*]] = load i16, i16* [[TMP4]]
; ALL-NEXT: [[TMP7]] = call i16 @llvm.bswap.i16(i16 [[TMP5]])
; ALL-NEXT: [[TMP8]] = call i16 @llvm.bswap.i16(i16 [[TMP6]])
; ALL-NEXT: [[TMP9:%.*]] = icmp eq i16 [[TMP7]], [[TMP8]]
; ALL-NEXT: br i1 [[TMP9]], label [[LOADBB1:%.*]], label [[RES_BLOCK:%.*]]
; ALL: loadbb1:
; ALL-NEXT: [[TMP9:%.*]] = getelementptr i8, i8* [[X]], i8 2
; ALL-NEXT: [[TMP10:%.*]] = getelementptr i8, i8* [[Y]], i8 2
; ALL-NEXT: [[TMP11:%.*]] = load i8, i8* [[TMP9]]
; ALL-NEXT: [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 2
; ALL-NEXT: [[TMP11:%.*]] = getelementptr i8, i8* [[Y]], i8 2
; ALL-NEXT: [[TMP12:%.*]] = load i8, i8* [[TMP10]]
; ALL-NEXT: [[TMP13:%.*]] = zext i8 [[TMP11]] to i32
; ALL-NEXT: [[TMP13:%.*]] = load i8, i8* [[TMP11]]
; ALL-NEXT: [[TMP14:%.*]] = zext i8 [[TMP12]] to i32
; ALL-NEXT: [[TMP15:%.*]] = sub i32 [[TMP13]], [[TMP14]]
; ALL-NEXT: [[TMP15:%.*]] = zext i8 [[TMP13]] to i32
; ALL-NEXT: [[TMP16:%.*]] = sub i32 [[TMP14]], [[TMP15]]
; ALL-NEXT: br label [[ENDBLOCK]]
; ALL: endblock:
; ALL-NEXT: [[PHI_RES:%.*]] = phi i32 [ [[TMP15]], [[LOADBB1]] ], [ [[TMP8]], [[RES_BLOCK]] ]
; ALL-NEXT: [[PHI_RES:%.*]] = phi i32 [ [[TMP16]], [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
; ALL-NEXT: ret i32 [[PHI_RES]]
;
%call = tail call i32 @memcmp(i8* %x, i8* %y, i64 3)
@ -74,30 +77,33 @@ define i32 @cmp4(i8* nocapture readonly %x, i8* nocapture readonly %y) {
define i32 @cmp5(i8* nocapture readonly %x, i8* nocapture readonly %y) {
; ALL-LABEL: @cmp5(
; ALL-NEXT: loadbb:
; ALL-NEXT: [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i32*
; ALL-NEXT: [[TMP1:%.*]] = bitcast i8* [[Y:%.*]] to i32*
; ALL-NEXT: [[TMP2:%.*]] = load i32, i32* [[TMP0]]
; ALL-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1]]
; ALL-NEXT: [[TMP4:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP2]])
; ALL-NEXT: [[TMP5:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP3]])
; ALL-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP4]], [[TMP5]]
; ALL-NEXT: br i1 [[TMP6]], label [[LOADBB1:%.*]], label [[RES_BLOCK:%.*]]
; ALL-NEXT: br label [[LOADBB:%.*]]
; ALL: res_block:
; ALL-NEXT: [[TMP7:%.*]] = icmp ult i32 [[TMP4]], [[TMP5]]
; ALL-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i32 -1, i32 1
; ALL-NEXT: [[PHI_SRC1:%.*]] = phi i32 [ [[TMP7:%.*]], [[LOADBB]] ]
; ALL-NEXT: [[PHI_SRC2:%.*]] = phi i32 [ [[TMP8:%.*]], [[LOADBB]] ]
; ALL-NEXT: [[TMP1:%.*]] = icmp ult i32 [[PHI_SRC1]], [[PHI_SRC2]]
; ALL-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
; ALL-NEXT: br label [[ENDBLOCK:%.*]]
; ALL: loadbb:
; ALL-NEXT: [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i32*
; ALL-NEXT: [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i32*
; ALL-NEXT: [[TMP5:%.*]] = load i32, i32* [[TMP3]]
; ALL-NEXT: [[TMP6:%.*]] = load i32, i32* [[TMP4]]
; ALL-NEXT: [[TMP7]] = call i32 @llvm.bswap.i32(i32 [[TMP5]])
; ALL-NEXT: [[TMP8]] = call i32 @llvm.bswap.i32(i32 [[TMP6]])
; ALL-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP7]], [[TMP8]]
; ALL-NEXT: br i1 [[TMP9]], label [[LOADBB1:%.*]], label [[RES_BLOCK:%.*]]
; ALL: loadbb1:
; ALL-NEXT: [[TMP9:%.*]] = getelementptr i8, i8* [[X]], i8 4
; ALL-NEXT: [[TMP10:%.*]] = getelementptr i8, i8* [[Y]], i8 4
; ALL-NEXT: [[TMP11:%.*]] = load i8, i8* [[TMP9]]
; ALL-NEXT: [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 4
; ALL-NEXT: [[TMP11:%.*]] = getelementptr i8, i8* [[Y]], i8 4
; ALL-NEXT: [[TMP12:%.*]] = load i8, i8* [[TMP10]]
; ALL-NEXT: [[TMP13:%.*]] = zext i8 [[TMP11]] to i32
; ALL-NEXT: [[TMP13:%.*]] = load i8, i8* [[TMP11]]
; ALL-NEXT: [[TMP14:%.*]] = zext i8 [[TMP12]] to i32
; ALL-NEXT: [[TMP15:%.*]] = sub i32 [[TMP13]], [[TMP14]]
; ALL-NEXT: [[TMP15:%.*]] = zext i8 [[TMP13]] to i32
; ALL-NEXT: [[TMP16:%.*]] = sub i32 [[TMP14]], [[TMP15]]
; ALL-NEXT: br label [[ENDBLOCK]]
; ALL: endblock:
; ALL-NEXT: [[PHI_RES:%.*]] = phi i32 [ [[TMP15]], [[LOADBB1]] ], [ [[TMP8]], [[RES_BLOCK]] ]
; ALL-NEXT: [[PHI_RES:%.*]] = phi i32 [ [[TMP16]], [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
; ALL-NEXT: ret i32 [[PHI_RES]]
;
%call = tail call i32 @memcmp(i8* %x, i8* %y, i64 5)
@ -106,36 +112,37 @@ define i32 @cmp5(i8* nocapture readonly %x, i8* nocapture readonly %y) {
define i32 @cmp6(i8* nocapture readonly %x, i8* nocapture readonly %y) {
; ALL-LABEL: @cmp6(
; ALL-NEXT: loadbb:
; ALL-NEXT: [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i32*
; ALL-NEXT: [[TMP1:%.*]] = bitcast i8* [[Y:%.*]] to i32*
; ALL-NEXT: [[TMP2:%.*]] = load i32, i32* [[TMP0]]
; ALL-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1]]
; ALL-NEXT: [[TMP4:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP2]])
; ALL-NEXT: [[TMP5:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP3]])
; ALL-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP4]], [[TMP5]]
; ALL-NEXT: br i1 [[TMP6]], label [[LOADBB1:%.*]], label [[RES_BLOCK:%.*]]
; ALL-NEXT: br label [[LOADBB:%.*]]
; ALL: res_block:
; ALL-NEXT: [[PHI_SRC1:%.*]] = phi i32 [ [[TMP4]], [[LOADBB:%.*]] ], [ [[TMP17:%.*]], [[LOADBB1]] ]
; ALL-NEXT: [[PHI_SRC2:%.*]] = phi i32 [ [[TMP5]], [[LOADBB]] ], [ [[TMP18:%.*]], [[LOADBB1]] ]
; ALL-NEXT: [[TMP7:%.*]] = icmp ult i32 [[PHI_SRC1]], [[PHI_SRC2]]
; ALL-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i32 -1, i32 1
; ALL-NEXT: [[PHI_SRC1:%.*]] = phi i32 [ [[TMP7:%.*]], [[LOADBB]] ], [ [[TMP18:%.*]], [[LOADBB1:%.*]] ]
; ALL-NEXT: [[PHI_SRC2:%.*]] = phi i32 [ [[TMP8:%.*]], [[LOADBB]] ], [ [[TMP19:%.*]], [[LOADBB1]] ]
; ALL-NEXT: [[TMP1:%.*]] = icmp ult i32 [[PHI_SRC1]], [[PHI_SRC2]]
; ALL-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
; ALL-NEXT: br label [[ENDBLOCK:%.*]]
; ALL: loadbb:
; ALL-NEXT: [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i32*
; ALL-NEXT: [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i32*
; ALL-NEXT: [[TMP5:%.*]] = load i32, i32* [[TMP3]]
; ALL-NEXT: [[TMP6:%.*]] = load i32, i32* [[TMP4]]
; ALL-NEXT: [[TMP7]] = call i32 @llvm.bswap.i32(i32 [[TMP5]])
; ALL-NEXT: [[TMP8]] = call i32 @llvm.bswap.i32(i32 [[TMP6]])
; ALL-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP7]], [[TMP8]]
; ALL-NEXT: br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
; ALL: loadbb1:
; ALL-NEXT: [[TMP9:%.*]] = bitcast i8* [[X]] to i16*
; ALL-NEXT: [[TMP10:%.*]] = bitcast i8* [[Y]] to i16*
; ALL-NEXT: [[TMP11:%.*]] = getelementptr i16, i16* [[TMP9]], i16 2
; ALL-NEXT: [[TMP10:%.*]] = bitcast i8* [[X]] to i16*
; ALL-NEXT: [[TMP11:%.*]] = bitcast i8* [[Y]] to i16*
; ALL-NEXT: [[TMP12:%.*]] = getelementptr i16, i16* [[TMP10]], i16 2
; ALL-NEXT: [[TMP13:%.*]] = load i16, i16* [[TMP11]]
; ALL-NEXT: [[TMP13:%.*]] = getelementptr i16, i16* [[TMP11]], i16 2
; ALL-NEXT: [[TMP14:%.*]] = load i16, i16* [[TMP12]]
; ALL-NEXT: [[TMP15:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP13]])
; ALL-NEXT: [[TMP15:%.*]] = load i16, i16* [[TMP13]]
; ALL-NEXT: [[TMP16:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP14]])
; ALL-NEXT: [[TMP17]] = zext i16 [[TMP15]] to i32
; ALL-NEXT: [[TMP17:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP15]])
; ALL-NEXT: [[TMP18]] = zext i16 [[TMP16]] to i32
; ALL-NEXT: [[TMP19:%.*]] = icmp eq i32 [[TMP17]], [[TMP18]]
; ALL-NEXT: br i1 [[TMP19]], label [[ENDBLOCK]], label [[RES_BLOCK]]
; ALL-NEXT: [[TMP19]] = zext i16 [[TMP17]] to i32
; ALL-NEXT: [[TMP20:%.*]] = icmp eq i32 [[TMP18]], [[TMP19]]
; ALL-NEXT: br i1 [[TMP20]], label [[ENDBLOCK]], label [[RES_BLOCK]]
; ALL: endblock:
; ALL-NEXT: [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP8]], [[RES_BLOCK]] ]
; ALL-NEXT: [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
; ALL-NEXT: ret i32 [[PHI_RES]]
;
%call = tail call i32 @memcmp(i8* %x, i8* %y, i64 6)
@ -153,34 +160,35 @@ define i32 @cmp7(i8* nocapture readonly %x, i8* nocapture readonly %y) {
define i32 @cmp8(i8* nocapture readonly %x, i8* nocapture readonly %y) {
; X32-LABEL: @cmp8(
; X32-NEXT: loadbb:
; X32-NEXT: [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i32*
; X32-NEXT: [[TMP1:%.*]] = bitcast i8* [[Y:%.*]] to i32*
; X32-NEXT: [[TMP2:%.*]] = load i32, i32* [[TMP0]]
; X32-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1]]
; X32-NEXT: [[TMP4:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP2]])
; X32-NEXT: [[TMP5:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP3]])
; X32-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP4]], [[TMP5]]
; X32-NEXT: br i1 [[TMP6]], label [[LOADBB1:%.*]], label [[RES_BLOCK:%.*]]
; X32-NEXT: br label [[LOADBB:%.*]]
; X32: res_block:
; X32-NEXT: [[PHI_SRC1:%.*]] = phi i32 [ [[TMP4]], [[LOADBB:%.*]] ], [ [[TMP15:%.*]], [[LOADBB1]] ]
; X32-NEXT: [[PHI_SRC2:%.*]] = phi i32 [ [[TMP5]], [[LOADBB]] ], [ [[TMP16:%.*]], [[LOADBB1]] ]
; X32-NEXT: [[TMP7:%.*]] = icmp ult i32 [[PHI_SRC1]], [[PHI_SRC2]]
; X32-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i32 -1, i32 1
; X32-NEXT: [[PHI_SRC1:%.*]] = phi i32 [ [[TMP7:%.*]], [[LOADBB]] ], [ [[TMP16:%.*]], [[LOADBB1:%.*]] ]
; X32-NEXT: [[PHI_SRC2:%.*]] = phi i32 [ [[TMP8:%.*]], [[LOADBB]] ], [ [[TMP17:%.*]], [[LOADBB1]] ]
; X32-NEXT: [[TMP1:%.*]] = icmp ult i32 [[PHI_SRC1]], [[PHI_SRC2]]
; X32-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
; X32-NEXT: br label [[ENDBLOCK:%.*]]
; X32: loadbb:
; X32-NEXT: [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i32*
; X32-NEXT: [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i32*
; X32-NEXT: [[TMP5:%.*]] = load i32, i32* [[TMP3]]
; X32-NEXT: [[TMP6:%.*]] = load i32, i32* [[TMP4]]
; X32-NEXT: [[TMP7]] = call i32 @llvm.bswap.i32(i32 [[TMP5]])
; X32-NEXT: [[TMP8]] = call i32 @llvm.bswap.i32(i32 [[TMP6]])
; X32-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP7]], [[TMP8]]
; X32-NEXT: br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
; X32: loadbb1:
; X32-NEXT: [[TMP9:%.*]] = bitcast i8* [[X]] to i32*
; X32-NEXT: [[TMP10:%.*]] = bitcast i8* [[Y]] to i32*
; X32-NEXT: [[TMP11:%.*]] = getelementptr i32, i32* [[TMP9]], i32 1
; X32-NEXT: [[TMP10:%.*]] = bitcast i8* [[X]] to i32*
; X32-NEXT: [[TMP11:%.*]] = bitcast i8* [[Y]] to i32*
; X32-NEXT: [[TMP12:%.*]] = getelementptr i32, i32* [[TMP10]], i32 1
; X32-NEXT: [[TMP13:%.*]] = load i32, i32* [[TMP11]]
; X32-NEXT: [[TMP13:%.*]] = getelementptr i32, i32* [[TMP11]], i32 1
; X32-NEXT: [[TMP14:%.*]] = load i32, i32* [[TMP12]]
; X32-NEXT: [[TMP15]] = call i32 @llvm.bswap.i32(i32 [[TMP13]])
; X32-NEXT: [[TMP15:%.*]] = load i32, i32* [[TMP13]]
; X32-NEXT: [[TMP16]] = call i32 @llvm.bswap.i32(i32 [[TMP14]])
; X32-NEXT: [[TMP17:%.*]] = icmp eq i32 [[TMP15]], [[TMP16]]
; X32-NEXT: br i1 [[TMP17]], label [[ENDBLOCK]], label [[RES_BLOCK]]
; X32-NEXT: [[TMP17]] = call i32 @llvm.bswap.i32(i32 [[TMP15]])
; X32-NEXT: [[TMP18:%.*]] = icmp eq i32 [[TMP16]], [[TMP17]]
; X32-NEXT: br i1 [[TMP18]], label [[ENDBLOCK]], label [[RES_BLOCK]]
; X32: endblock:
; X32-NEXT: [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP8]], [[RES_BLOCK]] ]
; X32-NEXT: [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
; X32-NEXT: ret i32 [[PHI_RES]]
;
; X64-LABEL: @cmp8(
@ -207,30 +215,33 @@ define i32 @cmp9(i8* nocapture readonly %x, i8* nocapture readonly %y) {
; X32-NEXT: ret i32 [[CALL]]
;
; X64-LABEL: @cmp9(
; X64-NEXT: loadbb:
; X64-NEXT: [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i64*
; X64-NEXT: [[TMP1:%.*]] = bitcast i8* [[Y:%.*]] to i64*
; X64-NEXT: [[TMP2:%.*]] = load i64, i64* [[TMP0]]
; X64-NEXT: [[TMP3:%.*]] = load i64, i64* [[TMP1]]
; X64-NEXT: [[TMP4:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP2]])
; X64-NEXT: [[TMP5:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP3]])
; X64-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP4]], [[TMP5]]
; X64-NEXT: br i1 [[TMP6]], label [[LOADBB1:%.*]], label [[RES_BLOCK:%.*]]
; X64-NEXT: br label [[LOADBB:%.*]]
; X64: res_block:
; X64-NEXT: [[TMP7:%.*]] = icmp ult i64 [[TMP4]], [[TMP5]]
; X64-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i32 -1, i32 1
; X64-NEXT: [[PHI_SRC1:%.*]] = phi i64 [ [[TMP7:%.*]], [[LOADBB]] ]
; X64-NEXT: [[PHI_SRC2:%.*]] = phi i64 [ [[TMP8:%.*]], [[LOADBB]] ]
; X64-NEXT: [[TMP1:%.*]] = icmp ult i64 [[PHI_SRC1]], [[PHI_SRC2]]
; X64-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
; X64-NEXT: br label [[ENDBLOCK:%.*]]
; X64: loadbb:
; X64-NEXT: [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i64*
; X64-NEXT: [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i64*
; X64-NEXT: [[TMP5:%.*]] = load i64, i64* [[TMP3]]
; X64-NEXT: [[TMP6:%.*]] = load i64, i64* [[TMP4]]
; X64-NEXT: [[TMP7]] = call i64 @llvm.bswap.i64(i64 [[TMP5]])
; X64-NEXT: [[TMP8]] = call i64 @llvm.bswap.i64(i64 [[TMP6]])
; X64-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
; X64-NEXT: br i1 [[TMP9]], label [[LOADBB1:%.*]], label [[RES_BLOCK:%.*]]
; X64: loadbb1:
; X64-NEXT: [[TMP9:%.*]] = getelementptr i8, i8* [[X]], i8 8
; X64-NEXT: [[TMP10:%.*]] = getelementptr i8, i8* [[Y]], i8 8
; X64-NEXT: [[TMP11:%.*]] = load i8, i8* [[TMP9]]
; X64-NEXT: [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 8
; X64-NEXT: [[TMP11:%.*]] = getelementptr i8, i8* [[Y]], i8 8
; X64-NEXT: [[TMP12:%.*]] = load i8, i8* [[TMP10]]
; X64-NEXT: [[TMP13:%.*]] = zext i8 [[TMP11]] to i32
; X64-NEXT: [[TMP13:%.*]] = load i8, i8* [[TMP11]]
; X64-NEXT: [[TMP14:%.*]] = zext i8 [[TMP12]] to i32
; X64-NEXT: [[TMP15:%.*]] = sub i32 [[TMP13]], [[TMP14]]
; X64-NEXT: [[TMP15:%.*]] = zext i8 [[TMP13]] to i32
; X64-NEXT: [[TMP16:%.*]] = sub i32 [[TMP14]], [[TMP15]]
; X64-NEXT: br label [[ENDBLOCK]]
; X64: endblock:
; X64-NEXT: [[PHI_RES:%.*]] = phi i32 [ [[TMP15]], [[LOADBB1]] ], [ [[TMP8]], [[RES_BLOCK]] ]
; X64-NEXT: [[PHI_RES:%.*]] = phi i32 [ [[TMP16]], [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
; X64-NEXT: ret i32 [[PHI_RES]]
;
%call = tail call i32 @memcmp(i8* %x, i8* %y, i64 9)
@ -243,36 +254,37 @@ define i32 @cmp10(i8* nocapture readonly %x, i8* nocapture readonly %y) {
; X32-NEXT: ret i32 [[CALL]]
;
; X64-LABEL: @cmp10(
; X64-NEXT: loadbb:
; X64-NEXT: [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i64*
; X64-NEXT: [[TMP1:%.*]] = bitcast i8* [[Y:%.*]] to i64*
; X64-NEXT: [[TMP2:%.*]] = load i64, i64* [[TMP0]]
; X64-NEXT: [[TMP3:%.*]] = load i64, i64* [[TMP1]]
; X64-NEXT: [[TMP4:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP2]])
; X64-NEXT: [[TMP5:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP3]])
; X64-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP4]], [[TMP5]]
; X64-NEXT: br i1 [[TMP6]], label [[LOADBB1:%.*]], label [[RES_BLOCK:%.*]]
; X64-NEXT: br label [[LOADBB:%.*]]
; X64: res_block:
; X64-NEXT: [[PHI_SRC1:%.*]] = phi i64 [ [[TMP4]], [[LOADBB:%.*]] ], [ [[TMP17:%.*]], [[LOADBB1]] ]
; X64-NEXT: [[PHI_SRC2:%.*]] = phi i64 [ [[TMP5]], [[LOADBB]] ], [ [[TMP18:%.*]], [[LOADBB1]] ]
; X64-NEXT: [[TMP7:%.*]] = icmp ult i64 [[PHI_SRC1]], [[PHI_SRC2]]
; X64-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i32 -1, i32 1
; X64-NEXT: [[PHI_SRC1:%.*]] = phi i64 [ [[TMP7:%.*]], [[LOADBB]] ], [ [[TMP18:%.*]], [[LOADBB1:%.*]] ]
; X64-NEXT: [[PHI_SRC2:%.*]] = phi i64 [ [[TMP8:%.*]], [[LOADBB]] ], [ [[TMP19:%.*]], [[LOADBB1]] ]
; X64-NEXT: [[TMP1:%.*]] = icmp ult i64 [[PHI_SRC1]], [[PHI_SRC2]]
; X64-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
; X64-NEXT: br label [[ENDBLOCK:%.*]]
; X64: loadbb:
; X64-NEXT: [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i64*
; X64-NEXT: [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i64*
; X64-NEXT: [[TMP5:%.*]] = load i64, i64* [[TMP3]]
; X64-NEXT: [[TMP6:%.*]] = load i64, i64* [[TMP4]]
; X64-NEXT: [[TMP7]] = call i64 @llvm.bswap.i64(i64 [[TMP5]])
; X64-NEXT: [[TMP8]] = call i64 @llvm.bswap.i64(i64 [[TMP6]])
; X64-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
; X64-NEXT: br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
; X64: loadbb1:
; X64-NEXT: [[TMP9:%.*]] = bitcast i8* [[X]] to i16*
; X64-NEXT: [[TMP10:%.*]] = bitcast i8* [[Y]] to i16*
; X64-NEXT: [[TMP11:%.*]] = getelementptr i16, i16* [[TMP9]], i16 4
; X64-NEXT: [[TMP10:%.*]] = bitcast i8* [[X]] to i16*
; X64-NEXT: [[TMP11:%.*]] = bitcast i8* [[Y]] to i16*
; X64-NEXT: [[TMP12:%.*]] = getelementptr i16, i16* [[TMP10]], i16 4
; X64-NEXT: [[TMP13:%.*]] = load i16, i16* [[TMP11]]
; X64-NEXT: [[TMP13:%.*]] = getelementptr i16, i16* [[TMP11]], i16 4
; X64-NEXT: [[TMP14:%.*]] = load i16, i16* [[TMP12]]
; X64-NEXT: [[TMP15:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP13]])
; X64-NEXT: [[TMP15:%.*]] = load i16, i16* [[TMP13]]
; X64-NEXT: [[TMP16:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP14]])
; X64-NEXT: [[TMP17]] = zext i16 [[TMP15]] to i64
; X64-NEXT: [[TMP17:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP15]])
; X64-NEXT: [[TMP18]] = zext i16 [[TMP16]] to i64
; X64-NEXT: [[TMP19:%.*]] = icmp eq i64 [[TMP17]], [[TMP18]]
; X64-NEXT: br i1 [[TMP19]], label [[ENDBLOCK]], label [[RES_BLOCK]]
; X64-NEXT: [[TMP19]] = zext i16 [[TMP17]] to i64
; X64-NEXT: [[TMP20:%.*]] = icmp eq i64 [[TMP18]], [[TMP19]]
; X64-NEXT: br i1 [[TMP20]], label [[ENDBLOCK]], label [[RES_BLOCK]]
; X64: endblock:
; X64-NEXT: [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP8]], [[RES_BLOCK]] ]
; X64-NEXT: [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
; X64-NEXT: ret i32 [[PHI_RES]]
;
%call = tail call i32 @memcmp(i8* %x, i8* %y, i64 10)
@ -294,36 +306,37 @@ define i32 @cmp12(i8* nocapture readonly %x, i8* nocapture readonly %y) {
; X32-NEXT: ret i32 [[CALL]]
;
; X64-LABEL: @cmp12(
; X64-NEXT: loadbb:
; X64-NEXT: [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i64*
; X64-NEXT: [[TMP1:%.*]] = bitcast i8* [[Y:%.*]] to i64*
; X64-NEXT: [[TMP2:%.*]] = load i64, i64* [[TMP0]]
; X64-NEXT: [[TMP3:%.*]] = load i64, i64* [[TMP1]]
; X64-NEXT: [[TMP4:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP2]])
; X64-NEXT: [[TMP5:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP3]])
; X64-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP4]], [[TMP5]]
; X64-NEXT: br i1 [[TMP6]], label [[LOADBB1:%.*]], label [[RES_BLOCK:%.*]]
; X64-NEXT: br label [[LOADBB:%.*]]
; X64: res_block:
; X64-NEXT: [[PHI_SRC1:%.*]] = phi i64 [ [[TMP4]], [[LOADBB:%.*]] ], [ [[TMP17:%.*]], [[LOADBB1]] ]
; X64-NEXT: [[PHI_SRC2:%.*]] = phi i64 [ [[TMP5]], [[LOADBB]] ], [ [[TMP18:%.*]], [[LOADBB1]] ]
; X64-NEXT: [[TMP7:%.*]] = icmp ult i64 [[PHI_SRC1]], [[PHI_SRC2]]
; X64-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i32 -1, i32 1
; X64-NEXT: [[PHI_SRC1:%.*]] = phi i64 [ [[TMP7:%.*]], [[LOADBB]] ], [ [[TMP18:%.*]], [[LOADBB1:%.*]] ]
; X64-NEXT: [[PHI_SRC2:%.*]] = phi i64 [ [[TMP8:%.*]], [[LOADBB]] ], [ [[TMP19:%.*]], [[LOADBB1]] ]
; X64-NEXT: [[TMP1:%.*]] = icmp ult i64 [[PHI_SRC1]], [[PHI_SRC2]]
; X64-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
; X64-NEXT: br label [[ENDBLOCK:%.*]]
; X64: loadbb:
; X64-NEXT: [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i64*
; X64-NEXT: [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i64*
; X64-NEXT: [[TMP5:%.*]] = load i64, i64* [[TMP3]]
; X64-NEXT: [[TMP6:%.*]] = load i64, i64* [[TMP4]]
; X64-NEXT: [[TMP7]] = call i64 @llvm.bswap.i64(i64 [[TMP5]])
; X64-NEXT: [[TMP8]] = call i64 @llvm.bswap.i64(i64 [[TMP6]])
; X64-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
; X64-NEXT: br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
; X64: loadbb1:
; X64-NEXT: [[TMP9:%.*]] = bitcast i8* [[X]] to i32*
; X64-NEXT: [[TMP10:%.*]] = bitcast i8* [[Y]] to i32*
; X64-NEXT: [[TMP11:%.*]] = getelementptr i32, i32* [[TMP9]], i32 2
; X64-NEXT: [[TMP10:%.*]] = bitcast i8* [[X]] to i32*
; X64-NEXT: [[TMP11:%.*]] = bitcast i8* [[Y]] to i32*
; X64-NEXT: [[TMP12:%.*]] = getelementptr i32, i32* [[TMP10]], i32 2
; X64-NEXT: [[TMP13:%.*]] = load i32, i32* [[TMP11]]
; X64-NEXT: [[TMP13:%.*]] = getelementptr i32, i32* [[TMP11]], i32 2
; X64-NEXT: [[TMP14:%.*]] = load i32, i32* [[TMP12]]
; X64-NEXT: [[TMP15:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP13]])
; X64-NEXT: [[TMP15:%.*]] = load i32, i32* [[TMP13]]
; X64-NEXT: [[TMP16:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP14]])
; X64-NEXT: [[TMP17]] = zext i32 [[TMP15]] to i64
; X64-NEXT: [[TMP17:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP15]])
; X64-NEXT: [[TMP18]] = zext i32 [[TMP16]] to i64
; X64-NEXT: [[TMP19:%.*]] = icmp eq i64 [[TMP17]], [[TMP18]]
; X64-NEXT: br i1 [[TMP19]], label [[ENDBLOCK]], label [[RES_BLOCK]]
; X64-NEXT: [[TMP19]] = zext i32 [[TMP17]] to i64
; X64-NEXT: [[TMP20:%.*]] = icmp eq i64 [[TMP18]], [[TMP19]]
; X64-NEXT: br i1 [[TMP20]], label [[ENDBLOCK]], label [[RES_BLOCK]]
; X64: endblock:
; X64-NEXT: [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP8]], [[RES_BLOCK]] ]
; X64-NEXT: [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
; X64-NEXT: ret i32 [[PHI_RES]]
;
%call = tail call i32 @memcmp(i8* %x, i8* %y, i64 12)
@ -363,34 +376,35 @@ define i32 @cmp16(i8* nocapture readonly %x, i8* nocapture readonly %y) {
; X32-NEXT: ret i32 [[CALL]]
;
; X64-LABEL: @cmp16(
; X64-NEXT: loadbb:
; X64-NEXT: [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i64*
; X64-NEXT: [[TMP1:%.*]] = bitcast i8* [[Y:%.*]] to i64*
; X64-NEXT: [[TMP2:%.*]] = load i64, i64* [[TMP0]]
; X64-NEXT: [[TMP3:%.*]] = load i64, i64* [[TMP1]]
; X64-NEXT: [[TMP4:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP2]])
; X64-NEXT: [[TMP5:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP3]])
; X64-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP4]], [[TMP5]]
; X64-NEXT: br i1 [[TMP6]], label [[LOADBB1:%.*]], label [[RES_BLOCK:%.*]]
; X64-NEXT: br label [[LOADBB:%.*]]
; X64: res_block:
; X64-NEXT: [[PHI_SRC1:%.*]] = phi i64 [ [[TMP4]], [[LOADBB:%.*]] ], [ [[TMP15:%.*]], [[LOADBB1]] ]
; X64-NEXT: [[PHI_SRC2:%.*]] = phi i64 [ [[TMP5]], [[LOADBB]] ], [ [[TMP16:%.*]], [[LOADBB1]] ]
; X64-NEXT: [[TMP7:%.*]] = icmp ult i64 [[PHI_SRC1]], [[PHI_SRC2]]
; X64-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i32 -1, i32 1
; X64-NEXT: [[PHI_SRC1:%.*]] = phi i64 [ [[TMP7:%.*]], [[LOADBB]] ], [ [[TMP16:%.*]], [[LOADBB1:%.*]] ]
; X64-NEXT: [[PHI_SRC2:%.*]] = phi i64 [ [[TMP8:%.*]], [[LOADBB]] ], [ [[TMP17:%.*]], [[LOADBB1]] ]
; X64-NEXT: [[TMP1:%.*]] = icmp ult i64 [[PHI_SRC1]], [[PHI_SRC2]]
; X64-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
; X64-NEXT: br label [[ENDBLOCK:%.*]]
; X64: loadbb:
; X64-NEXT: [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i64*
; X64-NEXT: [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i64*
; X64-NEXT: [[TMP5:%.*]] = load i64, i64* [[TMP3]]
; X64-NEXT: [[TMP6:%.*]] = load i64, i64* [[TMP4]]
; X64-NEXT: [[TMP7]] = call i64 @llvm.bswap.i64(i64 [[TMP5]])
; X64-NEXT: [[TMP8]] = call i64 @llvm.bswap.i64(i64 [[TMP6]])
; X64-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
; X64-NEXT: br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
; X64: loadbb1:
; X64-NEXT: [[TMP9:%.*]] = bitcast i8* [[X]] to i64*
; X64-NEXT: [[TMP10:%.*]] = bitcast i8* [[Y]] to i64*
; X64-NEXT: [[TMP11:%.*]] = getelementptr i64, i64* [[TMP9]], i64 1
; X64-NEXT: [[TMP10:%.*]] = bitcast i8* [[X]] to i64*
; X64-NEXT: [[TMP11:%.*]] = bitcast i8* [[Y]] to i64*
; X64-NEXT: [[TMP12:%.*]] = getelementptr i64, i64* [[TMP10]], i64 1
; X64-NEXT: [[TMP13:%.*]] = load i64, i64* [[TMP11]]
; X64-NEXT: [[TMP13:%.*]] = getelementptr i64, i64* [[TMP11]], i64 1
; X64-NEXT: [[TMP14:%.*]] = load i64, i64* [[TMP12]]
; X64-NEXT: [[TMP15]] = call i64 @llvm.bswap.i64(i64 [[TMP13]])
; X64-NEXT: [[TMP15:%.*]] = load i64, i64* [[TMP13]]
; X64-NEXT: [[TMP16]] = call i64 @llvm.bswap.i64(i64 [[TMP14]])
; X64-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP15]], [[TMP16]]
; X64-NEXT: br i1 [[TMP17]], label [[ENDBLOCK]], label [[RES_BLOCK]]
; X64-NEXT: [[TMP17]] = call i64 @llvm.bswap.i64(i64 [[TMP15]])
; X64-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP16]], [[TMP17]]
; X64-NEXT: br i1 [[TMP18]], label [[ENDBLOCK]], label [[RES_BLOCK]]
; X64: endblock:
; X64-NEXT: [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP8]], [[RES_BLOCK]] ]
; X64-NEXT: [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
; X64-NEXT: ret i32 [[PHI_RES]]
;
%call = tail call i32 @memcmp(i8* %x, i8* %y, i64 16)
@ -417,22 +431,23 @@ define i32 @cmp_eq2(i8* nocapture readonly %x, i8* nocapture readonly %y) {
define i32 @cmp_eq3(i8* nocapture readonly %x, i8* nocapture readonly %y) {
; ALL-LABEL: @cmp_eq3(
; ALL-NEXT: loadbb:
; ALL-NEXT: [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i16*
; ALL-NEXT: [[TMP1:%.*]] = bitcast i8* [[Y:%.*]] to i16*
; ALL-NEXT: [[TMP2:%.*]] = load i16, i16* [[TMP0]]
; ALL-NEXT: [[TMP3:%.*]] = load i16, i16* [[TMP1]]
; ALL-NEXT: [[TMP4:%.*]] = icmp ne i16 [[TMP2]], [[TMP3]]
; ALL-NEXT: br i1 [[TMP4]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
; ALL-NEXT: br label [[LOADBB:%.*]]
; ALL: res_block:
; ALL-NEXT: br label [[ENDBLOCK:%.*]]
; ALL: loadbb:
; ALL-NEXT: [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i16*
; ALL-NEXT: [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i16*
; ALL-NEXT: [[TMP3:%.*]] = load i16, i16* [[TMP1]]
; ALL-NEXT: [[TMP4:%.*]] = load i16, i16* [[TMP2]]
; ALL-NEXT: [[TMP5:%.*]] = icmp ne i16 [[TMP3]], [[TMP4]]
; ALL-NEXT: br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
; ALL: loadbb1:
; ALL-NEXT: [[TMP5:%.*]] = getelementptr i8, i8* [[X]], i8 2
; ALL-NEXT: [[TMP6:%.*]] = getelementptr i8, i8* [[Y]], i8 2
; ALL-NEXT: [[TMP7:%.*]] = load i8, i8* [[TMP5]]
; ALL-NEXT: [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 2
; ALL-NEXT: [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 2
; ALL-NEXT: [[TMP8:%.*]] = load i8, i8* [[TMP6]]
; ALL-NEXT: [[TMP9:%.*]] = icmp ne i8 [[TMP7]], [[TMP8]]
; ALL-NEXT: br i1 [[TMP9]], label [[RES_BLOCK]], label [[ENDBLOCK]]
; ALL-NEXT: [[TMP9:%.*]] = load i8, i8* [[TMP7]]
; ALL-NEXT: [[TMP10:%.*]] = icmp ne i8 [[TMP8]], [[TMP9]]
; ALL-NEXT: br i1 [[TMP10]], label [[RES_BLOCK]], label [[ENDBLOCK]]
; ALL: endblock:
; ALL-NEXT: [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
; ALL-NEXT: [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
@ -465,22 +480,23 @@ define i32 @cmp_eq4(i8* nocapture readonly %x, i8* nocapture readonly %y) {
define i32 @cmp_eq5(i8* nocapture readonly %x, i8* nocapture readonly %y) {
; ALL-LABEL: @cmp_eq5(
; ALL-NEXT: loadbb:
; ALL-NEXT: [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i32*
; ALL-NEXT: [[TMP1:%.*]] = bitcast i8* [[Y:%.*]] to i32*
; ALL-NEXT: [[TMP2:%.*]] = load i32, i32* [[TMP0]]
; ALL-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1]]
; ALL-NEXT: [[TMP4:%.*]] = icmp ne i32 [[TMP2]], [[TMP3]]
; ALL-NEXT: br i1 [[TMP4]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
; ALL-NEXT: br label [[LOADBB:%.*]]
; ALL: res_block:
; ALL-NEXT: br label [[ENDBLOCK:%.*]]
; ALL: loadbb:
; ALL-NEXT: [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
; ALL-NEXT: [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
; ALL-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1]]
; ALL-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP2]]
; ALL-NEXT: [[TMP5:%.*]] = icmp ne i32 [[TMP3]], [[TMP4]]
; ALL-NEXT: br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
; ALL: loadbb1:
; ALL-NEXT: [[TMP5:%.*]] = getelementptr i8, i8* [[X]], i8 4
; ALL-NEXT: [[TMP6:%.*]] = getelementptr i8, i8* [[Y]], i8 4
; ALL-NEXT: [[TMP7:%.*]] = load i8, i8* [[TMP5]]
; ALL-NEXT: [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
; ALL-NEXT: [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 4
; ALL-NEXT: [[TMP8:%.*]] = load i8, i8* [[TMP6]]
; ALL-NEXT: [[TMP9:%.*]] = icmp ne i8 [[TMP7]], [[TMP8]]
; ALL-NEXT: br i1 [[TMP9]], label [[RES_BLOCK]], label [[ENDBLOCK]]
; ALL-NEXT: [[TMP9:%.*]] = load i8, i8* [[TMP7]]
; ALL-NEXT: [[TMP10:%.*]] = icmp ne i8 [[TMP8]], [[TMP9]]
; ALL-NEXT: br i1 [[TMP10]], label [[RES_BLOCK]], label [[ENDBLOCK]]
; ALL: endblock:
; ALL-NEXT: [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
; ALL-NEXT: [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
@ -495,24 +511,25 @@ define i32 @cmp_eq5(i8* nocapture readonly %x, i8* nocapture readonly %y) {
define i32 @cmp_eq6(i8* nocapture readonly %x, i8* nocapture readonly %y) {
; ALL-LABEL: @cmp_eq6(
; ALL-NEXT: loadbb:
; ALL-NEXT: [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i32*
; ALL-NEXT: [[TMP1:%.*]] = bitcast i8* [[Y:%.*]] to i32*
; ALL-NEXT: [[TMP2:%.*]] = load i32, i32* [[TMP0]]
; ALL-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1]]
; ALL-NEXT: [[TMP4:%.*]] = icmp ne i32 [[TMP2]], [[TMP3]]
; ALL-NEXT: br i1 [[TMP4]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
; ALL-NEXT: br label [[LOADBB:%.*]]
; ALL: res_block:
; ALL-NEXT: br label [[ENDBLOCK:%.*]]
; ALL: loadbb:
; ALL-NEXT: [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
; ALL-NEXT: [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
; ALL-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1]]
; ALL-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP2]]
; ALL-NEXT: [[TMP5:%.*]] = icmp ne i32 [[TMP3]], [[TMP4]]
; ALL-NEXT: br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
; ALL: loadbb1:
; ALL-NEXT: [[TMP5:%.*]] = bitcast i8* [[X]] to i16*
; ALL-NEXT: [[TMP6:%.*]] = bitcast i8* [[Y]] to i16*
; ALL-NEXT: [[TMP7:%.*]] = getelementptr i16, i16* [[TMP5]], i16 2
; ALL-NEXT: [[TMP6:%.*]] = bitcast i8* [[X]] to i16*
; ALL-NEXT: [[TMP7:%.*]] = bitcast i8* [[Y]] to i16*
; ALL-NEXT: [[TMP8:%.*]] = getelementptr i16, i16* [[TMP6]], i16 2
; ALL-NEXT: [[TMP9:%.*]] = load i16, i16* [[TMP7]]
; ALL-NEXT: [[TMP9:%.*]] = getelementptr i16, i16* [[TMP7]], i16 2
; ALL-NEXT: [[TMP10:%.*]] = load i16, i16* [[TMP8]]
; ALL-NEXT: [[TMP11:%.*]] = icmp ne i16 [[TMP9]], [[TMP10]]
; ALL-NEXT: br i1 [[TMP11]], label [[RES_BLOCK]], label [[ENDBLOCK]]
; ALL-NEXT: [[TMP11:%.*]] = load i16, i16* [[TMP9]]
; ALL-NEXT: [[TMP12:%.*]] = icmp ne i16 [[TMP10]], [[TMP11]]
; ALL-NEXT: br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
; ALL: endblock:
; ALL-NEXT: [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
; ALL-NEXT: [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
@ -540,24 +557,25 @@ define i32 @cmp_eq7(i8* nocapture readonly %x, i8* nocapture readonly %y) {
define i32 @cmp_eq8(i8* nocapture readonly %x, i8* nocapture readonly %y) {
; X32-LABEL: @cmp_eq8(
; X32-NEXT: loadbb:
; X32-NEXT: [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i32*
; X32-NEXT: [[TMP1:%.*]] = bitcast i8* [[Y:%.*]] to i32*
; X32-NEXT: [[TMP2:%.*]] = load i32, i32* [[TMP0]]
; X32-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1]]
; X32-NEXT: [[TMP4:%.*]] = icmp ne i32 [[TMP2]], [[TMP3]]
; X32-NEXT: br i1 [[TMP4]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
; X32-NEXT: br label [[LOADBB:%.*]]
; X32: res_block:
; X32-NEXT: br label [[ENDBLOCK:%.*]]
; X32: loadbb:
; X32-NEXT: [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
; X32-NEXT: [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
; X32-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1]]
; X32-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP2]]
; X32-NEXT: [[TMP5:%.*]] = icmp ne i32 [[TMP3]], [[TMP4]]
; X32-NEXT: br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
; X32: loadbb1:
; X32-NEXT: [[TMP5:%.*]] = bitcast i8* [[X]] to i32*
; X32-NEXT: [[TMP6:%.*]] = bitcast i8* [[Y]] to i32*
; X32-NEXT: [[TMP7:%.*]] = getelementptr i32, i32* [[TMP5]], i32 1
; X32-NEXT: [[TMP6:%.*]] = bitcast i8* [[X]] to i32*
; X32-NEXT: [[TMP7:%.*]] = bitcast i8* [[Y]] to i32*
; X32-NEXT: [[TMP8:%.*]] = getelementptr i32, i32* [[TMP6]], i32 1
; X32-NEXT: [[TMP9:%.*]] = load i32, i32* [[TMP7]]
; X32-NEXT: [[TMP9:%.*]] = getelementptr i32, i32* [[TMP7]], i32 1
; X32-NEXT: [[TMP10:%.*]] = load i32, i32* [[TMP8]]
; X32-NEXT: [[TMP11:%.*]] = icmp ne i32 [[TMP9]], [[TMP10]]
; X32-NEXT: br i1 [[TMP11]], label [[RES_BLOCK]], label [[ENDBLOCK]]
; X32-NEXT: [[TMP11:%.*]] = load i32, i32* [[TMP9]]
; X32-NEXT: [[TMP12:%.*]] = icmp ne i32 [[TMP10]], [[TMP11]]
; X32-NEXT: br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
; X32: endblock:
; X32-NEXT: [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
; X32-NEXT: [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
@ -589,22 +607,23 @@ define i32 @cmp_eq9(i8* nocapture readonly %x, i8* nocapture readonly %y) {
; X32-NEXT: ret i32 [[CONV]]
;
; X64-LABEL: @cmp_eq9(
; X64-NEXT: loadbb:
; X64-NEXT: [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i64*
; X64-NEXT: [[TMP1:%.*]] = bitcast i8* [[Y:%.*]] to i64*
; X64-NEXT: [[TMP2:%.*]] = load i64, i64* [[TMP0]]
; X64-NEXT: [[TMP3:%.*]] = load i64, i64* [[TMP1]]
; X64-NEXT: [[TMP4:%.*]] = icmp ne i64 [[TMP2]], [[TMP3]]
; X64-NEXT: br i1 [[TMP4]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
; X64-NEXT: br label [[LOADBB:%.*]]
; X64: res_block:
; X64-NEXT: br label [[ENDBLOCK:%.*]]
; X64: loadbb:
; X64-NEXT: [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
; X64-NEXT: [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
; X64-NEXT: [[TMP3:%.*]] = load i64, i64* [[TMP1]]
; X64-NEXT: [[TMP4:%.*]] = load i64, i64* [[TMP2]]
; X64-NEXT: [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
; X64-NEXT: br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
; X64: loadbb1:
; X64-NEXT: [[TMP5:%.*]] = getelementptr i8, i8* [[X]], i8 8
; X64-NEXT: [[TMP6:%.*]] = getelementptr i8, i8* [[Y]], i8 8
; X64-NEXT: [[TMP7:%.*]] = load i8, i8* [[TMP5]]
; X64-NEXT: [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 8
; X64-NEXT: [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 8
; X64-NEXT: [[TMP8:%.*]] = load i8, i8* [[TMP6]]
; X64-NEXT: [[TMP9:%.*]] = icmp ne i8 [[TMP7]], [[TMP8]]
; X64-NEXT: br i1 [[TMP9]], label [[RES_BLOCK]], label [[ENDBLOCK]]
; X64-NEXT: [[TMP9:%.*]] = load i8, i8* [[TMP7]]
; X64-NEXT: [[TMP10:%.*]] = icmp ne i8 [[TMP8]], [[TMP9]]
; X64-NEXT: br i1 [[TMP10]], label [[RES_BLOCK]], label [[ENDBLOCK]]
; X64: endblock:
; X64-NEXT: [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
; X64-NEXT: [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
@ -625,24 +644,25 @@ define i32 @cmp_eq10(i8* nocapture readonly %x, i8* nocapture readonly %y) {
; X32-NEXT: ret i32 [[CONV]]
;
; X64-LABEL: @cmp_eq10(
; X64-NEXT: loadbb:
; X64-NEXT: [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i64*
; X64-NEXT: [[TMP1:%.*]] = bitcast i8* [[Y:%.*]] to i64*
; X64-NEXT: [[TMP2:%.*]] = load i64, i64* [[TMP0]]
; X64-NEXT: [[TMP3:%.*]] = load i64, i64* [[TMP1]]
; X64-NEXT: [[TMP4:%.*]] = icmp ne i64 [[TMP2]], [[TMP3]]
; X64-NEXT: br i1 [[TMP4]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
; X64-NEXT: br label [[LOADBB:%.*]]
; X64: res_block:
; X64-NEXT: br label [[ENDBLOCK:%.*]]
; X64: loadbb:
; X64-NEXT: [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
; X64-NEXT: [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
; X64-NEXT: [[TMP3:%.*]] = load i64, i64* [[TMP1]]
; X64-NEXT: [[TMP4:%.*]] = load i64, i64* [[TMP2]]
; X64-NEXT: [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
; X64-NEXT: br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
; X64: loadbb1:
; X64-NEXT: [[TMP5:%.*]] = bitcast i8* [[X]] to i16*
; X64-NEXT: [[TMP6:%.*]] = bitcast i8* [[Y]] to i16*
; X64-NEXT: [[TMP7:%.*]] = getelementptr i16, i16* [[TMP5]], i16 4
; X64-NEXT: [[TMP6:%.*]] = bitcast i8* [[X]] to i16*
; X64-NEXT: [[TMP7:%.*]] = bitcast i8* [[Y]] to i16*
; X64-NEXT: [[TMP8:%.*]] = getelementptr i16, i16* [[TMP6]], i16 4
; X64-NEXT: [[TMP9:%.*]] = load i16, i16* [[TMP7]]
; X64-NEXT: [[TMP9:%.*]] = getelementptr i16, i16* [[TMP7]], i16 4
; X64-NEXT: [[TMP10:%.*]] = load i16, i16* [[TMP8]]
; X64-NEXT: [[TMP11:%.*]] = icmp ne i16 [[TMP9]], [[TMP10]]
; X64-NEXT: br i1 [[TMP11]], label [[RES_BLOCK]], label [[ENDBLOCK]]
; X64-NEXT: [[TMP11:%.*]] = load i16, i16* [[TMP9]]
; X64-NEXT: [[TMP12:%.*]] = icmp ne i16 [[TMP10]], [[TMP11]]
; X64-NEXT: br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
; X64: endblock:
; X64-NEXT: [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
; X64-NEXT: [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
@ -676,24 +696,25 @@ define i32 @cmp_eq12(i8* nocapture readonly %x, i8* nocapture readonly %y) {
; X32-NEXT: ret i32 [[CONV]]
;
; X64-LABEL: @cmp_eq12(
; X64-NEXT: loadbb:
; X64-NEXT: [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i64*
; X64-NEXT: [[TMP1:%.*]] = bitcast i8* [[Y:%.*]] to i64*
; X64-NEXT: [[TMP2:%.*]] = load i64, i64* [[TMP0]]
; X64-NEXT: [[TMP3:%.*]] = load i64, i64* [[TMP1]]
; X64-NEXT: [[TMP4:%.*]] = icmp ne i64 [[TMP2]], [[TMP3]]
; X64-NEXT: br i1 [[TMP4]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
; X64-NEXT: br label [[LOADBB:%.*]]
; X64: res_block:
; X64-NEXT: br label [[ENDBLOCK:%.*]]
; X64: loadbb:
; X64-NEXT: [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
; X64-NEXT: [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
; X64-NEXT: [[TMP3:%.*]] = load i64, i64* [[TMP1]]
; X64-NEXT: [[TMP4:%.*]] = load i64, i64* [[TMP2]]
; X64-NEXT: [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
; X64-NEXT: br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
; X64: loadbb1:
; X64-NEXT: [[TMP5:%.*]] = bitcast i8* [[X]] to i32*
; X64-NEXT: [[TMP6:%.*]] = bitcast i8* [[Y]] to i32*
; X64-NEXT: [[TMP7:%.*]] = getelementptr i32, i32* [[TMP5]], i32 2
; X64-NEXT: [[TMP6:%.*]] = bitcast i8* [[X]] to i32*
; X64-NEXT: [[TMP7:%.*]] = bitcast i8* [[Y]] to i32*
; X64-NEXT: [[TMP8:%.*]] = getelementptr i32, i32* [[TMP6]], i32 2
; X64-NEXT: [[TMP9:%.*]] = load i32, i32* [[TMP7]]
; X64-NEXT: [[TMP9:%.*]] = getelementptr i32, i32* [[TMP7]], i32 2
; X64-NEXT: [[TMP10:%.*]] = load i32, i32* [[TMP8]]
; X64-NEXT: [[TMP11:%.*]] = icmp ne i32 [[TMP9]], [[TMP10]]
; X64-NEXT: br i1 [[TMP11]], label [[RES_BLOCK]], label [[ENDBLOCK]]
; X64-NEXT: [[TMP11:%.*]] = load i32, i32* [[TMP9]]
; X64-NEXT: [[TMP12:%.*]] = icmp ne i32 [[TMP10]], [[TMP11]]
; X64-NEXT: br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
; X64: endblock:
; X64-NEXT: [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
; X64-NEXT: [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0