2009-05-23 20:35:30 +08:00
|
|
|
//===-- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ---===//
|
|
|
|
//
|
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
//
|
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
// License. See LICENSE.TXT for details.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
|
|
|
// This file implements the SelectionDAG::LegalizeVectors method.
|
|
|
|
//
|
|
|
|
// The vector legalizer looks for vector operations which might need to be
|
2009-05-27 15:58:35 +08:00
|
|
|
// scalarized and legalizes them. This is a separate step from Legalize because
|
|
|
|
// scalarizing can introduce illegal types. For example, suppose we have an
|
2009-05-23 20:35:30 +08:00
|
|
|
// ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition
|
|
|
|
// on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
|
|
|
|
// operation, which introduces nodes with the illegal type i64 which must be
|
|
|
|
// expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
|
|
|
|
// the operation must be unrolled, which introduces nodes with the illegal
|
|
|
|
// type i8 which must be promoted.
|
|
|
|
//
|
|
|
|
// This does not legalize vector manipulations like ISD::BUILD_VECTOR,
|
Major calling convention code refactoring.
Instead of awkwardly encoding calling-convention information with ISD::CALL,
ISD::FORMAL_ARGUMENTS, ISD::RET, and ISD::ARG_FLAGS nodes, TargetLowering
provides three virtual functions for targets to override:
LowerFormalArguments, LowerCall, and LowerRet, which replace the custom
lowering done on the special nodes. They provide the same information, but
in a more immediately usable format.
This also reworks much of the target-independent tail call logic. The
decision of whether or not to perform a tail call is now cleanly split
between target-independent portions, and the target dependent portion
in IsEligibleForTailCallOptimization.
This also synchronizes all in-tree targets, to help enable future
refactoring and feature work.
llvm-svn: 78142
2009-08-05 09:29:28 +08:00
|
|
|
// or operations that happen to take a vector which are custom-lowered;
|
|
|
|
// the legalization for such operations never produces nodes
|
2009-05-23 20:35:30 +08:00
|
|
|
// with illegal types, so it's okay to put off legalizing them until
|
|
|
|
// SelectionDAG::Legalize runs.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
#include "llvm/CodeGen/SelectionDAG.h"
|
|
|
|
#include "llvm/Target/TargetLowering.h"
|
|
|
|
using namespace llvm;
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
class VectorLegalizer {
|
|
|
|
SelectionDAG& DAG;
|
2010-04-17 23:26:15 +08:00
|
|
|
const TargetLowering &TLI;
|
2009-05-23 20:35:30 +08:00
|
|
|
bool Changed; // Keep track of whether anything changed
|
|
|
|
|
2014-07-02 10:16:57 +08:00
|
|
|
/// For nodes that are of legal width, and that have more than one use, this
|
|
|
|
/// map indicates what regularized operand to use. This allows us to avoid
|
|
|
|
/// legalizing the same thing more than once.
|
2013-01-25 23:18:54 +08:00
|
|
|
SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes;
|
2009-05-23 20:35:30 +08:00
|
|
|
|
2014-07-02 10:16:57 +08:00
|
|
|
/// \brief Adds a node to the translation cache.
|
2009-05-23 20:35:30 +08:00
|
|
|
void AddLegalizedOperand(SDValue From, SDValue To) {
|
|
|
|
LegalizedNodes.insert(std::make_pair(From, To));
|
|
|
|
// If someone requests legalization of the new node, return itself.
|
|
|
|
if (From != To)
|
|
|
|
LegalizedNodes.insert(std::make_pair(To, To));
|
|
|
|
}
|
|
|
|
|
2014-07-02 10:16:57 +08:00
|
|
|
/// \brief Legalizes the given node.
|
2009-05-23 20:35:30 +08:00
|
|
|
SDValue LegalizeOp(SDValue Op);
|
2014-07-02 10:16:57 +08:00
|
|
|
|
|
|
|
/// \brief Assuming the node is legal, "legalize" the results.
|
2009-05-23 20:35:30 +08:00
|
|
|
SDValue TranslateLegalizeResults(SDValue Op, SDValue Result);
|
2014-07-02 10:16:57 +08:00
|
|
|
|
|
|
|
/// \brief Implements unrolling a VSETCC.
|
2009-05-23 20:35:30 +08:00
|
|
|
SDValue UnrollVSETCC(SDValue Op);
|
2014-07-02 10:16:57 +08:00
|
|
|
|
2014-07-02 14:23:34 +08:00
|
|
|
/// \brief Implement expand-based legalization of vector operations.
|
|
|
|
///
|
|
|
|
/// This is just a high-level routine to dispatch to specific code paths for
|
|
|
|
/// operations to legalize them.
|
|
|
|
SDValue Expand(SDValue Op);
|
|
|
|
|
2014-07-02 10:16:57 +08:00
|
|
|
/// \brief Implements expansion for FNEG; falls back to UnrollVectorOp if
|
|
|
|
/// FSUB isn't legal.
|
|
|
|
///
|
|
|
|
/// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
|
|
|
|
/// SINT_TO_FLOAT and SHR on vectors isn't legal.
|
2011-03-19 21:09:10 +08:00
|
|
|
SDValue ExpandUINT_TO_FLOAT(SDValue Op);
|
2014-07-02 10:16:57 +08:00
|
|
|
|
|
|
|
/// \brief Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
|
2013-01-12 06:57:48 +08:00
|
|
|
SDValue ExpandSEXTINREG(SDValue Op);
|
2014-07-02 10:16:57 +08:00
|
|
|
|
2014-07-10 20:32:32 +08:00
|
|
|
/// \brief Implement expansion for ANY_EXTEND_VECTOR_INREG.
|
|
|
|
///
|
|
|
|
/// Shuffles the low lanes of the operand into place and bitcasts to the proper
|
|
|
|
/// type. The contents of the bits in the extended part of each element are
|
|
|
|
/// undef.
|
|
|
|
SDValue ExpandANY_EXTEND_VECTOR_INREG(SDValue Op);
|
|
|
|
|
|
|
|
/// \brief Implement expansion for SIGN_EXTEND_VECTOR_INREG.
|
|
|
|
///
|
|
|
|
/// Shuffles the low lanes of the operand into place, bitcasts to the proper
|
|
|
|
/// type, then shifts left and arithmetic shifts right to introduce a sign
|
|
|
|
/// extension.
|
|
|
|
SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDValue Op);
|
|
|
|
|
2014-07-09 18:58:18 +08:00
|
|
|
/// \brief Implement expansion for ZERO_EXTEND_VECTOR_INREG.
|
|
|
|
///
|
|
|
|
/// Shuffles the low lanes of the operand into place and blends zeros into
|
|
|
|
/// the remaining lanes, finally bitcasting to the proper type.
|
|
|
|
SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDValue Op);
|
|
|
|
|
2014-07-02 10:16:57 +08:00
|
|
|
/// \brief Expand bswap of vectors into a shuffle if legal.
|
2014-05-19 21:12:38 +08:00
|
|
|
SDValue ExpandBSWAP(SDValue Op);
|
2014-07-02 10:16:57 +08:00
|
|
|
|
|
|
|
/// \brief Implement vselect in terms of XOR, AND, OR when blend is not
|
|
|
|
/// supported by the target.
|
2011-09-14 03:17:42 +08:00
|
|
|
SDValue ExpandVSELECT(SDValue Op);
|
2012-08-31 03:17:29 +08:00
|
|
|
SDValue ExpandSELECT(SDValue Op);
|
2011-10-15 15:41:10 +08:00
|
|
|
SDValue ExpandLoad(SDValue Op);
|
|
|
|
SDValue ExpandStore(SDValue Op);
|
2009-05-23 20:35:30 +08:00
|
|
|
SDValue ExpandFNEG(SDValue Op);
|
2015-07-16 23:22:46 +08:00
|
|
|
SDValue ExpandABSDIFF(SDValue Op);
|
2014-07-02 10:16:57 +08:00
|
|
|
|
|
|
|
/// \brief Implements vector promotion.
|
|
|
|
///
|
|
|
|
/// This is essentially just bitcasting the operands to a different type and
|
|
|
|
/// bitcasting the result back to the original type.
|
2014-07-02 11:07:11 +08:00
|
|
|
SDValue Promote(SDValue Op);
|
2014-07-02 10:16:57 +08:00
|
|
|
|
|
|
|
/// \brief Implements [SU]INT_TO_FP vector promotion.
|
|
|
|
///
|
|
|
|
/// This is a [zs]ext of the input operand to the next size up.
|
2014-07-02 11:07:11 +08:00
|
|
|
SDValue PromoteINT_TO_FP(SDValue Op);
|
2014-07-02 10:16:57 +08:00
|
|
|
|
|
|
|
/// \brief Implements FP_TO_[SU]INT vector promotion of the result type.
|
|
|
|
///
|
|
|
|
/// It is promoted to the next size up integer type. The result is then
|
|
|
|
/// truncated back to the original type.
|
2014-07-02 11:07:11 +08:00
|
|
|
SDValue PromoteFP_TO_INT(SDValue Op, bool isSigned);
|
2009-05-23 20:35:30 +08:00
|
|
|
|
2014-07-02 10:16:57 +08:00
|
|
|
public:
|
|
|
|
/// \brief Begin legalizer the vector operations in the DAG.
|
2009-05-23 20:35:30 +08:00
|
|
|
bool Run();
|
|
|
|
VectorLegalizer(SelectionDAG& dag) :
|
|
|
|
DAG(dag), TLI(dag.getTargetLoweringInfo()), Changed(false) {}
|
|
|
|
};
|
|
|
|
|
|
|
|
bool VectorLegalizer::Run() {
|
2013-02-23 07:33:30 +08:00
|
|
|
// Before we start legalizing vector nodes, check if there are any vectors.
|
|
|
|
bool HasVectors = false;
|
|
|
|
for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
|
2014-03-02 20:27:27 +08:00
|
|
|
E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) {
|
2013-02-23 07:33:30 +08:00
|
|
|
// Check if the values of the nodes contain vectors. We don't need to check
|
|
|
|
// the operands because we are going to check their values at some point.
|
|
|
|
for (SDNode::value_iterator J = I->value_begin(), E = I->value_end();
|
|
|
|
J != E; ++J)
|
|
|
|
HasVectors |= J->isVector();
|
|
|
|
|
|
|
|
// If we found a vector node we can start the legalization.
|
|
|
|
if (HasVectors)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
// If this basic block has no vectors then no need to legalize vectors.
|
|
|
|
if (!HasVectors)
|
|
|
|
return false;
|
|
|
|
|
2009-05-23 20:35:30 +08:00
|
|
|
// The legalize process is inherently a bottom-up recursive process (users
|
|
|
|
// legalize their uses before themselves). Given infinite stack space, we
|
|
|
|
// could just start legalizing on the root and traverse the whole graph. In
|
|
|
|
// practice however, this causes us to run out of stack space on large basic
|
|
|
|
// blocks. To avoid this problem, compute an ordering of the nodes where each
|
|
|
|
// node is only legalized after all of its operands are legalized.
|
|
|
|
DAG.AssignTopologicalOrder();
|
|
|
|
for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
|
2014-03-02 20:27:27 +08:00
|
|
|
E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I)
|
2015-10-14 03:47:46 +08:00
|
|
|
LegalizeOp(SDValue(&*I, 0));
|
2009-05-23 20:35:30 +08:00
|
|
|
|
|
|
|
// Finally, it's possible the root changed. Get the new root.
|
|
|
|
SDValue OldRoot = DAG.getRoot();
|
|
|
|
assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
|
|
|
|
DAG.setRoot(LegalizedNodes[OldRoot]);
|
|
|
|
|
|
|
|
LegalizedNodes.clear();
|
|
|
|
|
|
|
|
// Remove dead nodes now.
|
|
|
|
DAG.RemoveDeadNodes();
|
|
|
|
|
|
|
|
return Changed;
|
|
|
|
}
|
|
|
|
|
|
|
|
SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDValue Result) {
|
|
|
|
// Generic legalization: just pass the operand through.
|
|
|
|
for (unsigned i = 0, e = Op.getNode()->getNumValues(); i != e; ++i)
|
|
|
|
AddLegalizedOperand(Op.getValue(i), Result.getValue(i));
|
|
|
|
return Result.getValue(Op.getResNo());
|
|
|
|
}
|
|
|
|
|
|
|
|
SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
|
|
|
|
// Note that LegalizeOp may be reentered even from single-use nodes, which
|
|
|
|
// means that we always must cache transformed nodes.
|
|
|
|
DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);
|
|
|
|
if (I != LegalizedNodes.end()) return I->second;
|
|
|
|
|
|
|
|
SDNode* Node = Op.getNode();
|
|
|
|
|
|
|
|
// Legalize the operands
|
|
|
|
SmallVector<SDValue, 8> Ops;
|
2015-06-27 03:08:33 +08:00
|
|
|
for (const SDValue &Op : Node->op_values())
|
|
|
|
Ops.push_back(LegalizeOp(Op));
|
2009-05-23 20:35:30 +08:00
|
|
|
|
2014-04-28 13:57:50 +08:00
|
|
|
SDValue Result = SDValue(DAG.UpdateNodeOperands(Op.getNode(), Ops), 0);
|
2009-05-23 20:35:30 +08:00
|
|
|
|
2015-05-03 15:12:25 +08:00
|
|
|
bool HasVectorValue = false;
|
2011-10-15 15:41:10 +08:00
|
|
|
if (Op.getOpcode() == ISD::LOAD) {
|
|
|
|
LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
|
|
|
|
ISD::LoadExtType ExtType = LD->getExtensionType();
|
[x86] Make vector legalization of extloads work more like the "normal"
vector operation legalization with support for custom target lowering
and fallback to expand when it fails, and use this to implement sext and
anyext load lowering for x86 in a more principled way.
Previously, the x86 backend relied on a target DAG combine to "combine
away" sextload and extload nodes prior to legalization, or would expand
them during legalization with terrible code. This is particularly
problematic because the DAG combine relies on running over non-canonical
DAG nodes at just the right time to match several common and important
patterns. It used a combine rather than lowering because we didn't have
good lowering support, and to expose some tricks being employed to more
combine phases.
With this change it becomes a proper lowering operation, the backend
marks that it can lower these nodes, and I've added support for handling
the canonical forms that don't have direct legal representations such as
sextload of a v4i8 -> v4i64 on AVX1. With this change, our test cases
for this behavior continue to pass even after the DAG combiner beigns
running more systematically over every node.
There is some noise caused by this in the test suite where we actually
use vector extends instead of subregister extraction. This doesn't
really seem like the right thing to do, but is unlikely to be a critical
regression. We do regress in one case where by lowering to the
target-specific patterns early we were able to combine away extraneous
legal math nodes. However, this regression is completely addressed by
switching to a widening based legalization which is what I'm working
toward anyways, so I've just switched the test to that mode.
Differential Revision: http://reviews.llvm.org/D4654
llvm-svn: 213897
2014-07-25 06:09:56 +08:00
|
|
|
if (LD->getMemoryVT().isVector() && ExtType != ISD::NON_EXTLOAD)
|
[SelectionDAG] Allow targets to specify legality of extloads' result
type (in addition to the memory type).
The *LoadExt* legalization handling used to only have one type, the
memory type. This forced users to assume that as long as the extload
for the memory type was declared legal, and the result type was legal,
the whole extload was legal.
However, this isn't always the case. For instance, on X86, with AVX,
this is legal:
v4i32 load, zext from v4i8
but this isn't:
v4i64 load, zext from v4i8
Whereas v4i64 is (arguably) legal, even without AVX2.
Note that the same thing was done a while ago for truncstores (r46140),
but I assume no one needed it yet for extloads, so here we go.
Calls to getLoadExtAction were changed to add the value type, found
manually in the surrounding code.
Calls to setLoadExtAction were mechanically changed, by wrapping the
call in a loop, to match previous behavior. The loop iterates over
the MVT subrange corresponding to the memory type (FP vectors, etc...).
I also pulled neighboring setTruncStoreActions into some of the loops;
those shouldn't make a difference, as the additional types are illegal.
(e.g., i128->i1 truncstores on PPC.)
No functional change intended.
Differential Revision: http://reviews.llvm.org/D6532
llvm-svn: 225421
2015-01-08 08:51:32 +08:00
|
|
|
switch (TLI.getLoadExtAction(LD->getExtensionType(), LD->getValueType(0),
|
|
|
|
LD->getMemoryVT())) {
|
[x86] Make vector legalization of extloads work more like the "normal"
vector operation legalization with support for custom target lowering
and fallback to expand when it fails, and use this to implement sext and
anyext load lowering for x86 in a more principled way.
Previously, the x86 backend relied on a target DAG combine to "combine
away" sextload and extload nodes prior to legalization, or would expand
them during legalization with terrible code. This is particularly
problematic because the DAG combine relies on running over non-canonical
DAG nodes at just the right time to match several common and important
patterns. It used a combine rather than lowering because we didn't have
good lowering support, and to expose some tricks being employed to more
combine phases.
With this change it becomes a proper lowering operation, the backend
marks that it can lower these nodes, and I've added support for handling
the canonical forms that don't have direct legal representations such as
sextload of a v4i8 -> v4i64 on AVX1. With this change, our test cases
for this behavior continue to pass even after the DAG combiner beigns
running more systematically over every node.
There is some noise caused by this in the test suite where we actually
use vector extends instead of subregister extraction. This doesn't
really seem like the right thing to do, but is unlikely to be a critical
regression. We do regress in one case where by lowering to the
target-specific patterns early we were able to combine away extraneous
legal math nodes. However, this regression is completely addressed by
switching to a widening based legalization which is what I'm working
toward anyways, so I've just switched the test to that mode.
Differential Revision: http://reviews.llvm.org/D4654
llvm-svn: 213897
2014-07-25 06:09:56 +08:00
|
|
|
default: llvm_unreachable("This action is not supported yet!");
|
|
|
|
case TargetLowering::Legal:
|
2011-10-15 15:41:10 +08:00
|
|
|
return TranslateLegalizeResults(Op, Result);
|
[x86] Make vector legalization of extloads work more like the "normal"
vector operation legalization with support for custom target lowering
and fallback to expand when it fails, and use this to implement sext and
anyext load lowering for x86 in a more principled way.
Previously, the x86 backend relied on a target DAG combine to "combine
away" sextload and extload nodes prior to legalization, or would expand
them during legalization with terrible code. This is particularly
problematic because the DAG combine relies on running over non-canonical
DAG nodes at just the right time to match several common and important
patterns. It used a combine rather than lowering because we didn't have
good lowering support, and to expose some tricks being employed to more
combine phases.
With this change it becomes a proper lowering operation, the backend
marks that it can lower these nodes, and I've added support for handling
the canonical forms that don't have direct legal representations such as
sextload of a v4i8 -> v4i64 on AVX1. With this change, our test cases
for this behavior continue to pass even after the DAG combiner beigns
running more systematically over every node.
There is some noise caused by this in the test suite where we actually
use vector extends instead of subregister extraction. This doesn't
really seem like the right thing to do, but is unlikely to be a critical
regression. We do regress in one case where by lowering to the
target-specific patterns early we were able to combine away extraneous
legal math nodes. However, this regression is completely addressed by
switching to a widening based legalization which is what I'm working
toward anyways, so I've just switched the test to that mode.
Differential Revision: http://reviews.llvm.org/D4654
llvm-svn: 213897
2014-07-25 06:09:56 +08:00
|
|
|
case TargetLowering::Custom:
|
|
|
|
if (SDValue Lowered = TLI.LowerOperation(Result, DAG)) {
|
[SDAG] Handle LowerOperation returning its input consistently
For almost all node types, if the target requested custom lowering, and
LowerOperation returned its input, we'd treat the original node as legal. This
did not work, however, for many loads and stores, because they follow
slightly different code paths, and we did not account for the possibility of
LowerOperation returning its input at those call sites.
I think that we now handle this consistently everywhere. At the call sites in
LegalizeDAG, we used to assert in this case, so there's no functional change
for any existing code there. For the call sites in LegalizeVectorOps, this
really only affects whether or not we set Changed = true, but I think makes the
semantics clearer.
No test case here, but it will be covered by an upcoming PowerPC commit adding
QPX support.
llvm-svn: 230332
2015-02-24 20:59:47 +08:00
|
|
|
if (Lowered == Result)
|
|
|
|
return TranslateLegalizeResults(Op, Lowered);
|
[x86] Make vector legalization of extloads work more like the "normal"
vector operation legalization with support for custom target lowering
and fallback to expand when it fails, and use this to implement sext and
anyext load lowering for x86 in a more principled way.
Previously, the x86 backend relied on a target DAG combine to "combine
away" sextload and extload nodes prior to legalization, or would expand
them during legalization with terrible code. This is particularly
problematic because the DAG combine relies on running over non-canonical
DAG nodes at just the right time to match several common and important
patterns. It used a combine rather than lowering because we didn't have
good lowering support, and to expose some tricks being employed to more
combine phases.
With this change it becomes a proper lowering operation, the backend
marks that it can lower these nodes, and I've added support for handling
the canonical forms that don't have direct legal representations such as
sextload of a v4i8 -> v4i64 on AVX1. With this change, our test cases
for this behavior continue to pass even after the DAG combiner beigns
running more systematically over every node.
There is some noise caused by this in the test suite where we actually
use vector extends instead of subregister extraction. This doesn't
really seem like the right thing to do, but is unlikely to be a critical
regression. We do regress in one case where by lowering to the
target-specific patterns early we were able to combine away extraneous
legal math nodes. However, this regression is completely addressed by
switching to a widening based legalization which is what I'm working
toward anyways, so I've just switched the test to that mode.
Differential Revision: http://reviews.llvm.org/D4654
llvm-svn: 213897
2014-07-25 06:09:56 +08:00
|
|
|
Changed = true;
|
|
|
|
if (Lowered->getNumValues() != Op->getNumValues()) {
|
|
|
|
// This expanded to something other than the load. Assume the
|
|
|
|
// lowering code took care of any chain values, and just handle the
|
|
|
|
// returned value.
|
|
|
|
assert(Result.getValue(1).use_empty() &&
|
|
|
|
"There are still live users of the old chain!");
|
|
|
|
return LegalizeOp(Lowered);
|
|
|
|
}
|
2015-10-27 16:12:08 +08:00
|
|
|
return TranslateLegalizeResults(Op, Lowered);
|
[x86] Make vector legalization of extloads work more like the "normal"
vector operation legalization with support for custom target lowering
and fallback to expand when it fails, and use this to implement sext and
anyext load lowering for x86 in a more principled way.
Previously, the x86 backend relied on a target DAG combine to "combine
away" sextload and extload nodes prior to legalization, or would expand
them during legalization with terrible code. This is particularly
problematic because the DAG combine relies on running over non-canonical
DAG nodes at just the right time to match several common and important
patterns. It used a combine rather than lowering because we didn't have
good lowering support, and to expose some tricks being employed to more
combine phases.
With this change it becomes a proper lowering operation, the backend
marks that it can lower these nodes, and I've added support for handling
the canonical forms that don't have direct legal representations such as
sextload of a v4i8 -> v4i64 on AVX1. With this change, our test cases
for this behavior continue to pass even after the DAG combiner beigns
running more systematically over every node.
There is some noise caused by this in the test suite where we actually
use vector extends instead of subregister extraction. This doesn't
really seem like the right thing to do, but is unlikely to be a critical
regression. We do regress in one case where by lowering to the
target-specific patterns early we were able to combine away extraneous
legal math nodes. However, this regression is completely addressed by
switching to a widening based legalization which is what I'm working
toward anyways, so I've just switched the test to that mode.
Differential Revision: http://reviews.llvm.org/D4654
llvm-svn: 213897
2014-07-25 06:09:56 +08:00
|
|
|
}
|
|
|
|
case TargetLowering::Expand:
|
|
|
|
Changed = true;
|
|
|
|
return LegalizeOp(ExpandLoad(Op));
|
|
|
|
}
|
2011-10-15 15:41:10 +08:00
|
|
|
} else if (Op.getOpcode() == ISD::STORE) {
|
|
|
|
StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
|
|
|
|
EVT StVT = ST->getMemoryVT();
|
2012-12-19 16:28:51 +08:00
|
|
|
MVT ValVT = ST->getValue().getSimpleValueType();
|
2011-10-15 15:41:10 +08:00
|
|
|
if (StVT.isVector() && ST->isTruncatingStore())
|
2015-11-25 17:11:53 +08:00
|
|
|
switch (TLI.getTruncStoreAction(ValVT, StVT)) {
|
2012-02-05 16:31:47 +08:00
|
|
|
default: llvm_unreachable("This action is not supported yet!");
|
2011-10-15 15:41:10 +08:00
|
|
|
case TargetLowering::Legal:
|
|
|
|
return TranslateLegalizeResults(Op, Result);
|
[SDAG] Handle LowerOperation returning its input consistently
For almost all node types, if the target requested custom lowering, and
LowerOperation returned its input, we'd treat the original node as legal. This
did not work, however, for many loads and stores, because they follow
slightly different code paths, and we did not account for the possibility of
LowerOperation returning its input at those call sites.
I think that we now handle this consistently everywhere. At the call sites in
LegalizeDAG, we used to assert in this case, so there's no functional change
for any existing code there. For the call sites in LegalizeVectorOps, this
really only affects whether or not we set Changed = true, but I think makes the
semantics clearer.
No test case here, but it will be covered by an upcoming PowerPC commit adding
QPX support.
llvm-svn: 230332
2015-02-24 20:59:47 +08:00
|
|
|
case TargetLowering::Custom: {
|
|
|
|
SDValue Lowered = TLI.LowerOperation(Result, DAG);
|
|
|
|
Changed = Lowered != Result;
|
|
|
|
return TranslateLegalizeResults(Op, Lowered);
|
|
|
|
}
|
2011-10-15 15:41:10 +08:00
|
|
|
case TargetLowering::Expand:
|
|
|
|
Changed = true;
|
|
|
|
return LegalizeOp(ExpandStore(Op));
|
|
|
|
}
|
2015-05-03 15:12:25 +08:00
|
|
|
} else if (Op.getOpcode() == ISD::MSCATTER)
|
|
|
|
HasVectorValue = true;
|
2011-10-15 15:41:10 +08:00
|
|
|
|
2009-05-23 20:35:30 +08:00
|
|
|
for (SDNode::value_iterator J = Node->value_begin(), E = Node->value_end();
|
|
|
|
J != E;
|
|
|
|
++J)
|
|
|
|
HasVectorValue |= J->isVector();
|
|
|
|
if (!HasVectorValue)
|
|
|
|
return TranslateLegalizeResults(Op, Result);
|
|
|
|
|
2009-08-11 06:56:29 +08:00
|
|
|
EVT QueryType;
|
2009-05-23 20:35:30 +08:00
|
|
|
switch (Op.getOpcode()) {
|
|
|
|
default:
|
|
|
|
return TranslateLegalizeResults(Op, Result);
|
|
|
|
case ISD::ADD:
|
|
|
|
case ISD::SUB:
|
|
|
|
case ISD::MUL:
|
|
|
|
case ISD::SDIV:
|
|
|
|
case ISD::UDIV:
|
|
|
|
case ISD::SREM:
|
|
|
|
case ISD::UREM:
|
Combining DIV+REM->DIVREM doesn't belong in LegalizeDAG; move it over into DAGCombiner.
Summary:
In addition to moving the code over, this patch amends the DIV,REM -> DIVREM
combining to run on all affected nodes at once: if the nodes are converted
to DIVREM one at a time, then the resulting DIVREM may get legalized by the
backend into something target-specific that we won't be able to recognize
and correlate with the remaining nodes.
The motivation is to "prepare terrain" for D13862: when we set DIV and REM
to be legalized to libcalls, instead of the DIVREM, we otherwise lose the
ability to combine them together. To prevent this, we need to take the
DIV,REM -> DIVREM combining out of the lowering stage.
Reviewers: RKSimon, eli.friedman, rengolin
Subscribers: john.brawn, rengolin, llvm-commits
Differential Revision: http://reviews.llvm.org/D13733
llvm-svn: 250825
2015-10-20 21:06:02 +08:00
|
|
|
case ISD::SDIVREM:
|
|
|
|
case ISD::UDIVREM:
|
2009-05-23 20:35:30 +08:00
|
|
|
case ISD::FADD:
|
|
|
|
case ISD::FSUB:
|
|
|
|
case ISD::FMUL:
|
|
|
|
case ISD::FDIV:
|
|
|
|
case ISD::FREM:
|
|
|
|
case ISD::AND:
|
|
|
|
case ISD::OR:
|
|
|
|
case ISD::XOR:
|
|
|
|
case ISD::SHL:
|
|
|
|
case ISD::SRA:
|
|
|
|
case ISD::SRL:
|
|
|
|
case ISD::ROTL:
|
|
|
|
case ISD::ROTR:
|
2014-02-04 01:27:25 +08:00
|
|
|
case ISD::BSWAP:
|
2009-05-23 20:35:30 +08:00
|
|
|
case ISD::CTLZ:
|
2011-12-13 09:56:10 +08:00
|
|
|
case ISD::CTTZ:
|
|
|
|
case ISD::CTLZ_ZERO_UNDEF:
|
|
|
|
case ISD::CTTZ_ZERO_UNDEF:
|
2009-05-23 20:35:30 +08:00
|
|
|
case ISD::CTPOP:
|
|
|
|
case ISD::SELECT:
|
2011-09-14 03:17:42 +08:00
|
|
|
case ISD::VSELECT:
|
2009-05-23 20:35:30 +08:00
|
|
|
case ISD::SELECT_CC:
|
2011-09-07 03:07:46 +08:00
|
|
|
case ISD::SETCC:
|
2009-05-23 20:35:30 +08:00
|
|
|
case ISD::ZERO_EXTEND:
|
|
|
|
case ISD::ANY_EXTEND:
|
|
|
|
case ISD::TRUNCATE:
|
|
|
|
case ISD::SIGN_EXTEND:
|
|
|
|
case ISD::FP_TO_SINT:
|
|
|
|
case ISD::FP_TO_UINT:
|
|
|
|
case ISD::FNEG:
|
|
|
|
case ISD::FABS:
|
2014-10-22 07:01:01 +08:00
|
|
|
case ISD::FMINNUM:
|
|
|
|
case ISD::FMAXNUM:
|
2015-08-11 17:13:05 +08:00
|
|
|
case ISD::FMINNAN:
|
|
|
|
case ISD::FMAXNAN:
|
Add a llvm.copysign intrinsic
This adds a llvm.copysign intrinsic; We already have Libfunc recognition for
copysign (which is turned into the FCOPYSIGN SDAG node). In order to
autovectorize calls to copysign in the loop vectorizer, we need a corresponding
intrinsic as well.
In addition to the expected changes to the language reference, the loop
vectorizer, BasicTTI, and the SDAG builder (the intrinsic is transformed into
an FCOPYSIGN node, just like the function call), this also adds FCOPYSIGN to a
few lists in LegalizeVector{Ops,Types} so that vector copysigns can be
expanded.
In TargetLoweringBase::initActions, I've made the default action for FCOPYSIGN
be Expand for vector types. This seems correct for all in-tree targets, and I
think is the right thing to do because, previously, there was no way to generate
vector-values FCOPYSIGN nodes (and most targets don't specify an action for
vector-typed FCOPYSIGN).
llvm-svn: 188728
2013-08-20 07:35:46 +08:00
|
|
|
case ISD::FCOPYSIGN:
|
2009-05-23 20:35:30 +08:00
|
|
|
case ISD::FSQRT:
|
|
|
|
case ISD::FSIN:
|
|
|
|
case ISD::FCOS:
|
|
|
|
case ISD::FPOWI:
|
|
|
|
case ISD::FPOW:
|
|
|
|
case ISD::FLOG:
|
|
|
|
case ISD::FLOG2:
|
|
|
|
case ISD::FLOG10:
|
|
|
|
case ISD::FEXP:
|
|
|
|
case ISD::FEXP2:
|
|
|
|
case ISD::FCEIL:
|
|
|
|
case ISD::FTRUNC:
|
|
|
|
case ISD::FRINT:
|
|
|
|
case ISD::FNEARBYINT:
|
2013-08-08 06:49:12 +08:00
|
|
|
case ISD::FROUND:
|
2009-05-23 20:35:30 +08:00
|
|
|
case ISD::FFLOOR:
|
2012-11-16 06:44:27 +08:00
|
|
|
case ISD::FP_ROUND:
|
2012-11-17 09:52:46 +08:00
|
|
|
case ISD::FP_EXTEND:
|
2012-08-30 15:34:22 +08:00
|
|
|
case ISD::FMA:
|
2011-07-14 19:11:14 +08:00
|
|
|
case ISD::SIGN_EXTEND_INREG:
|
2014-07-10 20:32:32 +08:00
|
|
|
case ISD::ANY_EXTEND_VECTOR_INREG:
|
|
|
|
case ISD::SIGN_EXTEND_VECTOR_INREG:
|
2014-07-09 18:58:18 +08:00
|
|
|
case ISD::ZERO_EXTEND_VECTOR_INREG:
|
2015-05-15 17:03:15 +08:00
|
|
|
case ISD::SMIN:
|
|
|
|
case ISD::SMAX:
|
|
|
|
case ISD::UMIN:
|
|
|
|
case ISD::UMAX:
|
2015-07-16 23:22:46 +08:00
|
|
|
case ISD::UABSDIFF:
|
|
|
|
case ISD::SABSDIFF:
|
2009-06-06 11:27:50 +08:00
|
|
|
QueryType = Node->getValueType(0);
|
|
|
|
break;
|
2010-01-09 10:13:55 +08:00
|
|
|
case ISD::FP_ROUND_INREG:
|
|
|
|
QueryType = cast<VTSDNode>(Node->getOperand(1))->getVT();
|
|
|
|
break;
|
2009-06-06 11:27:50 +08:00
|
|
|
case ISD::SINT_TO_FP:
|
|
|
|
case ISD::UINT_TO_FP:
|
|
|
|
QueryType = Node->getOperand(0).getValueType();
|
2009-05-23 20:35:30 +08:00
|
|
|
break;
|
2015-05-03 15:12:25 +08:00
|
|
|
case ISD::MSCATTER:
|
|
|
|
QueryType = cast<MaskedScatterSDNode>(Node)->getValue().getValueType();
|
|
|
|
break;
|
2009-05-23 20:35:30 +08:00
|
|
|
}
|
|
|
|
|
2009-06-06 11:27:50 +08:00
|
|
|
switch (TLI.getOperationAction(Node->getOpcode(), QueryType)) {
|
2015-10-20 23:06:37 +08:00
|
|
|
default: llvm_unreachable("This action is not supported yet!");
|
2009-05-23 20:35:30 +08:00
|
|
|
case TargetLowering::Promote:
|
2014-07-02 11:07:15 +08:00
|
|
|
Result = Promote(Op);
|
|
|
|
Changed = true;
|
|
|
|
break;
|
|
|
|
case TargetLowering::Legal:
|
2009-05-23 20:35:30 +08:00
|
|
|
break;
|
|
|
|
case TargetLowering::Custom: {
|
|
|
|
SDValue Tmp1 = TLI.LowerOperation(Op, DAG);
|
|
|
|
if (Tmp1.getNode()) {
|
|
|
|
Result = Tmp1;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
// FALL THROUGH
|
|
|
|
}
|
|
|
|
case TargetLowering::Expand:
|
2014-07-02 14:23:34 +08:00
|
|
|
Result = Expand(Op);
|
2009-05-23 20:35:30 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Make sure that the generated code is itself legal.
|
|
|
|
if (Result != Op) {
|
|
|
|
Result = LegalizeOp(Result);
|
|
|
|
Changed = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Note that LegalizeOp may be reentered even from single-use nodes, which
|
|
|
|
// means that we always must cache transformed nodes.
|
|
|
|
AddLegalizedOperand(Op, Result);
|
|
|
|
return Result;
|
|
|
|
}
|
|
|
|
|
2014-07-02 11:07:11 +08:00
|
|
|
SDValue VectorLegalizer::Promote(SDValue Op) {
|
2014-07-02 11:07:15 +08:00
|
|
|
// For a few operations there is a specific concept for promotion based on
|
|
|
|
// the operand's type.
|
|
|
|
switch (Op.getOpcode()) {
|
|
|
|
case ISD::SINT_TO_FP:
|
|
|
|
case ISD::UINT_TO_FP:
|
|
|
|
// "Promote" the operation by extending the operand.
|
|
|
|
return PromoteINT_TO_FP(Op);
|
|
|
|
case ISD::FP_TO_UINT:
|
|
|
|
case ISD::FP_TO_SINT:
|
|
|
|
// Promote the operation by extending the operand.
|
|
|
|
return PromoteFP_TO_INT(Op, Op->getOpcode() == ISD::FP_TO_SINT);
|
|
|
|
}
|
|
|
|
|
2014-08-28 00:16:04 +08:00
|
|
|
// There are currently two cases of vector promotion:
|
|
|
|
// 1) Bitcasting a vector of integers to a different type to a vector of the
|
2015-03-28 05:45:18 +08:00
|
|
|
// same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.
|
|
|
|
// 2) Extending a vector of floats to a vector of the same number of larger
|
2014-08-28 00:16:04 +08:00
|
|
|
// floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.
|
2012-12-19 19:21:04 +08:00
|
|
|
MVT VT = Op.getSimpleValueType();
|
2009-05-23 20:35:30 +08:00
|
|
|
assert(Op.getNode()->getNumValues() == 1 &&
|
|
|
|
"Can't promote a vector with multiple results!");
|
2012-12-19 19:21:04 +08:00
|
|
|
MVT NVT = TLI.getTypeToPromoteTo(Op.getOpcode(), VT);
|
2013-05-29 00:31:26 +08:00
|
|
|
SDLoc dl(Op);
|
2009-05-23 20:35:30 +08:00
|
|
|
SmallVector<SDValue, 4> Operands(Op.getNumOperands());
|
|
|
|
|
|
|
|
for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
|
|
|
|
if (Op.getOperand(j).getValueType().isVector())
|
2014-08-28 00:16:04 +08:00
|
|
|
if (Op.getOperand(j)
|
|
|
|
.getValueType()
|
|
|
|
.getVectorElementType()
|
2015-02-13 06:43:52 +08:00
|
|
|
.isFloatingPoint() &&
|
|
|
|
NVT.isVector() && NVT.getVectorElementType().isFloatingPoint())
|
2014-08-28 00:16:04 +08:00
|
|
|
Operands[j] = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Op.getOperand(j));
|
|
|
|
else
|
|
|
|
Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Op.getOperand(j));
|
2009-05-23 20:35:30 +08:00
|
|
|
else
|
|
|
|
Operands[j] = Op.getOperand(j);
|
|
|
|
}
|
2015-09-17 00:31:21 +08:00
|
|
|
|
|
|
|
Op = DAG.getNode(Op.getOpcode(), dl, NVT, Operands, Op.getNode()->getFlags());
|
2015-02-13 06:43:52 +08:00
|
|
|
if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
|
|
|
|
(VT.isVector() && VT.getVectorElementType().isFloatingPoint() &&
|
|
|
|
NVT.isVector() && NVT.getVectorElementType().isFloatingPoint()))
|
2015-04-28 22:05:47 +08:00
|
|
|
return DAG.getNode(ISD::FP_ROUND, dl, VT, Op, DAG.getIntPtrConstant(0, dl));
|
2014-08-28 00:16:04 +08:00
|
|
|
else
|
|
|
|
return DAG.getNode(ISD::BITCAST, dl, VT, Op);
|
2009-05-23 20:35:30 +08:00
|
|
|
}
|
|
|
|
|
2014-07-02 11:07:11 +08:00
|
|
|
SDValue VectorLegalizer::PromoteINT_TO_FP(SDValue Op) {
|
2012-06-29 05:03:44 +08:00
|
|
|
// INT_TO_FP operations may require the input operand be promoted even
|
|
|
|
// when the type is otherwise legal.
|
|
|
|
EVT VT = Op.getOperand(0).getValueType();
|
|
|
|
assert(Op.getNode()->getNumValues() == 1 &&
|
|
|
|
"Can't promote a vector with multiple results!");
|
|
|
|
|
|
|
|
// Normal getTypeToPromoteTo() doesn't work here, as that will promote
|
|
|
|
// by widening the vector w/ the same element width and twice the number
|
|
|
|
// of elements. We want the other way around, the same number of elements,
|
|
|
|
// each twice the width.
|
|
|
|
//
|
|
|
|
// Increase the bitwidth of the element to the next pow-of-two
|
|
|
|
// (which is greater than 8 bits).
|
|
|
|
|
2014-03-18 01:06:14 +08:00
|
|
|
EVT NVT = VT.widenIntegerVectorElementType(*DAG.getContext());
|
|
|
|
assert(NVT.isSimple() && "Promoting to a non-simple vector type!");
|
2013-05-29 00:31:26 +08:00
|
|
|
SDLoc dl(Op);
|
2012-06-29 05:03:44 +08:00
|
|
|
SmallVector<SDValue, 4> Operands(Op.getNumOperands());
|
|
|
|
|
|
|
|
unsigned Opc = Op.getOpcode() == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND :
|
|
|
|
ISD::SIGN_EXTEND;
|
|
|
|
for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
|
|
|
|
if (Op.getOperand(j).getValueType().isVector())
|
|
|
|
Operands[j] = DAG.getNode(Opc, dl, NVT, Op.getOperand(j));
|
|
|
|
else
|
|
|
|
Operands[j] = Op.getOperand(j);
|
|
|
|
}
|
|
|
|
|
2014-04-27 02:35:24 +08:00
|
|
|
return DAG.getNode(Op.getOpcode(), dl, Op.getValueType(), Operands);
|
2012-06-29 05:03:44 +08:00
|
|
|
}
|
|
|
|
|
2014-03-18 01:06:14 +08:00
|
|
|
// For FP_TO_INT we promote the result type to a vector type with wider
|
|
|
|
// elements and then truncate the result. This is different from the default
|
|
|
|
// PromoteVector which uses bitcast to promote thus assumning that the
|
|
|
|
// promoted vector type has the same overall size.
|
2014-07-02 11:07:11 +08:00
|
|
|
SDValue VectorLegalizer::PromoteFP_TO_INT(SDValue Op, bool isSigned) {
|
2014-03-18 01:06:14 +08:00
|
|
|
assert(Op.getNode()->getNumValues() == 1 &&
|
|
|
|
"Can't promote a vector with multiple results!");
|
|
|
|
EVT VT = Op.getValueType();
|
|
|
|
|
|
|
|
EVT NewVT;
|
|
|
|
unsigned NewOpc;
|
|
|
|
while (1) {
|
|
|
|
NewVT = VT.widenIntegerVectorElementType(*DAG.getContext());
|
|
|
|
assert(NewVT.isSimple() && "Promoting to a non-simple vector type!");
|
|
|
|
if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewVT)) {
|
|
|
|
NewOpc = ISD::FP_TO_SINT;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewVT)) {
|
|
|
|
NewOpc = ISD::FP_TO_UINT;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
SDLoc loc(Op);
|
|
|
|
SDValue promoted = DAG.getNode(NewOpc, SDLoc(Op), NewVT, Op.getOperand(0));
|
|
|
|
return DAG.getNode(ISD::TRUNCATE, SDLoc(Op), VT, promoted);
|
|
|
|
}
|
|
|
|
|
2011-10-15 15:41:10 +08:00
|
|
|
|
|
|
|
SDValue VectorLegalizer::ExpandLoad(SDValue Op) {
|
2013-05-29 00:31:26 +08:00
|
|
|
SDLoc dl(Op);
|
2011-10-15 15:41:10 +08:00
|
|
|
LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
|
|
|
|
SDValue Chain = LD->getChain();
|
|
|
|
SDValue BasePTR = LD->getBasePtr();
|
|
|
|
EVT SrcVT = LD->getMemoryVT();
|
2011-10-19 06:32:43 +08:00
|
|
|
ISD::LoadExtType ExtType = LD->getExtensionType();
|
2011-10-15 15:41:10 +08:00
|
|
|
|
2013-02-21 02:04:21 +08:00
|
|
|
SmallVector<SDValue, 8> Vals;
|
2011-10-15 15:41:10 +08:00
|
|
|
SmallVector<SDValue, 8> LoadChains;
|
|
|
|
unsigned NumElem = SrcVT.getVectorNumElements();
|
|
|
|
|
2013-02-21 02:04:21 +08:00
|
|
|
EVT SrcEltVT = SrcVT.getScalarType();
|
|
|
|
EVT DstEltVT = Op.getNode()->getValueType(0).getScalarType();
|
|
|
|
|
|
|
|
if (SrcVT.getVectorNumElements() > 1 && !SrcEltVT.isByteSized()) {
|
|
|
|
// When elements in a vector is not byte-addressable, we cannot directly
|
|
|
|
// load each element by advancing pointer, which could only address bytes.
|
|
|
|
// Instead, we load all significant words, mask bits off, and concatenate
|
|
|
|
// them to form each element. Finally, they are extended to destination
|
|
|
|
// scalar type to build the destination vector.
|
2015-07-09 10:09:04 +08:00
|
|
|
EVT WideVT = TLI.getPointerTy(DAG.getDataLayout());
|
2013-02-21 02:04:21 +08:00
|
|
|
|
|
|
|
assert(WideVT.isRound() &&
|
|
|
|
"Could not handle the sophisticated case when the widest integer is"
|
|
|
|
" not power of 2.");
|
|
|
|
assert(WideVT.bitsGE(SrcEltVT) &&
|
|
|
|
"Type is not legalized?");
|
|
|
|
|
|
|
|
unsigned WideBytes = WideVT.getStoreSize();
|
|
|
|
unsigned Offset = 0;
|
|
|
|
unsigned RemainingBytes = SrcVT.getStoreSize();
|
|
|
|
SmallVector<SDValue, 8> LoadVals;
|
|
|
|
|
|
|
|
while (RemainingBytes > 0) {
|
|
|
|
SDValue ScalarLoad;
|
|
|
|
unsigned LoadBytes = WideBytes;
|
|
|
|
|
|
|
|
if (RemainingBytes >= LoadBytes) {
|
|
|
|
ScalarLoad = DAG.getLoad(WideVT, dl, Chain, BasePTR,
|
|
|
|
LD->getPointerInfo().getWithOffset(Offset),
|
|
|
|
LD->isVolatile(), LD->isNonTemporal(),
|
2015-02-22 23:58:04 +08:00
|
|
|
LD->isInvariant(),
|
|
|
|
MinAlign(LD->getAlignment(), Offset),
|
2014-07-24 20:16:19 +08:00
|
|
|
LD->getAAInfo());
|
2013-02-21 02:04:21 +08:00
|
|
|
} else {
|
|
|
|
EVT LoadVT = WideVT;
|
|
|
|
while (RemainingBytes < LoadBytes) {
|
|
|
|
LoadBytes >>= 1; // Reduce the load size by half.
|
|
|
|
LoadVT = EVT::getIntegerVT(*DAG.getContext(), LoadBytes << 3);
|
|
|
|
}
|
|
|
|
ScalarLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, WideVT, Chain, BasePTR,
|
|
|
|
LD->getPointerInfo().getWithOffset(Offset),
|
|
|
|
LoadVT, LD->isVolatile(),
|
2014-08-01 05:45:05 +08:00
|
|
|
LD->isNonTemporal(), LD->isInvariant(),
|
2015-02-22 23:58:04 +08:00
|
|
|
MinAlign(LD->getAlignment(), Offset),
|
|
|
|
LD->getAAInfo());
|
2013-02-21 02:04:21 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
RemainingBytes -= LoadBytes;
|
|
|
|
Offset += LoadBytes;
|
|
|
|
BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR,
|
2015-04-28 22:05:47 +08:00
|
|
|
DAG.getConstant(LoadBytes, dl,
|
|
|
|
BasePTR.getValueType()));
|
2013-02-21 02:04:21 +08:00
|
|
|
|
|
|
|
LoadVals.push_back(ScalarLoad.getValue(0));
|
|
|
|
LoadChains.push_back(ScalarLoad.getValue(1));
|
|
|
|
}
|
2011-10-15 15:41:10 +08:00
|
|
|
|
2013-02-21 02:04:21 +08:00
|
|
|
// Extract bits, pack and extend/trunc them into destination type.
|
|
|
|
unsigned SrcEltBits = SrcEltVT.getSizeInBits();
|
2015-04-28 22:05:47 +08:00
|
|
|
SDValue SrcEltBitMask = DAG.getConstant((1U << SrcEltBits) - 1, dl, WideVT);
|
2011-10-19 06:32:43 +08:00
|
|
|
|
2013-02-21 02:04:21 +08:00
|
|
|
unsigned BitOffset = 0;
|
|
|
|
unsigned WideIdx = 0;
|
|
|
|
unsigned WideBits = WideVT.getSizeInBits();
|
|
|
|
|
|
|
|
for (unsigned Idx = 0; Idx != NumElem; ++Idx) {
|
|
|
|
SDValue Lo, Hi, ShAmt;
|
|
|
|
|
|
|
|
if (BitOffset < WideBits) {
|
2015-07-09 10:09:20 +08:00
|
|
|
ShAmt = DAG.getConstant(
|
|
|
|
BitOffset, dl, TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
|
2013-02-21 02:04:21 +08:00
|
|
|
Lo = DAG.getNode(ISD::SRL, dl, WideVT, LoadVals[WideIdx], ShAmt);
|
|
|
|
Lo = DAG.getNode(ISD::AND, dl, WideVT, Lo, SrcEltBitMask);
|
|
|
|
}
|
|
|
|
|
|
|
|
BitOffset += SrcEltBits;
|
|
|
|
if (BitOffset >= WideBits) {
|
|
|
|
WideIdx++;
|
2015-02-05 02:54:01 +08:00
|
|
|
BitOffset -= WideBits;
|
|
|
|
if (BitOffset > 0) {
|
2015-07-09 10:09:20 +08:00
|
|
|
ShAmt = DAG.getConstant(
|
|
|
|
SrcEltBits - BitOffset, dl,
|
|
|
|
TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
|
2013-02-21 02:04:21 +08:00
|
|
|
Hi = DAG.getNode(ISD::SHL, dl, WideVT, LoadVals[WideIdx], ShAmt);
|
|
|
|
Hi = DAG.getNode(ISD::AND, dl, WideVT, Hi, SrcEltBitMask);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (Hi.getNode())
|
|
|
|
Lo = DAG.getNode(ISD::OR, dl, WideVT, Lo, Hi);
|
|
|
|
|
|
|
|
switch (ExtType) {
|
|
|
|
default: llvm_unreachable("Unknown extended-load op!");
|
|
|
|
case ISD::EXTLOAD:
|
|
|
|
Lo = DAG.getAnyExtOrTrunc(Lo, dl, DstEltVT);
|
|
|
|
break;
|
|
|
|
case ISD::ZEXTLOAD:
|
|
|
|
Lo = DAG.getZExtOrTrunc(Lo, dl, DstEltVT);
|
|
|
|
break;
|
|
|
|
case ISD::SEXTLOAD:
|
2015-07-09 10:09:20 +08:00
|
|
|
ShAmt =
|
|
|
|
DAG.getConstant(WideBits - SrcEltBits, dl,
|
|
|
|
TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
|
2013-02-21 02:04:21 +08:00
|
|
|
Lo = DAG.getNode(ISD::SHL, dl, WideVT, Lo, ShAmt);
|
|
|
|
Lo = DAG.getNode(ISD::SRA, dl, WideVT, Lo, ShAmt);
|
|
|
|
Lo = DAG.getSExtOrTrunc(Lo, dl, DstEltVT);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
Vals.push_back(Lo);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
unsigned Stride = SrcVT.getScalarType().getSizeInBits()/8;
|
|
|
|
|
|
|
|
for (unsigned Idx=0; Idx<NumElem; Idx++) {
|
|
|
|
SDValue ScalarLoad = DAG.getExtLoad(ExtType, dl,
|
|
|
|
Op.getNode()->getValueType(0).getScalarType(),
|
|
|
|
Chain, BasePTR, LD->getPointerInfo().getWithOffset(Idx * Stride),
|
|
|
|
SrcVT.getScalarType(),
|
2014-08-01 05:45:05 +08:00
|
|
|
LD->isVolatile(), LD->isNonTemporal(), LD->isInvariant(),
|
2015-02-22 23:58:04 +08:00
|
|
|
MinAlign(LD->getAlignment(), Idx * Stride), LD->getAAInfo());
|
2013-02-21 02:04:21 +08:00
|
|
|
|
|
|
|
BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR,
|
2015-04-28 22:05:47 +08:00
|
|
|
DAG.getConstant(Stride, dl, BasePTR.getValueType()));
|
2013-02-21 02:04:21 +08:00
|
|
|
|
|
|
|
Vals.push_back(ScalarLoad.getValue(0));
|
|
|
|
LoadChains.push_back(ScalarLoad.getValue(1));
|
|
|
|
}
|
2011-10-15 15:41:10 +08:00
|
|
|
}
|
2011-10-19 06:32:43 +08:00
|
|
|
|
2014-04-27 02:35:24 +08:00
|
|
|
SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
|
2011-10-15 15:41:10 +08:00
|
|
|
SDValue Value = DAG.getNode(ISD::BUILD_VECTOR, dl,
|
2014-04-27 02:35:24 +08:00
|
|
|
Op.getNode()->getValueType(0), Vals);
|
2011-10-15 15:41:10 +08:00
|
|
|
|
|
|
|
AddLegalizedOperand(Op.getValue(0), Value);
|
|
|
|
AddLegalizedOperand(Op.getValue(1), NewChain);
|
|
|
|
|
|
|
|
return (Op.getResNo() ? NewChain : Value);
|
|
|
|
}
|
|
|
|
|
|
|
|
SDValue VectorLegalizer::ExpandStore(SDValue Op) {
|
2013-05-29 00:31:26 +08:00
|
|
|
SDLoc dl(Op);
|
2011-10-15 15:41:10 +08:00
|
|
|
StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
|
|
|
|
SDValue Chain = ST->getChain();
|
|
|
|
SDValue BasePTR = ST->getBasePtr();
|
|
|
|
SDValue Value = ST->getValue();
|
|
|
|
EVT StVT = ST->getMemoryVT();
|
|
|
|
|
|
|
|
unsigned Alignment = ST->getAlignment();
|
|
|
|
bool isVolatile = ST->isVolatile();
|
|
|
|
bool isNonTemporal = ST->isNonTemporal();
|
2014-07-24 20:16:19 +08:00
|
|
|
AAMDNodes AAInfo = ST->getAAInfo();
|
2011-10-15 15:41:10 +08:00
|
|
|
|
|
|
|
unsigned NumElem = StVT.getVectorNumElements();
|
|
|
|
// The type of the data we want to save
|
|
|
|
EVT RegVT = Value.getValueType();
|
|
|
|
EVT RegSclVT = RegVT.getScalarType();
|
|
|
|
// The type of data as saved in memory.
|
|
|
|
EVT MemSclVT = StVT.getScalarType();
|
|
|
|
|
|
|
|
// Cast floats into integers
|
|
|
|
unsigned ScalarSize = MemSclVT.getSizeInBits();
|
|
|
|
|
|
|
|
// Round odd types to the next pow of two.
|
|
|
|
if (!isPowerOf2_32(ScalarSize))
|
|
|
|
ScalarSize = NextPowerOf2(ScalarSize);
|
|
|
|
|
|
|
|
// Store Stride in bytes
|
|
|
|
unsigned Stride = ScalarSize/8;
|
|
|
|
// Extract each of the elements from the original vector
|
|
|
|
// and save them into memory individually.
|
|
|
|
SmallVector<SDValue, 8> Stores;
|
|
|
|
for (unsigned Idx = 0; Idx < NumElem; Idx++) {
|
2015-07-09 10:09:04 +08:00
|
|
|
SDValue Ex = DAG.getNode(
|
|
|
|
ISD::EXTRACT_VECTOR_ELT, dl, RegSclVT, Value,
|
|
|
|
DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
|
2011-10-15 15:41:10 +08:00
|
|
|
|
|
|
|
// This scalar TruncStore may be illegal, but we legalize it later.
|
|
|
|
SDValue Store = DAG.getTruncStore(Chain, dl, Ex, BasePTR,
|
|
|
|
ST->getPointerInfo().getWithOffset(Idx*Stride), MemSclVT,
|
2015-02-22 23:58:04 +08:00
|
|
|
isVolatile, isNonTemporal, MinAlign(Alignment, Idx*Stride),
|
|
|
|
AAInfo);
|
2011-10-15 15:41:10 +08:00
|
|
|
|
2011-10-19 06:32:43 +08:00
|
|
|
BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR,
|
2015-04-28 22:05:47 +08:00
|
|
|
DAG.getConstant(Stride, dl, BasePTR.getValueType()));
|
2011-10-19 06:32:43 +08:00
|
|
|
|
2011-10-15 15:41:10 +08:00
|
|
|
Stores.push_back(Store);
|
|
|
|
}
|
2014-04-27 02:35:24 +08:00
|
|
|
SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
|
2011-10-15 15:41:10 +08:00
|
|
|
AddLegalizedOperand(Op, TF);
|
|
|
|
return TF;
|
|
|
|
}
|
|
|
|
|
2014-07-02 14:23:34 +08:00
|
|
|
SDValue VectorLegalizer::Expand(SDValue Op) {
|
|
|
|
switch (Op->getOpcode()) {
|
|
|
|
case ISD::SIGN_EXTEND_INREG:
|
|
|
|
return ExpandSEXTINREG(Op);
|
2014-07-10 20:32:32 +08:00
|
|
|
case ISD::ANY_EXTEND_VECTOR_INREG:
|
|
|
|
return ExpandANY_EXTEND_VECTOR_INREG(Op);
|
|
|
|
case ISD::SIGN_EXTEND_VECTOR_INREG:
|
|
|
|
return ExpandSIGN_EXTEND_VECTOR_INREG(Op);
|
2014-07-09 18:58:18 +08:00
|
|
|
case ISD::ZERO_EXTEND_VECTOR_INREG:
|
|
|
|
return ExpandZERO_EXTEND_VECTOR_INREG(Op);
|
2014-07-02 14:23:34 +08:00
|
|
|
case ISD::BSWAP:
|
|
|
|
return ExpandBSWAP(Op);
|
|
|
|
case ISD::VSELECT:
|
|
|
|
return ExpandVSELECT(Op);
|
|
|
|
case ISD::SELECT:
|
|
|
|
return ExpandSELECT(Op);
|
|
|
|
case ISD::UINT_TO_FP:
|
|
|
|
return ExpandUINT_TO_FLOAT(Op);
|
|
|
|
case ISD::FNEG:
|
|
|
|
return ExpandFNEG(Op);
|
|
|
|
case ISD::SETCC:
|
|
|
|
return UnrollVSETCC(Op);
|
2015-07-16 23:22:46 +08:00
|
|
|
case ISD::UABSDIFF:
|
|
|
|
case ISD::SABSDIFF:
|
|
|
|
return ExpandABSDIFF(Op);
|
2014-07-02 14:23:34 +08:00
|
|
|
default:
|
|
|
|
return DAG.UnrollVectorOp(Op.getNode());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-07-16 23:22:46 +08:00
|
|
|
SDValue VectorLegalizer::ExpandABSDIFF(SDValue Op) {
|
|
|
|
SDLoc dl(Op);
|
2015-09-24 18:35:03 +08:00
|
|
|
SDValue Op0 = Op.getOperand(0);
|
|
|
|
SDValue Op1 = Op.getOperand(1);
|
2015-07-16 23:22:46 +08:00
|
|
|
EVT VT = Op.getValueType();
|
2015-09-24 18:35:03 +08:00
|
|
|
|
|
|
|
// For unsigned intrinsic, promote the type to handle unsigned overflow.
|
|
|
|
bool isUabsdiff = (Op->getOpcode() == ISD::UABSDIFF);
|
|
|
|
if (isUabsdiff) {
|
|
|
|
VT = VT.widenIntegerVectorElementType(*DAG.getContext());
|
|
|
|
Op0 = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op0);
|
|
|
|
Op1 = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op1);
|
|
|
|
}
|
|
|
|
|
2015-07-16 23:22:46 +08:00
|
|
|
SDNodeFlags Flags;
|
2015-09-24 18:35:03 +08:00
|
|
|
Flags.setNoSignedWrap(!isUabsdiff);
|
|
|
|
SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Op0, Op1, &Flags);
|
|
|
|
if (isUabsdiff)
|
|
|
|
return DAG.getNode(ISD::TRUNCATE, dl, Op.getValueType(), Sub);
|
|
|
|
|
|
|
|
SDValue Cmp =
|
|
|
|
DAG.getNode(ISD::SETCC, dl, TLI.getSetCCResultType(DAG.getDataLayout(),
|
|
|
|
*DAG.getContext(), VT),
|
|
|
|
Sub, DAG.getConstant(0, dl, VT), DAG.getCondCode(ISD::SETGE));
|
|
|
|
SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), Sub, &Flags);
|
|
|
|
return DAG.getNode(ISD::VSELECT, dl, VT, Cmp, Sub, Neg);
|
2015-07-16 23:22:46 +08:00
|
|
|
}
|
|
|
|
|
2012-08-31 03:17:29 +08:00
|
|
|
SDValue VectorLegalizer::ExpandSELECT(SDValue Op) {
|
|
|
|
// Lower a select instruction where the condition is a scalar and the
|
|
|
|
// operands are vectors. Lower this select to VSELECT and implement it
|
2013-07-08 08:37:03 +08:00
|
|
|
// using XOR AND OR. The selector bit is broadcasted.
|
2012-08-31 03:17:29 +08:00
|
|
|
EVT VT = Op.getValueType();
|
2013-05-29 00:31:26 +08:00
|
|
|
SDLoc DL(Op);
|
2012-08-31 03:17:29 +08:00
|
|
|
|
|
|
|
SDValue Mask = Op.getOperand(0);
|
|
|
|
SDValue Op1 = Op.getOperand(1);
|
|
|
|
SDValue Op2 = Op.getOperand(2);
|
|
|
|
|
|
|
|
assert(VT.isVector() && !Mask.getValueType().isVector()
|
|
|
|
&& Op1.getValueType() == Op2.getValueType() && "Invalid type");
|
|
|
|
|
|
|
|
unsigned NumElem = VT.getVectorNumElements();
|
|
|
|
|
|
|
|
// If we can't even use the basic vector operations of
|
|
|
|
// AND,OR,XOR, we will have to scalarize the op.
|
|
|
|
// Notice that the operation may be 'promoted' which means that it is
|
|
|
|
// 'bitcasted' to another type which is handled.
|
|
|
|
// Also, we need to be able to construct a splat vector using BUILD_VECTOR.
|
|
|
|
if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
|
|
|
|
TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
|
|
|
|
TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand ||
|
|
|
|
TLI.getOperationAction(ISD::BUILD_VECTOR, VT) == TargetLowering::Expand)
|
|
|
|
return DAG.UnrollVectorOp(Op.getNode());
|
|
|
|
|
|
|
|
// Generate a mask operand.
|
2013-09-10 08:41:56 +08:00
|
|
|
EVT MaskTy = VT.changeVectorElementTypeToInteger();
|
2012-08-31 03:17:29 +08:00
|
|
|
|
|
|
|
// What is the size of each element in the vector mask.
|
|
|
|
EVT BitTy = MaskTy.getScalarType();
|
|
|
|
|
2013-06-15 06:04:37 +08:00
|
|
|
Mask = DAG.getSelect(DL, BitTy, Mask,
|
2015-04-28 22:05:47 +08:00
|
|
|
DAG.getConstant(APInt::getAllOnesValue(BitTy.getSizeInBits()), DL,
|
|
|
|
BitTy),
|
|
|
|
DAG.getConstant(0, DL, BitTy));
|
2012-08-31 03:17:29 +08:00
|
|
|
|
|
|
|
// Broadcast the mask so that the entire vector is all-one or all zero.
|
|
|
|
SmallVector<SDValue, 8> Ops(NumElem, Mask);
|
2014-04-27 02:35:24 +08:00
|
|
|
Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, MaskTy, Ops);
|
2012-08-31 03:17:29 +08:00
|
|
|
|
|
|
|
// Bitcast the operands to be the same type as the mask.
|
|
|
|
// This is needed when we select between FP types because
|
|
|
|
// the mask is a vector of integers.
|
|
|
|
Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1);
|
|
|
|
Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2);
|
|
|
|
|
|
|
|
SDValue AllOnes = DAG.getConstant(
|
2015-04-28 22:05:47 +08:00
|
|
|
APInt::getAllOnesValue(BitTy.getSizeInBits()), DL, MaskTy);
|
2012-08-31 03:17:29 +08:00
|
|
|
SDValue NotMask = DAG.getNode(ISD::XOR, DL, MaskTy, Mask, AllOnes);
|
|
|
|
|
|
|
|
Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask);
|
|
|
|
Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask);
|
|
|
|
SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2);
|
|
|
|
return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Val);
|
|
|
|
}
|
|
|
|
|
2013-01-12 06:57:48 +08:00
|
|
|
SDValue VectorLegalizer::ExpandSEXTINREG(SDValue Op) {
|
|
|
|
EVT VT = Op.getValueType();
|
|
|
|
|
2013-01-13 03:06:44 +08:00
|
|
|
// Make sure that the SRA and SHL instructions are available.
|
2013-01-12 06:57:48 +08:00
|
|
|
if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand ||
|
2013-01-13 03:06:44 +08:00
|
|
|
TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand)
|
2013-01-12 06:57:48 +08:00
|
|
|
return DAG.UnrollVectorOp(Op.getNode());
|
|
|
|
|
2013-05-29 00:31:26 +08:00
|
|
|
SDLoc DL(Op);
|
2013-01-12 06:57:48 +08:00
|
|
|
EVT OrigTy = cast<VTSDNode>(Op->getOperand(1))->getVT();
|
|
|
|
|
|
|
|
unsigned BW = VT.getScalarType().getSizeInBits();
|
|
|
|
unsigned OrigBW = OrigTy.getScalarType().getSizeInBits();
|
2015-04-28 22:05:47 +08:00
|
|
|
SDValue ShiftSz = DAG.getConstant(BW - OrigBW, DL, VT);
|
2013-01-12 06:57:48 +08:00
|
|
|
|
|
|
|
Op = Op.getOperand(0);
|
2013-01-13 03:06:44 +08:00
|
|
|
Op = DAG.getNode(ISD::SHL, DL, VT, Op, ShiftSz);
|
2013-01-12 06:57:48 +08:00
|
|
|
return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz);
|
|
|
|
}
|
|
|
|
|
2014-07-10 20:32:32 +08:00
|
|
|
// Generically expand a vector anyext in register to a shuffle of the relevant
|
|
|
|
// lanes into the appropriate locations, with other lanes left undef.
|
|
|
|
SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDValue Op) {
|
|
|
|
SDLoc DL(Op);
|
|
|
|
EVT VT = Op.getValueType();
|
|
|
|
int NumElements = VT.getVectorNumElements();
|
|
|
|
SDValue Src = Op.getOperand(0);
|
|
|
|
EVT SrcVT = Src.getValueType();
|
|
|
|
int NumSrcElements = SrcVT.getVectorNumElements();
|
|
|
|
|
|
|
|
// Build a base mask of undef shuffles.
|
|
|
|
SmallVector<int, 16> ShuffleMask;
|
|
|
|
ShuffleMask.resize(NumSrcElements, -1);
|
|
|
|
|
|
|
|
// Place the extended lanes into the correct locations.
|
|
|
|
int ExtLaneScale = NumSrcElements / NumElements;
|
2015-07-08 03:07:19 +08:00
|
|
|
int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
|
2014-07-10 20:32:32 +08:00
|
|
|
for (int i = 0; i < NumElements; ++i)
|
|
|
|
ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
|
|
|
|
|
|
|
|
return DAG.getNode(
|
|
|
|
ISD::BITCAST, DL, VT,
|
|
|
|
DAG.getVectorShuffle(SrcVT, DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
|
|
|
|
}
|
|
|
|
|
|
|
|
SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDValue Op) {
|
|
|
|
SDLoc DL(Op);
|
|
|
|
EVT VT = Op.getValueType();
|
|
|
|
SDValue Src = Op.getOperand(0);
|
|
|
|
EVT SrcVT = Src.getValueType();
|
|
|
|
|
|
|
|
// First build an any-extend node which can be legalized above when we
|
|
|
|
// recurse through it.
|
|
|
|
Op = DAG.getAnyExtendVectorInReg(Src, DL, VT);
|
|
|
|
|
|
|
|
// Now we need sign extend. Do this by shifting the elements. Even if these
|
|
|
|
// aren't legal operations, they have a better chance of being legalized
|
|
|
|
// without full scalarization than the sign extension does.
|
|
|
|
unsigned EltWidth = VT.getVectorElementType().getSizeInBits();
|
|
|
|
unsigned SrcEltWidth = SrcVT.getVectorElementType().getSizeInBits();
|
2015-04-28 22:05:47 +08:00
|
|
|
SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, DL, VT);
|
2014-07-10 20:32:32 +08:00
|
|
|
return DAG.getNode(ISD::SRA, DL, VT,
|
|
|
|
DAG.getNode(ISD::SHL, DL, VT, Op, ShiftAmount),
|
|
|
|
ShiftAmount);
|
|
|
|
}
|
|
|
|
|
2014-07-09 18:58:18 +08:00
|
|
|
// Generically expand a vector zext in register to a shuffle of the relevant
|
|
|
|
// lanes into the appropriate locations, a blend of zero into the high bits,
|
|
|
|
// and a bitcast to the wider element type.
|
|
|
|
SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDValue Op) {
|
|
|
|
SDLoc DL(Op);
|
|
|
|
EVT VT = Op.getValueType();
|
|
|
|
int NumElements = VT.getVectorNumElements();
|
|
|
|
SDValue Src = Op.getOperand(0);
|
|
|
|
EVT SrcVT = Src.getValueType();
|
|
|
|
int NumSrcElements = SrcVT.getVectorNumElements();
|
|
|
|
|
|
|
|
// Build up a zero vector to blend into this one.
|
|
|
|
EVT SrcScalarVT = SrcVT.getScalarType();
|
2015-04-28 22:05:47 +08:00
|
|
|
SDValue ScalarZero = DAG.getTargetConstant(0, DL, SrcScalarVT);
|
2014-07-09 18:58:18 +08:00
|
|
|
SmallVector<SDValue, 4> BuildVectorOperands(NumSrcElements, ScalarZero);
|
|
|
|
SDValue Zero = DAG.getNode(ISD::BUILD_VECTOR, DL, SrcVT, BuildVectorOperands);
|
|
|
|
|
|
|
|
// Shuffle the incoming lanes into the correct position, and pull all other
|
|
|
|
// lanes from the zero vector.
|
|
|
|
SmallVector<int, 16> ShuffleMask;
|
|
|
|
ShuffleMask.reserve(NumSrcElements);
|
|
|
|
for (int i = 0; i < NumSrcElements; ++i)
|
|
|
|
ShuffleMask.push_back(i);
|
|
|
|
|
|
|
|
int ExtLaneScale = NumSrcElements / NumElements;
|
2015-07-08 03:07:19 +08:00
|
|
|
int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
|
2014-07-09 18:58:18 +08:00
|
|
|
for (int i = 0; i < NumElements; ++i)
|
|
|
|
ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
|
|
|
|
|
|
|
|
return DAG.getNode(ISD::BITCAST, DL, VT,
|
|
|
|
DAG.getVectorShuffle(SrcVT, DL, Zero, Src, ShuffleMask));
|
|
|
|
}
|
|
|
|
|
2014-05-19 21:12:38 +08:00
|
|
|
SDValue VectorLegalizer::ExpandBSWAP(SDValue Op) {
|
|
|
|
EVT VT = Op.getValueType();
|
|
|
|
|
|
|
|
// Generate a byte wise shuffle mask for the BSWAP.
|
|
|
|
SmallVector<int, 16> ShuffleMask;
|
|
|
|
int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
|
|
|
|
for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)
|
|
|
|
for (int J = ScalarSizeInBytes - 1; J >= 0; --J)
|
|
|
|
ShuffleMask.push_back((I * ScalarSizeInBytes) + J);
|
|
|
|
|
|
|
|
EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, ShuffleMask.size());
|
|
|
|
|
|
|
|
// Only emit a shuffle if the mask is legal.
|
|
|
|
if (!TLI.isShuffleMaskLegal(ShuffleMask, ByteVT))
|
|
|
|
return DAG.UnrollVectorOp(Op.getNode());
|
|
|
|
|
|
|
|
SDLoc DL(Op);
|
|
|
|
Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Op.getOperand(0));
|
|
|
|
Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT),
|
|
|
|
ShuffleMask.data());
|
|
|
|
return DAG.getNode(ISD::BITCAST, DL, VT, Op);
|
|
|
|
}
|
|
|
|
|
2011-09-14 03:17:42 +08:00
|
|
|
SDValue VectorLegalizer::ExpandVSELECT(SDValue Op) {
|
|
|
|
// Implement VSELECT in terms of XOR, AND, OR
|
|
|
|
// on platforms which do not support blend natively.
|
2013-05-29 00:31:26 +08:00
|
|
|
SDLoc DL(Op);
|
2011-09-14 03:17:42 +08:00
|
|
|
|
|
|
|
SDValue Mask = Op.getOperand(0);
|
|
|
|
SDValue Op1 = Op.getOperand(1);
|
|
|
|
SDValue Op2 = Op.getOperand(2);
|
|
|
|
|
2013-05-08 04:24:18 +08:00
|
|
|
EVT VT = Mask.getValueType();
|
|
|
|
|
2011-09-14 03:17:42 +08:00
|
|
|
// If we can't even use the basic vector operations of
|
|
|
|
// AND,OR,XOR, we will have to scalarize the op.
|
2011-10-20 04:43:16 +08:00
|
|
|
// Notice that the operation may be 'promoted' which means that it is
|
|
|
|
// 'bitcasted' to another type which is handled.
|
2012-09-02 06:27:48 +08:00
|
|
|
// This operation also isn't safe with AND, OR, XOR when the boolean
|
|
|
|
// type is 0/1 as we need an all ones vector constant to mask with.
|
|
|
|
// FIXME: Sign extend 1 to all ones if thats legal on the target.
|
2011-10-20 04:43:16 +08:00
|
|
|
if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
|
|
|
|
TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
|
2014-07-10 18:18:12 +08:00
|
|
|
TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand ||
|
|
|
|
TLI.getBooleanContents(Op1.getValueType()) !=
|
|
|
|
TargetLowering::ZeroOrNegativeOneBooleanContent)
|
2011-10-20 04:43:16 +08:00
|
|
|
return DAG.UnrollVectorOp(Op.getNode());
|
2011-03-19 21:09:10 +08:00
|
|
|
|
2013-05-08 04:24:18 +08:00
|
|
|
// If the mask and the type are different sizes, unroll the vector op. This
|
|
|
|
// can occur when getSetCCResultType returns something that is different in
|
|
|
|
// size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
|
|
|
|
if (VT.getSizeInBits() != Op1.getValueType().getSizeInBits())
|
|
|
|
return DAG.UnrollVectorOp(Op.getNode());
|
|
|
|
|
2011-09-14 03:17:42 +08:00
|
|
|
// Bitcast the operands to be the same type as the mask.
|
|
|
|
// This is needed when we select between FP types because
|
|
|
|
// the mask is a vector of integers.
|
|
|
|
Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1);
|
|
|
|
Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2);
|
2011-03-19 21:09:10 +08:00
|
|
|
|
2011-09-14 03:17:42 +08:00
|
|
|
SDValue AllOnes = DAG.getConstant(
|
2015-04-28 22:05:47 +08:00
|
|
|
APInt::getAllOnesValue(VT.getScalarType().getSizeInBits()), DL, VT);
|
2011-09-14 03:17:42 +08:00
|
|
|
SDValue NotMask = DAG.getNode(ISD::XOR, DL, VT, Mask, AllOnes);
|
|
|
|
|
|
|
|
Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask);
|
|
|
|
Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask);
|
2012-04-15 23:08:09 +08:00
|
|
|
SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2);
|
|
|
|
return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Val);
|
2011-09-14 03:17:42 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
SDValue VectorLegalizer::ExpandUINT_TO_FLOAT(SDValue Op) {
|
2011-03-19 21:09:10 +08:00
|
|
|
EVT VT = Op.getOperand(0).getValueType();
|
2013-05-29 00:31:26 +08:00
|
|
|
SDLoc DL(Op);
|
2011-03-19 21:09:10 +08:00
|
|
|
|
|
|
|
// Make sure that the SINT_TO_FP and SRL instructions are available.
|
2011-10-20 04:43:16 +08:00
|
|
|
if (TLI.getOperationAction(ISD::SINT_TO_FP, VT) == TargetLowering::Expand ||
|
|
|
|
TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Expand)
|
|
|
|
return DAG.UnrollVectorOp(Op.getNode());
|
2011-03-19 21:09:10 +08:00
|
|
|
|
|
|
|
EVT SVT = VT.getScalarType();
|
|
|
|
assert((SVT.getSizeInBits() == 64 || SVT.getSizeInBits() == 32) &&
|
|
|
|
"Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
|
|
|
|
|
|
|
|
unsigned BW = SVT.getSizeInBits();
|
2015-04-28 22:05:47 +08:00
|
|
|
SDValue HalfWord = DAG.getConstant(BW/2, DL, VT);
|
2011-03-19 21:09:10 +08:00
|
|
|
|
|
|
|
// Constants to clear the upper part of the word.
|
|
|
|
// Notice that we can also use SHL+SHR, but using a constant is slightly
|
|
|
|
// faster on x86.
|
|
|
|
uint64_t HWMask = (SVT.getSizeInBits()==64)?0x00000000FFFFFFFF:0x0000FFFF;
|
2015-04-28 22:05:47 +08:00
|
|
|
SDValue HalfWordMask = DAG.getConstant(HWMask, DL, VT);
|
2011-03-19 21:09:10 +08:00
|
|
|
|
|
|
|
// Two to the power of half-word-size.
|
2015-04-28 22:05:47 +08:00
|
|
|
SDValue TWOHW = DAG.getConstantFP(1 << (BW/2), DL, Op.getValueType());
|
2011-03-19 21:09:10 +08:00
|
|
|
|
|
|
|
// Clear upper part of LO, lower HI
|
|
|
|
SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Op.getOperand(0), HalfWord);
|
|
|
|
SDValue LO = DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), HalfWordMask);
|
|
|
|
|
|
|
|
// Convert hi and lo to floats
|
|
|
|
// Convert the hi part back to the upper values
|
2015-09-17 00:31:21 +08:00
|
|
|
// TODO: Can any fast-math-flags be set on these nodes?
|
2011-03-19 21:09:10 +08:00
|
|
|
SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Op.getValueType(), HI);
|
|
|
|
fHI = DAG.getNode(ISD::FMUL, DL, Op.getValueType(), fHI, TWOHW);
|
|
|
|
SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Op.getValueType(), LO);
|
|
|
|
|
|
|
|
// Add the two halves
|
|
|
|
return DAG.getNode(ISD::FADD, DL, Op.getValueType(), fHI, fLO);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2009-05-23 20:35:30 +08:00
|
|
|
SDValue VectorLegalizer::ExpandFNEG(SDValue Op) {
|
|
|
|
if (TLI.isOperationLegalOrCustom(ISD::FSUB, Op.getValueType())) {
|
2015-04-28 22:05:47 +08:00
|
|
|
SDLoc DL(Op);
|
|
|
|
SDValue Zero = DAG.getConstantFP(-0.0, DL, Op.getValueType());
|
2015-09-17 00:31:21 +08:00
|
|
|
// TODO: If FNEG had fast-math-flags, they'd get propagated to this FSUB.
|
2015-04-28 22:05:47 +08:00
|
|
|
return DAG.getNode(ISD::FSUB, DL, Op.getValueType(),
|
2009-05-23 20:35:30 +08:00
|
|
|
Zero, Op.getOperand(0));
|
|
|
|
}
|
2009-11-30 10:42:02 +08:00
|
|
|
return DAG.UnrollVectorOp(Op.getNode());
|
2009-05-23 20:35:30 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
SDValue VectorLegalizer::UnrollVSETCC(SDValue Op) {
|
2009-08-11 06:56:29 +08:00
|
|
|
EVT VT = Op.getValueType();
|
2009-05-23 20:35:30 +08:00
|
|
|
unsigned NumElems = VT.getVectorNumElements();
|
2009-08-11 06:56:29 +08:00
|
|
|
EVT EltVT = VT.getVectorElementType();
|
2009-05-23 20:35:30 +08:00
|
|
|
SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1), CC = Op.getOperand(2);
|
2009-08-11 06:56:29 +08:00
|
|
|
EVT TmpEltVT = LHS.getValueType().getVectorElementType();
|
2013-05-29 00:31:26 +08:00
|
|
|
SDLoc dl(Op);
|
2009-05-23 20:35:30 +08:00
|
|
|
SmallVector<SDValue, 8> Ops(NumElems);
|
|
|
|
for (unsigned i = 0; i < NumElems; ++i) {
|
2015-07-09 10:09:04 +08:00
|
|
|
SDValue LHSElem = DAG.getNode(
|
|
|
|
ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
|
|
|
|
DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
|
|
|
|
SDValue RHSElem = DAG.getNode(
|
|
|
|
ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
|
|
|
|
DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
|
2013-05-18 08:21:46 +08:00
|
|
|
Ops[i] = DAG.getNode(ISD::SETCC, dl,
|
2015-07-09 10:09:04 +08:00
|
|
|
TLI.getSetCCResultType(DAG.getDataLayout(),
|
|
|
|
*DAG.getContext(), TmpEltVT),
|
2009-05-23 20:35:30 +08:00
|
|
|
LHSElem, RHSElem, CC);
|
2013-06-15 06:04:37 +08:00
|
|
|
Ops[i] = DAG.getSelect(dl, EltVT, Ops[i],
|
|
|
|
DAG.getConstant(APInt::getAllOnesValue
|
2015-04-28 22:05:47 +08:00
|
|
|
(EltVT.getSizeInBits()), dl, EltVT),
|
|
|
|
DAG.getConstant(0, dl, EltVT));
|
2009-05-23 20:35:30 +08:00
|
|
|
}
|
2014-04-27 02:35:24 +08:00
|
|
|
return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
|
2009-05-23 20:35:30 +08:00
|
|
|
}
|
|
|
|
|
2015-06-23 17:49:53 +08:00
|
|
|
}
|
2009-05-23 20:35:30 +08:00
|
|
|
|
|
|
|
bool SelectionDAG::LegalizeVectors() {
|
|
|
|
return VectorLegalizer(*this).Run();
|
|
|
|
}
|