diff --git a/llvm/lib/Target/Hexagon/Hexagon.td b/llvm/lib/Target/Hexagon/Hexagon.td index 6292e2a7a4ea..73617da25754 100644 --- a/llvm/lib/Target/Hexagon/Hexagon.td +++ b/llvm/lib/Target/Hexagon/Hexagon.td @@ -302,6 +302,7 @@ include "HexagonDepInstrFormats.td" include "HexagonDepInstrInfo.td" include "HexagonPseudo.td" include "HexagonPatterns.td" +include "HexagonPatternsHVX.td" include "HexagonPatternsV65.td" include "HexagonDepMappings.td" include "HexagonIntrinsics.td" diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp index b70e01b59ec1..ef387379bd08 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp +++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp @@ -129,11 +129,6 @@ namespace { // Implement calling convention for Hexagon. -static const MVT LegalV64[] = { MVT::v64i8, MVT::v32i16, MVT::v16i32 }; -static const MVT LegalW64[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 }; -static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 }; -static const MVT LegalW128[] = { MVT::v256i8, MVT::v128i16, MVT::v64i32 }; - static bool CC_Hexagon(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, @@ -1732,40 +1727,6 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM, addRegisterClass(MVT::f64, &Hexagon::DoubleRegsRegClass); } - if (Subtarget.hasV60TOps()) { - if (Subtarget.useHVX64BOps()) { - addRegisterClass(MVT::v64i8, &Hexagon::HvxVRRegClass); - addRegisterClass(MVT::v32i16, &Hexagon::HvxVRRegClass); - addRegisterClass(MVT::v16i32, &Hexagon::HvxVRRegClass); - addRegisterClass(MVT::v128i8, &Hexagon::HvxWRRegClass); - addRegisterClass(MVT::v64i16, &Hexagon::HvxWRRegClass); - addRegisterClass(MVT::v32i32, &Hexagon::HvxWRRegClass); - // These "short" boolean vector types should be legal because - // they will appear as results of vector compares. If they were - // not legal, type legalization would try to make them legal - // and that would require using operations that do not use or - // produce such types. That, in turn, would imply using custom - // nodes, which would be unoptimizable by the DAG combiner. - // The idea is to rely on target-independent operations as much - // as possible. - addRegisterClass(MVT::v16i1, &Hexagon::HvxQRRegClass); - addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass); - addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass); - addRegisterClass(MVT::v512i1, &Hexagon::HvxQRRegClass); - } else if (Subtarget.useHVX128BOps()) { - addRegisterClass(MVT::v128i8, &Hexagon::HvxVRRegClass); - addRegisterClass(MVT::v64i16, &Hexagon::HvxVRRegClass); - addRegisterClass(MVT::v32i32, &Hexagon::HvxVRRegClass); - addRegisterClass(MVT::v256i8, &Hexagon::HvxWRRegClass); - addRegisterClass(MVT::v128i16, &Hexagon::HvxWRRegClass); - addRegisterClass(MVT::v64i32, &Hexagon::HvxWRRegClass); - addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass); - addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass); - addRegisterClass(MVT::v128i1, &Hexagon::HvxQRRegClass); - addRegisterClass(MVT::v1024i1, &Hexagon::HvxQRRegClass); - } - } - // // Handling of scalar operations. // @@ -2002,11 +1963,6 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM, setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom); setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8, Custom); - auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) { - setOperationAction(Opc, FromTy, Promote); - AddPromotedToType(Opc, FromTy, ToTy); - }; - // Subtarget-specific operation actions. // if (Subtarget.hasV5TOps()) { @@ -2068,121 +2024,8 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM, setIndexedStoreAction(ISD::POST_INC, VT, Legal); } - if (Subtarget.useHVXOps()) { - bool Use64b = Subtarget.useHVX64BOps(); - ArrayRef LegalV = Use64b ? LegalV64 : LegalV128; - ArrayRef LegalW = Use64b ? LegalW64 : LegalW128; - MVT ByteV = Use64b ? MVT::v64i8 : MVT::v128i8; - MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8; - - setOperationAction(ISD::VECTOR_SHUFFLE, ByteV, Legal); - setOperationAction(ISD::VECTOR_SHUFFLE, ByteW, Legal); - setOperationAction(ISD::AND, ByteV, Legal); - setOperationAction(ISD::OR, ByteV, Legal); - setOperationAction(ISD::XOR, ByteV, Legal); - - for (MVT T : LegalV) { - setIndexedLoadAction(ISD::POST_INC, T, Legal); - setIndexedStoreAction(ISD::POST_INC, T, Legal); - - setOperationAction(ISD::ADD, T, Legal); - setOperationAction(ISD::SUB, T, Legal); - if (T != ByteV) { - setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal); - setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal); - } - - setOperationAction(ISD::MUL, T, Custom); - setOperationAction(ISD::MULHS, T, Custom); - setOperationAction(ISD::MULHU, T, Custom); - setOperationAction(ISD::BUILD_VECTOR, T, Custom); - // Make concat-vectors custom to handle concats of more than 2 vectors. - setOperationAction(ISD::CONCAT_VECTORS, T, Custom); - setOperationAction(ISD::INSERT_SUBVECTOR, T, Custom); - setOperationAction(ISD::INSERT_VECTOR_ELT, T, Custom); - setOperationAction(ISD::EXTRACT_SUBVECTOR, T, Custom); - setOperationAction(ISD::EXTRACT_VECTOR_ELT, T, Custom); - setOperationAction(ISD::ANY_EXTEND, T, Custom); - setOperationAction(ISD::SIGN_EXTEND, T, Custom); - setOperationAction(ISD::ZERO_EXTEND, T, Custom); - if (T != ByteV) { - setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom); - // HVX only has shifts of words and halfwords. - setOperationAction(ISD::SRA, T, Custom); - setOperationAction(ISD::SHL, T, Custom); - setOperationAction(ISD::SRL, T, Custom); - } - - setCondCodeAction(ISD::SETNE, T, Expand); - setCondCodeAction(ISD::SETLE, T, Expand); - setCondCodeAction(ISD::SETGE, T, Expand); - setCondCodeAction(ISD::SETLT, T, Expand); - setCondCodeAction(ISD::SETULE, T, Expand); - setCondCodeAction(ISD::SETUGE, T, Expand); - setCondCodeAction(ISD::SETULT, T, Expand); - } - - for (MVT T : LegalV) { - MVT BoolV = MVT::getVectorVT(MVT::i1, T.getVectorNumElements()); - setOperationAction(ISD::BUILD_VECTOR, BoolV, Custom); - setOperationAction(ISD::CONCAT_VECTORS, BoolV, Custom); - setOperationAction(ISD::INSERT_SUBVECTOR, BoolV, Custom); - setOperationAction(ISD::INSERT_VECTOR_ELT, BoolV, Custom); - setOperationAction(ISD::EXTRACT_SUBVECTOR, BoolV, Custom); - setOperationAction(ISD::EXTRACT_VECTOR_ELT, BoolV, Custom); - } - - for (MVT T : LegalV) { - if (T == ByteV) - continue; - // Promote all shuffles to operate on vectors of bytes. - setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteV); - setPromoteTo(ISD::AND, T, ByteV); - setPromoteTo(ISD::OR, T, ByteV); - setPromoteTo(ISD::XOR, T, ByteV); - } - - for (MVT T : LegalW) { - // Custom-lower BUILD_VECTOR for vector pairs. The standard (target- - // independent) handling of it would convert it to a load, which is - // not always the optimal choice. - setOperationAction(ISD::BUILD_VECTOR, T, Custom); - // Make concat-vectors custom to handle concats of more than 2 vectors. - setOperationAction(ISD::CONCAT_VECTORS, T, Custom); - - // Custom-lower these operations for pairs. Expand them into a concat - // of the corresponding operations on individual vectors. - setOperationAction(ISD::ANY_EXTEND, T, Custom); - setOperationAction(ISD::SIGN_EXTEND, T, Custom); - setOperationAction(ISD::ZERO_EXTEND, T, Custom); - setOperationAction(ISD::SIGN_EXTEND_INREG, T, Custom); - setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom); - setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal); - setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal); - - setOperationAction(ISD::ADD, T, Legal); - setOperationAction(ISD::SUB, T, Legal); - setOperationAction(ISD::MUL, T, Custom); - setOperationAction(ISD::MULHS, T, Custom); - setOperationAction(ISD::MULHU, T, Custom); - setOperationAction(ISD::AND, T, Custom); - setOperationAction(ISD::OR, T, Custom); - setOperationAction(ISD::XOR, T, Custom); - setOperationAction(ISD::SETCC, T, Custom); - setOperationAction(ISD::VSELECT, T, Custom); - if (T != ByteW) { - setOperationAction(ISD::SRA, T, Custom); - setOperationAction(ISD::SHL, T, Custom); - setOperationAction(ISD::SRL, T, Custom); - - // Promote all shuffles to operate on vectors of bytes. - setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteW); - } - - MVT BoolV = MVT::getVectorVT(MVT::i1, T.getVectorNumElements()); - setOperationAction(ISD::SETCC, BoolV, Custom); - } - } + if (Subtarget.useHVXOps()) + initializeHVXLowering(); computeRegisterProperties(&HRI); @@ -2444,6 +2287,8 @@ HexagonTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) unsigned VecLen = AM.size(); MVT VecTy = ty(Op); + assert(!Subtarget.isHVXVectorType(VecTy, true) && + "HVX shuffles should be legal"); assert(VecTy.getSizeInBits() <= 64 && "Unexpected vector length"); SDValue Op0 = Op.getOperand(0); @@ -2559,13 +2404,10 @@ HexagonTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) return SDValue(); } -// Lower a vector shift. Try to convert -// = SHL/SRA/SRL by to Hexagon specific -// = SHL/SRA/SRL by . +// Create a Hexagon-specific node for shifting a vector by an integer. SDValue -HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const { - const SDLoc dl(Op); - +HexagonTargetLowering::getVectorShiftByInt(SDValue Op, SelectionDAG &DAG) + const { if (auto *BVN = dyn_cast(Op.getOperand(1).getNode())) { if (SDValue S = BVN->getSplatValue()) { unsigned NewOpc; @@ -2582,16 +2424,18 @@ HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const { default: llvm_unreachable("Unexpected shift opcode"); } - return DAG.getNode(NewOpc, dl, ty(Op), Op.getOperand(0), S); + return DAG.getNode(NewOpc, SDLoc(Op), ty(Op), Op.getOperand(0), S); } } - if (Subtarget.useHVXOps() && Subtarget.isHVXVectorType(ty(Op))) - return LowerHvxShift(Op, DAG); - return SDValue(); } +SDValue +HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const { + return getVectorShiftByInt(Op, DAG); +} + SDValue HexagonTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const { MVT ResTy = ty(Op); @@ -2610,43 +2454,6 @@ HexagonTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const { return SDValue(); } -// Any-, sign-, and zero-extends of boolean vectors to integer types are -// all the same. - -SDValue -HexagonTargetLowering::LowerANY_EXTEND(SDValue Op, SelectionDAG &DAG) const { - // Lower any-extends of boolean vectors to sign-extends, since they - // translate directly to Q2V. Zero-extending could also be done equally - // fast, but Q2V is used/recognized in more places. - // For all other vectors, use zero-extend. - MVT ResTy = ty(Op); - SDValue InpV = Op.getOperand(0); - MVT ElemTy = ty(InpV).getVectorElementType(); - if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy)) - return LowerSIGN_EXTEND(Op, DAG); - return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Op), ResTy, InpV); -} - -SDValue -HexagonTargetLowering::LowerSIGN_EXTEND(SDValue Op, SelectionDAG &DAG) const { - MVT ResTy = ty(Op); - SDValue InpV = Op.getOperand(0); - MVT ElemTy = ty(InpV).getVectorElementType(); - if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy)) - return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), false, DAG); - return Op; -} - -SDValue -HexagonTargetLowering::LowerZERO_EXTEND(SDValue Op, SelectionDAG &DAG) const { - MVT ResTy = ty(Op); - SDValue InpV = Op.getOperand(0); - MVT ElemTy = ty(InpV).getVectorElementType(); - if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy)) - return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), true, DAG); - return Op; -} - bool HexagonTargetLowering::getBuildVectorConstInts(ArrayRef Values, MVT VecTy, SelectionDAG &DAG, @@ -3022,9 +2829,6 @@ HexagonTargetLowering::getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG) SDValue HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { MVT VecTy = ty(Op); - if (Subtarget.useHVXOps() && Subtarget.isHVXVectorType(VecTy, true)) - return LowerHvxBuildVector(Op, DAG); - unsigned BW = VecTy.getSizeInBits(); const SDLoc &dl(Op); SmallVector Ops; @@ -3064,9 +2868,6 @@ HexagonTargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const { MVT VecTy = ty(Op); const SDLoc &dl(Op); - if (Subtarget.useHVXOps() && Subtarget.isHVXVectorType(VecTy, true)) - return LowerHvxConcatVectors(Op, DAG); - if (VecTy.getSizeInBits() == 64) { assert(Op.getNumOperands() == 2); return DAG.getNode(HexagonISD::COMBINE, dl, VecTy, Op.getOperand(1), @@ -3131,10 +2932,6 @@ SDValue HexagonTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const { SDValue Vec = Op.getOperand(0); - MVT VecTy = ty(Vec); - if (Subtarget.useHVXOps() && Subtarget.isHVXVectorType(VecTy, true)) - return LowerHvxExtractElement(Op, DAG); - MVT ElemTy = ty(Vec).getVectorElementType(); return extractVector(Vec, Op.getOperand(1), SDLoc(Op), ElemTy, ty(Op), DAG); } @@ -3142,31 +2939,20 @@ HexagonTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, SDValue HexagonTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const { - SDValue Vec = Op.getOperand(0); - MVT VecTy = ty(Vec); - if (Subtarget.useHVXOps() && Subtarget.isHVXVectorType(VecTy, true)) - return LowerHvxExtractSubvector(Op, DAG); - - return extractVector(Vec, Op.getOperand(1), SDLoc(Op), ty(Op), ty(Op), DAG); + return extractVector(Op.getOperand(0), Op.getOperand(1), SDLoc(Op), + ty(Op), ty(Op), DAG); } SDValue HexagonTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const { - MVT VecTy = ty(Op); - if (Subtarget.useHVXOps() && Subtarget.isHVXVectorType(VecTy, true)) - return LowerHvxInsertElement(Op, DAG); - return insertVector(Op.getOperand(0), Op.getOperand(1), Op.getOperand(2), - SDLoc(Op), VecTy.getVectorElementType(), DAG); + SDLoc(Op), ty(Op).getVectorElementType(), DAG); } SDValue HexagonTargetLowering::LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const { - if (Subtarget.useHVXOps() && Subtarget.isHVXVectorType(ty(Op), true)) - return LowerHvxInsertSubvector(Op, DAG); - SDValue ValV = Op.getOperand(1); return insertVector(Op.getOperand(0), ValV, Op.getOperand(2), SDLoc(Op), ty(ValV), DAG); diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h index ab85893f03cc..a57acec82041 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h +++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h @@ -298,6 +298,8 @@ namespace HexagonISD { } private: + void initializeHVXLowering(); + bool getBuildVectorConstInts(ArrayRef Values, MVT VecTy, SelectionDAG &DAG, MutableArrayRef Consts) const; @@ -313,6 +315,7 @@ namespace HexagonISD { SelectionDAG &DAG) const; SDValue contractPredicate(SDValue Vec64, const SDLoc &dl, SelectionDAG &DAG) const; + SDValue getVectorShiftByInt(SDValue Op, SelectionDAG &DAG) const; bool isUndef(SDValue Op) const { if (Op.isMachineOpcode()) @@ -403,6 +406,10 @@ namespace HexagonISD { SDValue LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG) const; SDValue LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG) const; SDValue LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG) const; + + SDValue LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const; SDValue LowerHvxMul(SDValue Op, SelectionDAG &DAG) const; SDValue LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const; SDValue LowerHvxSetCC(SDValue Op, SelectionDAG &DAG) const; diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp index 7e9ef9dc85be..4b7321718052 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp +++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp @@ -13,6 +13,168 @@ using namespace llvm; +static const MVT LegalV64[] = { MVT::v64i8, MVT::v32i16, MVT::v16i32 }; +static const MVT LegalW64[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 }; +static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 }; +static const MVT LegalW128[] = { MVT::v256i8, MVT::v128i16, MVT::v64i32 }; + + +void +HexagonTargetLowering::initializeHVXLowering() { + if (Subtarget.useHVX64BOps()) { + addRegisterClass(MVT::v64i8, &Hexagon::HvxVRRegClass); + addRegisterClass(MVT::v32i16, &Hexagon::HvxVRRegClass); + addRegisterClass(MVT::v16i32, &Hexagon::HvxVRRegClass); + addRegisterClass(MVT::v128i8, &Hexagon::HvxWRRegClass); + addRegisterClass(MVT::v64i16, &Hexagon::HvxWRRegClass); + addRegisterClass(MVT::v32i32, &Hexagon::HvxWRRegClass); + // These "short" boolean vector types should be legal because + // they will appear as results of vector compares. If they were + // not legal, type legalization would try to make them legal + // and that would require using operations that do not use or + // produce such types. That, in turn, would imply using custom + // nodes, which would be unoptimizable by the DAG combiner. + // The idea is to rely on target-independent operations as much + // as possible. + addRegisterClass(MVT::v16i1, &Hexagon::HvxQRRegClass); + addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass); + addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass); + addRegisterClass(MVT::v512i1, &Hexagon::HvxQRRegClass); + } else if (Subtarget.useHVX128BOps()) { + addRegisterClass(MVT::v128i8, &Hexagon::HvxVRRegClass); + addRegisterClass(MVT::v64i16, &Hexagon::HvxVRRegClass); + addRegisterClass(MVT::v32i32, &Hexagon::HvxVRRegClass); + addRegisterClass(MVT::v256i8, &Hexagon::HvxWRRegClass); + addRegisterClass(MVT::v128i16, &Hexagon::HvxWRRegClass); + addRegisterClass(MVT::v64i32, &Hexagon::HvxWRRegClass); + addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass); + addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass); + addRegisterClass(MVT::v128i1, &Hexagon::HvxQRRegClass); + addRegisterClass(MVT::v1024i1, &Hexagon::HvxQRRegClass); + } + + // Set up operation actions. + + bool Use64b = Subtarget.useHVX64BOps(); + ArrayRef LegalV = Use64b ? LegalV64 : LegalV128; + ArrayRef LegalW = Use64b ? LegalW64 : LegalW128; + MVT ByteV = Use64b ? MVT::v64i8 : MVT::v128i8; + MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8; + + auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) { + setOperationAction(Opc, FromTy, Promote); + AddPromotedToType(Opc, FromTy, ToTy); + }; + + setOperationAction(ISD::VECTOR_SHUFFLE, ByteV, Legal); + setOperationAction(ISD::VECTOR_SHUFFLE, ByteW, Legal); + setOperationAction(ISD::AND, ByteV, Legal); + setOperationAction(ISD::OR, ByteV, Legal); + setOperationAction(ISD::XOR, ByteV, Legal); + + for (MVT T : LegalV) { + setIndexedLoadAction(ISD::POST_INC, T, Legal); + setIndexedStoreAction(ISD::POST_INC, T, Legal); + + setOperationAction(ISD::ADD, T, Legal); + setOperationAction(ISD::SUB, T, Legal); + if (T != ByteV) { + setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal); + setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal); + } + + setOperationAction(ISD::MUL, T, Custom); + setOperationAction(ISD::MULHS, T, Custom); + setOperationAction(ISD::MULHU, T, Custom); + setOperationAction(ISD::BUILD_VECTOR, T, Custom); + // Make concat-vectors custom to handle concats of more than 2 vectors. + setOperationAction(ISD::CONCAT_VECTORS, T, Custom); + setOperationAction(ISD::INSERT_SUBVECTOR, T, Custom); + setOperationAction(ISD::INSERT_VECTOR_ELT, T, Custom); + setOperationAction(ISD::EXTRACT_SUBVECTOR, T, Custom); + setOperationAction(ISD::EXTRACT_VECTOR_ELT, T, Custom); + setOperationAction(ISD::ANY_EXTEND, T, Custom); + setOperationAction(ISD::SIGN_EXTEND, T, Custom); + setOperationAction(ISD::ZERO_EXTEND, T, Custom); + if (T != ByteV) { + setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom); + // HVX only has shifts of words and halfwords. + setOperationAction(ISD::SRA, T, Custom); + setOperationAction(ISD::SHL, T, Custom); + setOperationAction(ISD::SRL, T, Custom); + } + + setCondCodeAction(ISD::SETNE, T, Expand); + setCondCodeAction(ISD::SETLE, T, Expand); + setCondCodeAction(ISD::SETGE, T, Expand); + setCondCodeAction(ISD::SETLT, T, Expand); + setCondCodeAction(ISD::SETULE, T, Expand); + setCondCodeAction(ISD::SETUGE, T, Expand); + setCondCodeAction(ISD::SETULT, T, Expand); + } + + for (MVT T : LegalV) { + MVT BoolV = MVT::getVectorVT(MVT::i1, T.getVectorNumElements()); + setOperationAction(ISD::BUILD_VECTOR, BoolV, Custom); + setOperationAction(ISD::CONCAT_VECTORS, BoolV, Custom); + setOperationAction(ISD::INSERT_SUBVECTOR, BoolV, Custom); + setOperationAction(ISD::INSERT_VECTOR_ELT, BoolV, Custom); + setOperationAction(ISD::EXTRACT_SUBVECTOR, BoolV, Custom); + setOperationAction(ISD::EXTRACT_VECTOR_ELT, BoolV, Custom); + } + + for (MVT T : LegalV) { + if (T == ByteV) + continue; + // Promote all shuffles to operate on vectors of bytes. + setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteV); + setPromoteTo(ISD::AND, T, ByteV); + setPromoteTo(ISD::OR, T, ByteV); + setPromoteTo(ISD::XOR, T, ByteV); + } + + for (MVT T : LegalW) { + // Custom-lower BUILD_VECTOR for vector pairs. The standard (target- + // independent) handling of it would convert it to a load, which is + // not always the optimal choice. + setOperationAction(ISD::BUILD_VECTOR, T, Custom); + // Make concat-vectors custom to handle concats of more than 2 vectors. + setOperationAction(ISD::CONCAT_VECTORS, T, Custom); + + // Custom-lower these operations for pairs. Expand them into a concat + // of the corresponding operations on individual vectors. + setOperationAction(ISD::ANY_EXTEND, T, Custom); + setOperationAction(ISD::SIGN_EXTEND, T, Custom); + setOperationAction(ISD::ZERO_EXTEND, T, Custom); + setOperationAction(ISD::SIGN_EXTEND_INREG, T, Custom); + setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom); + setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal); + setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal); + + setOperationAction(ISD::ADD, T, Legal); + setOperationAction(ISD::SUB, T, Legal); + setOperationAction(ISD::MUL, T, Custom); + setOperationAction(ISD::MULHS, T, Custom); + setOperationAction(ISD::MULHU, T, Custom); + setOperationAction(ISD::AND, T, Custom); + setOperationAction(ISD::OR, T, Custom); + setOperationAction(ISD::XOR, T, Custom); + setOperationAction(ISD::SETCC, T, Custom); + setOperationAction(ISD::VSELECT, T, Custom); + if (T != ByteW) { + setOperationAction(ISD::SRA, T, Custom); + setOperationAction(ISD::SHL, T, Custom); + setOperationAction(ISD::SRL, T, Custom); + + // Promote all shuffles to operate on vectors of bytes. + setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteW); + } + + MVT BoolV = MVT::getVectorVT(MVT::i1, T.getVectorNumElements()); + setOperationAction(ISD::SETCC, BoolV, Custom); + } +} + SDValue HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef Ops, const SDLoc &dl, SelectionDAG &DAG) const { @@ -916,6 +1078,40 @@ HexagonTargetLowering::LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG) return insertHvxSubvectorReg(VecV, ValV, IdxV, dl, DAG); } +SDValue +HexagonTargetLowering::LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const { + // Lower any-extends of boolean vectors to sign-extends, since they + // translate directly to Q2V. Zero-extending could also be done equally + // fast, but Q2V is used/recognized in more places. + // For all other vectors, use zero-extend. + MVT ResTy = ty(Op); + SDValue InpV = Op.getOperand(0); + MVT ElemTy = ty(InpV).getVectorElementType(); + if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy)) + return LowerHvxSignExt(Op, DAG); + return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Op), ResTy, InpV); +} + +SDValue +HexagonTargetLowering::LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const { + MVT ResTy = ty(Op); + SDValue InpV = Op.getOperand(0); + MVT ElemTy = ty(InpV).getVectorElementType(); + if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy)) + return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), false, DAG); + return Op; +} + +SDValue +HexagonTargetLowering::LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const { + MVT ResTy = ty(Op); + SDValue InpV = Op.getOperand(0); + MVT ElemTy = ty(InpV).getVectorElementType(); + if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy)) + return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), true, DAG); + return Op; +} + SDValue HexagonTargetLowering::LowerHvxMul(SDValue Op, SelectionDAG &DAG) const { MVT ResTy = ty(Op); @@ -1085,6 +1281,8 @@ HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const { SDValue HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const { + if (SDValue S = getVectorShiftByInt(Op, DAG)) + return S; return Op; } @@ -1152,19 +1350,19 @@ HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const { switch (Opc) { default: break; - case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); - case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, DAG); - case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); - case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG); - case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); - case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); - case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); - case ISD::ANY_EXTEND: return LowerANY_EXTEND(Op, DAG); - case ISD::SIGN_EXTEND: return LowerSIGN_EXTEND(Op, DAG); - case ISD::ZERO_EXTEND: return LowerZERO_EXTEND(Op, DAG); + case ISD::BUILD_VECTOR: return LowerHvxBuildVector(Op, DAG); + case ISD::CONCAT_VECTORS: return LowerHvxConcatVectors(Op, DAG); + case ISD::INSERT_SUBVECTOR: return LowerHvxInsertSubvector(Op, DAG); + case ISD::INSERT_VECTOR_ELT: return LowerHvxInsertElement(Op, DAG); + case ISD::EXTRACT_SUBVECTOR: return LowerHvxExtractSubvector(Op, DAG); + case ISD::EXTRACT_VECTOR_ELT: return LowerHvxExtractElement(Op, DAG); + + case ISD::ANY_EXTEND: return LowerHvxAnyExt(Op, DAG); + case ISD::SIGN_EXTEND: return LowerHvxSignExt(Op, DAG); + case ISD::ZERO_EXTEND: return LowerHvxZeroExt(Op, DAG); case ISD::SRA: case ISD::SHL: - case ISD::SRL: return LowerVECTOR_SHIFT(Op, DAG); + case ISD::SRL: return LowerHvxShift(Op, DAG); case ISD::MUL: return LowerHvxMul(Op, DAG); case ISD::MULHS: case ISD::MULHU: return LowerHvxMulh(Op, DAG); @@ -1182,7 +1380,7 @@ bool HexagonTargetLowering::isHvxOperation(SDValue Op) const { // If the type of the result, or any operand type are HVX vector types, // this is an HVX operation. - return Subtarget.isHVXVectorType(ty(Op)) || + return Subtarget.isHVXVectorType(ty(Op), true) || llvm::any_of(Op.getNode()->ops(), [this] (SDValue V) { return Subtarget.isHVXVectorType(ty(V), true); diff --git a/llvm/lib/Target/Hexagon/HexagonPatterns.td b/llvm/lib/Target/Hexagon/HexagonPatterns.td index 757e5c6f1714..6559881c59ff 100644 --- a/llvm/lib/Target/Hexagon/HexagonPatterns.td +++ b/llvm/lib/Target/Hexagon/HexagonPatterns.td @@ -839,20 +839,6 @@ def: Pat<(vselect V4I1:$Pu, V4I16:$Rs, V4I16:$Rt), def: Pat<(vselect V2I1:$Pu, V2I32:$Rs, V2I32:$Rt), (C2_vmux V2I1:$Pu, V2I32:$Rs, V2I32:$Rt)>; - -class HvxSel_pat - : Pat<(select I1:$Pu, RegPred:$Vs, RegPred:$Vt), - (MI I1:$Pu, RegPred:$Vs, RegPred:$Vt)>; - -let Predicates = [HasV60T,UseHVX] in { - def: HvxSel_pat; - def: HvxSel_pat; - def: HvxSel_pat; - def: HvxSel_pat; - def: HvxSel_pat; - def: HvxSel_pat; -} - // From LegalizeDAG.cpp: (Pu ? Pv : Pw) <=> (Pu & Pv) | (!Pu & Pw). def: Pat<(select I1:$Pu, I1:$Pv, I1:$Pw), (C2_or (C2_and I1:$Pu, I1:$Pv), @@ -2061,46 +2047,10 @@ def: Pat<(i1 (load (add I32:$Rs, anyimm0:$Off))), def: Pat<(i1 (load I32:$Rs)), (C2_tfrrp (L2_loadrub_io IntRegs:$Rs, 0))>; -// HVX loads - -multiclass HvxLd_pat { - def: Pat<(VT (Load I32:$Rt)), (MI I32:$Rt, 0)>; - def: Pat<(VT (Load (add I32:$Rt, ImmPred:$s))), (MI I32:$Rt, imm:$s)>; - // The HVX selection code for shuffles can generate vector constants. - // Calling "Select" on the resulting loads from CP fails without these - // patterns. - def: Pat<(VT (Load (HexagonCP tconstpool:$A))), (MI (A2_tfrsi imm:$A), 0)>; - def: Pat<(VT (Load (HexagonAtPcrel tconstpool:$A))), - (MI (C4_addipc imm:$A), 0)>; -} - - -let Predicates = [UseHVX] in { - multiclass HvxLdVs_pat { - defm: HvxLd_pat; - defm: HvxLd_pat; - defm: HvxLd_pat; - } - defm: HvxLdVs_pat; - defm: HvxLdVs_pat; - defm: HvxLdVs_pat; - - multiclass HvxLdWs_pat { - defm: HvxLd_pat; - defm: HvxLd_pat; - defm: HvxLd_pat; - } - defm: HvxLdWs_pat; - defm: HvxLdWs_pat; - defm: HvxLdWs_pat; -} - // --(13) Store ---------------------------------------------------------- // - class Storepi_pat : Pat<(Store Value:$Rt, I32:$Rx, Offset:$s4), (MI I32:$Rx, imm:$s4, Value:$Rt)>; @@ -2537,36 +2487,6 @@ let AddedComplexity = 10 in { def: Storexi_base_pat, I64, S2_storerd_io>; } -// HVX stores - -multiclass HvxSt_pat { - def: Pat<(Store Value:$Vs, I32:$Rt), - (MI I32:$Rt, 0, Value:$Vs)>; - def: Pat<(Store Value:$Vs, (add I32:$Rt, ImmPred:$s)), - (MI I32:$Rt, imm:$s, Value:$Vs)>; -} - -let Predicates = [UseHVX] in { - multiclass HvxStVs_pat { - defm: HvxSt_pat; - defm: HvxSt_pat; - defm: HvxSt_pat; - } - defm: HvxStVs_pat; - defm: HvxStVs_pat; - defm: HvxStVs_pat; - - multiclass HvxStWs_pat { - defm: HvxSt_pat; - defm: HvxSt_pat; - defm: HvxSt_pat; - } - defm: HvxStWs_pat; - defm: HvxStWs_pat; - defm: HvxStWs_pat; -} - // --(14) Memop ---------------------------------------------------------- // @@ -2975,262 +2895,3 @@ def HexagonREADCYCLE: SDNode<"HexagonISD::READCYCLE", SDTInt64Leaf, [SDNPHasChain]>; def: Pat<(HexagonREADCYCLE), (A4_tfrcpp UPCYCLE)>; - - -def SDTVecLeaf: - SDTypeProfile<1, 0, [SDTCisVec<0>]>; -def SDTVecBinOp: - SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>, SDTCisSameAs<1,2>]>; - -def SDTHexagonVEXTRACTW: SDTypeProfile<1, 2, - [SDTCisVT<0, i32>, SDTCisVec<1>, SDTCisVT<2, i32>]>; -def HexagonVEXTRACTW : SDNode<"HexagonISD::VEXTRACTW", SDTHexagonVEXTRACTW>; - -def SDTHexagonVINSERTW0: SDTypeProfile<1, 2, - [SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisVT<2, i32>]>; -def HexagonVINSERTW0 : SDNode<"HexagonISD::VINSERTW0", SDTHexagonVINSERTW0>; - -def HwLen2: SDNodeXForm(CurDAG->getSubtarget()); - return CurDAG->getTargetConstant(ST.getVectorLength()/2, SDLoc(N), MVT::i32); -}]>; - -def Q2V: OutPatFrag<(ops node:$Qs), (V6_vandqrt $Qs, (A2_tfrsi -1))>; - -def Combinev: OutPatFrag<(ops node:$Vs, node:$Vt), - (REG_SEQUENCE HvxWR, $Vs, vsub_hi, $Vt, vsub_lo)>; - -def Combineq: OutPatFrag<(ops node:$Qs, node:$Qt), - (V6_vandvrt - (V6_vor - (V6_vror (V6_vpackeb (V6_vd0), (Q2V $Qs)), - (A2_tfrsi (HwLen2 (i32 0)))), // Half the vector length - (V6_vpackeb (V6_vd0), (Q2V $Qt))), - (A2_tfrsi -1))>; - -def LoVec: OutPatFrag<(ops node:$Vs), (EXTRACT_SUBREG $Vs, vsub_lo)>; -def HiVec: OutPatFrag<(ops node:$Vs), (EXTRACT_SUBREG $Vs, vsub_hi)>; - -def HexagonVZERO: SDNode<"HexagonISD::VZERO", SDTVecLeaf>; -def HexagonQCAT: SDNode<"HexagonISD::QCAT", SDTVecBinOp>; -def HexagonQTRUE: SDNode<"HexagonISD::QTRUE", SDTVecLeaf>; -def HexagonQFALSE: SDNode<"HexagonISD::QFALSE", SDTVecLeaf>; -def vzero: PatFrag<(ops), (HexagonVZERO)>; -def qtrue: PatFrag<(ops), (HexagonQTRUE)>; -def qfalse: PatFrag<(ops), (HexagonQFALSE)>; -def qcat: PatFrag<(ops node:$Qs, node:$Qt), - (HexagonQCAT node:$Qs, node:$Qt)>; - -def qnot: PatFrag<(ops node:$Qs), (xor node:$Qs, qtrue)>; - -def VSxtb: OutPatFrag<(ops node:$Vs), (V6_vunpackb $Vs)>; -def VSxth: OutPatFrag<(ops node:$Vs), (V6_vunpackh $Vs)>; -def VZxtb: OutPatFrag<(ops node:$Vs), (V6_vunpackub $Vs)>; -def VZxth: OutPatFrag<(ops node:$Vs), (V6_vunpackuh $Vs)>; - -def SplatB: SDNodeXFormgetZExtValue(); - assert(isUInt<8>(V)); - uint32_t S = V << 24 | V << 16 | V << 8 | V; - return CurDAG->getTargetConstant(S, SDLoc(N), MVT::i32); -}]>; - -def SplatH: SDNodeXFormgetZExtValue(); - assert(isUInt<16>(V)); - return CurDAG->getTargetConstant(V << 16 | V, SDLoc(N), MVT::i32); -}]>; - -let Predicates = [UseHVX] in { - def: Pat<(VecI8 vzero), (V6_vd0)>; - def: Pat<(VecI16 vzero), (V6_vd0)>; - def: Pat<(VecI32 vzero), (V6_vd0)>; - // Use V6_vsubw_dv instead. - def: Pat<(VecPI8 vzero), (Combinev (V6_vd0), (V6_vd0))>; - def: Pat<(VecPI16 vzero), (Combinev (V6_vd0), (V6_vd0))>; - def: Pat<(VecPI32 vzero), (Combinev (V6_vd0), (V6_vd0))>; - - def: Pat<(VecPI8 (concat_vectors HVI8:$Vs, HVI8:$Vt)), - (Combinev HvxVR:$Vt, HvxVR:$Vs)>; - def: Pat<(VecPI16 (concat_vectors HVI16:$Vs, HVI16:$Vt)), - (Combinev HvxVR:$Vt, HvxVR:$Vs)>; - def: Pat<(VecPI32 (concat_vectors HVI32:$Vs, HVI32:$Vt)), - (Combinev HvxVR:$Vt, HvxVR:$Vs)>; - - def: Pat<(VecQ8 (qcat HQ16:$Qs, HQ16:$Qt)), (Combineq $Qs, $Qt)>; - def: Pat<(VecQ16 (qcat HQ32:$Qs, HQ32:$Qt)), (Combineq $Qs, $Qt)>; - - def: Pat<(HexagonVEXTRACTW HVI8:$Vu, I32:$Rs), - (V6_extractw HvxVR:$Vu, I32:$Rs)>; - def: Pat<(HexagonVEXTRACTW HVI16:$Vu, I32:$Rs), - (V6_extractw HvxVR:$Vu, I32:$Rs)>; - def: Pat<(HexagonVEXTRACTW HVI32:$Vu, I32:$Rs), - (V6_extractw HvxVR:$Vu, I32:$Rs)>; - - def: Pat<(HexagonVINSERTW0 HVI8:$Vu, I32:$Rt), - (V6_vinsertwr HvxVR:$Vu, I32:$Rt)>; - def: Pat<(HexagonVINSERTW0 HVI16:$Vu, I32:$Rt), - (V6_vinsertwr HvxVR:$Vu, I32:$Rt)>; - def: Pat<(HexagonVINSERTW0 HVI32:$Vu, I32:$Rt), - (V6_vinsertwr HvxVR:$Vu, I32:$Rt)>; - - let AddedComplexity = 10 in { - def: Pat<(VecI8 (HexagonVSPLAT u8_0ImmPred:$V)), - (V6_lvsplatw (ToI32 (SplatB $V)))>; - def: Pat<(VecI16 (HexagonVSPLAT u16_0ImmPred:$V)), - (V6_lvsplatw (ToI32 (SplatH $V)))>; - def: Pat<(VecI32 (HexagonVSPLAT anyimm:$V)), - (V6_lvsplatw (ToI32 $V))>; - def: Pat<(VecPI8 (HexagonVSPLAT u8_0ImmPred:$V)), - (Combinev (V6_lvsplatw (ToI32 (SplatB $V))), - (V6_lvsplatw (ToI32 (SplatB $V))))>; - def: Pat<(VecPI16 (HexagonVSPLAT u16_0ImmPred:$V)), - (Combinev (V6_lvsplatw (ToI32 (SplatH $V))), - (V6_lvsplatw (ToI32 (SplatH $V))))>; - def: Pat<(VecPI32 (HexagonVSPLAT anyimm:$V)), - (Combinev (V6_lvsplatw (ToI32 $V)), (V6_lvsplatw (ToI32 $V)))>; - } - def: Pat<(VecI8 (HexagonVSPLAT I32:$Rs)), - (V6_lvsplatw (S2_vsplatrb I32:$Rs))>; - def: Pat<(VecI16 (HexagonVSPLAT I32:$Rs)), - (V6_lvsplatw (A2_combine_ll I32:$Rs, I32:$Rs))>; - def: Pat<(VecI32 (HexagonVSPLAT I32:$Rs)), - (V6_lvsplatw I32:$Rs)>; - def: Pat<(VecPI8 (HexagonVSPLAT I32:$Rs)), - (Combinev (V6_lvsplatw (S2_vsplatrb I32:$Rs)), - (V6_lvsplatw (S2_vsplatrb I32:$Rs)))>; - def: Pat<(VecPI16 (HexagonVSPLAT I32:$Rs)), - (Combinev (V6_lvsplatw (A2_combine_ll I32:$Rs, I32:$Rs)), - (V6_lvsplatw (A2_combine_ll I32:$Rs, I32:$Rs)))>; - def: Pat<(VecPI32 (HexagonVSPLAT I32:$Rs)), - (Combinev (V6_lvsplatw I32:$Rs), (V6_lvsplatw I32:$Rs))>; - - def: Pat<(add HVI8:$Vs, HVI8:$Vt), (V6_vaddb HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(add HVI16:$Vs, HVI16:$Vt), (V6_vaddh HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(add HVI32:$Vs, HVI32:$Vt), (V6_vaddw HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(add HWI8:$Vs, HWI8:$Vt), (V6_vaddb_dv HvxWR:$Vs, HvxWR:$Vt)>; - def: Pat<(add HWI16:$Vs, HWI16:$Vt), (V6_vaddh_dv HvxWR:$Vs, HvxWR:$Vt)>; - def: Pat<(add HWI32:$Vs, HWI32:$Vt), (V6_vaddw_dv HvxWR:$Vs, HvxWR:$Vt)>; - - def: Pat<(sub HVI8:$Vs, HVI8:$Vt), (V6_vsubb HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(sub HVI16:$Vs, HVI16:$Vt), (V6_vsubh HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(sub HVI32:$Vs, HVI32:$Vt), (V6_vsubw HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(sub HWI8:$Vs, HWI8:$Vt), (V6_vsubb_dv HvxWR:$Vs, HvxWR:$Vt)>; - def: Pat<(sub HWI16:$Vs, HWI16:$Vt), (V6_vsubh_dv HvxWR:$Vs, HvxWR:$Vt)>; - def: Pat<(sub HWI32:$Vs, HWI32:$Vt), (V6_vsubw_dv HvxWR:$Vs, HvxWR:$Vt)>; - - def: Pat<(and HVI8:$Vs, HVI8:$Vt), (V6_vand HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(or HVI8:$Vs, HVI8:$Vt), (V6_vor HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(xor HVI8:$Vs, HVI8:$Vt), (V6_vxor HvxVR:$Vs, HvxVR:$Vt)>; - - def: Pat<(vselect HQ8:$Qu, HVI8:$Vs, HVI8:$Vt), - (V6_vmux HvxQR:$Qu, HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(vselect HQ16:$Qu, HVI16:$Vs, HVI16:$Vt), - (V6_vmux HvxQR:$Qu, HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(vselect HQ32:$Qu, HVI32:$Vs, HVI32:$Vt), - (V6_vmux HvxQR:$Qu, HvxVR:$Vs, HvxVR:$Vt)>; - - def: Pat<(vselect (qnot HQ8:$Qu), HVI8:$Vs, HVI8:$Vt), - (V6_vmux HvxQR:$Qu, HvxVR:$Vt, HvxVR:$Vs)>; - def: Pat<(vselect (qnot HQ16:$Qu), HVI16:$Vs, HVI16:$Vt), - (V6_vmux HvxQR:$Qu, HvxVR:$Vt, HvxVR:$Vs)>; - def: Pat<(vselect (qnot HQ32:$Qu), HVI32:$Vs, HVI32:$Vt), - (V6_vmux HvxQR:$Qu, HvxVR:$Vt, HvxVR:$Vs)>; - - def: Pat<(VecPI16 (sext HVI8:$Vs)), (VSxtb $Vs)>; - def: Pat<(VecPI32 (sext HVI16:$Vs)), (VSxth $Vs)>; - def: Pat<(VecPI16 (zext HVI8:$Vs)), (VZxtb $Vs)>; - def: Pat<(VecPI32 (zext HVI16:$Vs)), (VZxth $Vs)>; - - def: Pat<(VecI16 (sext_invec HVI8:$Vs)), (LoVec (VSxtb $Vs))>; - def: Pat<(VecI32 (sext_invec HVI16:$Vs)), (LoVec (VSxth $Vs))>; - def: Pat<(VecI32 (sext_invec HVI8:$Vs)), - (LoVec (VSxth (LoVec (VSxtb $Vs))))>; - def: Pat<(VecPI16 (sext_invec HWI8:$Vss)), (VSxtb (LoVec $Vss))>; - def: Pat<(VecPI32 (sext_invec HWI16:$Vss)), (VSxth (LoVec $Vss))>; - def: Pat<(VecPI32 (sext_invec HWI8:$Vss)), - (VSxth (LoVec (VSxtb (LoVec $Vss))))>; - - def: Pat<(VecI16 (zext_invec HVI8:$Vs)), (LoVec (VZxtb $Vs))>; - def: Pat<(VecI32 (zext_invec HVI16:$Vs)), (LoVec (VZxth $Vs))>; - def: Pat<(VecI32 (zext_invec HVI8:$Vs)), - (LoVec (VZxth (LoVec (VZxtb $Vs))))>; - def: Pat<(VecPI16 (zext_invec HWI8:$Vss)), (VZxtb (LoVec $Vss))>; - def: Pat<(VecPI32 (zext_invec HWI16:$Vss)), (VZxth (LoVec $Vss))>; - def: Pat<(VecPI32 (zext_invec HWI8:$Vss)), - (VZxth (LoVec (VZxtb (LoVec $Vss))))>; - - // The "source" types are not legal, and there are no parameterized - // definitions for them, but they are length-specific. - let Predicates = [UseHVX,UseHVX64B] in { - def: Pat<(VecI16 (sext_inreg HVI16:$Vs, v32i8)), - (V6_vasrh (V6_vaslh HVI16:$Vs, (A2_tfrsi 8)), (A2_tfrsi 8))>; - def: Pat<(VecI32 (sext_inreg HVI32:$Vs, v16i8)), - (V6_vasrw (V6_vaslw HVI32:$Vs, (A2_tfrsi 24)), (A2_tfrsi 24))>; - def: Pat<(VecI32 (sext_inreg HVI32:$Vs, v16i16)), - (V6_vasrw (V6_vaslw HVI32:$Vs, (A2_tfrsi 16)), (A2_tfrsi 16))>; - } - let Predicates = [UseHVX,UseHVX128B] in { - def: Pat<(VecI16 (sext_inreg HVI16:$Vs, v64i8)), - (V6_vasrh (V6_vaslh HVI16:$Vs, (A2_tfrsi 8)), (A2_tfrsi 8))>; - def: Pat<(VecI32 (sext_inreg HVI32:$Vs, v32i8)), - (V6_vasrw (V6_vaslw HVI32:$Vs, (A2_tfrsi 24)), (A2_tfrsi 24))>; - def: Pat<(VecI32 (sext_inreg HVI32:$Vs, v32i16)), - (V6_vasrw (V6_vaslw HVI32:$Vs, (A2_tfrsi 16)), (A2_tfrsi 16))>; - } - - def: Pat<(HexagonVASL HVI8:$Vs, I32:$Rt), - (V6_vpackeb (V6_vaslh (HiVec (VZxtb HvxVR:$Vs)), I32:$Rt), - (V6_vaslh (LoVec (VZxtb HvxVR:$Vs)), I32:$Rt))>; - def: Pat<(HexagonVASR HVI8:$Vs, I32:$Rt), - (V6_vpackeb (V6_vasrh (HiVec (VSxtb HvxVR:$Vs)), I32:$Rt), - (V6_vasrh (LoVec (VSxtb HvxVR:$Vs)), I32:$Rt))>; - def: Pat<(HexagonVLSR HVI8:$Vs, I32:$Rt), - (V6_vpackeb (V6_vlsrh (HiVec (VZxtb HvxVR:$Vs)), I32:$Rt), - (V6_vlsrh (LoVec (VZxtb HvxVR:$Vs)), I32:$Rt))>; - - def: Pat<(HexagonVASL HVI16:$Vs, I32:$Rt), (V6_vaslh HvxVR:$Vs, I32:$Rt)>; - def: Pat<(HexagonVASL HVI32:$Vs, I32:$Rt), (V6_vaslw HvxVR:$Vs, I32:$Rt)>; - def: Pat<(HexagonVASR HVI16:$Vs, I32:$Rt), (V6_vasrh HvxVR:$Vs, I32:$Rt)>; - def: Pat<(HexagonVASR HVI32:$Vs, I32:$Rt), (V6_vasrw HvxVR:$Vs, I32:$Rt)>; - def: Pat<(HexagonVLSR HVI16:$Vs, I32:$Rt), (V6_vlsrh HvxVR:$Vs, I32:$Rt)>; - def: Pat<(HexagonVLSR HVI32:$Vs, I32:$Rt), (V6_vlsrw HvxVR:$Vs, I32:$Rt)>; - - def: Pat<(shl HVI16:$Vs, HVI16:$Vt), (V6_vaslhv HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(shl HVI32:$Vs, HVI32:$Vt), (V6_vaslwv HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(sra HVI16:$Vs, HVI16:$Vt), (V6_vasrhv HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(sra HVI32:$Vs, HVI32:$Vt), (V6_vasrwv HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(srl HVI16:$Vs, HVI16:$Vt), (V6_vlsrhv HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(srl HVI32:$Vs, HVI32:$Vt), (V6_vlsrwv HvxVR:$Vs, HvxVR:$Vt)>; - - def: Pat<(vnot HQ8:$Qs), (V6_pred_not HvxQR:$Qs)>; - def: Pat<(vnot HQ16:$Qs), (V6_pred_not HvxQR:$Qs)>; - def: Pat<(vnot HQ32:$Qs), (V6_pred_not HvxQR:$Qs)>; - def: Pat<(qnot HQ8:$Qs), (V6_pred_not HvxQR:$Qs)>; - def: Pat<(qnot HQ16:$Qs), (V6_pred_not HvxQR:$Qs)>; - def: Pat<(qnot HQ32:$Qs), (V6_pred_not HvxQR:$Qs)>; - - def: Pat<(VecQ8 (seteq HVI8:$Vs, HVI8:$Vt)), - (V6_veqb HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(VecQ8 (setgt HVI8:$Vs, HVI8:$Vt)), - (V6_vgtb HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(VecQ8 (setugt HVI8:$Vs, HVI8:$Vt)), - (V6_vgtub HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(VecQ16 (seteq HVI16:$Vs, HVI16:$Vt)), - (V6_veqh HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(VecQ16 (setgt HVI16:$Vs, HVI16:$Vt)), - (V6_vgth HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(VecQ16 (setugt HVI16:$Vs, HVI16:$Vt)), - (V6_vgtuh HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(VecQ32 (seteq HVI32:$Vs, HVI32:$Vt)), - (V6_veqw HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(VecQ32 (setgt HVI32:$Vs, HVI32:$Vt)), - (V6_vgtw HvxVR:$Vs, HvxVR:$Vt)>; - def: Pat<(VecQ32 (setugt HVI32:$Vs, HVI32:$Vt)), - (V6_vgtuw HvxVR:$Vs, HvxVR:$Vt)>; - - def: Pat<(VecI8 (trunc HWI16:$Vss)), - (V6_vpackeb (HiVec $Vss), (LoVec $Vss))>; - def: Pat<(VecI16 (trunc HWI32:$Vss)), - (V6_vpackeh (HiVec $Vss), (LoVec $Vss))>; -} diff --git a/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td b/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td new file mode 100644 index 000000000000..0e5436411cf5 --- /dev/null +++ b/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td @@ -0,0 +1,338 @@ +def SDTVecLeaf: + SDTypeProfile<1, 0, [SDTCisVec<0>]>; +def SDTVecBinOp: + SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>, SDTCisSameAs<1,2>]>; + +def SDTHexagonVEXTRACTW: SDTypeProfile<1, 2, + [SDTCisVT<0, i32>, SDTCisVec<1>, SDTCisVT<2, i32>]>; +def HexagonVEXTRACTW : SDNode<"HexagonISD::VEXTRACTW", SDTHexagonVEXTRACTW>; + +def SDTHexagonVINSERTW0: SDTypeProfile<1, 2, + [SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisVT<2, i32>]>; +def HexagonVINSERTW0 : SDNode<"HexagonISD::VINSERTW0", SDTHexagonVINSERTW0>; + +def HwLen2: SDNodeXForm(CurDAG->getSubtarget()); + return CurDAG->getTargetConstant(ST.getVectorLength()/2, SDLoc(N), MVT::i32); +}]>; + +def Q2V: OutPatFrag<(ops node:$Qs), (V6_vandqrt $Qs, (A2_tfrsi -1))>; + +def Combinev: OutPatFrag<(ops node:$Vs, node:$Vt), + (REG_SEQUENCE HvxWR, $Vs, vsub_hi, $Vt, vsub_lo)>; + +def Combineq: OutPatFrag<(ops node:$Qs, node:$Qt), + (V6_vandvrt + (V6_vor + (V6_vror (V6_vpackeb (V6_vd0), (Q2V $Qs)), + (A2_tfrsi (HwLen2 (i32 0)))), // Half the vector length + (V6_vpackeb (V6_vd0), (Q2V $Qt))), + (A2_tfrsi -1))>; + +def LoVec: OutPatFrag<(ops node:$Vs), (EXTRACT_SUBREG $Vs, vsub_lo)>; +def HiVec: OutPatFrag<(ops node:$Vs), (EXTRACT_SUBREG $Vs, vsub_hi)>; + +def HexagonVZERO: SDNode<"HexagonISD::VZERO", SDTVecLeaf>; +def HexagonQCAT: SDNode<"HexagonISD::QCAT", SDTVecBinOp>; +def HexagonQTRUE: SDNode<"HexagonISD::QTRUE", SDTVecLeaf>; +def HexagonQFALSE: SDNode<"HexagonISD::QFALSE", SDTVecLeaf>; +def vzero: PatFrag<(ops), (HexagonVZERO)>; +def qtrue: PatFrag<(ops), (HexagonQTRUE)>; +def qfalse: PatFrag<(ops), (HexagonQFALSE)>; +def qcat: PatFrag<(ops node:$Qs, node:$Qt), + (HexagonQCAT node:$Qs, node:$Qt)>; + +def qnot: PatFrag<(ops node:$Qs), (xor node:$Qs, qtrue)>; + +def VSxtb: OutPatFrag<(ops node:$Vs), (V6_vunpackb $Vs)>; +def VSxth: OutPatFrag<(ops node:$Vs), (V6_vunpackh $Vs)>; +def VZxtb: OutPatFrag<(ops node:$Vs), (V6_vunpackub $Vs)>; +def VZxth: OutPatFrag<(ops node:$Vs), (V6_vunpackuh $Vs)>; + +def SplatB: SDNodeXFormgetZExtValue(); + assert(isUInt<8>(V)); + uint32_t S = V << 24 | V << 16 | V << 8 | V; + return CurDAG->getTargetConstant(S, SDLoc(N), MVT::i32); +}]>; + +def SplatH: SDNodeXFormgetZExtValue(); + assert(isUInt<16>(V)); + return CurDAG->getTargetConstant(V << 16 | V, SDLoc(N), MVT::i32); +}]>; + + +// HVX loads + +multiclass HvxLd_pat { + def: Pat<(VT (Load I32:$Rt)), (MI I32:$Rt, 0)>; + def: Pat<(VT (Load (add I32:$Rt, ImmPred:$s))), (MI I32:$Rt, imm:$s)>; + // The HVX selection code for shuffles can generate vector constants. + // Calling "Select" on the resulting loads from CP fails without these + // patterns. + def: Pat<(VT (Load (HexagonCP tconstpool:$A))), (MI (A2_tfrsi imm:$A), 0)>; + def: Pat<(VT (Load (HexagonAtPcrel tconstpool:$A))), + (MI (C4_addipc imm:$A), 0)>; +} + +let Predicates = [UseHVX] in { + multiclass HvxLdVs_pat { + defm: HvxLd_pat; + defm: HvxLd_pat; + defm: HvxLd_pat; + } + defm: HvxLdVs_pat; + defm: HvxLdVs_pat; + defm: HvxLdVs_pat; + + multiclass HvxLdWs_pat { + defm: HvxLd_pat; + defm: HvxLd_pat; + defm: HvxLd_pat; + } + defm: HvxLdWs_pat; + defm: HvxLdWs_pat; + defm: HvxLdWs_pat; +} + + +// HVX stores + +multiclass HvxSt_pat { + def: Pat<(Store Value:$Vs, I32:$Rt), + (MI I32:$Rt, 0, Value:$Vs)>; + def: Pat<(Store Value:$Vs, (add I32:$Rt, ImmPred:$s)), + (MI I32:$Rt, imm:$s, Value:$Vs)>; +} + +let Predicates = [UseHVX] in { + multiclass HvxStVs_pat { + defm: HvxSt_pat; + defm: HvxSt_pat; + defm: HvxSt_pat; + } + defm: HvxStVs_pat; + defm: HvxStVs_pat; + defm: HvxStVs_pat; + + multiclass HvxStWs_pat { + defm: HvxSt_pat; + defm: HvxSt_pat; + defm: HvxSt_pat; + } + defm: HvxStWs_pat; + defm: HvxStWs_pat; + defm: HvxStWs_pat; +} + + +let Predicates = [UseHVX] in { + def: Pat<(VecI8 vzero), (V6_vd0)>; + def: Pat<(VecI16 vzero), (V6_vd0)>; + def: Pat<(VecI32 vzero), (V6_vd0)>; + // Use V6_vsubw_dv instead. + def: Pat<(VecPI8 vzero), (Combinev (V6_vd0), (V6_vd0))>; + def: Pat<(VecPI16 vzero), (Combinev (V6_vd0), (V6_vd0))>; + def: Pat<(VecPI32 vzero), (Combinev (V6_vd0), (V6_vd0))>; + + def: Pat<(VecPI8 (concat_vectors HVI8:$Vs, HVI8:$Vt)), + (Combinev HvxVR:$Vt, HvxVR:$Vs)>; + def: Pat<(VecPI16 (concat_vectors HVI16:$Vs, HVI16:$Vt)), + (Combinev HvxVR:$Vt, HvxVR:$Vs)>; + def: Pat<(VecPI32 (concat_vectors HVI32:$Vs, HVI32:$Vt)), + (Combinev HvxVR:$Vt, HvxVR:$Vs)>; + + def: Pat<(VecQ8 (qcat HQ16:$Qs, HQ16:$Qt)), (Combineq $Qs, $Qt)>; + def: Pat<(VecQ16 (qcat HQ32:$Qs, HQ32:$Qt)), (Combineq $Qs, $Qt)>; + + def: Pat<(HexagonVEXTRACTW HVI8:$Vu, I32:$Rs), + (V6_extractw HvxVR:$Vu, I32:$Rs)>; + def: Pat<(HexagonVEXTRACTW HVI16:$Vu, I32:$Rs), + (V6_extractw HvxVR:$Vu, I32:$Rs)>; + def: Pat<(HexagonVEXTRACTW HVI32:$Vu, I32:$Rs), + (V6_extractw HvxVR:$Vu, I32:$Rs)>; + + def: Pat<(HexagonVINSERTW0 HVI8:$Vu, I32:$Rt), + (V6_vinsertwr HvxVR:$Vu, I32:$Rt)>; + def: Pat<(HexagonVINSERTW0 HVI16:$Vu, I32:$Rt), + (V6_vinsertwr HvxVR:$Vu, I32:$Rt)>; + def: Pat<(HexagonVINSERTW0 HVI32:$Vu, I32:$Rt), + (V6_vinsertwr HvxVR:$Vu, I32:$Rt)>; + + let AddedComplexity = 10 in { + def: Pat<(VecI8 (HexagonVSPLAT u8_0ImmPred:$V)), + (V6_lvsplatw (ToI32 (SplatB $V)))>; + def: Pat<(VecI16 (HexagonVSPLAT u16_0ImmPred:$V)), + (V6_lvsplatw (ToI32 (SplatH $V)))>; + def: Pat<(VecI32 (HexagonVSPLAT anyimm:$V)), + (V6_lvsplatw (ToI32 $V))>; + def: Pat<(VecPI8 (HexagonVSPLAT u8_0ImmPred:$V)), + (Combinev (V6_lvsplatw (ToI32 (SplatB $V))), + (V6_lvsplatw (ToI32 (SplatB $V))))>; + def: Pat<(VecPI16 (HexagonVSPLAT u16_0ImmPred:$V)), + (Combinev (V6_lvsplatw (ToI32 (SplatH $V))), + (V6_lvsplatw (ToI32 (SplatH $V))))>; + def: Pat<(VecPI32 (HexagonVSPLAT anyimm:$V)), + (Combinev (V6_lvsplatw (ToI32 $V)), (V6_lvsplatw (ToI32 $V)))>; + } + def: Pat<(VecI8 (HexagonVSPLAT I32:$Rs)), + (V6_lvsplatw (S2_vsplatrb I32:$Rs))>; + def: Pat<(VecI16 (HexagonVSPLAT I32:$Rs)), + (V6_lvsplatw (A2_combine_ll I32:$Rs, I32:$Rs))>; + def: Pat<(VecI32 (HexagonVSPLAT I32:$Rs)), + (V6_lvsplatw I32:$Rs)>; + def: Pat<(VecPI8 (HexagonVSPLAT I32:$Rs)), + (Combinev (V6_lvsplatw (S2_vsplatrb I32:$Rs)), + (V6_lvsplatw (S2_vsplatrb I32:$Rs)))>; + def: Pat<(VecPI16 (HexagonVSPLAT I32:$Rs)), + (Combinev (V6_lvsplatw (A2_combine_ll I32:$Rs, I32:$Rs)), + (V6_lvsplatw (A2_combine_ll I32:$Rs, I32:$Rs)))>; + def: Pat<(VecPI32 (HexagonVSPLAT I32:$Rs)), + (Combinev (V6_lvsplatw I32:$Rs), (V6_lvsplatw I32:$Rs))>; + + def: Pat<(add HVI8:$Vs, HVI8:$Vt), (V6_vaddb HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(add HVI16:$Vs, HVI16:$Vt), (V6_vaddh HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(add HVI32:$Vs, HVI32:$Vt), (V6_vaddw HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(add HWI8:$Vs, HWI8:$Vt), (V6_vaddb_dv HvxWR:$Vs, HvxWR:$Vt)>; + def: Pat<(add HWI16:$Vs, HWI16:$Vt), (V6_vaddh_dv HvxWR:$Vs, HvxWR:$Vt)>; + def: Pat<(add HWI32:$Vs, HWI32:$Vt), (V6_vaddw_dv HvxWR:$Vs, HvxWR:$Vt)>; + + def: Pat<(sub HVI8:$Vs, HVI8:$Vt), (V6_vsubb HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(sub HVI16:$Vs, HVI16:$Vt), (V6_vsubh HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(sub HVI32:$Vs, HVI32:$Vt), (V6_vsubw HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(sub HWI8:$Vs, HWI8:$Vt), (V6_vsubb_dv HvxWR:$Vs, HvxWR:$Vt)>; + def: Pat<(sub HWI16:$Vs, HWI16:$Vt), (V6_vsubh_dv HvxWR:$Vs, HvxWR:$Vt)>; + def: Pat<(sub HWI32:$Vs, HWI32:$Vt), (V6_vsubw_dv HvxWR:$Vs, HvxWR:$Vt)>; + + def: Pat<(and HVI8:$Vs, HVI8:$Vt), (V6_vand HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(or HVI8:$Vs, HVI8:$Vt), (V6_vor HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(xor HVI8:$Vs, HVI8:$Vt), (V6_vxor HvxVR:$Vs, HvxVR:$Vt)>; + + def: Pat<(vselect HQ8:$Qu, HVI8:$Vs, HVI8:$Vt), + (V6_vmux HvxQR:$Qu, HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(vselect HQ16:$Qu, HVI16:$Vs, HVI16:$Vt), + (V6_vmux HvxQR:$Qu, HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(vselect HQ32:$Qu, HVI32:$Vs, HVI32:$Vt), + (V6_vmux HvxQR:$Qu, HvxVR:$Vs, HvxVR:$Vt)>; + + def: Pat<(vselect (qnot HQ8:$Qu), HVI8:$Vs, HVI8:$Vt), + (V6_vmux HvxQR:$Qu, HvxVR:$Vt, HvxVR:$Vs)>; + def: Pat<(vselect (qnot HQ16:$Qu), HVI16:$Vs, HVI16:$Vt), + (V6_vmux HvxQR:$Qu, HvxVR:$Vt, HvxVR:$Vs)>; + def: Pat<(vselect (qnot HQ32:$Qu), HVI32:$Vs, HVI32:$Vt), + (V6_vmux HvxQR:$Qu, HvxVR:$Vt, HvxVR:$Vs)>; + + def: Pat<(VecPI16 (sext HVI8:$Vs)), (VSxtb $Vs)>; + def: Pat<(VecPI32 (sext HVI16:$Vs)), (VSxth $Vs)>; + def: Pat<(VecPI16 (zext HVI8:$Vs)), (VZxtb $Vs)>; + def: Pat<(VecPI32 (zext HVI16:$Vs)), (VZxth $Vs)>; + + def: Pat<(VecI16 (sext_invec HVI8:$Vs)), (LoVec (VSxtb $Vs))>; + def: Pat<(VecI32 (sext_invec HVI16:$Vs)), (LoVec (VSxth $Vs))>; + def: Pat<(VecI32 (sext_invec HVI8:$Vs)), + (LoVec (VSxth (LoVec (VSxtb $Vs))))>; + def: Pat<(VecPI16 (sext_invec HWI8:$Vss)), (VSxtb (LoVec $Vss))>; + def: Pat<(VecPI32 (sext_invec HWI16:$Vss)), (VSxth (LoVec $Vss))>; + def: Pat<(VecPI32 (sext_invec HWI8:$Vss)), + (VSxth (LoVec (VSxtb (LoVec $Vss))))>; + + def: Pat<(VecI16 (zext_invec HVI8:$Vs)), (LoVec (VZxtb $Vs))>; + def: Pat<(VecI32 (zext_invec HVI16:$Vs)), (LoVec (VZxth $Vs))>; + def: Pat<(VecI32 (zext_invec HVI8:$Vs)), + (LoVec (VZxth (LoVec (VZxtb $Vs))))>; + def: Pat<(VecPI16 (zext_invec HWI8:$Vss)), (VZxtb (LoVec $Vss))>; + def: Pat<(VecPI32 (zext_invec HWI16:$Vss)), (VZxth (LoVec $Vss))>; + def: Pat<(VecPI32 (zext_invec HWI8:$Vss)), + (VZxth (LoVec (VZxtb (LoVec $Vss))))>; + + // The "source" types are not legal, and there are no parameterized + // definitions for them, but they are length-specific. + let Predicates = [UseHVX,UseHVX64B] in { + def: Pat<(VecI16 (sext_inreg HVI16:$Vs, v32i8)), + (V6_vasrh (V6_vaslh HVI16:$Vs, (A2_tfrsi 8)), (A2_tfrsi 8))>; + def: Pat<(VecI32 (sext_inreg HVI32:$Vs, v16i8)), + (V6_vasrw (V6_vaslw HVI32:$Vs, (A2_tfrsi 24)), (A2_tfrsi 24))>; + def: Pat<(VecI32 (sext_inreg HVI32:$Vs, v16i16)), + (V6_vasrw (V6_vaslw HVI32:$Vs, (A2_tfrsi 16)), (A2_tfrsi 16))>; + } + let Predicates = [UseHVX,UseHVX128B] in { + def: Pat<(VecI16 (sext_inreg HVI16:$Vs, v64i8)), + (V6_vasrh (V6_vaslh HVI16:$Vs, (A2_tfrsi 8)), (A2_tfrsi 8))>; + def: Pat<(VecI32 (sext_inreg HVI32:$Vs, v32i8)), + (V6_vasrw (V6_vaslw HVI32:$Vs, (A2_tfrsi 24)), (A2_tfrsi 24))>; + def: Pat<(VecI32 (sext_inreg HVI32:$Vs, v32i16)), + (V6_vasrw (V6_vaslw HVI32:$Vs, (A2_tfrsi 16)), (A2_tfrsi 16))>; + } + + def: Pat<(HexagonVASL HVI8:$Vs, I32:$Rt), + (V6_vpackeb (V6_vaslh (HiVec (VZxtb HvxVR:$Vs)), I32:$Rt), + (V6_vaslh (LoVec (VZxtb HvxVR:$Vs)), I32:$Rt))>; + def: Pat<(HexagonVASR HVI8:$Vs, I32:$Rt), + (V6_vpackeb (V6_vasrh (HiVec (VSxtb HvxVR:$Vs)), I32:$Rt), + (V6_vasrh (LoVec (VSxtb HvxVR:$Vs)), I32:$Rt))>; + def: Pat<(HexagonVLSR HVI8:$Vs, I32:$Rt), + (V6_vpackeb (V6_vlsrh (HiVec (VZxtb HvxVR:$Vs)), I32:$Rt), + (V6_vlsrh (LoVec (VZxtb HvxVR:$Vs)), I32:$Rt))>; + + def: Pat<(HexagonVASL HVI16:$Vs, I32:$Rt), (V6_vaslh HvxVR:$Vs, I32:$Rt)>; + def: Pat<(HexagonVASL HVI32:$Vs, I32:$Rt), (V6_vaslw HvxVR:$Vs, I32:$Rt)>; + def: Pat<(HexagonVASR HVI16:$Vs, I32:$Rt), (V6_vasrh HvxVR:$Vs, I32:$Rt)>; + def: Pat<(HexagonVASR HVI32:$Vs, I32:$Rt), (V6_vasrw HvxVR:$Vs, I32:$Rt)>; + def: Pat<(HexagonVLSR HVI16:$Vs, I32:$Rt), (V6_vlsrh HvxVR:$Vs, I32:$Rt)>; + def: Pat<(HexagonVLSR HVI32:$Vs, I32:$Rt), (V6_vlsrw HvxVR:$Vs, I32:$Rt)>; + + def: Pat<(shl HVI16:$Vs, HVI16:$Vt), (V6_vaslhv HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(shl HVI32:$Vs, HVI32:$Vt), (V6_vaslwv HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(sra HVI16:$Vs, HVI16:$Vt), (V6_vasrhv HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(sra HVI32:$Vs, HVI32:$Vt), (V6_vasrwv HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(srl HVI16:$Vs, HVI16:$Vt), (V6_vlsrhv HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(srl HVI32:$Vs, HVI32:$Vt), (V6_vlsrwv HvxVR:$Vs, HvxVR:$Vt)>; + + def: Pat<(vnot HQ8:$Qs), (V6_pred_not HvxQR:$Qs)>; + def: Pat<(vnot HQ16:$Qs), (V6_pred_not HvxQR:$Qs)>; + def: Pat<(vnot HQ32:$Qs), (V6_pred_not HvxQR:$Qs)>; + def: Pat<(qnot HQ8:$Qs), (V6_pred_not HvxQR:$Qs)>; + def: Pat<(qnot HQ16:$Qs), (V6_pred_not HvxQR:$Qs)>; + def: Pat<(qnot HQ32:$Qs), (V6_pred_not HvxQR:$Qs)>; + + def: Pat<(VecQ8 (seteq HVI8:$Vs, HVI8:$Vt)), + (V6_veqb HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(VecQ8 (setgt HVI8:$Vs, HVI8:$Vt)), + (V6_vgtb HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(VecQ8 (setugt HVI8:$Vs, HVI8:$Vt)), + (V6_vgtub HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(VecQ16 (seteq HVI16:$Vs, HVI16:$Vt)), + (V6_veqh HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(VecQ16 (setgt HVI16:$Vs, HVI16:$Vt)), + (V6_vgth HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(VecQ16 (setugt HVI16:$Vs, HVI16:$Vt)), + (V6_vgtuh HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(VecQ32 (seteq HVI32:$Vs, HVI32:$Vt)), + (V6_veqw HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(VecQ32 (setgt HVI32:$Vs, HVI32:$Vt)), + (V6_vgtw HvxVR:$Vs, HvxVR:$Vt)>; + def: Pat<(VecQ32 (setugt HVI32:$Vs, HVI32:$Vt)), + (V6_vgtuw HvxVR:$Vs, HvxVR:$Vt)>; + + def: Pat<(VecI8 (trunc HWI16:$Vss)), + (V6_vpackeb (HiVec $Vss), (LoVec $Vss))>; + def: Pat<(VecI16 (trunc HWI32:$Vss)), + (V6_vpackeh (HiVec $Vss), (LoVec $Vss))>; +} + +class HvxSel_pat + : Pat<(select I1:$Pu, RegPred:$Vs, RegPred:$Vt), + (MI I1:$Pu, RegPred:$Vs, RegPred:$Vt)>; + +let Predicates = [HasV60T,UseHVX] in { + def: HvxSel_pat; + def: HvxSel_pat; + def: HvxSel_pat; + def: HvxSel_pat; + def: HvxSel_pat; + def: HvxSel_pat; +} +