forked from OSchip/llvm-project
[SVE] Implement lowering for fixed length vector multiplication.
NOTE: Also uses SVE code generation for NEON size vectors, instead of expanding i64 based vector multiplications. Differential Revision: https://reviews.llvm.org/D85327
This commit is contained in:
Paul Walker
parent
45676a8936
commit
3ed59b775d
|
|
@ -979,6 +979,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
|
|||
setOperationAction(ISD::TRUNCATE, VT, Custom);
|
||||
for (auto VT : {MVT::v8f16, MVT::v4f32})
|
||||
setOperationAction(ISD::FP_ROUND, VT, Expand);
|
||||
|
||||
// These operations are not supported on NEON but SVE can do them.
|
||||
setOperationAction(ISD::MUL, MVT::v1i64, Custom);
|
||||
setOperationAction(ISD::MUL, MVT::v2i64, Custom);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1083,6 +1087,7 @@ void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
|
|||
setOperationAction(ISD::FMUL, VT, Custom);
|
||||
setOperationAction(ISD::FSUB, VT, Custom);
|
||||
setOperationAction(ISD::LOAD, VT, Custom);
|
||||
setOperationAction(ISD::MUL, VT, Custom);
|
||||
setOperationAction(ISD::OR, VT, Custom);
|
||||
setOperationAction(ISD::STORE, VT, Custom);
|
||||
setOperationAction(ISD::TRUNCATE, VT, Custom);
|
||||
|
|
@ -1399,6 +1404,7 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
|
|||
MAKE_CASE(AArch64ISD::THREAD_POINTER)
|
||||
MAKE_CASE(AArch64ISD::TLSDESC_CALLSEQ)
|
||||
MAKE_CASE(AArch64ISD::ADD_PRED)
|
||||
MAKE_CASE(AArch64ISD::MUL_PRED)
|
||||
MAKE_CASE(AArch64ISD::SDIV_PRED)
|
||||
MAKE_CASE(AArch64ISD::SHL_PRED)
|
||||
MAKE_CASE(AArch64ISD::SMAX_PRED)
|
||||
|
|
@ -3090,10 +3096,17 @@ SDValue AArch64TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
|
|||
return DAG.getMergeValues({AND, Chain}, dl);
|
||||
}
|
||||
|
||||
static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
|
||||
SDValue AArch64TargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
|
||||
EVT VT = Op.getValueType();
|
||||
|
||||
// If SVE is available then i64 vector multiplications can also be made legal.
|
||||
bool OverrideNEON = VT == MVT::v2i64 || VT == MVT::v1i64;
|
||||
|
||||
if (useSVEForFixedLengthVectorVT(VT, OverrideNEON))
|
||||
return LowerToPredicatedOp(Op, DAG, AArch64ISD::MUL_PRED, OverrideNEON);
|
||||
|
||||
// Multiplications are only custom-lowered for 128-bit vectors so that
|
||||
// VMULL can be detected. Otherwise v2i64 multiplications are not legal.
|
||||
EVT VT = Op.getValueType();
|
||||
assert(VT.is128BitVector() && VT.isInteger() &&
|
||||
"unexpected type for custom-lowering ISD::MUL");
|
||||
SDNode *N0 = Op.getOperand(0).getNode();
|
||||
|
|
@ -3627,20 +3640,19 @@ bool AArch64TargetLowering::useSVEForFixedLengthVectors() const {
|
|||
return Subtarget->hasSVE() && Subtarget->getMinSVEVectorSizeInBits() >= 256;
|
||||
}
|
||||
|
||||
bool AArch64TargetLowering::useSVEForFixedLengthVectorVT(EVT VT) const {
|
||||
bool AArch64TargetLowering::useSVEForFixedLengthVectorVT(
|
||||
EVT VT, bool OverrideNEON) const {
|
||||
if (!useSVEForFixedLengthVectors())
|
||||
return false;
|
||||
|
||||
if (!VT.isFixedLengthVector())
|
||||
return false;
|
||||
|
||||
// Fixed length predicates should be promoted to i8.
|
||||
// NOTE: This is consistent with how NEON (and thus 64/128bit vectors) work.
|
||||
if (VT.getVectorElementType() == MVT::i1)
|
||||
return false;
|
||||
|
||||
// Don't use SVE for vectors we cannot scalarize if required.
|
||||
switch (VT.getVectorElementType().getSimpleVT().SimpleTy) {
|
||||
// Fixed length predicates should be promoted to i8.
|
||||
// NOTE: This is consistent with how NEON (and thus 64/128bit vectors) work.
|
||||
case MVT::i1:
|
||||
default:
|
||||
return false;
|
||||
case MVT::i8:
|
||||
|
|
@ -3653,6 +3665,10 @@ bool AArch64TargetLowering::useSVEForFixedLengthVectorVT(EVT VT) const {
|
|||
break;
|
||||
}
|
||||
|
||||
// All SVE implementations support NEON sized vectors.
|
||||
if (OverrideNEON && (VT.is128BitVector() || VT.is64BitVector()))
|
||||
return true;
|
||||
|
||||
// Ensure NEON MVTs only belong to a single register class.
|
||||
if (VT.getSizeInBits() <= 128)
|
||||
return false;
|
||||
|
|
@ -8801,7 +8817,7 @@ SDValue AArch64TargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
|
|||
unsigned Idx = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
|
||||
|
||||
// We don't have any patterns for scalable vector yet.
|
||||
if (InVT.isScalableVector() || !useSVEForFixedLengthVectorVT(InVT))
|
||||
if (InVT.isScalableVector())
|
||||
return SDValue();
|
||||
|
||||
// This will be matched by custom code during ISelDAGToDAG.
|
||||
|
|
@ -15252,12 +15268,13 @@ SDValue AArch64TargetLowering::LowerFixedLengthVectorTruncateToSVE(
|
|||
// the original operation's type is used to construct a suitable predicate.
|
||||
SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
|
||||
SelectionDAG &DAG,
|
||||
unsigned NewOp) const {
|
||||
unsigned NewOp,
|
||||
bool OverrideNEON) const {
|
||||
EVT VT = Op.getValueType();
|
||||
SDLoc DL(Op);
|
||||
auto Pg = getPredicateForVector(DAG, DL, VT);
|
||||
|
||||
if (useSVEForFixedLengthVectorVT(VT)) {
|
||||
if (useSVEForFixedLengthVectorVT(VT, OverrideNEON)) {
|
||||
EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
|
||||
|
||||
// Create list of operands by converting existing ones to scalable types.
|
||||
|
|
@ -15268,7 +15285,7 @@ SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
|
|||
continue;
|
||||
}
|
||||
|
||||
assert(useSVEForFixedLengthVectorVT(V.getValueType()) &&
|
||||
assert(useSVEForFixedLengthVectorVT(V.getValueType(), OverrideNEON) &&
|
||||
"Only fixed length vectors are supported!");
|
||||
Operands.push_back(convertToScalableVector(DAG, ContainerVT, V));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ enum NodeType : unsigned {
|
|||
FMA_PRED,
|
||||
FMUL_PRED,
|
||||
FSUB_PRED,
|
||||
MUL_PRED,
|
||||
SDIV_PRED,
|
||||
SHL_PRED,
|
||||
SMAX_PRED,
|
||||
|
|
@ -859,12 +860,13 @@ private:
|
|||
SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
|
||||
SDValue LowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG) const;
|
||||
SDValue LowerDUPQLane(SDValue Op, SelectionDAG &DAG) const;
|
||||
SDValue LowerToPredicatedOp(SDValue Op, SelectionDAG &DAG,
|
||||
unsigned NewOp) const;
|
||||
SDValue LowerToPredicatedOp(SDValue Op, SelectionDAG &DAG, unsigned NewOp,
|
||||
bool OverrideNEON = false) const;
|
||||
SDValue LowerToScalableOp(SDValue Op, SelectionDAG &DAG) const;
|
||||
SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
|
||||
SDValue LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
|
||||
SDValue LowerDIV(SDValue Op, SelectionDAG &DAG) const;
|
||||
SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const;
|
||||
SDValue LowerVectorSRA_SRL_SHL(SDValue Op, SelectionDAG &DAG) const;
|
||||
SDValue LowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
|
||||
SDValue LowerShiftRightParts(SDValue Op, SelectionDAG &DAG) const;
|
||||
|
|
@ -964,7 +966,10 @@ private:
|
|||
const TargetTransformInfo *TTI) const override;
|
||||
|
||||
bool useSVEForFixedLengthVectors() const;
|
||||
bool useSVEForFixedLengthVectorVT(EVT VT) const;
|
||||
// Normally SVE is only used for byte size vectors that do not fit within a
|
||||
// NEON vector. This changes when OverrideNEON is true, allowing SVE to be
|
||||
// used for 64bit and 128bit vectors as well.
|
||||
bool useSVEForFixedLengthVectorVT(EVT VT, bool OverrideNEON = false) const;
|
||||
};
|
||||
|
||||
namespace AArch64 {
|
||||
|
|
|
|||
|
|
@ -182,6 +182,7 @@ def AArch64fmul_p : SDNode<"AArch64ISD::FMUL_PRED", SDT_AArch64Arith>;
|
|||
def AArch64fsub_p : SDNode<"AArch64ISD::FSUB_PRED", SDT_AArch64Arith>;
|
||||
def AArch64lsl_p : SDNode<"AArch64ISD::SHL_PRED", SDT_AArch64Arith>;
|
||||
def AArch64lsr_p : SDNode<"AArch64ISD::SRL_PRED", SDT_AArch64Arith>;
|
||||
def AArch64mul_p : SDNode<"AArch64ISD::MUL_PRED", SDT_AArch64Arith>;
|
||||
def AArch64sdiv_p : SDNode<"AArch64ISD::SDIV_PRED", SDT_AArch64Arith>;
|
||||
def AArch64smax_p : SDNode<"AArch64ISD::SMAX_PRED", SDT_AArch64Arith>;
|
||||
def AArch64smin_p : SDNode<"AArch64ISD::SMIN_PRED", SDT_AArch64Arith>;
|
||||
|
|
@ -291,9 +292,11 @@ let Predicates = [HasSVE] in {
|
|||
defm UMIN_ZI : sve_int_arith_imm1_unsigned<0b11, "umin", AArch64umin_p>;
|
||||
|
||||
defm MUL_ZI : sve_int_arith_imm2<"mul", mul>;
|
||||
defm MUL_ZPmZ : sve_int_bin_pred_arit_2<0b000, "mul", int_aarch64_sve_mul>;
|
||||
defm SMULH_ZPmZ : sve_int_bin_pred_arit_2<0b010, "smulh", int_aarch64_sve_smulh>;
|
||||
defm UMULH_ZPmZ : sve_int_bin_pred_arit_2<0b011, "umulh", int_aarch64_sve_umulh>;
|
||||
defm MUL_ZPmZ : sve_int_bin_pred_arit_2<0b000, "mul", "MUL_ZPZZ", int_aarch64_sve_mul, DestructiveBinaryComm>;
|
||||
defm SMULH_ZPmZ : sve_int_bin_pred_arit_2<0b010, "smulh", "SMULH_ZPZZ", int_aarch64_sve_smulh, DestructiveBinaryComm>;
|
||||
defm UMULH_ZPmZ : sve_int_bin_pred_arit_2<0b011, "umulh", "UMULH_ZPZZ", int_aarch64_sve_umulh, DestructiveBinaryComm>;
|
||||
|
||||
defm MUL_ZPZZ : sve_int_bin_pred_bhsd<AArch64mul_p>;
|
||||
|
||||
// Add unpredicated alternative for the mul instruction.
|
||||
def : Pat<(mul nxv16i8:$Op1, nxv16i8:$Op2),
|
||||
|
|
|
|||
|
|
@ -2402,11 +2402,19 @@ multiclass sve_int_bin_pred_arit_1<bits<3> opc, string asm, string Ps,
|
|||
def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
|
||||
}
|
||||
|
||||
multiclass sve_int_bin_pred_arit_2<bits<3> opc, string asm, SDPatternOperator op> {
|
||||
def _B : sve_int_bin_pred_arit_log<0b00, 0b10, opc, asm, ZPR8>;
|
||||
def _H : sve_int_bin_pred_arit_log<0b01, 0b10, opc, asm, ZPR16>;
|
||||
def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>;
|
||||
def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>;
|
||||
multiclass sve_int_bin_pred_arit_2<bits<3> opc, string asm, string Ps,
|
||||
SDPatternOperator op,
|
||||
DestructiveInstTypeEnum flags> {
|
||||
let DestructiveInstType = flags in {
|
||||
def _B : sve_int_bin_pred_arit_log<0b00, 0b10, opc, asm, ZPR8>,
|
||||
SVEPseudo2Instr<Ps # _B, 1>;
|
||||
def _H : sve_int_bin_pred_arit_log<0b01, 0b10, opc, asm, ZPR16>,
|
||||
SVEPseudo2Instr<Ps # _H, 1>;
|
||||
def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>,
|
||||
SVEPseudo2Instr<Ps # _S, 1>;
|
||||
def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>,
|
||||
SVEPseudo2Instr<Ps # _D, 1>;
|
||||
}
|
||||
|
||||
def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
|
||||
def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
|
||||
|
|
|
|||
|
|
@ -410,4 +410,323 @@ define void @add_v32i64(<32 x i64>* %a, <32 x i64>* %b) #0 {
|
|||
ret void
|
||||
}
|
||||
|
||||
;
|
||||
; NOTE: Tests beyond this point only have CHECK lines to validate the first
|
||||
; VBYTES because the add tests already validate the legalisation code paths.
|
||||
;
|
||||
|
||||
; Don't use SVE for 64-bit vectors.
|
||||
define <8 x i8> @mul_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 {
|
||||
; CHECK-LABEL: @mul_v8i8
|
||||
; CHECK: mul v0.8b, v0.8b, v1.8b
|
||||
; CHECK: ret
|
||||
%res = mul <8 x i8> %op1, %op2
|
||||
ret <8 x i8> %res
|
||||
}
|
||||
|
||||
; Don't use SVE for 128-bit vectors.
|
||||
define <16 x i8> @mul_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 {
|
||||
; CHECK-LABEL: @mul_v16i8
|
||||
; CHECK: mul v0.16b, v0.16b, v1.16b
|
||||
; CHECK: ret
|
||||
%res = mul <16 x i8> %op1, %op2
|
||||
ret <16 x i8> %res
|
||||
}
|
||||
|
||||
define void @mul_v32i8(<32 x i8>* %a, <32 x i8>* %b) #0 {
|
||||
; CHECK-LABEL: @mul_v32i8
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].b, vl[[#min(VBYTES,32)]]
|
||||
; CHECK-DAG: ld1b { [[OP1:z[0-9]+]].b }, [[PG]]/z, [x0]
|
||||
; CHECK-DAG: ld1b { [[OP2:z[0-9]+]].b }, [[PG]]/z, [x1]
|
||||
; CHECK: mul [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[OP2]].b
|
||||
; CHECK: st1b { [[RES]].b }, [[PG]], [x0]
|
||||
; CHECK: ret
|
||||
%op1 = load <32 x i8>, <32 x i8>* %a
|
||||
%op2 = load <32 x i8>, <32 x i8>* %b
|
||||
%res = mul <32 x i8> %op1, %op2
|
||||
store <32 x i8> %res, <32 x i8>* %a
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @mul_v64i8(<64 x i8>* %a, <64 x i8>* %b) #0 {
|
||||
; CHECK-LABEL: @mul_v64i8
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].b, vl[[#min(VBYTES,64)]]
|
||||
; CHECK-DAG: ld1b { [[OP1:z[0-9]+]].b }, [[PG]]/z, [x0]
|
||||
; CHECK-DAG: ld1b { [[OP2:z[0-9]+]].b }, [[PG]]/z, [x1]
|
||||
; CHECK: mul [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[OP2]].b
|
||||
; CHECK: st1b { [[RES]].b }, [[PG]], [x0]
|
||||
; CHECK: ret
|
||||
%op1 = load <64 x i8>, <64 x i8>* %a
|
||||
%op2 = load <64 x i8>, <64 x i8>* %b
|
||||
%res = mul <64 x i8> %op1, %op2
|
||||
store <64 x i8> %res, <64 x i8>* %a
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @mul_v128i8(<128 x i8>* %a, <128 x i8>* %b) #0 {
|
||||
; CHECK-LABEL: @mul_v128i8
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].b, vl[[#min(VBYTES,128)]]
|
||||
; CHECK-DAG: ld1b { [[OP1:z[0-9]+]].b }, [[PG]]/z, [x0]
|
||||
; CHECK-DAG: ld1b { [[OP2:z[0-9]+]].b }, [[PG]]/z, [x1]
|
||||
; CHECK: mul [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[OP2]].b
|
||||
; CHECK: st1b { [[RES]].b }, [[PG]], [x0]
|
||||
; CHECK: ret
|
||||
%op1 = load <128 x i8>, <128 x i8>* %a
|
||||
%op2 = load <128 x i8>, <128 x i8>* %b
|
||||
%res = mul <128 x i8> %op1, %op2
|
||||
store <128 x i8> %res, <128 x i8>* %a
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @mul_v256i8(<256 x i8>* %a, <256 x i8>* %b) #0 {
|
||||
; CHECK-LABEL: @mul_v256i8
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].b, vl[[#min(VBYTES,256)]]
|
||||
; CHECK-DAG: ld1b { [[OP1:z[0-9]+]].b }, [[PG]]/z, [x0]
|
||||
; CHECK-DAG: ld1b { [[OP2:z[0-9]+]].b }, [[PG]]/z, [x1]
|
||||
; CHECK: mul [[RES:z[0-9]+]].b, [[PG]]/m, [[OP1]].b, [[OP2]].b
|
||||
; CHECK: st1b { [[RES]].b }, [[PG]], [x0]
|
||||
; CHECK: ret
|
||||
%op1 = load <256 x i8>, <256 x i8>* %a
|
||||
%op2 = load <256 x i8>, <256 x i8>* %b
|
||||
%res = mul <256 x i8> %op1, %op2
|
||||
store <256 x i8> %res, <256 x i8>* %a
|
||||
ret void
|
||||
}
|
||||
|
||||
; Don't use SVE for 64-bit vectors.
|
||||
define <4 x i16> @mul_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 {
|
||||
; CHECK-LABEL: @mul_v4i16
|
||||
; CHECK: mul v0.4h, v0.4h, v1.4h
|
||||
; CHECK: ret
|
||||
%res = mul <4 x i16> %op1, %op2
|
||||
ret <4 x i16> %res
|
||||
}
|
||||
|
||||
; Don't use SVE for 128-bit vectors.
|
||||
define <8 x i16> @mul_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 {
|
||||
; CHECK-LABEL: @mul_v8i16
|
||||
; CHECK: mul v0.8h, v0.8h, v1.8h
|
||||
; CHECK: ret
|
||||
%res = mul <8 x i16> %op1, %op2
|
||||
ret <8 x i16> %res
|
||||
}
|
||||
|
||||
define void @mul_v16i16(<16 x i16>* %a, <16 x i16>* %b) #0 {
|
||||
; CHECK-LABEL: @mul_v16i16
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),16)]]
|
||||
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
|
||||
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
|
||||
; CHECK: mul [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
|
||||
; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
|
||||
; CHECK: ret
|
||||
%op1 = load <16 x i16>, <16 x i16>* %a
|
||||
%op2 = load <16 x i16>, <16 x i16>* %b
|
||||
%res = mul <16 x i16> %op1, %op2
|
||||
store <16 x i16> %res, <16 x i16>* %a
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @mul_v32i16(<32 x i16>* %a, <32 x i16>* %b) #0 {
|
||||
; CHECK-LABEL: @mul_v32i16
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),32)]]
|
||||
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
|
||||
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
|
||||
; CHECK: mul [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
|
||||
; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
|
||||
; CHECK: ret
|
||||
%op1 = load <32 x i16>, <32 x i16>* %a
|
||||
%op2 = load <32 x i16>, <32 x i16>* %b
|
||||
%res = mul <32 x i16> %op1, %op2
|
||||
store <32 x i16> %res, <32 x i16>* %a
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @mul_v64i16(<64 x i16>* %a, <64 x i16>* %b) #0 {
|
||||
; CHECK-LABEL: @mul_v64i16
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),64)]]
|
||||
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
|
||||
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
|
||||
; CHECK: mul [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
|
||||
; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
|
||||
; CHECK: ret
|
||||
%op1 = load <64 x i16>, <64 x i16>* %a
|
||||
%op2 = load <64 x i16>, <64 x i16>* %b
|
||||
%res = mul <64 x i16> %op1, %op2
|
||||
store <64 x i16> %res, <64 x i16>* %a
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @mul_v128i16(<128 x i16>* %a, <128 x i16>* %b) #0 {
|
||||
; CHECK-LABEL: @mul_v128i16
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),128)]]
|
||||
; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
|
||||
; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
|
||||
; CHECK: mul [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
|
||||
; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
|
||||
; CHECK: ret
|
||||
%op1 = load <128 x i16>, <128 x i16>* %a
|
||||
%op2 = load <128 x i16>, <128 x i16>* %b
|
||||
%res = mul <128 x i16> %op1, %op2
|
||||
store <128 x i16> %res, <128 x i16>* %a
|
||||
ret void
|
||||
}
|
||||
|
||||
; Don't use SVE for 64-bit vectors.
|
||||
define <2 x i32> @mul_v2i32(<2 x i32> %op1, <2 x i32> %op2) #0 {
|
||||
; CHECK-LABEL: @mul_v2i32
|
||||
; CHECK: mul v0.2s, v0.2s, v1.2s
|
||||
; CHECK: ret
|
||||
%res = mul <2 x i32> %op1, %op2
|
||||
ret <2 x i32> %res
|
||||
}
|
||||
|
||||
; Don't use SVE for 128-bit vectors.
|
||||
define <4 x i32> @mul_v4i32(<4 x i32> %op1, <4 x i32> %op2) #0 {
|
||||
; CHECK-LABEL: @mul_v4i32
|
||||
; CHECK: mul v0.4s, v0.4s, v1.4s
|
||||
; CHECK: ret
|
||||
%res = mul <4 x i32> %op1, %op2
|
||||
ret <4 x i32> %res
|
||||
}
|
||||
|
||||
define void @mul_v8i32(<8 x i32>* %a, <8 x i32>* %b) #0 {
|
||||
; CHECK-LABEL: @mul_v8i32
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),8)]]
|
||||
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
|
||||
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
|
||||
; CHECK: mul [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
|
||||
; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
|
||||
; CHECK: ret
|
||||
%op1 = load <8 x i32>, <8 x i32>* %a
|
||||
%op2 = load <8 x i32>, <8 x i32>* %b
|
||||
%res = mul <8 x i32> %op1, %op2
|
||||
store <8 x i32> %res, <8 x i32>* %a
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @mul_v16i32(<16 x i32>* %a, <16 x i32>* %b) #0 {
|
||||
; CHECK-LABEL: @mul_v16i32
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]]
|
||||
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
|
||||
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
|
||||
; CHECK: mul [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
|
||||
; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
|
||||
; CHECK: ret
|
||||
%op1 = load <16 x i32>, <16 x i32>* %a
|
||||
%op2 = load <16 x i32>, <16 x i32>* %b
|
||||
%res = mul <16 x i32> %op1, %op2
|
||||
store <16 x i32> %res, <16 x i32>* %a
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @mul_v32i32(<32 x i32>* %a, <32 x i32>* %b) #0 {
|
||||
; CHECK-LABEL: @mul_v32i32
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]]
|
||||
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
|
||||
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
|
||||
; CHECK: mul [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
|
||||
; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
|
||||
; CHECK: ret
|
||||
%op1 = load <32 x i32>, <32 x i32>* %a
|
||||
%op2 = load <32 x i32>, <32 x i32>* %b
|
||||
%res = mul <32 x i32> %op1, %op2
|
||||
store <32 x i32> %res, <32 x i32>* %a
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @mul_v64i32(<64 x i32>* %a, <64 x i32>* %b) #0 {
|
||||
; CHECK-LABEL: @mul_v64i32
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]]
|
||||
; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
|
||||
; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
|
||||
; CHECK: mul [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
|
||||
; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
|
||||
; CHECK: ret
|
||||
%op1 = load <64 x i32>, <64 x i32>* %a
|
||||
%op2 = load <64 x i32>, <64 x i32>* %b
|
||||
%res = mul <64 x i32> %op1, %op2
|
||||
store <64 x i32> %res, <64 x i32>* %a
|
||||
ret void
|
||||
}
|
||||
|
||||
; Vector i64 multiplications are not legal for NEON so use SVE when available.
|
||||
define <1 x i64> @mul_v1i64(<1 x i64> %op1, <1 x i64> %op2) #0 {
|
||||
; CHECK-LABEL: @mul_v1i64
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].d, vl1
|
||||
; CHECK: mul z0.d, [[PG]]/m, z0.d, z1.d
|
||||
; CHECK: ret
|
||||
%res = mul <1 x i64> %op1, %op2
|
||||
ret <1 x i64> %res
|
||||
}
|
||||
|
||||
; Vector i64 multiplications are not legal for NEON so use SVE when available.
|
||||
define <2 x i64> @mul_v2i64(<2 x i64> %op1, <2 x i64> %op2) #0 {
|
||||
; CHECK-LABEL: @mul_v2i64
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].d, vl2
|
||||
; CHECK: mul z0.d, [[PG]]/m, z0.d, z1.d
|
||||
; CHECK: ret
|
||||
%res = mul <2 x i64> %op1, %op2
|
||||
ret <2 x i64> %res
|
||||
}
|
||||
|
||||
define void @mul_v4i64(<4 x i64>* %a, <4 x i64>* %b) #0 {
|
||||
; CHECK-LABEL: @mul_v4i64
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),4)]]
|
||||
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
|
||||
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
|
||||
; CHECK: mul [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
|
||||
; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
|
||||
; CHECK: ret
|
||||
%op1 = load <4 x i64>, <4 x i64>* %a
|
||||
%op2 = load <4 x i64>, <4 x i64>* %b
|
||||
%res = mul <4 x i64> %op1, %op2
|
||||
store <4 x i64> %res, <4 x i64>* %a
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @mul_v8i64(<8 x i64>* %a, <8 x i64>* %b) #0 {
|
||||
; CHECK-LABEL: @mul_v8i64
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),8)]]
|
||||
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
|
||||
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
|
||||
; CHECK: mul [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
|
||||
; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
|
||||
; CHECK: ret
|
||||
%op1 = load <8 x i64>, <8 x i64>* %a
|
||||
%op2 = load <8 x i64>, <8 x i64>* %b
|
||||
%res = mul <8 x i64> %op1, %op2
|
||||
store <8 x i64> %res, <8 x i64>* %a
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @mul_v16i64(<16 x i64>* %a, <16 x i64>* %b) #0 {
|
||||
; CHECK-LABEL: @mul_v16i64
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),16)]]
|
||||
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
|
||||
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
|
||||
; CHECK: mul [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
|
||||
; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
|
||||
; CHECK: ret
|
||||
%op1 = load <16 x i64>, <16 x i64>* %a
|
||||
%op2 = load <16 x i64>, <16 x i64>* %b
|
||||
%res = mul <16 x i64> %op1, %op2
|
||||
store <16 x i64> %res, <16 x i64>* %a
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @mul_v32i64(<32 x i64>* %a, <32 x i64>* %b) #0 {
|
||||
; CHECK-LABEL: @mul_v32i64
|
||||
; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),32)]]
|
||||
; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
|
||||
; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
|
||||
; CHECK: mul [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
|
||||
; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
|
||||
; CHECK: ret
|
||||
%op1 = load <32 x i64>, <32 x i64>* %a
|
||||
%op2 = load <32 x i64>, <32 x i64>* %b
|
||||
%res = mul <32 x i64> %op1, %op2
|
||||
store <32 x i64> %res, <32 x i64>* %a
|
||||
ret void
|
||||
}
|
||||
|
||||
attributes #0 = { "target-features"="+sve" }
|
||||
|
|
|
|||
Loading…
Reference in New Issue