maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

AMDGPU: Start selecting v_mad_mix_f32 

llvm-svn: 312732
This commit is contained in:
Matt Arsenault 2017-09-07 18:05:07 +00:00
parent 61ec738b60
commit d7e2303df2
5 changed files with 514 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
llvm/lib/Target/AMDGPU/AMDGPU.td
View File

@ -721,6 +721,9 @@ def HasDPP : Predicate<"Subtarget->hasDPP()">,
def HasIntClamp : Predicate<"Subtarget->hasIntClamp()">,
AssemblerPredicate<"FeatureIntClamp">;
def HasMadMix : Predicate<"Subtarget->hasMadMixInsts()">,
AssemblerPredicate<"FeatureGFX9Insts">;
class PredicateControl {
Predicate SubtargetPredicate;
Predicate SIAssemblerPredicate = isSICI;

101
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
View File

@ -170,6 +170,7 @@ private:
bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const;
bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods) const;
bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
@ -195,6 +196,8 @@ private:
bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
bool SelectVOP3OpSelMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
SDValue &Clamp) const;
bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods) const;
bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
void SelectADD_SUB_I64(SDNode *N);
void SelectUADDO_USUBO(SDNode *N);
@ -208,6 +211,7 @@ private:
void SelectS_BFE(SDNode *N);
bool isCBranchSCC(const SDNode *N) const;
void SelectBRCOND(SDNode *N);
void SelectFMAD(SDNode *N);
void SelectATOMIC_CMP_SWAP(SDNode *N);
protected:
@ -606,7 +610,9 @@ void AMDGPUDAGToDAGISel::Select(SDNode *N) {
case ISD::BRCOND:
SelectBRCOND(N);
return;
case ISD::FMAD:
SelectFMAD(N);
return;
case AMDGPUISD::ATOMIC_CMP_SWAP:
SelectATOMIC_CMP_SWAP(N);
return;
@ -1644,6 +1650,46 @@ void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) {
VCC.getValue(0));
}
void AMDGPUDAGToDAGISel::SelectFMAD(SDNode *N) {
MVT VT = N->getSimpleValueType(0);
if (VT != MVT::f32 || !Subtarget->hasMadMixInsts()) {
SelectCode(N);
return;
}
SDValue Src0 = N->getOperand(0);
SDValue Src1 = N->getOperand(1);
SDValue Src2 = N->getOperand(2);
unsigned Src0Mods, Src1Mods, Src2Mods;
// Avoid using v_mad_mix_f32 unless there is actually an operand using the
// conversion from f16.
bool Sel0 = SelectVOP3PMadMixModsImpl(Src0, Src0, Src0Mods);
bool Sel1 = SelectVOP3PMadMixModsImpl(Src1, Src1, Src1Mods);
bool Sel2 = SelectVOP3PMadMixModsImpl(Src2, Src2, Src2Mods);
assert(!Subtarget->hasFP32Denormals() &&
"fmad selected with denormals enabled");
// TODO: We can select this with f32 denormals enabled if all the sources are
// converted from f16 (in which case fmad isn't legal).
if (Sel0 || Sel1 || Sel2) {
// For dummy operands.
SDValue Zero = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
SDValue Ops[] = {
CurDAG->getTargetConstant(Src0Mods, SDLoc(), MVT::i32), Src0,
CurDAG->getTargetConstant(Src1Mods, SDLoc(), MVT::i32), Src1,
CurDAG->getTargetConstant(Src2Mods, SDLoc(), MVT::i32), Src2,
CurDAG->getTargetConstant(0, SDLoc(), MVT::i1),
Zero, Zero
};
CurDAG->SelectNodeTo(N, AMDGPU::V_MAD_MIX_F32, MVT::f32, Ops);
} else {
SelectCode(N);
}
}
// This is here because there isn't a way to use the generated sub0_sub1 as the
// subreg index to EXTRACT_SUBREG in tablegen.
void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) {
@ -1710,9 +1756,9 @@ void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) {
CurDAG->RemoveDeadNode(N);
}
bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
SDValue &SrcMods) const {
unsigned Mods = 0;
bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src,
unsigned &Mods) const {
Mods = 0;
Src = In;
if (Src.getOpcode() == ISD::FNEG) {
@ -1725,10 +1771,20 @@ bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
Src = Src.getOperand(0);
}
SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
return true;
}
bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
SDValue &SrcMods) const {
unsigned Mods;
if (SelectVOP3ModsImpl(In, Src, Mods)) {
SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
return true;
}
return false;
}
bool AMDGPUDAGToDAGISel::SelectVOP3Mods_NNaN(SDValue In, SDValue &Src,
SDValue &SrcMods) const {
SelectVOP3Mods(In, Src, SrcMods);
@ -1908,6 +1964,41 @@ bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods0(SDValue In, SDValue &Src,
return SelectVOP3OpSelMods(In, Src, SrcMods);
}
// The return value is not whether the match is possible (which it always is),
// but whether or not it a conversion is really used.
bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
unsigned &Mods) const {
Mods = 0;
SelectVOP3ModsImpl(In, Src, Mods);
if (Src.getOpcode() == ISD::FP_EXTEND) {
Src = Src.getOperand(0);
assert(Src.getValueType() == MVT::f16);
Src = stripBitcast(Src);
// op_sel/op_sel_hi decide the source type and source.
// If the source's op_sel_hi is set, it indicates to do a conversion from fp16.
// If the sources's op_sel is set, it picks the high half of the source
// register.
Mods |= SISrcMods::OP_SEL_1;
if (isExtractHiElt(Src, Src))
Mods |= SISrcMods::OP_SEL_0;
return true;
}
return false;
}
bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src,
SDValue &SrcMods) const {
unsigned Mods = 0;
SelectVOP3PMadMixModsImpl(In, Src, Mods);
SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
return true;
}
void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
const AMDGPUTargetLowering& Lowering =
*static_cast<const AMDGPUTargetLowering*>(getTargetLowering());

4
llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
View File

@ -313,6 +313,10 @@ public:
return getGeneration() >= GFX9;
}
bool hasMadMixInsts() const {
return getGeneration() >= GFX9;
}
bool hasCARRY() const {
return (getGeneration() >= EVERGREEN);
}

2
llvm/lib/Target/AMDGPU/SIInstrInfo.td
View File

@ -770,6 +770,8 @@ def VOP3OpSel0 : ComplexPattern<untyped, 3, "SelectVOP3OpSel0">;
def VOP3OpSelMods : ComplexPattern<untyped, 2, "SelectVOP3OpSelMods">;
def VOP3OpSelMods0 : ComplexPattern<untyped, 3, "SelectVOP3OpSelMods0">;
def VOP3PMadMixMods : ComplexPattern<untyped, 2, "SelectVOP3PMadMixMods">;
//===----------------------------------------------------------------------===//
// SI assembler operands
//===----------------------------------------------------------------------===//

409
llvm/test/CodeGen/AMDGPU/mad-mix.ll Normal file
View File

@ -0,0 +1,409 @@
; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,VI %s
; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,CI %s
; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f16lo:
; GFX9: v_mad_mix_f32 v0, v0, v1, v2{{$}}
; VI: v_mac_f32
; CI: v_mad_f32
define float @v_mad_mix_f32_f16lo_f16lo_f16lo(half %src0, half %src1, half %src2) #0 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%src2.ext = fpext half %src2 to float
%result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
ret float %result
}
; GCN-LABEL: {{^}}v_mad_mix_f32_f16hi_f16hi_f16hi_int:
; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel:[1,1,1]{{$}}
; CIVI: v_mac_f32
define float @v_mad_mix_f32_f16hi_f16hi_f16hi_int(i32 %src0, i32 %src1, i32 %src2) #0 {
%src0.hi = lshr i32 %src0, 16
%src1.hi = lshr i32 %src1, 16
%src2.hi = lshr i32 %src2, 16
%src0.i16 = trunc i32 %src0.hi to i16
%src1.i16 = trunc i32 %src1.hi to i16
%src2.i16 = trunc i32 %src2.hi to i16
%src0.fp16 = bitcast i16 %src0.i16 to half
%src1.fp16 = bitcast i16 %src1.i16 to half
%src2.fp16 = bitcast i16 %src2.i16 to half
%src0.ext = fpext half %src0.fp16 to float
%src1.ext = fpext half %src1.fp16 to float
%src2.ext = fpext half %src2.fp16 to float
%result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
ret float %result
}
; GCN-LABEL: {{^}}v_mad_mix_f32_f16hi_f16hi_f16hi_elt:
; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel:[1,1,1]{{$}}
; VI: v_mac_f32
; CI: v_mad_f32
define float @v_mad_mix_f32_f16hi_f16hi_f16hi_elt(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
%src0.hi = extractelement <2 x half> %src0, i32 1
%src1.hi = extractelement <2 x half> %src1, i32 1
%src2.hi = extractelement <2 x half> %src2, i32 1
%src0.ext = fpext half %src0.hi to float
%src1.ext = fpext half %src1.hi to float
%src2.ext = fpext half %src2.hi to float
%result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
ret float %result
}
; GCN-LABEL: {{^}}v_mad_mix_v2f32:
; GFX9: v_mov_b32_e32 v3, v1
; GFX9-NEXT: v_mad_mix_f32 v1, v0, v3, v2 op_sel:[1,1,1]
; GFX9-NEXT: v_mad_mix_f32 v0, v0, v3, v2
; CIVI: v_mac_f32
define <2 x float> @v_mad_mix_v2f32(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
%src0.ext = fpext <2 x half> %src0 to <2 x float>
%src1.ext = fpext <2 x half> %src1 to <2 x float>
%src2.ext = fpext <2 x half> %src2 to <2 x float>
%result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext)
ret <2 x float> %result
}
; GCN-LABEL: {{^}}v_mad_mix_v2f32_shuffle:
; GCN: s_waitcnt
; GFX9-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NEXT: v_mad_mix_f32 v1, v0, v3, v2 op_sel:[0,1,1]
; GFX9-NEXT: v_mad_mix_f32 v0, v0, v3, v2 op_sel:[1,0,1]
; GFX9-NEXT: s_setpc_b64
; CIVI: v_mac_f32
define <2 x float> @v_mad_mix_v2f32_shuffle(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
%src0.shuf = shufflevector <2 x half> %src0, <2 x half> undef, <2 x i32> <i32 1, i32 0>
%src1.shuf = shufflevector <2 x half> %src1, <2 x half> undef, <2 x i32> <i32 0, i32 1>
%src2.shuf = shufflevector <2 x half> %src2, <2 x half> undef, <2 x i32> <i32 1, i32 1>
%src0.ext = fpext <2 x half> %src0.shuf to <2 x float>
%src1.ext = fpext <2 x half> %src1.shuf to <2 x float>
%src2.ext = fpext <2 x half> %src2.shuf to <2 x float>
%result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext)
ret <2 x float> %result
}
; GCN-LABEL: {{^}}v_mad_mix_f32_negf16lo_f16lo_f16lo:
; GFX9: s_waitcnt
; GFX9-NEXT: v_mad_mix_f32 v0, -v0, v1, v2{{$}}
; GFX9-NEXT: s_setpc_b64
; CIVI: v_mad_f32
define float @v_mad_mix_f32_negf16lo_f16lo_f16lo(half %src0, half %src1, half %src2) #0 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%src2.ext = fpext half %src2 to float
%src0.ext.neg = fsub float -0.0, %src0.ext
%result = tail call float @llvm.fmuladd.f32(float %src0.ext.neg, float %src1.ext, float %src2.ext)
ret float %result
}
; GCN-LABEL: {{^}}v_mad_mix_f32_absf16lo_f16lo_f16lo:
; GFX9: v_mad_mix_f32 v0, |v0|, v1, v2
; CIVI: v_mad_f32
define float @v_mad_mix_f32_absf16lo_f16lo_f16lo(half %src0, half %src1, half %src2) #0 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%src2.ext = fpext half %src2 to float
%src0.ext.abs = call float @llvm.fabs.f32(float %src0.ext)
%result = tail call float @llvm.fmuladd.f32(float %src0.ext.abs, float %src1.ext, float %src2.ext)
ret float %result
}
; GCN-LABEL: {{^}}v_mad_mix_f32_negabsf16lo_f16lo_f16lo:
; GFX9: s_waitcnt
; GFX9-NEXT: v_mad_mix_f32 v0, -|v0|, v1, v2
; GFX9-NEXT: s_setpc_b64
; CIVI: v_mad_f32
define float @v_mad_mix_f32_negabsf16lo_f16lo_f16lo(half %src0, half %src1, half %src2) #0 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%src2.ext = fpext half %src2 to float
%src0.ext.abs = call float @llvm.fabs.f32(float %src0.ext)
%src0.ext.neg.abs = fsub float -0.0, %src0.ext.abs
%result = tail call float @llvm.fmuladd.f32(float %src0.ext.neg.abs, float %src1.ext, float %src2.ext)
ret float %result
}
; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32:
; GCN: s_waitcnt
; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]{{$}}
; GFX9-NEXT: s_setpc_b64
; CIVI: v_mad_f32
define float @v_mad_mix_f32_f16lo_f16lo_f32(half %src0, half %src1, float %src2) #0 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
ret float %result
}
; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_negf32:
; GCN: s_waitcnt
; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, -v2 op_sel_hi:[1,1,0]{{$}}
; GFX9-NEXT: s_setpc_b64
; CIVI: v_mad_f32
define float @v_mad_mix_f32_f16lo_f16lo_negf32(half %src0, half %src1, float %src2) #0 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%src2.neg = fsub float -0.0, %src2
%result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.neg)
ret float %result
}
; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_absf32:
; GCN: s_waitcnt
; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, |v2| op_sel_hi:[1,1,0]{{$}}
; GFX9-NEXT: s_setpc_b64
; CIVI: v_mad_f32
define float @v_mad_mix_f32_f16lo_f16lo_absf32(half %src0, half %src1, float %src2) #0 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%src2.abs = call float @llvm.fabs.f32(float %src2)
%result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.abs)
ret float %result
}
; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_negabsf32:
; GCN: s_waitcnt
; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, -|v2| op_sel_hi:[1,1,0]{{$}}
; GFX9-NEXT: s_setpc_b64
; CIVI: v_mad_f32
define float @v_mad_mix_f32_f16lo_f16lo_negabsf32(half %src0, half %src1, float %src2) #0 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%src2.abs = call float @llvm.fabs.f32(float %src2)
%src2.neg.abs = fsub float -0.0, %src2.abs
%result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.neg.abs)
ret float %result
}
; TODO: Fold inline immediates. Need to be careful because it is an
; f16 inline immediate that may be converted to f32, not an actual f32
; inline immediate.
; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32imm1:
; GCN: s_waitcnt
; GFX9: v_mov_b32_e32 v2, 1.0
; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]{{$}}
; CIVI: v_mad_f32 v0, v0, v1, 1.0
; GCN-NEXT: s_setpc_b64
define float @v_mad_mix_f32_f16lo_f16lo_f32imm1(half %src0, half %src1) #0 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float 1.0)
ret float %result
}
; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32imminv2pi:
; GCN: s_waitcnt
; GFX9: v_mov_b32_e32 v2, 0.15915494
; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]{{$}}
; VI: v_mad_f32 v0, v0, v1, 0.15915494
define float @v_mad_mix_f32_f16lo_f16lo_f32imminv2pi(half %src0, half %src1) #0 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float 0x3FC45F3060000000)
ret float %result
}
; Attempt to break inline immediate folding. If the operand is
; interpreted as f32, the inline immediate is really the f16 inline
; imm value converted to f32.
; fpext f16 1/2pi = 0x3e230000
; f32 1/2pi = 0x3e22f983
; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
; GFX9: v_mov_b32_e32 v2, 0x3e230000
; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]{{$}}
; CIVI: v_madak_f32 v0, v0, v1, 0x3e230000
define float @v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi(half %src0, half %src1) #0 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%src2 = fpext half 0xH3118 to float
%result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
ret float %result
}
; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
; GFX9: v_mov_b32_e32 v2, 0x367c0000
; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]{{$}}
; CIVI: v_madak_f32 v0, v0, v1, 0x367c0000
define float @v_mad_mix_f32_f16lo_f16lo_cvtf16imm63(half %src0, half %src1) #0 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%src2 = fpext half 0xH003F to float
%result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
ret float %result
}
; GCN-LABEL: {{^}}v_mad_mix_v2f32_f32imm1:
; GFX9: v_mov_b32_e32 v2, v1
; GFX9: v_mov_b32_e32 v3, 1.0
; GFX9: v_mad_mix_f32 v1, v0, v2, v3 op_sel:[1,1,0] op_sel_hi:[1,1,0]{{$}}
; GFX9: v_mad_mix_f32 v0, v0, v2, v3 op_sel_hi:[1,1,0]{{$}}
define <2 x float> @v_mad_mix_v2f32_f32imm1(<2 x half> %src0, <2 x half> %src1) #0 {
%src0.ext = fpext <2 x half> %src0 to <2 x float>
%src1.ext = fpext <2 x half> %src1 to <2 x float>
%result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> <float 1.0, float 1.0>)
ret <2 x float> %result
}
; GCN-LABEL: {{^}}v_mad_mix_v2f32_cvtf16imminv2pi:
; GFX9: v_mov_b32_e32 v2, v1
; GFX9: v_mov_b32_e32 v3, 0x3e230000
; GFX9: v_mad_mix_f32 v1, v0, v2, v3 op_sel:[1,1,0] op_sel_hi:[1,1,0]{{$}}
; GFX9: v_mad_mix_f32 v0, v0, v2, v3 op_sel_hi:[1,1,0]{{$}}
define <2 x float> @v_mad_mix_v2f32_cvtf16imminv2pi(<2 x half> %src0, <2 x half> %src1) #0 {
%src0.ext = fpext <2 x half> %src0 to <2 x float>
%src1.ext = fpext <2 x half> %src1 to <2 x float>
%src2 = fpext <2 x half> <half 0xH3118, half 0xH3118> to <2 x float>
%result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2)
ret <2 x float> %result
}
; GCN-LABEL: {{^}}v_mad_mix_v2f32_f32imminv2pi:
; GFX9: v_mov_b32_e32 v2, v1
; GFX9: v_mov_b32_e32 v3, 0.15915494
; GFX9: v_mad_mix_f32 v1, v0, v2, v3 op_sel:[1,1,0] op_sel_hi:[1,1,0]{{$}}
; GFX9: v_mad_mix_f32 v0, v0, v2, v3 op_sel_hi:[1,1,0]{{$}}
define <2 x float> @v_mad_mix_v2f32_f32imminv2pi(<2 x half> %src0, <2 x half> %src1) #0 {
%src0.ext = fpext <2 x half> %src0 to <2 x float>
%src1.ext = fpext <2 x half> %src1 to <2 x float>
%src2 = fpext <2 x half> <half 0xH3118, half 0xH3118> to <2 x float>
%result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> <float 0x3FC45F3060000000, float 0x3FC45F3060000000>)
ret <2 x float> %result
}
; GCN-LABEL: {{^}}v_mad_mix_clamp_f32_f16hi_f16hi_f16hi_elt:
; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel:[1,1,1] clamp{{$}}
; VI: v_mac_f32_e64 v{{[0-9]}}, v{{[0-9]}}, v{{[0-9]}} clamp{{$}}
; CI: v_mad_f32 v{{[0-9]}}, v{{[0-9]}}, v{{[0-9]}}, v{{[0-9]}} clamp{{$}}
define float @v_mad_mix_clamp_f32_f16hi_f16hi_f16hi_elt(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
%src0.hi = extractelement <2 x half> %src0, i32 1
%src1.hi = extractelement <2 x half> %src1, i32 1
%src2.hi = extractelement <2 x half> %src2, i32 1
%src0.ext = fpext half %src0.hi to float
%src1.ext = fpext half %src1.hi to float
%src2.ext = fpext half %src2.hi to float
%result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
%max = call float @llvm.maxnum.f32(float %result, float 0.0)
%clamp = call float @llvm.minnum.f32(float %max, float 1.0)
ret float %clamp
}
; GCN-LABEL: no_mix_simple:
; GCN: s_waitcnt
; GCN-NEXT: v_mad_f32 v0, v0, v1, v2{{$}}
; GCN-NEXT: s_setpc_b64
define float @no_mix_simple(float %src0, float %src1, float %src2) #0 {
%result = call float @llvm.fmuladd.f32(float %src0, float %src1, float %src2)
ret float %result
}
; GCN-LABEL: no_mix_simple_fabs:
; GCN: s_waitcnt
; GCN-NEXT: v_mad_f32 v0, |v0|, v1, v2{{$}}
; GCN-NEXT: s_setpc_b64
define float @no_mix_simple_fabs(float %src0, float %src1, float %src2) #0 {
%src0.fabs = call float @llvm.fabs.f32(float %src0)
%result = call float @llvm.fmuladd.f32(float %src0.fabs, float %src1, float %src2)
ret float %result
}
; FIXME: Should abe able to select in thits case
; All sources are converted from f16, so it doesn't matter
; v_mad_mix_f32 flushes.
; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals:
; GFX9: v_cvt_f32_f16
; GFX9: v_cvt_f32_f16
; GFX9: v_cvt_f32_f16
; GFX9: v_fma_f32
define float @v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals(half %src0, half %src1, half %src2) #1 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%src2.ext = fpext half %src2 to float
%result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
ret float %result
}
; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32_denormals:
; GFX9: v_cvt_f32_f16
; GFX9: v_cvt_f32_f16
; GFX9: v_fma_f32
define float @v_mad_mix_f32_f16lo_f16lo_f32_denormals(half %src0, half %src1, float %src2) #1 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
ret float %result
}
; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals_fmulfadd:
; GFX9: v_cvt_f32_f16
; GFX9: v_cvt_f32_f16
; GFX9: v_cvt_f32_f16
; GFX9: v_mul_f32
; GFX9: v_add_f32
define float @v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals_fmulfadd(half %src0, half %src1, half %src2) #1 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%src2.ext = fpext half %src2 to float
%mul = fmul float %src0.ext, %src1.ext
%result = fadd float %mul, %src2.ext
ret float %result
}
; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32_denormals_fmulfadd:
; GFX9: v_cvt_f32_f16
; GFX9: v_cvt_f32_f16
; GFX9: v_mul_f32
; GFX9: v_add_f32
define float @v_mad_mix_f32_f16lo_f16lo_f32_denormals_fmulfadd(half %src0, half %src1, float %src2) #1 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%mul = fmul float %src0.ext, %src1.ext
%result = fadd float %mul, %src2
ret float %result
}
; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f16lo_f32_flush_fmulfadd:
; GCN: s_waitcnt
; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2{{$}}
; GFX9-NEXT: s_setpc_b64
define float @v_mad_mix_f32_f16lo_f16lo_f16lo_f32_flush_fmulfadd(half %src0, half %src1, half %src2) #0 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%src2.ext = fpext half %src2 to float
%mul = fmul float %src0.ext, %src1.ext
%result = fadd float %mul, %src2.ext
ret float %result
}
; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32_flush_fmulfadd:
; GCN: s_waitcnt
; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]{{$}}
; GFX9-NEXT: s_setpc_b64
define float @v_mad_mix_f32_f16lo_f16lo_f32_flush_fmulfadd(half %src0, half %src1, float %src2) #0 {
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
%mul = fmul float %src0.ext, %src1.ext
%result = fadd float %mul, %src2
ret float %result
}
declare float @llvm.fabs.f32(float) #2
declare float @llvm.minnum.f32(float, float) #2
declare float @llvm.maxnum.f32(float, float) #2
declare float @llvm.fmuladd.f32(float, float, float) #2
declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>) #2
attributes #0 = { nounwind "target-features"="-fp32-denormals" }
attributes #1 = { nounwind "target-features"="+fp32-denormals" }
attributes #2 = { nounwind readnone speculatable }