R600/SI: Use mad for fsub + fmul

We can use a negate source modifier to match this for fsub. llvm-svn: 216735
2014-08-29 16:01:14 +00:00 · 2014-08-29 16:01:14 +00:00 · 8675db15da
parent 3c0915e858
commit 8675db15da
7 changed files with 227 additions and 0 deletions
--- a/llvm/lib/Target/R600/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/R600/AMDGPUISelLowering.cpp
@ -2157,6 +2157,7 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
  NODE_NAME_CASE(DWORDADDR)
  NODE_NAME_CASE(FRACT)
  NODE_NAME_CASE(CLAMP)
+  NODE_NAME_CASE(MAD)
  NODE_NAME_CASE(FMAX)
  NODE_NAME_CASE(SMAX)
  NODE_NAME_CASE(UMAX)
--- a/llvm/lib/Target/R600/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/R600/AMDGPUISelLowering.h
@ -180,6 +180,7 @@ enum {
  DWORDADDR,
  FRACT,
  CLAMP,
+  MAD, // Multiply + add with same result as the separate operations.

  // SIN_HW, COS_HW - f32 for SI, 1 ULP max error, valid from -100 pi to 100 pi.
  // Denormals handled on some parts.
--- a/llvm/lib/Target/R600/AMDGPUInstrInfo.td
+++ b/llvm/lib/Target/R600/AMDGPUInstrInfo.td
@ -64,6 +64,7 @@ def AMDGPUfmax : SDNode<"AMDGPUISD::FMAX", SDTFPBinOp,
 >;

 def AMDGPUclamp : SDNode<"AMDGPUISD::CLAMP", SDTFPTernaryOp, []>;
+def AMDGPUmad : SDNode<"AMDGPUISD::MAD", SDTFPTernaryOp, []>;

 // out = max(a, b) a and b are signed ints
 def AMDGPUsmax : SDNode<"AMDGPUISD::SMAX", SDTIntBinOp,
--- a/llvm/lib/Target/R600/SIISelLowering.cpp
+++ b/llvm/lib/Target/R600/SIISelLowering.cpp
@ -226,6 +226,7 @@ SITargetLowering::SITargetLowering(TargetMachine &TM) :

  setOperationAction(ISD::FDIV, MVT::f32, Custom);

+  setTargetDAGCombine(ISD::FSUB);
  setTargetDAGCombine(ISD::SELECT_CC);
  setTargetDAGCombine(ISD::SETCC);

@ -1386,6 +1387,42 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,

  case ISD::UINT_TO_FP: {
    return performUCharToFloatCombine(N, DCI);
+
+  case ISD::FSUB: {
+    if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
+      break;
+
+    EVT VT = N->getValueType(0);
+
+    // Try to get the fneg to fold into the source modifier. This undoes generic
+    // DAG combines and folds them into the mad.
+    if (VT == MVT::f32) {
+      SDValue LHS = N->getOperand(0);
+      SDValue RHS = N->getOperand(1);
+
+      if (LHS.getOpcode() == ISD::FMUL) {
+        // (fsub (fmul a, b), c) -> mad a, b, (fneg c)
+
+        SDValue A = LHS.getOperand(0);
+        SDValue B = LHS.getOperand(1);
+        SDValue C = DAG.getNode(ISD::FNEG, DL, VT, RHS);
+
+        return DAG.getNode(AMDGPUISD::MAD, DL, VT, A, B, C);
+      }
+
+      if (RHS.getOpcode() == ISD::FMUL) {
+        // (fsub c, (fmul a, b)) -> mad (fneg a), b, c
+
+        SDValue A = DAG.getNode(ISD::FNEG, DL, VT, RHS.getOperand(0));
+        SDValue B = RHS.getOperand(1);
+        SDValue C = LHS;
+
+        return DAG.getNode(AMDGPUISD::MAD, DL, VT, A, B, C);
+      }
+    }
+
+    break;
+  }
  }
  case ISD::LOAD:
  case ISD::STORE:
--- a/llvm/lib/Target/R600/SIInstrInfo.td
+++ b/llvm/lib/Target/R600/SIInstrInfo.td
@ -796,6 +796,17 @@ multiclass VOP3b_64 <bits<9> op, string opName, list<dag> pattern> :
 multiclass VOP3b_32 <bits<9> op, string opName, list<dag> pattern> :
  VOP3b_Helper <op, VReg_32, VSrc_32, opName, pattern>;

+
+class Vop3ModPat<Instruction Inst, VOPProfile P, SDPatternOperator node> : Pat<
+  (node (P.Src0VT (VOP3Mods0 P.Src0VT:$src0, i32:$src0_modifiers, i32:$clamp, i32:$omod)),
+        (P.Src1VT (VOP3Mods P.Src1VT:$src1, i32:$src1_modifiers)),
+        (P.Src2VT (VOP3Mods P.Src2VT:$src2, i32:$src2_modifiers))),
+  (Inst i32:$src0_modifiers, P.Src0VT:$src0,
+        i32:$src1_modifiers, P.Src1VT:$src1,
+        i32:$src2_modifiers, P.Src2VT:$src2,
+        i32:$clamp,
+        i32:$omod)>;
+
 //===----------------------------------------------------------------------===//
 // Vector I/O classes
 //===----------------------------------------------------------------------===//
--- a/llvm/lib/Target/R600/SIInstructions.td
+++ b/llvm/lib/Target/R600/SIInstructions.td
@ -2513,6 +2513,9 @@ def : Pat <
  (V_MUL_HI_I32 $src0, $src1)
 >;

+def : Vop3ModPat<V_MAD_F32, VOP_F32_F32_F32_F32, AMDGPUmad>;
+
+
 defm : BFIPatterns <V_BFI_B32, S_MOV_B32>;
 def : ROTRPattern <V_ALIGNBIT_B32>;

--- a/llvm/test/CodeGen/R600/mad-sub.ll
+++ b/llvm/test/CodeGen/R600/mad-sub.ll
@ -0,0 +1,173 @@
+; RUN: llc -march=r600 -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+
+declare i32 @llvm.r600.read.tidig.x() #0
+declare float @llvm.fabs.f32(float) #0
+
+; FUNC-LABEL: @mad_sub_f32
+; SI: BUFFER_LOAD_DWORD [[REGA:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGB:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGC:v[0-9]+]]
+; SI: V_MAD_F32 [[RESULT:v[0-9]+]], [[REGA]], [[REGB]], -[[REGC]]
+; SI: BUFFER_STORE_DWORD [[RESULT]]
+define void @mad_sub_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 {
+  %tid = tail call i32 @llvm.r600.read.tidig.x() #0
+  %tid.ext = sext i32 %tid to i64
+  %gep0 = getelementptr float addrspace(1)* %ptr, i64 %tid.ext
+  %add1 = add i64 %tid.ext, 1
+  %gep1 = getelementptr float addrspace(1)* %ptr, i64 %add1
+  %add2 = add i64 %tid.ext, 2
+  %gep2 = getelementptr float addrspace(1)* %ptr, i64 %add2
+  %outgep = getelementptr float addrspace(1)* %out, i64 %tid.ext
+  %a = load float addrspace(1)* %gep0, align 4
+  %b = load float addrspace(1)* %gep1, align 4
+  %c = load float addrspace(1)* %gep2, align 4
+  %mul = fmul float %a, %b
+  %sub = fsub float %mul, %c
+  store float %sub, float addrspace(1)* %outgep, align 4
+  ret void
+}
+
+; FUNC-LABEL: @mad_sub_inv_f32
+; SI: BUFFER_LOAD_DWORD [[REGA:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGB:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGC:v[0-9]+]]
+; SI: V_MAD_F32 [[RESULT:v[0-9]+]], -[[REGA]], [[REGB]], [[REGC]]
+; SI: BUFFER_STORE_DWORD [[RESULT]]
+define void @mad_sub_inv_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 {
+  %tid = tail call i32 @llvm.r600.read.tidig.x() #0
+  %tid.ext = sext i32 %tid to i64
+  %gep0 = getelementptr float addrspace(1)* %ptr, i64 %tid.ext
+  %add1 = add i64 %tid.ext, 1
+  %gep1 = getelementptr float addrspace(1)* %ptr, i64 %add1
+  %add2 = add i64 %tid.ext, 2
+  %gep2 = getelementptr float addrspace(1)* %ptr, i64 %add2
+  %outgep = getelementptr float addrspace(1)* %out, i64 %tid.ext
+  %a = load float addrspace(1)* %gep0, align 4
+  %b = load float addrspace(1)* %gep1, align 4
+  %c = load float addrspace(1)* %gep2, align 4
+  %mul = fmul float %a, %b
+  %sub = fsub float %c, %mul
+  store float %sub, float addrspace(1)* %outgep, align 4
+  ret void
+}
+
+; FUNC-LABEL: @mad_sub_f64
+; SI: V_MUL_F64
+; SI: V_ADD_F64
+define void @mad_sub_f64(double addrspace(1)* noalias nocapture %out, double addrspace(1)* noalias nocapture readonly %ptr) #1 {
+  %tid = tail call i32 @llvm.r600.read.tidig.x() #0
+  %tid.ext = sext i32 %tid to i64
+  %gep0 = getelementptr double addrspace(1)* %ptr, i64 %tid.ext
+  %add1 = add i64 %tid.ext, 1
+  %gep1 = getelementptr double addrspace(1)* %ptr, i64 %add1
+  %add2 = add i64 %tid.ext, 2
+  %gep2 = getelementptr double addrspace(1)* %ptr, i64 %add2
+  %outgep = getelementptr double addrspace(1)* %out, i64 %tid.ext
+  %a = load double addrspace(1)* %gep0, align 8
+  %b = load double addrspace(1)* %gep1, align 8
+  %c = load double addrspace(1)* %gep2, align 8
+  %mul = fmul double %a, %b
+  %sub = fsub double %mul, %c
+  store double %sub, double addrspace(1)* %outgep, align 8
+  ret void
+}
+
+; FUNC-LABEL: @mad_sub_fabs_f32
+; SI: BUFFER_LOAD_DWORD [[REGA:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGB:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGC:v[0-9]+]]
+; SI: V_MAD_F32 [[RESULT:v[0-9]+]], [[REGA]], [[REGB]], -|[[REGC]]|
+; SI: BUFFER_STORE_DWORD [[RESULT]]
+define void @mad_sub_fabs_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 {
+  %tid = tail call i32 @llvm.r600.read.tidig.x() #0
+  %tid.ext = sext i32 %tid to i64
+  %gep0 = getelementptr float addrspace(1)* %ptr, i64 %tid.ext
+  %add1 = add i64 %tid.ext, 1
+  %gep1 = getelementptr float addrspace(1)* %ptr, i64 %add1
+  %add2 = add i64 %tid.ext, 2
+  %gep2 = getelementptr float addrspace(1)* %ptr, i64 %add2
+  %outgep = getelementptr float addrspace(1)* %out, i64 %tid.ext
+  %a = load float addrspace(1)* %gep0, align 4
+  %b = load float addrspace(1)* %gep1, align 4
+  %c = load float addrspace(1)* %gep2, align 4
+  %c.abs = call float @llvm.fabs.f32(float %c) #0
+  %mul = fmul float %a, %b
+  %sub = fsub float %mul, %c.abs
+  store float %sub, float addrspace(1)* %outgep, align 4
+  ret void
+}
+
+; FUNC-LABEL: @mad_sub_fabs_inv_f32
+; SI: BUFFER_LOAD_DWORD [[REGA:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGB:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGC:v[0-9]+]]
+; SI: V_MAD_F32 [[RESULT:v[0-9]+]], -[[REGA]], [[REGB]], |[[REGC]]|
+; SI: BUFFER_STORE_DWORD [[RESULT]]
+define void @mad_sub_fabs_inv_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 {
+  %tid = tail call i32 @llvm.r600.read.tidig.x() #0
+  %tid.ext = sext i32 %tid to i64
+  %gep0 = getelementptr float addrspace(1)* %ptr, i64 %tid.ext
+  %add1 = add i64 %tid.ext, 1
+  %gep1 = getelementptr float addrspace(1)* %ptr, i64 %add1
+  %add2 = add i64 %tid.ext, 2
+  %gep2 = getelementptr float addrspace(1)* %ptr, i64 %add2
+  %outgep = getelementptr float addrspace(1)* %out, i64 %tid.ext
+  %a = load float addrspace(1)* %gep0, align 4
+  %b = load float addrspace(1)* %gep1, align 4
+  %c = load float addrspace(1)* %gep2, align 4
+  %c.abs = call float @llvm.fabs.f32(float %c) #0
+  %mul = fmul float %a, %b
+  %sub = fsub float %c.abs, %mul
+  store float %sub, float addrspace(1)* %outgep, align 4
+  ret void
+}
+
+; FUNC-LABEL: @neg_neg_mad_f32
+; SI: V_MAD_F32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
+define void @neg_neg_mad_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 {
+  %tid = tail call i32 @llvm.r600.read.tidig.x() #0
+  %tid.ext = sext i32 %tid to i64
+  %gep0 = getelementptr float addrspace(1)* %ptr, i64 %tid.ext
+  %add1 = add i64 %tid.ext, 1
+  %gep1 = getelementptr float addrspace(1)* %ptr, i64 %add1
+  %add2 = add i64 %tid.ext, 2
+  %gep2 = getelementptr float addrspace(1)* %ptr, i64 %add2
+  %outgep = getelementptr float addrspace(1)* %out, i64 %tid.ext
+  %a = load float addrspace(1)* %gep0, align 4
+  %b = load float addrspace(1)* %gep1, align 4
+  %c = load float addrspace(1)* %gep2, align 4
+  %nega = fsub float -0.000000e+00, %a
+  %negb = fsub float -0.000000e+00, %b
+  %mul = fmul float %nega, %negb
+  %sub = fadd float %mul, %c
+  store float %sub, float addrspace(1)* %outgep, align 4
+  ret void
+}
+
+; FUNC-LABEL: @mad_fabs_sub_f32
+; SI: BUFFER_LOAD_DWORD [[REGA:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGB:v[0-9]+]]
+; SI: BUFFER_LOAD_DWORD [[REGC:v[0-9]+]]
+; SI: V_MAD_F32 [[RESULT:v[0-9]+]], [[REGA]], |[[REGB]]|, -[[REGC]]
+; SI: BUFFER_STORE_DWORD [[RESULT]]
+define void @mad_fabs_sub_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 {
+  %tid = tail call i32 @llvm.r600.read.tidig.x() #0
+  %tid.ext = sext i32 %tid to i64
+  %gep0 = getelementptr float addrspace(1)* %ptr, i64 %tid.ext
+  %add1 = add i64 %tid.ext, 1
+  %gep1 = getelementptr float addrspace(1)* %ptr, i64 %add1
+  %add2 = add i64 %tid.ext, 2
+  %gep2 = getelementptr float addrspace(1)* %ptr, i64 %add2
+  %outgep = getelementptr float addrspace(1)* %out, i64 %tid.ext
+  %a = load float addrspace(1)* %gep0, align 4
+  %b = load float addrspace(1)* %gep1, align 4
+  %c = load float addrspace(1)* %gep2, align 4
+  %b.abs = call float @llvm.fabs.f32(float %b) #0
+  %mul = fmul float %a, %b.abs
+  %sub = fsub float %mul, %c
+  store float %sub, float addrspace(1)* %outgep, align 4
+  ret void
+}
+
+attributes #0 = { nounwind readnone }
+attributes #1 = { nounwind }