[InstructionSimplify] handle denormal input for fcmp

Handle denormal constant input for fcmp instructions based on the
denormal handling mode.

Reviewed By: spatel, dcandler

Differential Revision: https://reviews.llvm.org/D128647

llvm/include/llvm/Analysis/ConstantFolding.h

@@ -67,14 +67,13 @@ Constant *ConstantFoldInstOperands(Instruction *I, ArrayRef<Constant *> Ops,
                                    const DataLayout &DL,
                                    const TargetLibraryInfo *TLI = nullptr);
 
-/// ConstantFoldCompareInstOperands - Attempt to constant fold a compare
+/// Attempt to constant fold a compare instruction (icmp/fcmp) with the
-/// instruction (icmp/fcmp) with the specified operands.  If it fails, it
+/// specified operands.  If it fails, it returns a constant expression of the
-/// returns a constant expression of the specified operands.
+/// specified operands.
-///
+/// Denormal inputs may be flushed based on the denormal handling mode.
-Constant *
+Constant *ConstantFoldCompareInstOperands(
-ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS,
+    unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL,
-                                Constant *RHS, const DataLayout &DL,
+    const TargetLibraryInfo *TLI = nullptr, const Instruction *I = nullptr);
-                                const TargetLibraryInfo *TLI = nullptr);
 
 /// Attempt to constant fold a unary operation with the specified
 /// operand. If it fails, it returns a constant expression of the specified
@@ -95,6 +94,14 @@ Constant *ConstantFoldFPInstOperands(unsigned Opcode, Constant *LHS,
                                      Constant *RHS, const DataLayout &DL,
                                      const Instruction *I);
 
+/// Attempt to flush float point constant according to denormal mode set in the
+/// instruction's parent function attributes. If so, return a zero with the
+/// correct sign, otherwise return the original constant. Inputs and outputs to
+/// floating point instructions can have their mode set separately, so the
+/// direction is also needed.
+Constant *FlushFPConstant(Constant *Operand, const Instruction *I,
+                          bool IsOutput);
+
 /// Attempt to constant fold a select instruction with the specified
 /// operands. The constant result is returned if successful; if not, null is
 /// returned.

llvm/lib/Analysis/ConstantFolding.cpp

@@ -1071,9 +1071,11 @@ Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode,
   switch (Opcode) {
   default: return nullptr;
   case Instruction::ICmp:
-  case Instruction::FCmp:
+  case Instruction::FCmp: {
-    return ConstantFoldCompareInstOperands(
+    auto *C = cast<CmpInst>(InstOrCE);
-        cast<CmpInst>(InstOrCE)->getPredicate(), Ops[0], Ops[1], DL, TLI);
+    return ConstantFoldCompareInstOperands(C->getPredicate(), Ops[0], Ops[1],
+                                           DL, TLI, C);
+  }
   case Instruction::Freeze:
     return isGuaranteedNotToBeUndefOrPoison(Ops[0]) ? Ops[0] : nullptr;
   case Instruction::Call:
@@ -1210,10 +1212,9 @@ Constant *llvm::ConstantFoldInstOperands(Instruction *I,
   return ConstantFoldInstOperandsImpl(I, I->getOpcode(), Ops, DL, TLI);
 }
 
-Constant *llvm::ConstantFoldCompareInstOperands(unsigned IntPredicate,
+Constant *llvm::ConstantFoldCompareInstOperands(
-                                                Constant *Ops0, Constant *Ops1,
+    unsigned IntPredicate, Constant *Ops0, Constant *Ops1, const DataLayout &DL,
-                                                const DataLayout &DL,
+    const TargetLibraryInfo *TLI, const Instruction *I) {
-                                                const TargetLibraryInfo *TLI) {
   CmpInst::Predicate Predicate = (CmpInst::Predicate)IntPredicate;
   // fold: icmp (inttoptr x), null         -> icmp x, 0
   // fold: icmp null, (inttoptr x)         -> icmp 0, x
@@ -1315,6 +1316,11 @@ Constant *llvm::ConstantFoldCompareInstOperands(unsigned IntPredicate,
     return ConstantFoldCompareInstOperands(Predicate, Ops1, Ops0, DL, TLI);
   }
 
+  // Flush any denormal constant float input according to denormal handling
+  // mode.
+  Ops0 = FlushFPConstant(Ops0, I, /* IsOutput */ false);
+  Ops1 = FlushFPConstant(Ops1, I, /* IsOutput */ false);
+
   return ConstantExpr::getCompare(Predicate, Ops0, Ops1);
 }
 
@@ -1336,41 +1342,40 @@ Constant *llvm::ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS,
   return ConstantExpr::get(Opcode, LHS, RHS);
 }
 
-// Check whether a constant is a floating point denormal that should be flushed
+Constant *llvm::FlushFPConstant(Constant *Operand, const Instruction *I,
-// to zero according to the denormal handling mode set in the function
+                                bool IsOutput) {
-// attributes. If so, return a zero with the correct sign, otherwise return the
+  if (!I || !I->getParent() || !I->getFunction())
-// original constant. Inputs and outputs to floating point instructions can have
-// their mode set separately, so the direction is also needed.
-Constant *FlushFPConstant(Constant *Operand, const llvm::Function *F,
-                          bool IsOutput) {
-  if (F == nullptr)
     return Operand;
-  if (auto *CFP = dyn_cast<ConstantFP>(Operand)) {
+
-    const APFloat &APF = CFP->getValueAPF();
+  ConstantFP *CFP = dyn_cast<ConstantFP>(Operand);
-    Type *Ty = CFP->getType();
+  if (!CFP)
-    DenormalMode DenormMode = F->getDenormalMode(Ty->getFltSemantics());
+    return Operand;
-    DenormalMode::DenormalModeKind Mode =
+
-        IsOutput ? DenormMode.Output : DenormMode.Input;
+  const APFloat &APF = CFP->getValueAPF();
-    switch (Mode) {
+  Type *Ty = CFP->getType();
-    default:
+  DenormalMode DenormMode =
-      llvm_unreachable("unknown denormal mode");
+      I->getFunction()->getDenormalMode(Ty->getFltSemantics());
-      return Operand;
+  DenormalMode::DenormalModeKind Mode =
-    case DenormalMode::IEEE:
+      IsOutput ? DenormMode.Output : DenormMode.Input;
-      return Operand;
+  switch (Mode) {
-    case DenormalMode::PreserveSign:
+  default:
-      if (APF.isDenormal()) {
+    llvm_unreachable("unknown denormal mode");
-        return ConstantFP::get(
+    return Operand;
-            Ty->getContext(),
+  case DenormalMode::IEEE:
-            APFloat::getZero(Ty->getFltSemantics(), APF.isNegative()));
+    return Operand;
-      }
+  case DenormalMode::PreserveSign:
-      return Operand;
+    if (APF.isDenormal()) {
-    case DenormalMode::PositiveZero:
+      return ConstantFP::get(
-      if (APF.isDenormal()) {
+          Ty->getContext(),
-        return ConstantFP::get(Ty->getContext(),
+          APFloat::getZero(Ty->getFltSemantics(), APF.isNegative()));
-                               APFloat::getZero(Ty->getFltSemantics(), false));
-      }
-      return Operand;
     }
+    return Operand;
+  case DenormalMode::PositiveZero:
+    if (APF.isDenormal()) {
+      return ConstantFP::get(Ty->getContext(),
+                             APFloat::getZero(Ty->getFltSemantics(), false));
+    }
+    return Operand;
   }
   return Operand;
 }
@@ -1378,15 +1383,16 @@ Constant *FlushFPConstant(Constant *Operand, const llvm::Function *F,
 Constant *llvm::ConstantFoldFPInstOperands(unsigned Opcode, Constant *LHS,
                                            Constant *RHS, const DataLayout &DL,
                                            const Instruction *I) {
-  if (auto *BB = I->getParent()) {
+  if (Instruction::isBinaryOp(Opcode)) {
-    if (auto *F = BB->getParent()) {
+    // Flush denormal inputs if needed.
-      if (Instruction::isBinaryOp(Opcode)) {
+    Constant *Op0 = FlushFPConstant(LHS, I, /* IsOutput */ false);
-        Constant *Op0 = FlushFPConstant(LHS, F, false);
+    Constant *Op1 = FlushFPConstant(RHS, I, /* IsOutput */ false);
-        Constant *Op1 = FlushFPConstant(RHS, F, false);
+
-        Constant *C = ConstantFoldBinaryOpOperands(Opcode, Op0, Op1, DL);
+    // Calculate constant result.
-        return FlushFPConstant(C, F, true);
+    Constant *C = ConstantFoldBinaryOpOperands(Opcode, Op0, Op1, DL);
-      }
+
-    }
+    // Flush denormal output if needed.
+    return FlushFPConstant(C, I, /* IsOutput */ true);
   }
   // If instruction lacks a parent/function and the denormal mode cannot be
   // determined, use the default (IEEE).

llvm/lib/Analysis/InstructionSimplify.cpp

@@ -3900,7 +3900,8 @@ static Value *simplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
 
   if (Constant *CLHS = dyn_cast<Constant>(LHS)) {
     if (Constant *CRHS = dyn_cast<Constant>(RHS))
-      return ConstantFoldCompareInstOperands(Pred, CLHS, CRHS, Q.DL, Q.TLI);
+      return ConstantFoldCompareInstOperands(Pred, CLHS, CRHS, Q.DL, Q.TLI,
+                                             Q.CxtI);
 
     // If we have a constant, make sure it is on the RHS.
     std::swap(LHS, RHS);

llvm/test/Transforms/InstSimplify/constant-fold-fp-denormal.ll

@@ -763,7 +763,7 @@ entry:
 define i1 @fcmp_double_pz_in_pz_out() #6 {
 ; CHECK-LABEL: @fcmp_double_pz_in_pz_out(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    ret i1 false
 ;
 entry:
   %cmp = fcmp une double 0x0008000000000000, 0x0
@@ -773,7 +773,7 @@ entry:
 define i1 @fcmp_float_pz_in_pz_out() #6 {
 ; CHECK-LABEL: @fcmp_float_pz_in_pz_out(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    ret i1 false
 ;
 entry:
   %cmp = fcmp une float 0x3800000000000000, 0x0
@@ -783,7 +783,7 @@ entry:
 define i1 @fcmp_double_ps_in_ps_out() #7 {
 ; CHECK-LABEL: @fcmp_double_ps_in_ps_out(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    ret i1 false
 ;
 entry:
   %cmp = fcmp une double 0x0008000000000000, 0x0
@@ -793,7 +793,7 @@ entry:
 define i1 @fcmp_float_ps_in_ps_out() #7 {
 ; CHECK-LABEL: @fcmp_float_ps_in_ps_out(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    ret i1 false
 ;
 entry:
   %cmp = fcmp une float 0x3800000000000000, 0x0
@@ -823,7 +823,7 @@ entry:
 define i1 @fcmp_double_ieee_out_pz_in() #3 {
 ; CHECK-LABEL: @fcmp_double_ieee_out_pz_in(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    ret i1 false
 ;
 entry:
   %cmp = fcmp une double 0x0008000000000000, 0x0
@@ -833,7 +833,7 @@ entry:
 define i1 @fcmp_double_ieee_out_ps_in() #4 {
 ; CHECK-LABEL: @fcmp_double_ieee_out_ps_in(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    ret i1 false
 ;
 entry:
   %cmp = fcmp une double 0x0008000000000000, 0x0
@@ -853,7 +853,7 @@ entry:
 define i1 @fcmp_double_two_denormal_ins() #6 {
 ; CHECK-LABEL: @fcmp_double_two_denormal_ins(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    ret i1 false
 ;
 entry:
   %cmp = fcmp une double 0x0008100000000000, 0x0008000000000000
@@ -903,7 +903,7 @@ entry:
 define i1 @fcmp_double_ps_in_ps_out_oeq() #6 {
 ; CHECK-LABEL: @fcmp_double_ps_in_ps_out_oeq(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i1 false
+; CHECK-NEXT:    ret i1 true
 ;
 entry:
   %cmp = fcmp oeq double 0x0008100000000000, 0x0008000000000000
@@ -923,7 +923,7 @@ entry:
 define i1 @fcmp_double_ps_in_ps_out_one() #6 {
 ; CHECK-LABEL: @fcmp_double_ps_in_ps_out_one(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    ret i1 false
 ;
 entry:
   %cmp = fcmp one double 0x0008100000000000, 0x0008000000000000
@@ -983,7 +983,7 @@ entry:
 define i1 @fcmp_double_pz_in_pz_out_ugt() #7 {
 ; CHECK-LABEL: @fcmp_double_pz_in_pz_out_ugt(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    ret i1 false
 ;
 entry:
   %cmp = fcmp ugt double 0x0008000000000000, 0x0
@@ -1043,7 +1043,7 @@ entry:
 define i1 @fcmp_double_pz_in_pz_out_ule() #7 {
 ; CHECK-LABEL: @fcmp_double_pz_in_pz_out_ule(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i1 false
+; CHECK-NEXT:    ret i1 true
 ;
 entry:
   %cmp = fcmp ule double 0x0008000000000000, 0x0