Add InlineCost class for represent the estimated cost of inlining a

function.
 - This explicitly models the costs for functions which should
   "always" or "never" be inlined. This fixes bugs where such costs
   were not previously respected.

llvm-svn: 58450

llvm/include/llvm/Transforms/IPO/InlinerPass.h

@@ -18,6 +18,7 @@
 #define INLINER_H
 
 #include "llvm/CallGraphSCCPass.h"
+#include "llvm/Transforms/Utils/InlineCost.h"
 
 namespace llvm {
   class CallSite;
@@ -53,7 +54,7 @@ struct Inliner : public CallGraphSCCPass {
   /// returned is greater than the current inline threshold, the call site is
   /// not inlined.
   ///
-  virtual int getInlineCost(CallSite CS) = 0;
+  virtual InlineCost getInlineCost(CallSite CS) = 0;
 
   // getInlineFudgeFactor - Return a > 1.0 factor if the inliner should use a
   // higher threshold to determine if the function call should be inlined.

llvm/include/llvm/Transforms/Utils/InlineCost.h

@@ -15,6 +15,7 @@
 #define LLVM_TRANSFORMS_UTILS_INLINECOST_H
 
 #include "llvm/ADT/SmallPtrSet.h"
+#include <cassert>
 #include <map>
 #include <vector>
 
@@ -24,6 +25,41 @@ namespace llvm {
   class Function;
   class CallSite;
 
+  /// InlineCost - Represent the cost of inlining a function. This
+  /// supports special values for functions which should "always" or
+  /// "never" be inlined. Otherwise, the cost represents a unitless
+  /// amount; smaller values increase the likelyhood of the function
+  /// being inlined.
+  class InlineCost {
+    enum Kind {
+      Value,
+      Always,
+      Never
+    };
+
+    int Cost : 30;
+    unsigned Type :  2;
+
+    InlineCost(int C, int T) : Cost(C), Type(T) {
+      assert(Cost == C && "Cost exceeds InlineCost precision");
+    }
+  public:
+    static InlineCost get(int Cost) { return InlineCost(Cost, Value); }
+    static InlineCost getAlways() { return InlineCost(0, Always); }
+    static InlineCost getNever() { return InlineCost(0, Never); } 
+
+    bool isVariable() const { return Type == Value; }
+    bool isAlways() const { return Type == Always; }
+    bool isNever() const { return Type == Never; }
+
+    /// getValue() - Return a "variable" inline cost's amount. It is
+    /// an error to call this on an "always" or "never" InlineCost.
+    int getValue() const { 
+      assert(Type == Value && "Invalid access of InlineCost");
+      return Cost;
+    }
+  };
+  
   /// InlineCostAnalyzer - Cost analyzer used by inliner.
   class InlineCostAnalyzer {
     struct ArgInfo {
@@ -83,8 +119,8 @@ namespace llvm {
     /// getInlineCost - The heuristic used to determine if we should inline the
     /// function call or not.
     ///
-    int getInlineCost(CallSite CS,
+    InlineCost getInlineCost(CallSite CS,
-                      SmallPtrSet<const Function *, 16> &NeverInline);
+                             SmallPtrSet<const Function *, 16> &NeverInline);
 
     /// getInlineFudgeFactor - Return a > 1.0 factor if the inliner should use a
     /// higher threshold to determine if the function call should be inlined.

llvm/lib/Transforms/IPO/InlineAlways.cpp

@@ -39,7 +39,7 @@ namespace {
     // Use extremely low threshold. 
     AlwaysInliner() : Inliner(&ID, -2000000000) {}
     static char ID; // Pass identification, replacement for typeid
-    int getInlineCost(CallSite CS) {
+    InlineCost getInlineCost(CallSite CS) {
       return CA.getInlineCost(CS, NeverInline);
     }
     float getInlineFudgeFactor(CallSite CS) {

llvm/lib/Transforms/IPO/InlineSimple.cpp

@@ -37,7 +37,7 @@ namespace {
     SimpleInliner() : Inliner(&ID) {}
     SimpleInliner(int Threshold) : Inliner(&ID, Threshold) {}
     static char ID; // Pass identification, replacement for typeid
-    int getInlineCost(CallSite CS) {
+    InlineCost getInlineCost(CallSite CS) {
       return CA.getInlineCost(CS, NeverInline);
     }
     float getInlineFudgeFactor(CallSite CS) {

llvm/lib/Transforms/IPO/Inliner.cpp

@@ -76,9 +76,22 @@ static bool InlineCallIfPossible(CallSite CS, CallGraph &CG,
 /// shouldInline - Return true if the inliner should attempt to inline
 /// at the given CallSite.
 bool Inliner::shouldInline(CallSite CS) {
-  int Cost = getInlineCost(CS);
+  InlineCost IC = getInlineCost(CS);
   float FudgeFactor = getInlineFudgeFactor(CS);
   
+  if (IC.isAlways()) {
+    DOUT << "    Inlining: cost=always"
+         << ", Call: " << *CS.getInstruction();
+    return true;
+  }
+  
+  if (IC.isNever()) {
+    DOUT << "    NOT Inlining: cost=never"
+         << ", Call: " << *CS.getInstruction();
+    return false;
+  }
+  
+  int Cost = IC.getValue();
   int CurrentThreshold = InlineThreshold;
   Function *Fn = CS.getCaller();
   if (Fn && !Fn->isDeclaration() 

llvm/lib/Transforms/Utils/BasicInliner.cpp

@@ -107,16 +107,27 @@ void BasicInlinerImpl::inlineFunctions() {
           --index;
           continue;
         }
-        int InlineCost = CA.getInlineCost(CS, NeverInline);
+        InlineCost IC = CA.getInlineCost(CS, NeverInline);
-        if (InlineCost >= (int) BasicInlineThreshold) {
+        if (IC.isAlways()) {        
-          DOUT << "  NOT Inlining: cost = " << InlineCost
+          DOUT << "  Inlining: cost=always"
-               << ", call: " <<  *CS.getInstruction();
+               <<", call: " << *CS.getInstruction();
+        } else if (IC.isNever()) {
+          DOUT << "  NOT Inlining: cost=never"
+               <<", call: " << *CS.getInstruction();
           continue;
+        } else {
+          int Cost = IC.getValue();
+          
+          if (Cost >= BasicInlineThreshold) {
+            DOUT << "  NOT Inlining: cost = " << Cost
+                 << ", call: " <<  *CS.getInstruction();
+            continue;
+          } else {
+            DOUT << "  Inlining: cost = " << Cost
+                 << ", call: " <<  *CS.getInstruction();
+          }
         }
         
-        DOUT << "  Inlining: cost=" << InlineCost
-             <<", call: " << *CS.getInstruction();
-        
         // Inline
         if (InlineFunction(CS, NULL, TD)) {
           if (Callee->use_empty() && Callee->hasInternalLinkage())

llvm/lib/Transforms/Utils/InlineCost.cpp

@@ -169,7 +169,7 @@ void InlineCostAnalyzer::FunctionInfo::analyzeFunction(Function *F) {
 // getInlineCost - The heuristic used to determine if we should inline the
 // function call or not.
 //
-int InlineCostAnalyzer::getInlineCost(CallSite CS,
+InlineCost InlineCostAnalyzer::getInlineCost(CallSite CS,
                                SmallPtrSet<const Function *, 16> &NeverInline) {
   Instruction *TheCall = CS.getInstruction();
   Function *Callee = CS.getCalledFunction();
@@ -187,7 +187,7 @@ int InlineCostAnalyzer::getInlineCost(CallSite CS,
 
       // Don't inline functions marked noinline.
       NeverInline.count(Callee))
-    return 2000000000;
+    return llvm::InlineCost::getNever();
 
   // InlineCost - This value measures how good of an inline candidate this call
   // site is to inline.  A lower inline cost make is more likely for the call to
@@ -224,10 +224,14 @@ int InlineCostAnalyzer::getInlineCost(CallSite CS,
   
   // If we should never inline this, return a huge cost.
   if (CalleeFI.NeverInline)
-    return 2000000000;
+    return InlineCost::getNever();
 
+  // FIXME: It would be nice to kill off CalleeFI.NeverInline. Then we
+  // could move this up and avoid computing the FunctionInfo for
+  // things we are going to just return always inline for. This
+  // requires handling setjmp somewhere else, however.
   if (!Callee->isDeclaration() && Callee->hasFnAttr(Attribute::AlwaysInline))
-    return -2000000000;
+    return InlineCost::getAlways();
     
   // Add to the inline quality for properties that make the call valuable to
   // inline.  This includes factors that indicate that the result of inlining
@@ -274,7 +278,7 @@ int InlineCostAnalyzer::getInlineCost(CallSite CS,
   // Look at the size of the callee. Each instruction counts as 5.
   InlineCost += CalleeFI.NumInsts*5;
 
-  return InlineCost;
+  return llvm::InlineCost::get(InlineCost);
 }
 
 // getInlineFudgeFactor - Return a > 1.0 factor if the inliner should use a

llvm/test/Transforms/Inline/2008-10-30-AlwaysInline.ll

@@ -0,0 +1,14 @@
+; RUN: llvm-as < %s | opt -always-inline | llvm-dis | not grep call 
+
+; Ensure that threshold doesn't disrupt always inline.
+; RUN: llvm-as < %s | opt -inline-threshold=-2000000001 -always-inline | llvm-dis | not grep call 
+
+
+define internal i32 @if0() alwaysinline {
+       ret i32 1 
+}
+
+define i32 @f0() {
+       %r = call i32 @if0()
+       ret i32 %r
+}