[llvm] Add interface to drive inlining decision using ML model

Summary:

This change introduces InliningAdvisor (and related APIs), the interface
that abstracts decision making away from the inlining pass. We will use
this interface to delegate decision making to a trained ML model,
subsequently (see referenced RFC).

RFC: http://lists.llvm.org/pipermail/llvm-dev/2020-April/140763.html

Reviewers: davidxl, eraman, dblaikie

Subscribers: mgorny, hiraditya, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D79042

llvm/include/llvm/Analysis/InlineAdvisor.h

@@ -20,7 +20,178 @@ namespace llvm {
 class BasicBlock;
 class CallBase;
 class Function;
+class Module;
 class OptimizationRemarkEmitter;
+class PreservedAnalyses;
+
+/// There are 3 scenarios we can use the InlineAdvisor:
+/// - Default - use manual heuristics.
+///
+/// - Release mode, the expected mode for production, day to day deployments.
+/// In this mode, when building the compiler, we also compile a pre-trained ML
+/// model to native code, and link it as a static library. This mode has low
+/// overhead and no additional dependencies for the compiler runtime.
+///
+/// - Development mode, for training new models.
+/// In this mode, we trade off runtime performance for flexibility. This mode
+/// requires the full C Tensorflow API library, and evaluates models
+/// dynamically. This mode also permits generating training logs, for offline
+/// training.
+enum class InliningAdvisorMode : int { Default, Release, Development };
+
+class InlineAdvisor;
+/// Capture state between an inlining decision having had been made, and
+/// its impact being observable. When collecting model training data, this
+/// allows recording features/decisions/partial reward data sets.
+///
+/// Derivations of this type are expected to be tightly coupled with their
+/// InliningAdvisors. The base type implements the minimal contractual
+/// obligations.
+class InlineAdvice {
+public:
+  InlineAdvice(InlineAdvice &&) = delete;
+  InlineAdvice(const InlineAdvice &) = delete;
+  virtual ~InlineAdvice() {
+    assert(Recorded && "InlineAdvice should have been informed of the "
+                       "inliner's decision in all cases");
+  }
+
+  /// Exactly one of the record* APIs must be called. Implementers may extend
+  /// behavior by implementing the corresponding record*Impl.
+  ///
+  /// Call after inlining succeeded, and did not result in deleting the callee.
+  void recordInlining() {
+    markRecorded();
+    recordInliningImpl();
+  }
+
+  /// Call after inlining succeeded, and resulted in deleting the callee.
+  void recordInliningWithCalleeDeleted();
+
+  /// Call after the decision for a call site was to not inline.
+  void recordUnsuccessfulInlining(const InlineResult &Result) {
+    markRecorded();
+    recordUnsuccessfulInliningImpl(Result);
+  }
+
+  /// Call to indicate inlining was not attempted.
+  void recordUnattemptedInlining() {
+    markRecorded();
+    recordUnattemptedInliningImpl();
+  }
+
+  /// Get the inlining recommendation.
+  bool isInliningRecommended() const { return IsInliningRecommended; }
+
+protected:
+  InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
+               bool IsInliningRecommended);
+
+  virtual void recordInliningImpl() {}
+  virtual void recordInliningWithCalleeDeletedImpl() {}
+  virtual void recordUnsuccessfulInliningImpl(const InlineResult &Result) {}
+  virtual void recordUnattemptedInliningImpl() {}
+
+  InlineAdvisor *const Advisor;
+  /// Caller and Callee are pre-inlining.
+  Function *const Caller;
+  Function *const Callee;
+  const bool IsInliningRecommended;
+
+private:
+  void markRecorded() {
+    assert(!Recorded && "Recording should happen exactly once");
+    Recorded = true;
+  }
+
+  bool Recorded = false;
+};
+
+/// Interface for deciding whether to inline a call site or not.
+class InlineAdvisor {
+public:
+  InlineAdvisor(InlineAdvisor &&) = delete;
+  virtual ~InlineAdvisor() { freeDeletedFunctions(); }
+
+  /// Get an InlineAdvice containing a recommendation on whether to
+  /// inline or not. \p CB is assumed to be a direct call. \p FAM is assumed to
+  /// be up-to-date wrt previous inlining decisions.
+  /// Returns an InlineAdvice with the inlining recommendation.
+  virtual std::unique_ptr<InlineAdvice>
+  getAdvice(CallBase &CB, FunctionAnalysisManager &FAM) = 0;
+
+  /// This must be called when the Inliner pass is entered, to allow the
+  /// InlineAdvisor update internal state, as result of function passes run
+  /// between Inliner pass runs (for the same module).
+  virtual void OnPassEntry() {}
+
+  /// This must be called when the Inliner pass is exited, as function passes
+  /// may be run subsequently. This allows an implementation of InlineAdvisor
+  /// to prepare for a partial update.
+  virtual void OnPassExit() {}
+
+protected:
+  InlineAdvisor() = default;
+
+  /// We may want to defer deleting functions to after the inlining for a whole
+  /// module has finished. This allows us to reliably use function pointers as
+  /// unique identifiers, as an efficient implementation detail of the
+  /// InlineAdvisor. Otherwise, it is possible the memory allocator
+  /// re-allocate Function objects at the same address of a deleted Function;
+  /// and Functions are potentially created during the function passes called
+  /// after each SCC inlining (e.g. argument promotion does that).
+  void freeDeletedFunctions();
+
+  bool isFunctionDeleted(Function *F) const {
+    return DeletedFunctions.count(F);
+  }
+
+private:
+  friend class InlineAdvice;
+  void markFunctionAsDeleted(Function *F);
+  std::unordered_set<Function *> DeletedFunctions;
+};
+
+/// The default (manual heuristics) implementation of the InlineAdvisor. This
+/// implementation does not need to keep state between inliner pass runs, and is
+/// reusable as-is for inliner pass test scenarios, as well as for regular use.
+class DefaultInlineAdvisor : public InlineAdvisor {
+public:
+  DefaultInlineAdvisor(InlineParams Params) : Params(Params) {}
+
+private:
+  std::unique_ptr<InlineAdvice>
+  getAdvice(CallBase &CB, FunctionAnalysisManager &FAM) override;
+
+  void OnPassExit() override { freeDeletedFunctions(); }
+  InlineParams Params;
+};
+
+/// The InlineAdvisorAnalysis is a module pass because the InlineAdvisor
+/// needs to capture state right before inlining commences over a module.
+class InlineAdvisorAnalysis : public AnalysisInfoMixin<InlineAdvisorAnalysis> {
+public:
+  static AnalysisKey Key;
+  InlineAdvisorAnalysis() = default;
+  struct Result {
+    Result(Module &M, ModuleAnalysisManager &MAM) : M(M), MAM(MAM) {}
+    bool invalidate(Module &, const PreservedAnalyses &,
+                    ModuleAnalysisManager::Invalidator &) {
+      // InlineAdvisor must be preserved across analysis invalidations.
+      return false;
+    }
+    bool tryCreate(InlineParams Params, InliningAdvisorMode Mode);
+    InlineAdvisor *getAdvisor() const { return Advisor.get(); }
+    void clear() { Advisor.reset(); }
+
+  private:
+    Module &M;
+    ModuleAnalysisManager &MAM;
+    std::unique_ptr<InlineAdvisor> Advisor;
+  };
+
+  Result run(Module &M, ModuleAnalysisManager &MAM) { return Result(M, MAM); }
+};
 
 // Default (manual policy) decision making helper APIs. Shared with the legacy
 // pass manager inliner.

llvm/include/llvm/Passes/PassBuilder.h

@@ -19,6 +19,7 @@
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Transforms/IPO/Inliner.h"
 #include "llvm/Transforms/Instrumentation.h"
 #include "llvm/Transforms/Scalar/LoopPassManager.h"
 #include <vector>
@@ -345,9 +346,9 @@ public:
 
   /// Construct the module pipeline that performs inlining as well as
   /// the inlining-driven cleanups.
-  ModulePassManager buildInlinerPipeline(OptimizationLevel Level,
-                                         ThinLTOPhase Phase,
-                                         bool DebugLogging = false);
+  ModuleInlinerWrapperPass buildInlinerPipeline(OptimizationLevel Level,
+                                                ThinLTOPhase Phase,
+                                                bool DebugLogging = false);
 
   /// Construct the core LLVM module optimization pipeline.
   ///

llvm/include/llvm/Transforms/IPO/Inliner.h

@@ -11,6 +11,7 @@
 
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/Analysis/CallGraphSCCPass.h"
+#include "llvm/Analysis/InlineAdvisor.h"
 #include "llvm/Analysis/InlineCost.h"
 #include "llvm/Analysis/LazyCallGraph.h"
 #include "llvm/IR/PassManager.h"
@@ -93,21 +94,54 @@ protected:
 /// passes be composed to achieve the same end result.
 class InlinerPass : public PassInfoMixin<InlinerPass> {
 public:
-  InlinerPass(InlineParams Params = getInlineParams())
-      : Params(std::move(Params)) {}
+  InlinerPass() = default;
   ~InlinerPass();
   InlinerPass(InlinerPass &&Arg)
-      : Params(std::move(Arg.Params)),
-        ImportedFunctionsStats(std::move(Arg.ImportedFunctionsStats)) {}
+      : ImportedFunctionsStats(std::move(Arg.ImportedFunctionsStats)) {}
 
   PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
                         LazyCallGraph &CG, CGSCCUpdateResult &UR);
 
 private:
-  InlineParams Params;
+  InlineAdvisor &getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM,
+                            Module &M);
   std::unique_ptr<ImportedFunctionsInliningStatistics> ImportedFunctionsStats;
+  Optional<DefaultInlineAdvisor> OwnedDefaultAdvisor;
 };
 
+/// Module pass, wrapping the inliner pass. This works in conjunction with the
+/// InlineAdvisorAnalysis to facilitate inlining decisions taking into account
+/// module-wide state, that need to keep track of inter-inliner pass runs, for
+/// a given module. An InlineAdvisor is configured and kept alive for the
+/// duration of the ModuleInlinerWrapperPass::run.
+class ModuleInlinerWrapperPass
+    : public PassInfoMixin<ModuleInlinerWrapperPass> {
+public:
+  ModuleInlinerWrapperPass(
+      InlineParams Params = getInlineParams(), bool Debugging = false,
+      InliningAdvisorMode Mode = InliningAdvisorMode::Default,
+      unsigned MaxDevirtIterations = 0);
+  ModuleInlinerWrapperPass(ModuleInlinerWrapperPass &&Arg) = default;
+
+  PreservedAnalyses run(Module &, ModuleAnalysisManager &);
+
+  /// Allow adding more CGSCC passes, besides inlining. This should be called
+  /// before run is called, as part of pass pipeline building.
+  CGSCCPassManager &getPM() { return PM; }
+
+  /// Allow adding module-level analyses benefiting the contained CGSCC passes.
+  template <class T> void addRequiredModuleAnalysis() {
+    MPM.addPass(RequireAnalysisPass<T, Module>());
+  }
+
+private:
+  const InlineParams Params;
+  const InliningAdvisorMode Mode;
+  const unsigned MaxDevirtIterations;
+  const bool Debugging;
+  CGSCCPassManager PM;
+  ModulePassManager MPM;
+};
 } // end namespace llvm
 
 #endif // LLVM_TRANSFORMS_IPO_INLINER_H

llvm/lib/Analysis/InlineAdvisor.cpp

@@ -7,7 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements inlining decision-making APIs.
+// This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and
+// related types.
 //
 //===----------------------------------------------------------------------===//
 
@@ -46,6 +47,126 @@ static cl::opt<int>
                         cl::desc("Scale to limit the cost of inline deferral"),
                         cl::init(-1), cl::Hidden);
 
+namespace {
+class DefaultInlineAdvice : public InlineAdvice {
+public:
+  DefaultInlineAdvice(DefaultInlineAdvisor *Advisor, CallBase &CB,
+                      Optional<InlineCost> OIC, OptimizationRemarkEmitter &ORE)
+      : InlineAdvice(Advisor, CB, OIC.hasValue()), OriginalCB(&CB), OIC(OIC),
+        ORE(ORE), DLoc(CB.getDebugLoc()), Block(CB.getParent()) {}
+
+private:
+  void recordUnsuccessfulInliningImpl(const InlineResult &Result) override {
+    using namespace ore;
+    llvm::setInlineRemark(*OriginalCB, std::string(Result.getFailureReason()) +
+                                           "; " + inlineCostStr(*OIC));
+    ORE.emit([&]() {
+      return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block)
+             << NV("Callee", Callee) << " will not be inlined into "
+             << NV("Caller", Caller) << ": "
+             << NV("Reason", Result.getFailureReason());
+    });
+  }
+
+  void recordInliningWithCalleeDeletedImpl() override {
+    emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, *OIC);
+  }
+
+  void recordInliningImpl() override {
+    emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, *OIC);
+  }
+
+private:
+  CallBase *const OriginalCB;
+  Optional<InlineCost> OIC;
+  OptimizationRemarkEmitter &ORE;
+
+  // Capture the context of CB before inlining, as a successful inlining may
+  // change that context, and we want to report success or failure in the
+  // original context.
+  const DebugLoc DLoc;
+  const BasicBlock *const Block;
+};
+
+} // namespace
+
+std::unique_ptr<InlineAdvice>
+DefaultInlineAdvisor::getAdvice(CallBase &CB, FunctionAnalysisManager &FAM) {
+  Function &Callee = *CB.getCalledFunction();
+  Function &F = *CB.getCaller();
+  ProfileSummaryInfo *PSI = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(F)
+                                .getCachedResult<ProfileSummaryAnalysis>(
+                                    *CB.getParent()->getParent()->getParent());
+
+  auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
+  // FIXME: make GetAssumptionCache's decl similar to the other 2 below. May
+  // need changing the type of getInlineCost parameters? Also see similar case
+  // in Inliner.cpp
+  std::function<AssumptionCache &(Function &)> GetAssumptionCache =
+      [&](Function &F) -> AssumptionCache & {
+    return FAM.getResult<AssumptionAnalysis>(Callee);
+  };
+  auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
+    return FAM.getResult<BlockFrequencyAnalysis>(F);
+  };
+  auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
+    return FAM.getResult<TargetLibraryAnalysis>(F);
+  };
+
+  auto GetInlineCost = [&](CallBase &CB) {
+    Function &Callee = *CB.getCalledFunction();
+    auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
+    bool RemarksEnabled =
+        Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
+            DEBUG_TYPE);
+    return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, {GetBFI},
+                         GetTLI, PSI, RemarksEnabled ? &ORE : nullptr);
+  };
+  auto OIC = llvm::shouldInline(CB, GetInlineCost, ORE);
+  return std::make_unique<DefaultInlineAdvice>(this, CB, OIC, ORE);
+}
+
+InlineAdvice::InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
+                           bool IsInliningRecommended)
+    : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),
+      IsInliningRecommended(IsInliningRecommended) {}
+
+void InlineAdvisor::markFunctionAsDeleted(Function *F) {
+  assert((!DeletedFunctions.count(F)) &&
+         "Cannot put cause a function to become dead twice!");
+  DeletedFunctions.insert(F);
+}
+
+void InlineAdvisor::freeDeletedFunctions() {
+  for (auto *F : DeletedFunctions)
+    delete F;
+  DeletedFunctions.clear();
+}
+
+void InlineAdvice::recordInliningWithCalleeDeleted() {
+  markRecorded();
+  Advisor->markFunctionAsDeleted(Callee);
+  recordInliningWithCalleeDeletedImpl();
+}
+
+AnalysisKey InlineAdvisorAnalysis::Key;
+
+bool InlineAdvisorAnalysis::Result::tryCreate(InlineParams Params,
+                                              InliningAdvisorMode Mode) {
+  switch (Mode) {
+  case InliningAdvisorMode::Default:
+    Advisor.reset(new DefaultInlineAdvisor(Params));
+    break;
+  case InliningAdvisorMode::Development:
+    // To be added subsequently under conditional compilation.
+    break;
+  case InliningAdvisorMode::Release:
+    // To be added subsequently under conditional compilation.
+    break;
+  }
+  return !!Advisor;
+}
+
 /// Return true if inlining of CB can block the caller from being
 /// inlined which is proved to be more beneficial. \p IC is the
 /// estimated inline cost associated with callsite \p CB.

llvm/lib/Passes/PassBuilder.cpp

@@ -33,6 +33,7 @@
 #include "llvm/Analysis/DominanceFrontier.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/IVUsers.h"
+#include "llvm/Analysis/InlineAdvisor.h"
 #include "llvm/Analysis/LazyCallGraph.h"
 #include "llvm/Analysis/LazyValueInfo.h"
 #include "llvm/Analysis/LoopAccessAnalysis.h"
@@ -215,6 +216,16 @@ static cl::opt<bool> EnableGVNHoist(
     "enable-npm-gvn-hoist", cl::init(false), cl::Hidden,
     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
 
+static cl::opt<InliningAdvisorMode> UseInlineAdvisor(
+    "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden,
+    cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
+    cl::values(clEnumValN(InliningAdvisorMode::Default, "default",
+                          "Heuristics-based inliner version."),
+               clEnumValN(InliningAdvisorMode::Development, "development",
+                          "Use development mode (runtime-loadable model)."),
+               clEnumValN(InliningAdvisorMode::Release, "release",
+                          "Use release mode (AOT-compiled model).")));
+
 static cl::opt<bool> EnableGVNSink(
     "enable-npm-gvn-sink", cl::init(false), cl::Hidden,
     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
@@ -614,10 +625,8 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
     // This should probably be lowered after performance testing.
     // FIXME: this comment is cargo culted from the old pass manager, revisit).
     IP.HintThreshold = 325;
-
-    CGSCCPassManager CGPipeline(DebugLogging);
-
-    CGPipeline.addPass(InlinerPass(IP));
+    ModuleInlinerWrapperPass MIWP(IP, DebugLogging);
+    CGSCCPassManager &CGPipeline = MIWP.getPM();
 
     FunctionPassManager FPM;
     FPM.addPass(SROA());
@@ -628,7 +637,7 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
 
     CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
 
-    MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPipeline)));
+    MPM.addPass(std::move(MIWP));
 
     // Delete anything that is now dead to make sure that we don't instrument
     // dead code. Instrumentation can end up keeping dead code around and
@@ -693,41 +702,36 @@ getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) {
   return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel());
 }
 
-ModulePassManager PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
-                                                    ThinLTOPhase Phase,
-                                                    bool DebugLogging) {
-  ModulePassManager MPM(DebugLogging);
+ModuleInlinerWrapperPass
+PassBuilder::buildInlinerPipeline(OptimizationLevel Level, ThinLTOPhase Phase,
+                                  bool DebugLogging) {
+  InlineParams IP = getInlineParamsFromOptLevel(Level);
+  if (Phase == PassBuilder::ThinLTOPhase::PreLink && PGOOpt &&
+      PGOOpt->Action == PGOOptions::SampleUse)
+    IP.HotCallSiteThreshold = 0;
+
+  ModuleInlinerWrapperPass MIWP(IP, DebugLogging, UseInlineAdvisor,
+                                MaxDevirtIterations);
 
   // Require the GlobalsAA analysis for the module so we can query it within
   // the CGSCC pipeline.
-  MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
+  MIWP.addRequiredModuleAnalysis<GlobalsAA>();
 
   // Require the ProfileSummaryAnalysis for the module so we can query it within
   // the inliner pass.
-  MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
+  MIWP.addRequiredModuleAnalysis<ProfileSummaryAnalysis>();
 
   // Now begin the main postorder CGSCC pipeline.
   // FIXME: The current CGSCC pipeline has its origins in the legacy pass
   // manager and trying to emulate its precise behavior. Much of this doesn't
   // make a lot of sense and we should revisit the core CGSCC structure.
-  CGSCCPassManager MainCGPipeline(DebugLogging);
+  CGSCCPassManager &MainCGPipeline = MIWP.getPM();
 
   // Note: historically, the PruneEH pass was run first to deduce nounwind and
   // generally clean up exception handling overhead. It isn't clear this is
   // valuable as the inliner doesn't currently care whether it is inlining an
   // invoke or a call.
 
-  // Run the inliner first. The theory is that we are walking bottom-up and so
-  // the callees have already been fully optimized, and we want to inline them
-  // into the callers so that our optimizations can reflect that.
-  // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
-  // because it makes profile annotation in the backend inaccurate.
-  InlineParams IP = getInlineParamsFromOptLevel(Level);
-  if (Phase == ThinLTOPhase::PreLink && PGOOpt &&
-      PGOOpt->Action == PGOOptions::SampleUse)
-    IP.HotCallSiteThreshold = 0;
-  MainCGPipeline.addPass(InlinerPass(IP));
-
   if (AttributorRun & AttributorRunOption::CGSCC)
     MainCGPipeline.addPass(AttributorCGSCCPass());
 
@@ -755,15 +759,7 @@ ModulePassManager PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
   for (auto &C : CGSCCOptimizerLateEPCallbacks)
     C(MainCGPipeline, Level);
 
-  // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
-  // to detect when we devirtualize indirect calls and iterate the SCC passes
-  // in that case to try and catch knock-on inlining or function attrs
-  // opportunities. Then we add it to the module pipeline by walking the SCCs
-  // in postorder (or bottom-up).
-  MPM.addPass(
-      createModuleToPostOrderCGSCCPassAdaptor(createDevirtSCCRepeatedPass(
-          std::move(MainCGPipeline), MaxDevirtIterations)));
-  return MPM;
+  return MIWP;
 }
 
 ModulePassManager PassBuilder::buildModuleSimplificationPipeline(
@@ -1330,8 +1326,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging,
   // valuable as the inliner doesn't currently care whether it is inlining an
   // invoke or a call.
   // Run the inliner now.
-  MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
-      InlinerPass(getInlineParamsFromOptLevel(Level))));
+  MPM.addPass(ModuleInlinerWrapperPass(getInlineParamsFromOptLevel(Level),
+                                       DebugLogging));
 
   // Optimize globals again after we ran the inliner.
   MPM.addPass(GlobalOptPass());

llvm/lib/Passes/PassRegistry.def

@@ -27,6 +27,7 @@ MODULE_ANALYSIS("stack-safety", StackSafetyGlobalAnalysis())
 MODULE_ANALYSIS("verify", VerifierAnalysis())
 MODULE_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC))
 MODULE_ANALYSIS("asan-globals-md", ASanGlobalsMetadataAnalysis())
+MODULE_ANALYSIS("inline-advisor", InlineAdvisorAnalysis())
 
 #ifndef MODULE_ALIAS_ANALYSIS
 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS)                               \
@@ -57,6 +58,7 @@ MODULE_PASS("hotcoldsplit", HotColdSplittingPass())
 MODULE_PASS("hwasan", HWAddressSanitizerPass(false, false))
 MODULE_PASS("khwasan", HWAddressSanitizerPass(true, true))
 MODULE_PASS("inferattrs", InferFunctionAttrsPass())
+MODULE_PASS("inliner-wrapper", ModuleInlinerWrapperPass())
 MODULE_PASS("insert-gcov-profiling", GCOVProfilerPass())
 MODULE_PASS("instrorderfile", InstrOrderFilePass())
 MODULE_PASS("instrprof", InstrProfiling())

llvm/lib/Transforms/IPO/Inliner.cpp

@@ -17,6 +17,7 @@
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
@@ -28,6 +29,7 @@
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/Analysis/CallGraph.h"
+#include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/InlineAdvisor.h"
 #include "llvm/Analysis/InlineCost.h"
 #include "llvm/Analysis/LazyCallGraph.h"
@@ -664,6 +666,24 @@ InlinerPass::~InlinerPass() {
   }
 }
 
+InlineAdvisor &
+InlinerPass::getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM,
+                        Module &M) {
+  auto *IAA = MAM.getCachedResult<InlineAdvisorAnalysis>(M);
+  if (!IAA) {
+    // It should still be possible to run the inliner as a stand-alone SCC pass,
+    // for test scenarios. In that case, we default to the
+    // DefaultInlineAdvisor, which doesn't need to keep state between SCC pass
+    // runs. It also uses just the default InlineParams.
+    OwnedDefaultAdvisor.emplace(getInlineParams());
+    return *OwnedDefaultAdvisor;
+  }
+  assert(IAA->getAdvisor() &&
+         "Expected a present InlineAdvisorAnalysis also have an "
+         "InlineAdvisor initialized");
+  return *IAA->getAdvisor();
+}
+
 PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
                                    CGSCCAnalysisManager &AM, LazyCallGraph &CG,
                                    CGSCCUpdateResult &UR) {
@@ -675,6 +695,11 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
   Module &M = *InitialC.begin()->getFunction().getParent();
   ProfileSummaryInfo *PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(M);
 
+  InlineAdvisor &Advisor = getAdvisor(MAMProxy, M);
+  Advisor.OnPassEntry();
+
+  auto AdvisorOnExit = make_scope_exit([&] { Advisor.OnPassExit(); });
+
   if (!ImportedFunctionsStats &&
       InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) {
     ImportedFunctionsStats =
@@ -779,29 +804,10 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
 
     LLVM_DEBUG(dbgs() << "Inlining calls in: " << F.getName() << "\n");
 
-    // Get the remarks emission analysis for the caller.
-    auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
-
     std::function<AssumptionCache &(Function &)> GetAssumptionCache =
         [&](Function &F) -> AssumptionCache & {
       return FAM.getResult<AssumptionAnalysis>(F);
     };
-    auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
-      return FAM.getResult<BlockFrequencyAnalysis>(F);
-    };
-    auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
-      return FAM.getResult<TargetLibraryAnalysis>(F);
-    };
-
-    auto GetInlineCost = [&](CallBase &CB) {
-      Function &Callee = *CB.getCalledFunction();
-      auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
-      bool RemarksEnabled =
-          Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
-              DEBUG_TYPE);
-      return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, {GetBFI},
-                           GetTLI, PSI, RemarksEnabled ? &ORE : nullptr);
-    };
 
     // Now process as many calls as we have within this caller in the sequnece.
     // We bail out as soon as the caller has to change so we can update the
@@ -833,101 +839,88 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
         continue;
       }
 
-      auto OIC = shouldInline(*CB, GetInlineCost, ORE);
+      auto Advice = Advisor.getAdvice(*CB, FAM);
       // Check whether we want to inline this callsite.
-      if (!OIC)
-        continue;
-      auto DoInline = [&]() -> InlineResult {
-        // Setup the data structure used to plumb customization into the
-        // `InlineFunction` routine.
-        InlineFunctionInfo IFI(
-            /*cg=*/nullptr, &GetAssumptionCache, PSI,
-            &FAM.getResult<BlockFrequencyAnalysis>(*(CB->getCaller())),
-            &FAM.getResult<BlockFrequencyAnalysis>(Callee));
-
-        InlineResult IR = InlineFunction(*CB, IFI);
-        if (!IR.isSuccess())
-          return IR;
-
-        DidInline = true;
-        InlinedCallees.insert(&Callee);
-        ++NumInlined;
-
-        // Add any new callsites to defined functions to the worklist.
-        if (!IFI.InlinedCallSites.empty()) {
-          int NewHistoryID = InlineHistory.size();
-          InlineHistory.push_back({&Callee, InlineHistoryID});
-
-          for (CallBase *ICB : reverse(IFI.InlinedCallSites)) {
-            Function *NewCallee = ICB->getCalledFunction();
-            if (!NewCallee) {
-              // Try to promote an indirect (virtual) call without waiting for
-              // the post-inline cleanup and the next DevirtSCCRepeatedPass
-              // iteration because the next iteration may not happen and we may
-              // miss inlining it.
-              if (tryPromoteCall(*ICB))
-                NewCallee = ICB->getCalledFunction();
-            }
-            if (NewCallee)
-              if (!NewCallee->isDeclaration())
-                Calls.push_back({ICB, NewHistoryID});
-          }
-        }
-
-        if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No)
-          ImportedFunctionsStats->recordInline(F, Callee);
-
-        // Merge the attributes based on the inlining.
-        AttributeFuncs::mergeAttributesForInlining(F, Callee);
-
-        // For local functions, check whether this makes the callee trivially
-        // dead. In that case, we can drop the body of the function eagerly
-        // which may reduce the number of callers of other functions to one,
-        // changing inline cost thresholds.
-        if (Callee.hasLocalLinkage()) {
-          // To check this we also need to nuke any dead constant uses (perhaps
-          // made dead by this operation on other functions).
-          Callee.removeDeadConstantUsers();
-          if (Callee.use_empty() && !CG.isLibFunction(Callee)) {
-            Calls.erase(
-                std::remove_if(Calls.begin() + I + 1, Calls.end(),
-                               [&](const std::pair<CallBase *, int> &Call) {
-                                 return Call.first->getCaller() == &Callee;
-                               }),
-                Calls.end());
-            // Clear the body and queue the function itself for deletion when we
-            // finish inlining and call graph updates.
-            // Note that after this point, it is an error to do anything other
-            // than use the callee's address or delete it.
-            Callee.dropAllReferences();
-            assert(find(DeadFunctions, &Callee) == DeadFunctions.end() &&
-                   "Cannot put cause a function to become dead twice!");
-            DeadFunctions.push_back(&Callee);
-          }
-        }
-        return IR;
-      };
-      // Capture the context of CB before inlining, as a successful inlining may
-      // change that context, and we want to report success or failure in the
-      // original context.
-      auto DLoc = CB->getDebugLoc();
-      auto *Block = CB->getParent();
-
-      auto Outcome = DoInline();
-      if (!Outcome.isSuccess()) {
-        using namespace ore;
-        setInlineRemark(*CB, std::string(Outcome.getFailureReason()) + "; " +
-                                 inlineCostStr(*OIC));
-        ORE.emit([&]() {
-          return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block)
-                 << NV("Callee", &Callee) << " will not be inlined into "
-                 << NV("Caller", &F) << ": "
-                 << NV("Reason", Outcome.getFailureReason());
-        });
+      if (!Advice->isInliningRecommended()) {
+        Advice->recordUnattemptedInlining();
         continue;
       }
 
-      emitInlinedInto(ORE, DLoc, Block, Callee, F, *OIC);
+      // Setup the data structure used to plumb customization into the
+      // `InlineFunction` routine.
+      InlineFunctionInfo IFI(
+          /*cg=*/nullptr, &GetAssumptionCache, PSI,
+          &FAM.getResult<BlockFrequencyAnalysis>(*(CB->getCaller())),
+          &FAM.getResult<BlockFrequencyAnalysis>(Callee));
+
+      InlineResult IR = InlineFunction(*CB, IFI);
+      if (!IR.isSuccess()) {
+        Advice->recordUnsuccessfulInlining(IR);
+        continue;
+      }
+
+      DidInline = true;
+      InlinedCallees.insert(&Callee);
+      ++NumInlined;
+
+      // Add any new callsites to defined functions to the worklist.
+      if (!IFI.InlinedCallSites.empty()) {
+        int NewHistoryID = InlineHistory.size();
+        InlineHistory.push_back({&Callee, InlineHistoryID});
+
+        for (CallBase *ICB : reverse(IFI.InlinedCallSites)) {
+          Function *NewCallee = ICB->getCalledFunction();
+          if (!NewCallee) {
+            // Try to promote an indirect (virtual) call without waiting for
+            // the post-inline cleanup and the next DevirtSCCRepeatedPass
+            // iteration because the next iteration may not happen and we may
+            // miss inlining it.
+            if (tryPromoteCall(*ICB))
+              NewCallee = ICB->getCalledFunction();
+          }
+          if (NewCallee)
+            if (!NewCallee->isDeclaration())
+              Calls.push_back({ICB, NewHistoryID});
+        }
+      }
+
+      if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No)
+        ImportedFunctionsStats->recordInline(F, Callee);
+
+      // Merge the attributes based on the inlining.
+      AttributeFuncs::mergeAttributesForInlining(F, Callee);
+
+      // For local functions, check whether this makes the callee trivially
+      // dead. In that case, we can drop the body of the function eagerly
+      // which may reduce the number of callers of other functions to one,
+      // changing inline cost thresholds.
+      bool CalleeWasDeleted = false;
+      if (Callee.hasLocalLinkage()) {
+        // To check this we also need to nuke any dead constant uses (perhaps
+        // made dead by this operation on other functions).
+        Callee.removeDeadConstantUsers();
+        if (Callee.use_empty() && !CG.isLibFunction(Callee)) {
+          Calls.erase(
+              std::remove_if(Calls.begin() + I + 1, Calls.end(),
+                             [&](const std::pair<CallBase *, int> &Call) {
+                               return Call.first->getCaller() == &Callee;
+                             }),
+              Calls.end());
+          // Clear the body and queue the function itself for deletion when we
+          // finish inlining and call graph updates.
+          // Note that after this point, it is an error to do anything other
+          // than use the callee's address or delete it.
+          Callee.dropAllReferences();
+          assert(find(DeadFunctions, &Callee) == DeadFunctions.end() &&
+                 "Cannot put cause a function to become dead twice!");
+          DeadFunctions.push_back(&Callee);
+          CalleeWasDeleted = true;
+        }
+      }
+      if (CalleeWasDeleted)
+        Advice->recordInliningWithCalleeDeleted();
+      else
+        Advice->recordInlining();
     }
 
     // Back the call index up by one to put us in a good position to go around
@@ -1008,7 +1001,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
   // sets.
   for (Function *DeadF : DeadFunctions) {
     // Get the necessary information out of the call graph and nuke the
-    // function there. Also, cclear out any cached analyses.
+    // function there. Also, clear out any cached analyses.
     auto &DeadC = *CG.lookupSCC(*CG.lookup(*DeadF));
     FAM.clear(*DeadF, DeadF->getName());
     AM.clear(DeadC, DeadC.getName());
@@ -1021,7 +1014,15 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
     UR.InvalidatedRefSCCs.insert(&DeadRC);
 
     // And delete the actual function from the module.
-    M.getFunctionList().erase(DeadF);
+    // The Advisor may use Function pointers to efficiently index various
+    // internal maps, e.g. for memoization. Function cleanup passes like
+    // argument promotion create new functions. It is possible for a new
+    // function to be allocated at the address of a deleted function. We could
+    // index using names, but that's inefficient. Alternatively, we let the
+    // Advisor free the functions when it sees fit.
+    DeadF->getBasicBlockList().clear();
+    M.getFunctionList().remove(DeadF);
+
     ++NumDeleted;
   }
 
@@ -1034,3 +1035,45 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
   PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
   return PA;
 }
+
+ModuleInlinerWrapperPass::ModuleInlinerWrapperPass(InlineParams Params,
+                                                   bool Debugging,
+                                                   InliningAdvisorMode Mode,
+                                                   unsigned MaxDevirtIterations)
+    : Params(Params), Mode(Mode), MaxDevirtIterations(MaxDevirtIterations),
+      Debugging(Debugging), PM(Debugging), MPM(Debugging) {
+  // Run the inliner first. The theory is that we are walking bottom-up and so
+  // the callees have already been fully optimized, and we want to inline them
+  // into the callers so that our optimizations can reflect that.
+  // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
+  // because it makes profile annotation in the backend inaccurate.
+  PM.addPass(InlinerPass());
+}
+
+PreservedAnalyses ModuleInlinerWrapperPass::run(Module &M,
+                                                ModuleAnalysisManager &MAM) {
+  auto &IAA = MAM.getResult<InlineAdvisorAnalysis>(M);
+  if (!IAA.tryCreate(Params, Mode)) {
+    M.getContext().emitError(
+        "Could not setup Inlining Advisor for the requested "
+        "mode and/or options");
+    return PreservedAnalyses::all();
+  }
+
+  // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
+  // to detect when we devirtualize indirect calls and iterate the SCC passes
+  // in that case to try and catch knock-on inlining or function attrs
+  // opportunities. Then we add it to the module pipeline by walking the SCCs
+  // in postorder (or bottom-up).
+  // If MaxDevirtIterations is 0, we just don't use the devirtualization
+  // wrapper.
+  if (MaxDevirtIterations == 0)
+    MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(PM)));
+  else
+    MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+        createDevirtSCCRepeatedPass(std::move(PM), MaxDevirtIterations)));
+  auto Ret = MPM.run(M, MAM);
+
+  IAA.clear();
+  return Ret;
+}

llvm/test/Other/new-pm-defaults.ll

@@ -127,7 +127,8 @@
 ; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Finished llvm::Function pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Starting llvm::Module pass manager run.
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
 ; CHECK-O-NEXT: Running analysis: GlobalsAA

llvm/test/Other/new-pm-lto-defaults.ll

@@ -73,7 +73,14 @@
 ; CHECK-O2-NEXT: Running analysis: OuterAnalysisManagerProxy
 ; CHECK-EP-Peephole-NEXT: Running pass: NoOpFunctionPass
 ; CHECK-O2-NEXT: Finished llvm::Function pass manager run.
-; CHECK-O2-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}InlinerPass>
+; CHECK-O2-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O2-NEXT: Running analysis: InlineAdvisorAnalysis
+; CHECK-O2-NEXT: Starting llvm::Module pass manager run.
+; CHECK-O2-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}PassManager{{.*}}>
+; CHECK-O2-NEXT: Starting CGSCC pass manager run.
+; CHECK-O2-NEXT: Running pass: InlinerPass
+; CHECK-O2-NEXT: Finished CGSCC pass manager run.
+; CHECK-O2-NEXT: Finished llvm::Module pass manager run.
 ; CHECK-O2-NEXT: Running pass: GlobalOptPass
 ; CHECK-O2-NEXT: Running pass: GlobalDCEPass
 ; CHECK-O2-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>

llvm/test/Other/new-pm-thinlto-defaults.ll

@@ -92,7 +92,8 @@
 ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Finished llvm::Function pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Starting llvm::Module pass manager run.
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
 ; CHECK-O-NEXT: Running analysis: GlobalsAA

llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll

@@ -66,7 +66,8 @@
 ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis on foo
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Finished {{.*}}Function pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
 ; CHECK-O-NEXT: Running analysis: GlobalsAA

llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll

@@ -75,7 +75,8 @@
 ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis on foo
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass on foo
 ; CHECK-O-NEXT: Finished {{.*}}Function pass manager run
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
 ; CHECK-O-NEXT: Running analysis: GlobalsAA

llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll

@@ -1,7 +1,7 @@
 ; Validate ThinLTO prelink pipeline when we have instrumentation PGO
 ;
 ; RUN: llvm-profdata merge %S/Inputs/new-pm-thinlto-prelink-pgo-defaults.proftext -o %t.profdata
-; 
+;
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' \
 ; RUN:     -passes='thinlto-pre-link<O1>,name-anon-globals' -S %s 2>&1 \
@@ -65,6 +65,9 @@
 ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Finished {{.*}}Function pass manager run.
+; CHECK-O123-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O123-NEXT: Running analysis: InlineAdvisorAnalysis
+; CHECK-O123-NEXT: Starting {{.*}}Module pass manager run.
 ; CHECK-O123-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}PassManager<{{.*}}LazyCallGraph::SCC
 ; CHECK-O123-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O123-NEXT: Running analysis: LazyCallGraphAnalysis
@@ -75,6 +78,7 @@
 ; CHECK-O123-NEXT: Running pass: InlinerPass on (foo)
 ; CHECK-O123-NEXT: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
 ; CHECK-O123-NEXT: Finished CGSCC pass manager run.
+; CHECK-O123-NEXT: Finished {{.*}}Module pass manager run.
 ; CHECK-O123-NEXT: Running pass: GlobalDCEPass
 ; CHECK-O-NEXT: Running pass: PGOInstrumentationUse
 ; CHECK-O-NEXT: Running analysis: ProfileSummaryAnalysis
@@ -93,7 +97,9 @@
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis on foo
 ; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis on foo
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-Os-NEXT: Running analysis: InlineAdvisorAnalysis
+; CHECK-Oz-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
 ; CHECK-O-NEXT: Running analysis: GlobalsAA

llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll

@@ -75,7 +75,8 @@
 ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis on foo
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass on foo
 ; CHECK-O-NEXT: Finished {{.*}}Function pass manager run
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
 ; CHECK-O-NEXT: Running analysis: GlobalsAA

llvm/test/Other/scc-deleted-printer.ll

@@ -3,6 +3,11 @@
 ; RUN: opt < %s 2>&1 -disable-output \
 ; RUN: 	   -passes=inline -print-before-all -print-after-all -print-module-scope | FileCheck %s -check-prefix=INL-MOD
 
+; RUN: opt < %s 2>&1 -disable-output \
+; RUN: 	   -passes=inliner-wrapper -print-before-all -print-after-all | FileCheck %s -check-prefix=INL
+; RUN: opt < %s 2>&1 -disable-output \
+; RUN: 	   -passes=inliner-wrapper -print-before-all -print-after-all -print-module-scope | FileCheck %s -check-prefix=INL-MOD
+
 ; INL: IR Dump Before {{InlinerPass .*scc: .tester, foo}}
 ; INL-NOT: IR Dump After {{InlinerPass}}
 ; INL: IR Dump Before {{InlinerPass .*scc: .tester}}

llvm/test/Other/scc-pass-printer.ll

@@ -3,9 +3,13 @@
 ; RUN: opt < %s 2>&1 -disable-output \
 ; RUN: 	   -passes=inline -print-after-all | FileCheck %s -check-prefix=INL
 ; RUN: opt < %s 2>&1 -disable-output \
+; RUN: 	   -passes=inliner-wrapper -print-after-all | FileCheck %s -check-prefix=INL
+; RUN: opt < %s 2>&1 -disable-output \
 ; RUN: 	   -inline -print-after-all -print-module-scope | FileCheck %s -check-prefix=INL-MOD
 ; RUN: opt < %s 2>&1 -disable-output \
 ; RUN: 	   -passes=inline -print-after-all -print-module-scope | FileCheck %s -check-prefix=INL-MOD
+; RUN: opt < %s 2>&1 -disable-output \
+; RUN: 	   -passes=inliner-wrapper -print-after-all -print-module-scope | FileCheck %s -check-prefix=INL-MOD
 
 ; INL: IR Dump After {{Function Integration/Inlining|InlinerPass .*scc: .bar, foo}}
 ; INL: define void @bar()

llvm/test/Transforms/Inline/inline_stats.ll

@@ -6,6 +6,9 @@
 ; RUN: opt -S -passes=inline -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=CHECK-BASIC -check-prefix=CHECK
 ; RUN: opt -S -passes=inline -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix="CHECK-VERBOSE" -check-prefix=CHECK
 
+; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=CHECK-BASIC -check-prefix=CHECK
+; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix="CHECK-VERBOSE" -check-prefix=CHECK
+
 ; CHECK: ------- Dumping inliner stats for [<stdin>] -------
 ; CHECK-BASIC-NOT: -- List of inlined functions:
 ; CHECK-BASIC-NOT: -- Inlined not imported function

llvm/test/Transforms/Inline/inlining-advisor-default.ll

@@ -0,0 +1,9 @@
+; Check that, in the absence of dependencies, we emit an error message when
+; trying to use ML-driven inlining.
+;
+; RUN: not opt -passes=scc-oz-module-inliner -enable-ml-inliner=development -S < %s 2>&1 | FileCheck %s
+; RUN: not opt -passes=scc-oz-module-inliner -enable-ml-inliner=release -S < %s 2>&1 | FileCheck %s
+
+declare i64 @f1()
+
+; CHECK: Could not setup Inlining Advisor for the requested mode and/or options

llvm/test/Transforms/Inline/internal-scc-members.ll

@@ -3,6 +3,7 @@
 ;
 ; RUN: opt < %s -S -inline | FileCheck %s
 ; RUN: opt < %s -S -passes=inline | FileCheck %s
+; RUN: opt < %s -S -passes=inliner-wrapper | FileCheck %s
 
 ; CHECK-LABEL: define internal void @test1_scc0()
 ; CHECK-NOT: call

llvm/test/Transforms/Inline/module-inlining.ll

@@ -6,6 +6,7 @@
 ; a 'ret 10'
 ;
 ; RUN: opt -passes=inline -S < %s | FileCheck %s --check-prefix=INLINE --check-prefix=CHECK
+; RUN: opt -passes=inliner-wrapper -S < %s | FileCheck %s --check-prefix=INLINE --check-prefix=CHECK
 ; RUN: opt -passes=scc-oz-module-inliner -S < %s | FileCheck %s --check-prefix=MODULE --check-prefix=CHECK
 
 define void @modify_value({i32, float}* %v) {

llvm/test/Transforms/Inline/monster_scc.ll

@@ -41,6 +41,7 @@
 ;
 ; RUN: opt -S < %s -inline -inline-threshold=150 | FileCheck %s --check-prefixes=CHECK,OLD
 ; RUN: opt -S < %s -passes=inline -inline-threshold=150 | FileCheck %s --check-prefixes=CHECK,NEW
+; RUN: opt -S < %s -passes=inliner-wrapper -inline-threshold=150 | FileCheck %s --check-prefixes=CHECK,NEW
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 

llvm/test/Transforms/Inline/optimization-remarks-hotness-threshold.ll

@@ -5,6 +5,10 @@
 ; RUN:    -pass-remarks-with-hotness -pass-remarks-hotness-threshold=1 2>&1 | \
 ; RUN:    FileCheck -allow-empty -check-prefix=THRESHOLD %s
 
+; RUN: opt < %s -S -passes=inliner-wrapper -pass-remarks-output=%t -pass-remarks=inline \
+; RUN:    -pass-remarks-with-hotness -pass-remarks-hotness-threshold=1 2>&1 | \
+; RUN:    FileCheck -allow-empty -check-prefix=THRESHOLD %s
+
 ; Check that when any threshold is specified we ignore remarks with no
 ; hotness -- these are blocks that have not been executed during training.
 

llvm/test/Transforms/Inline/optimization-remarks-passed-yaml.ll

@@ -8,6 +8,11 @@
 ; RUN:    -pass-remarks-with-hotness 2>&1 | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=YAML %s
 
+; RUN: opt < %s -S -passes=inliner-wrapper -pass-remarks-output=%t -pass-remarks=inline \
+; RUN:    -pass-remarks-missed=inline -pass-remarks-analysis=inline \
+; RUN:    -pass-remarks-with-hotness 2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+
 ; Check the YAML file for inliner-generated passed and analysis remarks.  This
 ; is the input:
 

llvm/test/Transforms/Inline/optimization-remarks-with-hotness.ll

@@ -4,6 +4,9 @@
 ; RUN: opt < %s -passes=inline -pass-remarks=inline -pass-remarks-missed=inline \
 ; RUN:     -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 \
 ; RUN:     | FileCheck %s
+; RUN: opt < %s -passes=inliner-wrapper -pass-remarks=inline -pass-remarks-missed=inline \
+; RUN:     -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 \
+; RUN:     | FileCheck %s
 
 ; CHECK: foo inlined into bar with (cost=always): always inline attribute (hotness: 30)
 ; CHECK: foz not inlined into bar because it should never be inlined (cost=never): noinline function attribute (hotness: 30)

llvm/test/Transforms/Inline/optimization-remarks-yaml.ll

@@ -34,6 +34,25 @@
 ; RUN: opt < %s -S -passes=inline \
 ; RUN:     -pass-remarks-with-hotness -pass-remarks-hotness-threshold 100 \
 ; RUN:     -pass-remarks-output=%t.threshold
+
+; Inliner - Module Wrapper
+; RUN: opt < %s -S -passes=inliner-wrapper -pass-remarks-missed=inline \
+; RUN:     -pass-remarks-with-hotness -pass-remarks-hotness-threshold 15 \
+; RUN:     -pass-remarks-output=%t 2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+; RUN: opt < %s -S -passes=inliner-wrapper -pass-remarks-with-hotness -pass-remarks-output=%t
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+;
+; Verify that remarks that don't meet the hotness threshold are not output.
+; RUN: opt < %s -S -passes=inliner-wrapper -pass-remarks-missed=inline \
+; RUN:     -pass-remarks-with-hotness -pass-remarks-hotness-threshold 100 \
+; RUN:     -pass-remarks-output=%t.threshold 2>&1 | \
+; RUN:     FileCheck -check-prefix=THRESHOLD %s
+; RUN: test ! -s %t.threshold
+; RUN: opt < %s -S -passes=inliner-wrapper \
+; RUN:     -pass-remarks-with-hotness -pass-remarks-hotness-threshold 100 \
+; RUN:     -pass-remarks-output=%t.threshold
+
 ; The remarks output file should be empty.
 ; RUN: test ! -s %t.threshold
 

llvm/test/Transforms/Inline/optimization-remarks.ll

@@ -12,6 +12,13 @@
 ; RUN:       -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 | \
 ; RUN:       FileCheck -check-prefix=CHECK -check-prefix=HOTNESS %s
 
+; RUN: opt < %s -passes=inliner-wrapper -pass-remarks=inline -pass-remarks-missed=inline \
+; RUN:       -pass-remarks-analysis=inline -S 2>&1 | \
+; RUN:       FileCheck -check-prefix=CHECK -check-prefix=NO_HOTNESS %s
+; RUN: opt < %s -passes=inliner-wrapper -pass-remarks=inline -pass-remarks-missed=inline \
+; RUN:       -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 | \
+; RUN:       FileCheck -check-prefix=CHECK -check-prefix=HOTNESS %s
+
 ; HOTNESS: fox will not be inlined into bar because its definition is unavailable
 ; NO_HOTNESS-NOT: fox will not be inlined into bar because its definition is unavailable
 ; CHECK: foo inlined into bar with (cost=always): always inline attribute