[Inliner] Run always-inliner in inliner-wrapper

An alwaysinline function may not get inlined in inliner-wrapper due to
the inlining order.

Previously for the following, the inliner would first inline @a() into @b(),

```
define void @a() {
entry:
  call void @b()
  ret void
}

define void @b() alwaysinline {
entry:
  br label %for.cond

for.cond:
  call void @a()
  br label %for.cond
}
```

making @b() recursive and unable to be inlined into @a(), ending at

```
define void @a() {
entry:
  call void @b()
  ret void
}

define void @b() alwaysinline {
entry:
  br label %for.cond

for.cond:
  call void @b()
  br label %for.cond
}
```

Running always-inliner first makes sure that we respect alwaysinline in more cases.

Fixes https://bugs.llvm.org/show_bug.cgi?id=46945.

Reviewed By: davidxl, rnk

Differential Revision: https://reviews.llvm.org/D86988

llvm/lib/Transforms/IPO/Inliner.cpp

@@ -58,6 +58,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/IPO/AlwaysInliner.h"
 #include "llvm/Transforms/Utils/CallPromotionUtils.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h"
@@ -91,6 +92,11 @@ static cl::opt<bool>
     DisableInlinedAllocaMerging("disable-inlined-alloca-merging",
                                 cl::init(false), cl::Hidden);
 
+/// Flag to disable adding AlwaysInlinerPass to ModuleInlinerWrapperPass.
+/// TODO: remove this once this has is baked in for long enough.
+static cl::opt<bool> DisableAlwaysInlinerInModuleWrapper(
+    "disable-always-inliner-in-module-wrapper", cl::init(false), cl::Hidden);
+
 namespace {
 
 enum class InlinerFunctionImportStatsOpts {
@@ -1055,6 +1061,8 @@ PreservedAnalyses ModuleInlinerWrapperPass::run(Module &M,
     return PreservedAnalyses::all();
   }
 
+  if (!DisableAlwaysInlinerInModuleWrapper)
+    MPM.addPass(AlwaysInlinerPass());
   // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
   // to detect when we devirtualize indirect calls and iterate the SCC passes
   // in that case to try and catch knock-on inlining or function attrs

llvm/test/Other/new-pm-defaults.ll

@@ -126,6 +126,7 @@
 ; CHECK-O-NEXT: Running analysis: CallGraphAnalysis
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
 ; CHECK-O-NEXT: Running analysis: ProfileSummaryAnalysis
+; CHECK-O-NEXT: Running pass: AlwaysInlinerPass
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy

llvm/test/Other/new-pm-lto-defaults.ll

@@ -66,6 +66,7 @@
 ; CHECK-O2-NEXT: Running pass: ModuleInlinerWrapperPass
 ; CHECK-O2-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O2-NEXT: Starting llvm::Module pass manager run.
+; CHECK-O2-NEXT: Running pass: AlwaysInlinerPass
 ; CHECK-O2-NEXT: Starting CGSCC pass manager run.
 ; CHECK-O2-NEXT: Running pass: InlinerPass
 ; CHECK-O2-NEXT: Finished CGSCC pass manager run.

llvm/test/Other/new-pm-module-inliner-wrapper.ll

@@ -0,0 +1,10 @@
+; RUN: opt -disable-verify -debug-pass-manager -passes='inliner-wrapper' -S %s 2>&1 | FileCheck %s --check-prefixes=CHECK,ALWAYS
+; RUN: opt -disable-verify -disable-always-inliner-in-module-wrapper -debug-pass-manager -passes='inliner-wrapper' -S %s 2>&1 | FileCheck %s --check-prefixes=CHECK,DISABLEALWAYS
+
+; DISABLEALWAYS-NOT: Running pass: AlwaysInlinerPass
+; ALWAYS: Running pass: AlwaysInlinerPass
+; CHECK: Running pass: InlinerPass
+
+define void @foo() {
+  ret void
+}

llvm/test/Other/new-pm-thinlto-defaults.ll

@@ -91,6 +91,7 @@
 ; CHECK-O-NEXT: Running analysis: CallGraphAnalysis
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
 ; CHECK-PRELINK-O-NEXT: Running analysis: ProfileSummaryAnalysis
+; CHECK-O-NEXT: Running pass: AlwaysInlinerPass
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy

llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll

@@ -65,6 +65,7 @@
 ; CHECK-O-NEXT: Running analysis: GlobalsAA
 ; CHECK-O-NEXT: Running analysis: CallGraphAnalysis
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
+; CHECK-O-NEXT: Running pass: AlwaysInlinerPass
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy

llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll

@@ -73,6 +73,7 @@
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
 ; CHECK-O-NEXT: Running analysis: GlobalsAA
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
+; CHECK-O-NEXT: Running pass: AlwaysInlinerPass
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy

llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll

@@ -59,6 +59,8 @@
 ; CHECK-O123-NEXT: Running pass: ModuleInlinerWrapperPass
 ; CHECK-O123-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O123-NEXT: Starting {{.*}}Module pass manager run.
+; CHECK-O123-NEXT: Running pass: AlwaysInlinerPass
+; CHECK-O123-NEXT: Running analysis: ProfileSummaryAnalysis
 ; CHECK-O123-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O123-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O123-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy on (foo)
@@ -73,7 +75,6 @@
 ; CHECK-O123-NEXT: Finished {{.*}}Module pass manager run.
 ; CHECK-O123-NEXT: Running pass: GlobalDCEPass
 ; CHECK-O-NEXT: Running pass: PGOInstrumentationUse
-; CHECK-O-NEXT: Running analysis: ProfileSummaryAnalysis
 ; These next two can appear in any order since they are accessed as parameters
 ; on the same call to BlockFrequencyInfo::calculate.
 ; CHECK-O-DAG: Running analysis: BranchProbabilityAnalysis on foo
@@ -95,6 +96,7 @@
 ; CHECK-O-NEXT: Running analysis: GlobalsAA
 ; CHECK-O-NEXT: Running analysis: CallGraphAnalysis
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
+; CHECK-O-NEXT: Running pass: AlwaysInlinerPass
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis on foo

llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll

@@ -72,6 +72,7 @@
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
 ; CHECK-O-NEXT: Running analysis: GlobalsAA
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
+; CHECK-O-NEXT: Running pass: AlwaysInlinerPass
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy

llvm/test/Other/scc-deleted-printer.ll

@@ -3,11 +3,6 @@
 ; RUN: opt < %s 2>&1 -disable-output \
 ; RUN: 	   -passes=inline -print-before-all -print-after-all -print-module-scope | FileCheck %s -check-prefix=INL-MOD
 
-; RUN: opt < %s 2>&1 -disable-output \
-; RUN: 	   -passes=inliner-wrapper -print-before-all -print-after-all | FileCheck %s -check-prefix=INL
-; RUN: opt < %s 2>&1 -disable-output \
-; RUN: 	   -passes=inliner-wrapper -print-before-all -print-after-all -print-module-scope | FileCheck %s -check-prefix=INL-MOD
-
 ; INL: IR Dump Before {{InlinerPass .*scc: .tester, foo}}
 ; INL-NOT: IR Dump After {{InlinerPass}}
 ; INL: IR Dump Before {{InlinerPass .*scc: .tester}}

llvm/test/Transforms/Inline/inline_stats.ll

@@ -6,8 +6,8 @@
 ; RUN: opt -S -passes=inline -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=CHECK-BASIC -check-prefix=CHECK
 ; RUN: opt -S -passes=inline -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix="CHECK-VERBOSE" -check-prefix=CHECK
 
-; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=CHECK-BASIC -check-prefix=CHECK
-; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix="CHECK-VERBOSE" -check-prefix=CHECK
+; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=WRAPPER-BASIC -check-prefix=WRAPPER
+; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix=WRAPPER-VERBOSE -check-prefix=WRAPPER
 
 ; CHECK: ------- Dumping inliner stats for [<stdin>] -------
 ; CHECK-BASIC-NOT: -- List of inlined functions:
@@ -27,6 +27,16 @@
 ; CHECK: non-imported functions inlined anywhere: 1 [33.33% of non-imported functions]
 ; CHECK: non-imported functions inlined into importing module: 1 [33.33% of non-imported functions]
 
+; WRAPPER-VERBOSE: -- List of inlined functions:
+; WRAPPER-VERBOSE: Inlined imported function [external5]: #inlines = 1, #inlines_to_importing_module = 1
+; WRAPPER: -- Summary:
+; WRAPPER: All functions: 10, imported functions: 7
+; WRAPPER: inlined functions: 1 [10% of all functions]
+; WRAPPER: imported functions inlined anywhere: 1 [14.29% of imported functions]
+; WRAPPER: imported functions inlined into importing module: 1 [14.29% of imported functions], remaining: 6 [85.71% of imported functions]
+; WRAPPER: non-imported functions inlined anywhere: 0 [0% of non-imported functions]
+; WRAPPER: non-imported functions inlined into importing module: 0 [0% of non-imported functions]
+
 define void @internal() {
     call fastcc void @external1()
     call fastcc void @internal2()

llvm/test/Transforms/Inline/pr46945.ll

@@ -0,0 +1,18 @@
+; RUN: opt %s -o - -S -passes=inliner-wrapper | FileCheck %s
+
+; CHECK-NOT: call void @b()
+define void @a() {
+entry:
+  call void @b()
+  ret void
+}
+
+define void @b() alwaysinline {
+entry:
+  br label %for.cond
+
+for.cond:
+  call void @a()
+  br label %for.cond
+}
+