SelectionDAGBuilder, mach-o: Skip trap after noreturn call (for Mach-O)

Add NoTrapAfterNoreturn target option which skips emission of traps
behind noreturn calls even if TrapUnreachable is enabled.

Enable the feature on Mach-O to save code size; Comments suggest it is
not possible to enable it for the other users of TrapUnreachable.

rdar://41530228

DifferentialRevision: https://reviews.llvm.org/D48674
llvm-svn: 335877

llvm/include/llvm/Target/TargetOptions.h

@@ -108,6 +108,7 @@ namespace llvm {
           DisableIntegratedAS(false), RelaxELFRelocations(false),
           FunctionSections(false), DataSections(false),
           UniqueSectionNames(true), TrapUnreachable(false),
+          NoTrapAfterNoreturn(false),
           EmulatedTLS(false), ExplicitEmulatedTLS(false),
           EnableIPRA(false), EmitStackSizeSection(false) {}
 
@@ -213,6 +214,10 @@ namespace llvm {
     /// Emit target-specific trap instruction for 'unreachable' IR instructions.
     unsigned TrapUnreachable : 1;
 
+    /// Do not emit a trap instruction for 'unreachable' IR instructions behind
+    /// noreturn calls, even if TrapUnreachable is true.
+    unsigned NoTrapAfterNoreturn : 1;
+
     /// EmulatedTLS - This flag enables emulated TLS model, using emutls
     /// function in the runtime library..
     unsigned EmulatedTLS : 1;

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

@@ -2625,9 +2625,23 @@ void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
 }
 
 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
-  if (DAG.getTarget().Options.TrapUnreachable)
-    DAG.setRoot(
-        DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
+  if (!DAG.getTarget().Options.TrapUnreachable)
+    return;
+
+  // We may be able to ignore unreachable behind a noreturn call.
+  if (DAG.getTarget().Options.NoTrapAfterNoreturn) {
+    const BasicBlock &BB = *I.getParent();
+    if (&I != &BB.front()) {
+      BasicBlock::const_iterator PredI =
+        std::prev(BasicBlock::const_iterator(&I));
+      if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
+        if (Call->doesNotReturn())
+          return;
+      }
+    }
+  }
+
+  DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
 }
 
 void SelectionDAGBuilder::visitFSub(const User &I) {

llvm/lib/Target/AArch64/AArch64TargetMachine.cpp

@@ -244,8 +244,10 @@ AArch64TargetMachine::AArch64TargetMachine(const Target &T, const Triple &TT,
       TLOF(createTLOF(getTargetTriple())), isLittle(LittleEndian) {
   initAsmInfo();
 
-  if (TT.isOSBinFormatMachO())
+  if (TT.isOSBinFormatMachO()) {
     this->Options.TrapUnreachable = true;
+    this->Options.NoTrapAfterNoreturn = true;
+  }
 
   // Enable GlobalISel at or below EnableGlobalISelAt0.
   if (getOptLevel() <= EnableGlobalISelAtO)

llvm/lib/Target/ARM/ARMTargetMachine.cpp

@@ -239,8 +239,10 @@ ARMBaseTargetMachine::ARMBaseTargetMachine(const Target &T, const Triple &TT,
       this->Options.EABIVersion = EABI::EABI5;
   }
 
-  if (TT.isOSBinFormatMachO())
+  if (TT.isOSBinFormatMachO()) {
     this->Options.TrapUnreachable = true;
+    this->Options.NoTrapAfterNoreturn = true;
+  }
 
   initAsmInfo();
 }

llvm/lib/Target/X86/X86TargetMachine.cpp

@@ -229,8 +229,10 @@ X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT,
   // to ever want to mix 32 and 64-bit windows code in a single module
   // this should be fine.
   if ((TT.isOSWindows() && TT.getArch() == Triple::x86_64) || TT.isPS4() ||
-      TT.isOSBinFormatMachO())
+      TT.isOSBinFormatMachO()) {
     this->Options.TrapUnreachable = true;
+    this->Options.NoTrapAfterNoreturn = TT.isOSBinFormatMachO();
+  }
 
   initAsmInfo();
 }

llvm/test/CodeGen/X86/shadow-stack.ll

@@ -64,7 +64,6 @@ define i32 @bar(i32 %i) local_unnamed_addr {
 ; X86_64-NEXT:    movq 8(%rax), %rcx
 ; X86_64-NEXT:    movq 16(%rax), %rsp
 ; X86_64-NEXT:    jmpq *%rcx
-; X86_64-NEXT:    ud2
 ;
 ; X86-LABEL: bar:
 ; X86:       ## %bb.0: ## %entry
@@ -99,7 +98,6 @@ define i32 @bar(i32 %i) local_unnamed_addr {
 ; X86-NEXT:    movl 4(%eax), %ecx
 ; X86-NEXT:    movl 8(%eax), %esp
 ; X86-NEXT:    jmpl *%ecx
-; X86-NEXT:    ud2
 entry:
   %0 = load i8*, i8** @buf, align 8
   tail call void @llvm.eh.sjlj.longjmp(i8* %0)

llvm/test/CodeGen/X86/unreachable-trap.ll

@@ -0,0 +1,29 @@
+; RUN: llc -o - %s -mtriple=x86_64-windows-msvc | FileCheck %s --check-prefixes=CHECK,TRAP_AFTER_NORETURN
+; RUN: llc -o - %s -mtriple=x86_64-apple-darwin | FileCheck %s --check-prefixes=CHECK,NO_TRAP_AFTER_NORETURN
+
+; CHECK-LABEL: call_exit:
+; CHECK: callq {{_?}}exit
+; TRAP_AFTER_NORETURN: ud2
+; NO_TRAP_AFTER_NORETURN-NOT: ud2
+define i32 @call_exit() noreturn nounwind {
+  tail call void @exit(i32 0)
+  unreachable
+}
+
+; CHECK-LABEL: trap:
+; CHECK: ud2
+; TRAP_AFTER_NORETURN: ud2
+; NO_TRAP_AFTER_NORETURN-NOT: ud2
+define i32 @trap() noreturn nounwind {
+  tail call void @llvm.trap()
+  unreachable
+}
+
+; CHECK-LABEL: unreachable:
+; CHECK: ud2
+define i32 @unreachable() noreturn nounwind {
+  unreachable
+}
+
+declare void @llvm.trap() nounwind noreturn
+declare void @exit(i32 %rc) nounwind noreturn