[MS ABI] Don't crash while mangling recursive lambdas

We might get into bad situations where we try to embed the signature of
an inner lambda into an outer lambda which cannot work: the inner lambda
wants to embed the name of the outer lambda!

Instead, omit the return type for lambdas.

This fixes PR26105.

N.B.  While we are here, make lambdas nested within functions use an
artificial scope so that they can get demangled.

llvm-svn: 258003

clang/lib/AST/MicrosoftMangle.cpp

@@ -160,14 +160,17 @@ public:
                              raw_ostream &Out) override;
   void mangleStringLiteral(const StringLiteral *SL, raw_ostream &Out) override;
   bool getNextDiscriminator(const NamedDecl *ND, unsigned &disc) {
-    // Lambda closure types are already numbered.
-    if (isLambda(ND))
-      return false;
-
     const DeclContext *DC = getEffectiveDeclContext(ND);
     if (!DC->isFunctionOrMethod())
       return false;
 
+    // Lambda closure types are already numbered, give out a phony number so
+    // that they demangle nicely.
+    if (isLambda(ND)) {
+      disc = 1;
+      return true;
+    }
+
     // Use the canonical number for externally visible decls.
     if (ND->isExternallyVisible()) {
       disc = getASTContext().getManglingNumber(ND);
@@ -1799,9 +1802,12 @@ void MicrosoftCXXNameMangler::mangleFunctionType(const FunctionType *T,
   SourceRange Range;
   if (D) Range = D->getSourceRange();
 
+  bool IsInLambda = false;
   bool IsStructor = false, HasThisQuals = ForceThisQuals, IsCtorClosure = false;
   CallingConv CC = T->getCallConv();
   if (const CXXMethodDecl *MD = dyn_cast_or_null<CXXMethodDecl>(D)) {
+    if (MD->getParent()->isLambda())
+      IsInLambda = true;
     if (MD->isInstance())
       HasThisQuals = true;
     if (isa<CXXDestructorDecl>(MD)) {
@@ -1875,6 +1881,8 @@ void MicrosoftCXXNameMangler::mangleFunctionType(const FunctionType *T,
              "shouldn't need to mangle __auto_type!");
       mangleSourceName(AT->isDecltypeAuto() ? "<decltype-auto>" : "<auto>");
       Out << '@';
+    } else if (IsInLambda) {
+      Out << '@';
     } else {
       if (ResultType->isVoidType())
         ResultType = ResultType.getUnqualifiedType();

clang/test/CodeGenCXX/exceptions-seh-filter-captures.cpp

@@ -70,14 +70,14 @@ void test_lambda() {
   lambda();
 }
 
-// CHECK-LABEL: define internal void @"\01??R<lambda_0>@?test_lambda@@YAXXZ@QEBAXXZ"(%class.anon* %this)
+// CHECK-LABEL: define internal void @"\01??R<lambda_0>@?0??test_lambda@@YAXXZ@QEBA@XZ"(%class.anon* %this)
 // CHECK: @llvm.localescape(i32* %[[l2_addr:[^, ]*]])
 // CHECK: store i32 42, i32* %[[l2_addr]], align 4
 // CHECK: invoke void @might_crash()
 
-// CHECK-LABEL: define internal i32 @"\01?filt$0@0@?R<lambda_0>@?test_lambda@@YAXXZ@"(i8* %exception_pointers, i8* %frame_pointer)
-// CHECK: %[[fp:[^ ]*]] = call i8* @llvm.x86.seh.recoverfp(i8* bitcast (void (%class.anon*)* @"\01??R<lambda_0>@?test_lambda@@YAXXZ@QEBAXXZ" to i8*), i8* %frame_pointer)
-// CHECK: %[[l2_i8:[^ ]*]] = call i8* @llvm.localrecover(i8* bitcast (void (%class.anon*)* @"\01??R<lambda_0>@?test_lambda@@YAXXZ@QEBAXXZ" to i8*), i8* %[[fp]], i32 0)
+// CHECK-LABEL: define internal i32 @"\01?filt$0@0@?R<lambda_0>@?0??test_lambda@@YAXXZ@"(i8* %exception_pointers, i8* %frame_pointer)
+// CHECK: %[[fp:[^ ]*]] = call i8* @llvm.x86.seh.recoverfp(i8* bitcast (void (%class.anon*)* @"\01??R<lambda_0>@?0??test_lambda@@YAXXZ@QEBA@XZ" to i8*), i8* %frame_pointer)
+// CHECK: %[[l2_i8:[^ ]*]] = call i8* @llvm.localrecover(i8* bitcast (void (%class.anon*)* @"\01??R<lambda_0>@?0??test_lambda@@YAXXZ@QEBA@XZ" to i8*), i8* %[[fp]], i32 0)
 // CHECK: %[[l2_ptr:[^ ]*]] = bitcast i8* %[[l2_i8]] to i32*
 // CHECK: %[[l2:[^ ]*]] = load i32, i32* %[[l2_ptr]]
 // CHECK: call i32 (i32, ...) @basic_filter(i32 %[[l2]])

clang/test/CodeGenCXX/exceptions-seh.cpp

@@ -95,7 +95,7 @@ void use_seh_in_lambda() {
 // NOCXX-NOT: invoke
 // NOCXX: ret void
 
-// CHECK-LABEL: define internal void @"\01??R<lambda_0>@?use_seh_in_lambda@@YAXXZ@QEBAXXZ"(%class.anon* %this)
+// CHECK-LABEL: define internal void @"\01??R<lambda_0>@?0??use_seh_in_lambda@@YAXXZ@QEBA@XZ"(%class.anon* %this)
 // CXXEH-SAME:  personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*)
 // CHECK: invoke void @might_throw() #[[NOINLINE]]
 // CHECK: catchpad

clang/test/CodeGenCXX/mangle-ms-cxx11.cpp

@@ -166,14 +166,14 @@ inline int define_lambda() {
   static auto lambda = [] { static int local; ++local; return local; };
 // First, we have the static local variable of type "<lambda_1>" inside of
 // "define_lambda".
-// CHECK-DAG: @"\01?lambda@?1??define_lambda@@YAHXZ@4V<lambda_1>@?1@YAHXZ@A"
+// CHECK-DAG: @"\01?lambda@?1??define_lambda@@YAHXZ@4V<lambda_1>@?0??1@YAHXZ@A"
 // Next, we have the "operator()" for "<lambda_1>" which is inside of
 // "define_lambda".
-// CHECK-DAG: @"\01??R<lambda_1>@?define_lambda@@YAHXZ@QBEHXZ"
+// CHECK-DAG: @"\01??R<lambda_1>@?0??define_lambda@@YAHXZ@QBE@XZ"
 // Finally, we have the local which is inside of "<lambda_1>" which is inside of
 // "define_lambda". Hooray.
-// MSVC2013-DAG: @"\01?local@?2???R<lambda_1>@?define_lambda@@YAHXZ@QBEHXZ@4HA"
-// MSVC2015-DAG: @"\01?local@?1???R<lambda_1>@?define_lambda@@YAHXZ@QBEHXZ@4HA"
+// MSVC2013-DAG: @"\01?local@?2???R<lambda_1>@?0??define_lambda@@YAHXZ@QBE@XZ@4HA"
+// MSVC2015-DAG: @"\01?local@?1???R<lambda_1>@?0??define_lambda@@YAHXZ@QBE@XZ@4HA"
   return lambda();
 }
 
@@ -182,12 +182,12 @@ void use_lambda_arg(T) {}
 
 inline void call_with_lambda_arg1() {
   use_lambda_arg([]{});
-  // CHECK-DAG: @"\01??$use_lambda_arg@V<lambda_1>@?call_with_lambda_arg1@@YAXXZ@@@YAXV<lambda_1>@?call_with_lambda_arg1@@YAXXZ@@Z"
+  // CHECK-DAG: @"\01??$use_lambda_arg@V<lambda_1>@?0??call_with_lambda_arg1@@YAXXZ@@@YAXV<lambda_1>@?0??call_with_lambda_arg1@@YAXXZ@@Z"
 }
 
 inline void call_with_lambda_arg2() {
   use_lambda_arg([]{});
-  // CHECK-DAG: @"\01??$use_lambda_arg@V<lambda_1>@?call_with_lambda_arg2@@YAXXZ@@@YAXV<lambda_1>@?call_with_lambda_arg2@@YAXXZ@@Z"
+  // CHECK-DAG: @"\01??$use_lambda_arg@V<lambda_1>@?0??call_with_lambda_arg2@@YAXXZ@@@YAXV<lambda_1>@?0??call_with_lambda_arg2@@YAXXZ@@Z"
 }
 
 int call_lambda() {
@@ -286,3 +286,10 @@ static union {
 };
 // CHECK-DAG: @"\01??$f@T<unnamed-type-$S1>@PR18204@@@PR18204@@YAHPAT<unnamed-type-$S1>@0@@Z"
 }
+
+int PR26105() {
+  auto add = [](int x) { return ([x](int y) { return x + y; }); };
+  return add(3)(4);
+}
+// CHECK-DAG: @"\01??R<lambda_0>@?0??PR26105@@YAHXZ@QBE@H@Z"
+// CHECK-DAG: @"\01??R<lambda_1>@?0???R<lambda_0>@?0??PR26105@@YAHXZ@QBE@H@Z@QBE@H@Z"

clang/test/CodeGenCXX/mangle-ms-cxx14.cpp

@@ -35,12 +35,12 @@ auto TemplateFuncionWithLocalLambda(T) {
   return LocalLambdaWithLocalType();
 }
 
-// MSVC2013-DAG: "\01?ValueFromTemplateFuncionWithLocalLambda@@3ULocalType@?2???R<lambda_1>@??$TemplateFuncionWithLocalLambda@H@@YA?A?<auto>@@H@Z@QBE?A?3@XZ@A"
-// MSVC2013-DAG: "\01?ValueFromTemplateFuncionWithLocalLambda@@3ULocalType@?2???R<lambda_1>@??$TemplateFuncionWithLocalLambda@H@@YA?A?<auto>@@H@Z@QBE?A?3@XZ@A"
-// MSVC2015-DAG: "\01?ValueFromTemplateFuncionWithLocalLambda@@3ULocalType@?1???R<lambda_1>@??$TemplateFuncionWithLocalLambda@H@@YA?A?<auto>@@H@Z@QBE?A?3@XZ@A"
-// MSVC2015-DAG: "\01?ValueFromTemplateFuncionWithLocalLambda@@3ULocalType@?1???R<lambda_1>@??$TemplateFuncionWithLocalLambda@H@@YA?A?<auto>@@H@Z@QBE?A?3@XZ@A"
+// MSVC2013-DAG: "\01?ValueFromTemplateFuncionWithLocalLambda@@3ULocalType@?2???R<lambda_1>@?0???$TemplateFuncionWithLocalLambda@H@@YA?A?<auto>@@H@Z@QBE?A?3@XZ@A"
+// MSVC2013-DAG: "\01?ValueFromTemplateFuncionWithLocalLambda@@3ULocalType@?2???R<lambda_1>@?0???$TemplateFuncionWithLocalLambda@H@@YA?A?<auto>@@H@Z@QBE?A?3@XZ@A"
+// MSVC2015-DAG: "\01?ValueFromTemplateFuncionWithLocalLambda@@3ULocalType@?1???R<lambda_1>@?0???$TemplateFuncionWithLocalLambda@H@@YA?A?<auto>@@H@Z@QBE?A?3@XZ@A"
+// MSVC2015-DAG: "\01?ValueFromTemplateFuncionWithLocalLambda@@3ULocalType@?1???R<lambda_1>@?0???$TemplateFuncionWithLocalLambda@H@@YA?A?<auto>@@H@Z@QBE?A?3@XZ@A"
 // CHECK-DAG: "\01??$TemplateFuncionWithLocalLambda@H@@YA?A?<auto>@@H@Z"
-// CHECK-DAG: "\01??R<lambda_1>@??$TemplateFuncionWithLocalLambda@H@@YA?A?<auto>@@H@Z@QBE?A?1@XZ"
+// CHECK-DAG: "\01??R<lambda_1>@?0???$TemplateFuncionWithLocalLambda@H@@YA?A?<auto>@@H@Z@QBE?A?1@XZ"
 auto ValueFromTemplateFuncionWithLocalLambda = TemplateFuncionWithLocalLambda(0);
 
 struct S;

clang/test/CodeGenCXX/pr20719.cpp

@@ -2,8 +2,8 @@
 
 // Make sure that we emit H's constructor twice: once with the first lambda
 // inside of 'lep' and again with the second lambda inside of 'lep'.
-// CHECK-DAG: @"\01??0?$H@V<lambda_1>@??$lep@X@@YAXXZ@@@QAE@XZ"
-// CHECK-DAG: @"\01??0?$H@V<lambda_2>@??$lep@X@@YAXXZ@@@QAE@XZ"
+// CHECK-DAG: @"\01??0?$H@V<lambda_1>@?0???$lep@X@@YAXXZ@@@QAE@XZ"
+// CHECK-DAG: @"\01??0?$H@V<lambda_2>@?0???$lep@X@@YAXXZ@@@QAE@XZ"
 
 template <typename>
 struct H {