[X86][fastcall][vectorcall] Move capability check before free register update

When passing arguments with `__fastcall` or `__vectorcall` in 32-bit MSVC, the following arguments have chance to be passed by register if the current one failed. `__regcall` from ICC is on the contrary: https://godbolt.org/z/4MPbzhaMG
All the three calling conversions are not supported in GCC.

Fixes: #57737

Reviewed By: rnk

Differential Revision: https://reviews.llvm.org/D133920

clang/docs/ReleaseNotes.rst

@@ -358,6 +358,7 @@ RISC-V Support in Clang
 X86 Support in Clang
 --------------------
 - Support ``-mindirect-branch-cs-prefix`` for call and jmp to indirect thunk.
+- Fix 32-bit ``__fastcall`` and ``__vectorcall`` ABI mismatch with MSVC.
 
 DWARF Support in Clang
 ----------------------

clang/lib/CodeGen/TargetInfo.cpp

@@ -1771,23 +1771,22 @@ bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State,
 }
 
 bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const {
+  bool IsPtrOrInt = (getContext().getTypeSize(Ty) <= 32) &&
+                    (Ty->isIntegralOrEnumerationType() || Ty->isPointerType() ||
+                     Ty->isReferenceType());
+
+  if (!IsPtrOrInt && (State.CC == llvm::CallingConv::X86_FastCall ||
+                      State.CC == llvm::CallingConv::X86_VectorCall))
+    return false;
+
   if (!updateFreeRegs(Ty, State))
     return false;
 
-  if (IsMCUABI)
+  if (!IsPtrOrInt && State.CC == llvm::CallingConv::X86_RegCall)
     return false;
 
-  if (State.CC == llvm::CallingConv::X86_FastCall ||
-      State.CC == llvm::CallingConv::X86_VectorCall ||
-      State.CC == llvm::CallingConv::X86_RegCall) {
-    if (getContext().getTypeSize(Ty) > 32)
-      return false;
-
-    return (Ty->isIntegralOrEnumerationType() || Ty->isPointerType() ||
-        Ty->isReferenceType());
-  }
-
-  return true;
+  // Return true to apply inreg to all legal parameters except for MCU targets.
+  return !IsMCUABI;
 }
 
 void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const {

clang/test/CodeGen/mangle-windows.c

@@ -47,7 +47,7 @@ void __fastcall f8(long long a) {}
 // X64: define dso_local void @f8(
 
 void __fastcall f9(long long a, char b, char c, short d) {}
-// CHECK: define dso_local x86_fastcallcc void @"\01@f9@20"(i64 noundef %a, i8 noundef signext %b, i8 noundef signext %c, i16 noundef signext %d)
+// CHECK: define dso_local x86_fastcallcc void @"\01@f9@20"(i64 noundef %a, i8 inreg noundef signext %b, i8 inreg noundef signext %c, i16 noundef signext %d)
 // X64: define dso_local void @f9(
 
 void f12(void) {}
@@ -81,3 +81,6 @@ void __vectorcall v5(long long a) {}
 void __vectorcall v6(char a, char b) {}
 // CHECK: define dso_local x86_vectorcallcc void @"\01v6@@8"(
 // X64: define dso_local x86_vectorcallcc void @"\01v6@@16"(
+
+void __vectorcall v7(long long a, char b, char c, short d) {}
+// CHECK: define dso_local x86_vectorcallcc void @"\01v7@@20"(i64 noundef %a, i8 inreg noundef signext %b, i8 inreg noundef signext %c, i16 noundef signext %d)

clang/test/CodeGen/regcall.c

@@ -31,6 +31,10 @@ void __regcall v4(int a, struct Large b, int c) {}
 // Win64: define dso_local x86_regcallcc void @__regcall3__v4(i32 noundef %a, %struct.Large* noundef %b, i32 noundef %c)
 // Lin64: define dso_local x86_regcallcc void @__regcall3__v4(i32 noundef %a, [5 x i32] %b.coerce, i32 noundef %c)
 
+void __regcall v5(long long a, int b, int c) {}
+// X86: define dso_local x86_regcallcc void @__regcall3__v5(i64 noundef %a, i32 inreg noundef %b, i32 inreg noundef %c)
+// X64: define dso_local x86_regcallcc void @__regcall3__v5(i64 noundef %a, i32 noundef %b, i32 noundef %c)
+
 struct HFA2 { double x, y; };
 struct HFA4 { double w, x, y, z; };
 struct HFA5 { double v, w, x, y, z; };

clang/test/CodeGen/stdcall-fastcall.c

@@ -144,3 +144,10 @@ void bar12(struct S3 y, int x) {
   // CHECK: call x86_fastcallcc void @foo12(float %{{.*}}, i32 inreg noundef %
   foo12(y, x);
 }
+
+void __attribute__((fastcall)) foo13(long long a, int b, int c);
+void bar13(long long a, int b, int c) {
+  // CHECK-LABEL: define{{.*}} void @bar13
+  // CHECK: call x86_fastcallcc void @foo13(i64 noundef %{{.*}}, i32 inreg noundef %{{.*}}, i32 inreg noundef %
+  foo13(a, b, c);
+}

clang/test/CodeGen/vectorcall.c

@@ -19,6 +19,10 @@ void __vectorcall v4(int a, struct Large b, int c) {}
 // X86: define dso_local x86_vectorcallcc void @"\01v4@@28"(i32 inreg noundef %a, %struct.Large* noundef byval(%struct.Large) align 4 %b, i32 inreg noundef %c)
 // X64: define dso_local x86_vectorcallcc void @"\01v4@@40"(i32 noundef %a, %struct.Large* noundef %b, i32 noundef %c)
 
+void __vectorcall v5(long long a, int b, int c) {}
+// X86: define dso_local x86_vectorcallcc void @"\01v5@@16"(i64 noundef %a, i32 inreg noundef %b, i32 inreg noundef %c)
+// X64: define dso_local x86_vectorcallcc void @"\01v5@@24"(i64 noundef %a, i32 noundef %b, i32 noundef %c)
+
 struct HFA2 { double x, y; };
 struct HFA4 { double w, x, y, z; };
 struct HFA5 { double v, w, x, y, z; };