Misc fixes for atomics. Biggest fix is doing alignment correctly for _Atomic types.

llvm-svn: 142002

clang/include/clang/Basic/Builtins.def

@@ -596,8 +596,8 @@ BUILTIN(__atomic_fetch_sub, "v.", "t")
 BUILTIN(__atomic_fetch_and, "v.", "t")
 BUILTIN(__atomic_fetch_or, "v.", "t")
 BUILTIN(__atomic_fetch_xor, "v.", "t")
-BUILTIN(__atomic_thread_fence, "vi", "t")
-BUILTIN(__atomic_signal_fence, "vi", "t")
+BUILTIN(__atomic_thread_fence, "vi", "n")
+BUILTIN(__atomic_signal_fence, "vi", "n")
 
 // Non-overloaded atomic builtins.
 BUILTIN(__sync_synchronize, "v.", "n")

clang/include/clang/Basic/TargetInfo.h

@@ -75,6 +75,7 @@ protected:
   unsigned char LargeArrayMinWidth, LargeArrayAlign;
   unsigned char LongWidth, LongAlign;
   unsigned char LongLongWidth, LongLongAlign;
+  unsigned char MaxAtomicPromoteWidth, MaxAtomicInlineWidth;
   const char *DescriptionString;
   const char *UserLabelPrefix;
   const char *MCountName;
@@ -246,6 +247,14 @@ public:
   unsigned getLargeArrayMinWidth() const { return LargeArrayMinWidth; }
   unsigned getLargeArrayAlign() const { return LargeArrayAlign; }
 
+  /// getMaxAtomicPromoteWidth - Return the maximum width lock-free atomic
+  /// operation which will ever be supported for the given target
+  unsigned getMaxAtomicPromoteWidth() const { return MaxAtomicPromoteWidth; }
+  /// getMaxAtomicInlineWidth - Return the maximum width lock-free atomic
+  /// operation which can be inlined given the supported features of the
+  /// given target.
+  unsigned getMaxAtomicInlineWidth() const { return MaxAtomicInlineWidth; }
+
   /// getIntMaxTWidth - Return the size of intmax_t and uintmax_t for this
   /// target, in bits.
   unsigned getIntMaxTWidth() const {

clang/lib/AST/ASTContext.cpp

@@ -1064,13 +1064,25 @@ ASTContext::getTypeInfo(const Type *T) const {
   }
 
   case Type::Atomic: {
-    // FIXME: The alignment needs to be "fixed".
-    return getTypeInfo(cast<AtomicType>(T)->getValueType());
+    std::pair<uint64_t, unsigned> Info
+      = getTypeInfo(cast<AtomicType>(T)->getValueType());
+    Width = Info.first;
+    Align = Info.second;
+    if (Width != 0 && Width <= Target->getMaxAtomicPromoteWidth() &&
+        llvm::isPowerOf2_64(Width)) {
+      // We can potentially perform lock-free atomic operations for this
+      // type; promote the alignment appropriately.
+      // FIXME: We could potentially promote the width here as well...
+      // is that worthwhile?  (Non-struct atomic types generally have
+      // power-of-two size anyway, but structs might not.  Requires a bit
+      // of implementation work to make sure we zero out the extra bits.)
+      Align = static_cast<unsigned>(Width);
+    }
   }
 
   }
 
-  assert(Align && (Align & (Align-1)) == 0 && "Alignment must be power of 2");
+  assert(llvm::isPowerOf2_32(Align) && "Alignment must be power of 2");
   return std::make_pair(Width, Align);
 }
 

clang/lib/AST/StmtProfile.cpp

@@ -470,6 +470,7 @@ void StmtProfiler::VisitGenericSelectionExpr(const GenericSelectionExpr *S) {
 
 void StmtProfiler::VisitAtomicExpr(const AtomicExpr *S) {
   VisitExpr(S);
+  ID.AddInteger(S->getOp());
 }
 
 static Stmt::StmtClass DecodeOperatorCall(const CXXOperatorCallExpr *S,

clang/lib/Basic/TargetInfo.cpp

@@ -42,6 +42,7 @@ TargetInfo::TargetInfo(const std::string &T) : Triple(T) {
   LongDoubleAlign = 64;
   LargeArrayMinWidth = 0;
   LargeArrayAlign = 0;
+  MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 0;
   SizeType = UnsignedLong;
   PtrDiffType = SignedLong;
   IntMaxType = SignedLongLong;

clang/lib/Basic/Targets.cpp

@@ -1984,6 +1984,11 @@ public:
     RealTypeUsesObjCFPRet = ((1 << TargetInfo::Float) |
                              (1 << TargetInfo::Double) |
                              (1 << TargetInfo::LongDouble));
+
+    // x86-32 has atomics up to 8 bytes
+    // FIXME: Check that we actually have cmpxchg8b before setting
+    // MaxAtomicInlineWidth. (cmpxchg8b is an i586 instruction.)
+    MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 64;
   }
   virtual const char *getVAListDeclaration() const {
     return "typedef char* __builtin_va_list;";
@@ -2219,6 +2224,12 @@ public:
 
     // Use fpret only for long double.
     RealTypeUsesObjCFPRet = (1 << TargetInfo::LongDouble);
+
+    // x86-64 has atomics up to 16 bytes.
+    // FIXME: Once the backend is fixed, increase MaxAtomicInlineWidth to 128
+    // on CPUs with cmpxchg16b
+    MaxAtomicPromoteWidth = 128;
+    MaxAtomicInlineWidth = 64;
   }
   virtual const char *getVAListDeclaration() const {
     return "typedef struct __va_list_tag {"
@@ -2391,6 +2402,10 @@ public:
 
     // ARM targets default to using the ARM C++ ABI.
     CXXABI = CXXABI_ARM;
+
+    // ARM has atomics up to 8 bytes
+    // FIXME: Set MaxAtomicInlineWidth if we have the feature v6e
+    MaxAtomicPromoteWidth = 64;
   }
   virtual const char *getABI() const { return ABI.c_str(); }
   virtual bool setABI(const std::string &Name) {
@@ -2708,6 +2723,9 @@ public:
   DarwinARMTargetInfo(const std::string& triple)
     : DarwinTargetInfo<ARMTargetInfo>(triple) {
     HasAlignMac68kSupport = true;
+    // iOS always has 64-bit atomic instructions.
+    // FIXME: This should be based off of the target features in ARMTargetInfo.
+    MaxAtomicInlineWidth = 64;
   }
 };
 } // end anonymous namespace.

clang/lib/CodeGen/CGExpr.cpp

@@ -2567,9 +2567,9 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
   uint64_t Size = sizeChars.getQuantity();
   CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy);
   unsigned Align = alignChars.getQuantity();
-  // FIXME: Bound on Size should not be hardcoded.
-  bool UseLibcall = (sizeChars != alignChars || !llvm::isPowerOf2_64(Size) ||
-                     Size > 8);
+  unsigned MaxInlineWidth =
+      getContext().getTargetInfo().getMaxAtomicInlineWidth();
+  bool UseLibcall = (Size != Align || Size > MaxInlineWidth);
 
   llvm::Value *Ptr, *Order, *OrderFail = 0, *Val1 = 0, *Val2 = 0;
   Ptr = EmitScalarExpr(E->getPtr());
@@ -2585,11 +2585,9 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
     // is not the same as adding 1 to a uintptr_t.
     QualType Val1Ty = E->getVal1()->getType();
     llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
-    uint64_t PointeeIncAmt =
-        getContext().getTypeSizeInChars(MemTy->getPointeeType()).getQuantity();
-    llvm::Value *PointeeIncAmtVal =
-        llvm::ConstantInt::get(Val1Scalar->getType(), PointeeIncAmt);
-    Val1Scalar = Builder.CreateMul(Val1Scalar, PointeeIncAmtVal);
+    CharUnits PointeeIncAmt =
+        getContext().getTypeSizeInChars(MemTy->getPointeeType());
+    Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
     Val1 = CreateMemTemp(Val1Ty, ".atomictmp");
     EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty));
   } else if (E->getOp() != AtomicExpr::Load) {

clang/test/CodeGen/atomic-ops.c

@@ -63,9 +63,7 @@ int* fp2(_Atomic(int*) *p) {
   return __atomic_fetch_add(p, 1, memory_order_relaxed);
 }
 
-// FIXME: Alignment specification shouldn't be necessary
-typedef _Complex float ComplexAligned __attribute((aligned(8)));
-_Complex float fc(_Atomic(ComplexAligned) *c) {
+_Complex float fc(_Atomic(_Complex float) *c) {
   // CHECK: @fc
   // CHECK: atomicrmw xchg i64*
   return __atomic_exchange(c, 2, memory_order_seq_cst);