[globalisel][legalizer] Add AtomicOrdering to LegalityQuery and use it in AArch64

Now that we have the ability to legalize based on MMO's. Add support for
legalizing based on AtomicOrdering and use it to correct the legalization
of the atomic instructions.

Also extend all() to be a variadic template as this ruleset now requires
3 and 4 argument versions.

llvm-svn: 335767

llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h

@@ -122,6 +122,7 @@ struct LegalityQuery {
 
   struct MemDesc {
     uint64_t Size;
+    AtomicOrdering Ordering;
   };
 
   /// Operations which require memory can use this to place requirements on the
@@ -175,7 +176,19 @@ struct TypePairAndMemSize {
 };
 
 /// True iff P0 and P1 are true.
-LegalityPredicate all(LegalityPredicate P0, LegalityPredicate P1);
+template<typename Predicate>
+Predicate all(Predicate P0, Predicate P1) {
+  return [=](const LegalityQuery &Query) {
+    return P0(Query) && P1(Query);
+  };
+}
+/// True iff all given predicates are true.
+template<typename Predicate, typename... Args>
+Predicate all(Predicate P0, Predicate P1, Args... args) {
+  return all(all(P0, P1), args...);
+}
+/// True iff the given type index is the specified types.
+LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit);
 /// True iff the given type index is one of the specified types.
 LegalityPredicate typeInSet(unsigned TypeIdx,
                             std::initializer_list<LLT> TypesInit);
@@ -205,6 +218,10 @@ LegalityPredicate memSizeInBytesNotPow2(unsigned MMOIdx);
 /// True iff the specified type index is a vector whose element count is not a
 /// power of 2.
 LegalityPredicate numElementsNotPow2(unsigned TypeIdx);
+/// True iff the specified MMO index has at an atomic ordering of at Ordering or
+/// stronger.
+LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx,
+                                                      AtomicOrdering Ordering);
 } // end namespace LegalityPredicates
 
 namespace LegalizeMutations {

llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp

@@ -15,11 +15,9 @@
 
 using namespace llvm;
 
-LegalityPredicate
-LegalityPredicates::all(LegalityPredicate P0, LegalityPredicate P1) {
-  return [=](const LegalityQuery &Query) {
-    return P0(Query) && P1(Query);
-  };
+LegalityPredicate LegalityPredicates::typeIs(unsigned TypeIdx, LLT Type) {
+  return
+      [=](const LegalityQuery &Query) { return Query.Types[TypeIdx] == Type; };
 }
 
 LegalityPredicate
@@ -94,3 +92,10 @@ LegalityPredicate LegalityPredicates::numElementsNotPow2(unsigned TypeIdx) {
     return QueryTy.isVector() && isPowerOf2_32(QueryTy.getNumElements());
   };
 }
+
+LegalityPredicate LegalityPredicates::atomicOrderingAtLeastOrStrongerThan(
+    unsigned MMOIdx, AtomicOrdering Ordering) {
+  return [=](const LegalityQuery &Query) {
+    return isAtLeastOrStrongerThan(Query.MMODescrs[MMOIdx].Ordering, Ordering);
+  };
+}

llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp

@@ -364,7 +364,8 @@ LegalizerInfo::getAction(const MachineInstr &MI,
 
   SmallVector<LegalityQuery::MemDesc, 2> MemDescrs;
   for (const auto &MMO : MI.memoperands())
-    MemDescrs.push_back({MMO->getSize() /* in bytes */ * 8});
+    MemDescrs.push_back(
+        {MMO->getSize() /* in bytes */ * 8, MMO->getOrdering()});
 
   return getAction({MI.getOpcode(), Types, MemDescrs});
 }

llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp

@@ -280,13 +280,17 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
 
   if (ST.hasLSE()) {
     getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG_WITH_SUCCESS)
-        .lowerForCartesianProduct({s8, s16, s32, s64}, {s1}, {p0});
+        .lowerIf(all(
+            typeInSet(0, {s8, s16, s32, s64}), typeIs(1, s1), typeIs(2, p0),
+            atomicOrderingAtLeastOrStrongerThan(0, AtomicOrdering::Monotonic)));
 
     getActionDefinitionsBuilder(
         {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_AND,
          G_ATOMICRMW_OR, G_ATOMICRMW_XOR, G_ATOMICRMW_MIN, G_ATOMICRMW_MAX,
          G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX, G_ATOMIC_CMPXCHG})
-        .legalForCartesianProduct({s8, s16, s32, s64}, {p0});
+        .legalIf(all(
+            typeInSet(0, {s8, s16, s32, s64}), typeIs(1, p0),
+            atomicOrderingAtLeastOrStrongerThan(0, AtomicOrdering::Monotonic)));
   }
 
   // Merge/Unmerge

llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir

@@ -91,43 +91,43 @@
 # DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_ATOMIC_CMPXCHG_WITH_SUCCESS (opcode {{[0-9]+}}): 3 type indices
-# DEBUG:      .. the first uncovered type index: 3, OK
+# DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_ATOMIC_CMPXCHG (opcode {{[0-9]+}}): 2 type indices
-# DEBUG:      .. the first uncovered type index: 2, OK
+# DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_ATOMICRMW_XCHG (opcode {{[0-9]+}}): 2 type indices
-# DEBUG:      .. the first uncovered type index: 2, OK
+# DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_ATOMICRMW_ADD (opcode {{[0-9]+}}): 2 type indices
-# DEBUG:      .. the first uncovered type index: 2, OK
+# DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_ATOMICRMW_SUB (opcode {{[0-9]+}}): 2 type indices
-# DEBUG:      .. the first uncovered type index: 2, OK
+# DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_ATOMICRMW_AND (opcode {{[0-9]+}}): 2 type indices
-# DEBUG:      .. the first uncovered type index: 2, OK
+# DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_ATOMICRMW_NAND (opcode {{[0-9]+}}): 2 type indices
 # DEBUG:      .. type index coverage check SKIPPED: no rules defined
 #
 # DEBUG-NEXT: G_ATOMICRMW_OR (opcode {{[0-9]+}}): 2 type indices
-# DEBUG:      .. the first uncovered type index: 2, OK
+# DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_ATOMICRMW_XOR (opcode {{[0-9]+}}): 2 type indices
-# DEBUG:      .. the first uncovered type index: 2, OK
+# DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_ATOMICRMW_MAX (opcode {{[0-9]+}}): 2 type indices
-# DEBUG:      .. the first uncovered type index: 2, OK
+# DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_ATOMICRMW_MIN (opcode {{[0-9]+}}): 2 type indices
-# DEBUG:      .. the first uncovered type index: 2, OK
+# DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_ATOMICRMW_UMAX (opcode {{[0-9]+}}): 2 type indices
-# DEBUG:      .. the first uncovered type index: 2, OK
+# DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_ATOMICRMW_UMIN (opcode {{[0-9]+}}): 2 type indices
-# DEBUG:      .. the first uncovered type index: 2, OK
+# DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_BRCOND (opcode {{[0-9]+}}): 1 type index
 # DEBUG:      .. the first uncovered type index: 1, OK