[clang][SveEmitter] SVE builtins for `svusdot` and `svsudot` ACLE.

Summary:
Intrinsics, guarded by `__ARM_FEATURE_SVE_MATMUL_INT8`:

* svusdot[_s32]
* svusdot[_n_s32]
* svusdot_lane[_s32]
* svsudot[_s32]
* svsudot[_n_s32]
* svsudot_lane[_s32]

Reviewers: sdesmalen, efriedma, david-arm, rengolin

Subscribers: tschuett, kristof.beyls, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D79877

clang/include/clang/Basic/TargetBuiltins.h

@@ -244,6 +244,7 @@ namespace clang {
     bool isAppendSVALL() const { return Flags & IsAppendSVALL; }
     bool isInsertOp1SVALL() const { return Flags & IsInsertOp1SVALL; }
     bool isGatherPrefetch() const { return Flags & IsGatherPrefetch; }
+    bool isReverseUSDOT() const { return Flags & ReverseUSDOT; }
 
     uint64_t getBits() const { return Flags; }
     bool isFlagSet(uint64_t Flag) const { return Flags & Flag; }

clang/include/clang/Basic/arm_sve.td

@@ -68,6 +68,7 @@
 // a: scalar of element type (splat to vector type)
 // R: scalar of 1/2 width element type (splat to vector type)
 // r: scalar of 1/4 width element type (splat to vector type)
+// @: unsigned scalar of 1/4 width element type (splat to vector type)
 // e: 1/2 width unsigned elements, 2x element count
 // b: 1/4 width unsigned elements, 4x element count
 // h: 1/2 width elements, 2x element count
@@ -196,6 +197,7 @@ def IsInsertOp1SVALL          : FlagType<0x04000000>; // Inserts SV_ALL as the s
 def IsPrefetch                : FlagType<0x08000000>; // Contiguous prefetches.
 def IsGatherPrefetch          : FlagType<0x10000000>;
 def ReverseCompare            : FlagType<0x20000000>; // Compare operands must be swapped.
+def ReverseUSDOT              : FlagType<0x40000000>; // Unsigned/signed operands must be swapped.
 
 // These must be kept in sync with the flags in include/clang/Basic/TargetBuiltins.h
 class ImmCheckType<int val> {
@@ -1240,6 +1242,14 @@ let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_INT8)" in {
 def SVMLLA_S32   : SInst<"svmmla[_s32]",   "ddqq","i",  MergeNone, "aarch64_sve_smmla">;
 def SVMLLA_U32   : SInst<"svmmla[_u32]",   "ddqq","Ui", MergeNone, "aarch64_sve_ummla">;
 def SVUSMLLA_S32 : SInst<"svusmmla[_s32]", "ddbq","i",  MergeNone, "aarch64_sve_usmmla">;
+
+def SVUSDOT_S    : SInst<"svusdot[_s32]",    "ddbq", "i",       MergeNone, "aarch64_sve_usdot">;
+def SVUSDOT_N_S  : SInst<"svusdot[_n_s32]",  "ddbr", "i",       MergeNone, "aarch64_sve_usdot">;
+def SVSUDOT_S    : SInst<"svsudot[_s32]",    "ddqb", "i",       MergeNone, "aarch64_sve_usdot", [ReverseUSDOT]>;
+def SVSUDOT_N_S  : SInst<"svsudot[_n_s32]",  "ddq@", "i",       MergeNone, "aarch64_sve_usdot", [ReverseUSDOT]>;
+
+def SVUSDOT_LANE_S : SInst<"svusdot_lane[_s32]",  "ddbqi",  "i",   MergeNone, "aarch64_sve_usdot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>;
+def SVSUDOT_LANE_S : SInst<"svsudot_lane[_s32]",  "ddqbi",  "i",   MergeNone, "aarch64_sve_sudot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>;
 }
 
 let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_FP32)" in {

clang/lib/CodeGen/CGBuiltin.cpp

@@ -8040,6 +8040,9 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
     if (TypeFlags.isReverseCompare())
       std::swap(Ops[1], Ops[2]);
 
+    if (TypeFlags.isReverseUSDOT())
+      std::swap(Ops[1], Ops[2]);
+
     // Predicated intrinsics with _z suffix need a select w/ zeroinitializer.
     if (TypeFlags.getMergeType() == SVETypeFlags::MergeZero) {
       llvm::Type *OpndTy = Ops[1]->getType();

clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sudot.c

@@ -0,0 +1,54 @@
+// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_INT8 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_INT8 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svint32_t test_svsudot_s32(svint32_t x, svint8_t y, svuint8_t z) {
+  // CHECK-LABEL: test_svsudot_s32
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %z, <vscale x 16 x i8> %y)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svsudot, _s32, , )(x, y, z);
+}
+
+svint32_t test_svsudot_n_s32(svint32_t x, svint8_t y, uint8_t z) {
+  // CHECK-LABEL: test_svsudot_n_s32
+  // CHECK: %[[SPLAT:.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.dup.x.nxv16i8(i8 %z)
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %[[SPLAT]], <vscale x 16 x i8> %y)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svsudot, _n_s32, , )(x, y, z);
+}
+
+svint32_t test_svsudot_lane_s32_0(svint32_t x, svint8_t y, svuint8_t z) {
+  // CHECK-LABEL: test_svsudot_lane_s32_0
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sudot.lane.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z, i32 0)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svsudot_lane, _s32, , )(x, y, z, 0);
+}
+
+svint32_t test_svsudot_lane_s32_1(svint32_t x, svint8_t y, svuint8_t z) {
+  // CHECK-LABEL: test_svsudot_lane_s32_1
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sudot.lane.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z, i32 1)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svsudot_lane, _s32, , )(x, y, z, 1);
+}
+
+svint32_t test_svsudot_lane_s32_2(svint32_t x, svint8_t y, svuint8_t z) {
+  // CHECK-LABEL: test_svsudot_lane_s32_2
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sudot.lane.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z, i32 2)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svsudot_lane, _s32, , )(x, y, z, 2);
+}
+
+svint32_t test_svsudot_lane_s32_3(svint32_t x, svint8_t y, svuint8_t z) {
+  // CHECK-LABEL: test_svsudot_lane_s32_3
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sudot.lane.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z, i32 3)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svsudot_lane, _s32, , )(x, y, z, 3);
+}

clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_usdot.c

@@ -0,0 +1,54 @@
+// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_INT8 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_SVE_MATMUL_INT8 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+svint32_t test_svusdot_s32(svint32_t x, svuint8_t y, svint8_t z) {
+  // CHECK-LABEL: test_svusdot_s32
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svusdot, _s32, , )(x, y, z);
+}
+
+svint32_t test_svusdot_n_s32(svint32_t x, svuint8_t y, int8_t z) {
+  // CHECK-LABEL: test_svusdot_n_s32
+  // CHECK: %[[SPLAT:.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.dup.x.nxv16i8(i8 %z)
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %[[SPLAT]])
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svusdot, _n_s32, , )(x, y, z);
+}
+
+svint32_t test_svusdot_lane_s32_0(svint32_t x, svuint8_t y, svint8_t z) {
+  // CHECK-LABEL: test_svusdot_lane_s32_0
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.lane.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z, i32 0)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svusdot_lane, _s32, , )(x, y, z, 0);
+}
+
+svint32_t test_svusdot_lane_s32_1(svint32_t x, svuint8_t y, svint8_t z) {
+  // CHECK-LABEL: test_svusdot_lane_s32_1
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.lane.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z, i32 1)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svusdot_lane, _s32, , )(x, y, z, 1);
+}
+
+svint32_t test_svusdot_lane_s32_2(svint32_t x, svuint8_t y, svint8_t z) {
+  // CHECK-LABEL: test_svusdot_lane_s32_2
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.lane.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z, i32 2)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svusdot_lane, _s32, , )(x, y, z, 2);
+}
+
+svint32_t test_svusdot_lane_s32_3(svint32_t x, svuint8_t y, svint8_t z) {
+  // CHECK-LABEL: test_svusdot_lane_s32_3
+  // CHECK: %[[RET:.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.usdot.lane.nxv4i32(<vscale x 4 x i32> %x, <vscale x 16 x i8> %y, <vscale x 16 x i8> %z, i32 3)
+  // CHECK: ret <vscale x 4 x i32> %[[RET]]
+  return SVE_ACLE_FUNC(svusdot_lane, _s32, , )(x, y, z, 3);
+}

clang/utils/TableGen/SveEmitter.cpp

@@ -213,13 +213,13 @@ public:
   /// Return true if the intrinsic takes a splat operand.
   bool hasSplat() const {
     // These prototype modifiers are described in arm_sve.td.
-    return Proto.find_first_of("ajfrKLR") != std::string::npos;
+    return Proto.find_first_of("ajfrKLR@") != std::string::npos;
   }
 
   /// Return the parameter index of the splat operand.
   unsigned getSplatIdx() const {
     // These prototype modifiers are described in arm_sve.td.
-    auto Idx = Proto.find_first_of("ajfrKLR");
+    auto Idx = Proto.find_first_of("ajfrKLR@");
     assert(Idx != std::string::npos && Idx > 0 &&
            "Prototype has no splat operand");
     return Idx - 1;
@@ -541,6 +541,12 @@ void SVEType::applyModifier(char Mod) {
     ElementBitwidth /= 4;
     NumVectors = 0;
     break;
+  case '@':
+    Signed = false;
+    Float = false;
+    ElementBitwidth /= 4;
+    NumVectors = 0;
+    break;
   case 'K':
     Signed = true;
     Float = false;