[VectorUtils] Rework the Vector Function Database (VFDatabase).

Summary:
This commits is a rework of the patch in
https://reviews.llvm.org/D67572.

The rework was requested to prevent out-of-tree performance regression
when vectorizing out-of-tree IR intrinsics. The vectorization of such
intrinsics is enquired via the static function `isTLIScalarize`. For
detail see the discussion in https://reviews.llvm.org/D67572.

Reviewers: uabelho, fhahn, sdesmalen

Subscribers: hiraditya, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D72734

llvm/include/llvm/Analysis/TargetLibraryInfo.h

@@ -390,12 +390,16 @@ public:
                   FunctionAnalysisManager::Invalidator &) {
     return false;
   }
-
   /// Returns the largest vectorization factor used in the list of
   /// vector functions.
   unsigned getWidestVF(StringRef ScalarF) const {
     return Impl->getWidestVF(ScalarF);
   }
+
+  /// Check if the function "F" is listed in a library known to LLVM.
+  bool isKnownVectorFunctionInLibrary(StringRef F) const {
+    return this->isFunctionVectorizable(F);
+  }
 };
 
 /// Analysis pass providing the \c TargetLibraryInfo.

llvm/include/llvm/Analysis/VectorUtils.h

@@ -16,6 +16,7 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/Analysis/LoopAccessAnalysis.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/Support/CheckedArithmetic.h"
 
@@ -116,10 +117,10 @@ struct VFShape {
 
 /// Holds the VFShape for a specific scalar to vector function mapping.
 struct VFInfo {
-  VFShape Shape;        // Classification of the vector function.
-  StringRef ScalarName; // Scalar Function Name.
-  StringRef VectorName; // Vector Function Name associated to this VFInfo.
-  VFISAKind ISA;        // Instruction Set Architecture.
+  VFShape Shape;          /// Classification of the vector function.
+  std::string ScalarName; /// Scalar Function Name.
+  std::string VectorName; /// Vector Function Name associated to this VFInfo.
+  VFISAKind ISA;          /// Instruction Set Architecture.
 
   // Comparison operator.
   bool operator==(const VFInfo &Other) const {
@@ -131,6 +132,13 @@ struct VFInfo {
 namespace VFABI {
 /// LLVM Internal VFABI ISA token for vector functions.
 static constexpr char const *_LLVM_ = "_LLVM_";
+/// Prefix for internal name redirection for vector function that
+/// tells the compiler to scalarize the call using the scalar name
+/// of the function. For example, a mangled name like
+/// `_ZGV_LLVM_N2v_foo(_LLVM_Scalarize_foo)` would tell the
+/// vectorizer to vectorize the scalar call `foo`, and to scalarize
+/// it once vectorization is done.
+static constexpr char const *_LLVM_Scalarize_ = "_LLVM_Scalarize_";
 
 /// Function to contruct a VFInfo out of a mangled names in the
 /// following format:
@@ -167,6 +175,75 @@ void getVectorVariantNames(const CallInst &CI,
                            SmallVectorImpl<std::string> &VariantMappings);
 } // end namespace VFABI
 
+/// The Vector Function Database.
+///
+/// Helper class used to find the vector functions associated to a
+/// scalar CallInst.
+class VFDatabase {
+  /// The Module of the CallInst CI.
+  const Module *M;
+  /// List of vector functions descritors associated to the call
+  /// instruction.
+  const SmallVector<VFInfo, 8> ScalarToVectorMappings;
+
+  /// Retreive the scalar-to-vector mappings associated to the rule of
+  /// a vector Function ABI.
+  static void getVFABIMappings(const CallInst &CI,
+                               SmallVectorImpl<VFInfo> &Mappings) {
+    const StringRef ScalarName = CI.getCalledFunction()->getName();
+    const StringRef S =
+        CI.getAttribute(AttributeList::FunctionIndex, VFABI::MappingsAttrName)
+            .getValueAsString();
+    if (S.empty())
+      return;
+
+    SmallVector<std::string, 8> ListOfStrings;
+    VFABI::getVectorVariantNames(CI, ListOfStrings);
+    for (const auto &MangledName : ListOfStrings) {
+      const Optional<VFInfo> Shape = VFABI::tryDemangleForVFABI(MangledName);
+      // A match is found via scalar and vector names, and also by
+      // ensuring that the variant described in the attribute has a
+      // corresponding definition or declaration of the vector
+      // function in the Module M.
+      if (Shape.hasValue() && (Shape.getValue().ScalarName == ScalarName)) {
+        assert(CI.getModule()->getFunction(Shape.getValue().VectorName) &&
+               "Vector function is missing.");
+        Mappings.push_back(Shape.getValue());
+      }
+    }
+  }
+
+public:
+  /// Retrieve all the VFInfo instances associated to the CallInst CI.
+  static SmallVector<VFInfo, 8> getMappings(const CallInst &CI) {
+    SmallVector<VFInfo, 8> Ret;
+
+    // Get mappings from the Vector Function ABI variants.
+    getVFABIMappings(CI, Ret);
+
+    // Other non-VFABI variants should be retrieved here.
+
+    return Ret;
+  }
+
+  /// Constructor, requires a CallInst instance.
+  VFDatabase(CallInst &CI)
+      : M(CI.getModule()), ScalarToVectorMappings(VFDatabase::getMappings(CI)) {
+  }
+  /// \defgroup VFDatabase query interface.
+  ///
+  /// @{
+  /// Retrieve the Function with VFShape \p Shape.
+  Function *getVectorizedFunction(const VFShape &Shape) const {
+    for (const auto &Info : ScalarToVectorMappings)
+      if (Info.Shape == Shape)
+        return M->getFunction(Info.VectorName);
+
+    return nullptr;
+  }
+  /// @}
+};
+
 template <typename T> class ArrayRef;
 class DemandedBits;
 class GetElementPtrInst;

llvm/lib/Analysis/LazyCallGraph.cpp

@@ -15,6 +15,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/VectorUtils.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/Function.h"
@@ -146,8 +147,11 @@ LLVM_DUMP_METHOD void LazyCallGraph::Node::dump() const {
 static bool isKnownLibFunction(Function &F, TargetLibraryInfo &TLI) {
   LibFunc LF;
 
-  // Either this is a normal library function or a "vectorizable" function.
-  return TLI.getLibFunc(F, LF) || TLI.isFunctionVectorizable(F.getName());
+  // Either this is a normal library function or a "vectorizable"
+  // function.  Not using the VFDatabase here because this query
+  // is related only to libraries handled via the TLI.
+  return TLI.getLibFunc(F, LF) ||
+         TLI.isKnownVectorFunctionInLibrary(F.getName());
 }
 
 LazyCallGraph::LazyCallGraph(

llvm/lib/Analysis/LoopAccessAnalysis.cpp

@@ -1845,7 +1845,7 @@ void LoopAccessInfo::analyzeLoop(AliasAnalysis *AA, LoopInfo *LI,
         // If the function has an explicit vectorized counterpart, we can safely
         // assume that it can be vectorized.
         if (Call && !Call->isNoBuiltin() && Call->getCalledFunction() &&
-            TLI->isFunctionVectorizable(Call->getCalledFunction()->getName()))
+            !VFDatabase::getMappings(*Call).empty())
           continue;
 
         auto *Ld = dyn_cast<LoadInst>(&I);

llvm/lib/Analysis/VectorUtils.cpp

@@ -1174,6 +1174,7 @@ void VFABI::getVectorVariantNames(
 
   for (auto &S : SetVector<StringRef>(ListAttr.begin(), ListAttr.end())) {
 #ifndef NDEBUG
+    LLVM_DEBUG(dbgs() << "VFABI: adding mapping '" << S << "'\n");
     Optional<VFInfo> Info = VFABI::tryDemangleForVFABI(S);
     assert(Info.hasValue() && "Invalid name for a VFABI variant.");
     assert(CI.getModule()->getFunction(Info.getValue().VectorName) &&

llvm/lib/Transforms/Utils/InjectTLIMappings.cpp

@@ -13,8 +13,11 @@
 
 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/DemandedBits.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/InstIterator.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/Transforms/Utils.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
 
@@ -168,6 +171,11 @@ void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesCFG();
   AU.addRequired<TargetLibraryInfoWrapperPass>();
   AU.addPreserved<TargetLibraryInfoWrapperPass>();
+  AU.addPreserved<ScalarEvolutionWrapperPass>();
+  AU.addPreserved<AAResultsWrapperPass>();
+  AU.addPreserved<LoopAccessLegacyAnalysis>();
+  AU.addPreserved<DemandedBitsWrapperPass>();
+  AU.addPreserved<OptimizationRemarkEmitterWrapperPass>();
 }
 
 ////////////////////////////////////////////////////////////////////////////////

llvm/lib/Transforms/Utils/ModuleUtils.cpp

@@ -11,15 +11,17 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Utils/ModuleUtils.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/raw_ostream.h"
-
 using namespace llvm;
 
+#define DEBUG_TYPE "moduleutils"
+
 static void appendToGlobalArray(const char *Array, Module &M, Function *F,
                                 int Priority, Constant *Data) {
   IRBuilder<> IRB(M.getContext());
@@ -298,8 +300,9 @@ void VFABI::setVectorVariantNames(
   Module *M = CI->getModule();
 #ifndef NDEBUG
   for (const std::string &VariantMapping : VariantMappings) {
+    LLVM_DEBUG(dbgs() << "VFABI: adding mapping '" << VariantMapping << "'\n");
     Optional<VFInfo> VI = VFABI::tryDemangleForVFABI(VariantMapping);
-    assert(VI.hasValue() && "Canno add an invalid VFABI name.");
+    assert(VI.hasValue() && "Cannot add an invalid VFABI name.");
     assert(M->getNamedValue(VI.getValue().VectorName) &&
            "Cannot add variant to attribute: "
            "vector function declaration is missing.");

llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp

@@ -568,6 +568,28 @@ bool LoopVectorizationLegality::setupOuterLoopInductions() {
     return false;
 }
 
+/// Checks if a function is scalarizable according to the TLI, in
+/// the sense that it should be vectorized and then expanded in
+/// multiple scalarcalls. This is represented in the
+/// TLI via mappings that do not specify a vector name, as in the
+/// following example:
+///
+///    const VecDesc VecIntrinsics[] = {
+///      {"llvm.phx.abs.i32", "", 4}
+///    };
+static bool isTLIScalarize(const TargetLibraryInfo &TLI, const CallInst &CI) {
+  const StringRef ScalarName = CI.getCalledFunction()->getName();
+  bool Scalarize = TLI.isFunctionVectorizable(ScalarName);
+  // Check that all known VFs are not associated to a vector
+  // function, i.e. the vector name is emty.
+  if (Scalarize)
+    for (unsigned VF = 2, WidestVF = TLI.getWidestVF(ScalarName);
+         VF <= WidestVF; VF *= 2) {
+      Scalarize &= !TLI.isFunctionVectorizable(ScalarName, VF);
+    }
+  return Scalarize;
+}
+
 bool LoopVectorizationLegality::canVectorizeInstrs() {
   BasicBlock *Header = TheLoop->getHeader();
 
@@ -669,10 +691,12 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
       //   * Have a mapping to an IR intrinsic.
       //   * Have a vector version available.
       auto *CI = dyn_cast<CallInst>(&I);
+
       if (CI && !getVectorIntrinsicIDForCall(CI, TLI) &&
           !isa<DbgInfoIntrinsic>(CI) &&
           !(CI->getCalledFunction() && TLI &&
-            TLI->isFunctionVectorizable(CI->getCalledFunction()->getName()))) {
+            (!VFDatabase::getMappings(*CI).empty() ||
+             isTLIScalarize(*TLI, *CI)))) {
         // If the call is a recognized math libary call, it is likely that
         // we can vectorize it given loosened floating-point constraints.
         LibFunc Func;
@@ -687,7 +711,8 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
           // but it's hard to provide meaningful yet generic advice.
           // Also, should this be guarded by allowExtraAnalysis() and/or be part
           // of the returned info from isFunctionVectorizable()?
-          reportVectorizationFailure("Found a non-intrinsic callsite",
+          reportVectorizationFailure(
+              "Found a non-intrinsic callsite",
               "library call cannot be vectorized. "
               "Try compiling with -fno-math-errno, -ffast-math, "
               "or similar flags",

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -134,6 +134,7 @@
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/InjectTLIMappings.h"
 #include "llvm/Transforms/Utils/LoopSimplify.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 #include "llvm/Transforms/Utils/LoopVersioning.h"
@@ -1640,6 +1641,7 @@ struct LoopVectorize : public FunctionPass {
     AU.addRequired<LoopAccessLegacyAnalysis>();
     AU.addRequired<DemandedBitsWrapperPass>();
     AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
+    AU.addRequired<InjectTLIMappingsLegacy>();
 
     // We currently do not preserve loopinfo/dominator analyses with outer loop
     // vectorization. Until this is addressed, mark these analyses as preserved
@@ -3258,7 +3260,6 @@ unsigned LoopVectorizationCostModel::getVectorCallCost(CallInst *CI,
                                                        unsigned VF,
                                                        bool &NeedToScalarize) {
   Function *F = CI->getCalledFunction();
-  StringRef FnName = CI->getCalledFunction()->getName();
   Type *ScalarRetTy = CI->getType();
   SmallVector<Type *, 4> Tys, ScalarTys;
   for (auto &ArgOp : CI->arg_operands())
@@ -3286,7 +3287,7 @@ unsigned LoopVectorizationCostModel::getVectorCallCost(CallInst *CI,
   // If we can't emit a vector call for this function, then the currently found
   // cost is the cost we need to return.
   NeedToScalarize = true;
-  if (!TLI || !TLI->isFunctionVectorizable(FnName, VF) || CI->isNoBuiltin())
+  if (!TLI || CI->isNoBuiltin() || VFDatabase::getMappings(*CI).empty())
     return Cost;
 
   // If the corresponding vector cost is cheaper, return its cost.
@@ -4339,9 +4340,6 @@ void InnerLoopVectorizer::widenInstruction(Instruction &I) {
     Module *M = I.getParent()->getParent()->getParent();
     auto *CI = cast<CallInst>(&I);
 
-    StringRef FnName = CI->getCalledFunction()->getName();
-    Function *F = CI->getCalledFunction();
-    Type *RetTy = ToVectorTy(CI->getType(), VF);
     SmallVector<Type *, 4> Tys;
     for (Value *ArgOperand : CI->arg_operands())
       Tys.push_back(ToVectorTy(ArgOperand->getType(), VF));
@@ -4377,17 +4375,18 @@ void InnerLoopVectorizer::widenInstruction(Instruction &I) {
           TysForDecl[0] = VectorType::get(CI->getType()->getScalarType(), VF);
         VectorF = Intrinsic::getDeclaration(M, ID, TysForDecl);
       } else {
-        // Use vector version of the library call.
-        StringRef VFnName = TLI->getVectorizedFunction(FnName, VF);
-        assert(!VFnName.empty() && "Vector function name is empty.");
-        VectorF = M->getFunction(VFnName);
-        if (!VectorF) {
-          // Generate a declaration
-          FunctionType *FTy = FunctionType::get(RetTy, Tys, false);
-          VectorF =
-              Function::Create(FTy, Function::ExternalLinkage, VFnName, M);
-          VectorF->copyAttributesFrom(F);
-        }
+        // Use vector version of the function call.
+        const VFShape Shape =
+            VFShape::get(*CI, {VF, false} /*EC*/, false /*HasGlobalPred*/);
+#ifndef NDEBUG
+        const SmallVector<VFInfo, 8> Infos = VFDatabase::getMappings(*CI);
+        assert(std::find_if(Infos.begin(), Infos.end(),
+                            [&Shape](const VFInfo &Info) {
+                              return Info.Shape == Shape;
+                            }) != Infos.end() &&
+               "Vector function shape is missing from the database.");
+#endif
+        VectorF = VFDatabase(*CI).getVectorizedFunction(Shape);
       }
       assert(VectorF && "Can't create vector function.");
 
@@ -6397,6 +6396,7 @@ INITIALIZE_PASS_DEPENDENCY(LoopAccessLegacyAnalysis)
 INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(InjectTLIMappingsLegacy)
 INITIALIZE_PASS_END(LoopVectorize, LV_NAME, lv_name, false, false)
 
 namespace llvm {

llvm/test/Other/opt-O2-pipeline.ll

@@ -225,9 +225,11 @@
 ; CHECK-NEXT:       Lazy Branch Probability Analysis
 ; CHECK-NEXT:       Lazy Block Frequency Analysis
 ; CHECK-NEXT:       Optimization Remark Emitter
+; CHECK-NEXT:       Inject TLI Mappings
 ; CHECK-NEXT:       Loop Vectorization
 ; CHECK-NEXT:       Canonicalize natural loops
 ; CHECK-NEXT:       Scalar Evolution Analysis
+; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Loop Access Analysis
 ; CHECK-NEXT:       Lazy Branch Probability Analysis

llvm/test/Other/opt-O3-pipeline.ll

@@ -230,9 +230,11 @@
 ; CHECK-NEXT:       Lazy Branch Probability Analysis
 ; CHECK-NEXT:       Lazy Block Frequency Analysis
 ; CHECK-NEXT:       Optimization Remark Emitter
+; CHECK-NEXT:       Inject TLI Mappings
 ; CHECK-NEXT:       Loop Vectorization
 ; CHECK-NEXT:       Canonicalize natural loops
 ; CHECK-NEXT:       Scalar Evolution Analysis
+; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Loop Access Analysis
 ; CHECK-NEXT:       Lazy Branch Probability Analysis

llvm/test/Other/opt-Os-pipeline.ll

@@ -212,9 +212,11 @@
 ; CHECK-NEXT:       Lazy Branch Probability Analysis
 ; CHECK-NEXT:       Lazy Block Frequency Analysis
 ; CHECK-NEXT:       Optimization Remark Emitter
+; CHECK-NEXT:       Inject TLI Mappings
 ; CHECK-NEXT:       Loop Vectorization
 ; CHECK-NEXT:       Canonicalize natural loops
 ; CHECK-NEXT:       Scalar Evolution Analysis
+; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Loop Access Analysis
 ; CHECK-NEXT:       Lazy Branch Probability Analysis

llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll

@@ -107,3 +107,6 @@ loop.body:
 exit:
   ret void
 }
+
+
+

llvm/unittests/Analysis/VectorFunctionABITest.cpp

@@ -13,7 +13,7 @@
 
 using namespace llvm;
 
-// This test makes sure that the getFromVFABI method succeeds only on
+// This test makes sure that the demangling method succeeds only on
 // valid values of the string.
 TEST(VectorFunctionABITests, OnlyValidNames) {
   // Incomplete string.
@@ -91,8 +91,8 @@ protected:
   unsigned &VF = Info.Shape.VF;
   VFISAKind &ISA = Info.ISA;
   SmallVector<VFParameter, 8> &Parameters = Info.Shape.Parameters;
-  StringRef &ScalarName = Info.ScalarName;
-  StringRef &VectorName = Info.VectorName;
+  std::string &ScalarName = Info.ScalarName;
+  std::string &VectorName = Info.VectorName;
   bool &IsScalable = Info.Shape.IsScalable;
   // Invoke the parser.
   bool invokeParser(const StringRef MangledName) {
@@ -243,6 +243,12 @@ TEST_F(VFABIParserTest, ISA) {
   EXPECT_EQ(ISA, VFISAKind::AVX512);
 }
 
+TEST_F(VFABIParserTest, LLVM_ISA) {
+  EXPECT_FALSE(invokeParser("_ZGV_LLVM_N2v_sin"));
+  EXPECT_TRUE(invokeParser("_ZGV_LLVM_N2v_sin_(vector_name)"));
+  EXPECT_EQ(ISA, VFISAKind::LLVM);
+}
+
 TEST_F(VFABIParserTest, InvalidMask) {
   EXPECT_FALSE(invokeParser("_ZGVsK2v_sin"));
 }
@@ -525,3 +531,15 @@ TEST_F(VFABIParserTest, LLVM_InternalISA) {
   EXPECT_TRUE(invokeParser("_ZGV_LLVM_N2v_sin_(vector_name)"));
   EXPECT_EQ(ISA, VFISAKind::LLVM);
 }
+
+TEST_F(VFABIParserTest, IntrinsicsInLLVMIsa) {
+  EXPECT_TRUE(invokeParser("_ZGV_LLVM_N4vv_llvm.pow.f32(__svml_powf4)"));
+  EXPECT_EQ(VF, (unsigned)4);
+  EXPECT_FALSE(IsMasked());
+  EXPECT_FALSE(IsScalable);
+  EXPECT_EQ(ISA, VFISAKind::LLVM);
+  EXPECT_EQ(Parameters.size(), (unsigned)2);
+  EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector}));
+  EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::Vector}));
+  EXPECT_EQ(ScalarName, "llvm.pow.f32");
+}