2014-03-04 22:59:00 +08:00
|
|
|
//===------ PollyIRBuilder.cpp --------------------------------------------===//
|
|
|
|
//
|
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
//
|
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
// License. See LICENSE.TXT for details.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
|
|
|
// The Polly IRBuilder file contains Polly specific extensions for the IRBuilder
|
|
|
|
// that are used e.g. to emit the llvm.loop.parallel metadata.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
#include "polly/CodeGen/IRBuilder.h"
|
2014-10-02 23:31:24 +08:00
|
|
|
#include "polly/ScopInfo.h"
|
|
|
|
#include "polly/Support/ScopHelper.h"
|
2017-08-23 05:58:48 +08:00
|
|
|
#include "llvm/ADT/SmallVector.h"
|
2014-03-04 22:59:00 +08:00
|
|
|
#include "llvm/IR/Metadata.h"
|
|
|
|
#include "llvm/Support/Debug.h"
|
|
|
|
|
|
|
|
using namespace llvm;
|
|
|
|
using namespace polly;
|
|
|
|
|
2017-04-03 15:42:50 +08:00
|
|
|
static const int MaxArraysInAliasScops = 10;
|
|
|
|
|
2016-09-02 14:33:33 +08:00
|
|
|
/// Get a self referencing id metadata node.
|
2014-10-02 04:10:44 +08:00
|
|
|
///
|
2014-10-02 23:31:24 +08:00
|
|
|
/// The MDNode looks like this (if arg0/arg1 are not null):
|
2014-10-02 04:10:44 +08:00
|
|
|
///
|
2014-10-02 23:31:24 +08:00
|
|
|
/// '!n = metadata !{metadata !n, arg0, arg1}'
|
2014-10-02 04:10:44 +08:00
|
|
|
///
|
2014-10-02 23:31:24 +08:00
|
|
|
/// @return The self referencing id metadata node.
|
2014-12-10 06:02:16 +08:00
|
|
|
static MDNode *getID(LLVMContext &Ctx, Metadata *arg0 = nullptr,
|
|
|
|
Metadata *arg1 = nullptr) {
|
2014-10-02 23:31:24 +08:00
|
|
|
MDNode *ID;
|
2014-12-10 06:02:16 +08:00
|
|
|
SmallVector<Metadata *, 3> Args;
|
2014-11-08 05:44:18 +08:00
|
|
|
// Use a temporary node to safely create a unique pointer for the first arg.
|
2015-01-20 05:31:48 +08:00
|
|
|
auto TempNode = MDNode::getTemporary(Ctx, None);
|
2014-11-08 05:44:18 +08:00
|
|
|
// Reserve operand 0 for loop id self reference.
|
2015-01-20 05:31:48 +08:00
|
|
|
Args.push_back(TempNode.get());
|
2014-10-02 23:31:24 +08:00
|
|
|
|
|
|
|
if (arg0)
|
|
|
|
Args.push_back(arg0);
|
|
|
|
if (arg1)
|
|
|
|
Args.push_back(arg1);
|
|
|
|
|
|
|
|
ID = MDNode::get(Ctx, Args);
|
|
|
|
ID->replaceOperandWith(0, ID);
|
|
|
|
return ID;
|
|
|
|
}
|
|
|
|
|
2014-10-02 23:32:17 +08:00
|
|
|
ScopAnnotator::ScopAnnotator() : SE(nullptr), AliasScopeDomain(nullptr) {}
|
2014-10-02 23:31:24 +08:00
|
|
|
|
2014-10-02 23:32:17 +08:00
|
|
|
void ScopAnnotator::buildAliasScopes(Scop &S) {
|
2014-10-02 23:31:24 +08:00
|
|
|
SE = S.getSE();
|
|
|
|
|
|
|
|
LLVMContext &Ctx = SE->getContext();
|
|
|
|
AliasScopeDomain = getID(Ctx, MDString::get(Ctx, "polly.alias.scope.domain"));
|
|
|
|
|
|
|
|
AliasScopeMap.clear();
|
|
|
|
OtherAliasScopeListMap.clear();
|
|
|
|
|
2017-08-23 05:58:48 +08:00
|
|
|
// We are only interested in arrays, but no scalar references. Scalars should
|
|
|
|
// be handled easily by basicaa.
|
|
|
|
SmallVector<ScopArrayInfo *, 10> Arrays;
|
|
|
|
for (ScopArrayInfo *Array : S.arrays())
|
|
|
|
if (Array->isArrayKind())
|
|
|
|
Arrays.push_back(Array);
|
|
|
|
|
2017-04-03 15:42:50 +08:00
|
|
|
// The construction of alias scopes is quadratic in the number of arrays
|
|
|
|
// involved. In case of too many arrays, skip the construction of alias
|
|
|
|
// information to avoid quadratic increases in compile time and code size.
|
2017-08-23 05:58:48 +08:00
|
|
|
if (Arrays.size() > MaxArraysInAliasScops)
|
2017-04-03 15:42:50 +08:00
|
|
|
return;
|
|
|
|
|
2014-10-02 23:31:24 +08:00
|
|
|
std::string AliasScopeStr = "polly.alias.scope.";
|
2017-08-23 05:58:48 +08:00
|
|
|
for (const ScopArrayInfo *Array : Arrays) {
|
2017-06-19 18:19:29 +08:00
|
|
|
assert(Array->getBasePtr() && "Base pointer must be present");
|
2017-02-09 17:34:42 +08:00
|
|
|
AliasScopeMap[Array->getBasePtr()] =
|
|
|
|
getID(Ctx, AliasScopeDomain,
|
|
|
|
MDString::get(Ctx, (AliasScopeStr + Array->getName()).c_str()));
|
2017-06-19 18:19:29 +08:00
|
|
|
}
|
2014-10-02 23:31:24 +08:00
|
|
|
|
2017-08-23 05:58:48 +08:00
|
|
|
for (const ScopArrayInfo *Array : Arrays) {
|
2014-10-02 23:31:24 +08:00
|
|
|
MDNode *AliasScopeList = MDNode::get(Ctx, {});
|
|
|
|
for (const auto &AliasScopePair : AliasScopeMap) {
|
2017-02-09 17:34:42 +08:00
|
|
|
if (Array->getBasePtr() == AliasScopePair.first)
|
2014-10-02 23:31:24 +08:00
|
|
|
continue;
|
|
|
|
|
2014-12-10 06:02:16 +08:00
|
|
|
Metadata *Args = {AliasScopePair.second};
|
2014-10-02 23:31:24 +08:00
|
|
|
AliasScopeList =
|
|
|
|
MDNode::concatenate(AliasScopeList, MDNode::get(Ctx, Args));
|
|
|
|
}
|
|
|
|
|
2017-02-09 17:34:42 +08:00
|
|
|
OtherAliasScopeListMap[Array->getBasePtr()] = AliasScopeList;
|
2014-10-02 23:31:24 +08:00
|
|
|
}
|
2014-03-04 22:59:00 +08:00
|
|
|
}
|
|
|
|
|
2014-10-02 23:32:17 +08:00
|
|
|
void ScopAnnotator::pushLoop(Loop *L, bool IsParallel) {
|
2014-10-02 23:31:24 +08:00
|
|
|
|
2014-10-02 04:10:44 +08:00
|
|
|
ActiveLoops.push_back(L);
|
|
|
|
if (!IsParallel)
|
|
|
|
return;
|
|
|
|
|
|
|
|
BasicBlock *Header = L->getHeader();
|
2014-10-02 23:31:24 +08:00
|
|
|
MDNode *Id = getID(Header->getContext());
|
2014-12-08 05:12:10 +08:00
|
|
|
assert(Id->getOperand(0) == Id && "Expected Id to be a self-reference");
|
|
|
|
assert(Id->getNumOperands() == 1 && "Unexpected extra operands in Id");
|
2014-10-02 04:10:44 +08:00
|
|
|
MDNode *Ids = ParallelLoops.empty()
|
2014-12-08 05:12:10 +08:00
|
|
|
? Id
|
2014-10-02 04:10:44 +08:00
|
|
|
: MDNode::concatenate(ParallelLoops.back(), Id);
|
|
|
|
ParallelLoops.push_back(Ids);
|
2014-03-04 22:59:00 +08:00
|
|
|
}
|
|
|
|
|
2014-10-02 23:32:17 +08:00
|
|
|
void ScopAnnotator::popLoop(bool IsParallel) {
|
2014-10-02 04:10:44 +08:00
|
|
|
ActiveLoops.pop_back();
|
|
|
|
if (!IsParallel)
|
|
|
|
return;
|
2014-03-04 22:59:00 +08:00
|
|
|
|
2014-10-02 04:10:44 +08:00
|
|
|
assert(!ParallelLoops.empty() && "Expected a parallel loop to pop");
|
|
|
|
ParallelLoops.pop_back();
|
2014-03-04 22:59:00 +08:00
|
|
|
}
|
|
|
|
|
2017-08-23 01:38:46 +08:00
|
|
|
void ScopAnnotator::annotateLoopLatch(BranchInst *B, Loop *L, bool IsParallel,
|
|
|
|
bool IsLoopVectorizerDisabled) const {
|
|
|
|
MDNode *MData = nullptr;
|
|
|
|
|
|
|
|
if (IsLoopVectorizerDisabled) {
|
|
|
|
SmallVector<Metadata *, 3> Args;
|
|
|
|
LLVMContext &Ctx = SE->getContext();
|
|
|
|
Args.push_back(MDString::get(Ctx, "llvm.loop.vectorize.enable"));
|
|
|
|
auto *FalseValue = ConstantInt::get(Type::getInt1Ty(Ctx), 0);
|
|
|
|
Args.push_back(ValueAsMetadata::get(FalseValue));
|
|
|
|
MData = MDNode::concatenate(MData, getID(Ctx, MDNode::get(Ctx, Args)));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (IsParallel) {
|
|
|
|
assert(!ParallelLoops.empty() && "Expected a parallel loop to annotate");
|
|
|
|
MDNode *Ids = ParallelLoops.back();
|
|
|
|
MDNode *Id = cast<MDNode>(Ids->getOperand(Ids->getNumOperands() - 1));
|
|
|
|
MData = MDNode::concatenate(MData, Id);
|
|
|
|
}
|
2014-03-04 22:59:00 +08:00
|
|
|
|
2017-08-23 01:38:46 +08:00
|
|
|
B->setMetadata("llvm.loop", MData);
|
2014-10-02 04:10:44 +08:00
|
|
|
}
|
|
|
|
|
2017-03-22 22:25:24 +08:00
|
|
|
/// Get the pointer operand
|
|
|
|
///
|
|
|
|
/// @param Inst The instruction to be analyzed.
|
|
|
|
/// @return the pointer operand in case @p Inst is a memory access
|
|
|
|
/// instruction and nullptr otherwise.
|
|
|
|
static llvm::Value *getMemAccInstPointerOperand(Instruction *Inst) {
|
|
|
|
auto MemInst = MemAccInst::dyn_cast(Inst);
|
|
|
|
if (!MemInst)
|
|
|
|
return nullptr;
|
|
|
|
|
|
|
|
return MemInst.getPointerOperand();
|
|
|
|
}
|
|
|
|
|
|
|
|
void ScopAnnotator::annotateSecondLevel(llvm::Instruction *Inst,
|
|
|
|
llvm::Value *BasePtr) {
|
2017-08-09 00:50:28 +08:00
|
|
|
auto *PtrSCEV = SE->getSCEV(getMemAccInstPointerOperand(Inst));
|
|
|
|
auto *BasePtrSCEV = SE->getPointerBase(PtrSCEV);
|
|
|
|
|
|
|
|
if (!PtrSCEV)
|
2017-03-22 22:25:24 +08:00
|
|
|
return;
|
2017-08-09 00:50:28 +08:00
|
|
|
auto SecondLevelAliasScope = SecondLevelAliasScopeMap.lookup(PtrSCEV);
|
2017-03-22 22:25:24 +08:00
|
|
|
auto SecondLevelOtherAliasScopeList =
|
2017-08-09 00:50:28 +08:00
|
|
|
SecondLevelOtherAliasScopeListMap.lookup(PtrSCEV);
|
2017-03-22 22:25:24 +08:00
|
|
|
if (!SecondLevelAliasScope) {
|
|
|
|
auto AliasScope = AliasScopeMap.lookup(BasePtr);
|
|
|
|
if (!AliasScope)
|
|
|
|
return;
|
|
|
|
LLVMContext &Ctx = SE->getContext();
|
|
|
|
SecondLevelAliasScope = getID(
|
|
|
|
Ctx, AliasScope, MDString::get(Ctx, "second level alias metadata"));
|
2017-08-09 00:50:28 +08:00
|
|
|
SecondLevelAliasScopeMap[PtrSCEV] = SecondLevelAliasScope;
|
2017-03-22 22:25:24 +08:00
|
|
|
Metadata *Args = {SecondLevelAliasScope};
|
|
|
|
auto SecondLevelBasePtrAliasScopeList =
|
2017-08-09 00:50:28 +08:00
|
|
|
SecondLevelAliasScopeMap.lookup(BasePtrSCEV);
|
|
|
|
SecondLevelAliasScopeMap[BasePtrSCEV] = MDNode::concatenate(
|
2017-03-22 22:25:24 +08:00
|
|
|
SecondLevelBasePtrAliasScopeList, MDNode::get(Ctx, Args));
|
|
|
|
auto OtherAliasScopeList = OtherAliasScopeListMap.lookup(BasePtr);
|
|
|
|
SecondLevelOtherAliasScopeList = MDNode::concatenate(
|
|
|
|
OtherAliasScopeList, SecondLevelBasePtrAliasScopeList);
|
2017-08-09 00:50:28 +08:00
|
|
|
SecondLevelOtherAliasScopeListMap[PtrSCEV] = SecondLevelOtherAliasScopeList;
|
2017-03-22 22:25:24 +08:00
|
|
|
}
|
|
|
|
Inst->setMetadata("alias.scope", SecondLevelAliasScope);
|
|
|
|
Inst->setMetadata("noalias", SecondLevelOtherAliasScopeList);
|
|
|
|
}
|
|
|
|
|
2014-10-02 23:32:17 +08:00
|
|
|
void ScopAnnotator::annotate(Instruction *Inst) {
|
2014-10-02 23:31:24 +08:00
|
|
|
if (!Inst->mayReadOrWriteMemory())
|
|
|
|
return;
|
|
|
|
|
2015-10-07 21:19:03 +08:00
|
|
|
if (!ParallelLoops.empty())
|
|
|
|
Inst->setMetadata("llvm.mem.parallel_loop_access", ParallelLoops.back());
|
|
|
|
|
2014-10-02 23:31:24 +08:00
|
|
|
// TODO: Use the ScopArrayInfo once available here.
|
2015-10-07 21:19:03 +08:00
|
|
|
if (!AliasScopeDomain)
|
|
|
|
return;
|
2014-10-02 23:31:24 +08:00
|
|
|
|
2017-03-22 22:25:24 +08:00
|
|
|
auto *Ptr = getMemAccInstPointerOperand(Inst);
|
2016-02-25 22:08:48 +08:00
|
|
|
if (!Ptr)
|
|
|
|
return;
|
|
|
|
|
|
|
|
auto *PtrSCEV = SE->getSCEV(Ptr);
|
2015-10-07 21:19:03 +08:00
|
|
|
auto *BaseSCEV = SE->getPointerBase(PtrSCEV);
|
|
|
|
auto *SU = dyn_cast<SCEVUnknown>(BaseSCEV);
|
2015-08-20 00:04:35 +08:00
|
|
|
|
2015-10-07 21:19:03 +08:00
|
|
|
if (!SU)
|
|
|
|
return;
|
2015-08-20 00:04:35 +08:00
|
|
|
|
2015-10-07 21:19:03 +08:00
|
|
|
auto *BasePtr = SU->getValue();
|
2014-10-02 23:31:24 +08:00
|
|
|
|
2015-10-07 21:19:03 +08:00
|
|
|
if (!BasePtr)
|
2014-03-04 22:59:00 +08:00
|
|
|
return;
|
|
|
|
|
2015-10-07 21:35:20 +08:00
|
|
|
auto AliasScope = AliasScopeMap.lookup(BasePtr);
|
2015-10-07 21:19:03 +08:00
|
|
|
|
2015-10-07 21:35:20 +08:00
|
|
|
if (!AliasScope) {
|
|
|
|
BasePtr = AlternativeAliasBases.lookup(BasePtr);
|
|
|
|
if (!BasePtr)
|
2015-10-07 21:19:06 +08:00
|
|
|
return;
|
2015-10-07 21:19:03 +08:00
|
|
|
|
2015-10-07 21:35:20 +08:00
|
|
|
AliasScope = AliasScopeMap.lookup(BasePtr);
|
|
|
|
if (!AliasScope)
|
2015-10-07 21:19:06 +08:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
assert(OtherAliasScopeListMap.count(BasePtr) &&
|
|
|
|
"BasePtr either expected in AliasScopeMap and OtherAlias...Map");
|
2015-10-07 21:19:03 +08:00
|
|
|
auto *OtherAliasScopeList = OtherAliasScopeListMap[BasePtr];
|
|
|
|
|
2017-03-22 22:25:24 +08:00
|
|
|
if (InterIterationAliasFreeBasePtrs.count(BasePtr)) {
|
|
|
|
annotateSecondLevel(Inst, BasePtr);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2015-10-07 21:19:03 +08:00
|
|
|
Inst->setMetadata("alias.scope", AliasScope);
|
|
|
|
Inst->setMetadata("noalias", OtherAliasScopeList);
|
2014-03-04 22:59:00 +08:00
|
|
|
}
|
2017-03-22 22:25:24 +08:00
|
|
|
|
|
|
|
void ScopAnnotator::addInterIterationAliasFreeBasePtr(llvm::Value *BasePtr) {
|
|
|
|
if (!BasePtr)
|
|
|
|
return;
|
|
|
|
|
|
|
|
InterIterationAliasFreeBasePtrs.insert(BasePtr);
|
|
|
|
}
|