2015-02-02 00:56:15 +08:00
|
|
|
//===- LoopAccessAnalysis.cpp - Loop Access Analysis Implementation --------==//
|
|
|
|
//
|
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
//
|
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
// License. See LICENSE.TXT for details.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
|
|
|
// The implementation for the loop memory dependence that was originally
|
|
|
|
// developed for the loop vectorizer.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
#include "llvm/Analysis/LoopAccessAnalysis.h"
|
|
|
|
#include "llvm/Analysis/LoopInfo.h"
|
2015-02-07 02:31:04 +08:00
|
|
|
#include "llvm/Analysis/ScalarEvolutionExpander.h"
|
2015-03-24 03:32:43 +08:00
|
|
|
#include "llvm/Analysis/TargetLibraryInfo.h"
|
2015-02-02 00:56:15 +08:00
|
|
|
#include "llvm/Analysis/ValueTracking.h"
|
|
|
|
#include "llvm/IR/DiagnosticInfo.h"
|
|
|
|
#include "llvm/IR/Dominators.h"
|
2015-02-07 02:31:04 +08:00
|
|
|
#include "llvm/IR/IRBuilder.h"
|
2015-02-02 00:56:15 +08:00
|
|
|
#include "llvm/Support/Debug.h"
|
2015-03-24 03:32:43 +08:00
|
|
|
#include "llvm/Support/raw_ostream.h"
|
2015-06-27 02:02:52 +08:00
|
|
|
#include "llvm/Analysis/VectorUtils.h"
|
2015-02-02 00:56:15 +08:00
|
|
|
using namespace llvm;
|
|
|
|
|
2015-02-20 03:15:07 +08:00
|
|
|
#define DEBUG_TYPE "loop-accesses"
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-02-20 03:14:52 +08:00
|
|
|
static cl::opt<unsigned, true>
|
|
|
|
VectorizationFactor("force-vector-width", cl::Hidden,
|
|
|
|
cl::desc("Sets the SIMD width. Zero is autoselect."),
|
|
|
|
cl::location(VectorizerParams::VectorizationFactor));
|
2015-02-26 12:39:09 +08:00
|
|
|
unsigned VectorizerParams::VectorizationFactor;
|
2015-02-20 03:14:52 +08:00
|
|
|
|
|
|
|
static cl::opt<unsigned, true>
|
|
|
|
VectorizationInterleave("force-vector-interleave", cl::Hidden,
|
|
|
|
cl::desc("Sets the vectorization interleave count. "
|
|
|
|
"Zero is autoselect."),
|
|
|
|
cl::location(
|
|
|
|
VectorizerParams::VectorizationInterleave));
|
2015-02-26 12:39:09 +08:00
|
|
|
unsigned VectorizerParams::VectorizationInterleave;
|
2015-02-20 03:14:52 +08:00
|
|
|
|
2015-02-26 12:39:09 +08:00
|
|
|
static cl::opt<unsigned, true> RuntimeMemoryCheckThreshold(
|
|
|
|
"runtime-memory-check-threshold", cl::Hidden,
|
|
|
|
cl::desc("When performing memory disambiguation checks at runtime do not "
|
|
|
|
"generate more than this number of comparisons (default = 8)."),
|
|
|
|
cl::location(VectorizerParams::RuntimeMemoryCheckThreshold), cl::init(8));
|
|
|
|
unsigned VectorizerParams::RuntimeMemoryCheckThreshold;
|
2015-02-20 03:14:52 +08:00
|
|
|
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
/// \brief The maximum iterations used to merge memory checks
|
|
|
|
static cl::opt<unsigned> MemoryCheckMergeThreshold(
|
|
|
|
"memory-check-merge-threshold", cl::Hidden,
|
|
|
|
cl::desc("Maximum number of comparisons done when trying to merge "
|
|
|
|
"runtime memory checks. (default = 100)"),
|
|
|
|
cl::init(100));
|
|
|
|
|
2015-02-20 03:14:52 +08:00
|
|
|
/// Maximum SIMD width.
|
|
|
|
const unsigned VectorizerParams::MaxVectorWidth = 64;
|
|
|
|
|
2015-03-11 01:40:37 +08:00
|
|
|
/// \brief We collect interesting dependences up to this threshold.
|
|
|
|
static cl::opt<unsigned> MaxInterestingDependence(
|
|
|
|
"max-interesting-dependences", cl::Hidden,
|
|
|
|
cl::desc("Maximum number of interesting dependences collected by "
|
|
|
|
"loop-access analysis (default = 100)"),
|
|
|
|
cl::init(100));
|
|
|
|
|
2015-02-20 03:14:52 +08:00
|
|
|
bool VectorizerParams::isInterleaveForced() {
|
|
|
|
return ::VectorizationInterleave.getNumOccurrences() > 0;
|
|
|
|
}
|
|
|
|
|
2015-02-20 03:15:15 +08:00
|
|
|
void LoopAccessReport::emitAnalysis(const LoopAccessReport &Message,
|
|
|
|
const Function *TheFunction,
|
|
|
|
const Loop *TheLoop,
|
|
|
|
const char *PassName) {
|
2015-02-02 00:56:15 +08:00
|
|
|
DebugLoc DL = TheLoop->getStartLoc();
|
2015-02-20 03:15:13 +08:00
|
|
|
if (const Instruction *I = Message.getInstr())
|
2015-02-02 00:56:15 +08:00
|
|
|
DL = I->getDebugLoc();
|
2015-02-20 03:15:07 +08:00
|
|
|
emitOptimizationRemarkAnalysis(TheFunction->getContext(), PassName,
|
2015-02-02 00:56:15 +08:00
|
|
|
*TheFunction, DL, Message.str());
|
|
|
|
}
|
|
|
|
|
|
|
|
Value *llvm::stripIntegerCast(Value *V) {
|
|
|
|
if (CastInst *CI = dyn_cast<CastInst>(V))
|
|
|
|
if (CI->getOperand(0)->getType()->isIntegerTy())
|
|
|
|
return CI->getOperand(0);
|
|
|
|
return V;
|
|
|
|
}
|
|
|
|
|
|
|
|
const SCEV *llvm::replaceSymbolicStrideSCEV(ScalarEvolution *SE,
|
2015-02-24 08:41:59 +08:00
|
|
|
const ValueToValueMap &PtrToStride,
|
2015-02-02 00:56:15 +08:00
|
|
|
Value *Ptr, Value *OrigPtr) {
|
|
|
|
|
|
|
|
const SCEV *OrigSCEV = SE->getSCEV(Ptr);
|
|
|
|
|
|
|
|
// If there is an entry in the map return the SCEV of the pointer with the
|
|
|
|
// symbolic stride replaced by one.
|
2015-02-24 08:41:59 +08:00
|
|
|
ValueToValueMap::const_iterator SI =
|
|
|
|
PtrToStride.find(OrigPtr ? OrigPtr : Ptr);
|
2015-02-02 00:56:15 +08:00
|
|
|
if (SI != PtrToStride.end()) {
|
|
|
|
Value *StrideVal = SI->second;
|
|
|
|
|
|
|
|
// Strip casts.
|
|
|
|
StrideVal = stripIntegerCast(StrideVal);
|
|
|
|
|
|
|
|
// Replace symbolic stride by one.
|
|
|
|
Value *One = ConstantInt::get(StrideVal->getType(), 1);
|
|
|
|
ValueToValueMap RewriteMap;
|
|
|
|
RewriteMap[StrideVal] = One;
|
|
|
|
|
|
|
|
const SCEV *ByOne =
|
|
|
|
SCEVParameterRewriter::rewrite(OrigSCEV, *SE, RewriteMap, true);
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Replacing SCEV: " << *OrigSCEV << " by: " << *ByOne
|
2015-02-02 00:56:15 +08:00
|
|
|
<< "\n");
|
|
|
|
return ByOne;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Otherwise, just return the SCEV of the original pointer.
|
|
|
|
return SE->getSCEV(Ptr);
|
|
|
|
}
|
|
|
|
|
2015-07-15 06:32:44 +08:00
|
|
|
void RuntimePointerChecking::insert(Loop *Lp, Value *Ptr, bool WritePtr,
|
|
|
|
unsigned DepSetId, unsigned ASId,
|
|
|
|
const ValueToValueMap &Strides) {
|
2015-02-02 00:56:15 +08:00
|
|
|
// Get the stride replaced scev.
|
|
|
|
const SCEV *Sc = replaceSymbolicStrideSCEV(SE, Strides, Ptr);
|
|
|
|
const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Sc);
|
|
|
|
assert(AR && "Invalid addrec expression");
|
|
|
|
const SCEV *Ex = SE->getBackedgeTakenCount(Lp);
|
2015-07-16 22:02:58 +08:00
|
|
|
|
|
|
|
const SCEV *ScStart = AR->getStart();
|
2015-02-02 00:56:15 +08:00
|
|
|
const SCEV *ScEnd = AR->evaluateAtIteration(Ex, *SE);
|
2015-07-16 22:02:58 +08:00
|
|
|
const SCEV *Step = AR->getStepRecurrence(*SE);
|
|
|
|
|
|
|
|
// For expressions with negative step, the upper bound is ScStart and the
|
|
|
|
// lower bound is ScEnd.
|
|
|
|
if (const SCEVConstant *CStep = dyn_cast<const SCEVConstant>(Step)) {
|
|
|
|
if (CStep->getValue()->isNegative())
|
|
|
|
std::swap(ScStart, ScEnd);
|
|
|
|
} else {
|
|
|
|
// Fallback case: the step is not constant, but the we can still
|
|
|
|
// get the upper and lower bounds of the interval by using min/max
|
|
|
|
// expressions.
|
|
|
|
ScStart = SE->getUMinExpr(ScStart, ScEnd);
|
|
|
|
ScEnd = SE->getUMaxExpr(AR->getStart(), ScEnd);
|
|
|
|
}
|
|
|
|
|
|
|
|
Pointers.emplace_back(Ptr, ScStart, ScEnd, WritePtr, DepSetId, ASId, Sc);
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
}
|
|
|
|
|
2015-07-28 03:38:48 +08:00
|
|
|
SmallVector<RuntimePointerChecking::PointerCheck, 4>
|
2015-08-10 04:06:06 +08:00
|
|
|
RuntimePointerChecking::generateChecks() const {
|
2015-07-28 03:38:48 +08:00
|
|
|
SmallVector<PointerCheck, 4> Checks;
|
|
|
|
|
2015-07-28 03:38:50 +08:00
|
|
|
for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
|
|
|
|
for (unsigned J = I + 1; J < CheckingGroups.size(); ++J) {
|
|
|
|
const RuntimePointerChecking::CheckingPtrGroup &CGI = CheckingGroups[I];
|
|
|
|
const RuntimePointerChecking::CheckingPtrGroup &CGJ = CheckingGroups[J];
|
2015-07-28 03:38:48 +08:00
|
|
|
|
2015-08-10 04:06:06 +08:00
|
|
|
if (needsChecking(CGI, CGJ))
|
2015-07-28 03:38:48 +08:00
|
|
|
Checks.push_back(std::make_pair(&CGI, &CGJ));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return Checks;
|
|
|
|
}
|
|
|
|
|
2015-08-08 06:44:15 +08:00
|
|
|
void RuntimePointerChecking::generateChecks(
|
|
|
|
MemoryDepChecker::DepCandidates &DepCands, bool UseDependencies) {
|
|
|
|
assert(Checks.empty() && "Checks is not empty");
|
|
|
|
groupChecks(DepCands, UseDependencies);
|
|
|
|
Checks = generateChecks();
|
|
|
|
}
|
|
|
|
|
2015-07-15 06:32:44 +08:00
|
|
|
bool RuntimePointerChecking::needsChecking(
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
const CheckingPtrGroup &M, const CheckingPtrGroup &N,
|
|
|
|
const SmallVectorImpl<int> *PtrPartition) const {
|
|
|
|
for (unsigned I = 0, EI = M.Members.size(); EI != I; ++I)
|
|
|
|
for (unsigned J = 0, EJ = N.Members.size(); EJ != J; ++J)
|
|
|
|
if (needsChecking(M.Members[I], N.Members[J], PtrPartition))
|
|
|
|
return true;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Compare \p I and \p J and return the minimum.
|
|
|
|
/// Return nullptr in case we couldn't find an answer.
|
|
|
|
static const SCEV *getMinFromExprs(const SCEV *I, const SCEV *J,
|
|
|
|
ScalarEvolution *SE) {
|
|
|
|
const SCEV *Diff = SE->getMinusSCEV(J, I);
|
|
|
|
const SCEVConstant *C = dyn_cast<const SCEVConstant>(Diff);
|
|
|
|
|
|
|
|
if (!C)
|
|
|
|
return nullptr;
|
|
|
|
if (C->getValue()->isNegative())
|
|
|
|
return J;
|
|
|
|
return I;
|
|
|
|
}
|
|
|
|
|
2015-07-15 06:32:44 +08:00
|
|
|
bool RuntimePointerChecking::CheckingPtrGroup::addPointer(unsigned Index) {
|
2015-07-15 06:32:50 +08:00
|
|
|
const SCEV *Start = RtCheck.Pointers[Index].Start;
|
|
|
|
const SCEV *End = RtCheck.Pointers[Index].End;
|
|
|
|
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
// Compare the starts and ends with the known minimum and maximum
|
|
|
|
// of this set. We need to know how we compare against the min/max
|
|
|
|
// of the set in order to be able to emit memchecks.
|
2015-07-15 06:32:50 +08:00
|
|
|
const SCEV *Min0 = getMinFromExprs(Start, Low, RtCheck.SE);
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
if (!Min0)
|
|
|
|
return false;
|
|
|
|
|
2015-07-15 06:32:50 +08:00
|
|
|
const SCEV *Min1 = getMinFromExprs(End, High, RtCheck.SE);
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
if (!Min1)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
// Update the low bound expression if we've found a new min value.
|
2015-07-15 06:32:50 +08:00
|
|
|
if (Min0 == Start)
|
|
|
|
Low = Start;
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
|
|
|
|
// Update the high bound expression if we've found a new max value.
|
2015-07-15 06:32:50 +08:00
|
|
|
if (Min1 != End)
|
|
|
|
High = End;
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
|
|
|
|
Members.push_back(Index);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2015-07-15 06:32:44 +08:00
|
|
|
void RuntimePointerChecking::groupChecks(
|
|
|
|
MemoryDepChecker::DepCandidates &DepCands, bool UseDependencies) {
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
// We build the groups from dependency candidates equivalence classes
|
|
|
|
// because:
|
|
|
|
// - We know that pointers in the same equivalence class share
|
|
|
|
// the same underlying object and therefore there is a chance
|
|
|
|
// that we can compare pointers
|
|
|
|
// - We wouldn't be able to merge two pointers for which we need
|
|
|
|
// to emit a memcheck. The classes in DepCands are already
|
|
|
|
// conveniently built such that no two pointers in the same
|
|
|
|
// class need checking against each other.
|
|
|
|
|
|
|
|
// We use the following (greedy) algorithm to construct the groups
|
|
|
|
// For every pointer in the equivalence class:
|
|
|
|
// For each existing group:
|
|
|
|
// - if the difference between this pointer and the min/max bounds
|
|
|
|
// of the group is a constant, then make the pointer part of the
|
|
|
|
// group and update the min/max bounds of that group as required.
|
|
|
|
|
|
|
|
CheckingGroups.clear();
|
|
|
|
|
2015-07-28 21:44:08 +08:00
|
|
|
// If we need to check two pointers to the same underlying object
|
|
|
|
// with a non-constant difference, we shouldn't perform any pointer
|
|
|
|
// grouping with those pointers. This is because we can easily get
|
|
|
|
// into cases where the resulting check would return false, even when
|
|
|
|
// the accesses are safe.
|
|
|
|
//
|
|
|
|
// The following example shows this:
|
|
|
|
// for (i = 0; i < 1000; ++i)
|
|
|
|
// a[5000 + i * m] = a[i] + a[i + 9000]
|
|
|
|
//
|
|
|
|
// Here grouping gives a check of (5000, 5000 + 1000 * m) against
|
|
|
|
// (0, 10000) which is always false. However, if m is 1, there is no
|
|
|
|
// dependence. Not grouping the checks for a[i] and a[i + 9000] allows
|
|
|
|
// us to perform an accurate check in this case.
|
|
|
|
//
|
|
|
|
// The above case requires that we have an UnknownDependence between
|
|
|
|
// accesses to the same underlying object. This cannot happen unless
|
|
|
|
// ShouldRetryWithRuntimeCheck is set, and therefore UseDependencies
|
|
|
|
// is also false. In this case we will use the fallback path and create
|
|
|
|
// separate checking groups for all pointers.
|
|
|
|
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
// If we don't have the dependency partitions, construct a new
|
2015-07-28 21:44:08 +08:00
|
|
|
// checking pointer group for each pointer. This is also required
|
|
|
|
// for correctness, because in this case we can have checking between
|
|
|
|
// pointers to the same underlying object.
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
if (!UseDependencies) {
|
|
|
|
for (unsigned I = 0; I < Pointers.size(); ++I)
|
|
|
|
CheckingGroups.push_back(CheckingPtrGroup(I, *this));
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned TotalComparisons = 0;
|
|
|
|
|
|
|
|
DenseMap<Value *, unsigned> PositionMap;
|
2015-07-15 06:32:50 +08:00
|
|
|
for (unsigned Index = 0; Index < Pointers.size(); ++Index)
|
|
|
|
PositionMap[Pointers[Index].PointerValue] = Index;
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
|
2015-07-09 23:18:25 +08:00
|
|
|
// We need to keep track of what pointers we've already seen so we
|
|
|
|
// don't process them twice.
|
|
|
|
SmallSet<unsigned, 2> Seen;
|
|
|
|
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
// Go through all equivalence classes, get the the "pointer check groups"
|
2015-07-09 23:18:25 +08:00
|
|
|
// and add them to the overall solution. We use the order in which accesses
|
|
|
|
// appear in 'Pointers' to enforce determinism.
|
|
|
|
for (unsigned I = 0; I < Pointers.size(); ++I) {
|
|
|
|
// We've seen this pointer before, and therefore already processed
|
|
|
|
// its equivalence class.
|
|
|
|
if (Seen.count(I))
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
continue;
|
|
|
|
|
2015-07-15 06:32:50 +08:00
|
|
|
MemoryDepChecker::MemAccessInfo Access(Pointers[I].PointerValue,
|
|
|
|
Pointers[I].IsWritePtr);
|
2015-07-09 23:18:25 +08:00
|
|
|
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
SmallVector<CheckingPtrGroup, 2> Groups;
|
2015-07-09 23:18:25 +08:00
|
|
|
auto LeaderI = DepCands.findValue(DepCands.getLeaderValue(Access));
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
|
2015-07-13 22:48:24 +08:00
|
|
|
// Because DepCands is constructed by visiting accesses in the order in
|
|
|
|
// which they appear in alias sets (which is deterministic) and the
|
|
|
|
// iteration order within an equivalence class member is only dependent on
|
|
|
|
// the order in which unions and insertions are performed on the
|
|
|
|
// equivalence class, the iteration order is deterministic.
|
2015-07-09 23:18:25 +08:00
|
|
|
for (auto MI = DepCands.member_begin(LeaderI), ME = DepCands.member_end();
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
MI != ME; ++MI) {
|
|
|
|
unsigned Pointer = PositionMap[MI->getPointer()];
|
|
|
|
bool Merged = false;
|
2015-07-09 23:18:25 +08:00
|
|
|
// Mark this pointer as seen.
|
|
|
|
Seen.insert(Pointer);
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
|
|
|
|
// Go through all the existing sets and see if we can find one
|
|
|
|
// which can include this pointer.
|
|
|
|
for (CheckingPtrGroup &Group : Groups) {
|
|
|
|
// Don't perform more than a certain amount of comparisons.
|
|
|
|
// This should limit the cost of grouping the pointers to something
|
|
|
|
// reasonable. If we do end up hitting this threshold, the algorithm
|
|
|
|
// will create separate groups for all remaining pointers.
|
|
|
|
if (TotalComparisons > MemoryCheckMergeThreshold)
|
|
|
|
break;
|
|
|
|
|
|
|
|
TotalComparisons++;
|
|
|
|
|
|
|
|
if (Group.addPointer(Pointer)) {
|
|
|
|
Merged = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!Merged)
|
|
|
|
// We couldn't add this pointer to any existing set or the threshold
|
|
|
|
// for the number of comparisons has been reached. Create a new group
|
|
|
|
// to hold the current pointer.
|
|
|
|
Groups.push_back(CheckingPtrGroup(Pointer, *this));
|
|
|
|
}
|
|
|
|
|
|
|
|
// We've computed the grouped checks for this partition.
|
|
|
|
// Save the results and continue with the next one.
|
|
|
|
std::copy(Groups.begin(), Groups.end(), std::back_inserter(CheckingGroups));
|
|
|
|
}
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-07-16 10:48:05 +08:00
|
|
|
bool RuntimePointerChecking::arePointersInSamePartition(
|
|
|
|
const SmallVectorImpl<int> &PtrToPartition, unsigned PtrIdx1,
|
|
|
|
unsigned PtrIdx2) {
|
|
|
|
return (PtrToPartition[PtrIdx1] != -1 &&
|
|
|
|
PtrToPartition[PtrIdx1] == PtrToPartition[PtrIdx2]);
|
|
|
|
}
|
|
|
|
|
2015-07-15 06:32:44 +08:00
|
|
|
bool RuntimePointerChecking::needsChecking(
|
2015-03-11 02:54:26 +08:00
|
|
|
unsigned I, unsigned J, const SmallVectorImpl<int> *PtrPartition) const {
|
2015-07-15 06:32:50 +08:00
|
|
|
const PointerInfo &PointerI = Pointers[I];
|
|
|
|
const PointerInfo &PointerJ = Pointers[J];
|
|
|
|
|
2015-02-18 11:43:58 +08:00
|
|
|
// No need to check if two readonly pointers intersect.
|
2015-07-15 06:32:50 +08:00
|
|
|
if (!PointerI.IsWritePtr && !PointerJ.IsWritePtr)
|
2015-02-18 11:43:58 +08:00
|
|
|
return false;
|
|
|
|
|
|
|
|
// Only need to check pointers between two different dependency sets.
|
2015-07-15 06:32:50 +08:00
|
|
|
if (PointerI.DependencySetId == PointerJ.DependencySetId)
|
2015-02-18 11:43:58 +08:00
|
|
|
return false;
|
|
|
|
|
|
|
|
// Only need to check pointers in the same alias set.
|
2015-07-15 06:32:50 +08:00
|
|
|
if (PointerI.AliasSetId != PointerJ.AliasSetId)
|
2015-02-18 11:43:58 +08:00
|
|
|
return false;
|
|
|
|
|
2015-03-11 02:54:26 +08:00
|
|
|
// If PtrPartition is set omit checks between pointers of the same partition.
|
2015-07-16 10:48:05 +08:00
|
|
|
if (PtrPartition && arePointersInSamePartition(*PtrPartition, I, J))
|
2015-03-11 02:54:26 +08:00
|
|
|
return false;
|
|
|
|
|
2015-02-18 11:43:58 +08:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
[LAA] Split out a helper to print a collection of memchecks
This is effectively an NFC but we can no longer print the index of the
pointer group so instead I print its address. This still lets us
cross-check the section that list the checks against the section that
list the groups (see how I modified the test).
E.g. before we printed this:
Run-time memory checks:
Check 0:
Comparing group 0:
%arrayidxC = getelementptr inbounds i16, i16* %c, i64 %store_ind
%arrayidxC1 = getelementptr inbounds i16, i16* %c, i64 %store_ind_inc
Against group 1:
%arrayidxA = getelementptr i16, i16* %a, i64 %ind
%arrayidxA1 = getelementptr i16, i16* %a, i64 %add
...
Grouped accesses:
Group 0:
(Low: %c High: (78 + %c))
Member: {%c,+,4}<%for.body>
Member: {(2 + %c),+,4}<%for.body>
Now we print this (changes are underlined):
Run-time memory checks:
Check 0:
Comparing group (0x7f9c6040c320):
~~~~~~~~~~~~~~
%arrayidxC1 = getelementptr inbounds i16, i16* %c, i64 %store_ind_inc
%arrayidxC = getelementptr inbounds i16, i16* %c, i64 %store_ind
Against group (0x7f9c6040c358):
~~~~~~~~~~~~~~
%arrayidxA1 = getelementptr i16, i16* %a, i64 %add
%arrayidxA = getelementptr i16, i16* %a, i64 %ind
...
Grouped accesses:
Group 0x7f9c6040c320:
~~~~~~~~~~~~~~
(Low: %c High: (78 + %c))
Member: {(2 + %c),+,4}<%for.body>
Member: {%c,+,4}<%for.body>
llvm-svn: 243354
2015-07-28 07:54:41 +08:00
|
|
|
void RuntimePointerChecking::printChecks(
|
|
|
|
raw_ostream &OS, const SmallVectorImpl<PointerCheck> &Checks,
|
|
|
|
unsigned Depth) const {
|
|
|
|
unsigned N = 0;
|
|
|
|
for (const auto &Check : Checks) {
|
|
|
|
const auto &First = Check.first->Members, &Second = Check.second->Members;
|
|
|
|
|
|
|
|
OS.indent(Depth) << "Check " << N++ << ":\n";
|
|
|
|
|
|
|
|
OS.indent(Depth + 2) << "Comparing group (" << Check.first << "):\n";
|
|
|
|
for (unsigned K = 0; K < First.size(); ++K)
|
|
|
|
OS.indent(Depth + 2) << *Pointers[First[K]].PointerValue << "\n";
|
|
|
|
|
|
|
|
OS.indent(Depth + 2) << "Against group (" << Check.second << "):\n";
|
|
|
|
for (unsigned K = 0; K < Second.size(); ++K)
|
|
|
|
OS.indent(Depth + 2) << *Pointers[Second[K]].PointerValue << "\n";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-08-08 03:44:48 +08:00
|
|
|
void RuntimePointerChecking::print(raw_ostream &OS, unsigned Depth) const {
|
2015-02-20 03:15:19 +08:00
|
|
|
|
|
|
|
OS.indent(Depth) << "Run-time memory checks:\n";
|
2015-08-08 06:44:15 +08:00
|
|
|
printChecks(OS, Checks, Depth);
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
|
|
|
|
OS.indent(Depth) << "Grouped accesses:\n";
|
|
|
|
for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
|
[LAA] Split out a helper to print a collection of memchecks
This is effectively an NFC but we can no longer print the index of the
pointer group so instead I print its address. This still lets us
cross-check the section that list the checks against the section that
list the groups (see how I modified the test).
E.g. before we printed this:
Run-time memory checks:
Check 0:
Comparing group 0:
%arrayidxC = getelementptr inbounds i16, i16* %c, i64 %store_ind
%arrayidxC1 = getelementptr inbounds i16, i16* %c, i64 %store_ind_inc
Against group 1:
%arrayidxA = getelementptr i16, i16* %a, i64 %ind
%arrayidxA1 = getelementptr i16, i16* %a, i64 %add
...
Grouped accesses:
Group 0:
(Low: %c High: (78 + %c))
Member: {%c,+,4}<%for.body>
Member: {(2 + %c),+,4}<%for.body>
Now we print this (changes are underlined):
Run-time memory checks:
Check 0:
Comparing group (0x7f9c6040c320):
~~~~~~~~~~~~~~
%arrayidxC1 = getelementptr inbounds i16, i16* %c, i64 %store_ind_inc
%arrayidxC = getelementptr inbounds i16, i16* %c, i64 %store_ind
Against group (0x7f9c6040c358):
~~~~~~~~~~~~~~
%arrayidxA1 = getelementptr i16, i16* %a, i64 %add
%arrayidxA = getelementptr i16, i16* %a, i64 %ind
...
Grouped accesses:
Group 0x7f9c6040c320:
~~~~~~~~~~~~~~
(Low: %c High: (78 + %c))
Member: {(2 + %c),+,4}<%for.body>
Member: {%c,+,4}<%for.body>
llvm-svn: 243354
2015-07-28 07:54:41 +08:00
|
|
|
const auto &CG = CheckingGroups[I];
|
|
|
|
|
|
|
|
OS.indent(Depth + 2) << "Group " << &CG << ":\n";
|
|
|
|
OS.indent(Depth + 4) << "(Low: " << *CG.Low << " High: " << *CG.High
|
|
|
|
<< ")\n";
|
|
|
|
for (unsigned J = 0; J < CG.Members.size(); ++J) {
|
|
|
|
OS.indent(Depth + 6) << "Member: " << *Pointers[CG.Members[J]].Expr
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
<< "\n";
|
|
|
|
}
|
|
|
|
}
|
2015-02-20 03:15:19 +08:00
|
|
|
}
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
namespace {
|
|
|
|
/// \brief Analyses memory accesses in a loop.
|
|
|
|
///
|
|
|
|
/// Checks whether run time pointer checks are needed and builds sets for data
|
|
|
|
/// dependence checking.
|
|
|
|
class AccessAnalysis {
|
|
|
|
public:
|
|
|
|
/// \brief Read or write access location.
|
|
|
|
typedef PointerIntPair<Value *, 1, bool> MemAccessInfo;
|
|
|
|
typedef SmallPtrSet<MemAccessInfo, 8> MemAccessInfoSet;
|
|
|
|
|
2015-04-24 04:09:20 +08:00
|
|
|
AccessAnalysis(const DataLayout &Dl, AliasAnalysis *AA, LoopInfo *LI,
|
2015-03-11 01:40:34 +08:00
|
|
|
MemoryDepChecker::DepCandidates &DA)
|
2015-07-09 14:47:18 +08:00
|
|
|
: DL(Dl), AST(*AA), LI(LI), DepCands(DA),
|
|
|
|
IsRTCheckAnalysisNeeded(false) {}
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
/// \brief Register a load and whether it is only read from.
|
2015-06-17 15:18:54 +08:00
|
|
|
void addLoad(MemoryLocation &Loc, bool IsReadOnly) {
|
2015-02-02 00:56:15 +08:00
|
|
|
Value *Ptr = const_cast<Value*>(Loc.Ptr);
|
2015-06-17 15:21:38 +08:00
|
|
|
AST.add(Ptr, MemoryLocation::UnknownSize, Loc.AATags);
|
2015-02-02 00:56:15 +08:00
|
|
|
Accesses.insert(MemAccessInfo(Ptr, false));
|
|
|
|
if (IsReadOnly)
|
|
|
|
ReadOnlyPtr.insert(Ptr);
|
|
|
|
}
|
|
|
|
|
|
|
|
/// \brief Register a store.
|
2015-06-17 15:18:54 +08:00
|
|
|
void addStore(MemoryLocation &Loc) {
|
2015-02-02 00:56:15 +08:00
|
|
|
Value *Ptr = const_cast<Value*>(Loc.Ptr);
|
2015-06-17 15:21:38 +08:00
|
|
|
AST.add(Ptr, MemoryLocation::UnknownSize, Loc.AATags);
|
2015-02-02 00:56:15 +08:00
|
|
|
Accesses.insert(MemAccessInfo(Ptr, true));
|
|
|
|
}
|
|
|
|
|
|
|
|
/// \brief Check whether we can check the pointers at runtime for
|
2015-07-10 06:17:38 +08:00
|
|
|
/// non-intersection.
|
|
|
|
///
|
|
|
|
/// Returns true if we need no check or if we do and we can generate them
|
|
|
|
/// (i.e. the pointers have computable bounds).
|
2015-07-15 06:32:44 +08:00
|
|
|
bool canCheckPtrAtRT(RuntimePointerChecking &RtCheck, ScalarEvolution *SE,
|
|
|
|
Loop *TheLoop, const ValueToValueMap &Strides,
|
2015-02-02 00:56:15 +08:00
|
|
|
bool ShouldCheckStride = false);
|
|
|
|
|
|
|
|
/// \brief Goes over all memory accesses, checks whether a RT check is needed
|
|
|
|
/// and builds sets of dependent accesses.
|
|
|
|
void buildDependenceSets() {
|
|
|
|
processMemAccesses();
|
|
|
|
}
|
|
|
|
|
2015-07-09 14:47:18 +08:00
|
|
|
/// \brief Initial processing of memory accesses determined that we need to
|
|
|
|
/// perform dependency checking.
|
|
|
|
///
|
|
|
|
/// Note that this can later be cleared if we retry memcheck analysis without
|
|
|
|
/// dependency checking (i.e. ShouldRetryWithRuntimeCheck).
|
2015-02-02 00:56:15 +08:00
|
|
|
bool isDependencyCheckNeeded() { return !CheckDeps.empty(); }
|
2015-05-18 23:37:03 +08:00
|
|
|
|
|
|
|
/// We decided that no dependence analysis would be used. Reset the state.
|
|
|
|
void resetDepChecks(MemoryDepChecker &DepChecker) {
|
|
|
|
CheckDeps.clear();
|
|
|
|
DepChecker.clearInterestingDependences();
|
|
|
|
}
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
MemAccessInfoSet &getDependenciesToCheck() { return CheckDeps; }
|
|
|
|
|
|
|
|
private:
|
|
|
|
typedef SetVector<MemAccessInfo> PtrAccessSet;
|
|
|
|
|
|
|
|
/// \brief Go over all memory access and check whether runtime pointer checks
|
2015-07-09 14:47:21 +08:00
|
|
|
/// are needed and build sets of dependency check candidates.
|
2015-02-02 00:56:15 +08:00
|
|
|
void processMemAccesses();
|
|
|
|
|
|
|
|
/// Set of all accesses.
|
|
|
|
PtrAccessSet Accesses;
|
|
|
|
|
2015-03-10 10:37:25 +08:00
|
|
|
const DataLayout &DL;
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
/// Set of accesses that need a further dependence check.
|
|
|
|
MemAccessInfoSet CheckDeps;
|
|
|
|
|
|
|
|
/// Set of pointers that are read only.
|
|
|
|
SmallPtrSet<Value*, 16> ReadOnlyPtr;
|
|
|
|
|
|
|
|
/// An alias set tracker to partition the access set by underlying object and
|
|
|
|
//intrinsic property (such as TBAA metadata).
|
|
|
|
AliasSetTracker AST;
|
|
|
|
|
2015-04-24 04:09:20 +08:00
|
|
|
LoopInfo *LI;
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
/// Sets of potentially dependent accesses - members of one set share an
|
|
|
|
/// underlying pointer. The set "CheckDeps" identfies which sets really need a
|
|
|
|
/// dependence check.
|
2015-03-11 01:40:34 +08:00
|
|
|
MemoryDepChecker::DepCandidates &DepCands;
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-07-09 14:47:18 +08:00
|
|
|
/// \brief Initial processing of memory accesses determined that we may need
|
|
|
|
/// to add memchecks. Perform the analysis to determine the necessary checks.
|
|
|
|
///
|
|
|
|
/// Note that, this is different from isDependencyCheckNeeded. When we retry
|
|
|
|
/// memcheck analysis without dependency checking
|
|
|
|
/// (i.e. ShouldRetryWithRuntimeCheck), isDependencyCheckNeeded is cleared
|
|
|
|
/// while this remains set if we have potentially dependent accesses.
|
|
|
|
bool IsRTCheckAnalysisNeeded;
|
2015-02-02 00:56:15 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
} // end anonymous namespace
|
|
|
|
|
|
|
|
/// \brief Check whether a pointer can participate in a runtime bounds check.
|
2015-02-24 08:41:59 +08:00
|
|
|
static bool hasComputableBounds(ScalarEvolution *SE,
|
|
|
|
const ValueToValueMap &Strides, Value *Ptr) {
|
2015-02-02 00:56:15 +08:00
|
|
|
const SCEV *PtrScev = replaceSymbolicStrideSCEV(SE, Strides, Ptr);
|
|
|
|
const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(PtrScev);
|
|
|
|
if (!AR)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
return AR->isAffine();
|
|
|
|
}
|
|
|
|
|
2015-07-15 06:32:44 +08:00
|
|
|
bool AccessAnalysis::canCheckPtrAtRT(RuntimePointerChecking &RtCheck,
|
|
|
|
ScalarEvolution *SE, Loop *TheLoop,
|
|
|
|
const ValueToValueMap &StridesMap,
|
|
|
|
bool ShouldCheckStride) {
|
2015-02-02 00:56:15 +08:00
|
|
|
// Find pointers with computable bounds. We are going to use this information
|
|
|
|
// to place a runtime bound check.
|
|
|
|
bool CanDoRT = true;
|
|
|
|
|
2015-07-10 06:17:38 +08:00
|
|
|
bool NeedRTCheck = false;
|
2015-07-09 14:47:18 +08:00
|
|
|
if (!IsRTCheckAnalysisNeeded) return true;
|
2015-06-08 18:27:06 +08:00
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
bool IsDepCheckNeeded = isDependencyCheckNeeded();
|
|
|
|
|
|
|
|
// We assign a consecutive id to access from different alias sets.
|
|
|
|
// Accesses between different groups doesn't need to be checked.
|
|
|
|
unsigned ASId = 1;
|
|
|
|
for (auto &AS : AST) {
|
2015-07-09 06:58:48 +08:00
|
|
|
int NumReadPtrChecks = 0;
|
|
|
|
int NumWritePtrChecks = 0;
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
// We assign consecutive id to access from different dependence sets.
|
|
|
|
// Accesses within the same set don't need a runtime check.
|
|
|
|
unsigned RunningDepId = 1;
|
|
|
|
DenseMap<Value *, unsigned> DepSetId;
|
|
|
|
|
|
|
|
for (auto A : AS) {
|
|
|
|
Value *Ptr = A.getValue();
|
|
|
|
bool IsWrite = Accesses.count(MemAccessInfo(Ptr, true));
|
|
|
|
MemAccessInfo Access(Ptr, IsWrite);
|
|
|
|
|
2015-07-09 06:58:48 +08:00
|
|
|
if (IsWrite)
|
|
|
|
++NumWritePtrChecks;
|
|
|
|
else
|
|
|
|
++NumReadPtrChecks;
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
if (hasComputableBounds(SE, StridesMap, Ptr) &&
|
2015-03-10 10:37:25 +08:00
|
|
|
// When we run after a failing dependency check we have to make sure
|
|
|
|
// we don't have wrapping pointers.
|
2015-02-02 00:56:15 +08:00
|
|
|
(!ShouldCheckStride ||
|
2015-03-10 10:37:25 +08:00
|
|
|
isStridedPtr(SE, Ptr, TheLoop, StridesMap) == 1)) {
|
2015-02-02 00:56:15 +08:00
|
|
|
// The id of the dependence set.
|
|
|
|
unsigned DepId;
|
|
|
|
|
|
|
|
if (IsDepCheckNeeded) {
|
|
|
|
Value *Leader = DepCands.getLeaderValue(Access).getPointer();
|
|
|
|
unsigned &LeaderId = DepSetId[Leader];
|
|
|
|
if (!LeaderId)
|
|
|
|
LeaderId = RunningDepId++;
|
|
|
|
DepId = LeaderId;
|
|
|
|
} else
|
|
|
|
// Each access has its own dependence set.
|
|
|
|
DepId = RunningDepId++;
|
|
|
|
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
RtCheck.insert(TheLoop, Ptr, IsWrite, DepId, ASId, StridesMap);
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Found a runtime check ptr:" << *Ptr << '\n');
|
2015-02-02 00:56:15 +08:00
|
|
|
} else {
|
2015-05-18 23:36:52 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Can't find bounds for ptr:" << *Ptr << '\n');
|
2015-02-02 00:56:15 +08:00
|
|
|
CanDoRT = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-07-09 06:58:48 +08:00
|
|
|
// If we have at least two writes or one write and a read then we need to
|
|
|
|
// check them. But there is no need to checks if there is only one
|
|
|
|
// dependence set for this alias set.
|
|
|
|
//
|
|
|
|
// Note that this function computes CanDoRT and NeedRTCheck independently.
|
|
|
|
// For example CanDoRT=false, NeedRTCheck=false means that we have a pointer
|
|
|
|
// for which we couldn't find the bounds but we don't actually need to emit
|
|
|
|
// any checks so it does not matter.
|
|
|
|
if (!(IsDepCheckNeeded && CanDoRT && RunningDepId == 2))
|
|
|
|
NeedRTCheck |= (NumWritePtrChecks >= 2 || (NumReadPtrChecks >= 1 &&
|
|
|
|
NumWritePtrChecks >= 1));
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
++ASId;
|
|
|
|
}
|
|
|
|
|
|
|
|
// If the pointers that we would use for the bounds comparison have different
|
|
|
|
// address spaces, assume the values aren't directly comparable, so we can't
|
|
|
|
// use them for the runtime check. We also have to assume they could
|
|
|
|
// overlap. In the future there should be metadata for whether address spaces
|
|
|
|
// are disjoint.
|
|
|
|
unsigned NumPointers = RtCheck.Pointers.size();
|
|
|
|
for (unsigned i = 0; i < NumPointers; ++i) {
|
|
|
|
for (unsigned j = i + 1; j < NumPointers; ++j) {
|
|
|
|
// Only need to check pointers between two different dependency sets.
|
2015-07-15 06:32:50 +08:00
|
|
|
if (RtCheck.Pointers[i].DependencySetId ==
|
|
|
|
RtCheck.Pointers[j].DependencySetId)
|
2015-02-02 00:56:15 +08:00
|
|
|
continue;
|
|
|
|
// Only need to check pointers in the same alias set.
|
2015-07-15 06:32:50 +08:00
|
|
|
if (RtCheck.Pointers[i].AliasSetId != RtCheck.Pointers[j].AliasSetId)
|
2015-02-02 00:56:15 +08:00
|
|
|
continue;
|
|
|
|
|
2015-07-15 06:32:50 +08:00
|
|
|
Value *PtrI = RtCheck.Pointers[i].PointerValue;
|
|
|
|
Value *PtrJ = RtCheck.Pointers[j].PointerValue;
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
unsigned ASi = PtrI->getType()->getPointerAddressSpace();
|
|
|
|
unsigned ASj = PtrJ->getType()->getPointerAddressSpace();
|
|
|
|
if (ASi != ASj) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Runtime check would require comparison between"
|
2015-02-20 03:14:34 +08:00
|
|
|
" different address spaces\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
if (NeedRTCheck && CanDoRT)
|
2015-08-08 06:44:15 +08:00
|
|
|
RtCheck.generateChecks(DepCands, IsDepCheckNeeded);
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
|
2015-08-08 06:44:21 +08:00
|
|
|
DEBUG(dbgs() << "LAA: We need to do " << RtCheck.getNumberOfChecks()
|
2015-07-10 06:17:38 +08:00
|
|
|
<< " pointer comparisons.\n");
|
|
|
|
|
|
|
|
RtCheck.Need = NeedRTCheck;
|
|
|
|
|
|
|
|
bool CanDoRTIfNeeded = !NeedRTCheck || CanDoRT;
|
|
|
|
if (!CanDoRTIfNeeded)
|
|
|
|
RtCheck.reset();
|
|
|
|
return CanDoRTIfNeeded;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void AccessAnalysis::processMemAccesses() {
|
|
|
|
// We process the set twice: first we process read-write pointers, last we
|
|
|
|
// process read-only pointers. This allows us to skip dependence tests for
|
|
|
|
// read-only pointers.
|
|
|
|
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Processing memory accesses...\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
DEBUG(dbgs() << " AST: "; AST.dump());
|
2015-03-11 01:40:37 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Accesses(" << Accesses.size() << "):\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
DEBUG({
|
|
|
|
for (auto A : Accesses)
|
|
|
|
dbgs() << "\t" << *A.getPointer() << " (" <<
|
|
|
|
(A.getInt() ? "write" : (ReadOnlyPtr.count(A.getPointer()) ?
|
|
|
|
"read-only" : "read")) << ")\n";
|
|
|
|
});
|
|
|
|
|
|
|
|
// The AliasSetTracker has nicely partitioned our pointers by metadata
|
|
|
|
// compatibility and potential for underlying-object overlap. As a result, we
|
|
|
|
// only need to check for potential pointer dependencies within each alias
|
|
|
|
// set.
|
|
|
|
for (auto &AS : AST) {
|
|
|
|
// Note that both the alias-set tracker and the alias sets themselves used
|
|
|
|
// linked lists internally and so the iteration order here is deterministic
|
|
|
|
// (matching the original instruction order within each set).
|
|
|
|
|
|
|
|
bool SetHasWrite = false;
|
|
|
|
|
|
|
|
// Map of pointers to last access encountered.
|
|
|
|
typedef DenseMap<Value*, MemAccessInfo> UnderlyingObjToAccessMap;
|
|
|
|
UnderlyingObjToAccessMap ObjToLastAccess;
|
|
|
|
|
|
|
|
// Set of access to check after all writes have been processed.
|
|
|
|
PtrAccessSet DeferredAccesses;
|
|
|
|
|
|
|
|
// Iterate over each alias set twice, once to process read/write pointers,
|
|
|
|
// and then to process read-only pointers.
|
|
|
|
for (int SetIteration = 0; SetIteration < 2; ++SetIteration) {
|
|
|
|
bool UseDeferred = SetIteration > 0;
|
|
|
|
PtrAccessSet &S = UseDeferred ? DeferredAccesses : Accesses;
|
|
|
|
|
|
|
|
for (auto AV : AS) {
|
|
|
|
Value *Ptr = AV.getValue();
|
|
|
|
|
|
|
|
// For a single memory access in AliasSetTracker, Accesses may contain
|
|
|
|
// both read and write, and they both need to be handled for CheckDeps.
|
|
|
|
for (auto AC : S) {
|
|
|
|
if (AC.getPointer() != Ptr)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
bool IsWrite = AC.getInt();
|
|
|
|
|
|
|
|
// If we're using the deferred access set, then it contains only
|
|
|
|
// reads.
|
|
|
|
bool IsReadOnlyPtr = ReadOnlyPtr.count(Ptr) && !IsWrite;
|
|
|
|
if (UseDeferred && !IsReadOnlyPtr)
|
|
|
|
continue;
|
|
|
|
// Otherwise, the pointer must be in the PtrAccessSet, either as a
|
|
|
|
// read or a write.
|
|
|
|
assert(((IsReadOnlyPtr && UseDeferred) || IsWrite ||
|
|
|
|
S.count(MemAccessInfo(Ptr, false))) &&
|
|
|
|
"Alias-set pointer not in the access set?");
|
|
|
|
|
|
|
|
MemAccessInfo Access(Ptr, IsWrite);
|
|
|
|
DepCands.insert(Access);
|
|
|
|
|
|
|
|
// Memorize read-only pointers for later processing and skip them in
|
|
|
|
// the first round (they need to be checked after we have seen all
|
|
|
|
// write pointers). Note: we also mark pointer that are not
|
|
|
|
// consecutive as "read-only" pointers (so that we check
|
|
|
|
// "a[b[i]] +="). Hence, we need the second check for "!IsWrite".
|
|
|
|
if (!UseDeferred && IsReadOnlyPtr) {
|
|
|
|
DeferredAccesses.insert(Access);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
// If this is a write - check other reads and writes for conflicts. If
|
|
|
|
// this is a read only check other writes for conflicts (but only if
|
|
|
|
// there is no other write to the ptr - this is an optimization to
|
|
|
|
// catch "a[i] = a[i] + " without having to do a dependence check).
|
|
|
|
if ((IsWrite || IsReadOnlyPtr) && SetHasWrite) {
|
|
|
|
CheckDeps.insert(Access);
|
2015-07-09 14:47:18 +08:00
|
|
|
IsRTCheckAnalysisNeeded = true;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
if (IsWrite)
|
|
|
|
SetHasWrite = true;
|
|
|
|
|
|
|
|
// Create sets of pointers connected by a shared alias set and
|
|
|
|
// underlying object.
|
|
|
|
typedef SmallVector<Value *, 16> ValueVector;
|
|
|
|
ValueVector TempObjects;
|
2015-04-24 04:09:20 +08:00
|
|
|
|
|
|
|
GetUnderlyingObjects(Ptr, TempObjects, DL, LI);
|
|
|
|
DEBUG(dbgs() << "Underlying objects for pointer " << *Ptr << "\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
for (Value *UnderlyingObj : TempObjects) {
|
|
|
|
UnderlyingObjToAccessMap::iterator Prev =
|
|
|
|
ObjToLastAccess.find(UnderlyingObj);
|
|
|
|
if (Prev != ObjToLastAccess.end())
|
|
|
|
DepCands.unionSets(Access, Prev->second);
|
|
|
|
|
|
|
|
ObjToLastAccess[UnderlyingObj] = Access;
|
2015-04-24 04:09:20 +08:00
|
|
|
DEBUG(dbgs() << " " << *UnderlyingObj << "\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool isInBoundsGep(Value *Ptr) {
|
|
|
|
if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr))
|
|
|
|
return GEP->isInBounds();
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2015-06-27 01:25:43 +08:00
|
|
|
/// \brief Return true if an AddRec pointer \p Ptr is unsigned non-wrapping,
|
|
|
|
/// i.e. monotonically increasing/decreasing.
|
|
|
|
static bool isNoWrapAddRec(Value *Ptr, const SCEVAddRecExpr *AR,
|
|
|
|
ScalarEvolution *SE, const Loop *L) {
|
|
|
|
// FIXME: This should probably only return true for NUW.
|
|
|
|
if (AR->getNoWrapFlags(SCEV::NoWrapMask))
|
|
|
|
return true;
|
|
|
|
|
|
|
|
// Scalar evolution does not propagate the non-wrapping flags to values that
|
|
|
|
// are derived from a non-wrapping induction variable because non-wrapping
|
|
|
|
// could be flow-sensitive.
|
|
|
|
//
|
|
|
|
// Look through the potentially overflowing instruction to try to prove
|
|
|
|
// non-wrapping for the *specific* value of Ptr.
|
|
|
|
|
|
|
|
// The arithmetic implied by an inbounds GEP can't overflow.
|
|
|
|
auto *GEP = dyn_cast<GetElementPtrInst>(Ptr);
|
|
|
|
if (!GEP || !GEP->isInBounds())
|
|
|
|
return false;
|
|
|
|
|
|
|
|
// Make sure there is only one non-const index and analyze that.
|
|
|
|
Value *NonConstIndex = nullptr;
|
|
|
|
for (auto Index = GEP->idx_begin(); Index != GEP->idx_end(); ++Index)
|
|
|
|
if (!isa<ConstantInt>(*Index)) {
|
|
|
|
if (NonConstIndex)
|
|
|
|
return false;
|
|
|
|
NonConstIndex = *Index;
|
|
|
|
}
|
|
|
|
if (!NonConstIndex)
|
|
|
|
// The recurrence is on the pointer, ignore for now.
|
|
|
|
return false;
|
|
|
|
|
|
|
|
// The index in GEP is signed. It is non-wrapping if it's derived from a NSW
|
|
|
|
// AddRec using a NSW operation.
|
|
|
|
if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(NonConstIndex))
|
|
|
|
if (OBO->hasNoSignedWrap() &&
|
|
|
|
// Assume constant for other the operand so that the AddRec can be
|
|
|
|
// easily found.
|
|
|
|
isa<ConstantInt>(OBO->getOperand(1))) {
|
|
|
|
auto *OpScev = SE->getSCEV(OBO->getOperand(0));
|
|
|
|
|
|
|
|
if (auto *OpAR = dyn_cast<SCEVAddRecExpr>(OpScev))
|
|
|
|
return OpAR->getLoop() == L && OpAR->getNoWrapFlags(SCEV::FlagNSW);
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
/// \brief Check whether the access through \p Ptr has a constant stride.
|
[LoopVectorize] Teach Loop Vectorizor about interleaved memory accesses.
Interleaved memory accesses are grouped and vectorized into vector load/store and shufflevector.
E.g. for (i = 0; i < N; i+=2) {
a = A[i]; // load of even element
b = A[i+1]; // load of odd element
... // operations on a, b, c, d
A[i] = c; // store of even element
A[i+1] = d; // store of odd element
}
The loads of even and odd elements are identified as an interleave load group, which will be transfered into vectorized IRs like:
%wide.vec = load <8 x i32>, <8 x i32>* %ptr
%vec.even = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
%vec.odd = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
The stores of even and odd elements are identified as an interleave store group, which will be transfered into vectorized IRs like:
%interleaved.vec = shufflevector <4 x i32> %vec.even, %vec.odd, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
store <8 x i32> %interleaved.vec, <8 x i32>* %ptr
This optimization is currently disabled by defaut. To try it by adding '-enable-interleaved-mem-accesses=true'.
llvm-svn: 239291
2015-06-08 14:39:56 +08:00
|
|
|
int llvm::isStridedPtr(ScalarEvolution *SE, Value *Ptr, const Loop *Lp,
|
|
|
|
const ValueToValueMap &StridesMap) {
|
2015-08-02 06:20:21 +08:00
|
|
|
Type *Ty = Ptr->getType();
|
2015-02-02 00:56:15 +08:00
|
|
|
assert(Ty->isPointerTy() && "Unexpected non-ptr");
|
|
|
|
|
|
|
|
// Make sure that the pointer does not point to aggregate types.
|
2015-08-02 06:20:21 +08:00
|
|
|
auto *PtrTy = cast<PointerType>(Ty);
|
2015-02-02 00:56:15 +08:00
|
|
|
if (PtrTy->getElementType()->isAggregateType()) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Bad stride - Not a pointer to a scalar type"
|
|
|
|
<< *Ptr << "\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
const SCEV *PtrScev = replaceSymbolicStrideSCEV(SE, StridesMap, Ptr);
|
|
|
|
|
|
|
|
const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(PtrScev);
|
|
|
|
if (!AR) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Bad stride - Not an AddRecExpr pointer "
|
2015-02-20 03:14:34 +08:00
|
|
|
<< *Ptr << " SCEV: " << *PtrScev << "\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
// The accesss function must stride over the innermost loop.
|
|
|
|
if (Lp != AR->getLoop()) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Bad stride - Not striding over innermost loop " <<
|
2015-02-20 03:14:34 +08:00
|
|
|
*Ptr << " SCEV: " << *PtrScev << "\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// The address calculation must not wrap. Otherwise, a dependence could be
|
|
|
|
// inverted.
|
|
|
|
// An inbounds getelementptr that is a AddRec with a unit stride
|
|
|
|
// cannot wrap per definition. The unit stride requirement is checked later.
|
|
|
|
// An getelementptr without an inbounds attribute and unit stride would have
|
|
|
|
// to access the pointer value "0" which is undefined behavior in address
|
|
|
|
// space 0, therefore we can also vectorize this case.
|
|
|
|
bool IsInBoundsGEP = isInBoundsGep(Ptr);
|
2015-06-27 01:25:43 +08:00
|
|
|
bool IsNoWrapAddRec = isNoWrapAddRec(Ptr, AR, SE, Lp);
|
2015-02-02 00:56:15 +08:00
|
|
|
bool IsInAddressSpaceZero = PtrTy->getAddressSpace() == 0;
|
|
|
|
if (!IsNoWrapAddRec && !IsInBoundsGEP && !IsInAddressSpaceZero) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Bad stride - Pointer may wrap in the address space "
|
2015-02-20 03:14:34 +08:00
|
|
|
<< *Ptr << " SCEV: " << *PtrScev << "\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check the step is constant.
|
|
|
|
const SCEV *Step = AR->getStepRecurrence(*SE);
|
|
|
|
|
2015-07-09 08:03:22 +08:00
|
|
|
// Calculate the pointer stride and check if it is constant.
|
2015-02-02 00:56:15 +08:00
|
|
|
const SCEVConstant *C = dyn_cast<SCEVConstant>(Step);
|
|
|
|
if (!C) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Bad stride - Not a constant strided " << *Ptr <<
|
2015-02-20 03:14:34 +08:00
|
|
|
" SCEV: " << *PtrScev << "\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-03-10 10:37:25 +08:00
|
|
|
auto &DL = Lp->getHeader()->getModule()->getDataLayout();
|
|
|
|
int64_t Size = DL.getTypeAllocSize(PtrTy->getElementType());
|
2015-02-02 00:56:15 +08:00
|
|
|
const APInt &APStepVal = C->getValue()->getValue();
|
|
|
|
|
|
|
|
// Huge step value - give up.
|
|
|
|
if (APStepVal.getBitWidth() > 64)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
int64_t StepVal = APStepVal.getSExtValue();
|
|
|
|
|
|
|
|
// Strided access.
|
|
|
|
int64_t Stride = StepVal / Size;
|
|
|
|
int64_t Rem = StepVal % Size;
|
|
|
|
if (Rem)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
// If the SCEV could wrap but we have an inbounds gep with a unit stride we
|
|
|
|
// know we can't "wrap around the address space". In case of address space
|
|
|
|
// zero we know that this won't happen without triggering undefined behavior.
|
|
|
|
if (!IsNoWrapAddRec && (IsInBoundsGEP || IsInAddressSpaceZero) &&
|
|
|
|
Stride != 1 && Stride != -1)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
return Stride;
|
|
|
|
}
|
|
|
|
|
2015-03-11 01:40:37 +08:00
|
|
|
bool MemoryDepChecker::Dependence::isSafeForVectorization(DepType Type) {
|
|
|
|
switch (Type) {
|
|
|
|
case NoDep:
|
|
|
|
case Forward:
|
|
|
|
case BackwardVectorizable:
|
|
|
|
return true;
|
|
|
|
|
|
|
|
case Unknown:
|
|
|
|
case ForwardButPreventsForwarding:
|
|
|
|
case Backward:
|
|
|
|
case BackwardVectorizableButPreventsForwarding:
|
|
|
|
return false;
|
|
|
|
}
|
2015-03-11 04:23:29 +08:00
|
|
|
llvm_unreachable("unexpected DepType!");
|
2015-03-11 01:40:37 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
bool MemoryDepChecker::Dependence::isInterestingDependence(DepType Type) {
|
|
|
|
switch (Type) {
|
|
|
|
case NoDep:
|
|
|
|
case Forward:
|
|
|
|
return false;
|
|
|
|
|
|
|
|
case BackwardVectorizable:
|
|
|
|
case Unknown:
|
|
|
|
case ForwardButPreventsForwarding:
|
|
|
|
case Backward:
|
|
|
|
case BackwardVectorizableButPreventsForwarding:
|
|
|
|
return true;
|
|
|
|
}
|
2015-03-11 04:23:29 +08:00
|
|
|
llvm_unreachable("unexpected DepType!");
|
2015-03-11 01:40:37 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
bool MemoryDepChecker::Dependence::isPossiblyBackward() const {
|
|
|
|
switch (Type) {
|
|
|
|
case NoDep:
|
|
|
|
case Forward:
|
|
|
|
case ForwardButPreventsForwarding:
|
|
|
|
return false;
|
|
|
|
|
|
|
|
case Unknown:
|
|
|
|
case BackwardVectorizable:
|
|
|
|
case Backward:
|
|
|
|
case BackwardVectorizableButPreventsForwarding:
|
|
|
|
return true;
|
|
|
|
}
|
2015-03-11 04:23:29 +08:00
|
|
|
llvm_unreachable("unexpected DepType!");
|
2015-03-11 01:40:37 +08:00
|
|
|
}
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
bool MemoryDepChecker::couldPreventStoreLoadForward(unsigned Distance,
|
|
|
|
unsigned TypeByteSize) {
|
|
|
|
// If loads occur at a distance that is not a multiple of a feasible vector
|
|
|
|
// factor store-load forwarding does not take place.
|
|
|
|
// Positive dependences might cause troubles because vectorizing them might
|
|
|
|
// prevent store-load forwarding making vectorized code run a lot slower.
|
|
|
|
// a[i] = a[i-3] ^ a[i-8];
|
|
|
|
// The stores to a[i:i+1] don't align with the stores to a[i-3:i-2] and
|
|
|
|
// hence on your typical architecture store-load forwarding does not take
|
|
|
|
// place. Vectorizing in such cases does not make sense.
|
|
|
|
// Store-load forwarding distance.
|
|
|
|
const unsigned NumCyclesForStoreLoadThroughMemory = 8*TypeByteSize;
|
|
|
|
// Maximum vector factor.
|
2015-02-20 03:14:52 +08:00
|
|
|
unsigned MaxVFWithoutSLForwardIssues =
|
|
|
|
VectorizerParams::MaxVectorWidth * TypeByteSize;
|
2015-02-20 03:14:34 +08:00
|
|
|
if(MaxSafeDepDistBytes < MaxVFWithoutSLForwardIssues)
|
2015-02-02 00:56:15 +08:00
|
|
|
MaxVFWithoutSLForwardIssues = MaxSafeDepDistBytes;
|
|
|
|
|
|
|
|
for (unsigned vf = 2*TypeByteSize; vf <= MaxVFWithoutSLForwardIssues;
|
|
|
|
vf *= 2) {
|
|
|
|
if (Distance % vf && Distance / vf < NumCyclesForStoreLoadThroughMemory) {
|
|
|
|
MaxVFWithoutSLForwardIssues = (vf >>=1);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-02-20 03:14:34 +08:00
|
|
|
if (MaxVFWithoutSLForwardIssues< 2*TypeByteSize) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Distance " << Distance <<
|
2015-02-20 03:14:34 +08:00
|
|
|
" that could cause a store-load forwarding conflict\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (MaxVFWithoutSLForwardIssues < MaxSafeDepDistBytes &&
|
2015-02-20 03:14:52 +08:00
|
|
|
MaxVFWithoutSLForwardIssues !=
|
|
|
|
VectorizerParams::MaxVectorWidth * TypeByteSize)
|
2015-02-02 00:56:15 +08:00
|
|
|
MaxSafeDepDistBytes = MaxVFWithoutSLForwardIssues;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2015-06-08 12:48:37 +08:00
|
|
|
/// \brief Check the dependence for two accesses with the same stride \p Stride.
|
|
|
|
/// \p Distance is the positive distance and \p TypeByteSize is type size in
|
|
|
|
/// bytes.
|
|
|
|
///
|
|
|
|
/// \returns true if they are independent.
|
|
|
|
static bool areStridedAccessesIndependent(unsigned Distance, unsigned Stride,
|
|
|
|
unsigned TypeByteSize) {
|
|
|
|
assert(Stride > 1 && "The stride must be greater than 1");
|
|
|
|
assert(TypeByteSize > 0 && "The type size in byte must be non-zero");
|
|
|
|
assert(Distance > 0 && "The distance must be non-zero");
|
|
|
|
|
|
|
|
// Skip if the distance is not multiple of type byte size.
|
|
|
|
if (Distance % TypeByteSize)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
unsigned ScaledDist = Distance / TypeByteSize;
|
|
|
|
|
|
|
|
// No dependence if the scaled distance is not multiple of the stride.
|
|
|
|
// E.g.
|
|
|
|
// for (i = 0; i < 1024 ; i += 4)
|
|
|
|
// A[i+2] = A[i] + 1;
|
|
|
|
//
|
|
|
|
// Two accesses in memory (scaled distance is 2, stride is 4):
|
|
|
|
// | A[0] | | | | A[4] | | | |
|
|
|
|
// | | | A[2] | | | | A[6] | |
|
|
|
|
//
|
|
|
|
// E.g.
|
|
|
|
// for (i = 0; i < 1024 ; i += 3)
|
|
|
|
// A[i+4] = A[i] + 1;
|
|
|
|
//
|
|
|
|
// Two accesses in memory (scaled distance is 4, stride is 3):
|
|
|
|
// | A[0] | | | A[3] | | | A[6] | | |
|
|
|
|
// | | | | | A[4] | | | A[7] | |
|
|
|
|
return ScaledDist % Stride;
|
|
|
|
}
|
|
|
|
|
2015-03-11 01:40:37 +08:00
|
|
|
MemoryDepChecker::Dependence::DepType
|
|
|
|
MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
|
|
|
|
const MemAccessInfo &B, unsigned BIdx,
|
|
|
|
const ValueToValueMap &Strides) {
|
2015-02-02 00:56:15 +08:00
|
|
|
assert (AIdx < BIdx && "Must pass arguments in program order");
|
|
|
|
|
|
|
|
Value *APtr = A.getPointer();
|
|
|
|
Value *BPtr = B.getPointer();
|
|
|
|
bool AIsWrite = A.getInt();
|
|
|
|
bool BIsWrite = B.getInt();
|
|
|
|
|
|
|
|
// Two reads are independent.
|
|
|
|
if (!AIsWrite && !BIsWrite)
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::NoDep;
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
// We cannot check pointers in different address spaces.
|
|
|
|
if (APtr->getType()->getPointerAddressSpace() !=
|
|
|
|
BPtr->getType()->getPointerAddressSpace())
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::Unknown;
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
const SCEV *AScev = replaceSymbolicStrideSCEV(SE, Strides, APtr);
|
|
|
|
const SCEV *BScev = replaceSymbolicStrideSCEV(SE, Strides, BPtr);
|
|
|
|
|
2015-03-10 10:37:25 +08:00
|
|
|
int StrideAPtr = isStridedPtr(SE, APtr, InnermostLoop, Strides);
|
|
|
|
int StrideBPtr = isStridedPtr(SE, BPtr, InnermostLoop, Strides);
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
const SCEV *Src = AScev;
|
|
|
|
const SCEV *Sink = BScev;
|
|
|
|
|
|
|
|
// If the induction step is negative we have to invert source and sink of the
|
|
|
|
// dependence.
|
|
|
|
if (StrideAPtr < 0) {
|
|
|
|
//Src = BScev;
|
|
|
|
//Sink = AScev;
|
|
|
|
std::swap(APtr, BPtr);
|
|
|
|
std::swap(Src, Sink);
|
|
|
|
std::swap(AIsWrite, BIsWrite);
|
|
|
|
std::swap(AIdx, BIdx);
|
|
|
|
std::swap(StrideAPtr, StrideBPtr);
|
|
|
|
}
|
|
|
|
|
|
|
|
const SCEV *Dist = SE->getMinusSCEV(Sink, Src);
|
|
|
|
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Src Scev: " << *Src << "Sink Scev: " << *Sink
|
2015-02-20 03:14:34 +08:00
|
|
|
<< "(Induction step: " << StrideAPtr << ")\n");
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Distance for " << *InstMap[AIdx] << " to "
|
2015-02-20 03:14:34 +08:00
|
|
|
<< *InstMap[BIdx] << ": " << *Dist << "\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-07-09 08:03:22 +08:00
|
|
|
// Need accesses with constant stride. We don't want to vectorize
|
2015-02-02 00:56:15 +08:00
|
|
|
// "A[B[i]] += ..." and similar code or pointer arithmetic that could wrap in
|
|
|
|
// the address space.
|
|
|
|
if (!StrideAPtr || !StrideBPtr || StrideAPtr != StrideBPtr){
|
2015-07-09 08:03:22 +08:00
|
|
|
DEBUG(dbgs() << "Pointer access with non-constant stride\n");
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::Unknown;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
const SCEVConstant *C = dyn_cast<SCEVConstant>(Dist);
|
|
|
|
if (!C) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Dependence because of non-constant distance\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
ShouldRetryWithRuntimeCheck = true;
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::Unknown;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
Type *ATy = APtr->getType()->getPointerElementType();
|
|
|
|
Type *BTy = BPtr->getType()->getPointerElementType();
|
2015-03-10 10:37:25 +08:00
|
|
|
auto &DL = InnermostLoop->getHeader()->getModule()->getDataLayout();
|
|
|
|
unsigned TypeByteSize = DL.getTypeAllocSize(ATy);
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
// Negative distances are not plausible dependencies.
|
|
|
|
const APInt &Val = C->getValue()->getValue();
|
|
|
|
if (Val.isNegative()) {
|
|
|
|
bool IsTrueDataDependence = (AIsWrite && !BIsWrite);
|
|
|
|
if (IsTrueDataDependence &&
|
|
|
|
(couldPreventStoreLoadForward(Val.abs().getZExtValue(), TypeByteSize) ||
|
|
|
|
ATy != BTy))
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::ForwardButPreventsForwarding;
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Dependence is negative: NoDep\n");
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::Forward;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Write to the same location with the same size.
|
|
|
|
// Could be improved to assert type sizes are the same (i32 == float, etc).
|
|
|
|
if (Val == 0) {
|
|
|
|
if (ATy == BTy)
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::NoDep;
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Zero dependence difference but different types\n");
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::Unknown;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
assert(Val.isStrictlyPositive() && "Expect a positive value");
|
|
|
|
|
|
|
|
if (ATy != BTy) {
|
2015-02-20 03:14:34 +08:00
|
|
|
DEBUG(dbgs() <<
|
2015-02-20 03:15:07 +08:00
|
|
|
"LAA: ReadWrite-Write positive dependency with different types\n");
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::Unknown;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
unsigned Distance = (unsigned) Val.getZExtValue();
|
|
|
|
|
2015-06-08 12:48:37 +08:00
|
|
|
unsigned Stride = std::abs(StrideAPtr);
|
|
|
|
if (Stride > 1 &&
|
2015-07-09 02:47:38 +08:00
|
|
|
areStridedAccessesIndependent(Distance, Stride, TypeByteSize)) {
|
|
|
|
DEBUG(dbgs() << "LAA: Strided accesses are independent\n");
|
2015-06-08 12:48:37 +08:00
|
|
|
return Dependence::NoDep;
|
2015-07-09 02:47:38 +08:00
|
|
|
}
|
2015-06-08 12:48:37 +08:00
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
// Bail out early if passed-in parameters make vectorization not feasible.
|
2015-02-20 03:14:52 +08:00
|
|
|
unsigned ForcedFactor = (VectorizerParams::VectorizationFactor ?
|
|
|
|
VectorizerParams::VectorizationFactor : 1);
|
|
|
|
unsigned ForcedUnroll = (VectorizerParams::VectorizationInterleave ?
|
|
|
|
VectorizerParams::VectorizationInterleave : 1);
|
2015-06-08 12:48:37 +08:00
|
|
|
// The minimum number of iterations for a vectorized/unrolled version.
|
|
|
|
unsigned MinNumIter = std::max(ForcedFactor * ForcedUnroll, 2U);
|
|
|
|
|
|
|
|
// It's not vectorizable if the distance is smaller than the minimum distance
|
|
|
|
// needed for a vectroized/unrolled version. Vectorizing one iteration in
|
|
|
|
// front needs TypeByteSize * Stride. Vectorizing the last iteration needs
|
|
|
|
// TypeByteSize (No need to plus the last gap distance).
|
|
|
|
//
|
|
|
|
// E.g. Assume one char is 1 byte in memory and one int is 4 bytes.
|
|
|
|
// foo(int *A) {
|
|
|
|
// int *B = (int *)((char *)A + 14);
|
|
|
|
// for (i = 0 ; i < 1024 ; i += 2)
|
|
|
|
// B[i] = A[i] + 1;
|
|
|
|
// }
|
|
|
|
//
|
|
|
|
// Two accesses in memory (stride is 2):
|
|
|
|
// | A[0] | | A[2] | | A[4] | | A[6] | |
|
|
|
|
// | B[0] | | B[2] | | B[4] |
|
|
|
|
//
|
|
|
|
// Distance needs for vectorizing iterations except the last iteration:
|
|
|
|
// 4 * 2 * (MinNumIter - 1). Distance needs for the last iteration: 4.
|
|
|
|
// So the minimum distance needed is: 4 * 2 * (MinNumIter - 1) + 4.
|
|
|
|
//
|
|
|
|
// If MinNumIter is 2, it is vectorizable as the minimum distance needed is
|
|
|
|
// 12, which is less than distance.
|
|
|
|
//
|
|
|
|
// If MinNumIter is 4 (Say if a user forces the vectorization factor to be 4),
|
|
|
|
// the minimum distance needed is 28, which is greater than distance. It is
|
|
|
|
// not safe to do vectorization.
|
|
|
|
unsigned MinDistanceNeeded =
|
|
|
|
TypeByteSize * Stride * (MinNumIter - 1) + TypeByteSize;
|
|
|
|
if (MinDistanceNeeded > Distance) {
|
|
|
|
DEBUG(dbgs() << "LAA: Failure because of positive distance " << Distance
|
|
|
|
<< '\n');
|
|
|
|
return Dependence::Backward;
|
|
|
|
}
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-06-08 12:48:37 +08:00
|
|
|
// Unsafe if the minimum distance needed is greater than max safe distance.
|
|
|
|
if (MinDistanceNeeded > MaxSafeDepDistBytes) {
|
|
|
|
DEBUG(dbgs() << "LAA: Failure because it needs at least "
|
|
|
|
<< MinDistanceNeeded << " size in bytes");
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::Backward;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-02-27 01:58:48 +08:00
|
|
|
// Positive distance bigger than max vectorization factor.
|
2015-06-08 12:48:37 +08:00
|
|
|
// FIXME: Should use max factor instead of max distance in bytes, which could
|
|
|
|
// not handle different types.
|
|
|
|
// E.g. Assume one char is 1 byte in memory and one int is 4 bytes.
|
|
|
|
// void foo (int *A, char *B) {
|
|
|
|
// for (unsigned i = 0; i < 1024; i++) {
|
|
|
|
// A[i+2] = A[i] + 1;
|
|
|
|
// B[i+2] = B[i] + 1;
|
|
|
|
// }
|
|
|
|
// }
|
|
|
|
//
|
|
|
|
// This case is currently unsafe according to the max safe distance. If we
|
|
|
|
// analyze the two accesses on array B, the max safe dependence distance
|
|
|
|
// is 2. Then we analyze the accesses on array A, the minimum distance needed
|
|
|
|
// is 8, which is less than 2 and forbidden vectorization, But actually
|
|
|
|
// both A and B could be vectorized by 2 iterations.
|
|
|
|
MaxSafeDepDistBytes =
|
|
|
|
Distance < MaxSafeDepDistBytes ? Distance : MaxSafeDepDistBytes;
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
bool IsTrueDataDependence = (!AIsWrite && BIsWrite);
|
|
|
|
if (IsTrueDataDependence &&
|
|
|
|
couldPreventStoreLoadForward(Distance, TypeByteSize))
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::BackwardVectorizableButPreventsForwarding;
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-06-08 12:48:37 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Positive distance " << Val.getSExtValue()
|
|
|
|
<< " with max VF = "
|
|
|
|
<< MaxSafeDepDistBytes / (TypeByteSize * Stride) << '\n');
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::BackwardVectorizable;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-03-11 01:40:34 +08:00
|
|
|
bool MemoryDepChecker::areDepsSafe(DepCandidates &AccessSets,
|
2015-02-02 00:56:15 +08:00
|
|
|
MemAccessInfoSet &CheckDeps,
|
2015-02-24 08:41:59 +08:00
|
|
|
const ValueToValueMap &Strides) {
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
MaxSafeDepDistBytes = -1U;
|
|
|
|
while (!CheckDeps.empty()) {
|
|
|
|
MemAccessInfo CurAccess = *CheckDeps.begin();
|
|
|
|
|
|
|
|
// Get the relevant memory access set.
|
|
|
|
EquivalenceClasses<MemAccessInfo>::iterator I =
|
|
|
|
AccessSets.findValue(AccessSets.getLeaderValue(CurAccess));
|
|
|
|
|
|
|
|
// Check accesses within this set.
|
|
|
|
EquivalenceClasses<MemAccessInfo>::member_iterator AI, AE;
|
|
|
|
AI = AccessSets.member_begin(I), AE = AccessSets.member_end();
|
|
|
|
|
|
|
|
// Check every access pair.
|
|
|
|
while (AI != AE) {
|
|
|
|
CheckDeps.erase(*AI);
|
|
|
|
EquivalenceClasses<MemAccessInfo>::member_iterator OI = std::next(AI);
|
|
|
|
while (OI != AE) {
|
|
|
|
// Check every accessing instruction pair in program order.
|
|
|
|
for (std::vector<unsigned>::iterator I1 = Accesses[*AI].begin(),
|
|
|
|
I1E = Accesses[*AI].end(); I1 != I1E; ++I1)
|
|
|
|
for (std::vector<unsigned>::iterator I2 = Accesses[*OI].begin(),
|
|
|
|
I2E = Accesses[*OI].end(); I2 != I2E; ++I2) {
|
2015-03-11 01:40:37 +08:00
|
|
|
auto A = std::make_pair(&*AI, *I1);
|
|
|
|
auto B = std::make_pair(&*OI, *I2);
|
|
|
|
|
|
|
|
assert(*I1 != *I2);
|
|
|
|
if (*I1 > *I2)
|
|
|
|
std::swap(A, B);
|
|
|
|
|
|
|
|
Dependence::DepType Type =
|
|
|
|
isDependent(*A.first, A.second, *B.first, B.second, Strides);
|
|
|
|
SafeForVectorization &= Dependence::isSafeForVectorization(Type);
|
|
|
|
|
|
|
|
// Gather dependences unless we accumulated MaxInterestingDependence
|
|
|
|
// dependences. In that case return as soon as we find the first
|
|
|
|
// unsafe dependence. This puts a limit on this quadratic
|
|
|
|
// algorithm.
|
|
|
|
if (RecordInterestingDependences) {
|
|
|
|
if (Dependence::isInterestingDependence(Type))
|
|
|
|
InterestingDependences.push_back(
|
|
|
|
Dependence(A.second, B.second, Type));
|
|
|
|
|
|
|
|
if (InterestingDependences.size() >= MaxInterestingDependence) {
|
|
|
|
RecordInterestingDependences = false;
|
|
|
|
InterestingDependences.clear();
|
|
|
|
DEBUG(dbgs() << "Too many dependences, stopped recording\n");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!RecordInterestingDependences && !SafeForVectorization)
|
2015-02-02 00:56:15 +08:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
++OI;
|
|
|
|
}
|
|
|
|
AI++;
|
|
|
|
}
|
|
|
|
}
|
2015-03-11 01:40:37 +08:00
|
|
|
|
|
|
|
DEBUG(dbgs() << "Total Interesting Dependences: "
|
|
|
|
<< InterestingDependences.size() << "\n");
|
|
|
|
return SafeForVectorization;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-03-11 02:54:26 +08:00
|
|
|
SmallVector<Instruction *, 4>
|
|
|
|
MemoryDepChecker::getInstructionsForAccess(Value *Ptr, bool isWrite) const {
|
|
|
|
MemAccessInfo Access(Ptr, isWrite);
|
|
|
|
auto &IndexVector = Accesses.find(Access)->second;
|
|
|
|
|
|
|
|
SmallVector<Instruction *, 4> Insts;
|
|
|
|
std::transform(IndexVector.begin(), IndexVector.end(),
|
|
|
|
std::back_inserter(Insts),
|
|
|
|
[&](unsigned Idx) { return this->InstMap[Idx]; });
|
|
|
|
return Insts;
|
|
|
|
}
|
|
|
|
|
2015-03-11 01:40:43 +08:00
|
|
|
const char *MemoryDepChecker::Dependence::DepName[] = {
|
|
|
|
"NoDep", "Unknown", "Forward", "ForwardButPreventsForwarding", "Backward",
|
|
|
|
"BackwardVectorizable", "BackwardVectorizableButPreventsForwarding"};
|
|
|
|
|
|
|
|
void MemoryDepChecker::Dependence::print(
|
|
|
|
raw_ostream &OS, unsigned Depth,
|
|
|
|
const SmallVectorImpl<Instruction *> &Instrs) const {
|
|
|
|
OS.indent(Depth) << DepName[Type] << ":\n";
|
|
|
|
OS.indent(Depth + 2) << *Instrs[Source] << " -> \n";
|
|
|
|
OS.indent(Depth + 2) << *Instrs[Destination] << "\n";
|
|
|
|
}
|
|
|
|
|
2015-02-20 03:15:10 +08:00
|
|
|
bool LoopAccessInfo::canAnalyzeLoop() {
|
2015-04-18 06:43:10 +08:00
|
|
|
// We need to have a loop header.
|
|
|
|
DEBUG(dbgs() << "LAA: Found a loop: " <<
|
|
|
|
TheLoop->getHeader()->getName() << '\n');
|
|
|
|
|
2015-02-20 03:15:10 +08:00
|
|
|
// We can only analyze innermost loops.
|
|
|
|
if (!TheLoop->empty()) {
|
2015-04-18 06:43:10 +08:00
|
|
|
DEBUG(dbgs() << "LAA: loop is not the innermost loop\n");
|
2015-02-20 03:15:15 +08:00
|
|
|
emitAnalysis(LoopAccessReport() << "loop is not the innermost loop");
|
2015-02-20 03:15:10 +08:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// We must have a single backedge.
|
|
|
|
if (TheLoop->getNumBackEdges() != 1) {
|
2015-04-18 06:43:10 +08:00
|
|
|
DEBUG(dbgs() << "LAA: loop control flow is not understood by analyzer\n");
|
2015-02-20 03:15:10 +08:00
|
|
|
emitAnalysis(
|
2015-02-20 03:15:15 +08:00
|
|
|
LoopAccessReport() <<
|
2015-02-20 03:15:10 +08:00
|
|
|
"loop control flow is not understood by analyzer");
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// We must have a single exiting block.
|
|
|
|
if (!TheLoop->getExitingBlock()) {
|
2015-04-18 06:43:10 +08:00
|
|
|
DEBUG(dbgs() << "LAA: loop control flow is not understood by analyzer\n");
|
2015-02-20 03:15:10 +08:00
|
|
|
emitAnalysis(
|
2015-02-20 03:15:15 +08:00
|
|
|
LoopAccessReport() <<
|
2015-02-20 03:15:10 +08:00
|
|
|
"loop control flow is not understood by analyzer");
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// We only handle bottom-tested loops, i.e. loop in which the condition is
|
|
|
|
// checked at the end of each iteration. With that we can assume that all
|
|
|
|
// instructions in the loop are executed the same number of times.
|
|
|
|
if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {
|
2015-04-18 06:43:10 +08:00
|
|
|
DEBUG(dbgs() << "LAA: loop control flow is not understood by analyzer\n");
|
2015-02-20 03:15:10 +08:00
|
|
|
emitAnalysis(
|
2015-02-20 03:15:15 +08:00
|
|
|
LoopAccessReport() <<
|
2015-02-20 03:15:10 +08:00
|
|
|
"loop control flow is not understood by analyzer");
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// ScalarEvolution needs to be able to find the exit count.
|
|
|
|
const SCEV *ExitCount = SE->getBackedgeTakenCount(TheLoop);
|
|
|
|
if (ExitCount == SE->getCouldNotCompute()) {
|
2015-02-20 03:15:15 +08:00
|
|
|
emitAnalysis(LoopAccessReport() <<
|
2015-02-20 03:15:10 +08:00
|
|
|
"could not determine number of loop iterations");
|
|
|
|
DEBUG(dbgs() << "LAA: SCEV could not compute the loop exit count.\n");
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2015-02-24 08:41:59 +08:00
|
|
|
void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) {
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
typedef SmallVector<Value*, 16> ValueVector;
|
|
|
|
typedef SmallPtrSet<Value*, 16> ValueSet;
|
|
|
|
|
|
|
|
// Holds the Load and Store *instructions*.
|
|
|
|
ValueVector Loads;
|
|
|
|
ValueVector Stores;
|
|
|
|
|
|
|
|
// Holds all the different accesses in the loop.
|
|
|
|
unsigned NumReads = 0;
|
|
|
|
unsigned NumReadWrites = 0;
|
|
|
|
|
2015-07-15 06:32:44 +08:00
|
|
|
PtrRtChecking.Pointers.clear();
|
|
|
|
PtrRtChecking.Need = false;
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
const bool IsAnnotatedParallel = TheLoop->isAnnotatedParallel();
|
|
|
|
|
|
|
|
// For each block.
|
|
|
|
for (Loop::block_iterator bb = TheLoop->block_begin(),
|
|
|
|
be = TheLoop->block_end(); bb != be; ++bb) {
|
|
|
|
|
|
|
|
// Scan the BB and collect legal loads and stores.
|
|
|
|
for (BasicBlock::iterator it = (*bb)->begin(), e = (*bb)->end(); it != e;
|
|
|
|
++it) {
|
|
|
|
|
|
|
|
// If this is a load, save it. If this instruction can read from memory
|
|
|
|
// but is not a load, then we quit. Notice that we don't handle function
|
|
|
|
// calls that read or write.
|
|
|
|
if (it->mayReadFromMemory()) {
|
|
|
|
// Many math library functions read the rounding mode. We will only
|
|
|
|
// vectorize a loop if it contains known function calls that don't set
|
|
|
|
// the flag. Therefore, it is safe to ignore this read from memory.
|
|
|
|
CallInst *Call = dyn_cast<CallInst>(it);
|
|
|
|
if (Call && getIntrinsicIDForCall(Call, TLI))
|
|
|
|
continue;
|
|
|
|
|
2015-03-18 03:46:50 +08:00
|
|
|
// If the function has an explicit vectorized counterpart, we can safely
|
|
|
|
// assume that it can be vectorized.
|
|
|
|
if (Call && !Call->isNoBuiltin() && Call->getCalledFunction() &&
|
|
|
|
TLI->isFunctionVectorizable(Call->getCalledFunction()->getName()))
|
|
|
|
continue;
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
LoadInst *Ld = dyn_cast<LoadInst>(it);
|
|
|
|
if (!Ld || (!Ld->isSimple() && !IsAnnotatedParallel)) {
|
2015-02-20 03:15:15 +08:00
|
|
|
emitAnalysis(LoopAccessReport(Ld)
|
2015-02-02 00:56:15 +08:00
|
|
|
<< "read with atomic ordering or volatile read");
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Found a non-simple load.\n");
|
2015-02-20 03:15:00 +08:00
|
|
|
CanVecMem = false;
|
|
|
|
return;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
NumLoads++;
|
|
|
|
Loads.push_back(Ld);
|
|
|
|
DepChecker.addAccess(Ld);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Save 'store' instructions. Abort if other instructions write to memory.
|
|
|
|
if (it->mayWriteToMemory()) {
|
|
|
|
StoreInst *St = dyn_cast<StoreInst>(it);
|
|
|
|
if (!St) {
|
2015-02-20 03:15:15 +08:00
|
|
|
emitAnalysis(LoopAccessReport(it) <<
|
2015-02-20 03:14:34 +08:00
|
|
|
"instruction cannot be vectorized");
|
2015-02-20 03:15:00 +08:00
|
|
|
CanVecMem = false;
|
|
|
|
return;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
if (!St->isSimple() && !IsAnnotatedParallel) {
|
2015-02-20 03:15:15 +08:00
|
|
|
emitAnalysis(LoopAccessReport(St)
|
2015-02-02 00:56:15 +08:00
|
|
|
<< "write with atomic ordering or volatile write");
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Found a non-simple store.\n");
|
2015-02-20 03:15:00 +08:00
|
|
|
CanVecMem = false;
|
|
|
|
return;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
NumStores++;
|
|
|
|
Stores.push_back(St);
|
|
|
|
DepChecker.addAccess(St);
|
|
|
|
}
|
|
|
|
} // Next instr.
|
|
|
|
} // Next block.
|
|
|
|
|
|
|
|
// Now we have two lists that hold the loads and the stores.
|
|
|
|
// Next, we find the pointers that they use.
|
|
|
|
|
|
|
|
// Check if we see any stores. If there are no stores, then we don't
|
|
|
|
// care if the pointers are *restrict*.
|
|
|
|
if (!Stores.size()) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Found a read-only loop!\n");
|
2015-02-20 03:15:00 +08:00
|
|
|
CanVecMem = true;
|
|
|
|
return;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-03-11 01:40:34 +08:00
|
|
|
MemoryDepChecker::DepCandidates DependentAccesses;
|
2015-03-10 10:37:25 +08:00
|
|
|
AccessAnalysis Accesses(TheLoop->getHeader()->getModule()->getDataLayout(),
|
2015-04-24 04:09:20 +08:00
|
|
|
AA, LI, DependentAccesses);
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
// Holds the analyzed pointers. We don't want to call GetUnderlyingObjects
|
|
|
|
// multiple times on the same object. If the ptr is accessed twice, once
|
|
|
|
// for read and once for write, it will only appear once (on the write
|
|
|
|
// list). This is okay, since we are going to check for conflicts between
|
|
|
|
// writes and between reads and writes, but not between reads and reads.
|
|
|
|
ValueSet Seen;
|
|
|
|
|
|
|
|
ValueVector::iterator I, IE;
|
|
|
|
for (I = Stores.begin(), IE = Stores.end(); I != IE; ++I) {
|
|
|
|
StoreInst *ST = cast<StoreInst>(*I);
|
|
|
|
Value* Ptr = ST->getPointerOperand();
|
2015-04-09 01:48:40 +08:00
|
|
|
// Check for store to loop invariant address.
|
|
|
|
StoreToLoopInvariantAddress |= isUniform(Ptr);
|
2015-02-02 00:56:15 +08:00
|
|
|
// If we did *not* see this pointer before, insert it to the read-write
|
|
|
|
// list. At this phase it is only a 'write' list.
|
|
|
|
if (Seen.insert(Ptr).second) {
|
|
|
|
++NumReadWrites;
|
|
|
|
|
2015-06-17 15:18:54 +08:00
|
|
|
MemoryLocation Loc = MemoryLocation::get(ST);
|
2015-02-02 00:56:15 +08:00
|
|
|
// The TBAA metadata could have a control dependency on the predication
|
|
|
|
// condition, so we cannot rely on it when determining whether or not we
|
|
|
|
// need runtime pointer checks.
|
2015-02-18 11:43:19 +08:00
|
|
|
if (blockNeedsPredication(ST->getParent(), TheLoop, DT))
|
2015-02-02 00:56:15 +08:00
|
|
|
Loc.AATags.TBAA = nullptr;
|
|
|
|
|
|
|
|
Accesses.addStore(Loc);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (IsAnnotatedParallel) {
|
2015-02-20 03:14:34 +08:00
|
|
|
DEBUG(dbgs()
|
2015-02-20 03:15:07 +08:00
|
|
|
<< "LAA: A loop annotated parallel, ignore memory dependency "
|
2015-02-20 03:14:34 +08:00
|
|
|
<< "checks.\n");
|
2015-02-20 03:15:00 +08:00
|
|
|
CanVecMem = true;
|
|
|
|
return;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
for (I = Loads.begin(), IE = Loads.end(); I != IE; ++I) {
|
|
|
|
LoadInst *LD = cast<LoadInst>(*I);
|
|
|
|
Value* Ptr = LD->getPointerOperand();
|
|
|
|
// If we did *not* see this pointer before, insert it to the
|
|
|
|
// read list. If we *did* see it before, then it is already in
|
|
|
|
// the read-write list. This allows us to vectorize expressions
|
|
|
|
// such as A[i] += x; Because the address of A[i] is a read-write
|
|
|
|
// pointer. This only works if the index of A[i] is consecutive.
|
|
|
|
// If the address of i is unknown (for example A[B[i]]) then we may
|
|
|
|
// read a few words, modify, and write a few words, and some of the
|
|
|
|
// words may be written to the same address.
|
|
|
|
bool IsReadOnlyPtr = false;
|
2015-03-10 10:37:25 +08:00
|
|
|
if (Seen.insert(Ptr).second || !isStridedPtr(SE, Ptr, TheLoop, Strides)) {
|
2015-02-02 00:56:15 +08:00
|
|
|
++NumReads;
|
|
|
|
IsReadOnlyPtr = true;
|
|
|
|
}
|
|
|
|
|
2015-06-17 15:18:54 +08:00
|
|
|
MemoryLocation Loc = MemoryLocation::get(LD);
|
2015-02-02 00:56:15 +08:00
|
|
|
// The TBAA metadata could have a control dependency on the predication
|
|
|
|
// condition, so we cannot rely on it when determining whether or not we
|
|
|
|
// need runtime pointer checks.
|
2015-02-18 11:43:19 +08:00
|
|
|
if (blockNeedsPredication(LD->getParent(), TheLoop, DT))
|
2015-02-02 00:56:15 +08:00
|
|
|
Loc.AATags.TBAA = nullptr;
|
|
|
|
|
|
|
|
Accesses.addLoad(Loc, IsReadOnlyPtr);
|
|
|
|
}
|
|
|
|
|
|
|
|
// If we write (or read-write) to a single destination and there are no
|
|
|
|
// other reads in this loop then is it safe to vectorize.
|
|
|
|
if (NumReadWrites == 1 && NumReads == 0) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Found a write-only loop!\n");
|
2015-02-20 03:15:00 +08:00
|
|
|
CanVecMem = true;
|
|
|
|
return;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Build dependence sets and check whether we need a runtime pointer bounds
|
|
|
|
// check.
|
|
|
|
Accesses.buildDependenceSets();
|
|
|
|
|
|
|
|
// Find pointers with computable bounds. We are going to use this information
|
|
|
|
// to place a runtime bound check.
|
2015-07-10 06:17:38 +08:00
|
|
|
bool CanDoRTIfNeeded =
|
2015-07-15 06:32:44 +08:00
|
|
|
Accesses.canCheckPtrAtRT(PtrRtChecking, SE, TheLoop, Strides);
|
2015-07-10 06:17:38 +08:00
|
|
|
if (!CanDoRTIfNeeded) {
|
2015-02-20 03:15:15 +08:00
|
|
|
emitAnalysis(LoopAccessReport() << "cannot identify array bounds");
|
2015-07-10 06:17:38 +08:00
|
|
|
DEBUG(dbgs() << "LAA: We can't vectorize because we can't find "
|
|
|
|
<< "the array bounds.\n");
|
2015-02-20 03:15:00 +08:00
|
|
|
CanVecMem = false;
|
|
|
|
return;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-07-10 06:17:38 +08:00
|
|
|
DEBUG(dbgs() << "LAA: We can perform a memory runtime check if needed.\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-02-20 03:15:00 +08:00
|
|
|
CanVecMem = true;
|
2015-02-02 00:56:15 +08:00
|
|
|
if (Accesses.isDependencyCheckNeeded()) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Checking memory dependencies\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
CanVecMem = DepChecker.areDepsSafe(
|
|
|
|
DependentAccesses, Accesses.getDependenciesToCheck(), Strides);
|
|
|
|
MaxSafeDepDistBytes = DepChecker.getMaxSafeDepDistBytes();
|
|
|
|
|
|
|
|
if (!CanVecMem && DepChecker.shouldRetryWithRuntimeCheck()) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Retrying with memory checks\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
// Clear the dependency checks. We assume they are not needed.
|
2015-05-18 23:37:03 +08:00
|
|
|
Accesses.resetDepChecks(DepChecker);
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-07-15 06:32:44 +08:00
|
|
|
PtrRtChecking.reset();
|
|
|
|
PtrRtChecking.Need = true;
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-07-10 06:17:38 +08:00
|
|
|
CanDoRTIfNeeded =
|
2015-07-15 06:32:44 +08:00
|
|
|
Accesses.canCheckPtrAtRT(PtrRtChecking, SE, TheLoop, Strides, true);
|
2015-06-08 18:27:06 +08:00
|
|
|
|
2015-03-11 03:12:41 +08:00
|
|
|
// Check that we found the bounds for the pointer.
|
2015-07-10 06:17:38 +08:00
|
|
|
if (!CanDoRTIfNeeded) {
|
2015-03-11 02:54:19 +08:00
|
|
|
emitAnalysis(LoopAccessReport()
|
|
|
|
<< "cannot check memory dependencies at runtime");
|
|
|
|
DEBUG(dbgs() << "LAA: Can't vectorize with memory checks\n");
|
|
|
|
CanVecMem = false;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
CanVecMem = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-03-11 05:47:39 +08:00
|
|
|
if (CanVecMem)
|
|
|
|
DEBUG(dbgs() << "LAA: No unsafe dependent memory operations in loop. We"
|
2015-07-15 06:32:44 +08:00
|
|
|
<< (PtrRtChecking.Need ? "" : " don't")
|
2015-07-10 06:17:41 +08:00
|
|
|
<< " need runtime memory checks.\n");
|
2015-03-11 05:47:39 +08:00
|
|
|
else {
|
2015-02-20 03:15:15 +08:00
|
|
|
emitAnalysis(LoopAccessReport() <<
|
2015-02-20 03:14:34 +08:00
|
|
|
"unsafe dependent memory operations in loop");
|
2015-03-11 05:47:39 +08:00
|
|
|
DEBUG(dbgs() << "LAA: unsafe dependent memory operations in loop\n");
|
|
|
|
}
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-02-18 11:43:19 +08:00
|
|
|
bool LoopAccessInfo::blockNeedsPredication(BasicBlock *BB, Loop *TheLoop,
|
|
|
|
DominatorTree *DT) {
|
2015-02-02 00:56:15 +08:00
|
|
|
assert(TheLoop->contains(BB) && "Unknown block used");
|
|
|
|
|
|
|
|
// Blocks that do not dominate the latch need predication.
|
|
|
|
BasicBlock* Latch = TheLoop->getLoopLatch();
|
|
|
|
return !DT->dominates(BB, Latch);
|
|
|
|
}
|
|
|
|
|
2015-02-20 03:15:15 +08:00
|
|
|
void LoopAccessInfo::emitAnalysis(LoopAccessReport &Message) {
|
2015-02-20 03:14:56 +08:00
|
|
|
assert(!Report && "Multiple reports generated");
|
|
|
|
Report = Message;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-02-20 03:15:21 +08:00
|
|
|
bool LoopAccessInfo::isUniform(Value *V) const {
|
2015-02-02 00:56:15 +08:00
|
|
|
return (SE->isLoopInvariant(SE->getSCEV(V), TheLoop));
|
|
|
|
}
|
2015-02-07 02:31:04 +08:00
|
|
|
|
|
|
|
// FIXME: this function is currently a duplicate of the one in
|
|
|
|
// LoopVectorize.cpp.
|
|
|
|
static Instruction *getFirstInst(Instruction *FirstInst, Value *V,
|
|
|
|
Instruction *Loc) {
|
|
|
|
if (FirstInst)
|
|
|
|
return FirstInst;
|
|
|
|
if (Instruction *I = dyn_cast<Instruction>(V))
|
|
|
|
return I->getParent() == Loc->getParent() ? I : nullptr;
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
|
2015-07-26 13:32:14 +08:00
|
|
|
/// \brief IR Values for the lower and upper bounds of a pointer evolution.
|
|
|
|
struct PointerBounds {
|
|
|
|
Value *Start;
|
|
|
|
Value *End;
|
|
|
|
};
|
|
|
|
|
|
|
|
/// \brief Expand code for the lower and upper bound of the pointer group \p CG
|
|
|
|
/// in \p TheLoop. \return the values for the bounds.
|
|
|
|
static PointerBounds
|
|
|
|
expandBounds(const RuntimePointerChecking::CheckingPtrGroup *CG, Loop *TheLoop,
|
|
|
|
Instruction *Loc, SCEVExpander &Exp, ScalarEvolution *SE,
|
|
|
|
const RuntimePointerChecking &PtrRtChecking) {
|
|
|
|
Value *Ptr = PtrRtChecking.Pointers[CG->Members[0]].PointerValue;
|
|
|
|
const SCEV *Sc = SE->getSCEV(Ptr);
|
|
|
|
|
|
|
|
if (SE->isLoopInvariant(Sc, TheLoop)) {
|
|
|
|
DEBUG(dbgs() << "LAA: Adding RT check for a loop invariant ptr:" << *Ptr
|
|
|
|
<< "\n");
|
|
|
|
return {Ptr, Ptr};
|
|
|
|
} else {
|
|
|
|
unsigned AS = Ptr->getType()->getPointerAddressSpace();
|
|
|
|
LLVMContext &Ctx = Loc->getContext();
|
|
|
|
|
|
|
|
// Use this type for pointer arithmetic.
|
|
|
|
Type *PtrArithTy = Type::getInt8PtrTy(Ctx, AS);
|
|
|
|
Value *Start = nullptr, *End = nullptr;
|
|
|
|
|
|
|
|
DEBUG(dbgs() << "LAA: Adding RT check for range:\n");
|
|
|
|
Start = Exp.expandCodeFor(CG->Low, PtrArithTy, Loc);
|
|
|
|
End = Exp.expandCodeFor(CG->High, PtrArithTy, Loc);
|
|
|
|
DEBUG(dbgs() << "Start: " << *CG->Low << " End: " << *CG->High << "\n");
|
|
|
|
return {Start, End};
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/// \brief Turns a collection of checks into a collection of expanded upper and
|
|
|
|
/// lower bounds for both pointers in the check.
|
|
|
|
static SmallVector<std::pair<PointerBounds, PointerBounds>, 4> expandBounds(
|
|
|
|
const SmallVectorImpl<RuntimePointerChecking::PointerCheck> &PointerChecks,
|
|
|
|
Loop *L, Instruction *Loc, ScalarEvolution *SE, SCEVExpander &Exp,
|
|
|
|
const RuntimePointerChecking &PtrRtChecking) {
|
|
|
|
SmallVector<std::pair<PointerBounds, PointerBounds>, 4> ChecksWithBounds;
|
|
|
|
|
|
|
|
// Here we're relying on the SCEV Expander's cache to only emit code for the
|
|
|
|
// same bounds once.
|
|
|
|
std::transform(
|
|
|
|
PointerChecks.begin(), PointerChecks.end(),
|
|
|
|
std::back_inserter(ChecksWithBounds),
|
|
|
|
[&](const RuntimePointerChecking::PointerCheck &Check) {
|
2015-07-27 09:35:30 +08:00
|
|
|
PointerBounds
|
|
|
|
First = expandBounds(Check.first, L, Loc, Exp, SE, PtrRtChecking),
|
|
|
|
Second = expandBounds(Check.second, L, Loc, Exp, SE, PtrRtChecking);
|
|
|
|
return std::make_pair(First, Second);
|
2015-07-26 13:32:14 +08:00
|
|
|
});
|
|
|
|
|
|
|
|
return ChecksWithBounds;
|
|
|
|
}
|
|
|
|
|
2015-03-11 02:54:26 +08:00
|
|
|
std::pair<Instruction *, Instruction *> LoopAccessInfo::addRuntimeCheck(
|
2015-07-26 13:32:14 +08:00
|
|
|
Instruction *Loc,
|
|
|
|
const SmallVectorImpl<RuntimePointerChecking::PointerCheck> &PointerChecks)
|
|
|
|
const {
|
2015-02-07 02:31:04 +08:00
|
|
|
|
2015-07-26 13:32:14 +08:00
|
|
|
SCEVExpander Exp(*SE, DL, "induction");
|
|
|
|
auto ExpandedChecks =
|
|
|
|
expandBounds(PointerChecks, TheLoop, Loc, SE, Exp, PtrRtChecking);
|
2015-02-07 02:31:04 +08:00
|
|
|
|
|
|
|
LLVMContext &Ctx = Loc->getContext();
|
|
|
|
Instruction *FirstInst = nullptr;
|
|
|
|
IRBuilder<> ChkBuilder(Loc);
|
|
|
|
// Our instructions might fold to a constant.
|
|
|
|
Value *MemoryRuntimeCheck = nullptr;
|
2015-07-15 06:32:44 +08:00
|
|
|
|
2015-07-26 13:32:14 +08:00
|
|
|
for (const auto &Check : ExpandedChecks) {
|
|
|
|
const PointerBounds &A = Check.first, &B = Check.second;
|
|
|
|
unsigned AS0 = A.Start->getType()->getPointerAddressSpace();
|
|
|
|
unsigned AS1 = B.Start->getType()->getPointerAddressSpace();
|
|
|
|
|
|
|
|
assert((AS0 == B.End->getType()->getPointerAddressSpace()) &&
|
|
|
|
(AS1 == A.End->getType()->getPointerAddressSpace()) &&
|
|
|
|
"Trying to bounds check pointers with different address spaces");
|
|
|
|
|
|
|
|
Type *PtrArithTy0 = Type::getInt8PtrTy(Ctx, AS0);
|
|
|
|
Type *PtrArithTy1 = Type::getInt8PtrTy(Ctx, AS1);
|
|
|
|
|
|
|
|
Value *Start0 = ChkBuilder.CreateBitCast(A.Start, PtrArithTy0, "bc");
|
|
|
|
Value *Start1 = ChkBuilder.CreateBitCast(B.Start, PtrArithTy1, "bc");
|
|
|
|
Value *End0 = ChkBuilder.CreateBitCast(A.End, PtrArithTy1, "bc");
|
|
|
|
Value *End1 = ChkBuilder.CreateBitCast(B.End, PtrArithTy0, "bc");
|
|
|
|
|
|
|
|
Value *Cmp0 = ChkBuilder.CreateICmpULE(Start0, End1, "bound0");
|
|
|
|
FirstInst = getFirstInst(FirstInst, Cmp0, Loc);
|
|
|
|
Value *Cmp1 = ChkBuilder.CreateICmpULE(Start1, End0, "bound1");
|
|
|
|
FirstInst = getFirstInst(FirstInst, Cmp1, Loc);
|
|
|
|
Value *IsConflict = ChkBuilder.CreateAnd(Cmp0, Cmp1, "found.conflict");
|
|
|
|
FirstInst = getFirstInst(FirstInst, IsConflict, Loc);
|
|
|
|
if (MemoryRuntimeCheck) {
|
|
|
|
IsConflict =
|
|
|
|
ChkBuilder.CreateOr(MemoryRuntimeCheck, IsConflict, "conflict.rdx");
|
2015-02-07 02:31:04 +08:00
|
|
|
FirstInst = getFirstInst(FirstInst, IsConflict, Loc);
|
|
|
|
}
|
2015-07-26 13:32:14 +08:00
|
|
|
MemoryRuntimeCheck = IsConflict;
|
2015-02-07 02:31:04 +08:00
|
|
|
}
|
|
|
|
|
2015-04-03 01:51:57 +08:00
|
|
|
if (!MemoryRuntimeCheck)
|
|
|
|
return std::make_pair(nullptr, nullptr);
|
|
|
|
|
2015-02-07 02:31:04 +08:00
|
|
|
// We have to do this trickery because the IRBuilder might fold the check to a
|
|
|
|
// constant expression in which case there is no Instruction anchored in a
|
|
|
|
// the block.
|
|
|
|
Instruction *Check = BinaryOperator::CreateAnd(MemoryRuntimeCheck,
|
|
|
|
ConstantInt::getTrue(Ctx));
|
|
|
|
ChkBuilder.Insert(Check, "memcheck.conflict");
|
|
|
|
FirstInst = getFirstInst(FirstInst, Check, Loc);
|
|
|
|
return std::make_pair(FirstInst, Check);
|
|
|
|
}
|
2015-02-20 03:15:04 +08:00
|
|
|
|
2015-07-26 13:32:14 +08:00
|
|
|
std::pair<Instruction *, Instruction *> LoopAccessInfo::addRuntimeCheck(
|
2015-08-04 13:16:20 +08:00
|
|
|
Instruction *Loc) const {
|
2015-07-26 13:32:14 +08:00
|
|
|
if (!PtrRtChecking.Need)
|
|
|
|
return std::make_pair(nullptr, nullptr);
|
|
|
|
|
2015-08-08 06:44:15 +08:00
|
|
|
return addRuntimeCheck(Loc, PtrRtChecking.getChecks());
|
2015-07-26 13:32:14 +08:00
|
|
|
}
|
|
|
|
|
2015-02-20 03:15:04 +08:00
|
|
|
LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE,
|
2015-03-10 10:37:25 +08:00
|
|
|
const DataLayout &DL,
|
2015-02-20 03:15:04 +08:00
|
|
|
const TargetLibraryInfo *TLI, AliasAnalysis *AA,
|
2015-04-24 04:09:20 +08:00
|
|
|
DominatorTree *DT, LoopInfo *LI,
|
2015-02-24 08:41:59 +08:00
|
|
|
const ValueToValueMap &Strides)
|
2015-07-15 06:32:44 +08:00
|
|
|
: PtrRtChecking(SE), DepChecker(SE, L), TheLoop(L), SE(SE), DL(DL),
|
|
|
|
TLI(TLI), AA(AA), DT(DT), LI(LI), NumLoads(0), NumStores(0),
|
2015-04-09 01:48:40 +08:00
|
|
|
MaxSafeDepDistBytes(-1U), CanVecMem(false),
|
|
|
|
StoreToLoopInvariantAddress(false) {
|
2015-02-20 03:15:10 +08:00
|
|
|
if (canAnalyzeLoop())
|
|
|
|
analyzeLoop(Strides);
|
2015-02-20 03:15:04 +08:00
|
|
|
}
|
|
|
|
|
2015-02-20 03:15:19 +08:00
|
|
|
void LoopAccessInfo::print(raw_ostream &OS, unsigned Depth) const {
|
|
|
|
if (CanVecMem) {
|
2015-07-15 06:32:44 +08:00
|
|
|
if (PtrRtChecking.Need)
|
2015-02-20 03:15:19 +08:00
|
|
|
OS.indent(Depth) << "Memory dependences are safe with run-time checks\n";
|
2015-04-14 09:12:55 +08:00
|
|
|
else
|
|
|
|
OS.indent(Depth) << "Memory dependences are safe\n";
|
2015-02-20 03:15:19 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
if (Report)
|
|
|
|
OS.indent(Depth) << "Report: " << Report->str() << "\n";
|
|
|
|
|
2015-03-11 01:40:43 +08:00
|
|
|
if (auto *InterestingDependences = DepChecker.getInterestingDependences()) {
|
|
|
|
OS.indent(Depth) << "Interesting Dependences:\n";
|
|
|
|
for (auto &Dep : *InterestingDependences) {
|
|
|
|
Dep.print(OS, Depth + 2, DepChecker.getMemoryInstructions());
|
|
|
|
OS << "\n";
|
|
|
|
}
|
|
|
|
} else
|
|
|
|
OS.indent(Depth) << "Too many interesting dependences, not recorded\n";
|
2015-02-20 03:15:19 +08:00
|
|
|
|
|
|
|
// List the pair of accesses need run-time checks to prove independence.
|
2015-07-15 06:32:44 +08:00
|
|
|
PtrRtChecking.print(OS, Depth);
|
2015-02-20 03:15:19 +08:00
|
|
|
OS << "\n";
|
2015-05-18 23:36:57 +08:00
|
|
|
|
|
|
|
OS.indent(Depth) << "Store to invariant address was "
|
|
|
|
<< (StoreToLoopInvariantAddress ? "" : "not ")
|
|
|
|
<< "found in loop.\n";
|
2015-02-20 03:15:19 +08:00
|
|
|
}
|
|
|
|
|
2015-02-24 08:41:59 +08:00
|
|
|
const LoopAccessInfo &
|
|
|
|
LoopAccessAnalysis::getInfo(Loop *L, const ValueToValueMap &Strides) {
|
2015-02-20 03:15:04 +08:00
|
|
|
auto &LAI = LoopAccessInfoMap[L];
|
|
|
|
|
|
|
|
#ifndef NDEBUG
|
|
|
|
assert((!LAI || LAI->NumSymbolicStrides == Strides.size()) &&
|
|
|
|
"Symbolic strides changed for loop");
|
|
|
|
#endif
|
|
|
|
|
|
|
|
if (!LAI) {
|
2015-03-10 10:37:25 +08:00
|
|
|
const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
|
2015-04-24 04:09:20 +08:00
|
|
|
LAI = llvm::make_unique<LoopAccessInfo>(L, SE, DL, TLI, AA, DT, LI,
|
|
|
|
Strides);
|
2015-02-20 03:15:04 +08:00
|
|
|
#ifndef NDEBUG
|
|
|
|
LAI->NumSymbolicStrides = Strides.size();
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
return *LAI.get();
|
|
|
|
}
|
|
|
|
|
2015-02-20 03:15:19 +08:00
|
|
|
void LoopAccessAnalysis::print(raw_ostream &OS, const Module *M) const {
|
|
|
|
LoopAccessAnalysis &LAA = *const_cast<LoopAccessAnalysis *>(this);
|
|
|
|
|
|
|
|
ValueToValueMap NoSymbolicStrides;
|
|
|
|
|
|
|
|
for (Loop *TopLevelLoop : *LI)
|
|
|
|
for (Loop *L : depth_first(TopLevelLoop)) {
|
|
|
|
OS.indent(2) << L->getHeader()->getName() << ":\n";
|
|
|
|
auto &LAI = LAA.getInfo(L, NoSymbolicStrides);
|
|
|
|
LAI.print(OS, 4);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-02-20 03:15:04 +08:00
|
|
|
bool LoopAccessAnalysis::runOnFunction(Function &F) {
|
|
|
|
SE = &getAnalysis<ScalarEvolution>();
|
|
|
|
auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
|
|
|
|
TLI = TLIP ? &TLIP->getTLI() : nullptr;
|
|
|
|
AA = &getAnalysis<AliasAnalysis>();
|
|
|
|
DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
|
2015-04-24 04:09:20 +08:00
|
|
|
LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
|
2015-02-20 03:15:04 +08:00
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
void LoopAccessAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
|
|
|
|
AU.addRequired<ScalarEvolution>();
|
|
|
|
AU.addRequired<AliasAnalysis>();
|
|
|
|
AU.addRequired<DominatorTreeWrapperPass>();
|
2015-02-20 03:15:19 +08:00
|
|
|
AU.addRequired<LoopInfoWrapperPass>();
|
2015-02-20 03:15:04 +08:00
|
|
|
|
|
|
|
AU.setPreservesAll();
|
|
|
|
}
|
|
|
|
|
|
|
|
char LoopAccessAnalysis::ID = 0;
|
|
|
|
static const char laa_name[] = "Loop Access Analysis";
|
|
|
|
#define LAA_NAME "loop-accesses"
|
|
|
|
|
|
|
|
INITIALIZE_PASS_BEGIN(LoopAccessAnalysis, LAA_NAME, laa_name, false, true)
|
|
|
|
INITIALIZE_AG_DEPENDENCY(AliasAnalysis)
|
|
|
|
INITIALIZE_PASS_DEPENDENCY(ScalarEvolution)
|
|
|
|
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
|
2015-02-20 03:15:19 +08:00
|
|
|
INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
|
2015-02-20 03:15:04 +08:00
|
|
|
INITIALIZE_PASS_END(LoopAccessAnalysis, LAA_NAME, laa_name, false, true)
|
|
|
|
|
|
|
|
namespace llvm {
|
|
|
|
Pass *createLAAPass() {
|
|
|
|
return new LoopAccessAnalysis();
|
|
|
|
}
|
|
|
|
}
|