2015-02-02 00:56:15 +08:00
|
|
|
//===- LoopAccessAnalysis.cpp - Loop Access Analysis Implementation --------==//
|
|
|
|
//
|
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
//
|
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
// License. See LICENSE.TXT for details.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
|
|
|
// The implementation for the loop memory dependence that was originally
|
|
|
|
// developed for the loop vectorizer.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
#include "llvm/Analysis/LoopAccessAnalysis.h"
|
|
|
|
#include "llvm/Analysis/LoopInfo.h"
|
2015-02-07 02:31:04 +08:00
|
|
|
#include "llvm/Analysis/ScalarEvolutionExpander.h"
|
2015-03-24 03:32:43 +08:00
|
|
|
#include "llvm/Analysis/TargetLibraryInfo.h"
|
2015-02-02 00:56:15 +08:00
|
|
|
#include "llvm/Analysis/ValueTracking.h"
|
|
|
|
#include "llvm/IR/DiagnosticInfo.h"
|
|
|
|
#include "llvm/IR/Dominators.h"
|
2015-02-07 02:31:04 +08:00
|
|
|
#include "llvm/IR/IRBuilder.h"
|
2015-02-02 00:56:15 +08:00
|
|
|
#include "llvm/Support/Debug.h"
|
2015-03-24 03:32:43 +08:00
|
|
|
#include "llvm/Support/raw_ostream.h"
|
2015-06-27 02:02:52 +08:00
|
|
|
#include "llvm/Analysis/VectorUtils.h"
|
2015-02-02 00:56:15 +08:00
|
|
|
using namespace llvm;
|
|
|
|
|
2015-02-20 03:15:07 +08:00
|
|
|
#define DEBUG_TYPE "loop-accesses"
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-02-20 03:14:52 +08:00
|
|
|
static cl::opt<unsigned, true>
|
|
|
|
VectorizationFactor("force-vector-width", cl::Hidden,
|
|
|
|
cl::desc("Sets the SIMD width. Zero is autoselect."),
|
|
|
|
cl::location(VectorizerParams::VectorizationFactor));
|
2015-02-26 12:39:09 +08:00
|
|
|
unsigned VectorizerParams::VectorizationFactor;
|
2015-02-20 03:14:52 +08:00
|
|
|
|
|
|
|
static cl::opt<unsigned, true>
|
|
|
|
VectorizationInterleave("force-vector-interleave", cl::Hidden,
|
|
|
|
cl::desc("Sets the vectorization interleave count. "
|
|
|
|
"Zero is autoselect."),
|
|
|
|
cl::location(
|
|
|
|
VectorizerParams::VectorizationInterleave));
|
2015-02-26 12:39:09 +08:00
|
|
|
unsigned VectorizerParams::VectorizationInterleave;
|
2015-02-20 03:14:52 +08:00
|
|
|
|
2015-02-26 12:39:09 +08:00
|
|
|
static cl::opt<unsigned, true> RuntimeMemoryCheckThreshold(
|
|
|
|
"runtime-memory-check-threshold", cl::Hidden,
|
|
|
|
cl::desc("When performing memory disambiguation checks at runtime do not "
|
|
|
|
"generate more than this number of comparisons (default = 8)."),
|
|
|
|
cl::location(VectorizerParams::RuntimeMemoryCheckThreshold), cl::init(8));
|
|
|
|
unsigned VectorizerParams::RuntimeMemoryCheckThreshold;
|
2015-02-20 03:14:52 +08:00
|
|
|
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
/// \brief The maximum iterations used to merge memory checks
|
|
|
|
static cl::opt<unsigned> MemoryCheckMergeThreshold(
|
|
|
|
"memory-check-merge-threshold", cl::Hidden,
|
|
|
|
cl::desc("Maximum number of comparisons done when trying to merge "
|
|
|
|
"runtime memory checks. (default = 100)"),
|
|
|
|
cl::init(100));
|
|
|
|
|
2015-02-20 03:14:52 +08:00
|
|
|
/// Maximum SIMD width.
|
|
|
|
const unsigned VectorizerParams::MaxVectorWidth = 64;
|
|
|
|
|
2015-11-04 05:39:52 +08:00
|
|
|
/// \brief We collect dependences up to this threshold.
|
|
|
|
static cl::opt<unsigned>
|
|
|
|
MaxDependences("max-dependences", cl::Hidden,
|
|
|
|
cl::desc("Maximum number of dependences collected by "
|
|
|
|
"loop-access analysis (default = 100)"),
|
|
|
|
cl::init(100));
|
2015-03-11 01:40:37 +08:00
|
|
|
|
2015-02-20 03:14:52 +08:00
|
|
|
bool VectorizerParams::isInterleaveForced() {
|
|
|
|
return ::VectorizationInterleave.getNumOccurrences() > 0;
|
|
|
|
}
|
|
|
|
|
2015-02-20 03:15:15 +08:00
|
|
|
void LoopAccessReport::emitAnalysis(const LoopAccessReport &Message,
|
|
|
|
const Function *TheFunction,
|
|
|
|
const Loop *TheLoop,
|
|
|
|
const char *PassName) {
|
2015-02-02 00:56:15 +08:00
|
|
|
DebugLoc DL = TheLoop->getStartLoc();
|
2015-02-20 03:15:13 +08:00
|
|
|
if (const Instruction *I = Message.getInstr())
|
2015-02-02 00:56:15 +08:00
|
|
|
DL = I->getDebugLoc();
|
2015-02-20 03:15:07 +08:00
|
|
|
emitOptimizationRemarkAnalysis(TheFunction->getContext(), PassName,
|
2015-02-02 00:56:15 +08:00
|
|
|
*TheFunction, DL, Message.str());
|
|
|
|
}
|
|
|
|
|
|
|
|
Value *llvm::stripIntegerCast(Value *V) {
|
|
|
|
if (CastInst *CI = dyn_cast<CastInst>(V))
|
|
|
|
if (CI->getOperand(0)->getType()->isIntegerTy())
|
|
|
|
return CI->getOperand(0);
|
|
|
|
return V;
|
|
|
|
}
|
|
|
|
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
const SCEV *llvm::replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE,
|
2015-02-24 08:41:59 +08:00
|
|
|
const ValueToValueMap &PtrToStride,
|
2015-02-02 00:56:15 +08:00
|
|
|
Value *Ptr, Value *OrigPtr) {
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
const SCEV *OrigSCEV = PSE.getSCEV(Ptr);
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
// If there is an entry in the map return the SCEV of the pointer with the
|
|
|
|
// symbolic stride replaced by one.
|
2015-02-24 08:41:59 +08:00
|
|
|
ValueToValueMap::const_iterator SI =
|
|
|
|
PtrToStride.find(OrigPtr ? OrigPtr : Ptr);
|
2015-02-02 00:56:15 +08:00
|
|
|
if (SI != PtrToStride.end()) {
|
|
|
|
Value *StrideVal = SI->second;
|
|
|
|
|
|
|
|
// Strip casts.
|
|
|
|
StrideVal = stripIntegerCast(StrideVal);
|
|
|
|
|
|
|
|
// Replace symbolic stride by one.
|
|
|
|
Value *One = ConstantInt::get(StrideVal->getType(), 1);
|
|
|
|
ValueToValueMap RewriteMap;
|
|
|
|
RewriteMap[StrideVal] = One;
|
|
|
|
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
ScalarEvolution *SE = PSE.getSE();
|
[SCEV][LV] Add SCEV Predicates and use them to re-implement stride versioning
Summary:
SCEV Predicates represent conditions that typically cannot be derived from
static analysis, but can be used to reduce SCEV expressions to forms which are
usable for different optimizers.
ScalarEvolution now has the rewriteUsingPredicate method which can simplify a
SCEV expression using a SCEVPredicateSet. The normal workflow of a pass using
SCEVPredicates would be to hold a SCEVPredicateSet and every time assumptions
need to be made a new SCEV Predicate would be created and added to the set.
Each time after calling getSCEV, the user will call the rewriteUsingPredicate
method.
We add two types of predicates
SCEVPredicateSet - implements a set of predicates
SCEVEqualPredicate - tests for equality between two SCEV expressions
We use the SCEVEqualPredicate to re-implement stride versioning. Every time we
version a stride, we will add a SCEVEqualPredicate to the context.
Instead of adding specific stride checks, LoopVectorize now adds a more
generic SCEV check.
We only need to add support for this in the LoopVectorizer since this is the
only pass that will do stride versioning.
Reviewers: mzolotukhin, anemet, hfinkel, sanjoy
Subscribers: sanjoy, hfinkel, rengolin, jmolloy, llvm-commits
Differential Revision: http://reviews.llvm.org/D13595
llvm-svn: 251800
2015-11-02 22:41:02 +08:00
|
|
|
const auto *U = cast<SCEVUnknown>(SE->getSCEV(StrideVal));
|
|
|
|
const auto *CT =
|
|
|
|
static_cast<const SCEVConstant *>(SE->getOne(StrideVal->getType()));
|
|
|
|
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
PSE.addPredicate(*SE->getEqualPredicate(U, CT));
|
|
|
|
auto *Expr = PSE.getSCEV(Ptr);
|
[SCEV][LV] Add SCEV Predicates and use them to re-implement stride versioning
Summary:
SCEV Predicates represent conditions that typically cannot be derived from
static analysis, but can be used to reduce SCEV expressions to forms which are
usable for different optimizers.
ScalarEvolution now has the rewriteUsingPredicate method which can simplify a
SCEV expression using a SCEVPredicateSet. The normal workflow of a pass using
SCEVPredicates would be to hold a SCEVPredicateSet and every time assumptions
need to be made a new SCEV Predicate would be created and added to the set.
Each time after calling getSCEV, the user will call the rewriteUsingPredicate
method.
We add two types of predicates
SCEVPredicateSet - implements a set of predicates
SCEVEqualPredicate - tests for equality between two SCEV expressions
We use the SCEVEqualPredicate to re-implement stride versioning. Every time we
version a stride, we will add a SCEVEqualPredicate to the context.
Instead of adding specific stride checks, LoopVectorize now adds a more
generic SCEV check.
We only need to add support for this in the LoopVectorizer since this is the
only pass that will do stride versioning.
Reviewers: mzolotukhin, anemet, hfinkel, sanjoy
Subscribers: sanjoy, hfinkel, rengolin, jmolloy, llvm-commits
Differential Revision: http://reviews.llvm.org/D13595
llvm-svn: 251800
2015-11-02 22:41:02 +08:00
|
|
|
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Replacing SCEV: " << *OrigSCEV << " by: " << *Expr
|
2015-02-02 00:56:15 +08:00
|
|
|
<< "\n");
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
return Expr;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Otherwise, just return the SCEV of the original pointer.
|
[SCEV][LV] Add SCEV Predicates and use them to re-implement stride versioning
Summary:
SCEV Predicates represent conditions that typically cannot be derived from
static analysis, but can be used to reduce SCEV expressions to forms which are
usable for different optimizers.
ScalarEvolution now has the rewriteUsingPredicate method which can simplify a
SCEV expression using a SCEVPredicateSet. The normal workflow of a pass using
SCEVPredicates would be to hold a SCEVPredicateSet and every time assumptions
need to be made a new SCEV Predicate would be created and added to the set.
Each time after calling getSCEV, the user will call the rewriteUsingPredicate
method.
We add two types of predicates
SCEVPredicateSet - implements a set of predicates
SCEVEqualPredicate - tests for equality between two SCEV expressions
We use the SCEVEqualPredicate to re-implement stride versioning. Every time we
version a stride, we will add a SCEVEqualPredicate to the context.
Instead of adding specific stride checks, LoopVectorize now adds a more
generic SCEV check.
We only need to add support for this in the LoopVectorizer since this is the
only pass that will do stride versioning.
Reviewers: mzolotukhin, anemet, hfinkel, sanjoy
Subscribers: sanjoy, hfinkel, rengolin, jmolloy, llvm-commits
Differential Revision: http://reviews.llvm.org/D13595
llvm-svn: 251800
2015-11-02 22:41:02 +08:00
|
|
|
return OrigSCEV;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-07-15 06:32:44 +08:00
|
|
|
void RuntimePointerChecking::insert(Loop *Lp, Value *Ptr, bool WritePtr,
|
|
|
|
unsigned DepSetId, unsigned ASId,
|
[SCEV][LV] Add SCEV Predicates and use them to re-implement stride versioning
Summary:
SCEV Predicates represent conditions that typically cannot be derived from
static analysis, but can be used to reduce SCEV expressions to forms which are
usable for different optimizers.
ScalarEvolution now has the rewriteUsingPredicate method which can simplify a
SCEV expression using a SCEVPredicateSet. The normal workflow of a pass using
SCEVPredicates would be to hold a SCEVPredicateSet and every time assumptions
need to be made a new SCEV Predicate would be created and added to the set.
Each time after calling getSCEV, the user will call the rewriteUsingPredicate
method.
We add two types of predicates
SCEVPredicateSet - implements a set of predicates
SCEVEqualPredicate - tests for equality between two SCEV expressions
We use the SCEVEqualPredicate to re-implement stride versioning. Every time we
version a stride, we will add a SCEVEqualPredicate to the context.
Instead of adding specific stride checks, LoopVectorize now adds a more
generic SCEV check.
We only need to add support for this in the LoopVectorizer since this is the
only pass that will do stride versioning.
Reviewers: mzolotukhin, anemet, hfinkel, sanjoy
Subscribers: sanjoy, hfinkel, rengolin, jmolloy, llvm-commits
Differential Revision: http://reviews.llvm.org/D13595
llvm-svn: 251800
2015-11-02 22:41:02 +08:00
|
|
|
const ValueToValueMap &Strides,
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
PredicatedScalarEvolution &PSE) {
|
2015-02-02 00:56:15 +08:00
|
|
|
// Get the stride replaced scev.
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
const SCEV *Sc = replaceSymbolicStrideSCEV(PSE, Strides, Ptr);
|
|
|
|
ScalarEvolution *SE = PSE.getSE();
|
2016-03-24 12:28:47 +08:00
|
|
|
|
|
|
|
const SCEV *ScStart;
|
|
|
|
const SCEV *ScEnd;
|
|
|
|
|
2016-03-24 13:15:24 +08:00
|
|
|
if (SE->isLoopInvariant(Sc, Lp))
|
2016-03-24 12:28:47 +08:00
|
|
|
ScStart = ScEnd = Sc;
|
|
|
|
else {
|
|
|
|
const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Sc);
|
|
|
|
assert(AR && "Invalid addrec expression");
|
Re-commit [SCEV] Introduce a guarded backedge taken count and use it in LAA and LV
This re-commits r265535 which was reverted in r265541 because it
broke the windows bots. The problem was that we had a PointerIntPair
which took a pointer to a struct allocated with new. The problem
was that new doesn't provide sufficient alignment guarantees.
This pattern was already present before r265535 and it just happened
to work. To fix this, we now separate the PointerToIntPair from the
ExitNotTakenInfo struct into a pointer and a bool.
Original commit message:
Summary:
When the backedge taken codition is computed from an icmp, SCEV can
deduce the backedge taken count only if one of the sides of the icmp
is an AddRecExpr. However, due to sign/zero extensions, we sometimes
end up with something that is not an AddRecExpr.
However, we can use SCEV predicates to produce a 'guarded' expression.
This change adds a method to SCEV to get this expression, and the
SCEV predicate associated with it.
In HowManyGreaterThans and HowManyLessThans we will now add a SCEV
predicate associated with the guarded backedge taken count when the
analyzed SCEV expression is not an AddRecExpr. Note that we only do
this as an alternative to returning a 'CouldNotCompute'.
We use new feature in Loop Access Analysis and LoopVectorize to analyze
and transform more loops.
Reviewers: anemet, mzolotukhin, hfinkel, sanjoy
Subscribers: flyingforyou, mcrosier, atrick, mssimpso, sanjoy, mzolotukhin, llvm-commits
Differential Revision: http://reviews.llvm.org/D17201
llvm-svn: 265786
2016-04-08 22:29:09 +08:00
|
|
|
const SCEV *Ex = PSE.getBackedgeTakenCount();
|
2016-03-24 12:28:47 +08:00
|
|
|
|
|
|
|
ScStart = AR->getStart();
|
|
|
|
ScEnd = AR->evaluateAtIteration(Ex, *SE);
|
|
|
|
const SCEV *Step = AR->getStepRecurrence(*SE);
|
|
|
|
|
|
|
|
// For expressions with negative step, the upper bound is ScStart and the
|
|
|
|
// lower bound is ScEnd.
|
|
|
|
if (const SCEVConstant *CStep = dyn_cast<const SCEVConstant>(Step)) {
|
|
|
|
if (CStep->getValue()->isNegative())
|
|
|
|
std::swap(ScStart, ScEnd);
|
|
|
|
} else {
|
|
|
|
// Fallback case: the step is not constant, but the we can still
|
|
|
|
// get the upper and lower bounds of the interval by using min/max
|
|
|
|
// expressions.
|
|
|
|
ScStart = SE->getUMinExpr(ScStart, ScEnd);
|
|
|
|
ScEnd = SE->getUMaxExpr(AR->getStart(), ScEnd);
|
|
|
|
}
|
2015-07-16 22:02:58 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
Pointers.emplace_back(Ptr, ScStart, ScEnd, WritePtr, DepSetId, ASId, Sc);
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
}
|
|
|
|
|
2015-07-28 03:38:48 +08:00
|
|
|
SmallVector<RuntimePointerChecking::PointerCheck, 4>
|
2015-08-10 04:06:06 +08:00
|
|
|
RuntimePointerChecking::generateChecks() const {
|
2015-07-28 03:38:48 +08:00
|
|
|
SmallVector<PointerCheck, 4> Checks;
|
|
|
|
|
2015-07-28 03:38:50 +08:00
|
|
|
for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
|
|
|
|
for (unsigned J = I + 1; J < CheckingGroups.size(); ++J) {
|
|
|
|
const RuntimePointerChecking::CheckingPtrGroup &CGI = CheckingGroups[I];
|
|
|
|
const RuntimePointerChecking::CheckingPtrGroup &CGJ = CheckingGroups[J];
|
2015-07-28 03:38:48 +08:00
|
|
|
|
2015-08-10 04:06:06 +08:00
|
|
|
if (needsChecking(CGI, CGJ))
|
2015-07-28 03:38:48 +08:00
|
|
|
Checks.push_back(std::make_pair(&CGI, &CGJ));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return Checks;
|
|
|
|
}
|
|
|
|
|
2015-08-08 06:44:15 +08:00
|
|
|
void RuntimePointerChecking::generateChecks(
|
|
|
|
MemoryDepChecker::DepCandidates &DepCands, bool UseDependencies) {
|
|
|
|
assert(Checks.empty() && "Checks is not empty");
|
|
|
|
groupChecks(DepCands, UseDependencies);
|
|
|
|
Checks = generateChecks();
|
|
|
|
}
|
|
|
|
|
2015-08-10 04:06:08 +08:00
|
|
|
bool RuntimePointerChecking::needsChecking(const CheckingPtrGroup &M,
|
|
|
|
const CheckingPtrGroup &N) const {
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
for (unsigned I = 0, EI = M.Members.size(); EI != I; ++I)
|
|
|
|
for (unsigned J = 0, EJ = N.Members.size(); EJ != J; ++J)
|
2015-08-10 04:06:08 +08:00
|
|
|
if (needsChecking(M.Members[I], N.Members[J]))
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
return true;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Compare \p I and \p J and return the minimum.
|
|
|
|
/// Return nullptr in case we couldn't find an answer.
|
|
|
|
static const SCEV *getMinFromExprs(const SCEV *I, const SCEV *J,
|
|
|
|
ScalarEvolution *SE) {
|
|
|
|
const SCEV *Diff = SE->getMinusSCEV(J, I);
|
|
|
|
const SCEVConstant *C = dyn_cast<const SCEVConstant>(Diff);
|
|
|
|
|
|
|
|
if (!C)
|
|
|
|
return nullptr;
|
|
|
|
if (C->getValue()->isNegative())
|
|
|
|
return J;
|
|
|
|
return I;
|
|
|
|
}
|
|
|
|
|
2015-07-15 06:32:44 +08:00
|
|
|
bool RuntimePointerChecking::CheckingPtrGroup::addPointer(unsigned Index) {
|
2015-07-15 06:32:50 +08:00
|
|
|
const SCEV *Start = RtCheck.Pointers[Index].Start;
|
|
|
|
const SCEV *End = RtCheck.Pointers[Index].End;
|
|
|
|
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
// Compare the starts and ends with the known minimum and maximum
|
|
|
|
// of this set. We need to know how we compare against the min/max
|
|
|
|
// of the set in order to be able to emit memchecks.
|
2015-07-15 06:32:50 +08:00
|
|
|
const SCEV *Min0 = getMinFromExprs(Start, Low, RtCheck.SE);
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
if (!Min0)
|
|
|
|
return false;
|
|
|
|
|
2015-07-15 06:32:50 +08:00
|
|
|
const SCEV *Min1 = getMinFromExprs(End, High, RtCheck.SE);
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
if (!Min1)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
// Update the low bound expression if we've found a new min value.
|
2015-07-15 06:32:50 +08:00
|
|
|
if (Min0 == Start)
|
|
|
|
Low = Start;
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
|
|
|
|
// Update the high bound expression if we've found a new max value.
|
2015-07-15 06:32:50 +08:00
|
|
|
if (Min1 != End)
|
|
|
|
High = End;
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
|
|
|
|
Members.push_back(Index);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2015-07-15 06:32:44 +08:00
|
|
|
void RuntimePointerChecking::groupChecks(
|
|
|
|
MemoryDepChecker::DepCandidates &DepCands, bool UseDependencies) {
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
// We build the groups from dependency candidates equivalence classes
|
|
|
|
// because:
|
|
|
|
// - We know that pointers in the same equivalence class share
|
|
|
|
// the same underlying object and therefore there is a chance
|
|
|
|
// that we can compare pointers
|
|
|
|
// - We wouldn't be able to merge two pointers for which we need
|
|
|
|
// to emit a memcheck. The classes in DepCands are already
|
|
|
|
// conveniently built such that no two pointers in the same
|
|
|
|
// class need checking against each other.
|
|
|
|
|
|
|
|
// We use the following (greedy) algorithm to construct the groups
|
|
|
|
// For every pointer in the equivalence class:
|
|
|
|
// For each existing group:
|
|
|
|
// - if the difference between this pointer and the min/max bounds
|
|
|
|
// of the group is a constant, then make the pointer part of the
|
|
|
|
// group and update the min/max bounds of that group as required.
|
|
|
|
|
|
|
|
CheckingGroups.clear();
|
|
|
|
|
2015-07-28 21:44:08 +08:00
|
|
|
// If we need to check two pointers to the same underlying object
|
|
|
|
// with a non-constant difference, we shouldn't perform any pointer
|
|
|
|
// grouping with those pointers. This is because we can easily get
|
|
|
|
// into cases where the resulting check would return false, even when
|
|
|
|
// the accesses are safe.
|
|
|
|
//
|
|
|
|
// The following example shows this:
|
|
|
|
// for (i = 0; i < 1000; ++i)
|
|
|
|
// a[5000 + i * m] = a[i] + a[i + 9000]
|
|
|
|
//
|
|
|
|
// Here grouping gives a check of (5000, 5000 + 1000 * m) against
|
|
|
|
// (0, 10000) which is always false. However, if m is 1, there is no
|
|
|
|
// dependence. Not grouping the checks for a[i] and a[i + 9000] allows
|
|
|
|
// us to perform an accurate check in this case.
|
|
|
|
//
|
|
|
|
// The above case requires that we have an UnknownDependence between
|
|
|
|
// accesses to the same underlying object. This cannot happen unless
|
|
|
|
// ShouldRetryWithRuntimeCheck is set, and therefore UseDependencies
|
|
|
|
// is also false. In this case we will use the fallback path and create
|
|
|
|
// separate checking groups for all pointers.
|
2015-11-05 13:49:43 +08:00
|
|
|
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
// If we don't have the dependency partitions, construct a new
|
2015-07-28 21:44:08 +08:00
|
|
|
// checking pointer group for each pointer. This is also required
|
|
|
|
// for correctness, because in this case we can have checking between
|
|
|
|
// pointers to the same underlying object.
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
if (!UseDependencies) {
|
|
|
|
for (unsigned I = 0; I < Pointers.size(); ++I)
|
|
|
|
CheckingGroups.push_back(CheckingPtrGroup(I, *this));
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned TotalComparisons = 0;
|
|
|
|
|
|
|
|
DenseMap<Value *, unsigned> PositionMap;
|
2015-07-15 06:32:50 +08:00
|
|
|
for (unsigned Index = 0; Index < Pointers.size(); ++Index)
|
|
|
|
PositionMap[Pointers[Index].PointerValue] = Index;
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
|
2015-07-09 23:18:25 +08:00
|
|
|
// We need to keep track of what pointers we've already seen so we
|
|
|
|
// don't process them twice.
|
|
|
|
SmallSet<unsigned, 2> Seen;
|
|
|
|
|
2015-12-08 03:21:39 +08:00
|
|
|
// Go through all equivalence classes, get the "pointer check groups"
|
2015-07-09 23:18:25 +08:00
|
|
|
// and add them to the overall solution. We use the order in which accesses
|
|
|
|
// appear in 'Pointers' to enforce determinism.
|
|
|
|
for (unsigned I = 0; I < Pointers.size(); ++I) {
|
|
|
|
// We've seen this pointer before, and therefore already processed
|
|
|
|
// its equivalence class.
|
|
|
|
if (Seen.count(I))
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
continue;
|
|
|
|
|
2015-07-15 06:32:50 +08:00
|
|
|
MemoryDepChecker::MemAccessInfo Access(Pointers[I].PointerValue,
|
|
|
|
Pointers[I].IsWritePtr);
|
2015-07-09 23:18:25 +08:00
|
|
|
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
SmallVector<CheckingPtrGroup, 2> Groups;
|
2015-07-09 23:18:25 +08:00
|
|
|
auto LeaderI = DepCands.findValue(DepCands.getLeaderValue(Access));
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
|
2015-07-13 22:48:24 +08:00
|
|
|
// Because DepCands is constructed by visiting accesses in the order in
|
|
|
|
// which they appear in alias sets (which is deterministic) and the
|
|
|
|
// iteration order within an equivalence class member is only dependent on
|
|
|
|
// the order in which unions and insertions are performed on the
|
|
|
|
// equivalence class, the iteration order is deterministic.
|
2015-07-09 23:18:25 +08:00
|
|
|
for (auto MI = DepCands.member_begin(LeaderI), ME = DepCands.member_end();
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
MI != ME; ++MI) {
|
|
|
|
unsigned Pointer = PositionMap[MI->getPointer()];
|
|
|
|
bool Merged = false;
|
2015-07-09 23:18:25 +08:00
|
|
|
// Mark this pointer as seen.
|
|
|
|
Seen.insert(Pointer);
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
|
|
|
|
// Go through all the existing sets and see if we can find one
|
|
|
|
// which can include this pointer.
|
|
|
|
for (CheckingPtrGroup &Group : Groups) {
|
|
|
|
// Don't perform more than a certain amount of comparisons.
|
|
|
|
// This should limit the cost of grouping the pointers to something
|
|
|
|
// reasonable. If we do end up hitting this threshold, the algorithm
|
|
|
|
// will create separate groups for all remaining pointers.
|
|
|
|
if (TotalComparisons > MemoryCheckMergeThreshold)
|
|
|
|
break;
|
|
|
|
|
|
|
|
TotalComparisons++;
|
|
|
|
|
|
|
|
if (Group.addPointer(Pointer)) {
|
|
|
|
Merged = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!Merged)
|
|
|
|
// We couldn't add this pointer to any existing set or the threshold
|
|
|
|
// for the number of comparisons has been reached. Create a new group
|
|
|
|
// to hold the current pointer.
|
|
|
|
Groups.push_back(CheckingPtrGroup(Pointer, *this));
|
|
|
|
}
|
|
|
|
|
|
|
|
// We've computed the grouped checks for this partition.
|
|
|
|
// Save the results and continue with the next one.
|
|
|
|
std::copy(Groups.begin(), Groups.end(), std::back_inserter(CheckingGroups));
|
|
|
|
}
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-07-16 10:48:05 +08:00
|
|
|
bool RuntimePointerChecking::arePointersInSamePartition(
|
|
|
|
const SmallVectorImpl<int> &PtrToPartition, unsigned PtrIdx1,
|
|
|
|
unsigned PtrIdx2) {
|
|
|
|
return (PtrToPartition[PtrIdx1] != -1 &&
|
|
|
|
PtrToPartition[PtrIdx1] == PtrToPartition[PtrIdx2]);
|
|
|
|
}
|
|
|
|
|
2015-08-10 04:06:08 +08:00
|
|
|
bool RuntimePointerChecking::needsChecking(unsigned I, unsigned J) const {
|
2015-07-15 06:32:50 +08:00
|
|
|
const PointerInfo &PointerI = Pointers[I];
|
|
|
|
const PointerInfo &PointerJ = Pointers[J];
|
|
|
|
|
2015-02-18 11:43:58 +08:00
|
|
|
// No need to check if two readonly pointers intersect.
|
2015-07-15 06:32:50 +08:00
|
|
|
if (!PointerI.IsWritePtr && !PointerJ.IsWritePtr)
|
2015-02-18 11:43:58 +08:00
|
|
|
return false;
|
|
|
|
|
|
|
|
// Only need to check pointers between two different dependency sets.
|
2015-07-15 06:32:50 +08:00
|
|
|
if (PointerI.DependencySetId == PointerJ.DependencySetId)
|
2015-02-18 11:43:58 +08:00
|
|
|
return false;
|
|
|
|
|
|
|
|
// Only need to check pointers in the same alias set.
|
2015-07-15 06:32:50 +08:00
|
|
|
if (PointerI.AliasSetId != PointerJ.AliasSetId)
|
2015-02-18 11:43:58 +08:00
|
|
|
return false;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
[LAA] Split out a helper to print a collection of memchecks
This is effectively an NFC but we can no longer print the index of the
pointer group so instead I print its address. This still lets us
cross-check the section that list the checks against the section that
list the groups (see how I modified the test).
E.g. before we printed this:
Run-time memory checks:
Check 0:
Comparing group 0:
%arrayidxC = getelementptr inbounds i16, i16* %c, i64 %store_ind
%arrayidxC1 = getelementptr inbounds i16, i16* %c, i64 %store_ind_inc
Against group 1:
%arrayidxA = getelementptr i16, i16* %a, i64 %ind
%arrayidxA1 = getelementptr i16, i16* %a, i64 %add
...
Grouped accesses:
Group 0:
(Low: %c High: (78 + %c))
Member: {%c,+,4}<%for.body>
Member: {(2 + %c),+,4}<%for.body>
Now we print this (changes are underlined):
Run-time memory checks:
Check 0:
Comparing group (0x7f9c6040c320):
~~~~~~~~~~~~~~
%arrayidxC1 = getelementptr inbounds i16, i16* %c, i64 %store_ind_inc
%arrayidxC = getelementptr inbounds i16, i16* %c, i64 %store_ind
Against group (0x7f9c6040c358):
~~~~~~~~~~~~~~
%arrayidxA1 = getelementptr i16, i16* %a, i64 %add
%arrayidxA = getelementptr i16, i16* %a, i64 %ind
...
Grouped accesses:
Group 0x7f9c6040c320:
~~~~~~~~~~~~~~
(Low: %c High: (78 + %c))
Member: {(2 + %c),+,4}<%for.body>
Member: {%c,+,4}<%for.body>
llvm-svn: 243354
2015-07-28 07:54:41 +08:00
|
|
|
void RuntimePointerChecking::printChecks(
|
|
|
|
raw_ostream &OS, const SmallVectorImpl<PointerCheck> &Checks,
|
|
|
|
unsigned Depth) const {
|
|
|
|
unsigned N = 0;
|
|
|
|
for (const auto &Check : Checks) {
|
|
|
|
const auto &First = Check.first->Members, &Second = Check.second->Members;
|
|
|
|
|
|
|
|
OS.indent(Depth) << "Check " << N++ << ":\n";
|
|
|
|
|
|
|
|
OS.indent(Depth + 2) << "Comparing group (" << Check.first << "):\n";
|
|
|
|
for (unsigned K = 0; K < First.size(); ++K)
|
|
|
|
OS.indent(Depth + 2) << *Pointers[First[K]].PointerValue << "\n";
|
|
|
|
|
|
|
|
OS.indent(Depth + 2) << "Against group (" << Check.second << "):\n";
|
|
|
|
for (unsigned K = 0; K < Second.size(); ++K)
|
|
|
|
OS.indent(Depth + 2) << *Pointers[Second[K]].PointerValue << "\n";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-08-08 03:44:48 +08:00
|
|
|
void RuntimePointerChecking::print(raw_ostream &OS, unsigned Depth) const {
|
2015-02-20 03:15:19 +08:00
|
|
|
|
|
|
|
OS.indent(Depth) << "Run-time memory checks:\n";
|
2015-08-08 06:44:15 +08:00
|
|
|
printChecks(OS, Checks, Depth);
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
|
|
|
|
OS.indent(Depth) << "Grouped accesses:\n";
|
|
|
|
for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
|
[LAA] Split out a helper to print a collection of memchecks
This is effectively an NFC but we can no longer print the index of the
pointer group so instead I print its address. This still lets us
cross-check the section that list the checks against the section that
list the groups (see how I modified the test).
E.g. before we printed this:
Run-time memory checks:
Check 0:
Comparing group 0:
%arrayidxC = getelementptr inbounds i16, i16* %c, i64 %store_ind
%arrayidxC1 = getelementptr inbounds i16, i16* %c, i64 %store_ind_inc
Against group 1:
%arrayidxA = getelementptr i16, i16* %a, i64 %ind
%arrayidxA1 = getelementptr i16, i16* %a, i64 %add
...
Grouped accesses:
Group 0:
(Low: %c High: (78 + %c))
Member: {%c,+,4}<%for.body>
Member: {(2 + %c),+,4}<%for.body>
Now we print this (changes are underlined):
Run-time memory checks:
Check 0:
Comparing group (0x7f9c6040c320):
~~~~~~~~~~~~~~
%arrayidxC1 = getelementptr inbounds i16, i16* %c, i64 %store_ind_inc
%arrayidxC = getelementptr inbounds i16, i16* %c, i64 %store_ind
Against group (0x7f9c6040c358):
~~~~~~~~~~~~~~
%arrayidxA1 = getelementptr i16, i16* %a, i64 %add
%arrayidxA = getelementptr i16, i16* %a, i64 %ind
...
Grouped accesses:
Group 0x7f9c6040c320:
~~~~~~~~~~~~~~
(Low: %c High: (78 + %c))
Member: {(2 + %c),+,4}<%for.body>
Member: {%c,+,4}<%for.body>
llvm-svn: 243354
2015-07-28 07:54:41 +08:00
|
|
|
const auto &CG = CheckingGroups[I];
|
|
|
|
|
|
|
|
OS.indent(Depth + 2) << "Group " << &CG << ":\n";
|
|
|
|
OS.indent(Depth + 4) << "(Low: " << *CG.Low << " High: " << *CG.High
|
|
|
|
<< ")\n";
|
|
|
|
for (unsigned J = 0; J < CG.Members.size(); ++J) {
|
|
|
|
OS.indent(Depth + 6) << "Member: " << *Pointers[CG.Members[J]].Expr
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
<< "\n";
|
|
|
|
}
|
|
|
|
}
|
2015-02-20 03:15:19 +08:00
|
|
|
}
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
namespace {
|
|
|
|
/// \brief Analyses memory accesses in a loop.
|
|
|
|
///
|
|
|
|
/// Checks whether run time pointer checks are needed and builds sets for data
|
|
|
|
/// dependence checking.
|
|
|
|
class AccessAnalysis {
|
|
|
|
public:
|
|
|
|
/// \brief Read or write access location.
|
|
|
|
typedef PointerIntPair<Value *, 1, bool> MemAccessInfo;
|
|
|
|
typedef SmallPtrSet<MemAccessInfo, 8> MemAccessInfoSet;
|
|
|
|
|
2015-04-24 04:09:20 +08:00
|
|
|
AccessAnalysis(const DataLayout &Dl, AliasAnalysis *AA, LoopInfo *LI,
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
MemoryDepChecker::DepCandidates &DA,
|
|
|
|
PredicatedScalarEvolution &PSE)
|
[SCEV][LV] Add SCEV Predicates and use them to re-implement stride versioning
Summary:
SCEV Predicates represent conditions that typically cannot be derived from
static analysis, but can be used to reduce SCEV expressions to forms which are
usable for different optimizers.
ScalarEvolution now has the rewriteUsingPredicate method which can simplify a
SCEV expression using a SCEVPredicateSet. The normal workflow of a pass using
SCEVPredicates would be to hold a SCEVPredicateSet and every time assumptions
need to be made a new SCEV Predicate would be created and added to the set.
Each time after calling getSCEV, the user will call the rewriteUsingPredicate
method.
We add two types of predicates
SCEVPredicateSet - implements a set of predicates
SCEVEqualPredicate - tests for equality between two SCEV expressions
We use the SCEVEqualPredicate to re-implement stride versioning. Every time we
version a stride, we will add a SCEVEqualPredicate to the context.
Instead of adding specific stride checks, LoopVectorize now adds a more
generic SCEV check.
We only need to add support for this in the LoopVectorizer since this is the
only pass that will do stride versioning.
Reviewers: mzolotukhin, anemet, hfinkel, sanjoy
Subscribers: sanjoy, hfinkel, rengolin, jmolloy, llvm-commits
Differential Revision: http://reviews.llvm.org/D13595
llvm-svn: 251800
2015-11-02 22:41:02 +08:00
|
|
|
: DL(Dl), AST(*AA), LI(LI), DepCands(DA), IsRTCheckAnalysisNeeded(false),
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
PSE(PSE) {}
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
/// \brief Register a load and whether it is only read from.
|
2015-06-17 15:18:54 +08:00
|
|
|
void addLoad(MemoryLocation &Loc, bool IsReadOnly) {
|
2015-02-02 00:56:15 +08:00
|
|
|
Value *Ptr = const_cast<Value*>(Loc.Ptr);
|
2015-06-17 15:21:38 +08:00
|
|
|
AST.add(Ptr, MemoryLocation::UnknownSize, Loc.AATags);
|
2015-02-02 00:56:15 +08:00
|
|
|
Accesses.insert(MemAccessInfo(Ptr, false));
|
|
|
|
if (IsReadOnly)
|
|
|
|
ReadOnlyPtr.insert(Ptr);
|
|
|
|
}
|
|
|
|
|
|
|
|
/// \brief Register a store.
|
2015-06-17 15:18:54 +08:00
|
|
|
void addStore(MemoryLocation &Loc) {
|
2015-02-02 00:56:15 +08:00
|
|
|
Value *Ptr = const_cast<Value*>(Loc.Ptr);
|
2015-06-17 15:21:38 +08:00
|
|
|
AST.add(Ptr, MemoryLocation::UnknownSize, Loc.AATags);
|
2015-02-02 00:56:15 +08:00
|
|
|
Accesses.insert(MemAccessInfo(Ptr, true));
|
|
|
|
}
|
|
|
|
|
|
|
|
/// \brief Check whether we can check the pointers at runtime for
|
2015-07-10 06:17:38 +08:00
|
|
|
/// non-intersection.
|
|
|
|
///
|
|
|
|
/// Returns true if we need no check or if we do and we can generate them
|
|
|
|
/// (i.e. the pointers have computable bounds).
|
2015-07-15 06:32:44 +08:00
|
|
|
bool canCheckPtrAtRT(RuntimePointerChecking &RtCheck, ScalarEvolution *SE,
|
|
|
|
Loop *TheLoop, const ValueToValueMap &Strides,
|
2015-02-02 00:56:15 +08:00
|
|
|
bool ShouldCheckStride = false);
|
|
|
|
|
|
|
|
/// \brief Goes over all memory accesses, checks whether a RT check is needed
|
|
|
|
/// and builds sets of dependent accesses.
|
|
|
|
void buildDependenceSets() {
|
|
|
|
processMemAccesses();
|
|
|
|
}
|
|
|
|
|
2015-07-09 14:47:18 +08:00
|
|
|
/// \brief Initial processing of memory accesses determined that we need to
|
|
|
|
/// perform dependency checking.
|
|
|
|
///
|
|
|
|
/// Note that this can later be cleared if we retry memcheck analysis without
|
|
|
|
/// dependency checking (i.e. ShouldRetryWithRuntimeCheck).
|
2015-02-02 00:56:15 +08:00
|
|
|
bool isDependencyCheckNeeded() { return !CheckDeps.empty(); }
|
2015-05-18 23:37:03 +08:00
|
|
|
|
|
|
|
/// We decided that no dependence analysis would be used. Reset the state.
|
|
|
|
void resetDepChecks(MemoryDepChecker &DepChecker) {
|
|
|
|
CheckDeps.clear();
|
2015-11-04 05:39:52 +08:00
|
|
|
DepChecker.clearDependences();
|
2015-05-18 23:37:03 +08:00
|
|
|
}
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
MemAccessInfoSet &getDependenciesToCheck() { return CheckDeps; }
|
|
|
|
|
|
|
|
private:
|
|
|
|
typedef SetVector<MemAccessInfo> PtrAccessSet;
|
|
|
|
|
|
|
|
/// \brief Go over all memory access and check whether runtime pointer checks
|
2015-07-09 14:47:21 +08:00
|
|
|
/// are needed and build sets of dependency check candidates.
|
2015-02-02 00:56:15 +08:00
|
|
|
void processMemAccesses();
|
|
|
|
|
|
|
|
/// Set of all accesses.
|
|
|
|
PtrAccessSet Accesses;
|
|
|
|
|
2015-03-10 10:37:25 +08:00
|
|
|
const DataLayout &DL;
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
/// Set of accesses that need a further dependence check.
|
|
|
|
MemAccessInfoSet CheckDeps;
|
|
|
|
|
|
|
|
/// Set of pointers that are read only.
|
|
|
|
SmallPtrSet<Value*, 16> ReadOnlyPtr;
|
|
|
|
|
|
|
|
/// An alias set tracker to partition the access set by underlying object and
|
|
|
|
//intrinsic property (such as TBAA metadata).
|
|
|
|
AliasSetTracker AST;
|
|
|
|
|
2015-04-24 04:09:20 +08:00
|
|
|
LoopInfo *LI;
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
/// Sets of potentially dependent accesses - members of one set share an
|
|
|
|
/// underlying pointer. The set "CheckDeps" identfies which sets really need a
|
|
|
|
/// dependence check.
|
2015-03-11 01:40:34 +08:00
|
|
|
MemoryDepChecker::DepCandidates &DepCands;
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-07-09 14:47:18 +08:00
|
|
|
/// \brief Initial processing of memory accesses determined that we may need
|
|
|
|
/// to add memchecks. Perform the analysis to determine the necessary checks.
|
|
|
|
///
|
|
|
|
/// Note that, this is different from isDependencyCheckNeeded. When we retry
|
|
|
|
/// memcheck analysis without dependency checking
|
|
|
|
/// (i.e. ShouldRetryWithRuntimeCheck), isDependencyCheckNeeded is cleared
|
|
|
|
/// while this remains set if we have potentially dependent accesses.
|
|
|
|
bool IsRTCheckAnalysisNeeded;
|
[SCEV][LV] Add SCEV Predicates and use them to re-implement stride versioning
Summary:
SCEV Predicates represent conditions that typically cannot be derived from
static analysis, but can be used to reduce SCEV expressions to forms which are
usable for different optimizers.
ScalarEvolution now has the rewriteUsingPredicate method which can simplify a
SCEV expression using a SCEVPredicateSet. The normal workflow of a pass using
SCEVPredicates would be to hold a SCEVPredicateSet and every time assumptions
need to be made a new SCEV Predicate would be created and added to the set.
Each time after calling getSCEV, the user will call the rewriteUsingPredicate
method.
We add two types of predicates
SCEVPredicateSet - implements a set of predicates
SCEVEqualPredicate - tests for equality between two SCEV expressions
We use the SCEVEqualPredicate to re-implement stride versioning. Every time we
version a stride, we will add a SCEVEqualPredicate to the context.
Instead of adding specific stride checks, LoopVectorize now adds a more
generic SCEV check.
We only need to add support for this in the LoopVectorizer since this is the
only pass that will do stride versioning.
Reviewers: mzolotukhin, anemet, hfinkel, sanjoy
Subscribers: sanjoy, hfinkel, rengolin, jmolloy, llvm-commits
Differential Revision: http://reviews.llvm.org/D13595
llvm-svn: 251800
2015-11-02 22:41:02 +08:00
|
|
|
|
|
|
|
/// The SCEV predicate containing all the SCEV-related assumptions.
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
PredicatedScalarEvolution &PSE;
|
2015-02-02 00:56:15 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
} // end anonymous namespace
|
|
|
|
|
|
|
|
/// \brief Check whether a pointer can participate in a runtime bounds check.
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
static bool hasComputableBounds(PredicatedScalarEvolution &PSE,
|
[SCEV][LV] Add SCEV Predicates and use them to re-implement stride versioning
Summary:
SCEV Predicates represent conditions that typically cannot be derived from
static analysis, but can be used to reduce SCEV expressions to forms which are
usable for different optimizers.
ScalarEvolution now has the rewriteUsingPredicate method which can simplify a
SCEV expression using a SCEVPredicateSet. The normal workflow of a pass using
SCEVPredicates would be to hold a SCEVPredicateSet and every time assumptions
need to be made a new SCEV Predicate would be created and added to the set.
Each time after calling getSCEV, the user will call the rewriteUsingPredicate
method.
We add two types of predicates
SCEVPredicateSet - implements a set of predicates
SCEVEqualPredicate - tests for equality between two SCEV expressions
We use the SCEVEqualPredicate to re-implement stride versioning. Every time we
version a stride, we will add a SCEVEqualPredicate to the context.
Instead of adding specific stride checks, LoopVectorize now adds a more
generic SCEV check.
We only need to add support for this in the LoopVectorizer since this is the
only pass that will do stride versioning.
Reviewers: mzolotukhin, anemet, hfinkel, sanjoy
Subscribers: sanjoy, hfinkel, rengolin, jmolloy, llvm-commits
Differential Revision: http://reviews.llvm.org/D13595
llvm-svn: 251800
2015-11-02 22:41:02 +08:00
|
|
|
const ValueToValueMap &Strides, Value *Ptr,
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
Loop *L) {
|
|
|
|
const SCEV *PtrScev = replaceSymbolicStrideSCEV(PSE, Strides, Ptr);
|
2016-03-24 12:28:47 +08:00
|
|
|
|
|
|
|
// The bounds for loop-invariant pointer is trivial.
|
|
|
|
if (PSE.getSE()->isLoopInvariant(PtrScev, L))
|
|
|
|
return true;
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(PtrScev);
|
|
|
|
if (!AR)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
return AR->isAffine();
|
|
|
|
}
|
|
|
|
|
2015-07-15 06:32:44 +08:00
|
|
|
bool AccessAnalysis::canCheckPtrAtRT(RuntimePointerChecking &RtCheck,
|
|
|
|
ScalarEvolution *SE, Loop *TheLoop,
|
|
|
|
const ValueToValueMap &StridesMap,
|
|
|
|
bool ShouldCheckStride) {
|
2015-02-02 00:56:15 +08:00
|
|
|
// Find pointers with computable bounds. We are going to use this information
|
|
|
|
// to place a runtime bound check.
|
|
|
|
bool CanDoRT = true;
|
|
|
|
|
2015-07-10 06:17:38 +08:00
|
|
|
bool NeedRTCheck = false;
|
2015-07-09 14:47:18 +08:00
|
|
|
if (!IsRTCheckAnalysisNeeded) return true;
|
2015-06-08 18:27:06 +08:00
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
bool IsDepCheckNeeded = isDependencyCheckNeeded();
|
|
|
|
|
|
|
|
// We assign a consecutive id to access from different alias sets.
|
|
|
|
// Accesses between different groups doesn't need to be checked.
|
|
|
|
unsigned ASId = 1;
|
|
|
|
for (auto &AS : AST) {
|
2015-07-09 06:58:48 +08:00
|
|
|
int NumReadPtrChecks = 0;
|
|
|
|
int NumWritePtrChecks = 0;
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
// We assign consecutive id to access from different dependence sets.
|
|
|
|
// Accesses within the same set don't need a runtime check.
|
|
|
|
unsigned RunningDepId = 1;
|
|
|
|
DenseMap<Value *, unsigned> DepSetId;
|
|
|
|
|
|
|
|
for (auto A : AS) {
|
|
|
|
Value *Ptr = A.getValue();
|
|
|
|
bool IsWrite = Accesses.count(MemAccessInfo(Ptr, true));
|
|
|
|
MemAccessInfo Access(Ptr, IsWrite);
|
|
|
|
|
2015-07-09 06:58:48 +08:00
|
|
|
if (IsWrite)
|
|
|
|
++NumWritePtrChecks;
|
|
|
|
else
|
|
|
|
++NumReadPtrChecks;
|
|
|
|
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
if (hasComputableBounds(PSE, StridesMap, Ptr, TheLoop) &&
|
2015-03-10 10:37:25 +08:00
|
|
|
// When we run after a failing dependency check we have to make sure
|
|
|
|
// we don't have wrapping pointers.
|
2015-02-02 00:56:15 +08:00
|
|
|
(!ShouldCheckStride ||
|
2016-05-10 13:55:16 +08:00
|
|
|
getPtrStride(PSE, Ptr, TheLoop, StridesMap) == 1)) {
|
2015-02-02 00:56:15 +08:00
|
|
|
// The id of the dependence set.
|
|
|
|
unsigned DepId;
|
|
|
|
|
|
|
|
if (IsDepCheckNeeded) {
|
|
|
|
Value *Leader = DepCands.getLeaderValue(Access).getPointer();
|
|
|
|
unsigned &LeaderId = DepSetId[Leader];
|
|
|
|
if (!LeaderId)
|
|
|
|
LeaderId = RunningDepId++;
|
|
|
|
DepId = LeaderId;
|
|
|
|
} else
|
|
|
|
// Each access has its own dependence set.
|
|
|
|
DepId = RunningDepId++;
|
|
|
|
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
RtCheck.insert(TheLoop, Ptr, IsWrite, DepId, ASId, StridesMap, PSE);
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Found a runtime check ptr:" << *Ptr << '\n');
|
2015-02-02 00:56:15 +08:00
|
|
|
} else {
|
2015-05-18 23:36:52 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Can't find bounds for ptr:" << *Ptr << '\n');
|
2015-02-02 00:56:15 +08:00
|
|
|
CanDoRT = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-07-09 06:58:48 +08:00
|
|
|
// If we have at least two writes or one write and a read then we need to
|
|
|
|
// check them. But there is no need to checks if there is only one
|
|
|
|
// dependence set for this alias set.
|
|
|
|
//
|
|
|
|
// Note that this function computes CanDoRT and NeedRTCheck independently.
|
|
|
|
// For example CanDoRT=false, NeedRTCheck=false means that we have a pointer
|
|
|
|
// for which we couldn't find the bounds but we don't actually need to emit
|
|
|
|
// any checks so it does not matter.
|
|
|
|
if (!(IsDepCheckNeeded && CanDoRT && RunningDepId == 2))
|
|
|
|
NeedRTCheck |= (NumWritePtrChecks >= 2 || (NumReadPtrChecks >= 1 &&
|
|
|
|
NumWritePtrChecks >= 1));
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
++ASId;
|
|
|
|
}
|
|
|
|
|
|
|
|
// If the pointers that we would use for the bounds comparison have different
|
|
|
|
// address spaces, assume the values aren't directly comparable, so we can't
|
|
|
|
// use them for the runtime check. We also have to assume they could
|
|
|
|
// overlap. In the future there should be metadata for whether address spaces
|
|
|
|
// are disjoint.
|
|
|
|
unsigned NumPointers = RtCheck.Pointers.size();
|
|
|
|
for (unsigned i = 0; i < NumPointers; ++i) {
|
|
|
|
for (unsigned j = i + 1; j < NumPointers; ++j) {
|
|
|
|
// Only need to check pointers between two different dependency sets.
|
2015-07-15 06:32:50 +08:00
|
|
|
if (RtCheck.Pointers[i].DependencySetId ==
|
|
|
|
RtCheck.Pointers[j].DependencySetId)
|
2015-02-02 00:56:15 +08:00
|
|
|
continue;
|
|
|
|
// Only need to check pointers in the same alias set.
|
2015-07-15 06:32:50 +08:00
|
|
|
if (RtCheck.Pointers[i].AliasSetId != RtCheck.Pointers[j].AliasSetId)
|
2015-02-02 00:56:15 +08:00
|
|
|
continue;
|
|
|
|
|
2015-07-15 06:32:50 +08:00
|
|
|
Value *PtrI = RtCheck.Pointers[i].PointerValue;
|
|
|
|
Value *PtrJ = RtCheck.Pointers[j].PointerValue;
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
unsigned ASi = PtrI->getType()->getPointerAddressSpace();
|
|
|
|
unsigned ASj = PtrJ->getType()->getPointerAddressSpace();
|
|
|
|
if (ASi != ASj) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Runtime check would require comparison between"
|
2015-02-20 03:14:34 +08:00
|
|
|
" different address spaces\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
if (NeedRTCheck && CanDoRT)
|
2015-08-08 06:44:15 +08:00
|
|
|
RtCheck.generateChecks(DepCands, IsDepCheckNeeded);
|
[LAA] Merge memchecks for accesses separated by a constant offset
Summary:
Often filter-like loops will do memory accesses that are
separated by constant offsets. In these cases it is
common that we will exceed the threshold for the
allowable number of checks.
However, it should be possible to merge such checks,
sice a check of any interval againt two other intervals separated
by a constant offset (a,b), (a+c, b+c) will be equivalent with
a check againt (a, b+c), as long as (a,b) and (a+c, b+c) overlap.
Assuming the loop will be executed for a sufficient number of
iterations, this will be true. If not true, checking against
(a, b+c) is still safe (although not equivalent).
As long as there are no dependencies between two accesses,
we can merge their checks into a single one. We use this
technique to construct groups of accesses, and then check
the intervals associated with the groups instead of
checking the accesses directly.
Reviewers: anemet
Subscribers: llvm-commits
Differential Revision: http://reviews.llvm.org/D10386
llvm-svn: 241673
2015-07-08 17:16:33 +08:00
|
|
|
|
2015-08-08 06:44:21 +08:00
|
|
|
DEBUG(dbgs() << "LAA: We need to do " << RtCheck.getNumberOfChecks()
|
2015-07-10 06:17:38 +08:00
|
|
|
<< " pointer comparisons.\n");
|
|
|
|
|
|
|
|
RtCheck.Need = NeedRTCheck;
|
|
|
|
|
|
|
|
bool CanDoRTIfNeeded = !NeedRTCheck || CanDoRT;
|
|
|
|
if (!CanDoRTIfNeeded)
|
|
|
|
RtCheck.reset();
|
|
|
|
return CanDoRTIfNeeded;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void AccessAnalysis::processMemAccesses() {
|
|
|
|
// We process the set twice: first we process read-write pointers, last we
|
|
|
|
// process read-only pointers. This allows us to skip dependence tests for
|
|
|
|
// read-only pointers.
|
|
|
|
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Processing memory accesses...\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
DEBUG(dbgs() << " AST: "; AST.dump());
|
2015-03-11 01:40:37 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Accesses(" << Accesses.size() << "):\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
DEBUG({
|
|
|
|
for (auto A : Accesses)
|
|
|
|
dbgs() << "\t" << *A.getPointer() << " (" <<
|
|
|
|
(A.getInt() ? "write" : (ReadOnlyPtr.count(A.getPointer()) ?
|
|
|
|
"read-only" : "read")) << ")\n";
|
|
|
|
});
|
|
|
|
|
|
|
|
// The AliasSetTracker has nicely partitioned our pointers by metadata
|
|
|
|
// compatibility and potential for underlying-object overlap. As a result, we
|
|
|
|
// only need to check for potential pointer dependencies within each alias
|
|
|
|
// set.
|
|
|
|
for (auto &AS : AST) {
|
|
|
|
// Note that both the alias-set tracker and the alias sets themselves used
|
|
|
|
// linked lists internally and so the iteration order here is deterministic
|
|
|
|
// (matching the original instruction order within each set).
|
|
|
|
|
|
|
|
bool SetHasWrite = false;
|
|
|
|
|
|
|
|
// Map of pointers to last access encountered.
|
|
|
|
typedef DenseMap<Value*, MemAccessInfo> UnderlyingObjToAccessMap;
|
|
|
|
UnderlyingObjToAccessMap ObjToLastAccess;
|
|
|
|
|
|
|
|
// Set of access to check after all writes have been processed.
|
|
|
|
PtrAccessSet DeferredAccesses;
|
|
|
|
|
|
|
|
// Iterate over each alias set twice, once to process read/write pointers,
|
|
|
|
// and then to process read-only pointers.
|
|
|
|
for (int SetIteration = 0; SetIteration < 2; ++SetIteration) {
|
|
|
|
bool UseDeferred = SetIteration > 0;
|
|
|
|
PtrAccessSet &S = UseDeferred ? DeferredAccesses : Accesses;
|
|
|
|
|
|
|
|
for (auto AV : AS) {
|
|
|
|
Value *Ptr = AV.getValue();
|
|
|
|
|
|
|
|
// For a single memory access in AliasSetTracker, Accesses may contain
|
|
|
|
// both read and write, and they both need to be handled for CheckDeps.
|
|
|
|
for (auto AC : S) {
|
|
|
|
if (AC.getPointer() != Ptr)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
bool IsWrite = AC.getInt();
|
|
|
|
|
|
|
|
// If we're using the deferred access set, then it contains only
|
|
|
|
// reads.
|
|
|
|
bool IsReadOnlyPtr = ReadOnlyPtr.count(Ptr) && !IsWrite;
|
|
|
|
if (UseDeferred && !IsReadOnlyPtr)
|
|
|
|
continue;
|
|
|
|
// Otherwise, the pointer must be in the PtrAccessSet, either as a
|
|
|
|
// read or a write.
|
|
|
|
assert(((IsReadOnlyPtr && UseDeferred) || IsWrite ||
|
|
|
|
S.count(MemAccessInfo(Ptr, false))) &&
|
|
|
|
"Alias-set pointer not in the access set?");
|
|
|
|
|
|
|
|
MemAccessInfo Access(Ptr, IsWrite);
|
|
|
|
DepCands.insert(Access);
|
|
|
|
|
|
|
|
// Memorize read-only pointers for later processing and skip them in
|
|
|
|
// the first round (they need to be checked after we have seen all
|
|
|
|
// write pointers). Note: we also mark pointer that are not
|
|
|
|
// consecutive as "read-only" pointers (so that we check
|
|
|
|
// "a[b[i]] +="). Hence, we need the second check for "!IsWrite".
|
|
|
|
if (!UseDeferred && IsReadOnlyPtr) {
|
|
|
|
DeferredAccesses.insert(Access);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
// If this is a write - check other reads and writes for conflicts. If
|
|
|
|
// this is a read only check other writes for conflicts (but only if
|
|
|
|
// there is no other write to the ptr - this is an optimization to
|
|
|
|
// catch "a[i] = a[i] + " without having to do a dependence check).
|
|
|
|
if ((IsWrite || IsReadOnlyPtr) && SetHasWrite) {
|
|
|
|
CheckDeps.insert(Access);
|
2015-07-09 14:47:18 +08:00
|
|
|
IsRTCheckAnalysisNeeded = true;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
if (IsWrite)
|
|
|
|
SetHasWrite = true;
|
|
|
|
|
|
|
|
// Create sets of pointers connected by a shared alias set and
|
|
|
|
// underlying object.
|
|
|
|
typedef SmallVector<Value *, 16> ValueVector;
|
|
|
|
ValueVector TempObjects;
|
2015-04-24 04:09:20 +08:00
|
|
|
|
|
|
|
GetUnderlyingObjects(Ptr, TempObjects, DL, LI);
|
|
|
|
DEBUG(dbgs() << "Underlying objects for pointer " << *Ptr << "\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
for (Value *UnderlyingObj : TempObjects) {
|
2015-11-05 13:49:43 +08:00
|
|
|
// nullptr never alias, don't join sets for pointer that have "null"
|
|
|
|
// in their UnderlyingObjects list.
|
|
|
|
if (isa<ConstantPointerNull>(UnderlyingObj))
|
|
|
|
continue;
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
UnderlyingObjToAccessMap::iterator Prev =
|
|
|
|
ObjToLastAccess.find(UnderlyingObj);
|
|
|
|
if (Prev != ObjToLastAccess.end())
|
|
|
|
DepCands.unionSets(Access, Prev->second);
|
|
|
|
|
|
|
|
ObjToLastAccess[UnderlyingObj] = Access;
|
2015-04-24 04:09:20 +08:00
|
|
|
DEBUG(dbgs() << " " << *UnderlyingObj << "\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool isInBoundsGep(Value *Ptr) {
|
|
|
|
if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr))
|
|
|
|
return GEP->isInBounds();
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2015-06-27 01:25:43 +08:00
|
|
|
/// \brief Return true if an AddRec pointer \p Ptr is unsigned non-wrapping,
|
|
|
|
/// i.e. monotonically increasing/decreasing.
|
|
|
|
static bool isNoWrapAddRec(Value *Ptr, const SCEVAddRecExpr *AR,
|
[SCEV][LAA] Re-commit r260085 and r260086, this time with a fix for the memory
sanitizer issue. The PredicatedScalarEvolution's copy constructor
wasn't copying the Generation value, and was leaving it un-initialized.
Original commit message:
[SCEV][LAA] Add no wrap SCEV predicates and use use them to improve strided pointer detection
Summary:
This change adds no wrap SCEV predicates with:
- support for runtime checking
- support for expression rewriting:
(sext ({x,+,y}) -> {sext(x),+,sext(y)}
(zext ({x,+,y}) -> {zext(x),+,sext(y)}
Note that we are sign extending the increment of the SCEV, even for
the zext case. This is needed to cover the fairly common case where y would
be a (small) negative integer. In order to do this, this change adds two new
flags: nusw and nssw that are applicable to AddRecExprs and permit the
transformations above.
We also change isStridedPtr in LAA to be able to make use of
these predicates. With this feature we should now always be able to
work around overflow issues in the dependence analysis.
Reviewers: mzolotukhin, sanjoy, anemet
Subscribers: mzolotukhin, sanjoy, llvm-commits, rengolin, jmolloy, hfinkel
Differential Revision: http://reviews.llvm.org/D15412
llvm-svn: 260112
2016-02-09 01:02:45 +08:00
|
|
|
PredicatedScalarEvolution &PSE, const Loop *L) {
|
2015-06-27 01:25:43 +08:00
|
|
|
// FIXME: This should probably only return true for NUW.
|
|
|
|
if (AR->getNoWrapFlags(SCEV::NoWrapMask))
|
|
|
|
return true;
|
|
|
|
|
|
|
|
// Scalar evolution does not propagate the non-wrapping flags to values that
|
|
|
|
// are derived from a non-wrapping induction variable because non-wrapping
|
|
|
|
// could be flow-sensitive.
|
|
|
|
//
|
|
|
|
// Look through the potentially overflowing instruction to try to prove
|
|
|
|
// non-wrapping for the *specific* value of Ptr.
|
|
|
|
|
|
|
|
// The arithmetic implied by an inbounds GEP can't overflow.
|
|
|
|
auto *GEP = dyn_cast<GetElementPtrInst>(Ptr);
|
|
|
|
if (!GEP || !GEP->isInBounds())
|
|
|
|
return false;
|
|
|
|
|
|
|
|
// Make sure there is only one non-const index and analyze that.
|
|
|
|
Value *NonConstIndex = nullptr;
|
|
|
|
for (auto Index = GEP->idx_begin(); Index != GEP->idx_end(); ++Index)
|
|
|
|
if (!isa<ConstantInt>(*Index)) {
|
|
|
|
if (NonConstIndex)
|
|
|
|
return false;
|
|
|
|
NonConstIndex = *Index;
|
|
|
|
}
|
|
|
|
if (!NonConstIndex)
|
|
|
|
// The recurrence is on the pointer, ignore for now.
|
|
|
|
return false;
|
|
|
|
|
|
|
|
// The index in GEP is signed. It is non-wrapping if it's derived from a NSW
|
|
|
|
// AddRec using a NSW operation.
|
|
|
|
if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(NonConstIndex))
|
|
|
|
if (OBO->hasNoSignedWrap() &&
|
|
|
|
// Assume constant for other the operand so that the AddRec can be
|
|
|
|
// easily found.
|
|
|
|
isa<ConstantInt>(OBO->getOperand(1))) {
|
[SCEV][LAA] Re-commit r260085 and r260086, this time with a fix for the memory
sanitizer issue. The PredicatedScalarEvolution's copy constructor
wasn't copying the Generation value, and was leaving it un-initialized.
Original commit message:
[SCEV][LAA] Add no wrap SCEV predicates and use use them to improve strided pointer detection
Summary:
This change adds no wrap SCEV predicates with:
- support for runtime checking
- support for expression rewriting:
(sext ({x,+,y}) -> {sext(x),+,sext(y)}
(zext ({x,+,y}) -> {zext(x),+,sext(y)}
Note that we are sign extending the increment of the SCEV, even for
the zext case. This is needed to cover the fairly common case where y would
be a (small) negative integer. In order to do this, this change adds two new
flags: nusw and nssw that are applicable to AddRecExprs and permit the
transformations above.
We also change isStridedPtr in LAA to be able to make use of
these predicates. With this feature we should now always be able to
work around overflow issues in the dependence analysis.
Reviewers: mzolotukhin, sanjoy, anemet
Subscribers: mzolotukhin, sanjoy, llvm-commits, rengolin, jmolloy, hfinkel
Differential Revision: http://reviews.llvm.org/D15412
llvm-svn: 260112
2016-02-09 01:02:45 +08:00
|
|
|
auto *OpScev = PSE.getSCEV(OBO->getOperand(0));
|
2015-06-27 01:25:43 +08:00
|
|
|
|
|
|
|
if (auto *OpAR = dyn_cast<SCEVAddRecExpr>(OpScev))
|
|
|
|
return OpAR->getLoop() == L && OpAR->getNoWrapFlags(SCEV::FlagNSW);
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
/// \brief Check whether the access through \p Ptr has a constant stride.
|
2016-05-10 13:55:16 +08:00
|
|
|
int llvm::getPtrStride(PredicatedScalarEvolution &PSE, Value *Ptr,
|
[SCEV][LAA] Re-commit r260085 and r260086, this time with a fix for the memory
sanitizer issue. The PredicatedScalarEvolution's copy constructor
wasn't copying the Generation value, and was leaving it un-initialized.
Original commit message:
[SCEV][LAA] Add no wrap SCEV predicates and use use them to improve strided pointer detection
Summary:
This change adds no wrap SCEV predicates with:
- support for runtime checking
- support for expression rewriting:
(sext ({x,+,y}) -> {sext(x),+,sext(y)}
(zext ({x,+,y}) -> {zext(x),+,sext(y)}
Note that we are sign extending the increment of the SCEV, even for
the zext case. This is needed to cover the fairly common case where y would
be a (small) negative integer. In order to do this, this change adds two new
flags: nusw and nssw that are applicable to AddRecExprs and permit the
transformations above.
We also change isStridedPtr in LAA to be able to make use of
these predicates. With this feature we should now always be able to
work around overflow issues in the dependence analysis.
Reviewers: mzolotukhin, sanjoy, anemet
Subscribers: mzolotukhin, sanjoy, llvm-commits, rengolin, jmolloy, hfinkel
Differential Revision: http://reviews.llvm.org/D15412
llvm-svn: 260112
2016-02-09 01:02:45 +08:00
|
|
|
const Loop *Lp, const ValueToValueMap &StridesMap,
|
|
|
|
bool Assume) {
|
2015-08-02 06:20:21 +08:00
|
|
|
Type *Ty = Ptr->getType();
|
2015-02-02 00:56:15 +08:00
|
|
|
assert(Ty->isPointerTy() && "Unexpected non-ptr");
|
|
|
|
|
|
|
|
// Make sure that the pointer does not point to aggregate types.
|
2015-08-02 06:20:21 +08:00
|
|
|
auto *PtrTy = cast<PointerType>(Ty);
|
2015-02-02 00:56:15 +08:00
|
|
|
if (PtrTy->getElementType()->isAggregateType()) {
|
[SCEV][LAA] Re-commit r260085 and r260086, this time with a fix for the memory
sanitizer issue. The PredicatedScalarEvolution's copy constructor
wasn't copying the Generation value, and was leaving it un-initialized.
Original commit message:
[SCEV][LAA] Add no wrap SCEV predicates and use use them to improve strided pointer detection
Summary:
This change adds no wrap SCEV predicates with:
- support for runtime checking
- support for expression rewriting:
(sext ({x,+,y}) -> {sext(x),+,sext(y)}
(zext ({x,+,y}) -> {zext(x),+,sext(y)}
Note that we are sign extending the increment of the SCEV, even for
the zext case. This is needed to cover the fairly common case where y would
be a (small) negative integer. In order to do this, this change adds two new
flags: nusw and nssw that are applicable to AddRecExprs and permit the
transformations above.
We also change isStridedPtr in LAA to be able to make use of
these predicates. With this feature we should now always be able to
work around overflow issues in the dependence analysis.
Reviewers: mzolotukhin, sanjoy, anemet
Subscribers: mzolotukhin, sanjoy, llvm-commits, rengolin, jmolloy, hfinkel
Differential Revision: http://reviews.llvm.org/D15412
llvm-svn: 260112
2016-02-09 01:02:45 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Bad stride - Not a pointer to a scalar type" << *Ptr
|
|
|
|
<< "\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
const SCEV *PtrScev = replaceSymbolicStrideSCEV(PSE, StridesMap, Ptr);
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(PtrScev);
|
[SCEV][LAA] Re-commit r260085 and r260086, this time with a fix for the memory
sanitizer issue. The PredicatedScalarEvolution's copy constructor
wasn't copying the Generation value, and was leaving it un-initialized.
Original commit message:
[SCEV][LAA] Add no wrap SCEV predicates and use use them to improve strided pointer detection
Summary:
This change adds no wrap SCEV predicates with:
- support for runtime checking
- support for expression rewriting:
(sext ({x,+,y}) -> {sext(x),+,sext(y)}
(zext ({x,+,y}) -> {zext(x),+,sext(y)}
Note that we are sign extending the increment of the SCEV, even for
the zext case. This is needed to cover the fairly common case where y would
be a (small) negative integer. In order to do this, this change adds two new
flags: nusw and nssw that are applicable to AddRecExprs and permit the
transformations above.
We also change isStridedPtr in LAA to be able to make use of
these predicates. With this feature we should now always be able to
work around overflow issues in the dependence analysis.
Reviewers: mzolotukhin, sanjoy, anemet
Subscribers: mzolotukhin, sanjoy, llvm-commits, rengolin, jmolloy, hfinkel
Differential Revision: http://reviews.llvm.org/D15412
llvm-svn: 260112
2016-02-09 01:02:45 +08:00
|
|
|
if (Assume && !AR)
|
2016-03-23 23:29:30 +08:00
|
|
|
AR = PSE.getAsAddRec(Ptr);
|
[SCEV][LAA] Re-commit r260085 and r260086, this time with a fix for the memory
sanitizer issue. The PredicatedScalarEvolution's copy constructor
wasn't copying the Generation value, and was leaving it un-initialized.
Original commit message:
[SCEV][LAA] Add no wrap SCEV predicates and use use them to improve strided pointer detection
Summary:
This change adds no wrap SCEV predicates with:
- support for runtime checking
- support for expression rewriting:
(sext ({x,+,y}) -> {sext(x),+,sext(y)}
(zext ({x,+,y}) -> {zext(x),+,sext(y)}
Note that we are sign extending the increment of the SCEV, even for
the zext case. This is needed to cover the fairly common case where y would
be a (small) negative integer. In order to do this, this change adds two new
flags: nusw and nssw that are applicable to AddRecExprs and permit the
transformations above.
We also change isStridedPtr in LAA to be able to make use of
these predicates. With this feature we should now always be able to
work around overflow issues in the dependence analysis.
Reviewers: mzolotukhin, sanjoy, anemet
Subscribers: mzolotukhin, sanjoy, llvm-commits, rengolin, jmolloy, hfinkel
Differential Revision: http://reviews.llvm.org/D15412
llvm-svn: 260112
2016-02-09 01:02:45 +08:00
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
if (!AR) {
|
[SCEV][LAA] Re-commit r260085 and r260086, this time with a fix for the memory
sanitizer issue. The PredicatedScalarEvolution's copy constructor
wasn't copying the Generation value, and was leaving it un-initialized.
Original commit message:
[SCEV][LAA] Add no wrap SCEV predicates and use use them to improve strided pointer detection
Summary:
This change adds no wrap SCEV predicates with:
- support for runtime checking
- support for expression rewriting:
(sext ({x,+,y}) -> {sext(x),+,sext(y)}
(zext ({x,+,y}) -> {zext(x),+,sext(y)}
Note that we are sign extending the increment of the SCEV, even for
the zext case. This is needed to cover the fairly common case where y would
be a (small) negative integer. In order to do this, this change adds two new
flags: nusw and nssw that are applicable to AddRecExprs and permit the
transformations above.
We also change isStridedPtr in LAA to be able to make use of
these predicates. With this feature we should now always be able to
work around overflow issues in the dependence analysis.
Reviewers: mzolotukhin, sanjoy, anemet
Subscribers: mzolotukhin, sanjoy, llvm-commits, rengolin, jmolloy, hfinkel
Differential Revision: http://reviews.llvm.org/D15412
llvm-svn: 260112
2016-02-09 01:02:45 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Bad stride - Not an AddRecExpr pointer " << *Ptr
|
|
|
|
<< " SCEV: " << *PtrScev << "\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
// The accesss function must stride over the innermost loop.
|
|
|
|
if (Lp != AR->getLoop()) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Bad stride - Not striding over innermost loop " <<
|
[SCEV][LAA] Re-commit r260085 and r260086, this time with a fix for the memory
sanitizer issue. The PredicatedScalarEvolution's copy constructor
wasn't copying the Generation value, and was leaving it un-initialized.
Original commit message:
[SCEV][LAA] Add no wrap SCEV predicates and use use them to improve strided pointer detection
Summary:
This change adds no wrap SCEV predicates with:
- support for runtime checking
- support for expression rewriting:
(sext ({x,+,y}) -> {sext(x),+,sext(y)}
(zext ({x,+,y}) -> {zext(x),+,sext(y)}
Note that we are sign extending the increment of the SCEV, even for
the zext case. This is needed to cover the fairly common case where y would
be a (small) negative integer. In order to do this, this change adds two new
flags: nusw and nssw that are applicable to AddRecExprs and permit the
transformations above.
We also change isStridedPtr in LAA to be able to make use of
these predicates. With this feature we should now always be able to
work around overflow issues in the dependence analysis.
Reviewers: mzolotukhin, sanjoy, anemet
Subscribers: mzolotukhin, sanjoy, llvm-commits, rengolin, jmolloy, hfinkel
Differential Revision: http://reviews.llvm.org/D15412
llvm-svn: 260112
2016-02-09 01:02:45 +08:00
|
|
|
*Ptr << " SCEV: " << *AR << "\n");
|
2016-01-08 09:55:13 +08:00
|
|
|
return 0;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// The address calculation must not wrap. Otherwise, a dependence could be
|
|
|
|
// inverted.
|
|
|
|
// An inbounds getelementptr that is a AddRec with a unit stride
|
|
|
|
// cannot wrap per definition. The unit stride requirement is checked later.
|
|
|
|
// An getelementptr without an inbounds attribute and unit stride would have
|
|
|
|
// to access the pointer value "0" which is undefined behavior in address
|
|
|
|
// space 0, therefore we can also vectorize this case.
|
|
|
|
bool IsInBoundsGEP = isInBoundsGep(Ptr);
|
[SCEV][LAA] Re-commit r260085 and r260086, this time with a fix for the memory
sanitizer issue. The PredicatedScalarEvolution's copy constructor
wasn't copying the Generation value, and was leaving it un-initialized.
Original commit message:
[SCEV][LAA] Add no wrap SCEV predicates and use use them to improve strided pointer detection
Summary:
This change adds no wrap SCEV predicates with:
- support for runtime checking
- support for expression rewriting:
(sext ({x,+,y}) -> {sext(x),+,sext(y)}
(zext ({x,+,y}) -> {zext(x),+,sext(y)}
Note that we are sign extending the increment of the SCEV, even for
the zext case. This is needed to cover the fairly common case where y would
be a (small) negative integer. In order to do this, this change adds two new
flags: nusw and nssw that are applicable to AddRecExprs and permit the
transformations above.
We also change isStridedPtr in LAA to be able to make use of
these predicates. With this feature we should now always be able to
work around overflow issues in the dependence analysis.
Reviewers: mzolotukhin, sanjoy, anemet
Subscribers: mzolotukhin, sanjoy, llvm-commits, rengolin, jmolloy, hfinkel
Differential Revision: http://reviews.llvm.org/D15412
llvm-svn: 260112
2016-02-09 01:02:45 +08:00
|
|
|
bool IsNoWrapAddRec =
|
|
|
|
PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW) ||
|
|
|
|
isNoWrapAddRec(Ptr, AR, PSE, Lp);
|
2015-02-02 00:56:15 +08:00
|
|
|
bool IsInAddressSpaceZero = PtrTy->getAddressSpace() == 0;
|
|
|
|
if (!IsNoWrapAddRec && !IsInBoundsGEP && !IsInAddressSpaceZero) {
|
[SCEV][LAA] Re-commit r260085 and r260086, this time with a fix for the memory
sanitizer issue. The PredicatedScalarEvolution's copy constructor
wasn't copying the Generation value, and was leaving it un-initialized.
Original commit message:
[SCEV][LAA] Add no wrap SCEV predicates and use use them to improve strided pointer detection
Summary:
This change adds no wrap SCEV predicates with:
- support for runtime checking
- support for expression rewriting:
(sext ({x,+,y}) -> {sext(x),+,sext(y)}
(zext ({x,+,y}) -> {zext(x),+,sext(y)}
Note that we are sign extending the increment of the SCEV, even for
the zext case. This is needed to cover the fairly common case where y would
be a (small) negative integer. In order to do this, this change adds two new
flags: nusw and nssw that are applicable to AddRecExprs and permit the
transformations above.
We also change isStridedPtr in LAA to be able to make use of
these predicates. With this feature we should now always be able to
work around overflow issues in the dependence analysis.
Reviewers: mzolotukhin, sanjoy, anemet
Subscribers: mzolotukhin, sanjoy, llvm-commits, rengolin, jmolloy, hfinkel
Differential Revision: http://reviews.llvm.org/D15412
llvm-svn: 260112
2016-02-09 01:02:45 +08:00
|
|
|
if (Assume) {
|
|
|
|
PSE.setNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW);
|
|
|
|
IsNoWrapAddRec = true;
|
|
|
|
DEBUG(dbgs() << "LAA: Pointer may wrap in the address space:\n"
|
|
|
|
<< "LAA: Pointer: " << *Ptr << "\n"
|
|
|
|
<< "LAA: SCEV: " << *AR << "\n"
|
|
|
|
<< "LAA: Added an overflow assumption\n");
|
|
|
|
} else {
|
|
|
|
DEBUG(dbgs() << "LAA: Bad stride - Pointer may wrap in the address space "
|
|
|
|
<< *Ptr << " SCEV: " << *AR << "\n");
|
|
|
|
return 0;
|
|
|
|
}
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Check the step is constant.
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
const SCEV *Step = AR->getStepRecurrence(*PSE.getSE());
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-07-09 08:03:22 +08:00
|
|
|
// Calculate the pointer stride and check if it is constant.
|
2015-02-02 00:56:15 +08:00
|
|
|
const SCEVConstant *C = dyn_cast<SCEVConstant>(Step);
|
|
|
|
if (!C) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Bad stride - Not a constant strided " << *Ptr <<
|
[SCEV][LAA] Re-commit r260085 and r260086, this time with a fix for the memory
sanitizer issue. The PredicatedScalarEvolution's copy constructor
wasn't copying the Generation value, and was leaving it un-initialized.
Original commit message:
[SCEV][LAA] Add no wrap SCEV predicates and use use them to improve strided pointer detection
Summary:
This change adds no wrap SCEV predicates with:
- support for runtime checking
- support for expression rewriting:
(sext ({x,+,y}) -> {sext(x),+,sext(y)}
(zext ({x,+,y}) -> {zext(x),+,sext(y)}
Note that we are sign extending the increment of the SCEV, even for
the zext case. This is needed to cover the fairly common case where y would
be a (small) negative integer. In order to do this, this change adds two new
flags: nusw and nssw that are applicable to AddRecExprs and permit the
transformations above.
We also change isStridedPtr in LAA to be able to make use of
these predicates. With this feature we should now always be able to
work around overflow issues in the dependence analysis.
Reviewers: mzolotukhin, sanjoy, anemet
Subscribers: mzolotukhin, sanjoy, llvm-commits, rengolin, jmolloy, hfinkel
Differential Revision: http://reviews.llvm.org/D15412
llvm-svn: 260112
2016-02-09 01:02:45 +08:00
|
|
|
" SCEV: " << *AR << "\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-03-10 10:37:25 +08:00
|
|
|
auto &DL = Lp->getHeader()->getModule()->getDataLayout();
|
|
|
|
int64_t Size = DL.getTypeAllocSize(PtrTy->getElementType());
|
2015-12-18 04:28:46 +08:00
|
|
|
const APInt &APStepVal = C->getAPInt();
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
// Huge step value - give up.
|
|
|
|
if (APStepVal.getBitWidth() > 64)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
int64_t StepVal = APStepVal.getSExtValue();
|
|
|
|
|
|
|
|
// Strided access.
|
|
|
|
int64_t Stride = StepVal / Size;
|
|
|
|
int64_t Rem = StepVal % Size;
|
|
|
|
if (Rem)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
// If the SCEV could wrap but we have an inbounds gep with a unit stride we
|
|
|
|
// know we can't "wrap around the address space". In case of address space
|
|
|
|
// zero we know that this won't happen without triggering undefined behavior.
|
|
|
|
if (!IsNoWrapAddRec && (IsInBoundsGEP || IsInAddressSpaceZero) &&
|
[SCEV][LAA] Re-commit r260085 and r260086, this time with a fix for the memory
sanitizer issue. The PredicatedScalarEvolution's copy constructor
wasn't copying the Generation value, and was leaving it un-initialized.
Original commit message:
[SCEV][LAA] Add no wrap SCEV predicates and use use them to improve strided pointer detection
Summary:
This change adds no wrap SCEV predicates with:
- support for runtime checking
- support for expression rewriting:
(sext ({x,+,y}) -> {sext(x),+,sext(y)}
(zext ({x,+,y}) -> {zext(x),+,sext(y)}
Note that we are sign extending the increment of the SCEV, even for
the zext case. This is needed to cover the fairly common case where y would
be a (small) negative integer. In order to do this, this change adds two new
flags: nusw and nssw that are applicable to AddRecExprs and permit the
transformations above.
We also change isStridedPtr in LAA to be able to make use of
these predicates. With this feature we should now always be able to
work around overflow issues in the dependence analysis.
Reviewers: mzolotukhin, sanjoy, anemet
Subscribers: mzolotukhin, sanjoy, llvm-commits, rengolin, jmolloy, hfinkel
Differential Revision: http://reviews.llvm.org/D15412
llvm-svn: 260112
2016-02-09 01:02:45 +08:00
|
|
|
Stride != 1 && Stride != -1) {
|
|
|
|
if (Assume) {
|
|
|
|
// We can avoid this case by adding a run-time check.
|
|
|
|
DEBUG(dbgs() << "LAA: Non unit strided pointer which is not either "
|
|
|
|
<< "inbouds or in address space 0 may wrap:\n"
|
|
|
|
<< "LAA: Pointer: " << *Ptr << "\n"
|
|
|
|
<< "LAA: SCEV: " << *AR << "\n"
|
|
|
|
<< "LAA: Added an overflow assumption\n");
|
|
|
|
PSE.setNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW);
|
|
|
|
} else
|
|
|
|
return 0;
|
|
|
|
}
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
return Stride;
|
2016-01-26 10:27:47 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Take the pointer operand from the Load/Store instruction.
|
|
|
|
/// Returns NULL if this is not a valid Load/Store instruction.
|
|
|
|
static Value *getPointerOperand(Value *I) {
|
|
|
|
if (LoadInst *LI = dyn_cast<LoadInst>(I))
|
|
|
|
return LI->getPointerOperand();
|
|
|
|
if (StoreInst *SI = dyn_cast<StoreInst>(I))
|
|
|
|
return SI->getPointerOperand();
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Take the address space operand from the Load/Store instruction.
|
|
|
|
/// Returns -1 if this is not a valid Load/Store instruction.
|
|
|
|
static unsigned getAddressSpaceOperand(Value *I) {
|
|
|
|
if (LoadInst *L = dyn_cast<LoadInst>(I))
|
|
|
|
return L->getPointerAddressSpace();
|
|
|
|
if (StoreInst *S = dyn_cast<StoreInst>(I))
|
|
|
|
return S->getPointerAddressSpace();
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Returns true if the memory operations \p A and \p B are consecutive.
|
|
|
|
bool llvm::isConsecutiveAccess(Value *A, Value *B, const DataLayout &DL,
|
|
|
|
ScalarEvolution &SE, bool CheckType) {
|
|
|
|
Value *PtrA = getPointerOperand(A);
|
|
|
|
Value *PtrB = getPointerOperand(B);
|
|
|
|
unsigned ASA = getAddressSpaceOperand(A);
|
|
|
|
unsigned ASB = getAddressSpaceOperand(B);
|
|
|
|
|
|
|
|
// Check that the address spaces match and that the pointers are valid.
|
|
|
|
if (!PtrA || !PtrB || (ASA != ASB))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
// Make sure that A and B are different pointers.
|
|
|
|
if (PtrA == PtrB)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
// Make sure that A and B have the same type if required.
|
|
|
|
if(CheckType && PtrA->getType() != PtrB->getType())
|
|
|
|
return false;
|
|
|
|
|
|
|
|
unsigned PtrBitWidth = DL.getPointerSizeInBits(ASA);
|
|
|
|
Type *Ty = cast<PointerType>(PtrA->getType())->getElementType();
|
|
|
|
APInt Size(PtrBitWidth, DL.getTypeStoreSize(Ty));
|
|
|
|
|
|
|
|
APInt OffsetA(PtrBitWidth, 0), OffsetB(PtrBitWidth, 0);
|
|
|
|
PtrA = PtrA->stripAndAccumulateInBoundsConstantOffsets(DL, OffsetA);
|
|
|
|
PtrB = PtrB->stripAndAccumulateInBoundsConstantOffsets(DL, OffsetB);
|
|
|
|
|
|
|
|
// OffsetDelta = OffsetB - OffsetA;
|
|
|
|
const SCEV *OffsetSCEVA = SE.getConstant(OffsetA);
|
|
|
|
const SCEV *OffsetSCEVB = SE.getConstant(OffsetB);
|
|
|
|
const SCEV *OffsetDeltaSCEV = SE.getMinusSCEV(OffsetSCEVB, OffsetSCEVA);
|
|
|
|
const SCEVConstant *OffsetDeltaC = dyn_cast<SCEVConstant>(OffsetDeltaSCEV);
|
|
|
|
const APInt &OffsetDelta = OffsetDeltaC->getAPInt();
|
|
|
|
// Check if they are based on the same pointer. That makes the offsets
|
|
|
|
// sufficient.
|
|
|
|
if (PtrA == PtrB)
|
|
|
|
return OffsetDelta == Size;
|
|
|
|
|
|
|
|
// Compute the necessary base pointer delta to have the necessary final delta
|
|
|
|
// equal to the size.
|
|
|
|
// BaseDelta = Size - OffsetDelta;
|
|
|
|
const SCEV *SizeSCEV = SE.getConstant(Size);
|
|
|
|
const SCEV *BaseDelta = SE.getMinusSCEV(SizeSCEV, OffsetDeltaSCEV);
|
|
|
|
|
|
|
|
// Otherwise compute the distance with SCEV between the base pointers.
|
|
|
|
const SCEV *PtrSCEVA = SE.getSCEV(PtrA);
|
|
|
|
const SCEV *PtrSCEVB = SE.getSCEV(PtrB);
|
|
|
|
const SCEV *X = SE.getAddExpr(PtrSCEVA, BaseDelta);
|
|
|
|
return X == PtrSCEVB;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-03-11 01:40:37 +08:00
|
|
|
bool MemoryDepChecker::Dependence::isSafeForVectorization(DepType Type) {
|
|
|
|
switch (Type) {
|
|
|
|
case NoDep:
|
|
|
|
case Forward:
|
|
|
|
case BackwardVectorizable:
|
|
|
|
return true;
|
|
|
|
|
|
|
|
case Unknown:
|
|
|
|
case ForwardButPreventsForwarding:
|
|
|
|
case Backward:
|
|
|
|
case BackwardVectorizableButPreventsForwarding:
|
|
|
|
return false;
|
|
|
|
}
|
2015-03-11 04:23:29 +08:00
|
|
|
llvm_unreachable("unexpected DepType!");
|
2015-03-11 01:40:37 +08:00
|
|
|
}
|
|
|
|
|
2015-11-04 07:50:03 +08:00
|
|
|
bool MemoryDepChecker::Dependence::isBackward() const {
|
2015-03-11 01:40:37 +08:00
|
|
|
switch (Type) {
|
|
|
|
case NoDep:
|
|
|
|
case Forward:
|
|
|
|
case ForwardButPreventsForwarding:
|
2015-11-04 07:50:03 +08:00
|
|
|
case Unknown:
|
2015-03-11 01:40:37 +08:00
|
|
|
return false;
|
|
|
|
|
|
|
|
case BackwardVectorizable:
|
|
|
|
case Backward:
|
|
|
|
case BackwardVectorizableButPreventsForwarding:
|
|
|
|
return true;
|
|
|
|
}
|
2015-03-11 04:23:29 +08:00
|
|
|
llvm_unreachable("unexpected DepType!");
|
2015-03-11 01:40:37 +08:00
|
|
|
}
|
|
|
|
|
2015-11-04 07:50:03 +08:00
|
|
|
bool MemoryDepChecker::Dependence::isPossiblyBackward() const {
|
|
|
|
return isBackward() || Type == Unknown;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool MemoryDepChecker::Dependence::isForward() const {
|
|
|
|
switch (Type) {
|
|
|
|
case Forward:
|
|
|
|
case ForwardButPreventsForwarding:
|
|
|
|
return true;
|
|
|
|
|
|
|
|
case NoDep:
|
|
|
|
case Unknown:
|
|
|
|
case BackwardVectorizable:
|
|
|
|
case Backward:
|
|
|
|
case BackwardVectorizableButPreventsForwarding:
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
llvm_unreachable("unexpected DepType!");
|
|
|
|
}
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
bool MemoryDepChecker::couldPreventStoreLoadForward(unsigned Distance,
|
|
|
|
unsigned TypeByteSize) {
|
|
|
|
// If loads occur at a distance that is not a multiple of a feasible vector
|
|
|
|
// factor store-load forwarding does not take place.
|
|
|
|
// Positive dependences might cause troubles because vectorizing them might
|
|
|
|
// prevent store-load forwarding making vectorized code run a lot slower.
|
|
|
|
// a[i] = a[i-3] ^ a[i-8];
|
|
|
|
// The stores to a[i:i+1] don't align with the stores to a[i-3:i-2] and
|
|
|
|
// hence on your typical architecture store-load forwarding does not take
|
|
|
|
// place. Vectorizing in such cases does not make sense.
|
|
|
|
// Store-load forwarding distance.
|
|
|
|
const unsigned NumCyclesForStoreLoadThroughMemory = 8*TypeByteSize;
|
|
|
|
// Maximum vector factor.
|
2016-05-13 05:41:53 +08:00
|
|
|
unsigned MaxVFWithoutSLForwardIssues = std::min(
|
|
|
|
VectorizerParams::MaxVectorWidth * TypeByteSize, MaxSafeDepDistBytes);
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
for (unsigned vf = 2*TypeByteSize; vf <= MaxVFWithoutSLForwardIssues;
|
|
|
|
vf *= 2) {
|
|
|
|
if (Distance % vf && Distance / vf < NumCyclesForStoreLoadThroughMemory) {
|
|
|
|
MaxVFWithoutSLForwardIssues = (vf >>=1);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-02-20 03:14:34 +08:00
|
|
|
if (MaxVFWithoutSLForwardIssues< 2*TypeByteSize) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Distance " << Distance <<
|
2015-02-20 03:14:34 +08:00
|
|
|
" that could cause a store-load forwarding conflict\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (MaxVFWithoutSLForwardIssues < MaxSafeDepDistBytes &&
|
2015-02-20 03:14:52 +08:00
|
|
|
MaxVFWithoutSLForwardIssues !=
|
|
|
|
VectorizerParams::MaxVectorWidth * TypeByteSize)
|
2015-02-02 00:56:15 +08:00
|
|
|
MaxSafeDepDistBytes = MaxVFWithoutSLForwardIssues;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2015-06-08 12:48:37 +08:00
|
|
|
/// \brief Check the dependence for two accesses with the same stride \p Stride.
|
|
|
|
/// \p Distance is the positive distance and \p TypeByteSize is type size in
|
|
|
|
/// bytes.
|
|
|
|
///
|
|
|
|
/// \returns true if they are independent.
|
|
|
|
static bool areStridedAccessesIndependent(unsigned Distance, unsigned Stride,
|
|
|
|
unsigned TypeByteSize) {
|
|
|
|
assert(Stride > 1 && "The stride must be greater than 1");
|
|
|
|
assert(TypeByteSize > 0 && "The type size in byte must be non-zero");
|
|
|
|
assert(Distance > 0 && "The distance must be non-zero");
|
|
|
|
|
|
|
|
// Skip if the distance is not multiple of type byte size.
|
|
|
|
if (Distance % TypeByteSize)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
unsigned ScaledDist = Distance / TypeByteSize;
|
|
|
|
|
|
|
|
// No dependence if the scaled distance is not multiple of the stride.
|
|
|
|
// E.g.
|
|
|
|
// for (i = 0; i < 1024 ; i += 4)
|
|
|
|
// A[i+2] = A[i] + 1;
|
|
|
|
//
|
|
|
|
// Two accesses in memory (scaled distance is 2, stride is 4):
|
|
|
|
// | A[0] | | | | A[4] | | | |
|
|
|
|
// | | | A[2] | | | | A[6] | |
|
|
|
|
//
|
|
|
|
// E.g.
|
|
|
|
// for (i = 0; i < 1024 ; i += 3)
|
|
|
|
// A[i+4] = A[i] + 1;
|
|
|
|
//
|
|
|
|
// Two accesses in memory (scaled distance is 4, stride is 3):
|
|
|
|
// | A[0] | | | A[3] | | | A[6] | | |
|
|
|
|
// | | | | | A[4] | | | A[7] | |
|
|
|
|
return ScaledDist % Stride;
|
|
|
|
}
|
|
|
|
|
2015-03-11 01:40:37 +08:00
|
|
|
MemoryDepChecker::Dependence::DepType
|
|
|
|
MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
|
|
|
|
const MemAccessInfo &B, unsigned BIdx,
|
|
|
|
const ValueToValueMap &Strides) {
|
2015-02-02 00:56:15 +08:00
|
|
|
assert (AIdx < BIdx && "Must pass arguments in program order");
|
|
|
|
|
|
|
|
Value *APtr = A.getPointer();
|
|
|
|
Value *BPtr = B.getPointer();
|
|
|
|
bool AIsWrite = A.getInt();
|
|
|
|
bool BIsWrite = B.getInt();
|
|
|
|
|
|
|
|
// Two reads are independent.
|
|
|
|
if (!AIsWrite && !BIsWrite)
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::NoDep;
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
// We cannot check pointers in different address spaces.
|
|
|
|
if (APtr->getType()->getPointerAddressSpace() !=
|
|
|
|
BPtr->getType()->getPointerAddressSpace())
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::Unknown;
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2016-05-10 13:55:16 +08:00
|
|
|
int StrideAPtr = getPtrStride(PSE, APtr, InnermostLoop, Strides, true);
|
|
|
|
int StrideBPtr = getPtrStride(PSE, BPtr, InnermostLoop, Strides, true);
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2016-05-10 20:28:49 +08:00
|
|
|
const SCEV *Src = PSE.getSCEV(APtr);
|
|
|
|
const SCEV *Sink = PSE.getSCEV(BPtr);
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
// If the induction step is negative we have to invert source and sink of the
|
|
|
|
// dependence.
|
|
|
|
if (StrideAPtr < 0) {
|
|
|
|
std::swap(APtr, BPtr);
|
|
|
|
std::swap(Src, Sink);
|
|
|
|
std::swap(AIsWrite, BIsWrite);
|
|
|
|
std::swap(AIdx, BIdx);
|
|
|
|
std::swap(StrideAPtr, StrideBPtr);
|
|
|
|
}
|
|
|
|
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
const SCEV *Dist = PSE.getSE()->getMinusSCEV(Sink, Src);
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Src Scev: " << *Src << "Sink Scev: " << *Sink
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
<< "(Induction step: " << StrideAPtr << ")\n");
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Distance for " << *InstMap[AIdx] << " to "
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
<< *InstMap[BIdx] << ": " << *Dist << "\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-07-09 08:03:22 +08:00
|
|
|
// Need accesses with constant stride. We don't want to vectorize
|
2015-02-02 00:56:15 +08:00
|
|
|
// "A[B[i]] += ..." and similar code or pointer arithmetic that could wrap in
|
|
|
|
// the address space.
|
|
|
|
if (!StrideAPtr || !StrideBPtr || StrideAPtr != StrideBPtr){
|
2015-07-09 08:03:22 +08:00
|
|
|
DEBUG(dbgs() << "Pointer access with non-constant stride\n");
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::Unknown;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
const SCEVConstant *C = dyn_cast<SCEVConstant>(Dist);
|
|
|
|
if (!C) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Dependence because of non-constant distance\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
ShouldRetryWithRuntimeCheck = true;
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::Unknown;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
Type *ATy = APtr->getType()->getPointerElementType();
|
|
|
|
Type *BTy = BPtr->getType()->getPointerElementType();
|
2015-03-10 10:37:25 +08:00
|
|
|
auto &DL = InnermostLoop->getHeader()->getModule()->getDataLayout();
|
|
|
|
unsigned TypeByteSize = DL.getTypeAllocSize(ATy);
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
// Negative distances are not plausible dependencies.
|
2015-12-18 04:28:46 +08:00
|
|
|
const APInt &Val = C->getAPInt();
|
2015-02-02 00:56:15 +08:00
|
|
|
if (Val.isNegative()) {
|
|
|
|
bool IsTrueDataDependence = (AIsWrite && !BIsWrite);
|
|
|
|
if (IsTrueDataDependence &&
|
|
|
|
(couldPreventStoreLoadForward(Val.abs().getZExtValue(), TypeByteSize) ||
|
2016-03-01 08:50:08 +08:00
|
|
|
ATy != BTy)) {
|
|
|
|
DEBUG(dbgs() << "LAA: Forward but may prevent st->ld forwarding\n");
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::ForwardButPreventsForwarding;
|
2016-03-01 08:50:08 +08:00
|
|
|
}
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2016-05-06 07:41:28 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Dependence is negative\n");
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::Forward;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Write to the same location with the same size.
|
|
|
|
// Could be improved to assert type sizes are the same (i32 == float, etc).
|
|
|
|
if (Val == 0) {
|
|
|
|
if (ATy == BTy)
|
2015-11-04 04:13:43 +08:00
|
|
|
return Dependence::Forward;
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Zero dependence difference but different types\n");
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::Unknown;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
assert(Val.isStrictlyPositive() && "Expect a positive value");
|
|
|
|
|
|
|
|
if (ATy != BTy) {
|
2015-02-20 03:14:34 +08:00
|
|
|
DEBUG(dbgs() <<
|
2015-02-20 03:15:07 +08:00
|
|
|
"LAA: ReadWrite-Write positive dependency with different types\n");
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::Unknown;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
unsigned Distance = (unsigned) Val.getZExtValue();
|
|
|
|
|
2015-06-08 12:48:37 +08:00
|
|
|
unsigned Stride = std::abs(StrideAPtr);
|
|
|
|
if (Stride > 1 &&
|
2015-07-09 02:47:38 +08:00
|
|
|
areStridedAccessesIndependent(Distance, Stride, TypeByteSize)) {
|
|
|
|
DEBUG(dbgs() << "LAA: Strided accesses are independent\n");
|
2015-06-08 12:48:37 +08:00
|
|
|
return Dependence::NoDep;
|
2015-07-09 02:47:38 +08:00
|
|
|
}
|
2015-06-08 12:48:37 +08:00
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
// Bail out early if passed-in parameters make vectorization not feasible.
|
2015-02-20 03:14:52 +08:00
|
|
|
unsigned ForcedFactor = (VectorizerParams::VectorizationFactor ?
|
|
|
|
VectorizerParams::VectorizationFactor : 1);
|
|
|
|
unsigned ForcedUnroll = (VectorizerParams::VectorizationInterleave ?
|
|
|
|
VectorizerParams::VectorizationInterleave : 1);
|
2015-06-08 12:48:37 +08:00
|
|
|
// The minimum number of iterations for a vectorized/unrolled version.
|
|
|
|
unsigned MinNumIter = std::max(ForcedFactor * ForcedUnroll, 2U);
|
|
|
|
|
|
|
|
// It's not vectorizable if the distance is smaller than the minimum distance
|
|
|
|
// needed for a vectroized/unrolled version. Vectorizing one iteration in
|
|
|
|
// front needs TypeByteSize * Stride. Vectorizing the last iteration needs
|
|
|
|
// TypeByteSize (No need to plus the last gap distance).
|
|
|
|
//
|
|
|
|
// E.g. Assume one char is 1 byte in memory and one int is 4 bytes.
|
|
|
|
// foo(int *A) {
|
|
|
|
// int *B = (int *)((char *)A + 14);
|
|
|
|
// for (i = 0 ; i < 1024 ; i += 2)
|
|
|
|
// B[i] = A[i] + 1;
|
|
|
|
// }
|
|
|
|
//
|
|
|
|
// Two accesses in memory (stride is 2):
|
|
|
|
// | A[0] | | A[2] | | A[4] | | A[6] | |
|
|
|
|
// | B[0] | | B[2] | | B[4] |
|
|
|
|
//
|
|
|
|
// Distance needs for vectorizing iterations except the last iteration:
|
|
|
|
// 4 * 2 * (MinNumIter - 1). Distance needs for the last iteration: 4.
|
|
|
|
// So the minimum distance needed is: 4 * 2 * (MinNumIter - 1) + 4.
|
|
|
|
//
|
|
|
|
// If MinNumIter is 2, it is vectorizable as the minimum distance needed is
|
|
|
|
// 12, which is less than distance.
|
|
|
|
//
|
|
|
|
// If MinNumIter is 4 (Say if a user forces the vectorization factor to be 4),
|
|
|
|
// the minimum distance needed is 28, which is greater than distance. It is
|
|
|
|
// not safe to do vectorization.
|
|
|
|
unsigned MinDistanceNeeded =
|
|
|
|
TypeByteSize * Stride * (MinNumIter - 1) + TypeByteSize;
|
|
|
|
if (MinDistanceNeeded > Distance) {
|
|
|
|
DEBUG(dbgs() << "LAA: Failure because of positive distance " << Distance
|
|
|
|
<< '\n');
|
|
|
|
return Dependence::Backward;
|
|
|
|
}
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-06-08 12:48:37 +08:00
|
|
|
// Unsafe if the minimum distance needed is greater than max safe distance.
|
|
|
|
if (MinDistanceNeeded > MaxSafeDepDistBytes) {
|
|
|
|
DEBUG(dbgs() << "LAA: Failure because it needs at least "
|
|
|
|
<< MinDistanceNeeded << " size in bytes");
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::Backward;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-02-27 01:58:48 +08:00
|
|
|
// Positive distance bigger than max vectorization factor.
|
2015-06-08 12:48:37 +08:00
|
|
|
// FIXME: Should use max factor instead of max distance in bytes, which could
|
|
|
|
// not handle different types.
|
|
|
|
// E.g. Assume one char is 1 byte in memory and one int is 4 bytes.
|
|
|
|
// void foo (int *A, char *B) {
|
|
|
|
// for (unsigned i = 0; i < 1024; i++) {
|
|
|
|
// A[i+2] = A[i] + 1;
|
|
|
|
// B[i+2] = B[i] + 1;
|
|
|
|
// }
|
|
|
|
// }
|
|
|
|
//
|
|
|
|
// This case is currently unsafe according to the max safe distance. If we
|
|
|
|
// analyze the two accesses on array B, the max safe dependence distance
|
|
|
|
// is 2. Then we analyze the accesses on array A, the minimum distance needed
|
|
|
|
// is 8, which is less than 2 and forbidden vectorization, But actually
|
|
|
|
// both A and B could be vectorized by 2 iterations.
|
|
|
|
MaxSafeDepDistBytes =
|
|
|
|
Distance < MaxSafeDepDistBytes ? Distance : MaxSafeDepDistBytes;
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
bool IsTrueDataDependence = (!AIsWrite && BIsWrite);
|
|
|
|
if (IsTrueDataDependence &&
|
|
|
|
couldPreventStoreLoadForward(Distance, TypeByteSize))
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::BackwardVectorizableButPreventsForwarding;
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-06-08 12:48:37 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Positive distance " << Val.getSExtValue()
|
|
|
|
<< " with max VF = "
|
|
|
|
<< MaxSafeDepDistBytes / (TypeByteSize * Stride) << '\n');
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-03-11 01:40:37 +08:00
|
|
|
return Dependence::BackwardVectorizable;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-03-11 01:40:34 +08:00
|
|
|
bool MemoryDepChecker::areDepsSafe(DepCandidates &AccessSets,
|
2015-02-02 00:56:15 +08:00
|
|
|
MemAccessInfoSet &CheckDeps,
|
2015-02-24 08:41:59 +08:00
|
|
|
const ValueToValueMap &Strides) {
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
MaxSafeDepDistBytes = -1U;
|
|
|
|
while (!CheckDeps.empty()) {
|
|
|
|
MemAccessInfo CurAccess = *CheckDeps.begin();
|
|
|
|
|
|
|
|
// Get the relevant memory access set.
|
|
|
|
EquivalenceClasses<MemAccessInfo>::iterator I =
|
|
|
|
AccessSets.findValue(AccessSets.getLeaderValue(CurAccess));
|
|
|
|
|
|
|
|
// Check accesses within this set.
|
2016-02-19 06:09:30 +08:00
|
|
|
EquivalenceClasses<MemAccessInfo>::member_iterator AI =
|
|
|
|
AccessSets.member_begin(I);
|
|
|
|
EquivalenceClasses<MemAccessInfo>::member_iterator AE =
|
|
|
|
AccessSets.member_end();
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
// Check every access pair.
|
|
|
|
while (AI != AE) {
|
|
|
|
CheckDeps.erase(*AI);
|
|
|
|
EquivalenceClasses<MemAccessInfo>::member_iterator OI = std::next(AI);
|
|
|
|
while (OI != AE) {
|
|
|
|
// Check every accessing instruction pair in program order.
|
|
|
|
for (std::vector<unsigned>::iterator I1 = Accesses[*AI].begin(),
|
|
|
|
I1E = Accesses[*AI].end(); I1 != I1E; ++I1)
|
|
|
|
for (std::vector<unsigned>::iterator I2 = Accesses[*OI].begin(),
|
|
|
|
I2E = Accesses[*OI].end(); I2 != I2E; ++I2) {
|
2015-03-11 01:40:37 +08:00
|
|
|
auto A = std::make_pair(&*AI, *I1);
|
|
|
|
auto B = std::make_pair(&*OI, *I2);
|
|
|
|
|
|
|
|
assert(*I1 != *I2);
|
|
|
|
if (*I1 > *I2)
|
|
|
|
std::swap(A, B);
|
|
|
|
|
|
|
|
Dependence::DepType Type =
|
|
|
|
isDependent(*A.first, A.second, *B.first, B.second, Strides);
|
|
|
|
SafeForVectorization &= Dependence::isSafeForVectorization(Type);
|
|
|
|
|
2015-11-04 05:39:52 +08:00
|
|
|
// Gather dependences unless we accumulated MaxDependences
|
2015-03-11 01:40:37 +08:00
|
|
|
// dependences. In that case return as soon as we find the first
|
|
|
|
// unsafe dependence. This puts a limit on this quadratic
|
|
|
|
// algorithm.
|
2015-11-04 05:39:52 +08:00
|
|
|
if (RecordDependences) {
|
|
|
|
if (Type != Dependence::NoDep)
|
|
|
|
Dependences.push_back(Dependence(A.second, B.second, Type));
|
|
|
|
|
|
|
|
if (Dependences.size() >= MaxDependences) {
|
|
|
|
RecordDependences = false;
|
|
|
|
Dependences.clear();
|
2015-03-11 01:40:37 +08:00
|
|
|
DEBUG(dbgs() << "Too many dependences, stopped recording\n");
|
|
|
|
}
|
|
|
|
}
|
2015-11-04 05:39:52 +08:00
|
|
|
if (!RecordDependences && !SafeForVectorization)
|
2015-02-02 00:56:15 +08:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
++OI;
|
|
|
|
}
|
|
|
|
AI++;
|
|
|
|
}
|
|
|
|
}
|
2015-03-11 01:40:37 +08:00
|
|
|
|
2015-11-04 05:39:52 +08:00
|
|
|
DEBUG(dbgs() << "Total Dependences: " << Dependences.size() << "\n");
|
2015-03-11 01:40:37 +08:00
|
|
|
return SafeForVectorization;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-03-11 02:54:26 +08:00
|
|
|
SmallVector<Instruction *, 4>
|
|
|
|
MemoryDepChecker::getInstructionsForAccess(Value *Ptr, bool isWrite) const {
|
|
|
|
MemAccessInfo Access(Ptr, isWrite);
|
|
|
|
auto &IndexVector = Accesses.find(Access)->second;
|
|
|
|
|
|
|
|
SmallVector<Instruction *, 4> Insts;
|
|
|
|
std::transform(IndexVector.begin(), IndexVector.end(),
|
|
|
|
std::back_inserter(Insts),
|
|
|
|
[&](unsigned Idx) { return this->InstMap[Idx]; });
|
|
|
|
return Insts;
|
|
|
|
}
|
|
|
|
|
2015-03-11 01:40:43 +08:00
|
|
|
const char *MemoryDepChecker::Dependence::DepName[] = {
|
|
|
|
"NoDep", "Unknown", "Forward", "ForwardButPreventsForwarding", "Backward",
|
|
|
|
"BackwardVectorizable", "BackwardVectorizableButPreventsForwarding"};
|
|
|
|
|
|
|
|
void MemoryDepChecker::Dependence::print(
|
|
|
|
raw_ostream &OS, unsigned Depth,
|
|
|
|
const SmallVectorImpl<Instruction *> &Instrs) const {
|
|
|
|
OS.indent(Depth) << DepName[Type] << ":\n";
|
|
|
|
OS.indent(Depth + 2) << *Instrs[Source] << " -> \n";
|
|
|
|
OS.indent(Depth + 2) << *Instrs[Destination] << "\n";
|
|
|
|
}
|
|
|
|
|
2015-02-20 03:15:10 +08:00
|
|
|
bool LoopAccessInfo::canAnalyzeLoop() {
|
2015-04-18 06:43:10 +08:00
|
|
|
// We need to have a loop header.
|
2016-01-19 05:16:33 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Found a loop in "
|
|
|
|
<< TheLoop->getHeader()->getParent()->getName() << ": "
|
|
|
|
<< TheLoop->getHeader()->getName() << '\n');
|
2015-04-18 06:43:10 +08:00
|
|
|
|
2016-01-19 05:16:33 +08:00
|
|
|
// We can only analyze innermost loops.
|
2015-02-20 03:15:10 +08:00
|
|
|
if (!TheLoop->empty()) {
|
2015-04-18 06:43:10 +08:00
|
|
|
DEBUG(dbgs() << "LAA: loop is not the innermost loop\n");
|
2015-02-20 03:15:15 +08:00
|
|
|
emitAnalysis(LoopAccessReport() << "loop is not the innermost loop");
|
2015-02-20 03:15:10 +08:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// We must have a single backedge.
|
|
|
|
if (TheLoop->getNumBackEdges() != 1) {
|
2015-04-18 06:43:10 +08:00
|
|
|
DEBUG(dbgs() << "LAA: loop control flow is not understood by analyzer\n");
|
2015-02-20 03:15:10 +08:00
|
|
|
emitAnalysis(
|
2015-02-20 03:15:15 +08:00
|
|
|
LoopAccessReport() <<
|
2015-02-20 03:15:10 +08:00
|
|
|
"loop control flow is not understood by analyzer");
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// We must have a single exiting block.
|
|
|
|
if (!TheLoop->getExitingBlock()) {
|
2015-04-18 06:43:10 +08:00
|
|
|
DEBUG(dbgs() << "LAA: loop control flow is not understood by analyzer\n");
|
2015-02-20 03:15:10 +08:00
|
|
|
emitAnalysis(
|
2015-02-20 03:15:15 +08:00
|
|
|
LoopAccessReport() <<
|
2015-02-20 03:15:10 +08:00
|
|
|
"loop control flow is not understood by analyzer");
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// We only handle bottom-tested loops, i.e. loop in which the condition is
|
|
|
|
// checked at the end of each iteration. With that we can assume that all
|
|
|
|
// instructions in the loop are executed the same number of times.
|
|
|
|
if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {
|
2015-04-18 06:43:10 +08:00
|
|
|
DEBUG(dbgs() << "LAA: loop control flow is not understood by analyzer\n");
|
2015-02-20 03:15:10 +08:00
|
|
|
emitAnalysis(
|
2015-02-20 03:15:15 +08:00
|
|
|
LoopAccessReport() <<
|
2015-02-20 03:15:10 +08:00
|
|
|
"loop control flow is not understood by analyzer");
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// ScalarEvolution needs to be able to find the exit count.
|
Re-commit [SCEV] Introduce a guarded backedge taken count and use it in LAA and LV
This re-commits r265535 which was reverted in r265541 because it
broke the windows bots. The problem was that we had a PointerIntPair
which took a pointer to a struct allocated with new. The problem
was that new doesn't provide sufficient alignment guarantees.
This pattern was already present before r265535 and it just happened
to work. To fix this, we now separate the PointerToIntPair from the
ExitNotTakenInfo struct into a pointer and a bool.
Original commit message:
Summary:
When the backedge taken codition is computed from an icmp, SCEV can
deduce the backedge taken count only if one of the sides of the icmp
is an AddRecExpr. However, due to sign/zero extensions, we sometimes
end up with something that is not an AddRecExpr.
However, we can use SCEV predicates to produce a 'guarded' expression.
This change adds a method to SCEV to get this expression, and the
SCEV predicate associated with it.
In HowManyGreaterThans and HowManyLessThans we will now add a SCEV
predicate associated with the guarded backedge taken count when the
analyzed SCEV expression is not an AddRecExpr. Note that we only do
this as an alternative to returning a 'CouldNotCompute'.
We use new feature in Loop Access Analysis and LoopVectorize to analyze
and transform more loops.
Reviewers: anemet, mzolotukhin, hfinkel, sanjoy
Subscribers: flyingforyou, mcrosier, atrick, mssimpso, sanjoy, mzolotukhin, llvm-commits
Differential Revision: http://reviews.llvm.org/D17201
llvm-svn: 265786
2016-04-08 22:29:09 +08:00
|
|
|
const SCEV *ExitCount = PSE.getBackedgeTakenCount();
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
if (ExitCount == PSE.getSE()->getCouldNotCompute()) {
|
|
|
|
emitAnalysis(LoopAccessReport()
|
|
|
|
<< "could not determine number of loop iterations");
|
2015-02-20 03:15:10 +08:00
|
|
|
DEBUG(dbgs() << "LAA: SCEV could not compute the loop exit count.\n");
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2015-02-24 08:41:59 +08:00
|
|
|
void LoopAccessInfo::analyzeLoop(const ValueToValueMap &Strides) {
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
typedef SmallVector<Value*, 16> ValueVector;
|
|
|
|
typedef SmallPtrSet<Value*, 16> ValueSet;
|
|
|
|
|
|
|
|
// Holds the Load and Store *instructions*.
|
|
|
|
ValueVector Loads;
|
|
|
|
ValueVector Stores;
|
|
|
|
|
|
|
|
// Holds all the different accesses in the loop.
|
|
|
|
unsigned NumReads = 0;
|
|
|
|
unsigned NumReadWrites = 0;
|
|
|
|
|
2015-07-15 06:32:44 +08:00
|
|
|
PtrRtChecking.Pointers.clear();
|
|
|
|
PtrRtChecking.Need = false;
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
const bool IsAnnotatedParallel = TheLoop->isAnnotatedParallel();
|
|
|
|
|
|
|
|
// For each block.
|
|
|
|
for (Loop::block_iterator bb = TheLoop->block_begin(),
|
|
|
|
be = TheLoop->block_end(); bb != be; ++bb) {
|
|
|
|
|
|
|
|
// Scan the BB and collect legal loads and stores.
|
|
|
|
for (BasicBlock::iterator it = (*bb)->begin(), e = (*bb)->end(); it != e;
|
|
|
|
++it) {
|
|
|
|
|
|
|
|
// If this is a load, save it. If this instruction can read from memory
|
|
|
|
// but is not a load, then we quit. Notice that we don't handle function
|
|
|
|
// calls that read or write.
|
|
|
|
if (it->mayReadFromMemory()) {
|
|
|
|
// Many math library functions read the rounding mode. We will only
|
|
|
|
// vectorize a loop if it contains known function calls that don't set
|
|
|
|
// the flag. Therefore, it is safe to ignore this read from memory.
|
|
|
|
CallInst *Call = dyn_cast<CallInst>(it);
|
2016-04-20 03:10:21 +08:00
|
|
|
if (Call && getVectorIntrinsicIDForCall(Call, TLI))
|
2015-02-02 00:56:15 +08:00
|
|
|
continue;
|
|
|
|
|
2015-03-18 03:46:50 +08:00
|
|
|
// If the function has an explicit vectorized counterpart, we can safely
|
|
|
|
// assume that it can be vectorized.
|
|
|
|
if (Call && !Call->isNoBuiltin() && Call->getCalledFunction() &&
|
|
|
|
TLI->isFunctionVectorizable(Call->getCalledFunction()->getName()))
|
|
|
|
continue;
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
LoadInst *Ld = dyn_cast<LoadInst>(it);
|
|
|
|
if (!Ld || (!Ld->isSimple() && !IsAnnotatedParallel)) {
|
2015-02-20 03:15:15 +08:00
|
|
|
emitAnalysis(LoopAccessReport(Ld)
|
2015-02-02 00:56:15 +08:00
|
|
|
<< "read with atomic ordering or volatile read");
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Found a non-simple load.\n");
|
2015-02-20 03:15:00 +08:00
|
|
|
CanVecMem = false;
|
|
|
|
return;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
NumLoads++;
|
|
|
|
Loads.push_back(Ld);
|
|
|
|
DepChecker.addAccess(Ld);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Save 'store' instructions. Abort if other instructions write to memory.
|
|
|
|
if (it->mayWriteToMemory()) {
|
|
|
|
StoreInst *St = dyn_cast<StoreInst>(it);
|
|
|
|
if (!St) {
|
Analysis: Remove implicit ilist iterator conversions
Remove implicit ilist iterator conversions from LLVMAnalysis.
I came across something really scary in `llvm::isKnownNotFullPoison()`
which relied on `Instruction::getNextNode()` being completely broken
(not surprising, but scary nevertheless). This function is documented
(and coded to) return `nullptr` when it gets to the sentinel, but with
an `ilist_half_node` as a sentinel, the sentinel check looks into some
other memory and we don't recognize we've hit the end.
Rooting out these scary cases is the reason I'm removing the implicit
conversions before doing anything else with `ilist`; I'm not at all
surprised that clients rely on badness.
I found another scary case -- this time, not relying on badness, just
bad (but I guess getting lucky so far) -- in
`ObjectSizeOffsetEvaluator::compute_()`. Here, we save out the
insertion point, do some things, and then restore it. Previously, we
let the iterator auto-convert to `Instruction*`, and then set it back
using the `Instruction*` version:
Instruction *PrevInsertPoint = Builder.GetInsertPoint();
/* Logic that may change insert point */
if (PrevInsertPoint)
Builder.SetInsertPoint(PrevInsertPoint);
The check for `PrevInsertPoint` doesn't protect correctly against bad
accesses. If the insertion point has been set to the end of a basic
block (i.e., `SetInsertPoint(SomeBB)`), then `GetInsertPoint()` returns
an iterator pointing at the list sentinel. The version of
`SetInsertPoint()` that's getting called will then call
`PrevInsertPoint->getParent()`, which explodes horribly. The only
reason this hasn't blown up is that it's fairly unlikely the builder is
adding to the end of the block; usually, we're adding instructions
somewhere before the terminator.
llvm-svn: 249925
2015-10-10 08:53:03 +08:00
|
|
|
emitAnalysis(LoopAccessReport(&*it) <<
|
2015-02-20 03:14:34 +08:00
|
|
|
"instruction cannot be vectorized");
|
2015-02-20 03:15:00 +08:00
|
|
|
CanVecMem = false;
|
|
|
|
return;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
if (!St->isSimple() && !IsAnnotatedParallel) {
|
2015-02-20 03:15:15 +08:00
|
|
|
emitAnalysis(LoopAccessReport(St)
|
2015-02-02 00:56:15 +08:00
|
|
|
<< "write with atomic ordering or volatile write");
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Found a non-simple store.\n");
|
2015-02-20 03:15:00 +08:00
|
|
|
CanVecMem = false;
|
|
|
|
return;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
NumStores++;
|
|
|
|
Stores.push_back(St);
|
|
|
|
DepChecker.addAccess(St);
|
|
|
|
}
|
|
|
|
} // Next instr.
|
|
|
|
} // Next block.
|
|
|
|
|
|
|
|
// Now we have two lists that hold the loads and the stores.
|
|
|
|
// Next, we find the pointers that they use.
|
|
|
|
|
|
|
|
// Check if we see any stores. If there are no stores, then we don't
|
|
|
|
// care if the pointers are *restrict*.
|
|
|
|
if (!Stores.size()) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Found a read-only loop!\n");
|
2015-02-20 03:15:00 +08:00
|
|
|
CanVecMem = true;
|
|
|
|
return;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-03-11 01:40:34 +08:00
|
|
|
MemoryDepChecker::DepCandidates DependentAccesses;
|
2015-03-10 10:37:25 +08:00
|
|
|
AccessAnalysis Accesses(TheLoop->getHeader()->getModule()->getDataLayout(),
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
AA, LI, DependentAccesses, PSE);
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
// Holds the analyzed pointers. We don't want to call GetUnderlyingObjects
|
|
|
|
// multiple times on the same object. If the ptr is accessed twice, once
|
|
|
|
// for read and once for write, it will only appear once (on the write
|
|
|
|
// list). This is okay, since we are going to check for conflicts between
|
|
|
|
// writes and between reads and writes, but not between reads and reads.
|
|
|
|
ValueSet Seen;
|
|
|
|
|
|
|
|
ValueVector::iterator I, IE;
|
|
|
|
for (I = Stores.begin(), IE = Stores.end(); I != IE; ++I) {
|
|
|
|
StoreInst *ST = cast<StoreInst>(*I);
|
|
|
|
Value* Ptr = ST->getPointerOperand();
|
2015-04-09 01:48:40 +08:00
|
|
|
// Check for store to loop invariant address.
|
|
|
|
StoreToLoopInvariantAddress |= isUniform(Ptr);
|
2015-02-02 00:56:15 +08:00
|
|
|
// If we did *not* see this pointer before, insert it to the read-write
|
|
|
|
// list. At this phase it is only a 'write' list.
|
|
|
|
if (Seen.insert(Ptr).second) {
|
|
|
|
++NumReadWrites;
|
|
|
|
|
2015-06-17 15:18:54 +08:00
|
|
|
MemoryLocation Loc = MemoryLocation::get(ST);
|
2015-02-02 00:56:15 +08:00
|
|
|
// The TBAA metadata could have a control dependency on the predication
|
|
|
|
// condition, so we cannot rely on it when determining whether or not we
|
|
|
|
// need runtime pointer checks.
|
2015-02-18 11:43:19 +08:00
|
|
|
if (blockNeedsPredication(ST->getParent(), TheLoop, DT))
|
2015-02-02 00:56:15 +08:00
|
|
|
Loc.AATags.TBAA = nullptr;
|
|
|
|
|
|
|
|
Accesses.addStore(Loc);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (IsAnnotatedParallel) {
|
2015-02-20 03:14:34 +08:00
|
|
|
DEBUG(dbgs()
|
2015-02-20 03:15:07 +08:00
|
|
|
<< "LAA: A loop annotated parallel, ignore memory dependency "
|
2015-02-20 03:14:34 +08:00
|
|
|
<< "checks.\n");
|
2015-02-20 03:15:00 +08:00
|
|
|
CanVecMem = true;
|
|
|
|
return;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
for (I = Loads.begin(), IE = Loads.end(); I != IE; ++I) {
|
|
|
|
LoadInst *LD = cast<LoadInst>(*I);
|
|
|
|
Value* Ptr = LD->getPointerOperand();
|
|
|
|
// If we did *not* see this pointer before, insert it to the
|
|
|
|
// read list. If we *did* see it before, then it is already in
|
|
|
|
// the read-write list. This allows us to vectorize expressions
|
|
|
|
// such as A[i] += x; Because the address of A[i] is a read-write
|
|
|
|
// pointer. This only works if the index of A[i] is consecutive.
|
|
|
|
// If the address of i is unknown (for example A[B[i]]) then we may
|
|
|
|
// read a few words, modify, and write a few words, and some of the
|
|
|
|
// words may be written to the same address.
|
|
|
|
bool IsReadOnlyPtr = false;
|
2016-05-10 13:55:16 +08:00
|
|
|
if (Seen.insert(Ptr).second || !getPtrStride(PSE, Ptr, TheLoop, Strides)) {
|
2015-02-02 00:56:15 +08:00
|
|
|
++NumReads;
|
|
|
|
IsReadOnlyPtr = true;
|
|
|
|
}
|
|
|
|
|
2015-06-17 15:18:54 +08:00
|
|
|
MemoryLocation Loc = MemoryLocation::get(LD);
|
2015-02-02 00:56:15 +08:00
|
|
|
// The TBAA metadata could have a control dependency on the predication
|
|
|
|
// condition, so we cannot rely on it when determining whether or not we
|
|
|
|
// need runtime pointer checks.
|
2015-02-18 11:43:19 +08:00
|
|
|
if (blockNeedsPredication(LD->getParent(), TheLoop, DT))
|
2015-02-02 00:56:15 +08:00
|
|
|
Loc.AATags.TBAA = nullptr;
|
|
|
|
|
|
|
|
Accesses.addLoad(Loc, IsReadOnlyPtr);
|
|
|
|
}
|
|
|
|
|
|
|
|
// If we write (or read-write) to a single destination and there are no
|
|
|
|
// other reads in this loop then is it safe to vectorize.
|
|
|
|
if (NumReadWrites == 1 && NumReads == 0) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Found a write-only loop!\n");
|
2015-02-20 03:15:00 +08:00
|
|
|
CanVecMem = true;
|
|
|
|
return;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Build dependence sets and check whether we need a runtime pointer bounds
|
|
|
|
// check.
|
|
|
|
Accesses.buildDependenceSets();
|
|
|
|
|
|
|
|
// Find pointers with computable bounds. We are going to use this information
|
|
|
|
// to place a runtime bound check.
|
2015-07-10 06:17:38 +08:00
|
|
|
bool CanDoRTIfNeeded =
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
Accesses.canCheckPtrAtRT(PtrRtChecking, PSE.getSE(), TheLoop, Strides);
|
2015-07-10 06:17:38 +08:00
|
|
|
if (!CanDoRTIfNeeded) {
|
2015-02-20 03:15:15 +08:00
|
|
|
emitAnalysis(LoopAccessReport() << "cannot identify array bounds");
|
2015-07-10 06:17:38 +08:00
|
|
|
DEBUG(dbgs() << "LAA: We can't vectorize because we can't find "
|
|
|
|
<< "the array bounds.\n");
|
2015-02-20 03:15:00 +08:00
|
|
|
CanVecMem = false;
|
|
|
|
return;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-07-10 06:17:38 +08:00
|
|
|
DEBUG(dbgs() << "LAA: We can perform a memory runtime check if needed.\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-02-20 03:15:00 +08:00
|
|
|
CanVecMem = true;
|
2015-02-02 00:56:15 +08:00
|
|
|
if (Accesses.isDependencyCheckNeeded()) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Checking memory dependencies\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
CanVecMem = DepChecker.areDepsSafe(
|
|
|
|
DependentAccesses, Accesses.getDependenciesToCheck(), Strides);
|
|
|
|
MaxSafeDepDistBytes = DepChecker.getMaxSafeDepDistBytes();
|
|
|
|
|
|
|
|
if (!CanVecMem && DepChecker.shouldRetryWithRuntimeCheck()) {
|
2015-02-20 03:15:07 +08:00
|
|
|
DEBUG(dbgs() << "LAA: Retrying with memory checks\n");
|
2015-02-02 00:56:15 +08:00
|
|
|
|
|
|
|
// Clear the dependency checks. We assume they are not needed.
|
2015-05-18 23:37:03 +08:00
|
|
|
Accesses.resetDepChecks(DepChecker);
|
2015-02-02 00:56:15 +08:00
|
|
|
|
2015-07-15 06:32:44 +08:00
|
|
|
PtrRtChecking.reset();
|
|
|
|
PtrRtChecking.Need = true;
|
2015-02-02 00:56:15 +08:00
|
|
|
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
auto *SE = PSE.getSE();
|
2015-07-10 06:17:38 +08:00
|
|
|
CanDoRTIfNeeded =
|
2015-07-15 06:32:44 +08:00
|
|
|
Accesses.canCheckPtrAtRT(PtrRtChecking, SE, TheLoop, Strides, true);
|
2015-06-08 18:27:06 +08:00
|
|
|
|
2015-03-11 03:12:41 +08:00
|
|
|
// Check that we found the bounds for the pointer.
|
2015-07-10 06:17:38 +08:00
|
|
|
if (!CanDoRTIfNeeded) {
|
2015-03-11 02:54:19 +08:00
|
|
|
emitAnalysis(LoopAccessReport()
|
|
|
|
<< "cannot check memory dependencies at runtime");
|
|
|
|
DEBUG(dbgs() << "LAA: Can't vectorize with memory checks\n");
|
|
|
|
CanVecMem = false;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2015-02-02 00:56:15 +08:00
|
|
|
CanVecMem = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-03-11 05:47:39 +08:00
|
|
|
if (CanVecMem)
|
|
|
|
DEBUG(dbgs() << "LAA: No unsafe dependent memory operations in loop. We"
|
2015-07-15 06:32:44 +08:00
|
|
|
<< (PtrRtChecking.Need ? "" : " don't")
|
2015-07-10 06:17:41 +08:00
|
|
|
<< " need runtime memory checks.\n");
|
2015-03-11 05:47:39 +08:00
|
|
|
else {
|
2016-05-10 07:03:44 +08:00
|
|
|
emitAnalysis(
|
|
|
|
LoopAccessReport()
|
|
|
|
<< "unsafe dependent memory operations in loop. Use "
|
|
|
|
"#pragma loop distribute(enable) to allow loop distribution "
|
|
|
|
"to attempt to isolate the offending operations into a separate "
|
|
|
|
"loop");
|
2015-03-11 05:47:39 +08:00
|
|
|
DEBUG(dbgs() << "LAA: unsafe dependent memory operations in loop\n");
|
|
|
|
}
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-02-18 11:43:19 +08:00
|
|
|
bool LoopAccessInfo::blockNeedsPredication(BasicBlock *BB, Loop *TheLoop,
|
|
|
|
DominatorTree *DT) {
|
2015-02-02 00:56:15 +08:00
|
|
|
assert(TheLoop->contains(BB) && "Unknown block used");
|
|
|
|
|
|
|
|
// Blocks that do not dominate the latch need predication.
|
|
|
|
BasicBlock* Latch = TheLoop->getLoopLatch();
|
|
|
|
return !DT->dominates(BB, Latch);
|
|
|
|
}
|
|
|
|
|
2015-02-20 03:15:15 +08:00
|
|
|
void LoopAccessInfo::emitAnalysis(LoopAccessReport &Message) {
|
2015-02-20 03:14:56 +08:00
|
|
|
assert(!Report && "Multiple reports generated");
|
|
|
|
Report = Message;
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
|
|
|
|
2015-02-20 03:15:21 +08:00
|
|
|
bool LoopAccessInfo::isUniform(Value *V) const {
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
return (PSE.getSE()->isLoopInvariant(PSE.getSE()->getSCEV(V), TheLoop));
|
2015-02-02 00:56:15 +08:00
|
|
|
}
|
2015-02-07 02:31:04 +08:00
|
|
|
|
|
|
|
// FIXME: this function is currently a duplicate of the one in
|
|
|
|
// LoopVectorize.cpp.
|
|
|
|
static Instruction *getFirstInst(Instruction *FirstInst, Value *V,
|
|
|
|
Instruction *Loc) {
|
|
|
|
if (FirstInst)
|
|
|
|
return FirstInst;
|
|
|
|
if (Instruction *I = dyn_cast<Instruction>(V))
|
|
|
|
return I->getParent() == Loc->getParent() ? I : nullptr;
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
|
2015-10-28 21:54:36 +08:00
|
|
|
namespace {
|
2015-08-22 07:19:57 +08:00
|
|
|
/// \brief IR Values for the lower and upper bounds of a pointer evolution. We
|
|
|
|
/// need to use value-handles because SCEV expansion can invalidate previously
|
|
|
|
/// expanded values. Thus expansion of a pointer can invalidate the bounds for
|
|
|
|
/// a previous one.
|
2015-07-26 13:32:14 +08:00
|
|
|
struct PointerBounds {
|
2015-08-22 07:19:57 +08:00
|
|
|
TrackingVH<Value> Start;
|
|
|
|
TrackingVH<Value> End;
|
2015-07-26 13:32:14 +08:00
|
|
|
};
|
2015-10-28 21:54:36 +08:00
|
|
|
} // end anonymous namespace
|
2015-07-26 13:32:14 +08:00
|
|
|
|
|
|
|
/// \brief Expand code for the lower and upper bound of the pointer group \p CG
|
|
|
|
/// in \p TheLoop. \return the values for the bounds.
|
|
|
|
static PointerBounds
|
|
|
|
expandBounds(const RuntimePointerChecking::CheckingPtrGroup *CG, Loop *TheLoop,
|
|
|
|
Instruction *Loc, SCEVExpander &Exp, ScalarEvolution *SE,
|
|
|
|
const RuntimePointerChecking &PtrRtChecking) {
|
|
|
|
Value *Ptr = PtrRtChecking.Pointers[CG->Members[0]].PointerValue;
|
|
|
|
const SCEV *Sc = SE->getSCEV(Ptr);
|
|
|
|
|
|
|
|
if (SE->isLoopInvariant(Sc, TheLoop)) {
|
|
|
|
DEBUG(dbgs() << "LAA: Adding RT check for a loop invariant ptr:" << *Ptr
|
|
|
|
<< "\n");
|
|
|
|
return {Ptr, Ptr};
|
|
|
|
} else {
|
|
|
|
unsigned AS = Ptr->getType()->getPointerAddressSpace();
|
|
|
|
LLVMContext &Ctx = Loc->getContext();
|
|
|
|
|
|
|
|
// Use this type for pointer arithmetic.
|
|
|
|
Type *PtrArithTy = Type::getInt8PtrTy(Ctx, AS);
|
|
|
|
Value *Start = nullptr, *End = nullptr;
|
|
|
|
|
|
|
|
DEBUG(dbgs() << "LAA: Adding RT check for range:\n");
|
|
|
|
Start = Exp.expandCodeFor(CG->Low, PtrArithTy, Loc);
|
|
|
|
End = Exp.expandCodeFor(CG->High, PtrArithTy, Loc);
|
|
|
|
DEBUG(dbgs() << "Start: " << *CG->Low << " End: " << *CG->High << "\n");
|
|
|
|
return {Start, End};
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/// \brief Turns a collection of checks into a collection of expanded upper and
|
|
|
|
/// lower bounds for both pointers in the check.
|
|
|
|
static SmallVector<std::pair<PointerBounds, PointerBounds>, 4> expandBounds(
|
|
|
|
const SmallVectorImpl<RuntimePointerChecking::PointerCheck> &PointerChecks,
|
|
|
|
Loop *L, Instruction *Loc, ScalarEvolution *SE, SCEVExpander &Exp,
|
|
|
|
const RuntimePointerChecking &PtrRtChecking) {
|
|
|
|
SmallVector<std::pair<PointerBounds, PointerBounds>, 4> ChecksWithBounds;
|
|
|
|
|
|
|
|
// Here we're relying on the SCEV Expander's cache to only emit code for the
|
|
|
|
// same bounds once.
|
|
|
|
std::transform(
|
|
|
|
PointerChecks.begin(), PointerChecks.end(),
|
|
|
|
std::back_inserter(ChecksWithBounds),
|
|
|
|
[&](const RuntimePointerChecking::PointerCheck &Check) {
|
2015-07-27 09:35:30 +08:00
|
|
|
PointerBounds
|
|
|
|
First = expandBounds(Check.first, L, Loc, Exp, SE, PtrRtChecking),
|
|
|
|
Second = expandBounds(Check.second, L, Loc, Exp, SE, PtrRtChecking);
|
|
|
|
return std::make_pair(First, Second);
|
2015-07-26 13:32:14 +08:00
|
|
|
});
|
|
|
|
|
|
|
|
return ChecksWithBounds;
|
|
|
|
}
|
|
|
|
|
2015-08-11 08:09:37 +08:00
|
|
|
std::pair<Instruction *, Instruction *> LoopAccessInfo::addRuntimeChecks(
|
2015-07-26 13:32:14 +08:00
|
|
|
Instruction *Loc,
|
|
|
|
const SmallVectorImpl<RuntimePointerChecking::PointerCheck> &PointerChecks)
|
|
|
|
const {
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
auto *SE = PSE.getSE();
|
2015-07-26 13:32:14 +08:00
|
|
|
SCEVExpander Exp(*SE, DL, "induction");
|
|
|
|
auto ExpandedChecks =
|
|
|
|
expandBounds(PointerChecks, TheLoop, Loc, SE, Exp, PtrRtChecking);
|
2015-02-07 02:31:04 +08:00
|
|
|
|
|
|
|
LLVMContext &Ctx = Loc->getContext();
|
|
|
|
Instruction *FirstInst = nullptr;
|
|
|
|
IRBuilder<> ChkBuilder(Loc);
|
|
|
|
// Our instructions might fold to a constant.
|
|
|
|
Value *MemoryRuntimeCheck = nullptr;
|
2015-07-15 06:32:44 +08:00
|
|
|
|
2015-07-26 13:32:14 +08:00
|
|
|
for (const auto &Check : ExpandedChecks) {
|
|
|
|
const PointerBounds &A = Check.first, &B = Check.second;
|
2015-08-20 01:24:36 +08:00
|
|
|
// Check if two pointers (A and B) conflict where conflict is computed as:
|
|
|
|
// start(A) <= end(B) && start(B) <= end(A)
|
2015-07-26 13:32:14 +08:00
|
|
|
unsigned AS0 = A.Start->getType()->getPointerAddressSpace();
|
|
|
|
unsigned AS1 = B.Start->getType()->getPointerAddressSpace();
|
|
|
|
|
|
|
|
assert((AS0 == B.End->getType()->getPointerAddressSpace()) &&
|
|
|
|
(AS1 == A.End->getType()->getPointerAddressSpace()) &&
|
|
|
|
"Trying to bounds check pointers with different address spaces");
|
|
|
|
|
|
|
|
Type *PtrArithTy0 = Type::getInt8PtrTy(Ctx, AS0);
|
|
|
|
Type *PtrArithTy1 = Type::getInt8PtrTy(Ctx, AS1);
|
|
|
|
|
|
|
|
Value *Start0 = ChkBuilder.CreateBitCast(A.Start, PtrArithTy0, "bc");
|
|
|
|
Value *Start1 = ChkBuilder.CreateBitCast(B.Start, PtrArithTy1, "bc");
|
|
|
|
Value *End0 = ChkBuilder.CreateBitCast(A.End, PtrArithTy1, "bc");
|
|
|
|
Value *End1 = ChkBuilder.CreateBitCast(B.End, PtrArithTy0, "bc");
|
|
|
|
|
|
|
|
Value *Cmp0 = ChkBuilder.CreateICmpULE(Start0, End1, "bound0");
|
|
|
|
FirstInst = getFirstInst(FirstInst, Cmp0, Loc);
|
|
|
|
Value *Cmp1 = ChkBuilder.CreateICmpULE(Start1, End0, "bound1");
|
|
|
|
FirstInst = getFirstInst(FirstInst, Cmp1, Loc);
|
|
|
|
Value *IsConflict = ChkBuilder.CreateAnd(Cmp0, Cmp1, "found.conflict");
|
|
|
|
FirstInst = getFirstInst(FirstInst, IsConflict, Loc);
|
|
|
|
if (MemoryRuntimeCheck) {
|
|
|
|
IsConflict =
|
|
|
|
ChkBuilder.CreateOr(MemoryRuntimeCheck, IsConflict, "conflict.rdx");
|
2015-02-07 02:31:04 +08:00
|
|
|
FirstInst = getFirstInst(FirstInst, IsConflict, Loc);
|
|
|
|
}
|
2015-07-26 13:32:14 +08:00
|
|
|
MemoryRuntimeCheck = IsConflict;
|
2015-02-07 02:31:04 +08:00
|
|
|
}
|
|
|
|
|
2015-04-03 01:51:57 +08:00
|
|
|
if (!MemoryRuntimeCheck)
|
|
|
|
return std::make_pair(nullptr, nullptr);
|
|
|
|
|
2015-02-07 02:31:04 +08:00
|
|
|
// We have to do this trickery because the IRBuilder might fold the check to a
|
|
|
|
// constant expression in which case there is no Instruction anchored in a
|
|
|
|
// the block.
|
|
|
|
Instruction *Check = BinaryOperator::CreateAnd(MemoryRuntimeCheck,
|
|
|
|
ConstantInt::getTrue(Ctx));
|
|
|
|
ChkBuilder.Insert(Check, "memcheck.conflict");
|
|
|
|
FirstInst = getFirstInst(FirstInst, Check, Loc);
|
|
|
|
return std::make_pair(FirstInst, Check);
|
|
|
|
}
|
2015-02-20 03:15:04 +08:00
|
|
|
|
2015-08-11 08:09:37 +08:00
|
|
|
std::pair<Instruction *, Instruction *>
|
|
|
|
LoopAccessInfo::addRuntimeChecks(Instruction *Loc) const {
|
2015-07-26 13:32:14 +08:00
|
|
|
if (!PtrRtChecking.Need)
|
|
|
|
return std::make_pair(nullptr, nullptr);
|
|
|
|
|
2015-08-11 08:09:37 +08:00
|
|
|
return addRuntimeChecks(Loc, PtrRtChecking.getChecks());
|
2015-07-26 13:32:14 +08:00
|
|
|
}
|
|
|
|
|
2015-02-20 03:15:04 +08:00
|
|
|
LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE,
|
2015-03-10 10:37:25 +08:00
|
|
|
const DataLayout &DL,
|
2015-02-20 03:15:04 +08:00
|
|
|
const TargetLibraryInfo *TLI, AliasAnalysis *AA,
|
2015-04-24 04:09:20 +08:00
|
|
|
DominatorTree *DT, LoopInfo *LI,
|
2015-02-24 08:41:59 +08:00
|
|
|
const ValueToValueMap &Strides)
|
[SCEV][LAA] Re-commit r260085 and r260086, this time with a fix for the memory
sanitizer issue. The PredicatedScalarEvolution's copy constructor
wasn't copying the Generation value, and was leaving it un-initialized.
Original commit message:
[SCEV][LAA] Add no wrap SCEV predicates and use use them to improve strided pointer detection
Summary:
This change adds no wrap SCEV predicates with:
- support for runtime checking
- support for expression rewriting:
(sext ({x,+,y}) -> {sext(x),+,sext(y)}
(zext ({x,+,y}) -> {zext(x),+,sext(y)}
Note that we are sign extending the increment of the SCEV, even for
the zext case. This is needed to cover the fairly common case where y would
be a (small) negative integer. In order to do this, this change adds two new
flags: nusw and nssw that are applicable to AddRecExprs and permit the
transformations above.
We also change isStridedPtr in LAA to be able to make use of
these predicates. With this feature we should now always be able to
work around overflow issues in the dependence analysis.
Reviewers: mzolotukhin, sanjoy, anemet
Subscribers: mzolotukhin, sanjoy, llvm-commits, rengolin, jmolloy, hfinkel
Differential Revision: http://reviews.llvm.org/D15412
llvm-svn: 260112
2016-02-09 01:02:45 +08:00
|
|
|
: PSE(*SE, *L), PtrRtChecking(SE), DepChecker(PSE, L), TheLoop(L), DL(DL),
|
2015-07-15 06:32:44 +08:00
|
|
|
TLI(TLI), AA(AA), DT(DT), LI(LI), NumLoads(0), NumStores(0),
|
2015-04-09 01:48:40 +08:00
|
|
|
MaxSafeDepDistBytes(-1U), CanVecMem(false),
|
|
|
|
StoreToLoopInvariantAddress(false) {
|
2015-02-20 03:15:10 +08:00
|
|
|
if (canAnalyzeLoop())
|
|
|
|
analyzeLoop(Strides);
|
2015-02-20 03:15:04 +08:00
|
|
|
}
|
|
|
|
|
2015-02-20 03:15:19 +08:00
|
|
|
void LoopAccessInfo::print(raw_ostream &OS, unsigned Depth) const {
|
|
|
|
if (CanVecMem) {
|
2016-05-14 06:49:09 +08:00
|
|
|
OS.indent(Depth) << "Memory dependences are safe";
|
2016-05-14 06:49:13 +08:00
|
|
|
if (MaxSafeDepDistBytes != -1U)
|
|
|
|
OS << " with a maximum dependence distance of " << MaxSafeDepDistBytes
|
|
|
|
<< " bytes";
|
2015-07-15 06:32:44 +08:00
|
|
|
if (PtrRtChecking.Need)
|
2016-05-14 06:49:09 +08:00
|
|
|
OS << " with run-time checks";
|
|
|
|
OS << "\n";
|
2015-02-20 03:15:19 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
if (Report)
|
|
|
|
OS.indent(Depth) << "Report: " << Report->str() << "\n";
|
|
|
|
|
2015-11-04 05:39:52 +08:00
|
|
|
if (auto *Dependences = DepChecker.getDependences()) {
|
|
|
|
OS.indent(Depth) << "Dependences:\n";
|
|
|
|
for (auto &Dep : *Dependences) {
|
2015-03-11 01:40:43 +08:00
|
|
|
Dep.print(OS, Depth + 2, DepChecker.getMemoryInstructions());
|
|
|
|
OS << "\n";
|
|
|
|
}
|
|
|
|
} else
|
2015-11-04 05:39:52 +08:00
|
|
|
OS.indent(Depth) << "Too many dependences, not recorded\n";
|
2015-02-20 03:15:19 +08:00
|
|
|
|
|
|
|
// List the pair of accesses need run-time checks to prove independence.
|
2015-07-15 06:32:44 +08:00
|
|
|
PtrRtChecking.print(OS, Depth);
|
2015-02-20 03:15:19 +08:00
|
|
|
OS << "\n";
|
2015-05-18 23:36:57 +08:00
|
|
|
|
|
|
|
OS.indent(Depth) << "Store to invariant address was "
|
|
|
|
<< (StoreToLoopInvariantAddress ? "" : "not ")
|
|
|
|
<< "found in loop.\n";
|
[SCEV][LV] Add SCEV Predicates and use them to re-implement stride versioning
Summary:
SCEV Predicates represent conditions that typically cannot be derived from
static analysis, but can be used to reduce SCEV expressions to forms which are
usable for different optimizers.
ScalarEvolution now has the rewriteUsingPredicate method which can simplify a
SCEV expression using a SCEVPredicateSet. The normal workflow of a pass using
SCEVPredicates would be to hold a SCEVPredicateSet and every time assumptions
need to be made a new SCEV Predicate would be created and added to the set.
Each time after calling getSCEV, the user will call the rewriteUsingPredicate
method.
We add two types of predicates
SCEVPredicateSet - implements a set of predicates
SCEVEqualPredicate - tests for equality between two SCEV expressions
We use the SCEVEqualPredicate to re-implement stride versioning. Every time we
version a stride, we will add a SCEVEqualPredicate to the context.
Instead of adding specific stride checks, LoopVectorize now adds a more
generic SCEV check.
We only need to add support for this in the LoopVectorizer since this is the
only pass that will do stride versioning.
Reviewers: mzolotukhin, anemet, hfinkel, sanjoy
Subscribers: sanjoy, hfinkel, rengolin, jmolloy, llvm-commits
Differential Revision: http://reviews.llvm.org/D13595
llvm-svn: 251800
2015-11-02 22:41:02 +08:00
|
|
|
|
|
|
|
OS.indent(Depth) << "SCEV assumptions:\n";
|
Re-commit r255115, with the PredicatedScalarEvolution class moved to
ScalarEvolution.h, in order to avoid cyclic dependencies between the Transform
and Analysis modules:
[LV][LAA] Add a layer over SCEV to apply run-time checked knowledge on SCEV expressions
Summary:
This change creates a layer over ScalarEvolution for LAA and LV, and centralizes the
usage of SCEV predicates. The SCEVPredicatedLayer takes the statically deduced knowledge
by ScalarEvolution and applies the knowledge from the SCEV predicates. The end goal is
that both LAA and LV should use this interface everywhere.
This also solves a problem involving the result of SCEV expression rewritting when
the predicate changes. Suppose we have the expression (sext {a,+,b}) and two predicates
P1: {a,+,b} has nsw
P2: b = 1.
Applying P1 and then P2 gives us {a,+,1}, while applying P2 and the P1 gives us
sext({a,+,1}) (the AddRec expression was changed by P2 so P1 no longer applies).
The SCEVPredicatedLayer maintains the order of transformations by feeding back
the results of previous transformations into new transformations, and therefore
avoiding this issue.
The SCEVPredicatedLayer maintains a cache to remember the results of previous
SCEV rewritting results. This also has the benefit of reducing the overall number
of expression rewrites.
Reviewers: mzolotukhin, anemet
Subscribers: jmolloy, sanjoy, llvm-commits
Differential Revision: http://reviews.llvm.org/D14296
llvm-svn: 255122
2015-12-10 00:06:28 +08:00
|
|
|
PSE.getUnionPredicate().print(OS, Depth);
|
2016-04-15 00:08:45 +08:00
|
|
|
|
|
|
|
OS << "\n";
|
|
|
|
|
|
|
|
OS.indent(Depth) << "Expressions re-written:\n";
|
|
|
|
PSE.print(OS, Depth);
|
2015-02-20 03:15:19 +08:00
|
|
|
}
|
|
|
|
|
2015-02-24 08:41:59 +08:00
|
|
|
const LoopAccessInfo &
|
|
|
|
LoopAccessAnalysis::getInfo(Loop *L, const ValueToValueMap &Strides) {
|
2015-02-20 03:15:04 +08:00
|
|
|
auto &LAI = LoopAccessInfoMap[L];
|
|
|
|
|
|
|
|
#ifndef NDEBUG
|
|
|
|
assert((!LAI || LAI->NumSymbolicStrides == Strides.size()) &&
|
|
|
|
"Symbolic strides changed for loop");
|
|
|
|
#endif
|
|
|
|
|
|
|
|
if (!LAI) {
|
2015-03-10 10:37:25 +08:00
|
|
|
const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
|
[SCEV][LV] Add SCEV Predicates and use them to re-implement stride versioning
Summary:
SCEV Predicates represent conditions that typically cannot be derived from
static analysis, but can be used to reduce SCEV expressions to forms which are
usable for different optimizers.
ScalarEvolution now has the rewriteUsingPredicate method which can simplify a
SCEV expression using a SCEVPredicateSet. The normal workflow of a pass using
SCEVPredicates would be to hold a SCEVPredicateSet and every time assumptions
need to be made a new SCEV Predicate would be created and added to the set.
Each time after calling getSCEV, the user will call the rewriteUsingPredicate
method.
We add two types of predicates
SCEVPredicateSet - implements a set of predicates
SCEVEqualPredicate - tests for equality between two SCEV expressions
We use the SCEVEqualPredicate to re-implement stride versioning. Every time we
version a stride, we will add a SCEVEqualPredicate to the context.
Instead of adding specific stride checks, LoopVectorize now adds a more
generic SCEV check.
We only need to add support for this in the LoopVectorizer since this is the
only pass that will do stride versioning.
Reviewers: mzolotukhin, anemet, hfinkel, sanjoy
Subscribers: sanjoy, hfinkel, rengolin, jmolloy, llvm-commits
Differential Revision: http://reviews.llvm.org/D13595
llvm-svn: 251800
2015-11-02 22:41:02 +08:00
|
|
|
LAI =
|
|
|
|
llvm::make_unique<LoopAccessInfo>(L, SE, DL, TLI, AA, DT, LI, Strides);
|
2015-02-20 03:15:04 +08:00
|
|
|
#ifndef NDEBUG
|
|
|
|
LAI->NumSymbolicStrides = Strides.size();
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
return *LAI.get();
|
|
|
|
}
|
|
|
|
|
2015-02-20 03:15:19 +08:00
|
|
|
void LoopAccessAnalysis::print(raw_ostream &OS, const Module *M) const {
|
|
|
|
LoopAccessAnalysis &LAA = *const_cast<LoopAccessAnalysis *>(this);
|
|
|
|
|
|
|
|
ValueToValueMap NoSymbolicStrides;
|
|
|
|
|
|
|
|
for (Loop *TopLevelLoop : *LI)
|
|
|
|
for (Loop *L : depth_first(TopLevelLoop)) {
|
|
|
|
OS.indent(2) << L->getHeader()->getName() << ":\n";
|
|
|
|
auto &LAI = LAA.getInfo(L, NoSymbolicStrides);
|
|
|
|
LAI.print(OS, 4);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-02-20 03:15:04 +08:00
|
|
|
bool LoopAccessAnalysis::runOnFunction(Function &F) {
|
[PM] Port ScalarEvolution to the new pass manager.
This change makes ScalarEvolution a stand-alone object and just produces
one from a pass as needed. Making this work well requires making the
object movable, using references instead of overwritten pointers in
a number of places, and other refactorings.
I've also wired it up to the new pass manager and added a RUN line to
a test to exercise it under the new pass manager. This includes basic
printing support much like with other analyses.
But there is a big and somewhat scary change here. Prior to this patch
ScalarEvolution was never *actually* invalidated!!! Re-running the pass
just re-wired up the various other analyses and didn't remove any of the
existing entries in the SCEV caches or clear out anything at all. This
might seem OK as everything in SCEV that can uses ValueHandles to track
updates to the values that serve as SCEV keys. However, this still means
that as we ran SCEV over each function in the module, we kept
accumulating more and more SCEVs into the cache. At the end, we would
have a SCEV cache with every value that we ever needed a SCEV for in the
entire module!!! Yowzers. The releaseMemory routine would dump all of
this, but that isn't realy called during normal runs of the pipeline as
far as I can see.
To make matters worse, there *is* actually a key that we don't update
with value handles -- there is a map keyed off of Loop*s. Because
LoopInfo *does* release its memory from run to run, it is entirely
possible to run SCEV over one function, then over another function, and
then lookup a Loop* from the second function but find an entry inserted
for the first function! Ouch.
To make matters still worse, there are plenty of updates that *don't*
trip a value handle. It seems incredibly unlikely that today GVN or
another pass that invalidates SCEV can update values in *just* such
a way that a subsequent run of SCEV will incorrectly find lookups in
a cache, but it is theoretically possible and would be a nightmare to
debug.
With this refactoring, I've fixed all this by actually destroying and
recreating the ScalarEvolution object from run to run. Technically, this
could increase the amount of malloc traffic we see, but then again it is
also technically correct. ;] I don't actually think we're suffering from
tons of malloc traffic from SCEV because if we were, the fact that we
never clear the memory would seem more likely to have come up as an
actual problem before now. So, I've made the simple fix here. If in fact
there are serious issues with too much allocation and deallocation,
I can work on a clever fix that preserves the allocations (while
clearing the data) between each run, but I'd prefer to do that kind of
optimization with a test case / benchmark that shows why we need such
cleverness (and that can test that we actually make it faster). It's
possible that this will make some things faster by making the SCEV
caches have higher locality (due to being significantly smaller) so
until there is a clear benchmark, I think the simple change is best.
Differential Revision: http://reviews.llvm.org/D12063
llvm-svn: 245193
2015-08-17 10:08:17 +08:00
|
|
|
SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
|
2015-02-20 03:15:04 +08:00
|
|
|
auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
|
|
|
|
TLI = TLIP ? &TLIP->getTLI() : nullptr;
|
[PM/AA] Rebuild LLVM's alias analysis infrastructure in a way compatible
with the new pass manager, and no longer relying on analysis groups.
This builds essentially a ground-up new AA infrastructure stack for
LLVM. The core ideas are the same that are used throughout the new pass
manager: type erased polymorphism and direct composition. The design is
as follows:
- FunctionAAResults is a type-erasing alias analysis results aggregation
interface to walk a single query across a range of results from
different alias analyses. Currently this is function-specific as we
always assume that aliasing queries are *within* a function.
- AAResultBase is a CRTP utility providing stub implementations of
various parts of the alias analysis result concept, notably in several
cases in terms of other more general parts of the interface. This can
be used to implement only a narrow part of the interface rather than
the entire interface. This isn't really ideal, this logic should be
hoisted into FunctionAAResults as currently it will cause
a significant amount of redundant work, but it faithfully models the
behavior of the prior infrastructure.
- All the alias analysis passes are ported to be wrapper passes for the
legacy PM and new-style analysis passes for the new PM with a shared
result object. In some cases (most notably CFL), this is an extremely
naive approach that we should revisit when we can specialize for the
new pass manager.
- BasicAA has been restructured to reflect that it is much more
fundamentally a function analysis because it uses dominator trees and
loop info that need to be constructed for each function.
All of the references to getting alias analysis results have been
updated to use the new aggregation interface. All the preservation and
other pass management code has been updated accordingly.
The way the FunctionAAResultsWrapperPass works is to detect the
available alias analyses when run, and add them to the results object.
This means that we should be able to continue to respect when various
passes are added to the pipeline, for example adding CFL or adding TBAA
passes should just cause their results to be available and to get folded
into this. The exception to this rule is BasicAA which really needs to
be a function pass due to using dominator trees and loop info. As
a consequence, the FunctionAAResultsWrapperPass directly depends on
BasicAA and always includes it in the aggregation.
This has significant implications for preserving analyses. Generally,
most passes shouldn't bother preserving FunctionAAResultsWrapperPass
because rebuilding the results just updates the set of known AA passes.
The exception to this rule are LoopPass instances which need to preserve
all the function analyses that the loop pass manager will end up
needing. This means preserving both BasicAAWrapperPass and the
aggregating FunctionAAResultsWrapperPass.
Now, when preserving an alias analysis, you do so by directly preserving
that analysis. This is only necessary for non-immutable-pass-provided
alias analyses though, and there are only three of interest: BasicAA,
GlobalsAA (formerly GlobalsModRef), and SCEVAA. Usually BasicAA is
preserved when needed because it (like DominatorTree and LoopInfo) is
marked as a CFG-only pass. I've expanded GlobalsAA into the preserved
set everywhere we previously were preserving all of AliasAnalysis, and
I've added SCEVAA in the intersection of that with where we preserve
SCEV itself.
One significant challenge to all of this is that the CGSCC passes were
actually using the alias analysis implementations by taking advantage of
a pretty amazing set of loop holes in the old pass manager's analysis
management code which allowed analysis groups to slide through in many
cases. Moving away from analysis groups makes this problem much more
obvious. To fix it, I've leveraged the flexibility the design of the new
PM components provides to just directly construct the relevant alias
analyses for the relevant functions in the IPO passes that need them.
This is a bit hacky, but should go away with the new pass manager, and
is already in many ways cleaner than the prior state.
Another significant challenge is that various facilities of the old
alias analysis infrastructure just don't fit any more. The most
significant of these is the alias analysis 'counter' pass. That pass
relied on the ability to snoop on AA queries at different points in the
analysis group chain. Instead, I'm planning to build printing
functionality directly into the aggregation layer. I've not included
that in this patch merely to keep it smaller.
Note that all of this needs a nearly complete rewrite of the AA
documentation. I'm planning to do that, but I'd like to make sure the
new design settles, and to flesh out a bit more of what it looks like in
the new pass manager first.
Differential Revision: http://reviews.llvm.org/D12080
llvm-svn: 247167
2015-09-10 01:55:00 +08:00
|
|
|
AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
|
2015-02-20 03:15:04 +08:00
|
|
|
DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
|
2015-04-24 04:09:20 +08:00
|
|
|
LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
|
2015-02-20 03:15:04 +08:00
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
void LoopAccessAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
|
[PM] Port ScalarEvolution to the new pass manager.
This change makes ScalarEvolution a stand-alone object and just produces
one from a pass as needed. Making this work well requires making the
object movable, using references instead of overwritten pointers in
a number of places, and other refactorings.
I've also wired it up to the new pass manager and added a RUN line to
a test to exercise it under the new pass manager. This includes basic
printing support much like with other analyses.
But there is a big and somewhat scary change here. Prior to this patch
ScalarEvolution was never *actually* invalidated!!! Re-running the pass
just re-wired up the various other analyses and didn't remove any of the
existing entries in the SCEV caches or clear out anything at all. This
might seem OK as everything in SCEV that can uses ValueHandles to track
updates to the values that serve as SCEV keys. However, this still means
that as we ran SCEV over each function in the module, we kept
accumulating more and more SCEVs into the cache. At the end, we would
have a SCEV cache with every value that we ever needed a SCEV for in the
entire module!!! Yowzers. The releaseMemory routine would dump all of
this, but that isn't realy called during normal runs of the pipeline as
far as I can see.
To make matters worse, there *is* actually a key that we don't update
with value handles -- there is a map keyed off of Loop*s. Because
LoopInfo *does* release its memory from run to run, it is entirely
possible to run SCEV over one function, then over another function, and
then lookup a Loop* from the second function but find an entry inserted
for the first function! Ouch.
To make matters still worse, there are plenty of updates that *don't*
trip a value handle. It seems incredibly unlikely that today GVN or
another pass that invalidates SCEV can update values in *just* such
a way that a subsequent run of SCEV will incorrectly find lookups in
a cache, but it is theoretically possible and would be a nightmare to
debug.
With this refactoring, I've fixed all this by actually destroying and
recreating the ScalarEvolution object from run to run. Technically, this
could increase the amount of malloc traffic we see, but then again it is
also technically correct. ;] I don't actually think we're suffering from
tons of malloc traffic from SCEV because if we were, the fact that we
never clear the memory would seem more likely to have come up as an
actual problem before now. So, I've made the simple fix here. If in fact
there are serious issues with too much allocation and deallocation,
I can work on a clever fix that preserves the allocations (while
clearing the data) between each run, but I'd prefer to do that kind of
optimization with a test case / benchmark that shows why we need such
cleverness (and that can test that we actually make it faster). It's
possible that this will make some things faster by making the SCEV
caches have higher locality (due to being significantly smaller) so
until there is a clear benchmark, I think the simple change is best.
Differential Revision: http://reviews.llvm.org/D12063
llvm-svn: 245193
2015-08-17 10:08:17 +08:00
|
|
|
AU.addRequired<ScalarEvolutionWrapperPass>();
|
[PM/AA] Rebuild LLVM's alias analysis infrastructure in a way compatible
with the new pass manager, and no longer relying on analysis groups.
This builds essentially a ground-up new AA infrastructure stack for
LLVM. The core ideas are the same that are used throughout the new pass
manager: type erased polymorphism and direct composition. The design is
as follows:
- FunctionAAResults is a type-erasing alias analysis results aggregation
interface to walk a single query across a range of results from
different alias analyses. Currently this is function-specific as we
always assume that aliasing queries are *within* a function.
- AAResultBase is a CRTP utility providing stub implementations of
various parts of the alias analysis result concept, notably in several
cases in terms of other more general parts of the interface. This can
be used to implement only a narrow part of the interface rather than
the entire interface. This isn't really ideal, this logic should be
hoisted into FunctionAAResults as currently it will cause
a significant amount of redundant work, but it faithfully models the
behavior of the prior infrastructure.
- All the alias analysis passes are ported to be wrapper passes for the
legacy PM and new-style analysis passes for the new PM with a shared
result object. In some cases (most notably CFL), this is an extremely
naive approach that we should revisit when we can specialize for the
new pass manager.
- BasicAA has been restructured to reflect that it is much more
fundamentally a function analysis because it uses dominator trees and
loop info that need to be constructed for each function.
All of the references to getting alias analysis results have been
updated to use the new aggregation interface. All the preservation and
other pass management code has been updated accordingly.
The way the FunctionAAResultsWrapperPass works is to detect the
available alias analyses when run, and add them to the results object.
This means that we should be able to continue to respect when various
passes are added to the pipeline, for example adding CFL or adding TBAA
passes should just cause their results to be available and to get folded
into this. The exception to this rule is BasicAA which really needs to
be a function pass due to using dominator trees and loop info. As
a consequence, the FunctionAAResultsWrapperPass directly depends on
BasicAA and always includes it in the aggregation.
This has significant implications for preserving analyses. Generally,
most passes shouldn't bother preserving FunctionAAResultsWrapperPass
because rebuilding the results just updates the set of known AA passes.
The exception to this rule are LoopPass instances which need to preserve
all the function analyses that the loop pass manager will end up
needing. This means preserving both BasicAAWrapperPass and the
aggregating FunctionAAResultsWrapperPass.
Now, when preserving an alias analysis, you do so by directly preserving
that analysis. This is only necessary for non-immutable-pass-provided
alias analyses though, and there are only three of interest: BasicAA,
GlobalsAA (formerly GlobalsModRef), and SCEVAA. Usually BasicAA is
preserved when needed because it (like DominatorTree and LoopInfo) is
marked as a CFG-only pass. I've expanded GlobalsAA into the preserved
set everywhere we previously were preserving all of AliasAnalysis, and
I've added SCEVAA in the intersection of that with where we preserve
SCEV itself.
One significant challenge to all of this is that the CGSCC passes were
actually using the alias analysis implementations by taking advantage of
a pretty amazing set of loop holes in the old pass manager's analysis
management code which allowed analysis groups to slide through in many
cases. Moving away from analysis groups makes this problem much more
obvious. To fix it, I've leveraged the flexibility the design of the new
PM components provides to just directly construct the relevant alias
analyses for the relevant functions in the IPO passes that need them.
This is a bit hacky, but should go away with the new pass manager, and
is already in many ways cleaner than the prior state.
Another significant challenge is that various facilities of the old
alias analysis infrastructure just don't fit any more. The most
significant of these is the alias analysis 'counter' pass. That pass
relied on the ability to snoop on AA queries at different points in the
analysis group chain. Instead, I'm planning to build printing
functionality directly into the aggregation layer. I've not included
that in this patch merely to keep it smaller.
Note that all of this needs a nearly complete rewrite of the AA
documentation. I'm planning to do that, but I'd like to make sure the
new design settles, and to flesh out a bit more of what it looks like in
the new pass manager first.
Differential Revision: http://reviews.llvm.org/D12080
llvm-svn: 247167
2015-09-10 01:55:00 +08:00
|
|
|
AU.addRequired<AAResultsWrapperPass>();
|
2015-02-20 03:15:04 +08:00
|
|
|
AU.addRequired<DominatorTreeWrapperPass>();
|
2015-02-20 03:15:19 +08:00
|
|
|
AU.addRequired<LoopInfoWrapperPass>();
|
2015-02-20 03:15:04 +08:00
|
|
|
|
|
|
|
AU.setPreservesAll();
|
|
|
|
}
|
|
|
|
|
|
|
|
char LoopAccessAnalysis::ID = 0;
|
|
|
|
static const char laa_name[] = "Loop Access Analysis";
|
|
|
|
#define LAA_NAME "loop-accesses"
|
|
|
|
|
|
|
|
INITIALIZE_PASS_BEGIN(LoopAccessAnalysis, LAA_NAME, laa_name, false, true)
|
[PM/AA] Rebuild LLVM's alias analysis infrastructure in a way compatible
with the new pass manager, and no longer relying on analysis groups.
This builds essentially a ground-up new AA infrastructure stack for
LLVM. The core ideas are the same that are used throughout the new pass
manager: type erased polymorphism and direct composition. The design is
as follows:
- FunctionAAResults is a type-erasing alias analysis results aggregation
interface to walk a single query across a range of results from
different alias analyses. Currently this is function-specific as we
always assume that aliasing queries are *within* a function.
- AAResultBase is a CRTP utility providing stub implementations of
various parts of the alias analysis result concept, notably in several
cases in terms of other more general parts of the interface. This can
be used to implement only a narrow part of the interface rather than
the entire interface. This isn't really ideal, this logic should be
hoisted into FunctionAAResults as currently it will cause
a significant amount of redundant work, but it faithfully models the
behavior of the prior infrastructure.
- All the alias analysis passes are ported to be wrapper passes for the
legacy PM and new-style analysis passes for the new PM with a shared
result object. In some cases (most notably CFL), this is an extremely
naive approach that we should revisit when we can specialize for the
new pass manager.
- BasicAA has been restructured to reflect that it is much more
fundamentally a function analysis because it uses dominator trees and
loop info that need to be constructed for each function.
All of the references to getting alias analysis results have been
updated to use the new aggregation interface. All the preservation and
other pass management code has been updated accordingly.
The way the FunctionAAResultsWrapperPass works is to detect the
available alias analyses when run, and add them to the results object.
This means that we should be able to continue to respect when various
passes are added to the pipeline, for example adding CFL or adding TBAA
passes should just cause their results to be available and to get folded
into this. The exception to this rule is BasicAA which really needs to
be a function pass due to using dominator trees and loop info. As
a consequence, the FunctionAAResultsWrapperPass directly depends on
BasicAA and always includes it in the aggregation.
This has significant implications for preserving analyses. Generally,
most passes shouldn't bother preserving FunctionAAResultsWrapperPass
because rebuilding the results just updates the set of known AA passes.
The exception to this rule are LoopPass instances which need to preserve
all the function analyses that the loop pass manager will end up
needing. This means preserving both BasicAAWrapperPass and the
aggregating FunctionAAResultsWrapperPass.
Now, when preserving an alias analysis, you do so by directly preserving
that analysis. This is only necessary for non-immutable-pass-provided
alias analyses though, and there are only three of interest: BasicAA,
GlobalsAA (formerly GlobalsModRef), and SCEVAA. Usually BasicAA is
preserved when needed because it (like DominatorTree and LoopInfo) is
marked as a CFG-only pass. I've expanded GlobalsAA into the preserved
set everywhere we previously were preserving all of AliasAnalysis, and
I've added SCEVAA in the intersection of that with where we preserve
SCEV itself.
One significant challenge to all of this is that the CGSCC passes were
actually using the alias analysis implementations by taking advantage of
a pretty amazing set of loop holes in the old pass manager's analysis
management code which allowed analysis groups to slide through in many
cases. Moving away from analysis groups makes this problem much more
obvious. To fix it, I've leveraged the flexibility the design of the new
PM components provides to just directly construct the relevant alias
analyses for the relevant functions in the IPO passes that need them.
This is a bit hacky, but should go away with the new pass manager, and
is already in many ways cleaner than the prior state.
Another significant challenge is that various facilities of the old
alias analysis infrastructure just don't fit any more. The most
significant of these is the alias analysis 'counter' pass. That pass
relied on the ability to snoop on AA queries at different points in the
analysis group chain. Instead, I'm planning to build printing
functionality directly into the aggregation layer. I've not included
that in this patch merely to keep it smaller.
Note that all of this needs a nearly complete rewrite of the AA
documentation. I'm planning to do that, but I'd like to make sure the
new design settles, and to flesh out a bit more of what it looks like in
the new pass manager first.
Differential Revision: http://reviews.llvm.org/D12080
llvm-svn: 247167
2015-09-10 01:55:00 +08:00
|
|
|
INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
|
[PM] Port ScalarEvolution to the new pass manager.
This change makes ScalarEvolution a stand-alone object and just produces
one from a pass as needed. Making this work well requires making the
object movable, using references instead of overwritten pointers in
a number of places, and other refactorings.
I've also wired it up to the new pass manager and added a RUN line to
a test to exercise it under the new pass manager. This includes basic
printing support much like with other analyses.
But there is a big and somewhat scary change here. Prior to this patch
ScalarEvolution was never *actually* invalidated!!! Re-running the pass
just re-wired up the various other analyses and didn't remove any of the
existing entries in the SCEV caches or clear out anything at all. This
might seem OK as everything in SCEV that can uses ValueHandles to track
updates to the values that serve as SCEV keys. However, this still means
that as we ran SCEV over each function in the module, we kept
accumulating more and more SCEVs into the cache. At the end, we would
have a SCEV cache with every value that we ever needed a SCEV for in the
entire module!!! Yowzers. The releaseMemory routine would dump all of
this, but that isn't realy called during normal runs of the pipeline as
far as I can see.
To make matters worse, there *is* actually a key that we don't update
with value handles -- there is a map keyed off of Loop*s. Because
LoopInfo *does* release its memory from run to run, it is entirely
possible to run SCEV over one function, then over another function, and
then lookup a Loop* from the second function but find an entry inserted
for the first function! Ouch.
To make matters still worse, there are plenty of updates that *don't*
trip a value handle. It seems incredibly unlikely that today GVN or
another pass that invalidates SCEV can update values in *just* such
a way that a subsequent run of SCEV will incorrectly find lookups in
a cache, but it is theoretically possible and would be a nightmare to
debug.
With this refactoring, I've fixed all this by actually destroying and
recreating the ScalarEvolution object from run to run. Technically, this
could increase the amount of malloc traffic we see, but then again it is
also technically correct. ;] I don't actually think we're suffering from
tons of malloc traffic from SCEV because if we were, the fact that we
never clear the memory would seem more likely to have come up as an
actual problem before now. So, I've made the simple fix here. If in fact
there are serious issues with too much allocation and deallocation,
I can work on a clever fix that preserves the allocations (while
clearing the data) between each run, but I'd prefer to do that kind of
optimization with a test case / benchmark that shows why we need such
cleverness (and that can test that we actually make it faster). It's
possible that this will make some things faster by making the SCEV
caches have higher locality (due to being significantly smaller) so
until there is a clear benchmark, I think the simple change is best.
Differential Revision: http://reviews.llvm.org/D12063
llvm-svn: 245193
2015-08-17 10:08:17 +08:00
|
|
|
INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
|
2015-02-20 03:15:04 +08:00
|
|
|
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
|
2015-02-20 03:15:19 +08:00
|
|
|
INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
|
2015-02-20 03:15:04 +08:00
|
|
|
INITIALIZE_PASS_END(LoopAccessAnalysis, LAA_NAME, laa_name, false, true)
|
|
|
|
|
|
|
|
namespace llvm {
|
|
|
|
Pass *createLAAPass() {
|
|
|
|
return new LoopAccessAnalysis();
|
|
|
|
}
|
|
|
|
}
|