2012-06-08 23:38:21 +08:00
|
|
|
//===-- PPCCTRLoops.cpp - Identify and generate CTR loops -----------------===//
|
|
|
|
|
//
|
|
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
|
//
|
|
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
|
// License. See LICENSE.TXT for details.
|
|
|
|
|
//
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
//
|
|
|
|
|
// This pass identifies loops where we can generate the PPC branch instructions
|
|
|
|
|
// that decrement and test the count register (CTR) (bdnz and friends).
|
|
|
|
|
//
|
|
|
|
|
// The pattern that defines the induction variable can changed depending on
|
|
|
|
|
// prior optimizations. For example, the IndVarSimplify phase run by 'opt'
|
|
|
|
|
// normalizes induction variables, and the Loop Strength Reduction pass
|
|
|
|
|
// run by 'llc' may also make changes to the induction variable.
|
|
|
|
|
//
|
|
|
|
|
// Criteria for CTR loops:
|
|
|
|
|
// - Countable loops (w/ ind. var for a trip count)
|
|
|
|
|
// - Try inner-most loops first
|
|
|
|
|
// - No nested CTR loops.
|
|
|
|
|
// - No function calls in loops.
|
|
|
|
|
//
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
2014-01-07 19:48:04 +08:00
|
|
|
#include "PPC.h"
|
2017-06-30 07:28:45 +08:00
|
|
|
#include "PPCSubtarget.h"
|
2014-01-07 19:48:04 +08:00
|
|
|
#include "PPCTargetMachine.h"
|
2017-10-13 00:43:33 +08:00
|
|
|
#include "PPCTargetTransformInfo.h"
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
#include "llvm/ADT/STLExtras.h"
|
2014-01-07 19:48:04 +08:00
|
|
|
#include "llvm/ADT/Statistic.h"
|
2017-10-13 00:43:33 +08:00
|
|
|
#include "llvm/Analysis/AssumptionCache.h"
|
|
|
|
|
#include "llvm/Analysis/CodeMetrics.h"
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
#include "llvm/Analysis/LoopInfo.h"
|
|
|
|
|
#include "llvm/Analysis/ScalarEvolutionExpander.h"
|
2015-01-15 10:16:27 +08:00
|
|
|
#include "llvm/Analysis/TargetLibraryInfo.h"
|
2017-10-13 00:43:33 +08:00
|
|
|
#include "llvm/Analysis/TargetTransformInfo.h"
|
2018-03-22 06:34:23 +08:00
|
|
|
#include "llvm/Analysis/Utils/Local.h"
|
2017-06-30 07:28:45 +08:00
|
|
|
#include "llvm/CodeGen/TargetPassConfig.h"
|
2017-10-13 00:43:33 +08:00
|
|
|
#include "llvm/CodeGen/TargetSchedule.h"
|
2013-01-02 19:36:10 +08:00
|
|
|
#include "llvm/IR/Constants.h"
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
#include "llvm/IR/DerivedTypes.h"
|
2014-01-13 17:26:24 +08:00
|
|
|
#include "llvm/IR/Dominators.h"
|
2013-05-18 17:20:39 +08:00
|
|
|
#include "llvm/IR/InlineAsm.h"
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
#include "llvm/IR/Instructions.h"
|
|
|
|
|
#include "llvm/IR/IntrinsicInst.h"
|
|
|
|
|
#include "llvm/IR/Module.h"
|
2014-03-04 19:17:44 +08:00
|
|
|
#include "llvm/IR/ValueHandle.h"
|
2012-12-04 00:50:05 +08:00
|
|
|
#include "llvm/PassSupport.h"
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
#include "llvm/Support/CommandLine.h"
|
2012-06-08 23:38:21 +08:00
|
|
|
#include "llvm/Support/Debug.h"
|
|
|
|
|
#include "llvm/Support/raw_ostream.h"
|
2017-06-06 19:49:48 +08:00
|
|
|
#include "llvm/Transforms/Scalar.h"
|
2018-03-29 01:44:36 +08:00
|
|
|
#include "llvm/Transforms/Utils.h"
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
|
2013-05-21 04:46:30 +08:00
|
|
|
#include "llvm/Transforms/Utils/LoopUtils.h"
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
|
Add a PPCCTRLoops verification pass
When asserts are enabled, this adds a verification pass for PPC counter-loop
formation. Unfortunately, without sacrificing code quality, there is no better
way of forming counter-based loops except at the (late) IR level. This means
that we need to recognize, at the IR level, anything which might turn into a
function call (or indirect branch). Because this is currently a finite set of
things, and because SelectionDAG lowering is basic-block local, this can be
done. Nevertheless, it is fragile, and failure results in a miscompile. This
verification pass checks that all (reachable) counter-based branches are
dominated by a loop mtctr instruction, and that no instructions in between
clobber the counter register. If these conditions are not satisfied, then an
ICE will be triggered.
In short, this is to help us sleep better at night.
llvm-svn: 182295
2013-05-21 00:08:17 +08:00
|
|
|
#ifndef NDEBUG
|
|
|
|
|
#include "llvm/CodeGen/MachineDominators.h"
|
|
|
|
|
#include "llvm/CodeGen/MachineFunction.h"
|
|
|
|
|
#include "llvm/CodeGen/MachineFunctionPass.h"
|
|
|
|
|
#include "llvm/CodeGen/MachineRegisterInfo.h"
|
|
|
|
|
#endif
|
|
|
|
|
|
2012-06-08 23:38:21 +08:00
|
|
|
using namespace llvm;
|
|
|
|
|
|
2014-04-22 10:41:26 +08:00
|
|
|
#define DEBUG_TYPE "ctrloops"
|
|
|
|
|
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
#ifndef NDEBUG
|
|
|
|
|
static cl::opt<int> CTRLoopLimit("ppc-max-ctrloop", cl::Hidden, cl::init(-1));
|
|
|
|
|
#endif
|
|
|
|
|
|
2017-10-13 00:43:33 +08:00
|
|
|
// The latency of mtctr is only justified if there are more than 4
|
|
|
|
|
// comparisons that will be removed as a result.
|
|
|
|
|
static cl::opt<unsigned>
|
|
|
|
|
SmallCTRLoopThreshold("min-ctr-loop-threshold", cl::init(4), cl::Hidden,
|
|
|
|
|
cl::desc("Loops with a constant trip count smaller than "
|
|
|
|
|
"this value will not use the count register."));
|
|
|
|
|
|
2012-06-08 23:38:21 +08:00
|
|
|
STATISTIC(NumCTRLoops, "Number of loops converted to CTR loops");
|
|
|
|
|
|
2013-02-14 01:40:07 +08:00
|
|
|
namespace llvm {
|
|
|
|
|
void initializePPCCTRLoopsPass(PassRegistry&);
|
Add a PPCCTRLoops verification pass
When asserts are enabled, this adds a verification pass for PPC counter-loop
formation. Unfortunately, without sacrificing code quality, there is no better
way of forming counter-based loops except at the (late) IR level. This means
that we need to recognize, at the IR level, anything which might turn into a
function call (or indirect branch). Because this is currently a finite set of
things, and because SelectionDAG lowering is basic-block local, this can be
done. Nevertheless, it is fragile, and failure results in a miscompile. This
verification pass checks that all (reachable) counter-based branches are
dominated by a loop mtctr instruction, and that no instructions in between
clobber the counter register. If these conditions are not satisfied, then an
ICE will be triggered.
In short, this is to help us sleep better at night.
llvm-svn: 182295
2013-05-21 00:08:17 +08:00
|
|
|
#ifndef NDEBUG
|
|
|
|
|
void initializePPCCTRLoopsVerifyPass(PassRegistry&);
|
|
|
|
|
#endif
|
2013-02-14 01:40:07 +08:00
|
|
|
}
|
|
|
|
|
|
2012-06-08 23:38:21 +08:00
|
|
|
namespace {
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
struct PPCCTRLoops : public FunctionPass {
|
|
|
|
|
|
|
|
|
|
#ifndef NDEBUG
|
|
|
|
|
static int Counter;
|
|
|
|
|
#endif
|
2012-06-08 23:38:21 +08:00
|
|
|
|
|
|
|
|
public:
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
static char ID;
|
2012-06-08 23:38:21 +08:00
|
|
|
|
2017-06-30 07:28:45 +08:00
|
|
|
PPCCTRLoops() : FunctionPass(ID) {
|
2013-02-14 01:40:07 +08:00
|
|
|
initializePPCCTRLoopsPass(*PassRegistry::getPassRegistry());
|
|
|
|
|
}
|
2012-06-08 23:38:21 +08:00
|
|
|
|
2014-04-29 15:57:37 +08:00
|
|
|
bool runOnFunction(Function &F) override;
|
2012-06-08 23:38:21 +08:00
|
|
|
|
2014-04-29 15:57:37 +08:00
|
|
|
void getAnalysisUsage(AnalysisUsage &AU) const override {
|
2015-01-17 22:16:18 +08:00
|
|
|
AU.addRequired<LoopInfoWrapperPass>();
|
|
|
|
|
AU.addPreserved<LoopInfoWrapperPass>();
|
2014-01-13 21:07:17 +08:00
|
|
|
AU.addRequired<DominatorTreeWrapperPass>();
|
|
|
|
|
AU.addPreserved<DominatorTreeWrapperPass>();
|
[PM] Port ScalarEvolution to the new pass manager.
This change makes ScalarEvolution a stand-alone object and just produces
one from a pass as needed. Making this work well requires making the
object movable, using references instead of overwritten pointers in
a number of places, and other refactorings.
I've also wired it up to the new pass manager and added a RUN line to
a test to exercise it under the new pass manager. This includes basic
printing support much like with other analyses.
But there is a big and somewhat scary change here. Prior to this patch
ScalarEvolution was never *actually* invalidated!!! Re-running the pass
just re-wired up the various other analyses and didn't remove any of the
existing entries in the SCEV caches or clear out anything at all. This
might seem OK as everything in SCEV that can uses ValueHandles to track
updates to the values that serve as SCEV keys. However, this still means
that as we ran SCEV over each function in the module, we kept
accumulating more and more SCEVs into the cache. At the end, we would
have a SCEV cache with every value that we ever needed a SCEV for in the
entire module!!! Yowzers. The releaseMemory routine would dump all of
this, but that isn't realy called during normal runs of the pipeline as
far as I can see.
To make matters worse, there *is* actually a key that we don't update
with value handles -- there is a map keyed off of Loop*s. Because
LoopInfo *does* release its memory from run to run, it is entirely
possible to run SCEV over one function, then over another function, and
then lookup a Loop* from the second function but find an entry inserted
for the first function! Ouch.
To make matters still worse, there are plenty of updates that *don't*
trip a value handle. It seems incredibly unlikely that today GVN or
another pass that invalidates SCEV can update values in *just* such
a way that a subsequent run of SCEV will incorrectly find lookups in
a cache, but it is theoretically possible and would be a nightmare to
debug.
With this refactoring, I've fixed all this by actually destroying and
recreating the ScalarEvolution object from run to run. Technically, this
could increase the amount of malloc traffic we see, but then again it is
also technically correct. ;] I don't actually think we're suffering from
tons of malloc traffic from SCEV because if we were, the fact that we
never clear the memory would seem more likely to have come up as an
actual problem before now. So, I've made the simple fix here. If in fact
there are serious issues with too much allocation and deallocation,
I can work on a clever fix that preserves the allocations (while
clearing the data) between each run, but I'd prefer to do that kind of
optimization with a test case / benchmark that shows why we need such
cleverness (and that can test that we actually make it faster). It's
possible that this will make some things faster by making the SCEV
caches have higher locality (due to being significantly smaller) so
until there is a clear benchmark, I think the simple change is best.
Differential Revision: http://reviews.llvm.org/D12063
llvm-svn: 245193
2015-08-17 10:08:17 +08:00
|
|
|
AU.addRequired<ScalarEvolutionWrapperPass>();
|
2017-10-13 00:43:33 +08:00
|
|
|
AU.addRequired<AssumptionCacheTracker>();
|
|
|
|
|
AU.addRequired<TargetTransformInfoWrapperPass>();
|
2012-06-08 23:38:21 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private:
|
2017-06-30 07:28:45 +08:00
|
|
|
bool mightUseCTR(BasicBlock *BB);
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
bool convertToCTRLoop(Loop *L);
|
2013-05-21 00:47:10 +08:00
|
|
|
|
2012-06-08 23:38:21 +08:00
|
|
|
private:
|
2017-06-30 07:28:45 +08:00
|
|
|
const PPCTargetMachine *TM;
|
|
|
|
|
const PPCSubtarget *STI;
|
|
|
|
|
const PPCTargetLowering *TLI;
|
|
|
|
|
const DataLayout *DL;
|
|
|
|
|
const TargetLibraryInfo *LibInfo;
|
2017-10-13 00:43:33 +08:00
|
|
|
const TargetTransformInfo *TTI;
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
LoopInfo *LI;
|
|
|
|
|
ScalarEvolution *SE;
|
|
|
|
|
DominatorTree *DT;
|
2015-12-16 03:40:57 +08:00
|
|
|
bool PreserveLCSSA;
|
2017-10-13 00:43:33 +08:00
|
|
|
TargetSchedModel SchedModel;
|
2012-06-08 23:38:21 +08:00
|
|
|
};
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
|
|
|
|
|
char PPCCTRLoops::ID = 0;
|
|
|
|
|
#ifndef NDEBUG
|
|
|
|
|
int PPCCTRLoops::Counter = 0;
|
|
|
|
|
#endif
|
Add a PPCCTRLoops verification pass
When asserts are enabled, this adds a verification pass for PPC counter-loop
formation. Unfortunately, without sacrificing code quality, there is no better
way of forming counter-based loops except at the (late) IR level. This means
that we need to recognize, at the IR level, anything which might turn into a
function call (or indirect branch). Because this is currently a finite set of
things, and because SelectionDAG lowering is basic-block local, this can be
done. Nevertheless, it is fragile, and failure results in a miscompile. This
verification pass checks that all (reachable) counter-based branches are
dominated by a loop mtctr instruction, and that no instructions in between
clobber the counter register. If these conditions are not satisfied, then an
ICE will be triggered.
In short, this is to help us sleep better at night.
llvm-svn: 182295
2013-05-21 00:08:17 +08:00
|
|
|
|
|
|
|
|
#ifndef NDEBUG
|
|
|
|
|
struct PPCCTRLoopsVerify : public MachineFunctionPass {
|
|
|
|
|
public:
|
|
|
|
|
static char ID;
|
|
|
|
|
|
|
|
|
|
PPCCTRLoopsVerify() : MachineFunctionPass(ID) {
|
|
|
|
|
initializePPCCTRLoopsVerifyPass(*PassRegistry::getPassRegistry());
|
|
|
|
|
}
|
|
|
|
|
|
2014-04-29 15:57:37 +08:00
|
|
|
void getAnalysisUsage(AnalysisUsage &AU) const override {
|
Add a PPCCTRLoops verification pass
When asserts are enabled, this adds a verification pass for PPC counter-loop
formation. Unfortunately, without sacrificing code quality, there is no better
way of forming counter-based loops except at the (late) IR level. This means
that we need to recognize, at the IR level, anything which might turn into a
function call (or indirect branch). Because this is currently a finite set of
things, and because SelectionDAG lowering is basic-block local, this can be
done. Nevertheless, it is fragile, and failure results in a miscompile. This
verification pass checks that all (reachable) counter-based branches are
dominated by a loop mtctr instruction, and that no instructions in between
clobber the counter register. If these conditions are not satisfied, then an
ICE will be triggered.
In short, this is to help us sleep better at night.
llvm-svn: 182295
2013-05-21 00:08:17 +08:00
|
|
|
AU.addRequired<MachineDominatorTree>();
|
|
|
|
|
MachineFunctionPass::getAnalysisUsage(AU);
|
|
|
|
|
}
|
|
|
|
|
|
2014-04-29 15:57:37 +08:00
|
|
|
bool runOnMachineFunction(MachineFunction &MF) override;
|
Add a PPCCTRLoops verification pass
When asserts are enabled, this adds a verification pass for PPC counter-loop
formation. Unfortunately, without sacrificing code quality, there is no better
way of forming counter-based loops except at the (late) IR level. This means
that we need to recognize, at the IR level, anything which might turn into a
function call (or indirect branch). Because this is currently a finite set of
things, and because SelectionDAG lowering is basic-block local, this can be
done. Nevertheless, it is fragile, and failure results in a miscompile. This
verification pass checks that all (reachable) counter-based branches are
dominated by a loop mtctr instruction, and that no instructions in between
clobber the counter register. If these conditions are not satisfied, then an
ICE will be triggered.
In short, this is to help us sleep better at night.
llvm-svn: 182295
2013-05-21 00:08:17 +08:00
|
|
|
|
|
|
|
|
private:
|
|
|
|
|
MachineDominatorTree *MDT;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
char PPCCTRLoopsVerify::ID = 0;
|
|
|
|
|
#endif // NDEBUG
|
2012-06-08 23:38:21 +08:00
|
|
|
} // end anonymous namespace
|
|
|
|
|
|
2013-02-14 01:40:07 +08:00
|
|
|
INITIALIZE_PASS_BEGIN(PPCCTRLoops, "ppc-ctr-loops", "PowerPC CTR Loops",
|
|
|
|
|
false, false)
|
2014-01-13 21:07:17 +08:00
|
|
|
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
|
2015-01-17 22:16:18 +08:00
|
|
|
INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
|
[PM] Port ScalarEvolution to the new pass manager.
This change makes ScalarEvolution a stand-alone object and just produces
one from a pass as needed. Making this work well requires making the
object movable, using references instead of overwritten pointers in
a number of places, and other refactorings.
I've also wired it up to the new pass manager and added a RUN line to
a test to exercise it under the new pass manager. This includes basic
printing support much like with other analyses.
But there is a big and somewhat scary change here. Prior to this patch
ScalarEvolution was never *actually* invalidated!!! Re-running the pass
just re-wired up the various other analyses and didn't remove any of the
existing entries in the SCEV caches or clear out anything at all. This
might seem OK as everything in SCEV that can uses ValueHandles to track
updates to the values that serve as SCEV keys. However, this still means
that as we ran SCEV over each function in the module, we kept
accumulating more and more SCEVs into the cache. At the end, we would
have a SCEV cache with every value that we ever needed a SCEV for in the
entire module!!! Yowzers. The releaseMemory routine would dump all of
this, but that isn't realy called during normal runs of the pipeline as
far as I can see.
To make matters worse, there *is* actually a key that we don't update
with value handles -- there is a map keyed off of Loop*s. Because
LoopInfo *does* release its memory from run to run, it is entirely
possible to run SCEV over one function, then over another function, and
then lookup a Loop* from the second function but find an entry inserted
for the first function! Ouch.
To make matters still worse, there are plenty of updates that *don't*
trip a value handle. It seems incredibly unlikely that today GVN or
another pass that invalidates SCEV can update values in *just* such
a way that a subsequent run of SCEV will incorrectly find lookups in
a cache, but it is theoretically possible and would be a nightmare to
debug.
With this refactoring, I've fixed all this by actually destroying and
recreating the ScalarEvolution object from run to run. Technically, this
could increase the amount of malloc traffic we see, but then again it is
also technically correct. ;] I don't actually think we're suffering from
tons of malloc traffic from SCEV because if we were, the fact that we
never clear the memory would seem more likely to have come up as an
actual problem before now. So, I've made the simple fix here. If in fact
there are serious issues with too much allocation and deallocation,
I can work on a clever fix that preserves the allocations (while
clearing the data) between each run, but I'd prefer to do that kind of
optimization with a test case / benchmark that shows why we need such
cleverness (and that can test that we actually make it faster). It's
possible that this will make some things faster by making the SCEV
caches have higher locality (due to being significantly smaller) so
until there is a clear benchmark, I think the simple change is best.
Differential Revision: http://reviews.llvm.org/D12063
llvm-svn: 245193
2015-08-17 10:08:17 +08:00
|
|
|
INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
|
2013-02-14 01:40:07 +08:00
|
|
|
INITIALIZE_PASS_END(PPCCTRLoops, "ppc-ctr-loops", "PowerPC CTR Loops",
|
|
|
|
|
false, false)
|
2012-06-08 23:38:21 +08:00
|
|
|
|
2017-06-30 07:28:45 +08:00
|
|
|
FunctionPass *llvm::createPPCCTRLoops() { return new PPCCTRLoops(); }
|
2012-06-08 23:38:21 +08:00
|
|
|
|
Add a PPCCTRLoops verification pass
When asserts are enabled, this adds a verification pass for PPC counter-loop
formation. Unfortunately, without sacrificing code quality, there is no better
way of forming counter-based loops except at the (late) IR level. This means
that we need to recognize, at the IR level, anything which might turn into a
function call (or indirect branch). Because this is currently a finite set of
things, and because SelectionDAG lowering is basic-block local, this can be
done. Nevertheless, it is fragile, and failure results in a miscompile. This
verification pass checks that all (reachable) counter-based branches are
dominated by a loop mtctr instruction, and that no instructions in between
clobber the counter register. If these conditions are not satisfied, then an
ICE will be triggered.
In short, this is to help us sleep better at night.
llvm-svn: 182295
2013-05-21 00:08:17 +08:00
|
|
|
#ifndef NDEBUG
|
|
|
|
|
INITIALIZE_PASS_BEGIN(PPCCTRLoopsVerify, "ppc-ctr-loops-verify",
|
|
|
|
|
"PowerPC CTR Loops Verify", false, false)
|
|
|
|
|
INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
|
|
|
|
|
INITIALIZE_PASS_END(PPCCTRLoopsVerify, "ppc-ctr-loops-verify",
|
|
|
|
|
"PowerPC CTR Loops Verify", false, false)
|
|
|
|
|
|
|
|
|
|
FunctionPass *llvm::createPPCCTRLoopsVerify() {
|
|
|
|
|
return new PPCCTRLoopsVerify();
|
|
|
|
|
}
|
|
|
|
|
#endif // NDEBUG
|
|
|
|
|
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
bool PPCCTRLoops::runOnFunction(Function &F) {
|
2016-04-28 03:39:32 +08:00
|
|
|
if (skipFunction(F))
|
|
|
|
|
return false;
|
|
|
|
|
|
2017-06-30 07:28:45 +08:00
|
|
|
auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
|
|
|
|
|
if (!TPC)
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
TM = &TPC->getTM<PPCTargetMachine>();
|
|
|
|
|
STI = TM->getSubtargetImpl(F);
|
|
|
|
|
TLI = STI->getTargetLowering();
|
|
|
|
|
|
2015-01-17 22:16:18 +08:00
|
|
|
LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
|
[PM] Port ScalarEvolution to the new pass manager.
This change makes ScalarEvolution a stand-alone object and just produces
one from a pass as needed. Making this work well requires making the
object movable, using references instead of overwritten pointers in
a number of places, and other refactorings.
I've also wired it up to the new pass manager and added a RUN line to
a test to exercise it under the new pass manager. This includes basic
printing support much like with other analyses.
But there is a big and somewhat scary change here. Prior to this patch
ScalarEvolution was never *actually* invalidated!!! Re-running the pass
just re-wired up the various other analyses and didn't remove any of the
existing entries in the SCEV caches or clear out anything at all. This
might seem OK as everything in SCEV that can uses ValueHandles to track
updates to the values that serve as SCEV keys. However, this still means
that as we ran SCEV over each function in the module, we kept
accumulating more and more SCEVs into the cache. At the end, we would
have a SCEV cache with every value that we ever needed a SCEV for in the
entire module!!! Yowzers. The releaseMemory routine would dump all of
this, but that isn't realy called during normal runs of the pipeline as
far as I can see.
To make matters worse, there *is* actually a key that we don't update
with value handles -- there is a map keyed off of Loop*s. Because
LoopInfo *does* release its memory from run to run, it is entirely
possible to run SCEV over one function, then over another function, and
then lookup a Loop* from the second function but find an entry inserted
for the first function! Ouch.
To make matters still worse, there are plenty of updates that *don't*
trip a value handle. It seems incredibly unlikely that today GVN or
another pass that invalidates SCEV can update values in *just* such
a way that a subsequent run of SCEV will incorrectly find lookups in
a cache, but it is theoretically possible and would be a nightmare to
debug.
With this refactoring, I've fixed all this by actually destroying and
recreating the ScalarEvolution object from run to run. Technically, this
could increase the amount of malloc traffic we see, but then again it is
also technically correct. ;] I don't actually think we're suffering from
tons of malloc traffic from SCEV because if we were, the fact that we
never clear the memory would seem more likely to have come up as an
actual problem before now. So, I've made the simple fix here. If in fact
there are serious issues with too much allocation and deallocation,
I can work on a clever fix that preserves the allocations (while
clearing the data) between each run, but I'd prefer to do that kind of
optimization with a test case / benchmark that shows why we need such
cleverness (and that can test that we actually make it faster). It's
possible that this will make some things faster by making the SCEV
caches have higher locality (due to being significantly smaller) so
until there is a clear benchmark, I think the simple change is best.
Differential Revision: http://reviews.llvm.org/D12063
llvm-svn: 245193
2015-08-17 10:08:17 +08:00
|
|
|
SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
|
2014-01-13 21:07:17 +08:00
|
|
|
DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
|
2017-10-13 00:43:33 +08:00
|
|
|
TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
|
2015-03-05 02:43:29 +08:00
|
|
|
DL = &F.getParent()->getDataLayout();
|
2015-01-15 18:41:28 +08:00
|
|
|
auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
|
|
|
|
|
LibInfo = TLIP ? &TLIP->getTLI() : nullptr;
|
2015-12-16 03:40:57 +08:00
|
|
|
PreserveLCSSA = mustPreserveAnalysisID(LCSSAID);
|
2012-06-08 23:38:21 +08:00
|
|
|
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
bool MadeChange = false;
|
2012-06-08 23:38:21 +08:00
|
|
|
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
for (LoopInfo::iterator I = LI->begin(), E = LI->end();
|
2012-06-08 23:38:21 +08:00
|
|
|
I != E; ++I) {
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
Loop *L = *I;
|
|
|
|
|
if (!L->getParentLoop())
|
|
|
|
|
MadeChange |= convertToCTRLoop(L);
|
2012-06-08 23:38:21 +08:00
|
|
|
}
|
|
|
|
|
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
return MadeChange;
|
2012-06-08 23:38:21 +08:00
|
|
|
}
|
|
|
|
|
|
2014-02-26 04:51:50 +08:00
|
|
|
static bool isLargeIntegerTy(bool Is32Bit, Type *Ty) {
|
|
|
|
|
if (IntegerType *ITy = dyn_cast<IntegerType>(Ty))
|
2014-02-27 04:22:20 +08:00
|
|
|
return ITy->getBitWidth() > (Is32Bit ? 32U : 64U);
|
2014-02-26 04:51:50 +08:00
|
|
|
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
2014-12-28 03:45:38 +08:00
|
|
|
// Determining the address of a TLS variable results in a function call in
|
|
|
|
|
// certain TLS models.
|
2017-06-30 07:28:45 +08:00
|
|
|
static bool memAddrUsesCTR(const PPCTargetMachine &TM, const Value *MemAddr) {
|
2014-12-28 03:45:38 +08:00
|
|
|
const auto *GV = dyn_cast<GlobalValue>(MemAddr);
|
2015-10-29 07:43:00 +08:00
|
|
|
if (!GV) {
|
|
|
|
|
// Recurse to check for constants that refer to TLS global variables.
|
|
|
|
|
if (const auto *CV = dyn_cast<Constant>(MemAddr))
|
|
|
|
|
for (const auto &CO : CV->operands())
|
|
|
|
|
if (memAddrUsesCTR(TM, CO))
|
|
|
|
|
return true;
|
|
|
|
|
|
2014-12-28 03:45:38 +08:00
|
|
|
return false;
|
2015-10-29 07:43:00 +08:00
|
|
|
}
|
|
|
|
|
|
2014-12-28 03:45:38 +08:00
|
|
|
if (!GV->isThreadLocal())
|
|
|
|
|
return false;
|
2017-06-30 07:28:45 +08:00
|
|
|
TLSModel::Model Model = TM.getTLSModel(GV);
|
2014-12-28 03:45:38 +08:00
|
|
|
return Model == TLSModel::GeneralDynamic || Model == TLSModel::LocalDynamic;
|
|
|
|
|
}
|
|
|
|
|
|
2017-06-30 07:28:47 +08:00
|
|
|
// Loop through the inline asm constraints and look for something that clobbers
|
|
|
|
|
// ctr.
|
|
|
|
|
static bool asmClobbersCTR(InlineAsm *IA) {
|
|
|
|
|
InlineAsm::ConstraintInfoVector CIV = IA->ParseConstraints();
|
|
|
|
|
for (unsigned i = 0, ie = CIV.size(); i < ie; ++i) {
|
|
|
|
|
InlineAsm::ConstraintInfo &C = CIV[i];
|
|
|
|
|
if (C.Type != InlineAsm::isInput)
|
|
|
|
|
for (unsigned j = 0, je = C.Codes.size(); j < je; ++j)
|
|
|
|
|
if (StringRef(C.Codes[j]).equals_lower("{ctr}"))
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
2017-06-30 07:28:45 +08:00
|
|
|
bool PPCCTRLoops::mightUseCTR(BasicBlock *BB) {
|
2013-05-17 03:58:38 +08:00
|
|
|
for (BasicBlock::iterator J = BB->begin(), JE = BB->end();
|
|
|
|
|
J != JE; ++J) {
|
|
|
|
|
if (CallInst *CI = dyn_cast<CallInst>(J)) {
|
2017-06-30 07:28:47 +08:00
|
|
|
// Inline ASM is okay, unless it clobbers the ctr register.
|
2013-05-18 17:20:39 +08:00
|
|
|
if (InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledValue())) {
|
2017-08-28 14:47:47 +08:00
|
|
|
if (asmClobbersCTR(IA))
|
|
|
|
|
return true;
|
2013-05-18 17:20:39 +08:00
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
2013-05-17 03:58:38 +08:00
|
|
|
if (Function *F = CI->getCalledFunction()) {
|
|
|
|
|
// Most intrinsics don't become function calls, but some might.
|
|
|
|
|
// sin, cos, exp and log are always calls.
|
2016-03-27 13:40:56 +08:00
|
|
|
unsigned Opcode = 0;
|
2013-05-17 03:58:38 +08:00
|
|
|
if (F->getIntrinsicID() != Intrinsic::not_intrinsic) {
|
|
|
|
|
switch (F->getIntrinsicID()) {
|
|
|
|
|
default: continue;
|
2015-09-09 06:14:58 +08:00
|
|
|
// If we have a call to ppc_is_decremented_ctr_nonzero, or ppc_mtctr
|
|
|
|
|
// we're definitely using CTR.
|
|
|
|
|
case Intrinsic::ppc_is_decremented_ctr_nonzero:
|
2015-09-22 19:15:07 +08:00
|
|
|
case Intrinsic::ppc_mtctr:
|
|
|
|
|
return true;
|
2013-05-17 03:58:38 +08:00
|
|
|
|
|
|
|
|
// VisualStudio defines setjmp as _setjmp
|
|
|
|
|
#if defined(_MSC_VER) && defined(setjmp) && \
|
|
|
|
|
!defined(setjmp_undefined_for_msvc)
|
|
|
|
|
# pragma push_macro("setjmp")
|
|
|
|
|
# undef setjmp
|
|
|
|
|
# define setjmp_undefined_for_msvc
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
case Intrinsic::setjmp:
|
|
|
|
|
|
|
|
|
|
#if defined(_MSC_VER) && defined(setjmp_undefined_for_msvc)
|
|
|
|
|
// let's return it to _setjmp state
|
|
|
|
|
# pragma pop_macro("setjmp")
|
|
|
|
|
# undef setjmp_undefined_for_msvc
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
case Intrinsic::longjmp:
|
2013-07-17 13:35:44 +08:00
|
|
|
|
|
|
|
|
// Exclude eh_sjlj_setjmp; we don't need to exclude eh_sjlj_longjmp
|
|
|
|
|
// because, although it does clobber the counter register, the
|
|
|
|
|
// control can't then return to inside the loop unless there is also
|
|
|
|
|
// an eh_sjlj_setjmp.
|
|
|
|
|
case Intrinsic::eh_sjlj_setjmp:
|
|
|
|
|
|
2013-05-17 03:58:38 +08:00
|
|
|
case Intrinsic::memcpy:
|
|
|
|
|
case Intrinsic::memmove:
|
|
|
|
|
case Intrinsic::memset:
|
|
|
|
|
case Intrinsic::powi:
|
|
|
|
|
case Intrinsic::log:
|
|
|
|
|
case Intrinsic::log2:
|
|
|
|
|
case Intrinsic::log10:
|
|
|
|
|
case Intrinsic::exp:
|
|
|
|
|
case Intrinsic::exp2:
|
|
|
|
|
case Intrinsic::pow:
|
|
|
|
|
case Intrinsic::sin:
|
|
|
|
|
case Intrinsic::cos:
|
|
|
|
|
return true;
|
Add a llvm.copysign intrinsic
This adds a llvm.copysign intrinsic; We already have Libfunc recognition for
copysign (which is turned into the FCOPYSIGN SDAG node). In order to
autovectorize calls to copysign in the loop vectorizer, we need a corresponding
intrinsic as well.
In addition to the expected changes to the language reference, the loop
vectorizer, BasicTTI, and the SDAG builder (the intrinsic is transformed into
an FCOPYSIGN node, just like the function call), this also adds FCOPYSIGN to a
few lists in LegalizeVector{Ops,Types} so that vector copysigns can be
expanded.
In TargetLoweringBase::initActions, I've made the default action for FCOPYSIGN
be Expand for vector types. This seems correct for all in-tree targets, and I
think is the right thing to do because, previously, there was no way to generate
vector-values FCOPYSIGN nodes (and most targets don't specify an action for
vector-typed FCOPYSIGN).
llvm-svn: 188728
2013-08-20 07:35:46 +08:00
|
|
|
case Intrinsic::copysign:
|
|
|
|
|
if (CI->getArgOperand(0)->getType()->getScalarType()->
|
|
|
|
|
isPPC_FP128Ty())
|
|
|
|
|
return true;
|
|
|
|
|
else
|
|
|
|
|
continue; // ISD::FCOPYSIGN is never a library call.
|
2017-04-11 10:03:17 +08:00
|
|
|
case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
|
|
|
|
|
case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
|
|
|
|
|
case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
|
|
|
|
|
case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
|
|
|
|
|
case Intrinsic::rint: Opcode = ISD::FRINT; break;
|
|
|
|
|
case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
|
|
|
|
|
case Intrinsic::round: Opcode = ISD::FROUND; break;
|
|
|
|
|
case Intrinsic::minnum: Opcode = ISD::FMINNUM; break;
|
|
|
|
|
case Intrinsic::maxnum: Opcode = ISD::FMAXNUM; break;
|
|
|
|
|
case Intrinsic::umul_with_overflow: Opcode = ISD::UMULO; break;
|
|
|
|
|
case Intrinsic::smul_with_overflow: Opcode = ISD::SMULO; break;
|
2013-05-17 03:58:38 +08:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// PowerPC does not use [US]DIVREM or other library calls for
|
|
|
|
|
// operations on regular types which are not otherwise library calls
|
|
|
|
|
// (i.e. soft float or atomics). If adapting for targets that do,
|
|
|
|
|
// additional care is required here.
|
|
|
|
|
|
[Analysis] Add LibFunc_ prefix to enums in TargetLibraryInfo. (NFC)
Summary:
The LibFunc::Func enum holds enumerators named for libc functions.
Unfortunately, there are real situations, including libc implementations, where
function names are actually macros (musl uses "#define fopen64 fopen", for
example; any other transitively visible macro would have similar effects).
Strictly speaking, a conforming C++ Standard Library should provide any such
macros as functions instead (via <cstdio>). However, there are some "library"
functions which are not part of the standard, and thus not subject to this
rule (fopen64, for example). So, in order to be both portable and consistent,
the enum should not use the bare function names.
The old enum naming used a namespace LibFunc and an enum Func, with bare
enumerators. This patch changes LibFunc to be an enum with enumerators prefixed
with "LibFFunc_". (Unfortunately, a scoped enum is not sufficient to override
macros.)
There are additional changes required in clang.
Reviewers: rsmith
Subscribers: mehdi_amini, mzolotukhin, nemanjai, llvm-commits
Differential Revision: https://reviews.llvm.org/D28476
llvm-svn: 292848
2017-01-24 07:16:46 +08:00
|
|
|
LibFunc Func;
|
2013-05-17 03:58:38 +08:00
|
|
|
if (!F->hasLocalLinkage() && F->hasName() && LibInfo &&
|
|
|
|
|
LibInfo->getLibFunc(F->getName(), Func) &&
|
|
|
|
|
LibInfo->hasOptimizedCodeGen(Func)) {
|
|
|
|
|
// Non-read-only functions are never treated as intrinsics.
|
|
|
|
|
if (!CI->onlyReadsMemory())
|
|
|
|
|
return true;
|
|
|
|
|
|
|
|
|
|
// Conversion happens only for FP calls.
|
|
|
|
|
if (!CI->getArgOperand(0)->getType()->isFloatingPointTy())
|
|
|
|
|
return true;
|
|
|
|
|
|
|
|
|
|
switch (Func) {
|
|
|
|
|
default: return true;
|
[Analysis] Add LibFunc_ prefix to enums in TargetLibraryInfo. (NFC)
Summary:
The LibFunc::Func enum holds enumerators named for libc functions.
Unfortunately, there are real situations, including libc implementations, where
function names are actually macros (musl uses "#define fopen64 fopen", for
example; any other transitively visible macro would have similar effects).
Strictly speaking, a conforming C++ Standard Library should provide any such
macros as functions instead (via <cstdio>). However, there are some "library"
functions which are not part of the standard, and thus not subject to this
rule (fopen64, for example). So, in order to be both portable and consistent,
the enum should not use the bare function names.
The old enum naming used a namespace LibFunc and an enum Func, with bare
enumerators. This patch changes LibFunc to be an enum with enumerators prefixed
with "LibFFunc_". (Unfortunately, a scoped enum is not sufficient to override
macros.)
There are additional changes required in clang.
Reviewers: rsmith
Subscribers: mehdi_amini, mzolotukhin, nemanjai, llvm-commits
Differential Revision: https://reviews.llvm.org/D28476
llvm-svn: 292848
2017-01-24 07:16:46 +08:00
|
|
|
case LibFunc_copysign:
|
|
|
|
|
case LibFunc_copysignf:
|
2013-05-17 03:58:38 +08:00
|
|
|
continue; // ISD::FCOPYSIGN is never a library call.
|
[Analysis] Add LibFunc_ prefix to enums in TargetLibraryInfo. (NFC)
Summary:
The LibFunc::Func enum holds enumerators named for libc functions.
Unfortunately, there are real situations, including libc implementations, where
function names are actually macros (musl uses "#define fopen64 fopen", for
example; any other transitively visible macro would have similar effects).
Strictly speaking, a conforming C++ Standard Library should provide any such
macros as functions instead (via <cstdio>). However, there are some "library"
functions which are not part of the standard, and thus not subject to this
rule (fopen64, for example). So, in order to be both portable and consistent,
the enum should not use the bare function names.
The old enum naming used a namespace LibFunc and an enum Func, with bare
enumerators. This patch changes LibFunc to be an enum with enumerators prefixed
with "LibFFunc_". (Unfortunately, a scoped enum is not sufficient to override
macros.)
There are additional changes required in clang.
Reviewers: rsmith
Subscribers: mehdi_amini, mzolotukhin, nemanjai, llvm-commits
Differential Revision: https://reviews.llvm.org/D28476
llvm-svn: 292848
2017-01-24 07:16:46 +08:00
|
|
|
case LibFunc_copysignl:
|
2013-08-20 07:35:24 +08:00
|
|
|
return true;
|
[Analysis] Add LibFunc_ prefix to enums in TargetLibraryInfo. (NFC)
Summary:
The LibFunc::Func enum holds enumerators named for libc functions.
Unfortunately, there are real situations, including libc implementations, where
function names are actually macros (musl uses "#define fopen64 fopen", for
example; any other transitively visible macro would have similar effects).
Strictly speaking, a conforming C++ Standard Library should provide any such
macros as functions instead (via <cstdio>). However, there are some "library"
functions which are not part of the standard, and thus not subject to this
rule (fopen64, for example). So, in order to be both portable and consistent,
the enum should not use the bare function names.
The old enum naming used a namespace LibFunc and an enum Func, with bare
enumerators. This patch changes LibFunc to be an enum with enumerators prefixed
with "LibFFunc_". (Unfortunately, a scoped enum is not sufficient to override
macros.)
There are additional changes required in clang.
Reviewers: rsmith
Subscribers: mehdi_amini, mzolotukhin, nemanjai, llvm-commits
Differential Revision: https://reviews.llvm.org/D28476
llvm-svn: 292848
2017-01-24 07:16:46 +08:00
|
|
|
case LibFunc_fabs:
|
|
|
|
|
case LibFunc_fabsf:
|
|
|
|
|
case LibFunc_fabsl:
|
2013-05-17 03:58:38 +08:00
|
|
|
continue; // ISD::FABS is never a library call.
|
[Analysis] Add LibFunc_ prefix to enums in TargetLibraryInfo. (NFC)
Summary:
The LibFunc::Func enum holds enumerators named for libc functions.
Unfortunately, there are real situations, including libc implementations, where
function names are actually macros (musl uses "#define fopen64 fopen", for
example; any other transitively visible macro would have similar effects).
Strictly speaking, a conforming C++ Standard Library should provide any such
macros as functions instead (via <cstdio>). However, there are some "library"
functions which are not part of the standard, and thus not subject to this
rule (fopen64, for example). So, in order to be both portable and consistent,
the enum should not use the bare function names.
The old enum naming used a namespace LibFunc and an enum Func, with bare
enumerators. This patch changes LibFunc to be an enum with enumerators prefixed
with "LibFFunc_". (Unfortunately, a scoped enum is not sufficient to override
macros.)
There are additional changes required in clang.
Reviewers: rsmith
Subscribers: mehdi_amini, mzolotukhin, nemanjai, llvm-commits
Differential Revision: https://reviews.llvm.org/D28476
llvm-svn: 292848
2017-01-24 07:16:46 +08:00
|
|
|
case LibFunc_sqrt:
|
|
|
|
|
case LibFunc_sqrtf:
|
|
|
|
|
case LibFunc_sqrtl:
|
2013-05-17 03:58:38 +08:00
|
|
|
Opcode = ISD::FSQRT; break;
|
[Analysis] Add LibFunc_ prefix to enums in TargetLibraryInfo. (NFC)
Summary:
The LibFunc::Func enum holds enumerators named for libc functions.
Unfortunately, there are real situations, including libc implementations, where
function names are actually macros (musl uses "#define fopen64 fopen", for
example; any other transitively visible macro would have similar effects).
Strictly speaking, a conforming C++ Standard Library should provide any such
macros as functions instead (via <cstdio>). However, there are some "library"
functions which are not part of the standard, and thus not subject to this
rule (fopen64, for example). So, in order to be both portable and consistent,
the enum should not use the bare function names.
The old enum naming used a namespace LibFunc and an enum Func, with bare
enumerators. This patch changes LibFunc to be an enum with enumerators prefixed
with "LibFFunc_". (Unfortunately, a scoped enum is not sufficient to override
macros.)
There are additional changes required in clang.
Reviewers: rsmith
Subscribers: mehdi_amini, mzolotukhin, nemanjai, llvm-commits
Differential Revision: https://reviews.llvm.org/D28476
llvm-svn: 292848
2017-01-24 07:16:46 +08:00
|
|
|
case LibFunc_floor:
|
|
|
|
|
case LibFunc_floorf:
|
|
|
|
|
case LibFunc_floorl:
|
2013-05-17 03:58:38 +08:00
|
|
|
Opcode = ISD::FFLOOR; break;
|
[Analysis] Add LibFunc_ prefix to enums in TargetLibraryInfo. (NFC)
Summary:
The LibFunc::Func enum holds enumerators named for libc functions.
Unfortunately, there are real situations, including libc implementations, where
function names are actually macros (musl uses "#define fopen64 fopen", for
example; any other transitively visible macro would have similar effects).
Strictly speaking, a conforming C++ Standard Library should provide any such
macros as functions instead (via <cstdio>). However, there are some "library"
functions which are not part of the standard, and thus not subject to this
rule (fopen64, for example). So, in order to be both portable and consistent,
the enum should not use the bare function names.
The old enum naming used a namespace LibFunc and an enum Func, with bare
enumerators. This patch changes LibFunc to be an enum with enumerators prefixed
with "LibFFunc_". (Unfortunately, a scoped enum is not sufficient to override
macros.)
There are additional changes required in clang.
Reviewers: rsmith
Subscribers: mehdi_amini, mzolotukhin, nemanjai, llvm-commits
Differential Revision: https://reviews.llvm.org/D28476
llvm-svn: 292848
2017-01-24 07:16:46 +08:00
|
|
|
case LibFunc_nearbyint:
|
|
|
|
|
case LibFunc_nearbyintf:
|
|
|
|
|
case LibFunc_nearbyintl:
|
2013-05-17 03:58:38 +08:00
|
|
|
Opcode = ISD::FNEARBYINT; break;
|
[Analysis] Add LibFunc_ prefix to enums in TargetLibraryInfo. (NFC)
Summary:
The LibFunc::Func enum holds enumerators named for libc functions.
Unfortunately, there are real situations, including libc implementations, where
function names are actually macros (musl uses "#define fopen64 fopen", for
example; any other transitively visible macro would have similar effects).
Strictly speaking, a conforming C++ Standard Library should provide any such
macros as functions instead (via <cstdio>). However, there are some "library"
functions which are not part of the standard, and thus not subject to this
rule (fopen64, for example). So, in order to be both portable and consistent,
the enum should not use the bare function names.
The old enum naming used a namespace LibFunc and an enum Func, with bare
enumerators. This patch changes LibFunc to be an enum with enumerators prefixed
with "LibFFunc_". (Unfortunately, a scoped enum is not sufficient to override
macros.)
There are additional changes required in clang.
Reviewers: rsmith
Subscribers: mehdi_amini, mzolotukhin, nemanjai, llvm-commits
Differential Revision: https://reviews.llvm.org/D28476
llvm-svn: 292848
2017-01-24 07:16:46 +08:00
|
|
|
case LibFunc_ceil:
|
|
|
|
|
case LibFunc_ceilf:
|
|
|
|
|
case LibFunc_ceill:
|
2013-05-17 03:58:38 +08:00
|
|
|
Opcode = ISD::FCEIL; break;
|
[Analysis] Add LibFunc_ prefix to enums in TargetLibraryInfo. (NFC)
Summary:
The LibFunc::Func enum holds enumerators named for libc functions.
Unfortunately, there are real situations, including libc implementations, where
function names are actually macros (musl uses "#define fopen64 fopen", for
example; any other transitively visible macro would have similar effects).
Strictly speaking, a conforming C++ Standard Library should provide any such
macros as functions instead (via <cstdio>). However, there are some "library"
functions which are not part of the standard, and thus not subject to this
rule (fopen64, for example). So, in order to be both portable and consistent,
the enum should not use the bare function names.
The old enum naming used a namespace LibFunc and an enum Func, with bare
enumerators. This patch changes LibFunc to be an enum with enumerators prefixed
with "LibFFunc_". (Unfortunately, a scoped enum is not sufficient to override
macros.)
There are additional changes required in clang.
Reviewers: rsmith
Subscribers: mehdi_amini, mzolotukhin, nemanjai, llvm-commits
Differential Revision: https://reviews.llvm.org/D28476
llvm-svn: 292848
2017-01-24 07:16:46 +08:00
|
|
|
case LibFunc_rint:
|
|
|
|
|
case LibFunc_rintf:
|
|
|
|
|
case LibFunc_rintl:
|
2013-05-17 03:58:38 +08:00
|
|
|
Opcode = ISD::FRINT; break;
|
[Analysis] Add LibFunc_ prefix to enums in TargetLibraryInfo. (NFC)
Summary:
The LibFunc::Func enum holds enumerators named for libc functions.
Unfortunately, there are real situations, including libc implementations, where
function names are actually macros (musl uses "#define fopen64 fopen", for
example; any other transitively visible macro would have similar effects).
Strictly speaking, a conforming C++ Standard Library should provide any such
macros as functions instead (via <cstdio>). However, there are some "library"
functions which are not part of the standard, and thus not subject to this
rule (fopen64, for example). So, in order to be both portable and consistent,
the enum should not use the bare function names.
The old enum naming used a namespace LibFunc and an enum Func, with bare
enumerators. This patch changes LibFunc to be an enum with enumerators prefixed
with "LibFFunc_". (Unfortunately, a scoped enum is not sufficient to override
macros.)
There are additional changes required in clang.
Reviewers: rsmith
Subscribers: mehdi_amini, mzolotukhin, nemanjai, llvm-commits
Differential Revision: https://reviews.llvm.org/D28476
llvm-svn: 292848
2017-01-24 07:16:46 +08:00
|
|
|
case LibFunc_round:
|
|
|
|
|
case LibFunc_roundf:
|
|
|
|
|
case LibFunc_roundl:
|
2013-08-08 06:49:12 +08:00
|
|
|
Opcode = ISD::FROUND; break;
|
[Analysis] Add LibFunc_ prefix to enums in TargetLibraryInfo. (NFC)
Summary:
The LibFunc::Func enum holds enumerators named for libc functions.
Unfortunately, there are real situations, including libc implementations, where
function names are actually macros (musl uses "#define fopen64 fopen", for
example; any other transitively visible macro would have similar effects).
Strictly speaking, a conforming C++ Standard Library should provide any such
macros as functions instead (via <cstdio>). However, there are some "library"
functions which are not part of the standard, and thus not subject to this
rule (fopen64, for example). So, in order to be both portable and consistent,
the enum should not use the bare function names.
The old enum naming used a namespace LibFunc and an enum Func, with bare
enumerators. This patch changes LibFunc to be an enum with enumerators prefixed
with "LibFFunc_". (Unfortunately, a scoped enum is not sufficient to override
macros.)
There are additional changes required in clang.
Reviewers: rsmith
Subscribers: mehdi_amini, mzolotukhin, nemanjai, llvm-commits
Differential Revision: https://reviews.llvm.org/D28476
llvm-svn: 292848
2017-01-24 07:16:46 +08:00
|
|
|
case LibFunc_trunc:
|
|
|
|
|
case LibFunc_truncf:
|
|
|
|
|
case LibFunc_truncl:
|
2013-05-17 03:58:38 +08:00
|
|
|
Opcode = ISD::FTRUNC; break;
|
[Analysis] Add LibFunc_ prefix to enums in TargetLibraryInfo. (NFC)
Summary:
The LibFunc::Func enum holds enumerators named for libc functions.
Unfortunately, there are real situations, including libc implementations, where
function names are actually macros (musl uses "#define fopen64 fopen", for
example; any other transitively visible macro would have similar effects).
Strictly speaking, a conforming C++ Standard Library should provide any such
macros as functions instead (via <cstdio>). However, there are some "library"
functions which are not part of the standard, and thus not subject to this
rule (fopen64, for example). So, in order to be both portable and consistent,
the enum should not use the bare function names.
The old enum naming used a namespace LibFunc and an enum Func, with bare
enumerators. This patch changes LibFunc to be an enum with enumerators prefixed
with "LibFFunc_". (Unfortunately, a scoped enum is not sufficient to override
macros.)
There are additional changes required in clang.
Reviewers: rsmith
Subscribers: mehdi_amini, mzolotukhin, nemanjai, llvm-commits
Differential Revision: https://reviews.llvm.org/D28476
llvm-svn: 292848
2017-01-24 07:16:46 +08:00
|
|
|
case LibFunc_fmin:
|
|
|
|
|
case LibFunc_fminf:
|
|
|
|
|
case LibFunc_fminl:
|
2016-03-27 13:40:56 +08:00
|
|
|
Opcode = ISD::FMINNUM; break;
|
[Analysis] Add LibFunc_ prefix to enums in TargetLibraryInfo. (NFC)
Summary:
The LibFunc::Func enum holds enumerators named for libc functions.
Unfortunately, there are real situations, including libc implementations, where
function names are actually macros (musl uses "#define fopen64 fopen", for
example; any other transitively visible macro would have similar effects).
Strictly speaking, a conforming C++ Standard Library should provide any such
macros as functions instead (via <cstdio>). However, there are some "library"
functions which are not part of the standard, and thus not subject to this
rule (fopen64, for example). So, in order to be both portable and consistent,
the enum should not use the bare function names.
The old enum naming used a namespace LibFunc and an enum Func, with bare
enumerators. This patch changes LibFunc to be an enum with enumerators prefixed
with "LibFFunc_". (Unfortunately, a scoped enum is not sufficient to override
macros.)
There are additional changes required in clang.
Reviewers: rsmith
Subscribers: mehdi_amini, mzolotukhin, nemanjai, llvm-commits
Differential Revision: https://reviews.llvm.org/D28476
llvm-svn: 292848
2017-01-24 07:16:46 +08:00
|
|
|
case LibFunc_fmax:
|
|
|
|
|
case LibFunc_fmaxf:
|
|
|
|
|
case LibFunc_fmaxl:
|
2016-03-27 13:40:56 +08:00
|
|
|
Opcode = ISD::FMAXNUM; break;
|
2013-05-17 03:58:38 +08:00
|
|
|
}
|
2016-03-27 13:40:56 +08:00
|
|
|
}
|
2013-05-17 03:58:38 +08:00
|
|
|
|
2016-03-27 13:40:56 +08:00
|
|
|
if (Opcode) {
|
2018-01-09 11:03:41 +08:00
|
|
|
EVT EVTy =
|
|
|
|
|
TLI->getValueType(*DL, CI->getArgOperand(0)->getType(), true);
|
|
|
|
|
|
|
|
|
|
if (EVTy == MVT::Other)
|
2013-05-17 03:58:38 +08:00
|
|
|
return true;
|
2015-09-22 19:14:39 +08:00
|
|
|
|
2018-01-09 11:03:41 +08:00
|
|
|
if (TLI->isOperationLegalOrCustom(Opcode, EVTy))
|
2013-05-17 03:58:38 +08:00
|
|
|
continue;
|
2018-01-09 11:03:41 +08:00
|
|
|
else if (EVTy.isVector() &&
|
|
|
|
|
TLI->isOperationLegalOrCustom(Opcode, EVTy.getScalarType()))
|
2013-05-17 03:58:38 +08:00
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
} else if (isa<BinaryOperator>(J) &&
|
|
|
|
|
J->getType()->getScalarType()->isPPC_FP128Ty()) {
|
|
|
|
|
// Most operations on ppc_f128 values become calls.
|
|
|
|
|
return true;
|
|
|
|
|
} else if (isa<UIToFPInst>(J) || isa<SIToFPInst>(J) ||
|
|
|
|
|
isa<FPToUIInst>(J) || isa<FPToSIInst>(J)) {
|
|
|
|
|
CastInst *CI = cast<CastInst>(J);
|
|
|
|
|
if (CI->getSrcTy()->getScalarType()->isPPC_FP128Ty() ||
|
|
|
|
|
CI->getDestTy()->getScalarType()->isPPC_FP128Ty() ||
|
2017-06-30 07:28:45 +08:00
|
|
|
isLargeIntegerTy(!TM->isPPC64(), CI->getSrcTy()->getScalarType()) ||
|
|
|
|
|
isLargeIntegerTy(!TM->isPPC64(), CI->getDestTy()->getScalarType()))
|
2013-05-17 03:58:38 +08:00
|
|
|
return true;
|
2017-06-30 07:28:45 +08:00
|
|
|
} else if (isLargeIntegerTy(!TM->isPPC64(),
|
2014-02-26 04:51:50 +08:00
|
|
|
J->getType()->getScalarType()) &&
|
2013-06-08 06:16:19 +08:00
|
|
|
(J->getOpcode() == Instruction::UDiv ||
|
|
|
|
|
J->getOpcode() == Instruction::SDiv ||
|
|
|
|
|
J->getOpcode() == Instruction::URem ||
|
|
|
|
|
J->getOpcode() == Instruction::SRem)) {
|
|
|
|
|
return true;
|
2017-06-30 07:28:45 +08:00
|
|
|
} else if (!TM->isPPC64() &&
|
2014-05-12 00:23:29 +08:00
|
|
|
isLargeIntegerTy(false, J->getType()->getScalarType()) &&
|
|
|
|
|
(J->getOpcode() == Instruction::Shl ||
|
|
|
|
|
J->getOpcode() == Instruction::AShr ||
|
|
|
|
|
J->getOpcode() == Instruction::LShr)) {
|
|
|
|
|
// Only on PPC32, for 128-bit integers (specifically not 64-bit
|
|
|
|
|
// integers), these might be runtime calls.
|
|
|
|
|
return true;
|
2013-05-17 03:58:38 +08:00
|
|
|
} else if (isa<IndirectBrInst>(J) || isa<InvokeInst>(J)) {
|
|
|
|
|
// On PowerPC, indirect jumps use the counter register.
|
|
|
|
|
return true;
|
|
|
|
|
} else if (SwitchInst *SI = dyn_cast<SwitchInst>(J)) {
|
2014-09-03 06:28:02 +08:00
|
|
|
if (SI->getNumCases() + 1 >= (unsigned)TLI->getMinimumJumpTableEntries())
|
2013-05-17 03:58:38 +08:00
|
|
|
return true;
|
|
|
|
|
}
|
2016-03-18 01:11:33 +08:00
|
|
|
|
2018-02-22 11:02:41 +08:00
|
|
|
// FREM is always a call.
|
|
|
|
|
if (J->getOpcode() == Instruction::FRem)
|
|
|
|
|
return true;
|
|
|
|
|
|
2017-06-30 07:28:45 +08:00
|
|
|
if (STI->useSoftFloat()) {
|
2016-03-18 01:11:33 +08:00
|
|
|
switch(J->getOpcode()) {
|
|
|
|
|
case Instruction::FAdd:
|
|
|
|
|
case Instruction::FSub:
|
|
|
|
|
case Instruction::FMul:
|
|
|
|
|
case Instruction::FDiv:
|
|
|
|
|
case Instruction::FPTrunc:
|
|
|
|
|
case Instruction::FPExt:
|
|
|
|
|
case Instruction::FPToUI:
|
|
|
|
|
case Instruction::FPToSI:
|
|
|
|
|
case Instruction::UIToFP:
|
|
|
|
|
case Instruction::SIToFP:
|
|
|
|
|
case Instruction::FCmp:
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-12-28 03:45:38 +08:00
|
|
|
for (Value *Operand : J->operands())
|
2017-06-30 07:28:45 +08:00
|
|
|
if (memAddrUsesCTR(*TM, Operand))
|
2014-12-28 03:45:38 +08:00
|
|
|
return true;
|
2013-05-17 03:58:38 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return false;
|
|
|
|
|
}
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
bool PPCCTRLoops::convertToCTRLoop(Loop *L) {
|
|
|
|
|
bool MadeChange = false;
|
2012-06-08 23:38:21 +08:00
|
|
|
|
2017-10-13 00:43:33 +08:00
|
|
|
// Do not convert small short loops to CTR loop.
|
|
|
|
|
unsigned ConstTripCount = SE->getSmallConstantTripCount(L);
|
|
|
|
|
if (ConstTripCount && ConstTripCount < SmallCTRLoopThreshold) {
|
|
|
|
|
SmallPtrSet<const Value *, 32> EphValues;
|
|
|
|
|
auto AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
|
|
|
|
|
*L->getHeader()->getParent());
|
|
|
|
|
CodeMetrics::collectEphemeralValues(L, &AC, EphValues);
|
|
|
|
|
CodeMetrics Metrics;
|
|
|
|
|
for (BasicBlock *BB : L->blocks())
|
|
|
|
|
Metrics.analyzeBasicBlock(BB, *TTI, EphValues);
|
|
|
|
|
// 6 is an approximate latency for the mtctr instruction.
|
|
|
|
|
if (Metrics.NumInsts <= (6 * SchedModel.getIssueWidth()))
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
// Process nested loops first.
|
|
|
|
|
for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) {
|
|
|
|
|
MadeChange |= convertToCTRLoop(*I);
|
2015-09-09 06:14:58 +08:00
|
|
|
DEBUG(dbgs() << "Nested loop converted\n");
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
}
|
2013-03-19 01:40:27 +08:00
|
|
|
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
// If a nested loop has been converted, then we can't convert this loop.
|
|
|
|
|
if (MadeChange)
|
|
|
|
|
return MadeChange;
|
|
|
|
|
|
|
|
|
|
#ifndef NDEBUG
|
|
|
|
|
// Stop trying after reaching the limit (if any).
|
|
|
|
|
int Limit = CTRLoopLimit;
|
|
|
|
|
if (Limit >= 0) {
|
|
|
|
|
if (Counter >= CTRLoopLimit)
|
|
|
|
|
return false;
|
|
|
|
|
Counter++;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
// We don't want to spill/restore the counter register, and so we don't
|
|
|
|
|
// want to use the counter register if the loop contains calls.
|
|
|
|
|
for (Loop::block_iterator I = L->block_begin(), IE = L->block_end();
|
2013-05-17 03:58:38 +08:00
|
|
|
I != IE; ++I)
|
2017-06-30 07:28:45 +08:00
|
|
|
if (mightUseCTR(*I))
|
2013-05-17 03:58:38 +08:00
|
|
|
return MadeChange;
|
2012-06-08 23:38:21 +08:00
|
|
|
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
SmallVector<BasicBlock*, 4> ExitingBlocks;
|
|
|
|
|
L->getExitingBlocks(ExitingBlocks);
|
|
|
|
|
|
2018-02-05 20:25:29 +08:00
|
|
|
// If there is an exit edge known to be frequently taken,
|
|
|
|
|
// we should not transform this loop.
|
|
|
|
|
for (auto &BB : ExitingBlocks) {
|
|
|
|
|
Instruction *TI = BB->getTerminator();
|
|
|
|
|
if (!TI) continue;
|
|
|
|
|
|
|
|
|
|
if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
|
2018-02-06 19:34:16 +08:00
|
|
|
uint64_t TrueWeight = 0, FalseWeight = 0;
|
2018-02-05 20:25:29 +08:00
|
|
|
if (!BI->isConditional() ||
|
|
|
|
|
!BI->extractProfMetadata(TrueWeight, FalseWeight))
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
// If the exit path is more frequent than the loop path,
|
|
|
|
|
// we return here without further analysis for this loop.
|
|
|
|
|
bool TrueIsExit = !L->contains(BI->getSuccessor(0));
|
|
|
|
|
if (( TrueIsExit && FalseWeight < TrueWeight) ||
|
|
|
|
|
(!TrueIsExit && FalseWeight > TrueWeight))
|
|
|
|
|
return MadeChange;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-04-25 13:30:21 +08:00
|
|
|
BasicBlock *CountedExitBlock = nullptr;
|
|
|
|
|
const SCEV *ExitCount = nullptr;
|
|
|
|
|
BranchInst *CountedExitBranch = nullptr;
|
2013-07-04 09:31:24 +08:00
|
|
|
for (SmallVectorImpl<BasicBlock *>::iterator I = ExitingBlocks.begin(),
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
IE = ExitingBlocks.end(); I != IE; ++I) {
|
|
|
|
|
const SCEV *EC = SE->getExitCount(L, *I);
|
|
|
|
|
DEBUG(dbgs() << "Exit Count for " << *L << " from block " <<
|
|
|
|
|
(*I)->getName() << ": " << *EC << "\n");
|
|
|
|
|
if (isa<SCEVCouldNotCompute>(EC))
|
|
|
|
|
continue;
|
|
|
|
|
if (const SCEVConstant *ConstEC = dyn_cast<SCEVConstant>(EC)) {
|
|
|
|
|
if (ConstEC->getValue()->isZero())
|
|
|
|
|
continue;
|
|
|
|
|
} else if (!SE->isLoopInvariant(EC, L))
|
|
|
|
|
continue;
|
|
|
|
|
|
2017-06-30 07:28:45 +08:00
|
|
|
if (SE->getTypeSizeInBits(EC->getType()) > (TM->isPPC64() ? 64 : 32))
|
2013-07-02 03:34:59 +08:00
|
|
|
continue;
|
|
|
|
|
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
// We now have a loop-invariant count of loop iterations (which is not the
|
|
|
|
|
// constant zero) for which we know that this loop will not exit via this
|
|
|
|
|
// exisiting block.
|
|
|
|
|
|
|
|
|
|
// We need to make sure that this block will run on every loop iteration.
|
|
|
|
|
// For this to be true, we must dominate all blocks with backedges. Such
|
|
|
|
|
// blocks are in-loop predecessors to the header block.
|
|
|
|
|
bool NotAlways = false;
|
|
|
|
|
for (pred_iterator PI = pred_begin(L->getHeader()),
|
|
|
|
|
PIE = pred_end(L->getHeader()); PI != PIE; ++PI) {
|
|
|
|
|
if (!L->contains(*PI))
|
|
|
|
|
continue;
|
2012-06-08 23:38:21 +08:00
|
|
|
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
if (!DT->dominates(*I, *PI)) {
|
|
|
|
|
NotAlways = true;
|
|
|
|
|
break;
|
2012-06-08 23:38:21 +08:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
if (NotAlways)
|
|
|
|
|
continue;
|
2012-06-08 23:38:21 +08:00
|
|
|
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
// Make sure this blocks ends with a conditional branch.
|
|
|
|
|
Instruction *TI = (*I)->getTerminator();
|
|
|
|
|
if (!TI)
|
|
|
|
|
continue;
|
2012-06-08 23:38:21 +08:00
|
|
|
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
|
|
|
|
|
if (!BI->isConditional())
|
2012-06-08 23:38:21 +08:00
|
|
|
continue;
|
|
|
|
|
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
CountedExitBranch = BI;
|
|
|
|
|
} else
|
|
|
|
|
continue;
|
2012-06-08 23:38:21 +08:00
|
|
|
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
// Note that this block may not be the loop latch block, even if the loop
|
|
|
|
|
// has a latch block.
|
|
|
|
|
CountedExitBlock = *I;
|
|
|
|
|
ExitCount = EC;
|
|
|
|
|
break;
|
2012-06-08 23:38:21 +08:00
|
|
|
}
|
|
|
|
|
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
if (!CountedExitBlock)
|
|
|
|
|
return MadeChange;
|
|
|
|
|
|
|
|
|
|
BasicBlock *Preheader = L->getLoopPreheader();
|
2013-05-17 03:58:38 +08:00
|
|
|
|
|
|
|
|
// If we don't have a preheader, then insert one. If we already have a
|
|
|
|
|
// preheader, then we can use it (except if the preheader contains a use of
|
|
|
|
|
// the CTR register because some such uses might be reordered by the
|
|
|
|
|
// selection DAG after the mtctr instruction).
|
2017-06-30 07:28:45 +08:00
|
|
|
if (!Preheader || mightUseCTR(Preheader))
|
2015-12-16 03:40:57 +08:00
|
|
|
Preheader = InsertPreheaderForLoop(L, DT, LI, PreserveLCSSA);
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
if (!Preheader)
|
|
|
|
|
return MadeChange;
|
|
|
|
|
|
|
|
|
|
DEBUG(dbgs() << "Preheader for exit count: " << Preheader->getName() << "\n");
|
|
|
|
|
|
|
|
|
|
// Insert the count into the preheader and replace the condition used by the
|
|
|
|
|
// selected branch.
|
|
|
|
|
MadeChange = true;
|
|
|
|
|
|
2017-06-30 07:28:45 +08:00
|
|
|
SCEVExpander SCEVE(*SE, *DL, "loopcnt");
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
LLVMContext &C = SE->getContext();
|
2017-06-30 07:28:45 +08:00
|
|
|
Type *CountType = TM->isPPC64() ? Type::getInt64Ty(C) : Type::getInt32Ty(C);
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
if (!ExitCount->getType()->isPointerTy() &&
|
|
|
|
|
ExitCount->getType() != CountType)
|
|
|
|
|
ExitCount = SE->getZeroExtendExpr(ExitCount, CountType);
|
2015-09-23 09:59:04 +08:00
|
|
|
ExitCount = SE->getAddExpr(ExitCount, SE->getOne(CountType));
|
2015-09-22 19:13:55 +08:00
|
|
|
Value *ECValue =
|
|
|
|
|
SCEVE.expandCodeFor(ExitCount, CountType, Preheader->getTerminator());
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
|
|
|
|
|
IRBuilder<> CountBuilder(Preheader->getTerminator());
|
|
|
|
|
Module *M = Preheader->getParent()->getParent();
|
|
|
|
|
Value *MTCTRFunc = Intrinsic::getDeclaration(M, Intrinsic::ppc_mtctr,
|
|
|
|
|
CountType);
|
|
|
|
|
CountBuilder.CreateCall(MTCTRFunc, ECValue);
|
|
|
|
|
|
|
|
|
|
IRBuilder<> CondBuilder(CountedExitBranch);
|
|
|
|
|
Value *DecFunc =
|
|
|
|
|
Intrinsic::getDeclaration(M, Intrinsic::ppc_is_decremented_ctr_nonzero);
|
2015-05-19 06:13:54 +08:00
|
|
|
Value *NewCond = CondBuilder.CreateCall(DecFunc, {});
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
Value *OldCond = CountedExitBranch->getCondition();
|
|
|
|
|
CountedExitBranch->setCondition(NewCond);
|
|
|
|
|
|
|
|
|
|
// The false branch must exit the loop.
|
|
|
|
|
if (!L->contains(CountedExitBranch->getSuccessor(0)))
|
|
|
|
|
CountedExitBranch->swapSuccessors();
|
|
|
|
|
|
|
|
|
|
// The old condition may be dead now, and may have even created a dead PHI
|
|
|
|
|
// (the original induction variable).
|
|
|
|
|
RecursivelyDeleteTriviallyDeadInstructions(OldCond);
|
2017-07-06 01:57:57 +08:00
|
|
|
// Run through the basic blocks of the loop and see if any of them have dead
|
|
|
|
|
// PHIs that can be removed.
|
|
|
|
|
for (auto I : L->blocks())
|
|
|
|
|
DeleteDeadPHIs(I);
|
2013-03-19 01:40:44 +08:00
|
|
|
|
Implement PPC counter loops as a late IR-level pass
The old PPCCTRLoops pass, like the Hexagon pass version from which it was
derived, could only handle some simple loops in canonical form. We cannot
directly adapt the new Hexagon hardware loops pass, however, because the
Hexagon pass contains a fundamental assumption that non-constant-trip-count
loops will contain a guard, and this is not always true (the result being that
incorrect negative counts can be generated). With this commit, we replace the
pass with a late IR-level pass which makes use of SE to calculate the
backedge-taken counts and safely generate the loop-count expressions (including
any necessary max() parts). This IR level pass inserts custom intrinsics that
are lowered into the desired decrement-and-branch instructions.
The most fragile part of this new implementation is that interfering uses of
the counter register must be detected on the IR level (and, on PPC, this also
includes any indirect branches in addition to function calls). Also, to make
all of this work, we need a variant of the mtctr instruction that is marked
as having side effects. Without this, machine-code level CSE, DCE, etc.
illegally transform the resulting code. Hopefully, this can be improved
in the future.
This new pass is smaller than the original (and much smaller than the new
Hexagon hardware loops pass), and can handle many additional cases correctly.
In addition, the preheader-creation code has been copied from LoopSimplify, and
after we decide on where it belongs, this code will be refactored so that it
can be explicitly shared (making this implementation even smaller).
The new test-case files ctrloop-{le,lt,ne}.ll have been adapted from tests for
the new Hexagon pass. There are a few classes of loops that this pass does not
transform (noted by FIXMEs in the files), but these deficiencies can be
addressed within the SE infrastructure (thus helping many other passes as well).
llvm-svn: 181927
2013-05-16 05:37:41 +08:00
|
|
|
++NumCTRLoops;
|
|
|
|
|
return MadeChange;
|
|
|
|
|
}
|
2013-03-19 01:40:44 +08:00
|
|
|
|
Add a PPCCTRLoops verification pass
When asserts are enabled, this adds a verification pass for PPC counter-loop
formation. Unfortunately, without sacrificing code quality, there is no better
way of forming counter-based loops except at the (late) IR level. This means
that we need to recognize, at the IR level, anything which might turn into a
function call (or indirect branch). Because this is currently a finite set of
things, and because SelectionDAG lowering is basic-block local, this can be
done. Nevertheless, it is fragile, and failure results in a miscompile. This
verification pass checks that all (reachable) counter-based branches are
dominated by a loop mtctr instruction, and that no instructions in between
clobber the counter register. If these conditions are not satisfied, then an
ICE will be triggered.
In short, this is to help us sleep better at night.
llvm-svn: 182295
2013-05-21 00:08:17 +08:00
|
|
|
#ifndef NDEBUG
|
2016-07-27 21:24:16 +08:00
|
|
|
static bool clobbersCTR(const MachineInstr &MI) {
|
|
|
|
|
for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
|
|
|
|
|
const MachineOperand &MO = MI.getOperand(i);
|
Add a PPCCTRLoops verification pass
When asserts are enabled, this adds a verification pass for PPC counter-loop
formation. Unfortunately, without sacrificing code quality, there is no better
way of forming counter-based loops except at the (late) IR level. This means
that we need to recognize, at the IR level, anything which might turn into a
function call (or indirect branch). Because this is currently a finite set of
things, and because SelectionDAG lowering is basic-block local, this can be
done. Nevertheless, it is fragile, and failure results in a miscompile. This
verification pass checks that all (reachable) counter-based branches are
dominated by a loop mtctr instruction, and that no instructions in between
clobber the counter register. If these conditions are not satisfied, then an
ICE will be triggered.
In short, this is to help us sleep better at night.
llvm-svn: 182295
2013-05-21 00:08:17 +08:00
|
|
|
if (MO.isReg()) {
|
|
|
|
|
if (MO.isDef() && (MO.getReg() == PPC::CTR || MO.getReg() == PPC::CTR8))
|
|
|
|
|
return true;
|
|
|
|
|
} else if (MO.isRegMask()) {
|
|
|
|
|
if (MO.clobbersPhysReg(PPC::CTR) || MO.clobbersPhysReg(PPC::CTR8))
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static bool verifyCTRBranch(MachineBasicBlock *MBB,
|
|
|
|
|
MachineBasicBlock::iterator I) {
|
|
|
|
|
MachineBasicBlock::iterator BI = I;
|
|
|
|
|
SmallSet<MachineBasicBlock *, 16> Visited;
|
|
|
|
|
SmallVector<MachineBasicBlock *, 8> Preds;
|
|
|
|
|
bool CheckPreds;
|
|
|
|
|
|
|
|
|
|
if (I == MBB->begin()) {
|
|
|
|
|
Visited.insert(MBB);
|
|
|
|
|
goto queue_preds;
|
|
|
|
|
} else
|
|
|
|
|
--I;
|
|
|
|
|
|
|
|
|
|
check_block:
|
|
|
|
|
Visited.insert(MBB);
|
|
|
|
|
if (I == MBB->end())
|
|
|
|
|
goto queue_preds;
|
|
|
|
|
|
|
|
|
|
CheckPreds = true;
|
|
|
|
|
for (MachineBasicBlock::iterator IE = MBB->begin();; --I) {
|
|
|
|
|
unsigned Opc = I->getOpcode();
|
2013-05-21 00:08:37 +08:00
|
|
|
if (Opc == PPC::MTCTRloop || Opc == PPC::MTCTR8loop) {
|
Add a PPCCTRLoops verification pass
When asserts are enabled, this adds a verification pass for PPC counter-loop
formation. Unfortunately, without sacrificing code quality, there is no better
way of forming counter-based loops except at the (late) IR level. This means
that we need to recognize, at the IR level, anything which might turn into a
function call (or indirect branch). Because this is currently a finite set of
things, and because SelectionDAG lowering is basic-block local, this can be
done. Nevertheless, it is fragile, and failure results in a miscompile. This
verification pass checks that all (reachable) counter-based branches are
dominated by a loop mtctr instruction, and that no instructions in between
clobber the counter register. If these conditions are not satisfied, then an
ICE will be triggered.
In short, this is to help us sleep better at night.
llvm-svn: 182295
2013-05-21 00:08:17 +08:00
|
|
|
CheckPreds = false;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
2016-07-27 21:24:16 +08:00
|
|
|
if (I != BI && clobbersCTR(*I)) {
|
2017-12-05 01:18:51 +08:00
|
|
|
DEBUG(dbgs() << printMBBReference(*MBB) << " (" << MBB->getFullName()
|
|
|
|
|
<< ") instruction " << *I << " clobbers CTR, invalidating "
|
|
|
|
|
<< printMBBReference(*BI->getParent()) << " ("
|
|
|
|
|
<< BI->getParent()->getFullName() << ") instruction " << *BI
|
|
|
|
|
<< "\n");
|
Add a PPCCTRLoops verification pass
When asserts are enabled, this adds a verification pass for PPC counter-loop
formation. Unfortunately, without sacrificing code quality, there is no better
way of forming counter-based loops except at the (late) IR level. This means
that we need to recognize, at the IR level, anything which might turn into a
function call (or indirect branch). Because this is currently a finite set of
things, and because SelectionDAG lowering is basic-block local, this can be
done. Nevertheless, it is fragile, and failure results in a miscompile. This
verification pass checks that all (reachable) counter-based branches are
dominated by a loop mtctr instruction, and that no instructions in between
clobber the counter register. If these conditions are not satisfied, then an
ICE will be triggered.
In short, this is to help us sleep better at night.
llvm-svn: 182295
2013-05-21 00:08:17 +08:00
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (I == IE)
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!CheckPreds && Preds.empty())
|
|
|
|
|
return true;
|
|
|
|
|
|
|
|
|
|
if (CheckPreds) {
|
|
|
|
|
queue_preds:
|
|
|
|
|
if (MachineFunction::iterator(MBB) == MBB->getParent()->begin()) {
|
2017-12-05 01:18:51 +08:00
|
|
|
DEBUG(dbgs() << "Unable to find a MTCTR instruction for "
|
|
|
|
|
<< printMBBReference(*BI->getParent()) << " ("
|
|
|
|
|
<< BI->getParent()->getFullName() << ") instruction " << *BI
|
|
|
|
|
<< "\n");
|
Add a PPCCTRLoops verification pass
When asserts are enabled, this adds a verification pass for PPC counter-loop
formation. Unfortunately, without sacrificing code quality, there is no better
way of forming counter-based loops except at the (late) IR level. This means
that we need to recognize, at the IR level, anything which might turn into a
function call (or indirect branch). Because this is currently a finite set of
things, and because SelectionDAG lowering is basic-block local, this can be
done. Nevertheless, it is fragile, and failure results in a miscompile. This
verification pass checks that all (reachable) counter-based branches are
dominated by a loop mtctr instruction, and that no instructions in between
clobber the counter register. If these conditions are not satisfied, then an
ICE will be triggered.
In short, this is to help us sleep better at night.
llvm-svn: 182295
2013-05-21 00:08:17 +08:00
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
|
|
|
|
|
PIE = MBB->pred_end(); PI != PIE; ++PI)
|
|
|
|
|
Preds.push_back(*PI);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
do {
|
|
|
|
|
MBB = Preds.pop_back_val();
|
|
|
|
|
if (!Visited.count(MBB)) {
|
|
|
|
|
I = MBB->getLastNonDebugInstr();
|
|
|
|
|
goto check_block;
|
|
|
|
|
}
|
|
|
|
|
} while (!Preds.empty());
|
|
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool PPCCTRLoopsVerify::runOnMachineFunction(MachineFunction &MF) {
|
|
|
|
|
MDT = &getAnalysis<MachineDominatorTree>();
|
|
|
|
|
|
|
|
|
|
// Verify that all bdnz/bdz instructions are dominated by a loop mtctr before
|
|
|
|
|
// any other instructions that might clobber the ctr register.
|
|
|
|
|
for (MachineFunction::iterator I = MF.begin(), IE = MF.end();
|
|
|
|
|
I != IE; ++I) {
|
2015-10-20 09:07:37 +08:00
|
|
|
MachineBasicBlock *MBB = &*I;
|
Add a PPCCTRLoops verification pass
When asserts are enabled, this adds a verification pass for PPC counter-loop
formation. Unfortunately, without sacrificing code quality, there is no better
way of forming counter-based loops except at the (late) IR level. This means
that we need to recognize, at the IR level, anything which might turn into a
function call (or indirect branch). Because this is currently a finite set of
things, and because SelectionDAG lowering is basic-block local, this can be
done. Nevertheless, it is fragile, and failure results in a miscompile. This
verification pass checks that all (reachable) counter-based branches are
dominated by a loop mtctr instruction, and that no instructions in between
clobber the counter register. If these conditions are not satisfied, then an
ICE will be triggered.
In short, this is to help us sleep better at night.
llvm-svn: 182295
2013-05-21 00:08:17 +08:00
|
|
|
if (!MDT->isReachableFromEntry(MBB))
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
for (MachineBasicBlock::iterator MII = MBB->getFirstTerminator(),
|
|
|
|
|
MIIE = MBB->end(); MII != MIIE; ++MII) {
|
|
|
|
|
unsigned Opc = MII->getOpcode();
|
|
|
|
|
if (Opc == PPC::BDNZ8 || Opc == PPC::BDNZ ||
|
|
|
|
|
Opc == PPC::BDZ8 || Opc == PPC::BDZ)
|
|
|
|
|
if (!verifyCTRBranch(MBB, MII))
|
|
|
|
|
llvm_unreachable("Invalid PPC CTR loop!");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
#endif // NDEBUG
|