2017-08-10 08:46:15 +08:00
|
|
|
//===- GCNIterativeScheduler.cpp ------------------------------------------===//
|
2017-03-21 21:15:46 +08:00
|
|
|
//
|
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
//
|
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
// License. See LICENSE.TXT for details.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
#include "GCNIterativeScheduler.h"
|
2017-08-10 08:46:15 +08:00
|
|
|
#include "AMDGPUSubtarget.h"
|
|
|
|
#include "GCNRegPressure.h"
|
2017-03-21 21:15:46 +08:00
|
|
|
#include "GCNSchedStrategy.h"
|
AMDGPU: Remove #include "MCTargetDesc/AMDGPUMCTargetDesc.h" from common headers
Summary:
MCTargetDesc/AMDGPUMCTargetDesc.h contains enums for all the instuction
and register defintions, which are huge so we only want to include
them where needed.
This will also make it easier if we want to split the R600 and GCN
definitions into separate tablegenerated files.
I was unable to remove AMDGPUMCTargetDesc.h from SIMachineFunctionInfo.h
because it uses some enums from the header to initialize default values
for the SIMachineFunction class, so I ended up having to remove includes of
SIMachineFunctionInfo.h from headers too.
Reviewers: arsenm, nhaehnle
Reviewed By: nhaehnle
Subscribers: MatzeB, kzhuravl, wdng, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46272
llvm-svn: 332930
2018-05-22 10:03:23 +08:00
|
|
|
#include "SIMachineFunctionInfo.h"
|
2017-08-10 08:46:15 +08:00
|
|
|
#include "llvm/ADT/ArrayRef.h"
|
|
|
|
#include "llvm/ADT/STLExtras.h"
|
|
|
|
#include "llvm/ADT/SmallVector.h"
|
2017-12-13 10:51:04 +08:00
|
|
|
#include "llvm/CodeGen/LiveIntervals.h"
|
2017-08-10 08:46:15 +08:00
|
|
|
#include "llvm/CodeGen/MachineBasicBlock.h"
|
|
|
|
#include "llvm/CodeGen/MachineFunction.h"
|
|
|
|
#include "llvm/CodeGen/RegisterPressure.h"
|
|
|
|
#include "llvm/CodeGen/ScheduleDAG.h"
|
2018-04-30 22:59:11 +08:00
|
|
|
#include "llvm/Config/llvm-config.h"
|
2017-08-10 08:46:15 +08:00
|
|
|
#include "llvm/Support/Compiler.h"
|
|
|
|
#include "llvm/Support/Debug.h"
|
|
|
|
#include "llvm/Support/raw_ostream.h"
|
|
|
|
#include <algorithm>
|
|
|
|
#include <cassert>
|
|
|
|
#include <iterator>
|
|
|
|
#include <limits>
|
|
|
|
#include <memory>
|
|
|
|
#include <type_traits>
|
|
|
|
#include <vector>
|
2017-03-21 21:15:46 +08:00
|
|
|
|
|
|
|
using namespace llvm;
|
|
|
|
|
2017-07-12 06:08:28 +08:00
|
|
|
#define DEBUG_TYPE "machine-scheduler"
|
2017-03-21 21:15:46 +08:00
|
|
|
|
|
|
|
namespace llvm {
|
2017-08-10 08:46:15 +08:00
|
|
|
|
|
|
|
std::vector<const SUnit *> makeMinRegSchedule(ArrayRef<const SUnit *> TopRoots,
|
|
|
|
const ScheduleDAG &DAG);
|
|
|
|
|
2017-11-20 22:35:53 +08:00
|
|
|
std::vector<const SUnit*> makeGCNILPScheduler(ArrayRef<const SUnit*> BotRoots,
|
|
|
|
const ScheduleDAG &DAG);
|
|
|
|
}
|
2017-03-21 21:15:46 +08:00
|
|
|
|
|
|
|
// shim accessors for different order containers
|
|
|
|
static inline MachineInstr *getMachineInstr(MachineInstr *MI) {
|
|
|
|
return MI;
|
|
|
|
}
|
|
|
|
static inline MachineInstr *getMachineInstr(const SUnit *SU) {
|
|
|
|
return SU->getInstr();
|
|
|
|
}
|
|
|
|
static inline MachineInstr *getMachineInstr(const SUnit &SU) {
|
|
|
|
return SU.getInstr();
|
|
|
|
}
|
|
|
|
|
2017-10-15 22:32:27 +08:00
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
2017-03-21 21:15:46 +08:00
|
|
|
LLVM_DUMP_METHOD
|
|
|
|
static void printRegion(raw_ostream &OS,
|
|
|
|
MachineBasicBlock::iterator Begin,
|
|
|
|
MachineBasicBlock::iterator End,
|
|
|
|
const LiveIntervals *LIS,
|
|
|
|
unsigned MaxInstNum =
|
|
|
|
std::numeric_limits<unsigned>::max()) {
|
|
|
|
auto BB = Begin->getParent();
|
2017-12-05 01:18:51 +08:00
|
|
|
OS << BB->getParent()->getName() << ":" << printMBBReference(*BB) << ' '
|
|
|
|
<< BB->getName() << ":\n";
|
2017-03-21 21:15:46 +08:00
|
|
|
auto I = Begin;
|
|
|
|
MaxInstNum = std::max(MaxInstNum, 1u);
|
|
|
|
for (; I != End && MaxInstNum; ++I, --MaxInstNum) {
|
2018-05-09 10:42:00 +08:00
|
|
|
if (!I->isDebugInstr() && LIS)
|
2017-03-21 21:15:46 +08:00
|
|
|
OS << LIS->getInstructionIndex(*I);
|
|
|
|
OS << '\t' << *I;
|
|
|
|
}
|
|
|
|
if (I != End) {
|
|
|
|
OS << "\t...\n";
|
|
|
|
I = std::prev(End);
|
2018-05-09 10:42:00 +08:00
|
|
|
if (!I->isDebugInstr() && LIS)
|
2017-03-21 21:15:46 +08:00
|
|
|
OS << LIS->getInstructionIndex(*I);
|
|
|
|
OS << '\t' << *I;
|
|
|
|
}
|
|
|
|
if (End != BB->end()) { // print boundary inst if present
|
|
|
|
OS << "----\n";
|
|
|
|
if (LIS) OS << LIS->getInstructionIndex(*End) << '\t';
|
|
|
|
OS << *End;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
LLVM_DUMP_METHOD
|
|
|
|
static void printLivenessInfo(raw_ostream &OS,
|
|
|
|
MachineBasicBlock::iterator Begin,
|
|
|
|
MachineBasicBlock::iterator End,
|
|
|
|
const LiveIntervals *LIS) {
|
|
|
|
const auto BB = Begin->getParent();
|
|
|
|
const auto &MRI = BB->getParent()->getRegInfo();
|
|
|
|
|
|
|
|
const auto LiveIns = getLiveRegsBefore(*Begin, *LIS);
|
|
|
|
OS << "LIn RP: ";
|
|
|
|
getRegPressure(MRI, LiveIns).print(OS);
|
|
|
|
|
|
|
|
const auto BottomMI = End == BB->end() ? std::prev(End) : End;
|
|
|
|
const auto LiveOuts = getLiveRegsAfter(*BottomMI, *LIS);
|
|
|
|
OS << "LOt RP: ";
|
|
|
|
getRegPressure(MRI, LiveOuts).print(OS);
|
|
|
|
}
|
|
|
|
|
|
|
|
LLVM_DUMP_METHOD
|
|
|
|
void GCNIterativeScheduler::printRegions(raw_ostream &OS) const {
|
|
|
|
const auto &ST = MF.getSubtarget<SISubtarget>();
|
|
|
|
for (const auto R : Regions) {
|
|
|
|
OS << "Region to schedule ";
|
|
|
|
printRegion(OS, R->Begin, R->End, LIS, 1);
|
|
|
|
printLivenessInfo(OS, R->Begin, R->End, LIS);
|
|
|
|
OS << "Max RP: ";
|
|
|
|
R->MaxPressure.print(OS, &ST);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
LLVM_DUMP_METHOD
|
|
|
|
void GCNIterativeScheduler::printSchedResult(raw_ostream &OS,
|
|
|
|
const Region *R,
|
|
|
|
const GCNRegPressure &RP) const {
|
|
|
|
OS << "\nAfter scheduling ";
|
|
|
|
printRegion(OS, R->Begin, R->End, LIS);
|
|
|
|
printSchedRP(OS, R->MaxPressure, RP);
|
|
|
|
OS << '\n';
|
|
|
|
}
|
|
|
|
|
|
|
|
LLVM_DUMP_METHOD
|
|
|
|
void GCNIterativeScheduler::printSchedRP(raw_ostream &OS,
|
|
|
|
const GCNRegPressure &Before,
|
|
|
|
const GCNRegPressure &After) const {
|
|
|
|
const auto &ST = MF.getSubtarget<SISubtarget>();
|
|
|
|
OS << "RP before: ";
|
|
|
|
Before.print(OS, &ST);
|
|
|
|
OS << "RP after: ";
|
|
|
|
After.print(OS, &ST);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
// DAG builder helper
|
|
|
|
class GCNIterativeScheduler::BuildDAG {
|
|
|
|
GCNIterativeScheduler &Sch;
|
2017-08-10 08:46:15 +08:00
|
|
|
SmallVector<SUnit *, 8> TopRoots;
|
|
|
|
|
2017-11-20 22:35:53 +08:00
|
|
|
SmallVector<SUnit*, 8> BotRoots;
|
2017-03-21 21:15:46 +08:00
|
|
|
public:
|
|
|
|
BuildDAG(const Region &R, GCNIterativeScheduler &_Sch)
|
|
|
|
: Sch(_Sch) {
|
|
|
|
auto BB = R.Begin->getParent();
|
|
|
|
Sch.BaseClass::startBlock(BB);
|
|
|
|
Sch.BaseClass::enterRegion(BB, R.Begin, R.End, R.NumRegionInstrs);
|
|
|
|
|
|
|
|
Sch.buildSchedGraph(Sch.AA, nullptr, nullptr, nullptr,
|
|
|
|
/*TrackLaneMask*/true);
|
|
|
|
Sch.Topo.InitDAGTopologicalSorting();
|
|
|
|
Sch.findRootsAndBiasEdges(TopRoots, BotRoots);
|
|
|
|
}
|
2017-08-10 08:46:15 +08:00
|
|
|
|
2017-03-21 21:15:46 +08:00
|
|
|
~BuildDAG() {
|
|
|
|
Sch.BaseClass::exitRegion();
|
|
|
|
Sch.BaseClass::finishBlock();
|
|
|
|
}
|
2017-08-10 08:46:15 +08:00
|
|
|
|
|
|
|
ArrayRef<const SUnit *> getTopRoots() const {
|
2017-03-21 21:15:46 +08:00
|
|
|
return TopRoots;
|
|
|
|
}
|
2017-11-20 22:35:53 +08:00
|
|
|
ArrayRef<SUnit*> getBottomRoots() const {
|
|
|
|
return BotRoots;
|
|
|
|
}
|
2017-03-21 21:15:46 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
class GCNIterativeScheduler::OverrideLegacyStrategy {
|
|
|
|
GCNIterativeScheduler &Sch;
|
|
|
|
Region &Rgn;
|
|
|
|
std::unique_ptr<MachineSchedStrategy> SaveSchedImpl;
|
|
|
|
GCNRegPressure SaveMaxRP;
|
2017-08-10 08:46:15 +08:00
|
|
|
|
2017-03-21 21:15:46 +08:00
|
|
|
public:
|
|
|
|
OverrideLegacyStrategy(Region &R,
|
|
|
|
MachineSchedStrategy &OverrideStrategy,
|
|
|
|
GCNIterativeScheduler &_Sch)
|
|
|
|
: Sch(_Sch)
|
|
|
|
, Rgn(R)
|
|
|
|
, SaveSchedImpl(std::move(_Sch.SchedImpl))
|
|
|
|
, SaveMaxRP(R.MaxPressure) {
|
|
|
|
Sch.SchedImpl.reset(&OverrideStrategy);
|
|
|
|
auto BB = R.Begin->getParent();
|
|
|
|
Sch.BaseClass::startBlock(BB);
|
|
|
|
Sch.BaseClass::enterRegion(BB, R.Begin, R.End, R.NumRegionInstrs);
|
|
|
|
}
|
2017-08-10 08:46:15 +08:00
|
|
|
|
2017-03-21 21:15:46 +08:00
|
|
|
~OverrideLegacyStrategy() {
|
|
|
|
Sch.BaseClass::exitRegion();
|
|
|
|
Sch.BaseClass::finishBlock();
|
|
|
|
Sch.SchedImpl.release();
|
|
|
|
Sch.SchedImpl = std::move(SaveSchedImpl);
|
|
|
|
}
|
2017-08-10 08:46:15 +08:00
|
|
|
|
2017-03-21 21:15:46 +08:00
|
|
|
void schedule() {
|
|
|
|
assert(Sch.RegionBegin == Rgn.Begin && Sch.RegionEnd == Rgn.End);
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "\nScheduling ";
|
|
|
|
printRegion(dbgs(), Rgn.Begin, Rgn.End, Sch.LIS, 2));
|
2017-03-21 21:15:46 +08:00
|
|
|
Sch.BaseClass::schedule();
|
|
|
|
|
|
|
|
// Unfortunatelly placeDebugValues incorrectly modifies RegionEnd, restore
|
|
|
|
Sch.RegionEnd = Rgn.End;
|
|
|
|
//assert(Rgn.End == Sch.RegionEnd);
|
|
|
|
Rgn.Begin = Sch.RegionBegin;
|
|
|
|
Rgn.MaxPressure.clear();
|
|
|
|
}
|
2017-08-10 08:46:15 +08:00
|
|
|
|
2017-03-21 21:15:46 +08:00
|
|
|
void restoreOrder() {
|
|
|
|
assert(Sch.RegionBegin == Rgn.Begin && Sch.RegionEnd == Rgn.End);
|
|
|
|
// DAG SUnits are stored using original region's order
|
|
|
|
// so just use SUnits as the restoring schedule
|
|
|
|
Sch.scheduleRegion(Rgn, Sch.SUnits, SaveMaxRP);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2017-05-27 04:09:00 +08:00
|
|
|
namespace {
|
2017-08-10 08:46:15 +08:00
|
|
|
|
2017-03-21 21:15:46 +08:00
|
|
|
// just a stub to make base class happy
|
|
|
|
class SchedStrategyStub : public MachineSchedStrategy {
|
|
|
|
public:
|
|
|
|
bool shouldTrackPressure() const override { return false; }
|
|
|
|
bool shouldTrackLaneMasks() const override { return false; }
|
|
|
|
void initialize(ScheduleDAGMI *DAG) override {}
|
|
|
|
SUnit *pickNode(bool &IsTopNode) override { return nullptr; }
|
|
|
|
void schedNode(SUnit *SU, bool IsTopNode) override {}
|
|
|
|
void releaseTopNode(SUnit *SU) override {}
|
|
|
|
void releaseBottomNode(SUnit *SU) override {}
|
|
|
|
};
|
2017-08-10 08:46:15 +08:00
|
|
|
|
|
|
|
} // end anonymous namespace
|
2017-03-21 21:15:46 +08:00
|
|
|
|
|
|
|
GCNIterativeScheduler::GCNIterativeScheduler(MachineSchedContext *C,
|
|
|
|
StrategyKind S)
|
|
|
|
: BaseClass(C, llvm::make_unique<SchedStrategyStub>())
|
|
|
|
, Context(C)
|
|
|
|
, Strategy(S)
|
|
|
|
, UPTracker(*LIS) {
|
|
|
|
}
|
|
|
|
|
|
|
|
// returns max pressure for a region
|
|
|
|
GCNRegPressure
|
|
|
|
GCNIterativeScheduler::getRegionPressure(MachineBasicBlock::iterator Begin,
|
|
|
|
MachineBasicBlock::iterator End)
|
|
|
|
const {
|
|
|
|
// For the purpose of pressure tracking bottom inst of the region should
|
|
|
|
// be also processed. End is either BB end, BB terminator inst or sched
|
|
|
|
// boundary inst.
|
|
|
|
auto const BBEnd = Begin->getParent()->end();
|
|
|
|
auto const BottomMI = End == BBEnd ? std::prev(End) : End;
|
|
|
|
|
|
|
|
// scheduleRegions walks bottom to top, so its likely we just get next
|
|
|
|
// instruction to track
|
|
|
|
auto AfterBottomMI = std::next(BottomMI);
|
|
|
|
if (AfterBottomMI == BBEnd ||
|
|
|
|
&*AfterBottomMI != UPTracker.getLastTrackedMI()) {
|
|
|
|
UPTracker.reset(*BottomMI);
|
|
|
|
} else {
|
|
|
|
assert(UPTracker.isValid());
|
|
|
|
}
|
|
|
|
|
|
|
|
for (auto I = BottomMI; I != Begin; --I)
|
|
|
|
UPTracker.recede(*I);
|
|
|
|
|
|
|
|
UPTracker.recede(*Begin);
|
|
|
|
|
|
|
|
assert(UPTracker.isValid() ||
|
|
|
|
(dbgs() << "Tracked region ",
|
|
|
|
printRegion(dbgs(), Begin, End, LIS), false));
|
|
|
|
return UPTracker.moveMaxPressure();
|
|
|
|
}
|
|
|
|
|
|
|
|
// returns max pressure for a tentative schedule
|
|
|
|
template <typename Range> GCNRegPressure
|
|
|
|
GCNIterativeScheduler::getSchedulePressure(const Region &R,
|
|
|
|
Range &&Schedule) const {
|
|
|
|
auto const BBEnd = R.Begin->getParent()->end();
|
|
|
|
GCNUpwardRPTracker RPTracker(*LIS);
|
|
|
|
if (R.End != BBEnd) {
|
|
|
|
// R.End points to the boundary instruction but the
|
|
|
|
// schedule doesn't include it
|
|
|
|
RPTracker.reset(*R.End);
|
|
|
|
RPTracker.recede(*R.End);
|
|
|
|
} else {
|
|
|
|
// R.End doesn't point to the boundary instruction
|
|
|
|
RPTracker.reset(*std::prev(BBEnd));
|
|
|
|
}
|
|
|
|
for (auto I = Schedule.end(), B = Schedule.begin(); I != B;) {
|
|
|
|
RPTracker.recede(*getMachineInstr(*--I));
|
|
|
|
}
|
|
|
|
return RPTracker.moveMaxPressure();
|
|
|
|
}
|
|
|
|
|
|
|
|
void GCNIterativeScheduler::enterRegion(MachineBasicBlock *BB, // overriden
|
|
|
|
MachineBasicBlock::iterator Begin,
|
|
|
|
MachineBasicBlock::iterator End,
|
|
|
|
unsigned NumRegionInstrs) {
|
|
|
|
BaseClass::enterRegion(BB, Begin, End, NumRegionInstrs);
|
|
|
|
if (NumRegionInstrs > 2) {
|
|
|
|
Regions.push_back(
|
|
|
|
new (Alloc.Allocate())
|
|
|
|
Region { Begin, End, NumRegionInstrs,
|
|
|
|
getRegionPressure(Begin, End), nullptr });
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void GCNIterativeScheduler::schedule() { // overriden
|
|
|
|
// do nothing
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(printLivenessInfo(dbgs(), RegionBegin, RegionEnd, LIS);
|
|
|
|
if (!Regions.empty() && Regions.back()->Begin == RegionBegin) {
|
|
|
|
dbgs() << "Max RP: ";
|
|
|
|
Regions.back()->MaxPressure.print(
|
|
|
|
dbgs(), &MF.getSubtarget<SISubtarget>());
|
|
|
|
} dbgs()
|
|
|
|
<< '\n';);
|
2017-03-21 21:15:46 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void GCNIterativeScheduler::finalizeSchedule() { // overriden
|
|
|
|
if (Regions.empty())
|
|
|
|
return;
|
|
|
|
switch (Strategy) {
|
|
|
|
case SCHEDULE_MINREGONLY: scheduleMinReg(); break;
|
|
|
|
case SCHEDULE_MINREGFORCED: scheduleMinReg(true); break;
|
|
|
|
case SCHEDULE_LEGACYMAXOCCUPANCY: scheduleLegacyMaxOccupancy(); break;
|
2017-11-20 22:35:53 +08:00
|
|
|
case SCHEDULE_ILP: scheduleILP(false); break;
|
2017-03-21 21:15:46 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Detach schedule from SUnits and interleave it with debug values.
|
|
|
|
// Returned schedule becomes independent of DAG state.
|
|
|
|
std::vector<MachineInstr*>
|
|
|
|
GCNIterativeScheduler::detachSchedule(ScheduleRef Schedule) const {
|
|
|
|
std::vector<MachineInstr*> Res;
|
|
|
|
Res.reserve(Schedule.size() * 2);
|
|
|
|
|
|
|
|
if (FirstDbgValue)
|
|
|
|
Res.push_back(FirstDbgValue);
|
|
|
|
|
|
|
|
const auto DbgB = DbgValues.begin(), DbgE = DbgValues.end();
|
|
|
|
for (auto SU : Schedule) {
|
|
|
|
Res.push_back(SU->getInstr());
|
|
|
|
const auto &D = std::find_if(DbgB, DbgE, [SU](decltype(*DbgB) &P) {
|
|
|
|
return P.second == SU->getInstr();
|
|
|
|
});
|
|
|
|
if (D != DbgE)
|
|
|
|
Res.push_back(D->first);
|
|
|
|
}
|
|
|
|
return Res;
|
|
|
|
}
|
|
|
|
|
|
|
|
void GCNIterativeScheduler::setBestSchedule(Region &R,
|
|
|
|
ScheduleRef Schedule,
|
|
|
|
const GCNRegPressure &MaxRP) {
|
|
|
|
R.BestSchedule.reset(
|
|
|
|
new TentativeSchedule{ detachSchedule(Schedule), MaxRP });
|
|
|
|
}
|
|
|
|
|
|
|
|
void GCNIterativeScheduler::scheduleBest(Region &R) {
|
|
|
|
assert(R.BestSchedule.get() && "No schedule specified");
|
|
|
|
scheduleRegion(R, R.BestSchedule->Schedule, R.BestSchedule->MaxPressure);
|
|
|
|
R.BestSchedule.reset();
|
|
|
|
}
|
|
|
|
|
|
|
|
// minimal required region scheduler, works for ranges of SUnits*,
|
|
|
|
// SUnits or MachineIntrs*
|
|
|
|
template <typename Range>
|
|
|
|
void GCNIterativeScheduler::scheduleRegion(Region &R, Range &&Schedule,
|
|
|
|
const GCNRegPressure &MaxRP) {
|
|
|
|
assert(RegionBegin == R.Begin && RegionEnd == R.End);
|
|
|
|
assert(LIS != nullptr);
|
|
|
|
#ifndef NDEBUG
|
|
|
|
const auto SchedMaxRP = getSchedulePressure(R, Schedule);
|
|
|
|
#endif
|
|
|
|
auto BB = R.Begin->getParent();
|
|
|
|
auto Top = R.Begin;
|
|
|
|
for (const auto &I : Schedule) {
|
|
|
|
auto MI = getMachineInstr(I);
|
|
|
|
if (MI != &*Top) {
|
|
|
|
BB->remove(MI);
|
|
|
|
BB->insert(Top, MI);
|
2018-05-09 10:42:00 +08:00
|
|
|
if (!MI->isDebugInstr())
|
2017-03-21 21:15:46 +08:00
|
|
|
LIS->handleMove(*MI, true);
|
|
|
|
}
|
2018-05-09 10:42:00 +08:00
|
|
|
if (!MI->isDebugInstr()) {
|
2017-03-21 21:15:46 +08:00
|
|
|
// Reset read - undef flags and update them later.
|
|
|
|
for (auto &Op : MI->operands())
|
|
|
|
if (Op.isReg() && Op.isDef())
|
|
|
|
Op.setIsUndef(false);
|
|
|
|
|
|
|
|
RegisterOperands RegOpers;
|
|
|
|
RegOpers.collect(*MI, *TRI, MRI, /*ShouldTrackLaneMasks*/true,
|
|
|
|
/*IgnoreDead*/false);
|
|
|
|
// Adjust liveness and add missing dead+read-undef flags.
|
|
|
|
auto SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
|
|
|
|
RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
|
|
|
|
}
|
|
|
|
Top = std::next(MI->getIterator());
|
|
|
|
}
|
|
|
|
RegionBegin = getMachineInstr(Schedule.front());
|
|
|
|
|
|
|
|
// Schedule consisting of MachineInstr* is considered 'detached'
|
|
|
|
// and already interleaved with debug values
|
|
|
|
if (!std::is_same<decltype(*Schedule.begin()), MachineInstr*>::value) {
|
|
|
|
placeDebugValues();
|
|
|
|
// Unfortunatelly placeDebugValues incorrectly modifies RegionEnd, restore
|
|
|
|
//assert(R.End == RegionEnd);
|
|
|
|
RegionEnd = R.End;
|
|
|
|
}
|
|
|
|
|
|
|
|
R.Begin = RegionBegin;
|
|
|
|
R.MaxPressure = MaxRP;
|
|
|
|
|
|
|
|
#ifndef NDEBUG
|
|
|
|
const auto RegionMaxRP = getRegionPressure(R);
|
|
|
|
const auto &ST = MF.getSubtarget<SISubtarget>();
|
|
|
|
#endif
|
|
|
|
assert((SchedMaxRP == RegionMaxRP && (MaxRP.empty() || SchedMaxRP == MaxRP))
|
|
|
|
|| (dbgs() << "Max RP mismatch!!!\n"
|
|
|
|
"RP for schedule (calculated): ",
|
|
|
|
SchedMaxRP.print(dbgs(), &ST),
|
|
|
|
dbgs() << "RP for schedule (reported): ",
|
|
|
|
MaxRP.print(dbgs(), &ST),
|
|
|
|
dbgs() << "RP after scheduling: ",
|
|
|
|
RegionMaxRP.print(dbgs(), &ST),
|
|
|
|
false));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Sort recorded regions by pressure - highest at the front
|
|
|
|
void GCNIterativeScheduler::sortRegionsByPressure(unsigned TargetOcc) {
|
|
|
|
const auto &ST = MF.getSubtarget<SISubtarget>();
|
[AMDGPU] Change std::sort to llvm::sort in response to r327219
Summary:
r327219 added wrappers to std::sort which randomly shuffle the container before sorting.
This will help in uncovering non-determinism caused due to undefined sorting
order of objects having the same key.
To make use of that infrastructure we need to invoke llvm::sort instead of std::sort.
Reviewers: tstellar, RKSimon, arsenm
Reviewed By: arsenm
Subscribers: arsenm, kzhuravl, wdng, nhaehnle, yaxunl, dstuttard, tpr, llvm-commits, t-tye
Differential Revision: https://reviews.llvm.org/D44856
llvm-svn: 328429
2018-03-25 01:15:04 +08:00
|
|
|
llvm::sort(Regions.begin(), Regions.end(),
|
2017-03-21 21:15:46 +08:00
|
|
|
[&ST, TargetOcc](const Region *R1, const Region *R2) {
|
|
|
|
return R2->MaxPressure.less(ST, R1->MaxPressure, TargetOcc);
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
|
|
// Legacy MaxOccupancy Strategy
|
|
|
|
|
|
|
|
// Tries to increase occupancy applying minreg scheduler for a sequence of
|
|
|
|
// most demanding regions. Obtained schedules are saved as BestSchedule for a
|
|
|
|
// region.
|
|
|
|
// TargetOcc is the best achievable occupancy for a kernel.
|
|
|
|
// Returns better occupancy on success or current occupancy on fail.
|
|
|
|
// BestSchedules aren't deleted on fail.
|
|
|
|
unsigned GCNIterativeScheduler::tryMaximizeOccupancy(unsigned TargetOcc) {
|
|
|
|
// TODO: assert Regions are sorted descending by pressure
|
|
|
|
const auto &ST = MF.getSubtarget<SISubtarget>();
|
|
|
|
const auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "Trying to improve occupancy, target = " << TargetOcc
|
|
|
|
<< ", current = " << Occ << '\n');
|
2017-03-21 21:15:46 +08:00
|
|
|
|
|
|
|
auto NewOcc = TargetOcc;
|
|
|
|
for (auto R : Regions) {
|
|
|
|
if (R->MaxPressure.getOccupancy(ST) >= NewOcc)
|
|
|
|
break;
|
|
|
|
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(printRegion(dbgs(), R->Begin, R->End, LIS, 3);
|
|
|
|
printLivenessInfo(dbgs(), R->Begin, R->End, LIS));
|
2017-03-21 21:15:46 +08:00
|
|
|
|
|
|
|
BuildDAG DAG(*R, *this);
|
|
|
|
const auto MinSchedule = makeMinRegSchedule(DAG.getTopRoots(), *this);
|
|
|
|
const auto MaxRP = getSchedulePressure(*R, MinSchedule);
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "Occupancy improvement attempt:\n";
|
|
|
|
printSchedRP(dbgs(), R->MaxPressure, MaxRP));
|
2017-03-21 21:15:46 +08:00
|
|
|
|
|
|
|
NewOcc = std::min(NewOcc, MaxRP.getOccupancy(ST));
|
|
|
|
if (NewOcc <= Occ)
|
|
|
|
break;
|
|
|
|
|
|
|
|
setBestSchedule(*R, MinSchedule, MaxRP);
|
|
|
|
}
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "New occupancy = " << NewOcc
|
|
|
|
<< ", prev occupancy = " << Occ << '\n');
|
2018-05-31 13:36:04 +08:00
|
|
|
if (NewOcc > Occ) {
|
|
|
|
SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
|
|
|
|
MFI->increaseOccupancy(MF, NewOcc);
|
|
|
|
}
|
|
|
|
|
2017-03-21 21:15:46 +08:00
|
|
|
return std::max(NewOcc, Occ);
|
|
|
|
}
|
|
|
|
|
|
|
|
void GCNIterativeScheduler::scheduleLegacyMaxOccupancy(
|
|
|
|
bool TryMaximizeOccupancy) {
|
|
|
|
const auto &ST = MF.getSubtarget<SISubtarget>();
|
2018-05-31 13:36:04 +08:00
|
|
|
SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
|
|
|
|
auto TgtOcc = MFI->getMinAllowedOccupancy();
|
2017-03-21 21:15:46 +08:00
|
|
|
|
|
|
|
sortRegionsByPressure(TgtOcc);
|
|
|
|
auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
|
|
|
|
|
|
|
|
if (TryMaximizeOccupancy && Occ < TgtOcc)
|
|
|
|
Occ = tryMaximizeOccupancy(TgtOcc);
|
|
|
|
|
|
|
|
// This is really weird but for some magic scheduling regions twice
|
|
|
|
// gives performance improvement
|
|
|
|
const int NumPasses = Occ < TgtOcc ? 2 : 1;
|
|
|
|
|
|
|
|
TgtOcc = std::min(Occ, TgtOcc);
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "Scheduling using default scheduler, "
|
|
|
|
"target occupancy = "
|
|
|
|
<< TgtOcc << '\n');
|
2017-03-21 21:15:46 +08:00
|
|
|
GCNMaxOccupancySchedStrategy LStrgy(Context);
|
2018-05-31 13:36:04 +08:00
|
|
|
unsigned FinalOccupancy = std::min(Occ, MFI->getOccupancy());
|
2017-03-21 21:15:46 +08:00
|
|
|
|
|
|
|
for (int I = 0; I < NumPasses; ++I) {
|
|
|
|
// running first pass with TargetOccupancy = 0 mimics previous scheduling
|
|
|
|
// approach and is a performance magic
|
|
|
|
LStrgy.setTargetOccupancy(I == 0 ? 0 : TgtOcc);
|
|
|
|
for (auto R : Regions) {
|
|
|
|
OverrideLegacyStrategy Ovr(*R, LStrgy, *this);
|
|
|
|
|
|
|
|
Ovr.schedule();
|
|
|
|
const auto RP = getRegionPressure(*R);
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(printSchedRP(dbgs(), R->MaxPressure, RP));
|
2017-03-21 21:15:46 +08:00
|
|
|
|
|
|
|
if (RP.getOccupancy(ST) < TgtOcc) {
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "Didn't fit into target occupancy O" << TgtOcc);
|
2017-03-21 21:15:46 +08:00
|
|
|
if (R->BestSchedule.get() &&
|
|
|
|
R->BestSchedule->MaxPressure.getOccupancy(ST) >= TgtOcc) {
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << ", scheduling minimal register\n");
|
2017-03-21 21:15:46 +08:00
|
|
|
scheduleBest(*R);
|
|
|
|
} else {
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << ", restoring\n");
|
2017-03-21 21:15:46 +08:00
|
|
|
Ovr.restoreOrder();
|
|
|
|
assert(R->MaxPressure.getOccupancy(ST) >= TgtOcc);
|
|
|
|
}
|
|
|
|
}
|
2018-05-31 13:36:04 +08:00
|
|
|
FinalOccupancy = std::min(FinalOccupancy, RP.getOccupancy(ST));
|
2017-03-21 21:15:46 +08:00
|
|
|
}
|
|
|
|
}
|
2018-05-31 13:36:04 +08:00
|
|
|
MFI->limitOccupancy(FinalOccupancy);
|
2017-03-21 21:15:46 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
|
|
// Minimal Register Strategy
|
|
|
|
|
|
|
|
void GCNIterativeScheduler::scheduleMinReg(bool force) {
|
|
|
|
const auto &ST = MF.getSubtarget<SISubtarget>();
|
2018-05-31 13:36:04 +08:00
|
|
|
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
|
|
|
|
const auto TgtOcc = MFI->getOccupancy();
|
2017-03-21 21:15:46 +08:00
|
|
|
sortRegionsByPressure(TgtOcc);
|
|
|
|
|
|
|
|
auto MaxPressure = Regions.front()->MaxPressure;
|
|
|
|
for (auto R : Regions) {
|
|
|
|
if (!force && R->MaxPressure.less(ST, MaxPressure, TgtOcc))
|
|
|
|
break;
|
|
|
|
|
|
|
|
BuildDAG DAG(*R, *this);
|
|
|
|
const auto MinSchedule = makeMinRegSchedule(DAG.getTopRoots(), *this);
|
|
|
|
|
|
|
|
const auto RP = getSchedulePressure(*R, MinSchedule);
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(if (R->MaxPressure.less(ST, RP, TgtOcc)) {
|
2017-03-21 21:15:46 +08:00
|
|
|
dbgs() << "\nWarning: Pressure becomes worse after minreg!";
|
|
|
|
printSchedRP(dbgs(), R->MaxPressure, RP);
|
|
|
|
});
|
|
|
|
|
|
|
|
if (!force && MaxPressure.less(ST, RP, TgtOcc))
|
|
|
|
break;
|
|
|
|
|
|
|
|
scheduleRegion(*R, MinSchedule, RP);
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(printSchedResult(dbgs(), R, RP));
|
2017-03-21 21:15:46 +08:00
|
|
|
|
|
|
|
MaxPressure = RP;
|
|
|
|
}
|
|
|
|
}
|
2017-11-20 22:35:53 +08:00
|
|
|
|
|
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
|
|
// ILP scheduler port
|
|
|
|
|
|
|
|
void GCNIterativeScheduler::scheduleILP(
|
|
|
|
bool TryMaximizeOccupancy) {
|
|
|
|
const auto &ST = MF.getSubtarget<SISubtarget>();
|
2018-05-31 13:36:04 +08:00
|
|
|
SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
|
|
|
|
auto TgtOcc = MFI->getMinAllowedOccupancy();
|
2017-11-20 22:35:53 +08:00
|
|
|
|
|
|
|
sortRegionsByPressure(TgtOcc);
|
|
|
|
auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
|
|
|
|
|
|
|
|
if (TryMaximizeOccupancy && Occ < TgtOcc)
|
|
|
|
Occ = tryMaximizeOccupancy(TgtOcc);
|
|
|
|
|
|
|
|
TgtOcc = std::min(Occ, TgtOcc);
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "Scheduling using default scheduler, "
|
|
|
|
"target occupancy = "
|
|
|
|
<< TgtOcc << '\n');
|
2017-11-20 22:35:53 +08:00
|
|
|
|
2018-05-31 13:36:04 +08:00
|
|
|
unsigned FinalOccupancy = std::min(Occ, MFI->getOccupancy());
|
2017-11-20 22:35:53 +08:00
|
|
|
for (auto R : Regions) {
|
|
|
|
BuildDAG DAG(*R, *this);
|
|
|
|
const auto ILPSchedule = makeGCNILPScheduler(DAG.getBottomRoots(), *this);
|
|
|
|
|
|
|
|
const auto RP = getSchedulePressure(*R, ILPSchedule);
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(printSchedRP(dbgs(), R->MaxPressure, RP));
|
2017-11-20 22:35:53 +08:00
|
|
|
|
|
|
|
if (RP.getOccupancy(ST) < TgtOcc) {
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "Didn't fit into target occupancy O" << TgtOcc);
|
2017-11-20 22:35:53 +08:00
|
|
|
if (R->BestSchedule.get() &&
|
|
|
|
R->BestSchedule->MaxPressure.getOccupancy(ST) >= TgtOcc) {
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << ", scheduling minimal register\n");
|
2017-11-20 22:35:53 +08:00
|
|
|
scheduleBest(*R);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
scheduleRegion(*R, ILPSchedule, RP);
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(printSchedResult(dbgs(), R, RP));
|
2018-05-31 13:36:04 +08:00
|
|
|
FinalOccupancy = std::min(FinalOccupancy, RP.getOccupancy(ST));
|
2017-11-20 22:35:53 +08:00
|
|
|
}
|
|
|
|
}
|
2018-05-31 13:36:04 +08:00
|
|
|
MFI->limitOccupancy(FinalOccupancy);
|
2017-11-20 22:35:53 +08:00
|
|
|
}
|