From 6a989c358cc79928a8cff0b51913d11417866dc4 Mon Sep 17 00:00:00 2001 From: Andrea Di Biagio Date: Sat, 1 Jun 2019 15:22:37 +0000 Subject: [PATCH] [MCA][Scheduler] Change how memory instructions are dispatched to the pending set. NFCI llvm-svn: 362302 --- llvm/include/llvm/MCA/HardwareUnits/LSUnit.h | 33 ++++++++---- llvm/lib/MCA/HardwareUnits/Scheduler.cpp | 55 +++++++++----------- 2 files changed, 48 insertions(+), 40 deletions(-) diff --git a/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h b/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h index e55b70088429..f2a5cf86ca49 100644 --- a/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h +++ b/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h @@ -70,7 +70,7 @@ public: unsigned getNumExecuting() const { return NumExecuting; } unsigned getNumExecuted() const { return NumExecuted; } - const InstRef &getCriticalMemoryInstruction() const { + const InstRef &getCriticalMemoryInstruction() const { return CriticalMemoryInstruction; } const CriticalDependency &getCriticalPredecessor() const { @@ -96,7 +96,7 @@ public: } bool isReady() const { return NumExecutedPredecessors == NumPredecessors; } bool isExecuting() const { - return NumExecuting == NumInstructions - NumExecuted; + return NumExecuting && (NumExecuting == (NumInstructions - NumExecuted)); } bool isExecuted() const { return NumInstructions == NumExecuted; } @@ -247,22 +247,32 @@ public: /// Check if a peviously dispatched instruction IR is now ready for execution. bool isReady(const InstRef &IR) const { unsigned GroupID = IR.getInstruction()->getLSUTokenID(); - assert(isValidGroupID(GroupID) && - "Invalid group associated with this instruction!"); - const MemoryGroup &Group = *Groups.find(GroupID)->second; + const MemoryGroup &Group = getGroup(GroupID); return Group.isReady(); } - /// Check if a previously dispatched instruction IR only depends on - /// instructions that are currently executing. + /// Check if instruction IR only depends on memory instructions that are + /// currently executing. bool isPending(const InstRef &IR) const { unsigned GroupID = IR.getInstruction()->getLSUTokenID(); - assert(isValidGroupID(GroupID) && - "Invalid group associated with this instruction!"); - const MemoryGroup &Group = *Groups.find(GroupID)->second; + const MemoryGroup &Group = getGroup(GroupID); return Group.isPending(); } + /// Check if instruction IR is still waiting on memory operations, and the + /// wait time is still unknown. + bool isWaiting(const InstRef &IR) const { + unsigned GroupID = IR.getInstruction()->getLSUTokenID(); + const MemoryGroup &Group = getGroup(GroupID); + return Group.isWaiting(); + } + + bool hasDependentUsers(const InstRef &IR) const { + unsigned GroupID = IR.getInstruction()->getLSUTokenID(); + const MemoryGroup &Group = getGroup(GroupID); + return !Group.isExecuted() && Group.getNumSuccessors(); + } + const MemoryGroup &getGroup(unsigned Index) const { assert(isValidGroupID(Index) && "Group doesn't exist!"); return *Groups.find(Index)->second; @@ -274,7 +284,8 @@ public: } unsigned createMemoryGroup() { - Groups.insert(std::make_pair(NextGroupID, llvm::make_unique())); + Groups.insert( + std::make_pair(NextGroupID, llvm::make_unique())); return NextGroupID++; } diff --git a/llvm/lib/MCA/HardwareUnits/Scheduler.cpp b/llvm/lib/MCA/HardwareUnits/Scheduler.cpp index 3afc0ac89ef0..c7091203595c 100644 --- a/llvm/lib/MCA/HardwareUnits/Scheduler.cpp +++ b/llvm/lib/MCA/HardwareUnits/Scheduler.cpp @@ -104,6 +104,7 @@ void Scheduler::issueInstruction( SmallVectorImpl &ReadyInstructions) { const Instruction &Inst = *IR.getInstruction(); bool HasDependentUsers = Inst.hasDependentUsers(); + HasDependentUsers |= Inst.isMemOp() && LSU.hasDependentUsers(IR); Resources->releaseBuffers(Inst.getDesc().Buffers); issueInstructionImpl(IR, UsedResources); @@ -111,14 +112,9 @@ void Scheduler::issueInstruction( // other dependent instructions. Dependent instructions may be issued during // this same cycle if operands have ReadAdvance entries. Promote those // instructions to the ReadySet and notify the caller that those are ready. - // If IR is a memory operation, then always call method `promoteToReadySet()` - // to notify any dependent memory operations that IR started execution. - bool ShouldPromoteInstructions = Inst.isMemOp(); if (HasDependentUsers) - ShouldPromoteInstructions |= promoteToPendingSet(PendingInstructions); - - if (ShouldPromoteInstructions) - promoteToReadySet(ReadyInstructions); + if (promoteToPendingSet(PendingInstructions)) + promoteToReadySet(ReadyInstructions); } bool Scheduler::promoteToReadySet(SmallVectorImpl &Ready) { @@ -130,18 +126,18 @@ bool Scheduler::promoteToReadySet(SmallVectorImpl &Ready) { if (!IR) break; - // Check if there are unsolved memory dependencies. + // Check if there are unsolved register dependencies. Instruction &IS = *IR.getInstruction(); + if (!IS.isReady() && !IS.updatePending()) { + ++I; + continue; + } + // Check if there are unsolved memory dependencies. if (IS.isMemOp() && !LSU.isReady(IR)) { ++I; continue; } - // Check if there are unsolved register dependencies. - if (!IS.isReady() && !IS.updatePending()) { - ++I; - continue; - } LLVM_DEBUG(dbgs() << "[SCHEDULER]: Instruction #" << IR << " promoted to the READY set.\n"); @@ -173,6 +169,12 @@ bool Scheduler::promoteToPendingSet(SmallVectorImpl &Pending) { ++I; continue; } + + if (IS.isMemOp() && LSU.isWaiting(IR)) { + ++I; + continue; + } + LLVM_DEBUG(dbgs() << "[SCHEDULER]: Instruction #" << IR << " promoted to the PENDING set.\n"); @@ -251,13 +253,8 @@ void Scheduler::analyzeDataDependencies(SmallVectorImpl &RegDeps, if (Resources->checkAvailability(IS.getDesc())) continue; - if (IS.isMemOp()) { - const MemoryGroup &Group = LSU.getGroup(IS.getLSUTokenID()); - if (Group.isWaiting()) - continue; - if (Group.isPending()) - MemDeps.emplace_back(IR); - } + if (IS.isMemOp() && LSU.isPending(IR)) + MemDeps.emplace_back(IR); if (IS.isPending()) RegDeps.emplace_back(IR); @@ -309,7 +306,13 @@ bool Scheduler::dispatch(InstRef &IR) { if (IS.isMemOp()) IS.setLSUTokenID(LSU.dispatch(IR)); - if (IS.isPending()) { + if (IS.isDispatched() || (IS.isMemOp() && LSU.isWaiting(IR))) { + LLVM_DEBUG(dbgs() << "[SCHEDULER] Adding #" << IR << " to the WaitSet\n"); + WaitSet.push_back(IR); + return false; + } + + if (IS.isPending() || (IS.isMemOp() && LSU.isPending(IR))) { LLVM_DEBUG(dbgs() << "[SCHEDULER] Adding #" << IR << " to the PendingSet\n"); PendingSet.push_back(IR); @@ -317,14 +320,8 @@ bool Scheduler::dispatch(InstRef &IR) { return false; } - // Memory operations that still have unsolved memory dependencies are - // initially dispatched to the WaitSet. - if (!IS.isReady() || (IS.isMemOp() && !LSU.isReady(IR))) { - LLVM_DEBUG(dbgs() << "[SCHEDULER] Adding #" << IR << " to the WaitSet\n"); - WaitSet.push_back(IR); - return false; - } - + assert(IS.isReady() && (!IS.isMemOp() || LSU.isReady(IR)) && + "Unexpected internal state found!"); // Don't add a zero-latency instruction to the Ready queue. // A zero-latency instruction doesn't consume any scheduler resources. That is // because it doesn't need to be executed, and it is often removed at register