mi-sched: update PressureDiffs on-the-fly for liveness.

This removes all expensive pressure tracking logic from the scheduling
critical path of node comparison.

llvm-svn: 189643

llvm/include/llvm/CodeGen/LiveInterval.h

@@ -296,6 +296,12 @@ namespace llvm {
       return r != end() && r->end.getBaseIndex() == BaseIdx;
     }
 
+    /// Return true if a live range starts at the instruction at this index.
+    bool isDefinedByInstr(SlotIndex index) const {
+      const_iterator r = find(index.getDeadSlot());
+      return r != end() && r->end.getBaseIndex() == index.getBaseIndex();
+    }
+
     /// getLiveRangeContaining - Return the live range that contains the
     /// specified index, or null if there is none.
     const LiveRange *getLiveRangeContaining(SlotIndex Idx) const {

llvm/include/llvm/CodeGen/MachineScheduler.h

@@ -221,7 +221,9 @@ protected:
 
   MachineBasicBlock::iterator LiveRegionEnd;
 
-  // Map each SU to its summary of pressure changes.
+  // Map each SU to its summary of pressure changes. This array is updated for
+  // liveness during bottom-up scheduling. Top-down scheduling may proceed but
+  // has no affect on the pressure diffs.
   PressureDiffs SUPressureDiffs;
 
   /// Register pressure in this region computed by initRegPressure.
@@ -376,6 +378,8 @@ protected:
 
   void initRegPressure();
 
+  void updatePressureDiffs(ArrayRef<unsigned> LiveUses);
+
   void updateScheduledPressure(const std::vector<unsigned> &NewMaxPressure);
 
   bool checkSchedLimit();

llvm/include/llvm/CodeGen/RegisterPressure.h

@@ -311,7 +311,7 @@ public:
   SlotIndex getCurrSlot() const;
 
   /// Recede across the previous instruction.
-  bool recede(PressureDiff *PDiff = 0);
+  bool recede(SmallVectorImpl<unsigned> *LiveUses = 0, PressureDiff *PDiff = 0);
 
   /// Advance across the current instruction.
   bool advance();

llvm/lib/CodeGen/MachineScheduler.cpp

@@ -156,8 +156,9 @@ static ScheduleDAGInstrs *createConvergingSched(MachineSchedContext *C);
 
 
 /// Decrement this iterator until reaching the top or a non-debug instr.
-static MachineBasicBlock::iterator
-priorNonDebug(MachineBasicBlock::iterator I, MachineBasicBlock::iterator Beg) {
+static MachineBasicBlock::const_iterator
+priorNonDebug(MachineBasicBlock::const_iterator I,
+              MachineBasicBlock::const_iterator Beg) {
   assert(I != Beg && "reached the top of the region, cannot decrement");
   while (--I != Beg) {
     if (!I->isDebugValue())
@@ -166,6 +167,14 @@ priorNonDebug(MachineBasicBlock::iterator I, MachineBasicBlock::iterator Beg) {
   return I;
 }
 
+/// Non-const version.
+static MachineBasicBlock::iterator
+priorNonDebug(MachineBasicBlock::iterator I,
+              MachineBasicBlock::const_iterator Beg) {
+  return const_cast<MachineInstr*>(
+    &*priorNonDebug(MachineBasicBlock::const_iterator(I), Beg));
+}
+
 /// If this iterator is a debug value, increment until reaching the End or a
 /// non-debug instruction.
 static MachineBasicBlock::iterator
@@ -488,9 +497,16 @@ void ScheduleDAGMI::initRegPressure() {
           dumpRegSetPressure(BotRPTracker.getLiveThru(), TRI));
   };
 
+  // For each live out vreg reduce the pressure change associated with other
+  // uses of the same vreg below the live-out reaching def.
+  updatePressureDiffs(RPTracker.getPressure().LiveOutRegs);
+
   // Account for liveness generated by the region boundary.
-  if (LiveRegionEnd != RegionEnd)
-    BotRPTracker.recede();
+  if (LiveRegionEnd != RegionEnd) {
+    SmallVector<unsigned, 8> LiveUses;
+    BotRPTracker.recede(&LiveUses);
+    updatePressureDiffs(LiveUses);
+  }
 
   assert(BotRPTracker.getPos() == RegionEnd && "Can't find the region bottom");
 
@@ -535,6 +551,42 @@ updateScheduledPressure(const std::vector<unsigned> &NewMaxPressure) {
     });
 }
 
+/// Update the PressureDiff array for liveness after scheduling this
+/// instruction.
+void ScheduleDAGMI::updatePressureDiffs(ArrayRef<unsigned> LiveUses) {
+  for (unsigned LUIdx = 0, LUEnd = LiveUses.size(); LUIdx != LUEnd; ++LUIdx) {
+    /// FIXME: Currently assuming single-use physregs.
+    unsigned Reg = LiveUses[LUIdx];
+    if (!TRI->isVirtualRegister(Reg))
+      continue;
+    // This may be called before CurrentBottom has been initialized. However,
+    // BotRPTracker must have a valid position. We want the value live into the
+    // instruction or live out of the block, so ask for the previous
+    // instruction's live-out.
+    const LiveInterval &LI = LIS->getInterval(Reg);
+    VNInfo *VNI;
+    if (BotRPTracker.getPos() == BB->end())
+      VNI = LI.getVNInfoBefore(LIS->getMBBEndIdx(BB));
+    else {
+      LiveRangeQuery LRQ(LI, LIS->getInstructionIndex(BotRPTracker.getPos()));
+      VNI = LRQ.valueIn();
+    }
+    // RegisterPressureTracker guarantees that readsReg is true for LiveUses.
+    assert(VNI && "No live value at use.");
+    for (VReg2UseMap::iterator
+           UI = VRegUses.find(Reg); UI != VRegUses.end(); ++UI) {
+      SUnit *SU = UI->SU;
+      // If this use comes before the reaching def, it cannot be a last use, so
+      // descrease its pressure change.
+      if (!SU->isScheduled && SU != &ExitSU) {
+        LiveRangeQuery LRQ(LI, LIS->getInstructionIndex(SU->getInstr()));
+        if (LRQ.valueIn() == VNI)
+          getPressureDiff(SU).addPressureChange(Reg, true, &MRI);
+      }
+    }
+  }
+}
+
 /// schedule - Called back from MachineScheduler::runOnMachineFunction
 /// after setting up the current scheduling region. [RegionBegin, RegionEnd)
 /// only includes instructions that have DAG nodes, not scheduling boundaries.
@@ -794,8 +846,10 @@ void ScheduleDAGMI::scheduleMI(SUnit *SU, bool IsTopNode) {
       CurrentBottom = MI;
     }
     // Update bottom scheduled pressure.
-    BotRPTracker.recede();
+    SmallVector<unsigned, 8> LiveUses;
+    BotRPTracker.recede(&LiveUses);
     assert(BotRPTracker.getPos() == CurrentBottom && "out of sync");
+    updatePressureDiffs(LiveUses);
     updateScheduledPressure(BotRPTracker.getPressure().MaxSetPressure);
   }
 }

llvm/lib/CodeGen/RegisterPressure.cpp

@@ -399,10 +399,9 @@ static void collectPDiff(PressureDiff &PDiff, RegisterOperands &RegOpers,
                          const MachineRegisterInfo *MRI) {
   assert(!PDiff.begin()->isValid() && "stale PDiff");
 
-  for (unsigned i = 0, e = RegOpers.Defs.size(); i != e; ++i) {
-    if (!containsReg(RegOpers.Uses, RegOpers.Defs[i]))
-      PDiff.addPressureChange(RegOpers.Defs[i], true, MRI);
-  }
+  for (unsigned i = 0, e = RegOpers.Defs.size(); i != e; ++i)
+    PDiff.addPressureChange(RegOpers.Defs[i], true, MRI);
+
   for (unsigned i = 0, e = RegOpers.Uses.size(); i != e; ++i)
     PDiff.addPressureChange(RegOpers.Uses[i], false, MRI);
 }
@@ -437,9 +436,13 @@ void RegPressureTracker::discoverLiveOut(unsigned Reg) {
   increaseSetPressure(P.MaxSetPressure, MRI->getPressureSets(Reg));
 }
 
-/// Recede across the previous instruction.
-/// Record the pressure difference if it is provided.
-bool RegPressureTracker::recede(PressureDiff *PDiff) {
+/// Recede across the previous instruction. If LiveUses is provided, record any
+/// RegUnits that are made live by the current instruction's uses. This includes
+/// registers that are both defined and used by the instruction.  If a pressure
+/// difference pointer is provided record the changes is pressure caused by this
+/// instruction independent of liveness.
+bool RegPressureTracker::recede(SmallVectorImpl<unsigned> *LiveUses,
+                                PressureDiff *PDiff) {
   // Check for the top of the analyzable region.
   if (CurrPos == MBB->begin()) {
     closeRegion();
@@ -496,11 +499,16 @@ bool RegPressureTracker::recede(PressureDiff *PDiff) {
       // Adjust liveouts if LiveIntervals are available.
       if (RequireIntervals) {
         const LiveInterval *LI = getInterval(Reg);
-        if (LI && !LI->isKilledAtInstr(SlotIdx))
-          discoverLiveOut(Reg);
+        // Check if this LR is killed and not redefined here.
+        if (LI && !LI->isKilledAtInstr(SlotIdx)
+            && !LI->isDefinedByInstr(SlotIdx)) {
+            discoverLiveOut(Reg);
+        }
       }
       increaseRegPressure(Reg);
       LiveRegs.insert(Reg);
+      if (LiveUses && !containsReg(*LiveUses, Reg))
+        LiveUses->push_back(Reg);
     }
   }
   if (TrackUntiedDefs) {
@@ -773,22 +781,10 @@ getMaxUpwardPressureDelta(const MachineInstr *MI, PressureDiff *PDiff,
 /// @param MaxPressureLimit Is the max pressure within the region, not
 /// necessarily at the current position.
 void RegPressureTracker::
-getUpwardPressureDelta(const MachineInstr *MI, /*const*/ PressureDiff &PDiff1,
+getUpwardPressureDelta(const MachineInstr *MI, /*const*/ PressureDiff &PDiff,
                        RegPressureDelta &Delta,
                        ArrayRef<PressureChange> CriticalPSets,
                        ArrayRef<unsigned> MaxPressureLimit) const {
-  RegisterOperands RegOpers(TRI, MRI, /*IgnoreDead=*/true);
-  collectOperands(MI, RegOpers);
-
-  // Decrease the pressure change for live uses.
-  PressureDiff PDiff = PDiff1;
-  for (unsigned i = 0, e = RegOpers.Uses.size(); i != e; ++i) {
-    if (LiveRegs.contains(RegOpers.Uses[i]))
-      PDiff.addPressureChange(RegOpers.Uses[i], true, MRI);
-  }
-
-  // Now directly query pressure from PDiff. Everything above this can be
-  // cached and updated independent of the query.
   unsigned CritIdx = 0, CritEnd = CriticalPSets.size();
   for (PressureDiff::const_iterator
          PDiffI = PDiff.begin(), PDiffE = PDiff.end();

llvm/lib/CodeGen/ScheduleDAGInstrs.cpp

@@ -408,7 +408,13 @@ void ScheduleDAGInstrs::addVRegUseDeps(SUnit *SU, unsigned OperIdx) {
   unsigned Reg = MI->getOperand(OperIdx).getReg();
 
   // Record this local VReg use.
-  VRegUses.insert(VReg2SUnit(Reg, SU));
+  VReg2UseMap::iterator UI = VRegUses.find(Reg);
+  for (; UI != VRegUses.end(); ++UI) {
+    if (UI->SU == SU)
+      break;
+  }
+  if (UI == VRegUses.end())
+    VRegUses.insert(VReg2SUnit(Reg, SU));
 
   // Lookup this operand's reaching definition.
   assert(LIS && "vreg dependencies requires LiveIntervals");
@@ -755,7 +761,7 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA,
 
     if (RPTracker) {
       PressureDiff *PDiff = PDiffs ? &(*PDiffs)[SU->NodeNum] : 0;
-      RPTracker->recede(PDiff);
+      RPTracker->recede(/*LiveUses=*/0, PDiff);
       assert(RPTracker->getPos() == prior(MII) && "RPTracker can't find MI");
     }