diff --git a/llvm/lib/CodeGen/RegisterPressure.cpp b/llvm/lib/CodeGen/RegisterPressure.cpp index e07efd33021f..a21d6c1d4d63 100644 --- a/llvm/lib/CodeGen/RegisterPressure.cpp +++ b/llvm/lib/CodeGen/RegisterPressure.cpp @@ -382,7 +382,7 @@ static void removeRegLanes(SmallVectorImpl &RegUnits, static LaneBitmask getLanesWithProperty(const LiveIntervals &LIS, const MachineRegisterInfo &MRI, bool TrackLaneMasks, unsigned RegUnit, - SlotIndex Pos, + SlotIndex Pos, LaneBitmask SafeDefault, bool(*Property)(const LiveRange &LR, SlotIndex Pos)) { if (TargetRegisterInfo::isVirtualRegister(RegUnit)) { const LiveInterval &LI = LIS.getInterval(RegUnit); @@ -402,7 +402,7 @@ static LaneBitmask getLanesWithProperty(const LiveIntervals &LIS, // Be prepared for missing liveranges: We usually do not compute liveranges // for physical registers on targets with many registers (GPUs). if (LR == nullptr) - return 0; + return SafeDefault; return Property(*LR, Pos) ? ~0u : 0; } } @@ -411,7 +411,7 @@ static LaneBitmask getLiveLanesAt(const LiveIntervals &LIS, const MachineRegisterInfo &MRI, bool TrackLaneMasks, unsigned RegUnit, SlotIndex Pos) { - return getLanesWithProperty(LIS, MRI, TrackLaneMasks, RegUnit, Pos, + return getLanesWithProperty(LIS, MRI, TrackLaneMasks, RegUnit, Pos, ~0u, [](const LiveRange &LR, SlotIndex Pos) { return LR.liveAt(Pos); }); @@ -570,11 +570,11 @@ void RegisterOperands::adjustLaneLiveness(const LiveIntervals &LIS, AddFlagsMI != nullptr && (LiveAfter & ~I->LaneMask) == 0) AddFlagsMI->setRegisterDefReadUndef(RegUnit); - LaneBitmask LaneMask = I->LaneMask & LiveAfter; - if (LaneMask == 0) { + LaneBitmask ActualDef = I->LaneMask & LiveAfter; + if (ActualDef == 0) { I = Defs.erase(I); } else { - I->LaneMask = LaneMask; + I->LaneMask = ActualDef; ++I; } } @@ -790,9 +790,11 @@ void RegPressureTracker::recede(const RegisterOperands &RegOpers, } // Discover live outs if this may be the first occurance of this register. - LaneBitmask LiveOut = getLiveThroughAt(Reg, SlotIdx); - if (LiveOut != 0) - discoverLiveOut(RegisterMaskPair(Reg, LiveOut)); + if (RequireIntervals) { + LaneBitmask LiveOut = getLiveThroughAt(Reg, SlotIdx); + if (LiveOut != 0) + discoverLiveOut(RegisterMaskPair(Reg, LiveOut)); + } } increaseRegPressure(Reg, PreviousMask, NewMask); @@ -875,10 +877,12 @@ void RegPressureTracker::advance(const RegisterOperands &RegOpers) { LiveRegs.insert(RegisterMaskPair(Reg, LiveIn)); } // Kill liveness at last uses. - LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx); - if (LastUseMask != 0) { - LiveRegs.erase(RegisterMaskPair(Reg, LastUseMask)); - decreaseRegPressure(Reg, LiveMask, LiveMask & ~LastUseMask); + if (RequireIntervals) { + LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx); + if (LastUseMask != 0) { + LiveRegs.erase(RegisterMaskPair(Reg, LastUseMask)); + decreaseRegPressure(Reg, LiveMask, LiveMask & ~LastUseMask); + } } } @@ -1197,10 +1201,8 @@ static LaneBitmask findUseBetween(unsigned Reg, LaneBitmask LastUseMask, LaneBitmask RegPressureTracker::getLiveLanesAt(unsigned RegUnit, SlotIndex Pos) const { - if (!RequireIntervals) - return 0; - - return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, Pos, + assert(RequireIntervals); + return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, Pos, ~0u, [](const LiveRange &LR, SlotIndex Pos) { return LR.liveAt(Pos); }); @@ -1208,11 +1210,9 @@ LaneBitmask RegPressureTracker::getLiveLanesAt(unsigned RegUnit, LaneBitmask RegPressureTracker::getLastUsedLanes(unsigned RegUnit, SlotIndex Pos) const { - if (!RequireIntervals) - return 0; - + assert(RequireIntervals); return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, - Pos.getBaseIndex(), + Pos.getBaseIndex(), 0, [](const LiveRange &LR, SlotIndex Pos) { const LiveRange::Segment *S = LR.getSegmentContaining(Pos); return S != nullptr && S->end == Pos.getRegSlot(); @@ -1221,10 +1221,8 @@ LaneBitmask RegPressureTracker::getLastUsedLanes(unsigned RegUnit, LaneBitmask RegPressureTracker::getLiveThroughAt(unsigned RegUnit, SlotIndex Pos) const { - if (!RequireIntervals) - return 0; - - return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, Pos, + assert(RequireIntervals); + return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, Pos, 0u, [](const LiveRange &LR, SlotIndex Pos) { const LiveRange::Segment *S = LR.getSegmentContaining(Pos); return S != nullptr && S->start < Pos.getRegSlot(true) && @@ -1251,12 +1249,12 @@ void RegPressureTracker::bumpDownwardPressure(const MachineInstr *MI) { if (TrackLaneMasks) RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx); - for (const RegisterMaskPair &Use : RegOpers.Uses) { - unsigned Reg = Use.RegUnit; - LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx); - if (LastUseMask == 0) - continue; - if (RequireIntervals) { + if (RequireIntervals) { + for (const RegisterMaskPair &Use : RegOpers.Uses) { + unsigned Reg = Use.RegUnit; + LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx); + if (LastUseMask == 0) + continue; // The LastUseMask is queried from the liveness information of instruction // which may be further down the schedule. Some lanes may actually not be // last uses for the current position. @@ -1267,11 +1265,11 @@ void RegPressureTracker::bumpDownwardPressure(const MachineInstr *MI) { = findUseBetween(Reg, LastUseMask, CurrIdx, SlotIdx, *MRI, LIS); if (LastUseMask == 0) continue; - } - LaneBitmask LiveMask = LiveRegs.contains(Reg); - LaneBitmask NewMask = LiveMask & ~LastUseMask; - decreaseRegPressure(Reg, LiveMask, NewMask); + LaneBitmask LiveMask = LiveRegs.contains(Reg); + LaneBitmask NewMask = LiveMask & ~LastUseMask; + decreaseRegPressure(Reg, LiveMask, NewMask); + } } // Generate liveness for defs. diff --git a/llvm/test/CodeGen/AMDGPU/captured-frame-index.ll b/llvm/test/CodeGen/AMDGPU/captured-frame-index.ll index f1ab48b5048b..6cf19677812a 100644 --- a/llvm/test/CodeGen/AMDGPU/captured-frame-index.ll +++ b/llvm/test/CodeGen/AMDGPU/captured-frame-index.ll @@ -1,10 +1,10 @@ ; RUN: llc -march=amdgcn -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s ; GCN-LABEL: {{^}}stored_fi_to_lds: -; GCN-DAG: s_load_dword [[LDSPTR:s[0-9]+]] -; GCN-DAG: v_mov_b32_e32 [[ZERO1:v[0-9]+]], 0{{$}} -; GCN-DAG: buffer_store_dword v{{[0-9]+}}, [[ZERO1]] +; GCN: s_load_dword [[LDSPTR:s[0-9]+]] ; GCN: v_mov_b32_e32 [[ZERO0:v[0-9]+]], 0{{$}} +; GCN: v_mov_b32_e32 [[ZERO1:v[0-9]+]], 0{{$}} +; GCN: buffer_store_dword v{{[0-9]+}}, [[ZERO1]] ; GCN: v_mov_b32_e32 [[VLDSPTR:v[0-9]+]], [[LDSPTR]] ; GCN: ds_write_b32 [[VLDSPTR]], [[ZERO0]] define void @stored_fi_to_lds(float* addrspace(3)* %ptr) #0 { diff --git a/llvm/test/CodeGen/AMDGPU/valu-i1.ll b/llvm/test/CodeGen/AMDGPU/valu-i1.ll index ffa0110a14b3..c9776dd74600 100644 --- a/llvm/test/CodeGen/AMDGPU/valu-i1.ll +++ b/llvm/test/CodeGen/AMDGPU/valu-i1.ll @@ -137,7 +137,7 @@ exit: ; SI: BB#4: ; SI: buffer_store_dword -; SI: v_cmp_ge_i64_e64 [[CMP:s\[[0-9]+:[0-9]+\]]] +; SI: v_cmp_ge_i64_e32 [[CMP:s\[[0-9]+:[0-9]+\]|vcc]] ; SI: s_or_b64 [[TMP:s\[[0-9]+:[0-9]+\]]], [[CMP]], [[COND_STATE]] ; SI: BB3_5: