maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity
llvm-project/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp

628 lines
20 KiB
C++
Raw Blame History

//===- SIMemoryLegalizer.cpp ----------------------------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
/// \file
/// \brief Memory legalizer - implements memory model. More information can be
/// found here:
/// http://llvm.org/docs/AMDGPUUsage.html#memory-model
//
//===----------------------------------------------------------------------===//
#include "AMDGPU.h"
#include "AMDGPUMachineModuleInfo.h"
#include "AMDGPUSubtarget.h"
#include "SIDefines.h"
#include "SIInstrInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/MC/MCInstrDesc.h"
#include "llvm/Pass.h"
#include "llvm/Support/AtomicOrdering.h"
#include <cassert>
#include <list>
using namespace llvm;
using namespace llvm::AMDGPU;
#define DEBUG_TYPE "si-memory-legalizer"
#define PASS_NAME "SI Memory Legalizer"
namespace {
class SIMemOpInfo final {
private:
SyncScope::ID SSID = SyncScope::System;
AtomicOrdering Ordering = AtomicOrdering::NotAtomic;
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic;
bool IsNonTemporal = false;
SIMemOpInfo(SyncScope::ID SSID, AtomicOrdering Ordering)
: SSID(SSID), Ordering(Ordering) {}
SIMemOpInfo(SyncScope::ID SSID, AtomicOrdering Ordering,
AtomicOrdering FailureOrdering, bool IsNonTemporal = false)
: SSID(SSID), Ordering(Ordering), FailureOrdering(FailureOrdering),
IsNonTemporal(IsNonTemporal) {}
/// \returns Info constructed from \p MI, which has at least machine memory
/// operand.
static Optional<SIMemOpInfo> constructFromMIWithMMO(
const MachineBasicBlock::iterator &MI);
public:
/// \returns Synchronization scope ID of the machine instruction used to
/// create this SIMemOpInfo.
SyncScope::ID getSSID() const {
return SSID;
}
/// \returns Ordering constraint of the machine instruction used to
/// create this SIMemOpInfo.
AtomicOrdering getOrdering() const {
return Ordering;
}
/// \returns Failure ordering constraint of the machine instruction used to
/// create this SIMemOpInfo.
AtomicOrdering getFailureOrdering() const {
return FailureOrdering;
}
/// \returns True if memory access of the machine instruction used to
/// create this SIMemOpInfo is non-temporal, false otherwise.
bool isNonTemporal() const {
return IsNonTemporal;
}
/// \returns True if ordering constraint of the machine instruction used to
/// create this SIMemOpInfo is unordered or higher, false otherwise.
bool isAtomic() const {
return Ordering != AtomicOrdering::NotAtomic;
}
/// \returns Load info if \p MI is a load operation, "None" otherwise.
static Optional<SIMemOpInfo> getLoadInfo(
const MachineBasicBlock::iterator &MI);
/// \returns Store info if \p MI is a store operation, "None" otherwise.
static Optional<SIMemOpInfo> getStoreInfo(
const MachineBasicBlock::iterator &MI);
/// \returns Atomic fence info if \p MI is an atomic fence operation,
/// "None" otherwise.
static Optional<SIMemOpInfo> getAtomicFenceInfo(
const MachineBasicBlock::iterator &MI);
/// \returns Atomic cmpxchg info if \p MI is an atomic cmpxchg operation,
/// "None" otherwise.
static Optional<SIMemOpInfo> getAtomicCmpxchgInfo(
const MachineBasicBlock::iterator &MI);
/// \returns Atomic rmw info if \p MI is an atomic rmw operation,
/// "None" otherwise.
static Optional<SIMemOpInfo> getAtomicRmwInfo(
const MachineBasicBlock::iterator &MI);
/// \brief Reports unknown synchronization scope used in \p MI to LLVM
/// context.
static void reportUnknownSyncScope(
const MachineBasicBlock::iterator &MI);
};
class SIMemoryLegalizer final : public MachineFunctionPass {
private:
/// \brief Machine module info.
const AMDGPUMachineModuleInfo *MMI = nullptr;
/// \brief Instruction info.
const SIInstrInfo *TII = nullptr;
/// \brief Immediate for "vmcnt(0)".
unsigned Vmcnt0Immediate = 0;
/// \brief Opcode for cache invalidation instruction (L1).
unsigned Wbinvl1Opcode = 0;
/// \brief List of atomic pseudo instructions.
std::list<MachineBasicBlock::iterator> AtomicPseudoMIs;
/// \brief Sets named bit (BitName) to "true" if present in \p MI. Returns
/// true if \p MI is modified, false otherwise.
template <uint16_t BitName>
bool enableNamedBit(const MachineBasicBlock::iterator &MI) const {
int BitIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), BitName);
if (BitIdx == -1)
return false;
MachineOperand &Bit = MI->getOperand(BitIdx);
if (Bit.getImm() != 0)
return false;
Bit.setImm(1);
return true;
}
/// \brief Sets GLC bit to "true" if present in \p MI. Returns true if \p MI
/// is modified, false otherwise.
bool enableGLCBit(const MachineBasicBlock::iterator &MI) const {
return enableNamedBit<AMDGPU::OpName::glc>(MI);
}
/// \brief Sets SLC bit to "true" if present in \p MI. Returns true if \p MI
/// is modified, false otherwise.
bool enableSLCBit(const MachineBasicBlock::iterator &MI) const {
return enableNamedBit<AMDGPU::OpName::slc>(MI);
}
/// \brief Inserts "buffer_wbinvl1_vol" instruction \p Before or after \p MI.
/// Always returns true.
bool insertBufferWbinvl1Vol(MachineBasicBlock::iterator &MI,
bool Before = true) const;
/// \brief Inserts "s_waitcnt vmcnt(0)" instruction \p Before or after \p MI.
/// Always returns true.
bool insertWaitcntVmcnt0(MachineBasicBlock::iterator &MI,
bool Before = true) const;
/// \brief Removes all processed atomic pseudo instructions from the current
/// function. Returns true if current function is modified, false otherwise.
bool removeAtomicPseudoMIs();
/// \brief Expands load operation \p MI. Returns true if instructions are
/// added/deleted or \p MI is modified, false otherwise.
bool expandLoad(const SIMemOpInfo &MOI,
MachineBasicBlock::iterator &MI);
/// \brief Expands store operation \p MI. Returns true if instructions are
/// added/deleted or \p MI is modified, false otherwise.
bool expandStore(const SIMemOpInfo &MOI,
MachineBasicBlock::iterator &MI);
/// \brief Expands atomic fence operation \p MI. Returns true if
/// instructions are added/deleted or \p MI is modified, false otherwise.
bool expandAtomicFence(const SIMemOpInfo &MOI,
MachineBasicBlock::iterator &MI);
/// \brief Expands atomic cmpxchg operation \p MI. Returns true if
/// instructions are added/deleted or \p MI is modified, false otherwise.
bool expandAtomicCmpxchg(const SIMemOpInfo &MOI,
MachineBasicBlock::iterator &MI);
/// \brief Expands atomic rmw operation \p MI. Returns true if
/// instructions are added/deleted or \p MI is modified, false otherwise.
bool expandAtomicRmw(const SIMemOpInfo &MOI,
MachineBasicBlock::iterator &MI);
public:
static char ID;
SIMemoryLegalizer() : MachineFunctionPass(ID) {}
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesCFG();
MachineFunctionPass::getAnalysisUsage(AU);
}
StringRef getPassName() const override {
return PASS_NAME;
}
bool runOnMachineFunction(MachineFunction &MF) override;
};
} // end namespace anonymous
/* static */
Optional<SIMemOpInfo> SIMemOpInfo::constructFromMIWithMMO(
const MachineBasicBlock::iterator &MI) {
assert(MI->getNumMemOperands() > 0);
const MachineFunction *MF = MI->getParent()->getParent();
const AMDGPUMachineModuleInfo *MMI =
&MF->getMMI().getObjFileInfo<AMDGPUMachineModuleInfo>();
SyncScope::ID SSID = SyncScope::SingleThread;
AtomicOrdering Ordering = AtomicOrdering::NotAtomic;
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic;
bool IsNonTemporal = true;
// Validator should check whether or not MMOs cover the entire set of
// locations accessed by the memory instruction.
for (const auto &MMO : MI->memoperands()) {
const auto &IsSyncScopeInclusion =
MMI->isSyncScopeInclusion(SSID, MMO->getSyncScopeID());
if (!IsSyncScopeInclusion) {
reportUnknownSyncScope(MI);
return None;
}
SSID = IsSyncScopeInclusion.getValue() ? SSID : MMO->getSyncScopeID();
Ordering =
isStrongerThan(Ordering, MMO->getOrdering()) ?
Ordering : MMO->getOrdering();
FailureOrdering =
isStrongerThan(FailureOrdering, MMO->getFailureOrdering()) ?
FailureOrdering : MMO->getFailureOrdering();
if (!(MMO->getFlags() & MachineMemOperand::MONonTemporal))
IsNonTemporal = false;
}
return SIMemOpInfo(SSID, Ordering, FailureOrdering, IsNonTemporal);
}
/* static */
Optional<SIMemOpInfo> SIMemOpInfo::getLoadInfo(
const MachineBasicBlock::iterator &MI) {
assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
if (!(MI->mayLoad() && !MI->mayStore()))
return None;
// Be conservative if there are no memory operands.
if (MI->getNumMemOperands() == 0)
return SIMemOpInfo(SyncScope::System,
AtomicOrdering::SequentiallyConsistent);
return SIMemOpInfo::constructFromMIWithMMO(MI);
}
/* static */
Optional<SIMemOpInfo> SIMemOpInfo::getStoreInfo(
const MachineBasicBlock::iterator &MI) {
assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
if (!(!MI->mayLoad() && MI->mayStore()))
return None;
// Be conservative if there are no memory operands.
if (MI->getNumMemOperands() == 0)
return SIMemOpInfo(SyncScope::System,
AtomicOrdering::SequentiallyConsistent);
return SIMemOpInfo::constructFromMIWithMMO(MI);
}
/* static */
Optional<SIMemOpInfo> SIMemOpInfo::getAtomicFenceInfo(
const MachineBasicBlock::iterator &MI) {
assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
if (MI->getOpcode() != AMDGPU::ATOMIC_FENCE)
return None;
SyncScope::ID SSID =
static_cast<SyncScope::ID>(MI->getOperand(1).getImm());
AtomicOrdering Ordering =
static_cast<AtomicOrdering>(MI->getOperand(0).getImm());
return SIMemOpInfo(SSID, Ordering);
}
/* static */
Optional<SIMemOpInfo> SIMemOpInfo::getAtomicCmpxchgInfo(
const MachineBasicBlock::iterator &MI) {
assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
if (!(MI->mayLoad() && MI->mayStore()))
return None;
// Be conservative if there are no memory operands.
if (MI->getNumMemOperands() == 0)
return SIMemOpInfo(SyncScope::System,
AtomicOrdering::SequentiallyConsistent,
AtomicOrdering::SequentiallyConsistent);
return SIMemOpInfo::constructFromMIWithMMO(MI);
}
/* static */
Optional<SIMemOpInfo> SIMemOpInfo::getAtomicRmwInfo(
const MachineBasicBlock::iterator &MI) {
assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
if (!(MI->mayLoad() && MI->mayStore()))
return None;
// Be conservative if there are no memory operands.
if (MI->getNumMemOperands() == 0)
return SIMemOpInfo(SyncScope::System,
AtomicOrdering::SequentiallyConsistent);
return SIMemOpInfo::constructFromMIWithMMO(MI);
}
/* static */
void SIMemOpInfo::reportUnknownSyncScope(
const MachineBasicBlock::iterator &MI) {
DiagnosticInfoUnsupported Diag(MI->getParent()->getParent()->getFunction(),
"Unsupported synchronization scope");
LLVMContext *CTX = &MI->getParent()->getParent()->getFunction().getContext();
CTX->diagnose(Diag);
}
bool SIMemoryLegalizer::insertBufferWbinvl1Vol(MachineBasicBlock::iterator &MI,
bool Before) const {
MachineBasicBlock &MBB = *MI->getParent();
DebugLoc DL = MI->getDebugLoc();
if (!Before)
++MI;
BuildMI(MBB, MI, DL, TII->get(Wbinvl1Opcode));
if (!Before)
--MI;
return true;
}
bool SIMemoryLegalizer::insertWaitcntVmcnt0(MachineBasicBlock::iterator &MI,
bool Before) const {
MachineBasicBlock &MBB = *MI->getParent();
DebugLoc DL = MI->getDebugLoc();
if (!Before)
++MI;
BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAITCNT)).addImm(Vmcnt0Immediate);
if (!Before)
--MI;
return true;
}
bool SIMemoryLegalizer::removeAtomicPseudoMIs() {
if (AtomicPseudoMIs.empty())
return false;
for (auto &MI : AtomicPseudoMIs)
MI->eraseFromParent();
AtomicPseudoMIs.clear();
return true;
}
bool SIMemoryLegalizer::expandLoad(const SIMemOpInfo &MOI,
MachineBasicBlock::iterator &MI) {
assert(MI->mayLoad() && !MI->mayStore());
bool Changed = false;
if (MOI.isAtomic()) {
if (MOI.getSSID() == SyncScope::System ||
MOI.getSSID() == MMI->getAgentSSID()) {
if (MOI.getOrdering() == AtomicOrdering::Acquire ||
MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
Changed |= enableGLCBit(MI);
if (MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
Changed |= insertWaitcntVmcnt0(MI);
if (MOI.getOrdering() == AtomicOrdering::Acquire ||
MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) {
Changed |= insertWaitcntVmcnt0(MI, false);
Changed |= insertBufferWbinvl1Vol(MI, false);
}
return Changed;
}
if (MOI.getSSID() == SyncScope::SingleThread ||
MOI.getSSID() == MMI->getWorkgroupSSID() ||
MOI.getSSID() == MMI->getWavefrontSSID()) {
return Changed;
}
llvm_unreachable("Unsupported synchronization scope");
}
// Atomic instructions do not have the nontemporal attribute.
if (MOI.isNonTemporal()) {
Changed |= enableGLCBit(MI);
Changed |= enableSLCBit(MI);
return Changed;
}
return Changed;
}
bool SIMemoryLegalizer::expandStore(const SIMemOpInfo &MOI,
MachineBasicBlock::iterator &MI) {
assert(!MI->mayLoad() && MI->mayStore());
bool Changed = false;
if (MOI.isAtomic()) {
if (MOI.getSSID() == SyncScope::System ||
MOI.getSSID() == MMI->getAgentSSID()) {
if (MOI.getOrdering() == AtomicOrdering::Release ||
MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
Changed |= insertWaitcntVmcnt0(MI);
return Changed;
}
if (MOI.getSSID() == SyncScope::SingleThread ||
MOI.getSSID() == MMI->getWorkgroupSSID() ||
MOI.getSSID() == MMI->getWavefrontSSID()) {
return Changed;
}
llvm_unreachable("Unsupported synchronization scope");
}
// Atomic instructions do not have the nontemporal attribute.
if (MOI.isNonTemporal()) {
Changed |= enableGLCBit(MI);
Changed |= enableSLCBit(MI);
return Changed;
}
return Changed;
}
bool SIMemoryLegalizer::expandAtomicFence(const SIMemOpInfo &MOI,
MachineBasicBlock::iterator &MI) {
assert(MI->getOpcode() == AMDGPU::ATOMIC_FENCE);
bool Changed = false;
if (MOI.isAtomic()) {
if (MOI.getSSID() == SyncScope::System ||
MOI.getSSID() == MMI->getAgentSSID()) {
if (MOI.getOrdering() == AtomicOrdering::Acquire ||
MOI.getOrdering() == AtomicOrdering::Release ||
MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
Changed |= insertWaitcntVmcnt0(MI);
if (MOI.getOrdering() == AtomicOrdering::Acquire ||
MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
Changed |= insertBufferWbinvl1Vol(MI);
AtomicPseudoMIs.push_back(MI);
return Changed;
}
if (MOI.getSSID() == SyncScope::SingleThread ||
MOI.getSSID() == MMI->getWorkgroupSSID() ||
MOI.getSSID() == MMI->getWavefrontSSID()) {
AtomicPseudoMIs.push_back(MI);
return Changed;
}
SIMemOpInfo::reportUnknownSyncScope(MI);
}
return Changed;
}
bool SIMemoryLegalizer::expandAtomicCmpxchg(const SIMemOpInfo &MOI,
MachineBasicBlock::iterator &MI) {
assert(MI->mayLoad() && MI->mayStore());
bool Changed = false;
if (MOI.isAtomic()) {
if (MOI.getSSID() == SyncScope::System ||
MOI.getSSID() == MMI->getAgentSSID()) {
if (MOI.getOrdering() == AtomicOrdering::Release ||
MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent ||
MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent)
Changed |= insertWaitcntVmcnt0(MI);
if (MOI.getOrdering() == AtomicOrdering::Acquire ||
MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent ||
MOI.getFailureOrdering() == AtomicOrdering::Acquire ||
MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent) {
Changed |= insertWaitcntVmcnt0(MI, false);
Changed |= insertBufferWbinvl1Vol(MI, false);
}
return Changed;
}
if (MOI.getSSID() == SyncScope::SingleThread ||
MOI.getSSID() == MMI->getWorkgroupSSID() ||
MOI.getSSID() == MMI->getWavefrontSSID()) {
Changed |= enableGLCBit(MI);
return Changed;
}
llvm_unreachable("Unsupported synchronization scope");
}
return Changed;
}
bool SIMemoryLegalizer::expandAtomicRmw(const SIMemOpInfo &MOI,
MachineBasicBlock::iterator &MI) {
assert(MI->mayLoad() && MI->mayStore());
bool Changed = false;
if (MOI.isAtomic()) {
if (MOI.getSSID() == SyncScope::System ||
MOI.getSSID() == MMI->getAgentSSID()) {
if (MOI.getOrdering() == AtomicOrdering::Release ||
MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
Changed |= insertWaitcntVmcnt0(MI);
if (MOI.getOrdering() == AtomicOrdering::Acquire ||
MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) {
Changed |= insertWaitcntVmcnt0(MI, false);
Changed |= insertBufferWbinvl1Vol(MI, false);
}
return Changed;
}
if (MOI.getSSID() == SyncScope::SingleThread ||
MOI.getSSID() == MMI->getWorkgroupSSID() ||
MOI.getSSID() == MMI->getWavefrontSSID()) {
Changed |= enableGLCBit(MI);
return Changed;
}
llvm_unreachable("Unsupported synchronization scope");
}
return Changed;
}
bool SIMemoryLegalizer::runOnMachineFunction(MachineFunction &MF) {
bool Changed = false;
const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
const IsaInfo::IsaVersion IV = IsaInfo::getIsaVersion(ST.getFeatureBits());
MMI = &MF.getMMI().getObjFileInfo<AMDGPUMachineModuleInfo>();
TII = ST.getInstrInfo();
Vmcnt0Immediate =
AMDGPU::encodeWaitcnt(IV, 0, getExpcntBitMask(IV), getLgkmcntBitMask(IV));
Wbinvl1Opcode = ST.getGeneration() <= AMDGPUSubtarget::SOUTHERN_ISLANDS ?
AMDGPU::BUFFER_WBINVL1 : AMDGPU::BUFFER_WBINVL1_VOL;
for (auto &MBB : MF) {
for (auto MI = MBB.begin(); MI != MBB.end(); ++MI) {
if (!(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic))
continue;
if (const auto &MOI = SIMemOpInfo::getLoadInfo(MI))
Changed |= expandLoad(MOI.getValue(), MI);
else if (const auto &MOI = SIMemOpInfo::getStoreInfo(MI))
Changed |= expandStore(MOI.getValue(), MI);
else if (const auto &MOI = SIMemOpInfo::getAtomicFenceInfo(MI))
Changed |= expandAtomicFence(MOI.getValue(), MI);
else if (const auto &MOI = SIMemOpInfo::getAtomicCmpxchgInfo(MI))
Changed |= expandAtomicCmpxchg(MOI.getValue(), MI);
else if (const auto &MOI = SIMemOpInfo::getAtomicRmwInfo(MI))
Changed |= expandAtomicRmw(MOI.getValue(), MI);
}
}
Changed |= removeAtomicPseudoMIs();
return Changed;
}
INITIALIZE_PASS(SIMemoryLegalizer, DEBUG_TYPE, PASS_NAME, false, false)
char SIMemoryLegalizer::ID = 0;
char &llvm::SIMemoryLegalizerID = SIMemoryLegalizer::ID;
FunctionPass *llvm::createSIMemoryLegalizerPass() {
return new SIMemoryLegalizer();
}
Reference in New Issue View Git Blame Copy Permalink