From d0943537e10e25281164bb27df843e283dc6666c Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Sun, 12 Jan 2020 15:54:09 -0500 Subject: [PATCH] GlobalISel: Apply target MMO flags to atomics Unify MMO flag handling with SelectionDAG like with loads and stores. --- llvm/include/llvm/CodeGen/TargetLowering.h | 2 + llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 11 +-- .../SelectionDAG/SelectionDAGBuilder.cpp | 14 +-- llvm/lib/CodeGen/TargetLoweringBase.cpp | 19 ++++ .../irtranslator-atomic-metadata.ll | 88 +++++++++++++++++++ 5 files changed, 117 insertions(+), 17 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-atomic-metadata.ll diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 8e3ca15488b8..1714da2c2a9d 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -295,6 +295,8 @@ public: const DataLayout &DL) const; MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const; + MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, + const DataLayout &DL) const; virtual bool isSelectSupported(SelectSupportKind /*kind*/) const { return true; diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 2d2cc2257cda..640ba851f1ad 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1955,9 +1955,8 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U, if (I.isWeak()) return false; - auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile - : MachineMemOperand::MONone; - Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore; + auto &TLI = *MF->getSubtarget().getTargetLowering(); + auto Flags = TLI.getAtomicMemOperandFlags(I, *DL); Type *ResType = I.getType(); Type *ValType = ResType->Type::getStructElementType(0); @@ -1985,10 +1984,8 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U, bool IRTranslator::translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder) { const AtomicRMWInst &I = cast(U); - - auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile - : MachineMemOperand::MONone; - Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore; + auto &TLI = *MF->getSubtarget().getTargetLowering(); + auto Flags = TLI.getAtomicMemOperandFlags(I, *DL); Type *ResType = I.getType(); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 3c405f40a6e4..eef899034549 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4619,11 +4619,8 @@ void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other); auto Alignment = DAG.getEVTAlignment(MemVT); - - auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; - if (I.isVolatile()) - Flags |= MachineMemOperand::MOVolatile; - Flags |= DAG.getTargetLoweringInfo().getTargetMMOFlags(I); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout()); MachineFunction &MF = DAG.getMachineFunction(); MachineMemOperand *MMO = @@ -4670,11 +4667,8 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) { auto MemVT = getValue(I.getValOperand()).getSimpleValueType(); auto Alignment = DAG.getEVTAlignment(MemVT); - - auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; - if (I.isVolatile()) - Flags |= MachineMemOperand::MOVolatile; - Flags |= DAG.getTargetLoweringInfo().getTargetMMOFlags(I); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout()); MachineFunction &MF = DAG.getMachineFunction(); MachineMemOperand *MMO = diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp index 25173d64f8af..db077bb678ac 100644 --- a/llvm/lib/CodeGen/TargetLoweringBase.cpp +++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -2042,3 +2042,22 @@ TargetLoweringBase::getStoreMemOperandFlags(const StoreInst &SI, Flags |= getTargetMMOFlags(SI); return Flags; } + +MachineMemOperand::Flags +TargetLoweringBase::getAtomicMemOperandFlags(const Instruction &AI, + const DataLayout &DL) const { + auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; + + if (const AtomicRMWInst *RMW = dyn_cast(&AI)) { + if (RMW->isVolatile()) + Flags |= MachineMemOperand::MOVolatile; + } else if (const AtomicCmpXchgInst *CmpX = dyn_cast(&AI)) { + if (CmpX->isVolatile()) + Flags |= MachineMemOperand::MOVolatile; + } else + llvm_unreachable("not an atomic instruction"); + + // FIXME: Not preserving dereferenceable + Flags |= getTargetMMOFlags(AI); + return Flags; +} diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-atomic-metadata.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-atomic-metadata.ll new file mode 100644 index 000000000000..e1c59711c5af --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-atomic-metadata.ll @@ -0,0 +1,88 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +; RUN: llc -mtriple=aarch64-- -mcpu=falkor -mattr=+lse -O0 -aarch64-enable-atomic-cfg-tidy=0 -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - | FileCheck %s + +define i32 @atomicrmw_volatile(i32* %ptr) { + ; CHECK-LABEL: name: atomicrmw_volatile + ; CHECK: bb.1 (%ir-block.0): + ; CHECK: liveins: $x0 + ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[C]] :: (volatile load store monotonic 4 on %ir.ptr) + ; CHECK: $w0 = COPY [[ATOMICRMW_ADD]](s32) + ; CHECK: RET_ReallyLR implicit $w0 + %oldval = atomicrmw volatile add i32* %ptr, i32 1 monotonic + ret i32 %oldval +} + +define i32 @atomicrmw_falkor(i32* %ptr) { + ; CHECK-LABEL: name: atomicrmw_falkor + ; CHECK: bb.1 (%ir-block.0): + ; CHECK: liveins: $x0 + ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[C]] :: ("aarch64-strided-access" load store monotonic 4 on %ir.ptr) + ; CHECK: $w0 = COPY [[ATOMICRMW_ADD]](s32) + ; CHECK: RET_ReallyLR implicit $w0 + %oldval = atomicrmw add i32* %ptr, i32 1 monotonic, !falkor.strided.access !0 + ret i32 %oldval +} + +define i32 @atomicrmw_volatile_falkor(i32* %ptr) { + ; CHECK-LABEL: name: atomicrmw_volatile_falkor + ; CHECK: bb.1 (%ir-block.0): + ; CHECK: liveins: $x0 + ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[C]] :: (volatile "aarch64-strided-access" load store monotonic 4 on %ir.ptr) + ; CHECK: $w0 = COPY [[ATOMICRMW_ADD]](s32) + ; CHECK: RET_ReallyLR implicit $w0 + %oldval = atomicrmw volatile add i32* %ptr, i32 1 monotonic, !falkor.strided.access !0 + ret i32 %oldval +} + +define i32 @cmpxchg_volatile(i32* %addr) { + ; CHECK-LABEL: name: cmpxchg_volatile + ; CHECK: bb.1 (%ir-block.0): + ; CHECK: liveins: $x0 + ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 + ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; CHECK: [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(s32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p0), [[C]], [[C1]] :: (volatile load store monotonic monotonic 4 on %ir.addr) + ; CHECK: $w0 = COPY [[ATOMIC_CMPXCHG_WITH_SUCCESS]](s32) + ; CHECK: RET_ReallyLR implicit $w0 + %val_success = cmpxchg volatile i32* %addr, i32 0, i32 1 monotonic monotonic + %value_loaded = extractvalue { i32, i1 } %val_success, 0 + ret i32 %value_loaded +} + +define i32 @cmpxchg_falkor(i32* %addr) { + ; CHECK-LABEL: name: cmpxchg_falkor + ; CHECK: bb.1 (%ir-block.0): + ; CHECK: liveins: $x0 + ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 + ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; CHECK: [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(s32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p0), [[C]], [[C1]] :: ("aarch64-strided-access" load store monotonic monotonic 4 on %ir.addr) + ; CHECK: $w0 = COPY [[ATOMIC_CMPXCHG_WITH_SUCCESS]](s32) + ; CHECK: RET_ReallyLR implicit $w0 + %val_success = cmpxchg i32* %addr, i32 0, i32 1 monotonic monotonic, !falkor.strided.access !0 + %value_loaded = extractvalue { i32, i1 } %val_success, 0 + ret i32 %value_loaded +} + +define i32 @cmpxchg_volatile_falkor(i32* %addr) { + ; CHECK-LABEL: name: cmpxchg_volatile_falkor + ; CHECK: bb.1 (%ir-block.0): + ; CHECK: liveins: $x0 + ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 + ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; CHECK: [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(s32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p0), [[C]], [[C1]] :: (volatile "aarch64-strided-access" load store monotonic monotonic 4 on %ir.addr) + ; CHECK: $w0 = COPY [[ATOMIC_CMPXCHG_WITH_SUCCESS]](s32) + ; CHECK: RET_ReallyLR implicit $w0 + %val_success = cmpxchg volatile i32* %addr, i32 0, i32 1 monotonic monotonic, !falkor.strided.access !0 + %value_loaded = extractvalue { i32, i1 } %val_success, 0 + ret i32 %value_loaded +} + +!0 = !{}