2016-09-29 09:44:16 +08:00
|
|
|
//===-- SIOptimizeExecMasking.cpp -----------------------------------------===//
|
|
|
|
//
|
2019-01-19 16:50:56 +08:00
|
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
2016-09-29 09:44:16 +08:00
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
#include "AMDGPU.h"
|
2021-01-20 20:48:02 +08:00
|
|
|
#include "GCNSubtarget.h"
|
|
|
|
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
|
2016-09-29 09:44:16 +08:00
|
|
|
#include "llvm/CodeGen/MachineFunctionPass.h"
|
Sink all InitializePasses.h includes
This file lists every pass in LLVM, and is included by Pass.h, which is
very popular. Every time we add, remove, or rename a pass in LLVM, it
caused lots of recompilation.
I found this fact by looking at this table, which is sorted by the
number of times a file was changed over the last 100,000 git commits
multiplied by the number of object files that depend on it in the
current checkout:
recompiles touches affected_files header
342380 95 3604 llvm/include/llvm/ADT/STLExtras.h
314730 234 1345 llvm/include/llvm/InitializePasses.h
307036 118 2602 llvm/include/llvm/ADT/APInt.h
213049 59 3611 llvm/include/llvm/Support/MathExtras.h
170422 47 3626 llvm/include/llvm/Support/Compiler.h
162225 45 3605 llvm/include/llvm/ADT/Optional.h
158319 63 2513 llvm/include/llvm/ADT/Triple.h
140322 39 3598 llvm/include/llvm/ADT/StringRef.h
137647 59 2333 llvm/include/llvm/Support/Error.h
131619 73 1803 llvm/include/llvm/Support/FileSystem.h
Before this change, touching InitializePasses.h would cause 1345 files
to recompile. After this change, touching it only causes 550 compiles in
an incremental rebuild.
Reviewers: bkramer, asbirlea, bollu, jdoerfert
Differential Revision: https://reviews.llvm.org/D70211
2019-11-14 05:15:01 +08:00
|
|
|
#include "llvm/InitializePasses.h"
|
2016-09-29 09:44:16 +08:00
|
|
|
|
|
|
|
using namespace llvm;
|
|
|
|
|
|
|
|
#define DEBUG_TYPE "si-optimize-exec-masking"
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
|
|
|
class SIOptimizeExecMasking : public MachineFunctionPass {
|
|
|
|
public:
|
|
|
|
static char ID;
|
|
|
|
|
|
|
|
public:
|
|
|
|
SIOptimizeExecMasking() : MachineFunctionPass(ID) {
|
|
|
|
initializeSIOptimizeExecMaskingPass(*PassRegistry::getPassRegistry());
|
|
|
|
}
|
|
|
|
|
|
|
|
bool runOnMachineFunction(MachineFunction &MF) override;
|
|
|
|
|
2016-10-01 10:56:57 +08:00
|
|
|
StringRef getPassName() const override {
|
2016-09-29 09:44:16 +08:00
|
|
|
return "SI optimize exec mask operations";
|
|
|
|
}
|
|
|
|
|
|
|
|
void getAnalysisUsage(AnalysisUsage &AU) const override {
|
|
|
|
AU.setPreservesCFG();
|
|
|
|
MachineFunctionPass::getAnalysisUsage(AU);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
} // End anonymous namespace.
|
|
|
|
|
|
|
|
INITIALIZE_PASS_BEGIN(SIOptimizeExecMasking, DEBUG_TYPE,
|
|
|
|
"SI optimize exec mask operations", false, false)
|
|
|
|
INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
|
|
|
|
INITIALIZE_PASS_END(SIOptimizeExecMasking, DEBUG_TYPE,
|
|
|
|
"SI optimize exec mask operations", false, false)
|
|
|
|
|
|
|
|
char SIOptimizeExecMasking::ID = 0;
|
|
|
|
|
|
|
|
char &llvm::SIOptimizeExecMaskingID = SIOptimizeExecMasking::ID;
|
|
|
|
|
|
|
|
/// If \p MI is a copy from exec, return the register copied to.
|
2019-12-28 04:21:44 +08:00
|
|
|
static Register isCopyFromExec(const MachineInstr &MI, const GCNSubtarget &ST) {
|
2016-09-29 09:44:16 +08:00
|
|
|
switch (MI.getOpcode()) {
|
|
|
|
case AMDGPU::COPY:
|
|
|
|
case AMDGPU::S_MOV_B64:
|
2019-06-17 01:13:09 +08:00
|
|
|
case AMDGPU::S_MOV_B64_term:
|
|
|
|
case AMDGPU::S_MOV_B32:
|
|
|
|
case AMDGPU::S_MOV_B32_term: {
|
2016-09-29 09:44:16 +08:00
|
|
|
const MachineOperand &Src = MI.getOperand(1);
|
2019-06-17 01:13:09 +08:00
|
|
|
if (Src.isReg() &&
|
|
|
|
Src.getReg() == (ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC))
|
2016-09-29 09:44:16 +08:00
|
|
|
return MI.getOperand(0).getReg();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return AMDGPU::NoRegister;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// If \p MI is a copy to exec, return the register copied from.
|
2019-12-28 04:21:44 +08:00
|
|
|
static Register isCopyToExec(const MachineInstr &MI, const GCNSubtarget &ST) {
|
2016-09-29 09:44:16 +08:00
|
|
|
switch (MI.getOpcode()) {
|
|
|
|
case AMDGPU::COPY:
|
2019-06-17 01:13:09 +08:00
|
|
|
case AMDGPU::S_MOV_B64:
|
|
|
|
case AMDGPU::S_MOV_B32: {
|
2016-09-29 09:44:16 +08:00
|
|
|
const MachineOperand &Dst = MI.getOperand(0);
|
2019-06-17 01:13:09 +08:00
|
|
|
if (Dst.isReg() &&
|
|
|
|
Dst.getReg() == (ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC) &&
|
|
|
|
MI.getOperand(1).isReg())
|
2016-09-29 09:44:16 +08:00
|
|
|
return MI.getOperand(1).getReg();
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case AMDGPU::S_MOV_B64_term:
|
2019-06-17 01:13:09 +08:00
|
|
|
case AMDGPU::S_MOV_B32_term:
|
2016-09-29 09:44:16 +08:00
|
|
|
llvm_unreachable("should have been replaced");
|
|
|
|
}
|
|
|
|
|
2019-12-28 04:21:44 +08:00
|
|
|
return Register();
|
2016-09-29 09:44:16 +08:00
|
|
|
}
|
|
|
|
|
2017-08-02 07:44:35 +08:00
|
|
|
/// If \p MI is a logical operation on an exec value,
|
|
|
|
/// return the register copied to.
|
2019-12-28 04:21:44 +08:00
|
|
|
static Register isLogicalOpOnExec(const MachineInstr &MI) {
|
2017-08-02 07:44:35 +08:00
|
|
|
switch (MI.getOpcode()) {
|
|
|
|
case AMDGPU::S_AND_B64:
|
|
|
|
case AMDGPU::S_OR_B64:
|
|
|
|
case AMDGPU::S_XOR_B64:
|
|
|
|
case AMDGPU::S_ANDN2_B64:
|
|
|
|
case AMDGPU::S_ORN2_B64:
|
|
|
|
case AMDGPU::S_NAND_B64:
|
|
|
|
case AMDGPU::S_NOR_B64:
|
|
|
|
case AMDGPU::S_XNOR_B64: {
|
|
|
|
const MachineOperand &Src1 = MI.getOperand(1);
|
|
|
|
if (Src1.isReg() && Src1.getReg() == AMDGPU::EXEC)
|
|
|
|
return MI.getOperand(0).getReg();
|
|
|
|
const MachineOperand &Src2 = MI.getOperand(2);
|
|
|
|
if (Src2.isReg() && Src2.getReg() == AMDGPU::EXEC)
|
|
|
|
return MI.getOperand(0).getReg();
|
2019-06-17 01:13:09 +08:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
case AMDGPU::S_AND_B32:
|
|
|
|
case AMDGPU::S_OR_B32:
|
|
|
|
case AMDGPU::S_XOR_B32:
|
|
|
|
case AMDGPU::S_ANDN2_B32:
|
|
|
|
case AMDGPU::S_ORN2_B32:
|
|
|
|
case AMDGPU::S_NAND_B32:
|
|
|
|
case AMDGPU::S_NOR_B32:
|
|
|
|
case AMDGPU::S_XNOR_B32: {
|
|
|
|
const MachineOperand &Src1 = MI.getOperand(1);
|
|
|
|
if (Src1.isReg() && Src1.getReg() == AMDGPU::EXEC_LO)
|
|
|
|
return MI.getOperand(0).getReg();
|
|
|
|
const MachineOperand &Src2 = MI.getOperand(2);
|
|
|
|
if (Src2.isReg() && Src2.getReg() == AMDGPU::EXEC_LO)
|
|
|
|
return MI.getOperand(0).getReg();
|
|
|
|
break;
|
2017-08-02 07:44:35 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return AMDGPU::NoRegister;
|
|
|
|
}
|
|
|
|
|
2016-09-29 09:44:16 +08:00
|
|
|
static unsigned getSaveExecOp(unsigned Opc) {
|
|
|
|
switch (Opc) {
|
|
|
|
case AMDGPU::S_AND_B64:
|
|
|
|
return AMDGPU::S_AND_SAVEEXEC_B64;
|
|
|
|
case AMDGPU::S_OR_B64:
|
|
|
|
return AMDGPU::S_OR_SAVEEXEC_B64;
|
|
|
|
case AMDGPU::S_XOR_B64:
|
|
|
|
return AMDGPU::S_XOR_SAVEEXEC_B64;
|
|
|
|
case AMDGPU::S_ANDN2_B64:
|
|
|
|
return AMDGPU::S_ANDN2_SAVEEXEC_B64;
|
|
|
|
case AMDGPU::S_ORN2_B64:
|
|
|
|
return AMDGPU::S_ORN2_SAVEEXEC_B64;
|
|
|
|
case AMDGPU::S_NAND_B64:
|
|
|
|
return AMDGPU::S_NAND_SAVEEXEC_B64;
|
|
|
|
case AMDGPU::S_NOR_B64:
|
|
|
|
return AMDGPU::S_NOR_SAVEEXEC_B64;
|
|
|
|
case AMDGPU::S_XNOR_B64:
|
|
|
|
return AMDGPU::S_XNOR_SAVEEXEC_B64;
|
2019-06-17 01:13:09 +08:00
|
|
|
case AMDGPU::S_AND_B32:
|
|
|
|
return AMDGPU::S_AND_SAVEEXEC_B32;
|
|
|
|
case AMDGPU::S_OR_B32:
|
|
|
|
return AMDGPU::S_OR_SAVEEXEC_B32;
|
|
|
|
case AMDGPU::S_XOR_B32:
|
|
|
|
return AMDGPU::S_XOR_SAVEEXEC_B32;
|
|
|
|
case AMDGPU::S_ANDN2_B32:
|
|
|
|
return AMDGPU::S_ANDN2_SAVEEXEC_B32;
|
|
|
|
case AMDGPU::S_ORN2_B32:
|
|
|
|
return AMDGPU::S_ORN2_SAVEEXEC_B32;
|
|
|
|
case AMDGPU::S_NAND_B32:
|
|
|
|
return AMDGPU::S_NAND_SAVEEXEC_B32;
|
|
|
|
case AMDGPU::S_NOR_B32:
|
|
|
|
return AMDGPU::S_NOR_SAVEEXEC_B32;
|
|
|
|
case AMDGPU::S_XNOR_B32:
|
|
|
|
return AMDGPU::S_XNOR_SAVEEXEC_B32;
|
2016-09-29 09:44:16 +08:00
|
|
|
default:
|
|
|
|
return AMDGPU::INSTRUCTION_LIST_END;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// These are only terminators to get correct spill code placement during
|
2020-07-24 06:05:16 +08:00
|
|
|
// register allocation, so turn them back into normal instructions.
|
2016-09-29 09:44:16 +08:00
|
|
|
static bool removeTerminatorBit(const SIInstrInfo &TII, MachineInstr &MI) {
|
|
|
|
switch (MI.getOpcode()) {
|
2019-06-17 01:13:09 +08:00
|
|
|
case AMDGPU::S_MOV_B32_term: {
|
2020-11-10 11:16:26 +08:00
|
|
|
bool RegSrc = MI.getOperand(1).isReg();
|
|
|
|
MI.setDesc(TII.get(RegSrc ? AMDGPU::COPY : AMDGPU::S_MOV_B32));
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
case AMDGPU::S_MOV_B64_term: {
|
|
|
|
bool RegSrc = MI.getOperand(1).isReg();
|
|
|
|
MI.setDesc(TII.get(RegSrc ? AMDGPU::COPY : AMDGPU::S_MOV_B64));
|
2016-09-29 09:44:16 +08:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
case AMDGPU::S_XOR_B64_term: {
|
|
|
|
// This is only a terminator to get the correct spill code placement during
|
|
|
|
// register allocation.
|
|
|
|
MI.setDesc(TII.get(AMDGPU::S_XOR_B64));
|
|
|
|
return true;
|
|
|
|
}
|
2019-06-17 01:13:09 +08:00
|
|
|
case AMDGPU::S_XOR_B32_term: {
|
|
|
|
// This is only a terminator to get the correct spill code placement during
|
|
|
|
// register allocation.
|
|
|
|
MI.setDesc(TII.get(AMDGPU::S_XOR_B32));
|
|
|
|
return true;
|
|
|
|
}
|
2020-09-11 03:49:09 +08:00
|
|
|
case AMDGPU::S_OR_B64_term: {
|
|
|
|
// This is only a terminator to get the correct spill code placement during
|
|
|
|
// register allocation.
|
|
|
|
MI.setDesc(TII.get(AMDGPU::S_OR_B64));
|
|
|
|
return true;
|
|
|
|
}
|
2019-06-17 01:13:09 +08:00
|
|
|
case AMDGPU::S_OR_B32_term: {
|
|
|
|
// This is only a terminator to get the correct spill code placement during
|
|
|
|
// register allocation.
|
|
|
|
MI.setDesc(TII.get(AMDGPU::S_OR_B32));
|
|
|
|
return true;
|
|
|
|
}
|
2016-09-29 09:44:16 +08:00
|
|
|
case AMDGPU::S_ANDN2_B64_term: {
|
|
|
|
// This is only a terminator to get the correct spill code placement during
|
|
|
|
// register allocation.
|
|
|
|
MI.setDesc(TII.get(AMDGPU::S_ANDN2_B64));
|
|
|
|
return true;
|
|
|
|
}
|
2019-06-17 01:13:09 +08:00
|
|
|
case AMDGPU::S_ANDN2_B32_term: {
|
|
|
|
// This is only a terminator to get the correct spill code placement during
|
|
|
|
// register allocation.
|
|
|
|
MI.setDesc(TII.get(AMDGPU::S_ANDN2_B32));
|
|
|
|
return true;
|
|
|
|
}
|
2021-02-10 11:56:45 +08:00
|
|
|
case AMDGPU::S_AND_B64_term: {
|
|
|
|
// This is only a terminator to get the correct spill code placement during
|
|
|
|
// register allocation.
|
|
|
|
MI.setDesc(TII.get(AMDGPU::S_AND_B64));
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
case AMDGPU::S_AND_B32_term: {
|
|
|
|
// This is only a terminator to get the correct spill code placement during
|
|
|
|
// register allocation.
|
|
|
|
MI.setDesc(TII.get(AMDGPU::S_AND_B32));
|
|
|
|
return true;
|
|
|
|
}
|
2016-09-29 09:44:16 +08:00
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-07-24 06:05:16 +08:00
|
|
|
// Turn all pseudoterminators in the block into their equivalent non-terminator
|
|
|
|
// instructions. Returns the reverse iterator to the first non-terminator
|
|
|
|
// instruction in the block.
|
2016-09-29 09:44:16 +08:00
|
|
|
static MachineBasicBlock::reverse_iterator fixTerminators(
|
|
|
|
const SIInstrInfo &TII,
|
|
|
|
MachineBasicBlock &MBB) {
|
|
|
|
MachineBasicBlock::reverse_iterator I = MBB.rbegin(), E = MBB.rend();
|
2020-07-24 06:05:16 +08:00
|
|
|
|
|
|
|
bool Seen = false;
|
|
|
|
MachineBasicBlock::reverse_iterator FirstNonTerm = I;
|
2016-09-29 09:44:16 +08:00
|
|
|
for (; I != E; ++I) {
|
|
|
|
if (!I->isTerminator())
|
2020-07-24 06:05:16 +08:00
|
|
|
return Seen ? FirstNonTerm : I;
|
|
|
|
|
|
|
|
if (removeTerminatorBit(TII, *I)) {
|
|
|
|
if (!Seen) {
|
|
|
|
FirstNonTerm = I;
|
|
|
|
Seen = true;
|
|
|
|
}
|
|
|
|
}
|
2016-09-29 09:44:16 +08:00
|
|
|
}
|
|
|
|
|
2020-07-24 06:05:16 +08:00
|
|
|
return FirstNonTerm;
|
2016-09-29 09:44:16 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static MachineBasicBlock::reverse_iterator findExecCopy(
|
|
|
|
const SIInstrInfo &TII,
|
2019-06-17 01:13:09 +08:00
|
|
|
const GCNSubtarget &ST,
|
2016-09-29 09:44:16 +08:00
|
|
|
MachineBasicBlock &MBB,
|
|
|
|
MachineBasicBlock::reverse_iterator I,
|
|
|
|
unsigned CopyToExec) {
|
|
|
|
const unsigned InstLimit = 25;
|
|
|
|
|
|
|
|
auto E = MBB.rend();
|
|
|
|
for (unsigned N = 0; N <= InstLimit && I != E; ++I, ++N) {
|
2019-12-28 04:21:44 +08:00
|
|
|
Register CopyFromExec = isCopyFromExec(*I, ST);
|
|
|
|
if (CopyFromExec.isValid())
|
2016-09-29 09:44:16 +08:00
|
|
|
return I;
|
|
|
|
}
|
|
|
|
|
|
|
|
return E;
|
|
|
|
}
|
|
|
|
|
|
|
|
// XXX - Seems LivePhysRegs doesn't work correctly since it will incorrectly
|
2019-07-12 05:19:33 +08:00
|
|
|
// report the register as unavailable because a super-register with a lane mask
|
|
|
|
// is unavailable.
|
2016-09-29 09:44:16 +08:00
|
|
|
static bool isLiveOut(const MachineBasicBlock &MBB, unsigned Reg) {
|
|
|
|
for (MachineBasicBlock *Succ : MBB.successors()) {
|
|
|
|
if (Succ->isLiveIn(Reg))
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool SIOptimizeExecMasking::runOnMachineFunction(MachineFunction &MF) {
|
2017-12-16 06:22:58 +08:00
|
|
|
if (skipFunction(MF.getFunction()))
|
2017-10-11 04:48:36 +08:00
|
|
|
return false;
|
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
|
2016-09-29 09:44:16 +08:00
|
|
|
const SIRegisterInfo *TRI = ST.getRegisterInfo();
|
|
|
|
const SIInstrInfo *TII = ST.getInstrInfo();
|
2019-12-28 04:21:44 +08:00
|
|
|
MCRegister Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
|
2016-09-29 09:44:16 +08:00
|
|
|
|
|
|
|
// Optimize sequences emitted for control flow lowering. They are originally
|
|
|
|
// emitted as the separate operations because spill code may need to be
|
|
|
|
// inserted for the saved copy of exec.
|
|
|
|
//
|
|
|
|
// x = copy exec
|
|
|
|
// z = s_<op>_b64 x, y
|
|
|
|
// exec = copy z
|
|
|
|
// =>
|
|
|
|
// x = s_<op>_saveexec_b64 y
|
|
|
|
//
|
|
|
|
|
|
|
|
for (MachineBasicBlock &MBB : MF) {
|
|
|
|
MachineBasicBlock::reverse_iterator I = fixTerminators(*TII, MBB);
|
|
|
|
MachineBasicBlock::reverse_iterator E = MBB.rend();
|
|
|
|
if (I == E)
|
|
|
|
continue;
|
|
|
|
|
2020-07-24 07:42:37 +08:00
|
|
|
// It's possible to see other terminator copies after the exec copy. This
|
|
|
|
// can happen if control flow pseudos had their outputs used by phis.
|
|
|
|
Register CopyToExec;
|
|
|
|
|
|
|
|
unsigned SearchCount = 0;
|
|
|
|
const unsigned SearchLimit = 5;
|
|
|
|
while (I != E && SearchCount++ < SearchLimit) {
|
|
|
|
CopyToExec = isCopyToExec(*I, ST);
|
|
|
|
if (CopyToExec)
|
|
|
|
break;
|
|
|
|
++I;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!CopyToExec)
|
2016-09-29 09:44:16 +08:00
|
|
|
continue;
|
|
|
|
|
|
|
|
// Scan backwards to find the def.
|
|
|
|
auto CopyToExecInst = &*I;
|
2019-06-17 01:13:09 +08:00
|
|
|
auto CopyFromExecInst = findExecCopy(*TII, ST, MBB, I, CopyToExec);
|
2017-08-02 07:44:35 +08:00
|
|
|
if (CopyFromExecInst == E) {
|
|
|
|
auto PrepareExecInst = std::next(I);
|
|
|
|
if (PrepareExecInst == E)
|
|
|
|
continue;
|
|
|
|
// Fold exec = COPY (S_AND_B64 reg, exec) -> exec = S_AND_B64 reg, exec
|
|
|
|
if (CopyToExecInst->getOperand(1).isKill() &&
|
|
|
|
isLogicalOpOnExec(*PrepareExecInst) == CopyToExec) {
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "Fold exec copy: " << *PrepareExecInst);
|
2017-08-02 07:44:35 +08:00
|
|
|
|
2019-06-17 01:13:09 +08:00
|
|
|
PrepareExecInst->getOperand(0).setReg(Exec);
|
2017-08-02 07:44:35 +08:00
|
|
|
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "into: " << *PrepareExecInst << '\n');
|
2017-08-02 07:44:35 +08:00
|
|
|
|
|
|
|
CopyToExecInst->eraseFromParent();
|
|
|
|
}
|
|
|
|
|
2016-09-29 09:44:16 +08:00
|
|
|
continue;
|
2017-08-02 07:44:35 +08:00
|
|
|
}
|
2016-09-29 09:44:16 +08:00
|
|
|
|
|
|
|
if (isLiveOut(MBB, CopyToExec)) {
|
|
|
|
// The copied register is live out and has a second use in another block.
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "Exec copy source register is live out\n");
|
2016-09-29 09:44:16 +08:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
Apply llvm-prefer-register-over-unsigned from clang-tidy to LLVM
Summary:
This clang-tidy check is looking for unsigned integer variables whose initializer
starts with an implicit cast from llvm::Register and changes the type of the
variable to llvm::Register (dropping the llvm:: where possible).
Partial reverts in:
X86FrameLowering.cpp - Some functions return unsigned and arguably should be MCRegister
X86FixupLEAs.cpp - Some functions return unsigned and arguably should be MCRegister
X86FrameLowering.cpp - Some functions return unsigned and arguably should be MCRegister
HexagonBitSimplify.cpp - Function takes BitTracker::RegisterRef which appears to be unsigned&
MachineVerifier.cpp - Ambiguous operator==() given MCRegister and const Register
PPCFastISel.cpp - No Register::operator-=()
PeepholeOptimizer.cpp - TargetInstrInfo::optimizeLoadInstr() takes an unsigned&
MachineTraceMetrics.cpp - MachineTraceMetrics lacks a suitable constructor
Manual fixups in:
ARMFastISel.cpp - ARMEmitLoad() now takes a Register& instead of unsigned&
HexagonSplitDouble.cpp - Ternary operator was ambiguous between unsigned/Register
HexagonConstExtenders.cpp - Has a local class named Register, used llvm::Register instead of Register.
PPCFastISel.cpp - PPCEmitLoad() now takes a Register& instead of unsigned&
Depends on D65919
Reviewers: arsenm, bogner, craig.topper, RKSimon
Reviewed By: arsenm
Subscribers: RKSimon, craig.topper, lenary, aemerson, wuzish, jholewinski, MatzeB, qcolombet, dschuff, jyknight, dylanmckay, sdardis, nemanjai, jvesely, wdng, nhaehnle, sbc100, jgravelle-google, kristof.beyls, hiraditya, aheejin, kbarton, fedor.sergeev, javed.absar, asb, rbar, johnrusso, simoncook, apazos, sabuasal, niosHD, jrtc27, MaskRay, zzheng, edward-jones, atanasyan, rogfer01, MartinMosbeck, brucehoult, the_o, tpr, PkmX, jocewei, jsji, Petar.Avramovic, asbirlea, Jim, s.egerton, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D65962
llvm-svn: 369041
2019-08-16 03:22:08 +08:00
|
|
|
Register CopyFromExec = CopyFromExecInst->getOperand(0).getReg();
|
2016-09-29 09:44:16 +08:00
|
|
|
MachineInstr *SaveExecInst = nullptr;
|
|
|
|
SmallVector<MachineInstr *, 4> OtherUseInsts;
|
|
|
|
|
|
|
|
for (MachineBasicBlock::iterator J
|
|
|
|
= std::next(CopyFromExecInst->getIterator()), JE = I->getIterator();
|
|
|
|
J != JE; ++J) {
|
2019-06-17 01:13:09 +08:00
|
|
|
if (SaveExecInst && J->readsRegister(Exec, TRI)) {
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "exec read prevents saveexec: " << *J << '\n');
|
2016-09-29 09:44:16 +08:00
|
|
|
// Make sure this is inserted after any VALU ops that may have been
|
|
|
|
// scheduled in between.
|
|
|
|
SaveExecInst = nullptr;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2017-11-14 10:16:54 +08:00
|
|
|
bool ReadsCopyFromExec = J->readsRegister(CopyFromExec, TRI);
|
|
|
|
|
2016-09-29 09:44:16 +08:00
|
|
|
if (J->modifiesRegister(CopyToExec, TRI)) {
|
|
|
|
if (SaveExecInst) {
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "Multiple instructions modify "
|
|
|
|
<< printReg(CopyToExec, TRI) << '\n');
|
2016-09-29 09:44:16 +08:00
|
|
|
SaveExecInst = nullptr;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned SaveExecOp = getSaveExecOp(J->getOpcode());
|
|
|
|
if (SaveExecOp == AMDGPU::INSTRUCTION_LIST_END)
|
|
|
|
break;
|
|
|
|
|
2017-11-14 10:16:54 +08:00
|
|
|
if (ReadsCopyFromExec) {
|
2016-09-29 09:44:16 +08:00
|
|
|
SaveExecInst = &*J;
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "Found save exec op: " << *SaveExecInst << '\n');
|
AMDGPU: Fix use-after-free in SIOptimizeExecMasking
Summary:
There was a bug with sequences like
s_mov_b64 s[0:1], exec
s_and_b64 s[2:3]<def>, s[0:1], s[2:3]<kill>
...
s_mov_b64_term exec, s[2:3]
because s[2:3] was defined and used in the same instruction, ending up with
SaveExecInst inside OtherUseInsts.
Note that the test case also exposes an unrelated bug.
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=98028
Reviewers: tstellarAMD, arsenm
Subscribers: kzhuravl, wdng, yaxunl, llvm-commits, tony-tye
Differential Revision: https://reviews.llvm.org/D25306
llvm-svn: 283528
2016-10-07 16:40:14 +08:00
|
|
|
continue;
|
2016-09-29 09:44:16 +08:00
|
|
|
} else {
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs()
|
|
|
|
<< "Instruction does not read exec copy: " << *J << '\n');
|
2016-09-29 09:44:16 +08:00
|
|
|
break;
|
|
|
|
}
|
2017-11-14 10:16:54 +08:00
|
|
|
} else if (ReadsCopyFromExec && !SaveExecInst) {
|
|
|
|
// Make sure no other instruction is trying to use this copy, before it
|
|
|
|
// will be rewritten by the saveexec, i.e. hasOneUse. There may have
|
|
|
|
// been another use, such as an inserted spill. For example:
|
|
|
|
//
|
|
|
|
// %sgpr0_sgpr1 = COPY %exec
|
|
|
|
// spill %sgpr0_sgpr1
|
|
|
|
// %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1
|
|
|
|
//
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "Found second use of save inst candidate: " << *J
|
|
|
|
<< '\n');
|
2017-11-14 10:16:54 +08:00
|
|
|
break;
|
2016-09-29 09:44:16 +08:00
|
|
|
}
|
|
|
|
|
AMDGPU: Fix use-after-free in SIOptimizeExecMasking
Summary:
There was a bug with sequences like
s_mov_b64 s[0:1], exec
s_and_b64 s[2:3]<def>, s[0:1], s[2:3]<kill>
...
s_mov_b64_term exec, s[2:3]
because s[2:3] was defined and used in the same instruction, ending up with
SaveExecInst inside OtherUseInsts.
Note that the test case also exposes an unrelated bug.
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=98028
Reviewers: tstellarAMD, arsenm
Subscribers: kzhuravl, wdng, yaxunl, llvm-commits, tony-tye
Differential Revision: https://reviews.llvm.org/D25306
llvm-svn: 283528
2016-10-07 16:40:14 +08:00
|
|
|
if (SaveExecInst && J->readsRegister(CopyToExec, TRI)) {
|
|
|
|
assert(SaveExecInst != &*J);
|
2016-09-29 09:44:16 +08:00
|
|
|
OtherUseInsts.push_back(&*J);
|
AMDGPU: Fix use-after-free in SIOptimizeExecMasking
Summary:
There was a bug with sequences like
s_mov_b64 s[0:1], exec
s_and_b64 s[2:3]<def>, s[0:1], s[2:3]<kill>
...
s_mov_b64_term exec, s[2:3]
because s[2:3] was defined and used in the same instruction, ending up with
SaveExecInst inside OtherUseInsts.
Note that the test case also exposes an unrelated bug.
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=98028
Reviewers: tstellarAMD, arsenm
Subscribers: kzhuravl, wdng, yaxunl, llvm-commits, tony-tye
Differential Revision: https://reviews.llvm.org/D25306
llvm-svn: 283528
2016-10-07 16:40:14 +08:00
|
|
|
}
|
2016-09-29 09:44:16 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
if (!SaveExecInst)
|
|
|
|
continue;
|
|
|
|
|
2018-05-14 20:53:11 +08:00
|
|
|
LLVM_DEBUG(dbgs() << "Insert save exec op: " << *SaveExecInst << '\n');
|
2016-09-29 09:44:16 +08:00
|
|
|
|
|
|
|
MachineOperand &Src0 = SaveExecInst->getOperand(1);
|
|
|
|
MachineOperand &Src1 = SaveExecInst->getOperand(2);
|
|
|
|
|
|
|
|
MachineOperand *OtherOp = nullptr;
|
|
|
|
|
|
|
|
if (Src0.isReg() && Src0.getReg() == CopyFromExec) {
|
|
|
|
OtherOp = &Src1;
|
|
|
|
} else if (Src1.isReg() && Src1.getReg() == CopyFromExec) {
|
|
|
|
if (!SaveExecInst->isCommutable())
|
|
|
|
break;
|
|
|
|
|
|
|
|
OtherOp = &Src0;
|
|
|
|
} else
|
|
|
|
llvm_unreachable("unexpected");
|
|
|
|
|
|
|
|
CopyFromExecInst->eraseFromParent();
|
|
|
|
|
|
|
|
auto InsPt = SaveExecInst->getIterator();
|
|
|
|
const DebugLoc &DL = SaveExecInst->getDebugLoc();
|
|
|
|
|
|
|
|
BuildMI(MBB, InsPt, DL, TII->get(getSaveExecOp(SaveExecInst->getOpcode())),
|
|
|
|
CopyFromExec)
|
|
|
|
.addReg(OtherOp->getReg());
|
|
|
|
SaveExecInst->eraseFromParent();
|
|
|
|
|
|
|
|
CopyToExecInst->eraseFromParent();
|
|
|
|
|
|
|
|
for (MachineInstr *OtherInst : OtherUseInsts) {
|
2019-06-17 01:13:09 +08:00
|
|
|
OtherInst->substituteRegister(CopyToExec, Exec,
|
[MachineOperand][Target] MachineOperand::isRenamable semantics changes
Summary:
Add a target option AllowRegisterRenaming that is used to opt in to
post-register-allocation renaming of registers. This is set to 0 by
default, which causes the hasExtraSrcRegAllocReq/hasExtraDstRegAllocReq
fields of all opcodes to be set to 1, causing
MachineOperand::isRenamable to always return false.
Set the AllowRegisterRenaming flag to 1 for all in-tree targets that
have lit tests that were effected by enabling COPY forwarding in
MachineCopyPropagation (AArch64, AMDGPU, ARM, Hexagon, Mips, PowerPC,
RISCV, Sparc, SystemZ and X86).
Add some more comments describing the semantics of the
MachineOperand::isRenamable function and how it is set and maintained.
Change isRenamable to check the operand's opcode
hasExtraSrcRegAllocReq/hasExtraDstRegAllocReq bit directly instead of
relying on it being consistently reflected in the IsRenamable bit
setting.
Clear the IsRenamable bit when changing an operand's register value.
Remove target code that was clearing the IsRenamable bit when changing
registers/opcodes now that this is done conservatively by default.
Change setting of hasExtraSrcRegAllocReq in AMDGPU target to be done in
one place covering all opcodes that have constant pipe read limit
restrictions.
Reviewers: qcolombet, MatzeB
Subscribers: aemerson, arsenm, jyknight, mcrosier, sdardis, nhaehnle, javed.absar, tpr, arichardson, kristof.beyls, kbarton, fedor.sergeev, asb, rbar, johnrusso, simoncook, jordy.potman.lists, apazos, sabuasal, niosHD, escha, nemanjai, llvm-commits
Differential Revision: https://reviews.llvm.org/D43042
llvm-svn: 325931
2018-02-24 02:25:08 +08:00
|
|
|
AMDGPU::NoSubRegister, *TRI);
|
2016-09-29 09:44:16 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
|
|
|
}
|