2012-12-12 05:25:42 +08:00
|
|
|
//===-- AMDGPUSubtarget.cpp - AMDGPU Subtarget Information ----------------===//
|
|
|
|
|
//
|
2019-01-19 16:50:56 +08:00
|
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
|
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
2012-12-12 05:25:42 +08:00
|
|
|
//
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
//
|
|
|
|
|
/// \file
|
2018-05-01 23:54:18 +08:00
|
|
|
/// Implements the AMDGPU specific subclass of TargetSubtarget.
|
2012-12-12 05:25:42 +08:00
|
|
|
//
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
|
|
#include "AMDGPUSubtarget.h"
|
2017-07-06 02:40:56 +08:00
|
|
|
#include "AMDGPU.h"
|
|
|
|
|
#include "AMDGPUTargetMachine.h"
|
|
|
|
|
#include "AMDGPUCallLowering.h"
|
|
|
|
|
#include "AMDGPUInstructionSelector.h"
|
|
|
|
|
#include "AMDGPULegalizerInfo.h"
|
|
|
|
|
#include "AMDGPURegisterBankInfo.h"
|
2017-02-08 21:02:33 +08:00
|
|
|
#include "SIMachineFunctionInfo.h"
|
AMDGPU: Remove #include "MCTargetDesc/AMDGPUMCTargetDesc.h" from common headers
Summary:
MCTargetDesc/AMDGPUMCTargetDesc.h contains enums for all the instuction
and register defintions, which are huge so we only want to include
them where needed.
This will also make it easier if we want to split the R600 and GCN
definitions into separate tablegenerated files.
I was unable to remove AMDGPUMCTargetDesc.h from SIMachineFunctionInfo.h
because it uses some enums from the header to initialize default values
for the SIMachineFunction class, so I ended up having to remove includes of
SIMachineFunctionInfo.h from headers too.
Reviewers: arsenm, nhaehnle
Reviewed By: nhaehnle
Subscribers: MatzeB, kzhuravl, wdng, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46272
llvm-svn: 332930
2018-05-22 10:03:23 +08:00
|
|
|
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
|
2014-07-13 10:08:26 +08:00
|
|
|
#include "llvm/ADT/SmallString.h"
|
2015-01-30 00:55:25 +08:00
|
|
|
#include "llvm/CodeGen/MachineScheduler.h"
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
#include "llvm/MC/MCSubtargetInfo.h"
|
2017-04-13 04:48:56 +08:00
|
|
|
#include "llvm/IR/MDBuilder.h"
|
2017-11-04 06:32:11 +08:00
|
|
|
#include "llvm/CodeGen/TargetFrameLowering.h"
|
2016-12-13 06:23:53 +08:00
|
|
|
#include <algorithm>
|
2014-07-13 10:08:26 +08:00
|
|
|
|
2012-12-12 05:25:42 +08:00
|
|
|
using namespace llvm;
|
|
|
|
|
|
2014-04-22 06:55:11 +08:00
|
|
|
#define DEBUG_TYPE "amdgpu-subtarget"
|
|
|
|
|
|
2012-12-12 05:25:42 +08:00
|
|
|
#define GET_SUBTARGETINFO_TARGET_DESC
|
|
|
|
|
#define GET_SUBTARGETINFO_CTOR
|
2018-07-12 04:59:01 +08:00
|
|
|
#define AMDGPUSubtarget GCNSubtarget
|
2012-12-12 05:25:42 +08:00
|
|
|
#include "AMDGPUGenSubtargetInfo.inc"
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
#define GET_SUBTARGETINFO_TARGET_DESC
|
|
|
|
|
#define GET_SUBTARGETINFO_CTOR
|
2018-07-12 04:59:01 +08:00
|
|
|
#undef AMDGPUSubtarget
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
#include "R600GenSubtargetInfo.inc"
|
2012-12-12 05:25:42 +08:00
|
|
|
|
2019-07-12 05:25:00 +08:00
|
|
|
static cl::opt<bool> DisablePowerSched(
|
|
|
|
|
"amdgpu-disable-power-sched",
|
|
|
|
|
cl::desc("Disable scheduling to minimize mAI power bursts"),
|
|
|
|
|
cl::init(false));
|
|
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
GCNSubtarget::~GCNSubtarget() = default;
|
2016-06-24 14:30:11 +08:00
|
|
|
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
R600Subtarget &
|
|
|
|
|
R600Subtarget::initializeSubtargetDependencies(const Triple &TT,
|
|
|
|
|
StringRef GPU, StringRef FS) {
|
2019-03-30 03:14:54 +08:00
|
|
|
SmallString<256> FullFS("+promote-alloca,");
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
FullFS += FS;
|
|
|
|
|
ParseSubtargetFeatures(GPU, FullFS);
|
|
|
|
|
|
|
|
|
|
// FIXME: I don't think think Evergreen has any useful support for
|
|
|
|
|
// denormals, but should be checked. Should we issue a warning somewhere
|
|
|
|
|
// if someone tries to enable these?
|
2018-07-12 04:59:01 +08:00
|
|
|
if (getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS) {
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
FP32Denormals = false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
HasMulU24 = getGeneration() >= EVERGREEN;
|
|
|
|
|
HasMulI24 = hasCaymanISA();
|
|
|
|
|
|
|
|
|
|
return *this;
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
GCNSubtarget &
|
|
|
|
|
GCNSubtarget::initializeSubtargetDependencies(const Triple &TT,
|
2019-04-03 09:58:57 +08:00
|
|
|
StringRef GPU, StringRef FS) {
|
2014-07-26 06:22:39 +08:00
|
|
|
// Determine default and user-specified characteristics
|
2014-07-15 07:40:49 +08:00
|
|
|
// On SI+, we want FP64 denormals to be on by default. FP32 denormals can be
|
|
|
|
|
// enabled, but some instructions do not respect them and they run at the
|
|
|
|
|
// double precision rate, so don't enable by default.
|
|
|
|
|
//
|
|
|
|
|
// We want to be able to turn these off, but making this a subtarget feature
|
|
|
|
|
// for SI has the unhelpful behavior that it unsets everything else if you
|
|
|
|
|
// disable it.
|
[AMDGPU] Add support for TFE/LWE in image intrinsics. 2nd try
TFE and LWE support requires extra result registers that are written in the
event of a failure in order to detect that failure case.
The specific use-case that initiated these changes is sparse texture support.
This means that if image intrinsics are used with either option turned on, the
programmer must ensure that the return type can contain all of the expected
results. This can result in redundant registers since the vector size must be a
power-of-2.
This change takes roughly 6 parts:
1. Modify the instruction defs in tablegen to add new instruction variants that
can accomodate the extra return values.
2. Updates to lowerImage in SIISelLowering.cpp to accomodate setting TFE or LWE
(where the bulk of the work for these instruction types is now done)
3. Extra verification code to catch cases where intrinsics have been used but
insufficient return registers are used.
4. Modification to the adjustWritemask optimisation to account for TFE/LWE being
enabled (requires extra registers to be maintained for error return value).
5. An extra pass to zero initialize the error value return - this is because if
the error does not occur, the register is not written and thus must be zeroed
before use. Also added a new (on by default) option to ensure ALL return values
are zero-initialized that is required for sparse texture support.
6. Disable the inst_combine optimization in the presence of tfe/lwe (later TODO
for this to re-enable and handle correctly).
There's an additional fix now to avoid a dmask=0
For an image intrinsic with tfe where all result channels except tfe
were unused, I was getting an image instruction with dmask=0 and only a
single vgpr result for tfe. That is incorrect because the hardware
assumes there is at least one vgpr result, plus the one for tfe.
Fixed by forcing dmask to 1, which gives the desired two vgpr result
with tfe in the second one.
The TFE or LWE result is returned from the intrinsics using an aggregate
type. Look in the test code provided to see how this works, but in essence IR
code to invoke the intrinsic looks as follows:
%v = call {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 15,
i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
%v.vec = extractvalue {<4 x float>, i32} %v, 0
%v.err = extractvalue {<4 x float>, i32} %v, 1
This re-submit of the change also includes a slight modification in
SIISelLowering.cpp to work-around a compiler bug for the powerpc_le
platform that caused a buildbot failure on a previous submission.
Differential revision: https://reviews.llvm.org/D48826
Change-Id: If222bc03642e76cf98059a6bef5d5bffeda38dda
Work around for ppcle compiler bug
Change-Id: Ie284cf24b2271215be1b9dc95b485fd15000e32b
llvm-svn: 351054
2019-01-14 19:55:24 +08:00
|
|
|
//
|
|
|
|
|
// Similarly we want enable-prt-strict-null to be on by default and not to
|
|
|
|
|
// unset everything else if it is disabled
|
2014-07-13 10:08:26 +08:00
|
|
|
|
2019-04-03 09:58:57 +08:00
|
|
|
// Assuming ECC is enabled is the conservative default.
|
2019-05-16 22:48:34 +08:00
|
|
|
SmallString<256> FullFS("+promote-alloca,+load-store-opt,+sram-ecc,+xnack,");
|
2017-12-05 06:57:29 +08:00
|
|
|
|
2015-12-23 04:55:23 +08:00
|
|
|
if (isAmdHsaOS()) // Turn on FlatForGlobal for HSA.
|
2019-03-18 05:31:35 +08:00
|
|
|
FullFS += "+flat-for-global,+unaligned-buffer-access,+trap-handler,";
|
2017-01-24 06:31:03 +08:00
|
|
|
|
2017-12-05 06:57:29 +08:00
|
|
|
// FIXME: I don't think think Evergreen has any useful support for
|
|
|
|
|
// denormals, but should be checked. Should we issue a warning somewhere
|
|
|
|
|
// if someone tries to enable these?
|
|
|
|
|
if (getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS) {
|
|
|
|
|
FullFS += "+fp64-fp16-denormals,";
|
|
|
|
|
} else {
|
|
|
|
|
FullFS += "-fp32-denormals,";
|
|
|
|
|
}
|
|
|
|
|
|
[AMDGPU] Add support for TFE/LWE in image intrinsics. 2nd try
TFE and LWE support requires extra result registers that are written in the
event of a failure in order to detect that failure case.
The specific use-case that initiated these changes is sparse texture support.
This means that if image intrinsics are used with either option turned on, the
programmer must ensure that the return type can contain all of the expected
results. This can result in redundant registers since the vector size must be a
power-of-2.
This change takes roughly 6 parts:
1. Modify the instruction defs in tablegen to add new instruction variants that
can accomodate the extra return values.
2. Updates to lowerImage in SIISelLowering.cpp to accomodate setting TFE or LWE
(where the bulk of the work for these instruction types is now done)
3. Extra verification code to catch cases where intrinsics have been used but
insufficient return registers are used.
4. Modification to the adjustWritemask optimisation to account for TFE/LWE being
enabled (requires extra registers to be maintained for error return value).
5. An extra pass to zero initialize the error value return - this is because if
the error does not occur, the register is not written and thus must be zeroed
before use. Also added a new (on by default) option to ensure ALL return values
are zero-initialized that is required for sparse texture support.
6. Disable the inst_combine optimization in the presence of tfe/lwe (later TODO
for this to re-enable and handle correctly).
There's an additional fix now to avoid a dmask=0
For an image intrinsic with tfe where all result channels except tfe
were unused, I was getting an image instruction with dmask=0 and only a
single vgpr result for tfe. That is incorrect because the hardware
assumes there is at least one vgpr result, plus the one for tfe.
Fixed by forcing dmask to 1, which gives the desired two vgpr result
with tfe in the second one.
The TFE or LWE result is returned from the intrinsics using an aggregate
type. Look in the test code provided to see how this works, but in essence IR
code to invoke the intrinsic looks as follows:
%v = call {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 15,
i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
%v.vec = extractvalue {<4 x float>, i32} %v, 0
%v.err = extractvalue {<4 x float>, i32} %v, 1
This re-submit of the change also includes a slight modification in
SIISelLowering.cpp to work-around a compiler bug for the powerpc_le
platform that caused a buildbot failure on a previous submission.
Differential revision: https://reviews.llvm.org/D48826
Change-Id: If222bc03642e76cf98059a6bef5d5bffeda38dda
Work around for ppcle compiler bug
Change-Id: Ie284cf24b2271215be1b9dc95b485fd15000e32b
llvm-svn: 351054
2019-01-14 19:55:24 +08:00
|
|
|
FullFS += "+enable-prt-strict-null,"; // This is overridden by a disable in FS
|
|
|
|
|
|
2019-06-14 03:18:29 +08:00
|
|
|
// Disable mutually exclusive bits.
|
|
|
|
|
if (FS.find_lower("+wavefrontsize") != StringRef::npos) {
|
|
|
|
|
if (FS.find_lower("wavefrontsize16") == StringRef::npos)
|
|
|
|
|
FullFS += "-wavefrontsize16,";
|
|
|
|
|
if (FS.find_lower("wavefrontsize32") == StringRef::npos)
|
|
|
|
|
FullFS += "-wavefrontsize32,";
|
|
|
|
|
if (FS.find_lower("wavefrontsize64") == StringRef::npos)
|
|
|
|
|
FullFS += "-wavefrontsize64,";
|
|
|
|
|
}
|
|
|
|
|
|
2014-07-13 10:08:26 +08:00
|
|
|
FullFS += FS;
|
|
|
|
|
|
|
|
|
|
ParseSubtargetFeatures(GPU, FullFS);
|
2014-06-13 09:32:00 +08:00
|
|
|
|
2017-12-05 06:57:29 +08:00
|
|
|
// We don't support FP64 for EG/NI atm.
|
|
|
|
|
assert(!hasFP64() || (getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS));
|
|
|
|
|
|
2017-01-28 01:42:26 +08:00
|
|
|
// Unless +-flat-for-global is specified, turn on FlatForGlobal for all OS-es
|
|
|
|
|
// on VI and newer hardware to avoid assertion failures due to missing ADDR64
|
|
|
|
|
// variants of MUBUF instructions.
|
|
|
|
|
if (!hasAddr64() && !FS.contains("flat-for-global")) {
|
|
|
|
|
FlatForGlobal = true;
|
|
|
|
|
}
|
|
|
|
|
|
2016-02-12 10:40:47 +08:00
|
|
|
// Set defaults if needed.
|
|
|
|
|
if (MaxPrivateElementSize == 0)
|
2016-05-11 08:28:54 +08:00
|
|
|
MaxPrivateElementSize = 4;
|
2016-02-12 10:40:47 +08:00
|
|
|
|
2017-08-07 22:58:04 +08:00
|
|
|
if (LDSBankCount == 0)
|
|
|
|
|
LDSBankCount = 32;
|
|
|
|
|
|
|
|
|
|
if (TT.getArch() == Triple::amdgcn) {
|
|
|
|
|
if (LocalMemorySize == 0)
|
|
|
|
|
LocalMemorySize = 32768;
|
|
|
|
|
|
|
|
|
|
// Do something sensible for unspecified target.
|
|
|
|
|
if (!HasMovrel && !HasVGPRIndexMode)
|
|
|
|
|
HasMovrel = true;
|
|
|
|
|
}
|
|
|
|
|
|
2019-02-09 03:18:01 +08:00
|
|
|
// Don't crash on invalid devices.
|
|
|
|
|
if (WavefrontSize == 0)
|
|
|
|
|
WavefrontSize = 64;
|
|
|
|
|
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
HasFminFmaxLegacy = getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS;
|
|
|
|
|
|
2019-05-16 22:48:34 +08:00
|
|
|
if (DoesNotSupportXNACK && EnableXNACK) {
|
|
|
|
|
ToggleFeature(AMDGPU::FeatureXNACK);
|
|
|
|
|
EnableXNACK = false;
|
|
|
|
|
}
|
|
|
|
|
|
2019-04-03 09:58:57 +08:00
|
|
|
// ECC is on by default, but turn it off if the hardware doesn't support it
|
|
|
|
|
// anyway. This matters for the gfx9 targets with d16 loads, but don't support
|
|
|
|
|
// ECC.
|
|
|
|
|
if (DoesNotSupportSRAMECC && EnableSRAMECC) {
|
|
|
|
|
ToggleFeature(AMDGPU::FeatureSRAMECC);
|
|
|
|
|
EnableSRAMECC = false;
|
|
|
|
|
}
|
|
|
|
|
|
2014-07-26 06:22:39 +08:00
|
|
|
return *this;
|
|
|
|
|
}
|
|
|
|
|
|
2018-09-13 02:50:47 +08:00
|
|
|
AMDGPUSubtarget::AMDGPUSubtarget(const Triple &TT) :
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
TargetTriple(TT),
|
|
|
|
|
Has16BitInsts(false),
|
|
|
|
|
HasMadMixInsts(false),
|
|
|
|
|
FP32Denormals(false),
|
|
|
|
|
FPExceptions(false),
|
|
|
|
|
HasSDWA(false),
|
|
|
|
|
HasVOP3PInsts(false),
|
|
|
|
|
HasMulI24(true),
|
|
|
|
|
HasMulU24(true),
|
2018-08-16 05:03:55 +08:00
|
|
|
HasInv2PiInlineImm(false),
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
HasFminFmaxLegacy(true),
|
|
|
|
|
EnablePromoteAlloca(false),
|
2018-09-14 18:27:19 +08:00
|
|
|
HasTrigReducedRange(false),
|
2019-07-31 09:07:10 +08:00
|
|
|
MaxWavesPerEU(10),
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
LocalMemorySize(0),
|
|
|
|
|
WavefrontSize(0)
|
|
|
|
|
{ }
|
|
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
GCNSubtarget::GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
|
2018-09-13 02:50:47 +08:00
|
|
|
const GCNTargetMachine &TM) :
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
AMDGPUGenSubtargetInfo(TT, GPU, FS),
|
2018-09-13 02:50:47 +08:00
|
|
|
AMDGPUSubtarget(TT),
|
2016-06-24 14:30:11 +08:00
|
|
|
TargetTriple(TT),
|
2019-03-18 05:31:35 +08:00
|
|
|
Gen(TT.getOS() == Triple::AMDHSA ? SEA_ISLANDS : SOUTHERN_ISLANDS),
|
2018-09-18 00:04:32 +08:00
|
|
|
InstrItins(getInstrItineraryForCPU(GPU)),
|
2016-06-24 14:30:11 +08:00
|
|
|
LDSBankCount(0),
|
|
|
|
|
MaxPrivateElementSize(0),
|
|
|
|
|
|
|
|
|
|
FastFMAF32(false),
|
|
|
|
|
HalfRate64Ops(false),
|
|
|
|
|
|
2017-01-24 06:31:03 +08:00
|
|
|
FP64FP16Denormals(false),
|
2016-06-24 14:30:11 +08:00
|
|
|
FlatForGlobal(false),
|
2017-06-03 01:40:26 +08:00
|
|
|
AutoWaitcntBeforeBarrier(false),
|
2017-10-14 23:59:07 +08:00
|
|
|
CodeObjectV3(false),
|
2016-10-15 02:10:39 +08:00
|
|
|
UnalignedScratchAccess(false),
|
2016-07-02 07:03:44 +08:00
|
|
|
UnalignedBufferAccess(false),
|
|
|
|
|
|
2017-02-19 02:29:53 +08:00
|
|
|
HasApertureRegs(false),
|
2016-06-24 14:30:11 +08:00
|
|
|
EnableXNACK(false),
|
2019-05-16 22:48:34 +08:00
|
|
|
DoesNotSupportXNACK(false),
|
2019-04-25 01:03:15 +08:00
|
|
|
EnableCuMode(false),
|
2017-02-10 10:15:29 +08:00
|
|
|
TrapHandler(false),
|
2016-06-24 14:30:11 +08:00
|
|
|
|
|
|
|
|
EnableLoadStoreOpt(false),
|
|
|
|
|
EnableUnsafeDSOffsetFolding(false),
|
|
|
|
|
EnableSIScheduler(false),
|
2018-04-11 06:48:23 +08:00
|
|
|
EnableDS128(false),
|
[AMDGPU] Add support for TFE/LWE in image intrinsics. 2nd try
TFE and LWE support requires extra result registers that are written in the
event of a failure in order to detect that failure case.
The specific use-case that initiated these changes is sparse texture support.
This means that if image intrinsics are used with either option turned on, the
programmer must ensure that the return type can contain all of the expected
results. This can result in redundant registers since the vector size must be a
power-of-2.
This change takes roughly 6 parts:
1. Modify the instruction defs in tablegen to add new instruction variants that
can accomodate the extra return values.
2. Updates to lowerImage in SIISelLowering.cpp to accomodate setting TFE or LWE
(where the bulk of the work for these instruction types is now done)
3. Extra verification code to catch cases where intrinsics have been used but
insufficient return registers are used.
4. Modification to the adjustWritemask optimisation to account for TFE/LWE being
enabled (requires extra registers to be maintained for error return value).
5. An extra pass to zero initialize the error value return - this is because if
the error does not occur, the register is not written and thus must be zeroed
before use. Also added a new (on by default) option to ensure ALL return values
are zero-initialized that is required for sparse texture support.
6. Disable the inst_combine optimization in the presence of tfe/lwe (later TODO
for this to re-enable and handle correctly).
There's an additional fix now to avoid a dmask=0
For an image intrinsic with tfe where all result channels except tfe
were unused, I was getting an image instruction with dmask=0 and only a
single vgpr result for tfe. That is incorrect because the hardware
assumes there is at least one vgpr result, plus the one for tfe.
Fixed by forcing dmask to 1, which gives the desired two vgpr result
with tfe in the second one.
The TFE or LWE result is returned from the intrinsics using an aggregate
type. Look in the test code provided to see how this works, but in essence IR
code to invoke the intrinsic looks as follows:
%v = call {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 15,
i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
%v.vec = extractvalue {<4 x float>, i32} %v, 0
%v.err = extractvalue {<4 x float>, i32} %v, 1
This re-submit of the change also includes a slight modification in
SIISelLowering.cpp to work-around a compiler bug for the powerpc_le
platform that caused a buildbot failure on a previous submission.
Differential revision: https://reviews.llvm.org/D48826
Change-Id: If222bc03642e76cf98059a6bef5d5bffeda38dda
Work around for ppcle compiler bug
Change-Id: Ie284cf24b2271215be1b9dc95b485fd15000e32b
llvm-svn: 351054
2019-01-14 19:55:24 +08:00
|
|
|
EnablePRTStrictNull(false),
|
2016-06-24 14:30:11 +08:00
|
|
|
DumpCode(false),
|
|
|
|
|
|
|
|
|
|
FP64(false),
|
|
|
|
|
GCN3Encoding(false),
|
|
|
|
|
CIInsts(false),
|
2019-04-06 02:24:34 +08:00
|
|
|
GFX8Insts(false),
|
2017-02-19 03:12:26 +08:00
|
|
|
GFX9Insts(false),
|
2019-04-25 01:03:15 +08:00
|
|
|
GFX10Insts(false),
|
2019-04-06 02:24:34 +08:00
|
|
|
GFX7GFX8GFX9Insts(false),
|
2016-06-24 14:30:11 +08:00
|
|
|
SGPRInitBug(false),
|
|
|
|
|
HasSMemRealTime(false),
|
2017-08-16 21:51:56 +08:00
|
|
|
HasIntClamp(false),
|
2018-05-01 03:08:16 +08:00
|
|
|
HasFmaMixInsts(false),
|
2016-10-13 02:00:51 +08:00
|
|
|
HasMovrel(false),
|
|
|
|
|
HasVGPRIndexMode(false),
|
2016-10-29 12:05:06 +08:00
|
|
|
HasScalarStores(false),
|
2018-04-03 00:10:25 +08:00
|
|
|
HasScalarAtomics(false),
|
2017-06-22 14:26:41 +08:00
|
|
|
HasSDWAOmod(false),
|
|
|
|
|
HasSDWAScalar(false),
|
|
|
|
|
HasSDWASdst(false),
|
|
|
|
|
HasSDWAMac(false),
|
[AMDGPU] SDWA: several fixes for V_CVT and VOPC instructions
Summary:
1. Instruction V_CVT_U32_F32 allow omod operand (see SIInstrInfo.td:1435). In fact this operand shouldn't be allowed here. This fix checks if SDWA pseudo instruction has OMod operand and then copy it.
2. There were several problems with support of VOPC instructions in SDWA peephole pass.
Reviewers: tstellar, arsenm, vpykhtin, airlied, kzhuravl
Subscribers: wdng, nhaehnle, yaxunl, dstuttard, tpr, sarnex, t-tye
Differential Revision: https://reviews.llvm.org/D34626
llvm-svn: 306413
2017-06-27 23:02:23 +08:00
|
|
|
HasSDWAOutModsVOPC(false),
|
2017-01-20 18:01:25 +08:00
|
|
|
HasDPP(false),
|
2019-06-13 02:02:41 +08:00
|
|
|
HasDPP8(false),
|
2018-08-28 23:07:30 +08:00
|
|
|
HasR128A16(false),
|
2019-04-25 01:03:15 +08:00
|
|
|
HasNSAEncoding(false),
|
2018-05-01 03:08:16 +08:00
|
|
|
HasDLInsts(false),
|
2019-02-09 08:34:21 +08:00
|
|
|
HasDot1Insts(false),
|
|
|
|
|
HasDot2Insts(false),
|
2019-07-10 02:10:06 +08:00
|
|
|
HasDot3Insts(false),
|
|
|
|
|
HasDot4Insts(false),
|
2019-06-14 08:33:31 +08:00
|
|
|
HasDot5Insts(false),
|
|
|
|
|
HasDot6Insts(false),
|
2019-07-10 02:10:06 +08:00
|
|
|
HasMAIInsts(false),
|
|
|
|
|
HasPkFmacF16Inst(false),
|
|
|
|
|
HasAtomicFaddInsts(false),
|
2018-11-06 06:44:19 +08:00
|
|
|
EnableSRAMECC(false),
|
2019-04-03 09:58:57 +08:00
|
|
|
DoesNotSupportSRAMECC(false),
|
2019-04-25 01:03:15 +08:00
|
|
|
HasNoSdstCMPX(false),
|
|
|
|
|
HasVscnt(false),
|
|
|
|
|
HasRegisterBanking(false),
|
|
|
|
|
HasVOP3Literal(false),
|
|
|
|
|
HasNoDataDepHazard(false),
|
2016-06-24 14:30:11 +08:00
|
|
|
FlatAddressSpace(false),
|
2017-05-11 05:19:05 +08:00
|
|
|
FlatInstOffsets(false),
|
|
|
|
|
FlatGlobalInsts(false),
|
|
|
|
|
FlatScratchInsts(false),
|
2019-04-25 01:03:15 +08:00
|
|
|
ScalarFlatScratchInsts(false),
|
2017-07-21 01:42:47 +08:00
|
|
|
AddNoCarryInsts(false),
|
2018-01-13 05:12:19 +08:00
|
|
|
HasUnpackedD16VMem(false),
|
2019-04-25 01:03:15 +08:00
|
|
|
LDSMisalignedBug(false),
|
2019-08-24 06:09:58 +08:00
|
|
|
HasMFMAInlineLiteralBug(false),
|
2016-06-24 14:30:11 +08:00
|
|
|
|
2016-12-09 01:28:47 +08:00
|
|
|
ScalarizeGlobal(false),
|
2016-06-24 14:30:11 +08:00
|
|
|
|
2019-04-25 01:03:15 +08:00
|
|
|
HasVcmpxPermlaneHazard(false),
|
|
|
|
|
HasVMEMtoScalarWriteHazard(false),
|
|
|
|
|
HasSMEMtoVectorWriteHazard(false),
|
|
|
|
|
HasInstFwdPrefetchBug(false),
|
|
|
|
|
HasVcmpxExecWARHazard(false),
|
|
|
|
|
HasLdsBranchVmemWARHazard(false),
|
|
|
|
|
HasNSAtoVMEMBug(false),
|
2019-06-27 01:34:57 +08:00
|
|
|
HasOffset3fBug(false),
|
2019-04-25 01:03:15 +08:00
|
|
|
HasFlatSegmentOffsetBug(false),
|
|
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
FeatureDisable(false),
|
2018-07-12 06:15:15 +08:00
|
|
|
InstrInfo(initializeSubtargetDependencies(TT, GPU, FS)),
|
2018-07-20 17:05:08 +08:00
|
|
|
TLInfo(TM, *this),
|
2018-07-12 04:59:01 +08:00
|
|
|
FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0) {
|
2019-07-31 09:07:10 +08:00
|
|
|
MaxWavesPerEU = AMDGPU::IsaInfo::getMaxWavesPerEU(this);
|
2018-07-12 04:59:01 +08:00
|
|
|
CallLoweringInfo.reset(new AMDGPUCallLowering(*getTargetLowering()));
|
|
|
|
|
Legalizer.reset(new AMDGPULegalizerInfo(*this, TM));
|
2019-09-20 00:26:14 +08:00
|
|
|
RegBankInfo.reset(new AMDGPURegisterBankInfo(*this));
|
2018-07-12 04:59:01 +08:00
|
|
|
InstSelector.reset(new AMDGPUInstructionSelector(
|
|
|
|
|
*this, *static_cast<AMDGPURegisterBankInfo *>(RegBankInfo.get()), TM));
|
2014-01-23 05:55:43 +08:00
|
|
|
}
|
2014-12-03 06:00:07 +08:00
|
|
|
|
2019-05-02 11:47:23 +08:00
|
|
|
unsigned GCNSubtarget::getConstantBusLimit(unsigned Opcode) const {
|
|
|
|
|
if (getGeneration() < GFX10)
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
|
|
switch (Opcode) {
|
|
|
|
|
case AMDGPU::V_LSHLREV_B64:
|
|
|
|
|
case AMDGPU::V_LSHLREV_B64_gfx10:
|
|
|
|
|
case AMDGPU::V_LSHL_B64:
|
|
|
|
|
case AMDGPU::V_LSHRREV_B64:
|
|
|
|
|
case AMDGPU::V_LSHRREV_B64_gfx10:
|
|
|
|
|
case AMDGPU::V_LSHR_B64:
|
|
|
|
|
case AMDGPU::V_ASHRREV_I64:
|
|
|
|
|
case AMDGPU::V_ASHRREV_I64_gfx10:
|
|
|
|
|
case AMDGPU::V_ASHR_I64:
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 2;
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
unsigned AMDGPUSubtarget::getMaxLocalMemSizeWithWaveCount(unsigned NWaves,
|
2017-02-02 06:59:50 +08:00
|
|
|
const Function &F) const {
|
|
|
|
|
if (NWaves == 1)
|
2016-05-17 05:19:59 +08:00
|
|
|
return getLocalMemorySize();
|
2017-02-02 06:59:50 +08:00
|
|
|
unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second;
|
|
|
|
|
unsigned WorkGroupsPerCu = getMaxWorkGroupsPerCU(WorkGroupSize);
|
2019-04-25 01:03:15 +08:00
|
|
|
if (!WorkGroupsPerCu)
|
|
|
|
|
return 0;
|
2017-02-02 06:59:50 +08:00
|
|
|
unsigned MaxWaves = getMaxWavesPerEU();
|
|
|
|
|
return getLocalMemorySize() * MaxWaves / WorkGroupsPerCu / NWaves;
|
2016-05-17 05:19:59 +08:00
|
|
|
}
|
|
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
unsigned AMDGPUSubtarget::getOccupancyWithLocalMemSize(uint32_t Bytes,
|
2017-02-02 06:59:50 +08:00
|
|
|
const Function &F) const {
|
|
|
|
|
unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second;
|
|
|
|
|
unsigned WorkGroupsPerCu = getMaxWorkGroupsPerCU(WorkGroupSize);
|
2019-04-25 01:03:15 +08:00
|
|
|
if (!WorkGroupsPerCu)
|
|
|
|
|
return 0;
|
2017-02-02 06:59:50 +08:00
|
|
|
unsigned MaxWaves = getMaxWavesPerEU();
|
|
|
|
|
unsigned Limit = getLocalMemorySize() * MaxWaves / WorkGroupsPerCu;
|
|
|
|
|
unsigned NumWaves = Limit / (Bytes ? Bytes : 1u);
|
|
|
|
|
NumWaves = std::min(NumWaves, MaxWaves);
|
|
|
|
|
NumWaves = std::max(NumWaves, 1u);
|
|
|
|
|
return NumWaves;
|
2016-05-17 05:19:59 +08:00
|
|
|
}
|
|
|
|
|
|
AMDGPU: Remove #include "MCTargetDesc/AMDGPUMCTargetDesc.h" from common headers
Summary:
MCTargetDesc/AMDGPUMCTargetDesc.h contains enums for all the instuction
and register defintions, which are huge so we only want to include
them where needed.
This will also make it easier if we want to split the R600 and GCN
definitions into separate tablegenerated files.
I was unable to remove AMDGPUMCTargetDesc.h from SIMachineFunctionInfo.h
because it uses some enums from the header to initialize default values
for the SIMachineFunction class, so I ended up having to remove includes of
SIMachineFunctionInfo.h from headers too.
Reviewers: arsenm, nhaehnle
Reviewed By: nhaehnle
Subscribers: MatzeB, kzhuravl, wdng, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46272
llvm-svn: 332930
2018-05-22 10:03:23 +08:00
|
|
|
unsigned
|
2018-07-12 04:59:01 +08:00
|
|
|
AMDGPUSubtarget::getOccupancyWithLocalMemSize(const MachineFunction &MF) const {
|
AMDGPU: Remove #include "MCTargetDesc/AMDGPUMCTargetDesc.h" from common headers
Summary:
MCTargetDesc/AMDGPUMCTargetDesc.h contains enums for all the instuction
and register defintions, which are huge so we only want to include
them where needed.
This will also make it easier if we want to split the R600 and GCN
definitions into separate tablegenerated files.
I was unable to remove AMDGPUMCTargetDesc.h from SIMachineFunctionInfo.h
because it uses some enums from the header to initialize default values
for the SIMachineFunction class, so I ended up having to remove includes of
SIMachineFunctionInfo.h from headers too.
Reviewers: arsenm, nhaehnle
Reviewed By: nhaehnle
Subscribers: MatzeB, kzhuravl, wdng, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46272
llvm-svn: 332930
2018-05-22 10:03:23 +08:00
|
|
|
const auto *MFI = MF.getInfo<SIMachineFunctionInfo>();
|
|
|
|
|
return getOccupancyWithLocalMemSize(MFI->getLDSSize(), MF.getFunction());
|
|
|
|
|
}
|
|
|
|
|
|
2017-10-24 01:09:35 +08:00
|
|
|
std::pair<unsigned, unsigned>
|
2018-07-12 04:59:01 +08:00
|
|
|
AMDGPUSubtarget::getDefaultFlatWorkGroupSize(CallingConv::ID CC) const {
|
2017-10-24 01:09:35 +08:00
|
|
|
switch (CC) {
|
|
|
|
|
case CallingConv::AMDGPU_CS:
|
|
|
|
|
case CallingConv::AMDGPU_KERNEL:
|
|
|
|
|
case CallingConv::SPIR_KERNEL:
|
2019-04-25 01:03:15 +08:00
|
|
|
return std::make_pair(getWavefrontSize() * 2,
|
|
|
|
|
std::max(getWavefrontSize() * 4, 256u));
|
2017-10-24 01:09:35 +08:00
|
|
|
case CallingConv::AMDGPU_VS:
|
|
|
|
|
case CallingConv::AMDGPU_LS:
|
|
|
|
|
case CallingConv::AMDGPU_HS:
|
|
|
|
|
case CallingConv::AMDGPU_ES:
|
|
|
|
|
case CallingConv::AMDGPU_GS:
|
|
|
|
|
case CallingConv::AMDGPU_PS:
|
|
|
|
|
return std::make_pair(1, getWavefrontSize());
|
|
|
|
|
default:
|
|
|
|
|
return std::make_pair(1, 16 * getWavefrontSize());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
std::pair<unsigned, unsigned> AMDGPUSubtarget::getFlatWorkGroupSizes(
|
2016-09-07 04:22:28 +08:00
|
|
|
const Function &F) const {
|
2017-10-24 01:09:35 +08:00
|
|
|
// FIXME: 1024 if function.
|
2016-09-07 04:22:28 +08:00
|
|
|
// Default minimum/maximum flat work group sizes.
|
|
|
|
|
std::pair<unsigned, unsigned> Default =
|
2017-10-24 01:09:35 +08:00
|
|
|
getDefaultFlatWorkGroupSize(F.getCallingConv());
|
2016-09-07 04:22:28 +08:00
|
|
|
|
|
|
|
|
// Requested minimum/maximum flat work group sizes.
|
|
|
|
|
std::pair<unsigned, unsigned> Requested = AMDGPU::getIntegerPairAttribute(
|
|
|
|
|
F, "amdgpu-flat-work-group-size", Default);
|
|
|
|
|
|
|
|
|
|
// Make sure requested minimum is less than requested maximum.
|
|
|
|
|
if (Requested.first > Requested.second)
|
|
|
|
|
return Default;
|
|
|
|
|
|
|
|
|
|
// Make sure requested values do not violate subtarget's specifications.
|
|
|
|
|
if (Requested.first < getMinFlatWorkGroupSize())
|
|
|
|
|
return Default;
|
|
|
|
|
if (Requested.second > getMaxFlatWorkGroupSize())
|
|
|
|
|
return Default;
|
|
|
|
|
|
|
|
|
|
return Requested;
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
std::pair<unsigned, unsigned> AMDGPUSubtarget::getWavesPerEU(
|
2016-09-07 04:22:28 +08:00
|
|
|
const Function &F) const {
|
|
|
|
|
// Default minimum/maximum number of waves per execution unit.
|
2017-02-10 05:33:23 +08:00
|
|
|
std::pair<unsigned, unsigned> Default(1, getMaxWavesPerEU());
|
2016-09-07 04:22:28 +08:00
|
|
|
|
|
|
|
|
// Default/requested minimum/maximum flat work group sizes.
|
|
|
|
|
std::pair<unsigned, unsigned> FlatWorkGroupSizes = getFlatWorkGroupSizes(F);
|
|
|
|
|
|
|
|
|
|
// If minimum/maximum flat work group sizes were explicitly requested using
|
|
|
|
|
// "amdgpu-flat-work-group-size" attribute, then set default minimum/maximum
|
|
|
|
|
// number of waves per execution unit to values implied by requested
|
|
|
|
|
// minimum/maximum flat work group sizes.
|
|
|
|
|
unsigned MinImpliedByFlatWorkGroupSize =
|
|
|
|
|
getMaxWavesPerEU(FlatWorkGroupSizes.second);
|
|
|
|
|
bool RequestedFlatWorkGroupSize = false;
|
|
|
|
|
|
2019-06-06 04:32:32 +08:00
|
|
|
if (F.hasFnAttribute("amdgpu-flat-work-group-size")) {
|
2016-09-07 04:22:28 +08:00
|
|
|
Default.first = MinImpliedByFlatWorkGroupSize;
|
|
|
|
|
RequestedFlatWorkGroupSize = true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Requested minimum/maximum number of waves per execution unit.
|
|
|
|
|
std::pair<unsigned, unsigned> Requested = AMDGPU::getIntegerPairAttribute(
|
|
|
|
|
F, "amdgpu-waves-per-eu", Default, true);
|
|
|
|
|
|
|
|
|
|
// Make sure requested minimum is less than requested maximum.
|
|
|
|
|
if (Requested.second && Requested.first > Requested.second)
|
|
|
|
|
return Default;
|
|
|
|
|
|
|
|
|
|
// Make sure requested values do not violate subtarget's specifications.
|
|
|
|
|
if (Requested.first < getMinWavesPerEU() ||
|
|
|
|
|
Requested.first > getMaxWavesPerEU())
|
|
|
|
|
return Default;
|
|
|
|
|
if (Requested.second > getMaxWavesPerEU())
|
|
|
|
|
return Default;
|
|
|
|
|
|
|
|
|
|
// Make sure requested values are compatible with values implied by requested
|
|
|
|
|
// minimum/maximum flat work group sizes.
|
|
|
|
|
if (RequestedFlatWorkGroupSize &&
|
2017-07-17 03:38:47 +08:00
|
|
|
Requested.first < MinImpliedByFlatWorkGroupSize)
|
2016-09-07 04:22:28 +08:00
|
|
|
return Default;
|
|
|
|
|
|
|
|
|
|
return Requested;
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
bool AMDGPUSubtarget::makeLIDRangeMetadata(Instruction *I) const {
|
2017-04-13 04:48:56 +08:00
|
|
|
Function *Kernel = I->getParent()->getParent();
|
|
|
|
|
unsigned MinSize = 0;
|
|
|
|
|
unsigned MaxSize = getFlatWorkGroupSizes(*Kernel).second;
|
|
|
|
|
bool IdQuery = false;
|
|
|
|
|
|
|
|
|
|
// If reqd_work_group_size is present it narrows value down.
|
|
|
|
|
if (auto *CI = dyn_cast<CallInst>(I)) {
|
|
|
|
|
const Function *F = CI->getCalledFunction();
|
|
|
|
|
if (F) {
|
|
|
|
|
unsigned Dim = UINT_MAX;
|
|
|
|
|
switch (F->getIntrinsicID()) {
|
|
|
|
|
case Intrinsic::amdgcn_workitem_id_x:
|
|
|
|
|
case Intrinsic::r600_read_tidig_x:
|
|
|
|
|
IdQuery = true;
|
2017-07-07 18:18:57 +08:00
|
|
|
LLVM_FALLTHROUGH;
|
2017-04-13 04:48:56 +08:00
|
|
|
case Intrinsic::r600_read_local_size_x:
|
|
|
|
|
Dim = 0;
|
|
|
|
|
break;
|
|
|
|
|
case Intrinsic::amdgcn_workitem_id_y:
|
|
|
|
|
case Intrinsic::r600_read_tidig_y:
|
|
|
|
|
IdQuery = true;
|
2017-07-07 18:18:57 +08:00
|
|
|
LLVM_FALLTHROUGH;
|
2017-04-13 04:48:56 +08:00
|
|
|
case Intrinsic::r600_read_local_size_y:
|
|
|
|
|
Dim = 1;
|
|
|
|
|
break;
|
|
|
|
|
case Intrinsic::amdgcn_workitem_id_z:
|
|
|
|
|
case Intrinsic::r600_read_tidig_z:
|
|
|
|
|
IdQuery = true;
|
2017-07-07 18:18:57 +08:00
|
|
|
LLVM_FALLTHROUGH;
|
2017-04-13 04:48:56 +08:00
|
|
|
case Intrinsic::r600_read_local_size_z:
|
|
|
|
|
Dim = 2;
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
if (Dim <= 3) {
|
|
|
|
|
if (auto Node = Kernel->getMetadata("reqd_work_group_size"))
|
|
|
|
|
if (Node->getNumOperands() == 3)
|
|
|
|
|
MinSize = MaxSize = mdconst::extract<ConstantInt>(
|
|
|
|
|
Node->getOperand(Dim))->getZExtValue();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!MaxSize)
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
// Range metadata is [Lo, Hi). For ID query we need to pass max size
|
|
|
|
|
// as Hi. For size query we need to pass Hi + 1.
|
|
|
|
|
if (IdQuery)
|
|
|
|
|
MinSize = 0;
|
|
|
|
|
else
|
|
|
|
|
++MaxSize;
|
|
|
|
|
|
|
|
|
|
MDBuilder MDB(I->getContext());
|
|
|
|
|
MDNode *MaxWorkGroupSizeRange = MDB.createRange(APInt(32, MinSize),
|
|
|
|
|
APInt(32, MaxSize));
|
|
|
|
|
I->setMetadata(LLVMContext::MD_range, MaxWorkGroupSizeRange);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-20 17:05:08 +08:00
|
|
|
uint64_t AMDGPUSubtarget::getExplicitKernArgSize(const Function &F,
|
2019-10-15 20:56:24 +08:00
|
|
|
Align &MaxAlign) const {
|
2018-07-20 17:05:08 +08:00
|
|
|
assert(F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
|
|
|
|
|
F.getCallingConv() == CallingConv::SPIR_KERNEL);
|
|
|
|
|
|
|
|
|
|
const DataLayout &DL = F.getParent()->getDataLayout();
|
|
|
|
|
uint64_t ExplicitArgBytes = 0;
|
2019-10-15 20:56:24 +08:00
|
|
|
MaxAlign = Align::None();
|
2018-07-20 17:05:08 +08:00
|
|
|
|
|
|
|
|
for (const Argument &Arg : F.args()) {
|
|
|
|
|
Type *ArgTy = Arg.getType();
|
|
|
|
|
|
2019-10-15 20:56:24 +08:00
|
|
|
const Align Alignment(DL.getABITypeAlignment(ArgTy));
|
2018-07-20 17:05:08 +08:00
|
|
|
uint64_t AllocSize = DL.getTypeAllocSize(ArgTy);
|
2019-10-15 20:56:24 +08:00
|
|
|
ExplicitArgBytes = alignTo(ExplicitArgBytes, Alignment) + AllocSize;
|
|
|
|
|
MaxAlign = std::max(MaxAlign, Alignment);
|
2018-07-20 17:05:08 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return ExplicitArgBytes;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
unsigned AMDGPUSubtarget::getKernArgSegmentSize(const Function &F,
|
2019-10-15 20:56:24 +08:00
|
|
|
Align &MaxAlign) const {
|
2018-07-20 17:05:08 +08:00
|
|
|
uint64_t ExplicitArgBytes = getExplicitKernArgSize(F, MaxAlign);
|
|
|
|
|
|
|
|
|
|
unsigned ExplicitOffset = getExplicitKernelArgOffset(F);
|
|
|
|
|
|
|
|
|
|
uint64_t TotalSize = ExplicitOffset + ExplicitArgBytes;
|
|
|
|
|
unsigned ImplicitBytes = getImplicitArgNumBytes(F);
|
|
|
|
|
if (ImplicitBytes != 0) {
|
2019-10-15 20:56:24 +08:00
|
|
|
const Align Alignment = getAlignmentForImplicitArgPtr();
|
2018-07-20 17:05:08 +08:00
|
|
|
TotalSize = alignTo(ExplicitArgBytes, Alignment) + ImplicitBytes;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Being able to dereference past the end is useful for emitting scalar loads.
|
|
|
|
|
return alignTo(TotalSize, 4);
|
|
|
|
|
}
|
|
|
|
|
|
2016-06-24 14:30:11 +08:00
|
|
|
R600Subtarget::R600Subtarget(const Triple &TT, StringRef GPU, StringRef FS,
|
|
|
|
|
const TargetMachine &TM) :
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
R600GenSubtargetInfo(TT, GPU, FS),
|
2018-09-13 02:50:47 +08:00
|
|
|
AMDGPUSubtarget(TT),
|
2016-06-24 14:30:11 +08:00
|
|
|
InstrInfo(*this),
|
|
|
|
|
FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0),
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
FMA(false),
|
|
|
|
|
CaymanISA(false),
|
|
|
|
|
CFALUBug(false),
|
|
|
|
|
HasVertexCache(false),
|
|
|
|
|
R600ALUInst(false),
|
|
|
|
|
FP64(false),
|
|
|
|
|
TexVTXClauseSize(0),
|
|
|
|
|
Gen(R600),
|
|
|
|
|
TLInfo(TM, initializeSubtargetDependencies(TT, GPU, FS)),
|
2018-08-31 13:49:54 +08:00
|
|
|
InstrItins(getInstrItineraryForCPU(GPU)) { }
|
2016-06-24 14:30:11 +08:00
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
void GCNSubtarget::overrideSchedPolicy(MachineSchedPolicy &Policy,
|
2016-06-28 08:11:26 +08:00
|
|
|
unsigned NumRegionInstrs) const {
|
2016-06-24 14:30:11 +08:00
|
|
|
// Track register pressure so the scheduler can try to decrease
|
|
|
|
|
// pressure once register usage is above the threshold defined by
|
|
|
|
|
// SIRegisterInfo::getRegPressureSetLimit()
|
|
|
|
|
Policy.ShouldTrackPressure = true;
|
|
|
|
|
|
|
|
|
|
// Enabling both top down and bottom up scheduling seems to give us less
|
|
|
|
|
// register spills than just using one of these approaches on its own.
|
|
|
|
|
Policy.OnlyTopDown = false;
|
|
|
|
|
Policy.OnlyBottomUp = false;
|
|
|
|
|
|
2017-02-14 22:29:05 +08:00
|
|
|
// Enabling ShouldTrackLaneMasks crashes the SI Machine Scheduler.
|
|
|
|
|
if (!enableSIScheduler())
|
|
|
|
|
Policy.ShouldTrackLaneMasks = true;
|
2016-06-24 14:30:11 +08:00
|
|
|
}
|
2015-01-30 00:55:25 +08:00
|
|
|
|
2019-04-25 01:03:15 +08:00
|
|
|
bool GCNSubtarget::hasMadF16() const {
|
|
|
|
|
return InstrInfo.pseudoToMCOpcode(AMDGPU::V_MAD_F16) != -1;
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
unsigned GCNSubtarget::getOccupancyWithNumSGPRs(unsigned SGPRs) const {
|
2019-04-25 01:03:15 +08:00
|
|
|
if (getGeneration() >= AMDGPUSubtarget::GFX10)
|
2019-07-31 09:07:10 +08:00
|
|
|
return getMaxWavesPerEU();
|
2019-04-25 01:03:15 +08:00
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
|
2016-08-30 03:42:52 +08:00
|
|
|
if (SGPRs <= 80)
|
|
|
|
|
return 10;
|
|
|
|
|
if (SGPRs <= 88)
|
|
|
|
|
return 9;
|
|
|
|
|
if (SGPRs <= 100)
|
|
|
|
|
return 8;
|
|
|
|
|
return 7;
|
|
|
|
|
}
|
|
|
|
|
if (SGPRs <= 48)
|
|
|
|
|
return 10;
|
|
|
|
|
if (SGPRs <= 56)
|
|
|
|
|
return 9;
|
|
|
|
|
if (SGPRs <= 64)
|
|
|
|
|
return 8;
|
|
|
|
|
if (SGPRs <= 72)
|
|
|
|
|
return 7;
|
|
|
|
|
if (SGPRs <= 80)
|
|
|
|
|
return 6;
|
|
|
|
|
return 5;
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
unsigned GCNSubtarget::getOccupancyWithNumVGPRs(unsigned VGPRs) const {
|
2019-07-20 05:29:51 +08:00
|
|
|
unsigned MaxWaves = getMaxWavesPerEU();
|
|
|
|
|
unsigned Granule = getVGPRAllocGranule();
|
|
|
|
|
if (VGPRs < Granule)
|
|
|
|
|
return MaxWaves;
|
|
|
|
|
unsigned RoundedRegs = ((VGPRs + Granule - 1) / Granule) * Granule;
|
2019-09-20 04:09:04 +08:00
|
|
|
return std::min(std::max(getTotalNumVGPRs() / RoundedRegs, 1u), MaxWaves);
|
2016-08-30 03:42:52 +08:00
|
|
|
}
|
2016-10-29 04:31:47 +08:00
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
unsigned GCNSubtarget::getReservedNumSGPRs(const MachineFunction &MF) const {
|
2017-02-08 21:02:33 +08:00
|
|
|
const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
|
2019-04-25 01:03:15 +08:00
|
|
|
if (getGeneration() >= AMDGPUSubtarget::GFX10)
|
|
|
|
|
return 2; // VCC. FLAT_SCRATCH and XNACK are no longer in SGPRs.
|
|
|
|
|
|
2017-02-08 21:02:33 +08:00
|
|
|
if (MFI.hasFlatScratchInit()) {
|
|
|
|
|
if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
|
|
|
|
|
return 6; // FLAT_SCRATCH, XNACK, VCC (in that order).
|
|
|
|
|
if (getGeneration() == AMDGPUSubtarget::SEA_ISLANDS)
|
|
|
|
|
return 4; // FLAT_SCRATCH, VCC (in that order).
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (isXNACKEnabled())
|
|
|
|
|
return 4; // XNACK, VCC (in that order).
|
|
|
|
|
return 2; // VCC.
|
|
|
|
|
}
|
|
|
|
|
|
2019-07-31 09:07:10 +08:00
|
|
|
unsigned GCNSubtarget::computeOccupancy(const MachineFunction &MF,
|
|
|
|
|
unsigned LDSSize,
|
|
|
|
|
unsigned NumSGPRs,
|
|
|
|
|
unsigned NumVGPRs) const {
|
|
|
|
|
unsigned Occupancy =
|
|
|
|
|
std::min(getMaxWavesPerEU(),
|
|
|
|
|
getOccupancyWithLocalMemSize(LDSSize, MF.getFunction()));
|
|
|
|
|
if (NumSGPRs)
|
|
|
|
|
Occupancy = std::min(Occupancy, getOccupancyWithNumSGPRs(NumSGPRs));
|
|
|
|
|
if (NumVGPRs)
|
|
|
|
|
Occupancy = std::min(Occupancy, getOccupancyWithNumVGPRs(NumVGPRs));
|
|
|
|
|
return Occupancy;
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
unsigned GCNSubtarget::getMaxNumSGPRs(const MachineFunction &MF) const {
|
2017-12-16 06:22:58 +08:00
|
|
|
const Function &F = MF.getFunction();
|
2017-02-08 21:02:33 +08:00
|
|
|
const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
|
|
|
|
|
|
|
|
|
|
// Compute maximum number of SGPRs function can use using default/requested
|
|
|
|
|
// minimum number of waves per execution unit.
|
|
|
|
|
std::pair<unsigned, unsigned> WavesPerEU = MFI.getWavesPerEU();
|
|
|
|
|
unsigned MaxNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, false);
|
|
|
|
|
unsigned MaxAddressableNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, true);
|
|
|
|
|
|
|
|
|
|
// Check if maximum number of SGPRs was explicitly requested using
|
|
|
|
|
// "amdgpu-num-sgpr" attribute.
|
|
|
|
|
if (F.hasFnAttribute("amdgpu-num-sgpr")) {
|
|
|
|
|
unsigned Requested = AMDGPU::getIntegerAttribute(
|
|
|
|
|
F, "amdgpu-num-sgpr", MaxNumSGPRs);
|
|
|
|
|
|
|
|
|
|
// Make sure requested value does not violate subtarget's specifications.
|
|
|
|
|
if (Requested && (Requested <= getReservedNumSGPRs(MF)))
|
|
|
|
|
Requested = 0;
|
|
|
|
|
|
|
|
|
|
// If more SGPRs are required to support the input user/system SGPRs,
|
|
|
|
|
// increase to accommodate them.
|
|
|
|
|
//
|
|
|
|
|
// FIXME: This really ends up using the requested number of SGPRs + number
|
|
|
|
|
// of reserved special registers in total. Theoretically you could re-use
|
|
|
|
|
// the last input registers for these special registers, but this would
|
|
|
|
|
// require a lot of complexity to deal with the weird aliasing.
|
|
|
|
|
unsigned InputNumSGPRs = MFI.getNumPreloadedSGPRs();
|
|
|
|
|
if (Requested && Requested < InputNumSGPRs)
|
|
|
|
|
Requested = InputNumSGPRs;
|
|
|
|
|
|
|
|
|
|
// Make sure requested value is compatible with values implied by
|
|
|
|
|
// default/requested minimum/maximum number of waves per execution unit.
|
|
|
|
|
if (Requested && Requested > getMaxNumSGPRs(WavesPerEU.first, false))
|
|
|
|
|
Requested = 0;
|
|
|
|
|
if (WavesPerEU.second &&
|
|
|
|
|
Requested && Requested < getMinNumSGPRs(WavesPerEU.second))
|
|
|
|
|
Requested = 0;
|
|
|
|
|
|
|
|
|
|
if (Requested)
|
|
|
|
|
MaxNumSGPRs = Requested;
|
|
|
|
|
}
|
|
|
|
|
|
2016-10-29 04:31:47 +08:00
|
|
|
if (hasSGPRInitBug())
|
2017-02-08 22:05:23 +08:00
|
|
|
MaxNumSGPRs = AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG;
|
2017-02-08 21:02:33 +08:00
|
|
|
|
|
|
|
|
return std::min(MaxNumSGPRs - getReservedNumSGPRs(MF),
|
|
|
|
|
MaxAddressableNumSGPRs);
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
unsigned GCNSubtarget::getMaxNumVGPRs(const MachineFunction &MF) const {
|
2017-12-16 06:22:58 +08:00
|
|
|
const Function &F = MF.getFunction();
|
2017-02-08 21:02:33 +08:00
|
|
|
const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
|
|
|
|
|
|
|
|
|
|
// Compute maximum number of VGPRs function can use using default/requested
|
|
|
|
|
// minimum number of waves per execution unit.
|
|
|
|
|
std::pair<unsigned, unsigned> WavesPerEU = MFI.getWavesPerEU();
|
|
|
|
|
unsigned MaxNumVGPRs = getMaxNumVGPRs(WavesPerEU.first);
|
|
|
|
|
|
|
|
|
|
// Check if maximum number of VGPRs was explicitly requested using
|
|
|
|
|
// "amdgpu-num-vgpr" attribute.
|
|
|
|
|
if (F.hasFnAttribute("amdgpu-num-vgpr")) {
|
|
|
|
|
unsigned Requested = AMDGPU::getIntegerAttribute(
|
|
|
|
|
F, "amdgpu-num-vgpr", MaxNumVGPRs);
|
|
|
|
|
|
|
|
|
|
// Make sure requested value is compatible with values implied by
|
|
|
|
|
// default/requested minimum/maximum number of waves per execution unit.
|
|
|
|
|
if (Requested && Requested > getMaxNumVGPRs(WavesPerEU.first))
|
|
|
|
|
Requested = 0;
|
|
|
|
|
if (WavesPerEU.second &&
|
|
|
|
|
Requested && Requested < getMinNumVGPRs(WavesPerEU.second))
|
|
|
|
|
Requested = 0;
|
|
|
|
|
|
|
|
|
|
if (Requested)
|
|
|
|
|
MaxNumVGPRs = Requested;
|
|
|
|
|
}
|
2016-10-29 04:31:47 +08:00
|
|
|
|
2018-06-22 04:28:19 +08:00
|
|
|
return MaxNumVGPRs;
|
2016-10-29 04:31:47 +08:00
|
|
|
}
|
2017-09-20 04:54:38 +08:00
|
|
|
|
2017-11-01 07:21:30 +08:00
|
|
|
namespace {
|
2017-09-20 04:54:38 +08:00
|
|
|
struct MemOpClusterMutation : ScheduleDAGMutation {
|
|
|
|
|
const SIInstrInfo *TII;
|
|
|
|
|
|
|
|
|
|
MemOpClusterMutation(const SIInstrInfo *tii) : TII(tii) {}
|
|
|
|
|
|
2019-03-29 16:33:05 +08:00
|
|
|
void apply(ScheduleDAGInstrs *DAG) override {
|
2017-09-20 04:54:38 +08:00
|
|
|
SUnit *SUa = nullptr;
|
|
|
|
|
// Search for two consequent memory operations and link them
|
|
|
|
|
// to prevent scheduler from moving them apart.
|
|
|
|
|
// In DAG pre-process SUnits are in the original order of
|
|
|
|
|
// the instructions before scheduling.
|
|
|
|
|
for (SUnit &SU : DAG->SUnits) {
|
|
|
|
|
MachineInstr &MI2 = *SU.getInstr();
|
|
|
|
|
if (!MI2.mayLoad() && !MI2.mayStore()) {
|
|
|
|
|
SUa = nullptr;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (!SUa) {
|
|
|
|
|
SUa = &SU;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
MachineInstr &MI1 = *SUa->getInstr();
|
|
|
|
|
if ((TII->isVMEM(MI1) && TII->isVMEM(MI2)) ||
|
|
|
|
|
(TII->isFLAT(MI1) && TII->isFLAT(MI2)) ||
|
|
|
|
|
(TII->isSMRD(MI1) && TII->isSMRD(MI2)) ||
|
|
|
|
|
(TII->isDS(MI1) && TII->isDS(MI2))) {
|
|
|
|
|
SU.addPredBarrier(SUa);
|
|
|
|
|
|
|
|
|
|
for (const SDep &SI : SU.Preds) {
|
|
|
|
|
if (SI.getSUnit() != SUa)
|
|
|
|
|
SUa->addPred(SDep(SI.getSUnit(), SDep::Artificial));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (&SU != &DAG->ExitSU) {
|
|
|
|
|
for (const SDep &SI : SUa->Succs) {
|
|
|
|
|
if (SI.getSUnit() != &SU)
|
|
|
|
|
SI.getSUnit()->addPred(SDep(&SU, SDep::Artificial));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
SUa = &SU;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
};
|
2019-07-12 05:25:00 +08:00
|
|
|
|
|
|
|
|
struct FillMFMAShadowMutation : ScheduleDAGMutation {
|
|
|
|
|
const SIInstrInfo *TII;
|
|
|
|
|
|
|
|
|
|
ScheduleDAGMI *DAG;
|
|
|
|
|
|
|
|
|
|
FillMFMAShadowMutation(const SIInstrInfo *tii) : TII(tii) {}
|
|
|
|
|
|
|
|
|
|
bool isSALU(const SUnit *SU) const {
|
2019-07-15 23:34:05 +08:00
|
|
|
const MachineInstr *MI = SU->getInstr();
|
|
|
|
|
return MI && TII->isSALU(*MI) && !MI->isTerminator();
|
2019-07-12 05:25:00 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool canAddEdge(const SUnit *Succ, const SUnit *Pred) const {
|
|
|
|
|
if (Pred->NodeNum < Succ->NodeNum)
|
|
|
|
|
return true;
|
|
|
|
|
|
|
|
|
|
SmallVector<const SUnit*, 64> Succs({Succ}), Preds({Pred});
|
|
|
|
|
|
|
|
|
|
for (unsigned I = 0; I < Succs.size(); ++I) {
|
|
|
|
|
for (const SDep &SI : Succs[I]->Succs) {
|
|
|
|
|
const SUnit *SU = SI.getSUnit();
|
|
|
|
|
if (SU != Succs[I] && llvm::find(Succs, SU) == Succs.end())
|
|
|
|
|
Succs.push_back(SU);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
SmallPtrSet<const SUnit*, 32> Visited;
|
|
|
|
|
while (!Preds.empty()) {
|
|
|
|
|
const SUnit *SU = Preds.pop_back_val();
|
|
|
|
|
if (llvm::find(Succs, SU) != Succs.end())
|
|
|
|
|
return false;
|
|
|
|
|
Visited.insert(SU);
|
|
|
|
|
for (const SDep &SI : SU->Preds)
|
|
|
|
|
if (SI.getSUnit() != SU && !Visited.count(SI.getSUnit()))
|
|
|
|
|
Preds.push_back(SI.getSUnit());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Link as much SALU intructions in chain as possible. Return the size
|
|
|
|
|
// of the chain. Links up to MaxChain instructions.
|
|
|
|
|
unsigned linkSALUChain(SUnit *From, SUnit *To, unsigned MaxChain,
|
|
|
|
|
SmallPtrSetImpl<SUnit *> &Visited) const {
|
|
|
|
|
SmallVector<SUnit *, 8> Worklist({To});
|
|
|
|
|
unsigned Linked = 0;
|
|
|
|
|
|
|
|
|
|
while (!Worklist.empty() && MaxChain-- > 0) {
|
|
|
|
|
SUnit *SU = Worklist.pop_back_val();
|
|
|
|
|
if (!Visited.insert(SU).second)
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
LLVM_DEBUG(dbgs() << "Inserting edge from\n" ; DAG->dumpNode(*From);
|
|
|
|
|
dbgs() << "to\n"; DAG->dumpNode(*SU); dbgs() << '\n');
|
|
|
|
|
|
|
|
|
|
if (SU->addPred(SDep(From, SDep::Artificial), false))
|
|
|
|
|
++Linked;
|
|
|
|
|
|
|
|
|
|
for (SDep &SI : From->Succs) {
|
|
|
|
|
SUnit *SUv = SI.getSUnit();
|
|
|
|
|
if (SUv != From && TII->isVALU(*SUv->getInstr()) && canAddEdge(SUv, SU))
|
|
|
|
|
SUv->addPred(SDep(SU, SDep::Artificial), false);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (SDep &SI : SU->Succs) {
|
|
|
|
|
SUnit *Succ = SI.getSUnit();
|
|
|
|
|
if (Succ != SU && isSALU(Succ) && canAddEdge(From, Succ))
|
|
|
|
|
Worklist.push_back(Succ);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return Linked;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void apply(ScheduleDAGInstrs *DAGInstrs) override {
|
|
|
|
|
const GCNSubtarget &ST = DAGInstrs->MF.getSubtarget<GCNSubtarget>();
|
|
|
|
|
if (!ST.hasMAIInsts() || DisablePowerSched)
|
|
|
|
|
return;
|
|
|
|
|
DAG = static_cast<ScheduleDAGMI*>(DAGInstrs);
|
|
|
|
|
const TargetSchedModel *TSchedModel = DAGInstrs->getSchedModel();
|
|
|
|
|
if (!TSchedModel || DAG->SUnits.empty())
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
// Scan for MFMA long latency instructions and try to add a dependency
|
|
|
|
|
// of available SALU instructions to give them a chance to fill MFMA
|
|
|
|
|
// shadow. That is desirable to fill MFMA shadow with SALU instructions
|
|
|
|
|
// rather than VALU to prevent power consumption bursts and throttle.
|
|
|
|
|
auto LastSALU = DAG->SUnits.begin();
|
|
|
|
|
auto E = DAG->SUnits.end();
|
|
|
|
|
SmallPtrSet<SUnit*, 32> Visited;
|
|
|
|
|
for (SUnit &SU : DAG->SUnits) {
|
|
|
|
|
MachineInstr &MAI = *SU.getInstr();
|
|
|
|
|
if (!TII->isMAI(MAI) ||
|
|
|
|
|
MAI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32 ||
|
|
|
|
|
MAI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32)
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
unsigned Lat = TSchedModel->computeInstrLatency(&MAI) - 1;
|
|
|
|
|
|
|
|
|
|
LLVM_DEBUG(dbgs() << "Found MFMA: "; DAG->dumpNode(SU);
|
|
|
|
|
dbgs() << "Need " << Lat
|
|
|
|
|
<< " instructions to cover latency.\n");
|
|
|
|
|
|
|
|
|
|
// Find up to Lat independent scalar instructions as early as
|
|
|
|
|
// possible such that they can be scheduled after this MFMA.
|
|
|
|
|
for ( ; Lat && LastSALU != E; ++LastSALU) {
|
|
|
|
|
if (Visited.count(&*LastSALU))
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
if (!isSALU(&*LastSALU) || !canAddEdge(&*LastSALU, &SU))
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
Lat -= linkSALUChain(&SU, &*LastSALU, Lat, Visited);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
};
|
2017-11-01 07:21:30 +08:00
|
|
|
} // namespace
|
2017-09-20 04:54:38 +08:00
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
void GCNSubtarget::getPostRAMutations(
|
2017-09-20 04:54:38 +08:00
|
|
|
std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
|
2019-08-15 23:54:37 +08:00
|
|
|
Mutations.push_back(std::make_unique<MemOpClusterMutation>(&InstrInfo));
|
|
|
|
|
Mutations.push_back(std::make_unique<FillMFMAShadowMutation>(&InstrInfo));
|
2017-09-20 04:54:38 +08:00
|
|
|
}
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
const AMDGPUSubtarget &AMDGPUSubtarget::get(const MachineFunction &MF) {
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
if (MF.getTarget().getTargetTriple().getArch() == Triple::amdgcn)
|
2018-07-12 04:59:01 +08:00
|
|
|
return static_cast<const AMDGPUSubtarget&>(MF.getSubtarget<GCNSubtarget>());
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
else
|
2018-07-12 04:59:01 +08:00
|
|
|
return static_cast<const AMDGPUSubtarget&>(MF.getSubtarget<R600Subtarget>());
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
}
|
|
|
|
|
|
2018-07-12 04:59:01 +08:00
|
|
|
const AMDGPUSubtarget &AMDGPUSubtarget::get(const TargetMachine &TM, const Function &F) {
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
if (TM.getTargetTriple().getArch() == Triple::amdgcn)
|
2018-07-12 04:59:01 +08:00
|
|
|
return static_cast<const AMDGPUSubtarget&>(TM.getSubtarget<GCNSubtarget>(F));
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
else
|
2018-07-12 04:59:01 +08:00
|
|
|
return static_cast<const AMDGPUSubtarget&>(TM.getSubtarget<R600Subtarget>(F));
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 07:47:12 +08:00
|
|
|
}
|