2015-04-21 04:55:21 +08:00
|
|
|
/*
|
|
|
|
* Copyright 2014 Advanced Micro Devices, Inc.
|
|
|
|
* Copyright 2008 Red Hat Inc.
|
|
|
|
* Copyright 2009 Jerome Glisse.
|
|
|
|
*
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
|
|
* to deal in the Software without restriction, including without limitation
|
|
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
|
|
*
|
|
|
|
* The above copyright notice and this permission notice shall be included in
|
|
|
|
* all copies or substantial portions of the Software.
|
|
|
|
*
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
|
|
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
|
|
|
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
|
|
|
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
|
|
* OTHER DEALINGS IN THE SOFTWARE.
|
|
|
|
*
|
|
|
|
*/
|
2019-06-10 06:07:56 +08:00
|
|
|
|
2015-04-21 04:55:21 +08:00
|
|
|
#include "amdgpu.h"
|
2016-10-22 16:48:25 +08:00
|
|
|
#include "amdgpu_gfx.h"
|
2018-11-08 13:43:46 +08:00
|
|
|
#include "amdgpu_rlc.h"
|
2019-09-03 06:06:08 +08:00
|
|
|
#include "amdgpu_ras.h"
|
2015-04-21 04:55:21 +08:00
|
|
|
|
2018-08-06 19:45:04 +08:00
|
|
|
/* delay 0.1 second to enable gfx off feature */
|
|
|
|
#define GFX_OFF_DELAY_ENABLE msecs_to_jiffies(100)
|
2018-07-27 21:06:30 +08:00
|
|
|
|
2015-04-21 04:55:21 +08:00
|
|
|
/*
|
2018-08-02 16:12:39 +08:00
|
|
|
* GPU GFX IP block helpers function.
|
2015-04-21 04:55:21 +08:00
|
|
|
*/
|
2018-08-02 16:12:39 +08:00
|
|
|
|
2018-07-31 15:43:10 +08:00
|
|
|
int amdgpu_gfx_mec_queue_to_bit(struct amdgpu_device *adev, int mec,
|
|
|
|
int pipe, int queue)
|
2018-08-02 16:12:39 +08:00
|
|
|
{
|
|
|
|
int bit = 0;
|
|
|
|
|
|
|
|
bit += mec * adev->gfx.mec.num_pipe_per_mec
|
|
|
|
* adev->gfx.mec.num_queue_per_pipe;
|
|
|
|
bit += pipe * adev->gfx.mec.num_queue_per_pipe;
|
|
|
|
bit += queue;
|
|
|
|
|
|
|
|
return bit;
|
|
|
|
}
|
|
|
|
|
2018-07-31 15:43:10 +08:00
|
|
|
void amdgpu_gfx_bit_to_mec_queue(struct amdgpu_device *adev, int bit,
|
|
|
|
int *mec, int *pipe, int *queue)
|
2018-08-02 16:12:39 +08:00
|
|
|
{
|
|
|
|
*queue = bit % adev->gfx.mec.num_queue_per_pipe;
|
|
|
|
*pipe = (bit / adev->gfx.mec.num_queue_per_pipe)
|
|
|
|
% adev->gfx.mec.num_pipe_per_mec;
|
|
|
|
*mec = (bit / adev->gfx.mec.num_queue_per_pipe)
|
|
|
|
/ adev->gfx.mec.num_pipe_per_mec;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
bool amdgpu_gfx_is_mec_queue_enabled(struct amdgpu_device *adev,
|
|
|
|
int mec, int pipe, int queue)
|
|
|
|
{
|
2018-07-31 15:43:10 +08:00
|
|
|
return test_bit(amdgpu_gfx_mec_queue_to_bit(adev, mec, pipe, queue),
|
2018-08-02 16:12:39 +08:00
|
|
|
adev->gfx.mec.queue_bitmap);
|
|
|
|
}
|
|
|
|
|
2018-07-31 15:43:10 +08:00
|
|
|
int amdgpu_gfx_me_queue_to_bit(struct amdgpu_device *adev,
|
|
|
|
int me, int pipe, int queue)
|
|
|
|
{
|
|
|
|
int bit = 0;
|
|
|
|
|
|
|
|
bit += me * adev->gfx.me.num_pipe_per_me
|
|
|
|
* adev->gfx.me.num_queue_per_pipe;
|
|
|
|
bit += pipe * adev->gfx.me.num_queue_per_pipe;
|
|
|
|
bit += queue;
|
|
|
|
|
|
|
|
return bit;
|
|
|
|
}
|
|
|
|
|
|
|
|
void amdgpu_gfx_bit_to_me_queue(struct amdgpu_device *adev, int bit,
|
|
|
|
int *me, int *pipe, int *queue)
|
|
|
|
{
|
|
|
|
*queue = bit % adev->gfx.me.num_queue_per_pipe;
|
|
|
|
*pipe = (bit / adev->gfx.me.num_queue_per_pipe)
|
|
|
|
% adev->gfx.me.num_pipe_per_me;
|
|
|
|
*me = (bit / adev->gfx.me.num_queue_per_pipe)
|
|
|
|
/ adev->gfx.me.num_pipe_per_me;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool amdgpu_gfx_is_me_queue_enabled(struct amdgpu_device *adev,
|
|
|
|
int me, int pipe, int queue)
|
|
|
|
{
|
|
|
|
return test_bit(amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue),
|
|
|
|
adev->gfx.me.queue_bitmap);
|
|
|
|
}
|
|
|
|
|
2015-04-21 04:55:21 +08:00
|
|
|
/**
|
|
|
|
* amdgpu_gfx_scratch_get - Allocate a scratch register
|
|
|
|
*
|
|
|
|
* @adev: amdgpu_device pointer
|
|
|
|
* @reg: scratch register mmio offset
|
|
|
|
*
|
|
|
|
* Allocate a CP scratch register for use by the driver (all asics).
|
|
|
|
* Returns 0 on success or -EINVAL on failure.
|
|
|
|
*/
|
|
|
|
int amdgpu_gfx_scratch_get(struct amdgpu_device *adev, uint32_t *reg)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2017-01-17 04:56:48 +08:00
|
|
|
i = ffs(adev->gfx.scratch.free_mask);
|
|
|
|
if (i != 0 && i <= adev->gfx.scratch.num_reg) {
|
|
|
|
i--;
|
|
|
|
adev->gfx.scratch.free_mask &= ~(1u << i);
|
|
|
|
*reg = adev->gfx.scratch.reg_base + i;
|
|
|
|
return 0;
|
2015-04-21 04:55:21 +08:00
|
|
|
}
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* amdgpu_gfx_scratch_free - Free a scratch register
|
|
|
|
*
|
|
|
|
* @adev: amdgpu_device pointer
|
|
|
|
* @reg: scratch register mmio offset
|
|
|
|
*
|
|
|
|
* Free a CP scratch register allocated for use by the driver (all asics)
|
|
|
|
*/
|
|
|
|
void amdgpu_gfx_scratch_free(struct amdgpu_device *adev, uint32_t reg)
|
|
|
|
{
|
2017-01-17 04:56:48 +08:00
|
|
|
adev->gfx.scratch.free_mask |= 1u << (reg - adev->gfx.scratch.reg_base);
|
2015-04-21 04:55:21 +08:00
|
|
|
}
|
2016-06-18 01:31:33 +08:00
|
|
|
|
|
|
|
/**
|
|
|
|
* amdgpu_gfx_parse_disable_cu - Parse the disable_cu module parameter
|
|
|
|
*
|
|
|
|
* @mask: array in which the per-shader array disable masks will be stored
|
|
|
|
* @max_se: number of SEs
|
|
|
|
* @max_sh: number of SHs
|
|
|
|
*
|
|
|
|
* The bitmask of CUs to be disabled in the shader array determined by se and
|
|
|
|
* sh is stored in mask[se * max_sh + sh].
|
|
|
|
*/
|
|
|
|
void amdgpu_gfx_parse_disable_cu(unsigned *mask, unsigned max_se, unsigned max_sh)
|
|
|
|
{
|
|
|
|
unsigned se, sh, cu;
|
|
|
|
const char *p;
|
|
|
|
|
|
|
|
memset(mask, 0, sizeof(*mask) * max_se * max_sh);
|
|
|
|
|
|
|
|
if (!amdgpu_disable_cu || !*amdgpu_disable_cu)
|
|
|
|
return;
|
|
|
|
|
|
|
|
p = amdgpu_disable_cu;
|
|
|
|
for (;;) {
|
|
|
|
char *next;
|
|
|
|
int ret = sscanf(p, "%u.%u.%u", &se, &sh, &cu);
|
|
|
|
if (ret < 3) {
|
|
|
|
DRM_ERROR("amdgpu: could not parse disable_cu\n");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (se < max_se && sh < max_sh && cu < 16) {
|
|
|
|
DRM_INFO("amdgpu: disabling CU %u.%u.%u\n", se, sh, cu);
|
|
|
|
mask[se * max_sh + sh] |= 1u << cu;
|
|
|
|
} else {
|
|
|
|
DRM_ERROR("amdgpu: disable_cu %u.%u.%u is out of range\n",
|
|
|
|
se, sh, cu);
|
|
|
|
}
|
|
|
|
|
|
|
|
next = strchr(p, ',');
|
|
|
|
if (!next)
|
|
|
|
break;
|
|
|
|
p = next + 1;
|
|
|
|
}
|
|
|
|
}
|
2017-06-07 23:05:26 +08:00
|
|
|
|
2017-09-27 00:22:45 +08:00
|
|
|
static bool amdgpu_gfx_is_multipipe_capable(struct amdgpu_device *adev)
|
|
|
|
{
|
2017-09-27 00:22:46 +08:00
|
|
|
if (amdgpu_compute_multipipe != -1) {
|
|
|
|
DRM_INFO("amdgpu: forcing compute pipe policy %d\n",
|
|
|
|
amdgpu_compute_multipipe);
|
|
|
|
return amdgpu_compute_multipipe == 1;
|
|
|
|
}
|
|
|
|
|
2017-09-27 00:22:45 +08:00
|
|
|
/* FIXME: spreading the queues across pipes causes perf regressions
|
|
|
|
* on POLARIS11 compute workloads */
|
|
|
|
if (adev->asic_type == CHIP_POLARIS11)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
return adev->gfx.mec.num_mec > 1;
|
|
|
|
}
|
|
|
|
|
2017-06-07 23:05:26 +08:00
|
|
|
void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev)
|
|
|
|
{
|
|
|
|
int i, queue, pipe, mec;
|
2017-09-27 00:22:45 +08:00
|
|
|
bool multipipe_policy = amdgpu_gfx_is_multipipe_capable(adev);
|
2017-06-07 23:05:26 +08:00
|
|
|
|
|
|
|
/* policy for amdgpu compute queue ownership */
|
|
|
|
for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {
|
|
|
|
queue = i % adev->gfx.mec.num_queue_per_pipe;
|
|
|
|
pipe = (i / adev->gfx.mec.num_queue_per_pipe)
|
|
|
|
% adev->gfx.mec.num_pipe_per_mec;
|
|
|
|
mec = (i / adev->gfx.mec.num_queue_per_pipe)
|
|
|
|
/ adev->gfx.mec.num_pipe_per_mec;
|
|
|
|
|
|
|
|
/* we've run out of HW */
|
|
|
|
if (mec >= adev->gfx.mec.num_mec)
|
|
|
|
break;
|
|
|
|
|
2017-09-27 00:22:45 +08:00
|
|
|
if (multipipe_policy) {
|
2017-06-07 23:05:26 +08:00
|
|
|
/* policy: amdgpu owns the first two queues of the first MEC */
|
|
|
|
if (mec == 0 && queue < 2)
|
|
|
|
set_bit(i, adev->gfx.mec.queue_bitmap);
|
|
|
|
} else {
|
|
|
|
/* policy: amdgpu owns all queues in the first pipe */
|
|
|
|
if (mec == 0 && pipe == 0)
|
|
|
|
set_bit(i, adev->gfx.mec.queue_bitmap);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* update the number of active compute rings */
|
|
|
|
adev->gfx.num_compute_rings =
|
|
|
|
bitmap_weight(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
|
|
|
|
|
|
|
|
/* If you hit this case and edited the policy, you probably just
|
|
|
|
* need to increase AMDGPU_MAX_COMPUTE_RINGS */
|
|
|
|
if (WARN_ON(adev->gfx.num_compute_rings > AMDGPU_MAX_COMPUTE_RINGS))
|
|
|
|
adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS;
|
|
|
|
}
|
2017-06-08 01:31:32 +08:00
|
|
|
|
2018-08-08 15:16:43 +08:00
|
|
|
void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev)
|
|
|
|
{
|
2019-09-29 20:38:43 +08:00
|
|
|
int i, queue, me;
|
2018-08-08 15:16:43 +08:00
|
|
|
|
|
|
|
for (i = 0; i < AMDGPU_MAX_GFX_QUEUES; ++i) {
|
|
|
|
queue = i % adev->gfx.me.num_queue_per_pipe;
|
|
|
|
me = (i / adev->gfx.me.num_queue_per_pipe)
|
|
|
|
/ adev->gfx.me.num_pipe_per_me;
|
|
|
|
|
|
|
|
if (me >= adev->gfx.me.num_me)
|
|
|
|
break;
|
|
|
|
/* policy: amdgpu owns the first queue per pipe at this stage
|
|
|
|
* will extend to mulitple queues per pipe later */
|
|
|
|
if (me == 0 && queue < 1)
|
|
|
|
set_bit(i, adev->gfx.me.queue_bitmap);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* update the number of active graphics rings */
|
|
|
|
adev->gfx.num_gfx_rings =
|
|
|
|
bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
|
|
|
|
}
|
|
|
|
|
2017-06-08 01:31:32 +08:00
|
|
|
static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev,
|
|
|
|
struct amdgpu_ring *ring)
|
|
|
|
{
|
|
|
|
int queue_bit;
|
|
|
|
int mec, pipe, queue;
|
|
|
|
|
|
|
|
queue_bit = adev->gfx.mec.num_mec
|
|
|
|
* adev->gfx.mec.num_pipe_per_mec
|
|
|
|
* adev->gfx.mec.num_queue_per_pipe;
|
|
|
|
|
|
|
|
while (queue_bit-- >= 0) {
|
|
|
|
if (test_bit(queue_bit, adev->gfx.mec.queue_bitmap))
|
|
|
|
continue;
|
|
|
|
|
2018-07-31 15:43:10 +08:00
|
|
|
amdgpu_gfx_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue);
|
2017-06-08 01:31:32 +08:00
|
|
|
|
2017-12-15 09:33:21 +08:00
|
|
|
/*
|
|
|
|
* 1. Using pipes 2/3 from MEC 2 seems cause problems.
|
|
|
|
* 2. It must use queue id 0, because CGPG_IDLE/SAVE/LOAD/RUN
|
|
|
|
* only can be issued on queue 0.
|
|
|
|
*/
|
|
|
|
if ((mec == 1 && pipe > 1) || queue != 0)
|
2017-06-08 01:31:32 +08:00
|
|
|
continue;
|
|
|
|
|
|
|
|
ring->me = mec + 1;
|
|
|
|
ring->pipe = pipe;
|
|
|
|
ring->queue = queue;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
dev_err(adev->dev, "Failed to find a queue for KIQ\n");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,
|
|
|
|
struct amdgpu_ring *ring,
|
|
|
|
struct amdgpu_irq_src *irq)
|
|
|
|
{
|
|
|
|
struct amdgpu_kiq *kiq = &adev->gfx.kiq;
|
|
|
|
int r = 0;
|
|
|
|
|
2017-10-13 15:38:35 +08:00
|
|
|
spin_lock_init(&kiq->ring_lock);
|
2017-06-08 01:31:32 +08:00
|
|
|
|
2020-01-15 14:32:00 +08:00
|
|
|
r = amdgpu_device_wb_get(adev, &kiq->reg_val_offs);
|
2017-06-08 01:31:32 +08:00
|
|
|
if (r)
|
|
|
|
return r;
|
|
|
|
|
|
|
|
ring->adev = NULL;
|
|
|
|
ring->ring_obj = NULL;
|
|
|
|
ring->use_doorbell = true;
|
2018-11-20 05:20:07 +08:00
|
|
|
ring->doorbell_index = adev->doorbell_index.kiq;
|
2017-06-08 01:31:32 +08:00
|
|
|
|
|
|
|
r = amdgpu_gfx_kiq_acquire(adev, ring);
|
|
|
|
if (r)
|
|
|
|
return r;
|
|
|
|
|
|
|
|
ring->eop_gpu_addr = kiq->eop_gpu_addr;
|
2017-06-12 21:05:04 +08:00
|
|
|
sprintf(ring->name, "kiq_%d.%d.%d", ring->me, ring->pipe, ring->queue);
|
2017-06-08 01:31:32 +08:00
|
|
|
r = amdgpu_ring_init(adev, ring, 1024,
|
|
|
|
irq, AMDGPU_CP_KIQ_IRQ_DRIVER0);
|
|
|
|
if (r)
|
|
|
|
dev_warn(adev->dev, "(%d) failed to init kiq ring\n", r);
|
|
|
|
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
2019-10-23 22:33:52 +08:00
|
|
|
void amdgpu_gfx_kiq_free_ring(struct amdgpu_ring *ring)
|
2017-06-08 01:31:32 +08:00
|
|
|
{
|
2020-01-15 14:32:00 +08:00
|
|
|
amdgpu_device_wb_free(ring->adev, ring->adev->gfx.kiq.reg_val_offs);
|
2017-06-08 01:31:32 +08:00
|
|
|
amdgpu_ring_fini(ring);
|
|
|
|
}
|
|
|
|
|
|
|
|
void amdgpu_gfx_kiq_fini(struct amdgpu_device *adev)
|
|
|
|
{
|
|
|
|
struct amdgpu_kiq *kiq = &adev->gfx.kiq;
|
|
|
|
|
|
|
|
amdgpu_bo_free_kernel(&kiq->eop_obj, &kiq->eop_gpu_addr, NULL);
|
|
|
|
}
|
|
|
|
|
|
|
|
int amdgpu_gfx_kiq_init(struct amdgpu_device *adev,
|
|
|
|
unsigned hpd_size)
|
|
|
|
{
|
|
|
|
int r;
|
|
|
|
u32 *hpd;
|
|
|
|
struct amdgpu_kiq *kiq = &adev->gfx.kiq;
|
|
|
|
|
|
|
|
r = amdgpu_bo_create_kernel(adev, hpd_size, PAGE_SIZE,
|
|
|
|
AMDGPU_GEM_DOMAIN_GTT, &kiq->eop_obj,
|
|
|
|
&kiq->eop_gpu_addr, (void **)&hpd);
|
|
|
|
if (r) {
|
|
|
|
dev_warn(adev->dev, "failed to create KIQ bo (%d).\n", r);
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
|
|
|
memset(hpd, 0, hpd_size);
|
|
|
|
|
|
|
|
r = amdgpu_bo_reserve(kiq->eop_obj, true);
|
|
|
|
if (unlikely(r != 0))
|
|
|
|
dev_warn(adev->dev, "(%d) reserve kiq eop bo failed\n", r);
|
|
|
|
amdgpu_bo_kunmap(kiq->eop_obj);
|
|
|
|
amdgpu_bo_unreserve(kiq->eop_obj);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
2017-06-08 03:27:52 +08:00
|
|
|
|
2018-08-01 12:03:20 +08:00
|
|
|
/* create MQD for each compute/gfx queue */
|
|
|
|
int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
|
|
|
|
unsigned mqd_size)
|
2017-06-08 03:27:52 +08:00
|
|
|
{
|
|
|
|
struct amdgpu_ring *ring = NULL;
|
|
|
|
int r, i;
|
|
|
|
|
|
|
|
/* create MQD for KIQ */
|
|
|
|
ring = &adev->gfx.kiq.ring;
|
|
|
|
if (!ring->mqd_obj) {
|
2017-09-21 15:10:06 +08:00
|
|
|
/* originaly the KIQ MQD is put in GTT domain, but for SRIOV VRAM domain is a must
|
|
|
|
* otherwise hypervisor trigger SAVE_VF fail after driver unloaded which mean MQD
|
|
|
|
* deallocated and gart_unbind, to strict diverage we decide to use VRAM domain for
|
|
|
|
* KIQ MQD no matter SRIOV or Bare-metal
|
|
|
|
*/
|
2017-06-08 03:27:52 +08:00
|
|
|
r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
|
2017-09-21 15:10:06 +08:00
|
|
|
AMDGPU_GEM_DOMAIN_VRAM, &ring->mqd_obj,
|
2017-06-08 03:27:52 +08:00
|
|
|
&ring->mqd_gpu_addr, &ring->mqd_ptr);
|
|
|
|
if (r) {
|
|
|
|
dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r);
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* prepare MQD backup */
|
|
|
|
adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS] = kmalloc(mqd_size, GFP_KERNEL);
|
|
|
|
if (!adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS])
|
|
|
|
dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
|
|
|
|
}
|
|
|
|
|
2019-03-28 16:43:16 +08:00
|
|
|
if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
|
2018-08-14 20:54:35 +08:00
|
|
|
/* create MQD for each KGQ */
|
|
|
|
for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
|
|
|
|
ring = &adev->gfx.gfx_ring[i];
|
|
|
|
if (!ring->mqd_obj) {
|
|
|
|
r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
|
|
|
|
AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
|
|
|
|
&ring->mqd_gpu_addr, &ring->mqd_ptr);
|
|
|
|
if (r) {
|
|
|
|
dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* prepare MQD backup */
|
|
|
|
adev->gfx.me.mqd_backup[i] = kmalloc(mqd_size, GFP_KERNEL);
|
|
|
|
if (!adev->gfx.me.mqd_backup[i])
|
|
|
|
dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-06-08 03:27:52 +08:00
|
|
|
/* create MQD for each KCQ */
|
|
|
|
for (i = 0; i < adev->gfx.num_compute_rings; i++) {
|
|
|
|
ring = &adev->gfx.compute_ring[i];
|
|
|
|
if (!ring->mqd_obj) {
|
|
|
|
r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
|
|
|
|
AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
|
|
|
|
&ring->mqd_gpu_addr, &ring->mqd_ptr);
|
|
|
|
if (r) {
|
2018-08-14 20:54:35 +08:00
|
|
|
dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
|
2017-06-08 03:27:52 +08:00
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* prepare MQD backup */
|
|
|
|
adev->gfx.mec.mqd_backup[i] = kmalloc(mqd_size, GFP_KERNEL);
|
|
|
|
if (!adev->gfx.mec.mqd_backup[i])
|
|
|
|
dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-08-01 12:03:20 +08:00
|
|
|
void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev)
|
2017-06-08 03:27:52 +08:00
|
|
|
{
|
|
|
|
struct amdgpu_ring *ring = NULL;
|
|
|
|
int i;
|
|
|
|
|
2019-03-28 16:43:16 +08:00
|
|
|
if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
|
2018-08-14 20:54:35 +08:00
|
|
|
for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
|
|
|
|
ring = &adev->gfx.gfx_ring[i];
|
|
|
|
kfree(adev->gfx.me.mqd_backup[i]);
|
|
|
|
amdgpu_bo_free_kernel(&ring->mqd_obj,
|
|
|
|
&ring->mqd_gpu_addr,
|
|
|
|
&ring->mqd_ptr);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-06-08 03:27:52 +08:00
|
|
|
for (i = 0; i < adev->gfx.num_compute_rings; i++) {
|
|
|
|
ring = &adev->gfx.compute_ring[i];
|
|
|
|
kfree(adev->gfx.mec.mqd_backup[i]);
|
|
|
|
amdgpu_bo_free_kernel(&ring->mqd_obj,
|
|
|
|
&ring->mqd_gpu_addr,
|
|
|
|
&ring->mqd_ptr);
|
|
|
|
}
|
|
|
|
|
|
|
|
ring = &adev->gfx.kiq.ring;
|
|
|
|
kfree(adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS]);
|
|
|
|
amdgpu_bo_free_kernel(&ring->mqd_obj,
|
|
|
|
&ring->mqd_gpu_addr,
|
|
|
|
&ring->mqd_ptr);
|
|
|
|
}
|
2018-07-30 16:59:09 +08:00
|
|
|
|
2018-08-22 13:45:25 +08:00
|
|
|
int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev)
|
|
|
|
{
|
|
|
|
struct amdgpu_kiq *kiq = &adev->gfx.kiq;
|
|
|
|
struct amdgpu_ring *kiq_ring = &kiq->ring;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *
|
|
|
|
adev->gfx.num_compute_rings))
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
for (i = 0; i < adev->gfx.num_compute_rings; i++)
|
2019-01-08 13:33:46 +08:00
|
|
|
kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.compute_ring[i],
|
|
|
|
RESET_QUEUES, 0, 0);
|
2018-08-22 13:45:25 +08:00
|
|
|
|
|
|
|
return amdgpu_ring_test_ring(kiq_ring);
|
|
|
|
}
|
|
|
|
|
2019-03-05 22:05:02 +08:00
|
|
|
int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev)
|
|
|
|
{
|
|
|
|
struct amdgpu_kiq *kiq = &adev->gfx.kiq;
|
|
|
|
struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
|
|
|
|
uint64_t queue_mask = 0;
|
|
|
|
int r, i;
|
|
|
|
|
|
|
|
if (!kiq->pmf || !kiq->pmf->kiq_map_queues || !kiq->pmf->kiq_set_resources)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {
|
|
|
|
if (!test_bit(i, adev->gfx.mec.queue_bitmap))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* This situation may be hit in the future if a new HW
|
|
|
|
* generation exposes more than 64 queues. If so, the
|
|
|
|
* definition of queue_mask needs updating */
|
|
|
|
if (WARN_ON(i > (sizeof(queue_mask)*8))) {
|
|
|
|
DRM_ERROR("Invalid KCQ enabled: %d\n", i);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
queue_mask |= (1ull << i);
|
|
|
|
}
|
|
|
|
|
|
|
|
DRM_INFO("kiq ring mec %d pipe %d q %d\n", kiq_ring->me, kiq_ring->pipe,
|
|
|
|
kiq_ring->queue);
|
|
|
|
|
|
|
|
r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size *
|
|
|
|
adev->gfx.num_compute_rings +
|
|
|
|
kiq->pmf->set_resources_size);
|
|
|
|
if (r) {
|
|
|
|
DRM_ERROR("Failed to lock KIQ (%d).\n", r);
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
|
|
|
kiq->pmf->kiq_set_resources(kiq_ring, queue_mask);
|
|
|
|
for (i = 0; i < adev->gfx.num_compute_rings; i++)
|
|
|
|
kiq->pmf->kiq_map_queues(kiq_ring, &adev->gfx.compute_ring[i]);
|
|
|
|
|
|
|
|
r = amdgpu_ring_test_helper(kiq_ring);
|
|
|
|
if (r)
|
|
|
|
DRM_ERROR("KCQ enable failed\n");
|
|
|
|
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
2018-07-30 16:59:09 +08:00
|
|
|
/* amdgpu_gfx_off_ctrl - Handle gfx off feature enable/disable
|
|
|
|
*
|
|
|
|
* @adev: amdgpu_device pointer
|
|
|
|
* @bool enable true: enable gfx off feature, false: disable gfx off feature
|
|
|
|
*
|
|
|
|
* 1. gfx off feature will be enabled by gfx ip after gfx cg gp enabled.
|
|
|
|
* 2. other client can send request to disable gfx off feature, the request should be honored.
|
|
|
|
* 3. other client can cancel their request of disable gfx off feature
|
|
|
|
* 4. other client should not send request to enable gfx off feature before disable gfx off feature.
|
|
|
|
*/
|
|
|
|
|
|
|
|
void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable)
|
|
|
|
{
|
2019-01-31 14:11:04 +08:00
|
|
|
if (!(adev->pm.pp_feature & PP_GFXOFF_MASK))
|
2018-07-30 16:59:09 +08:00
|
|
|
return;
|
|
|
|
|
|
|
|
mutex_lock(&adev->gfx.gfx_off_mutex);
|
|
|
|
|
|
|
|
if (!enable)
|
|
|
|
adev->gfx.gfx_off_req_count++;
|
|
|
|
else if (adev->gfx.gfx_off_req_count > 0)
|
|
|
|
adev->gfx.gfx_off_req_count--;
|
|
|
|
|
|
|
|
if (enable && !adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) {
|
2018-07-27 21:06:30 +08:00
|
|
|
schedule_delayed_work(&adev->gfx.gfx_off_delay_work, GFX_OFF_DELAY_ENABLE);
|
2018-07-30 16:59:09 +08:00
|
|
|
} else if (!enable && adev->gfx.gfx_off_state) {
|
|
|
|
if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false))
|
|
|
|
adev->gfx.gfx_off_state = false;
|
|
|
|
}
|
2018-07-27 21:06:30 +08:00
|
|
|
|
2018-07-30 16:59:09 +08:00
|
|
|
mutex_unlock(&adev->gfx.gfx_off_mutex);
|
|
|
|
}
|
2019-09-03 06:06:08 +08:00
|
|
|
|
2019-09-19 11:46:11 +08:00
|
|
|
int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev)
|
2019-09-03 06:06:08 +08:00
|
|
|
{
|
|
|
|
int r;
|
|
|
|
struct ras_fs_if fs_info = {
|
|
|
|
.sysfs_name = "gfx_err_count",
|
|
|
|
.debugfs_name = "gfx_err_inject",
|
|
|
|
};
|
2019-09-19 11:46:11 +08:00
|
|
|
struct ras_ih_if ih_info = {
|
|
|
|
.cb = amdgpu_gfx_process_ras_data_cb,
|
|
|
|
};
|
2019-09-03 06:06:08 +08:00
|
|
|
|
|
|
|
if (!adev->gfx.ras_if) {
|
|
|
|
adev->gfx.ras_if = kmalloc(sizeof(struct ras_common_if), GFP_KERNEL);
|
|
|
|
if (!adev->gfx.ras_if)
|
|
|
|
return -ENOMEM;
|
|
|
|
adev->gfx.ras_if->block = AMDGPU_RAS_BLOCK__GFX;
|
|
|
|
adev->gfx.ras_if->type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
|
|
|
|
adev->gfx.ras_if->sub_block_index = 0;
|
|
|
|
strcpy(adev->gfx.ras_if->name, "gfx");
|
|
|
|
}
|
2019-09-19 11:46:11 +08:00
|
|
|
fs_info.head = ih_info.head = *adev->gfx.ras_if;
|
2019-09-03 06:06:08 +08:00
|
|
|
|
|
|
|
r = amdgpu_ras_late_init(adev, adev->gfx.ras_if,
|
2019-09-19 11:46:11 +08:00
|
|
|
&fs_info, &ih_info);
|
2019-09-03 06:06:08 +08:00
|
|
|
if (r)
|
|
|
|
goto free;
|
|
|
|
|
|
|
|
if (amdgpu_ras_is_supported(adev, adev->gfx.ras_if->block)) {
|
|
|
|
r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0);
|
|
|
|
if (r)
|
|
|
|
goto late_fini;
|
|
|
|
} else {
|
|
|
|
/* free gfx ras_if if ras is not supported */
|
|
|
|
r = 0;
|
|
|
|
goto free;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
late_fini:
|
2019-09-19 11:46:11 +08:00
|
|
|
amdgpu_ras_late_fini(adev, adev->gfx.ras_if, &ih_info);
|
2019-09-03 06:06:08 +08:00
|
|
|
free:
|
|
|
|
kfree(adev->gfx.ras_if);
|
|
|
|
adev->gfx.ras_if = NULL;
|
|
|
|
return r;
|
|
|
|
}
|
2019-09-12 14:06:35 +08:00
|
|
|
|
2019-09-12 17:44:49 +08:00
|
|
|
void amdgpu_gfx_ras_fini(struct amdgpu_device *adev)
|
|
|
|
{
|
|
|
|
if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX) &&
|
|
|
|
adev->gfx.ras_if) {
|
|
|
|
struct ras_common_if *ras_if = adev->gfx.ras_if;
|
|
|
|
struct ras_ih_if ih_info = {
|
|
|
|
.head = *ras_if,
|
|
|
|
.cb = amdgpu_gfx_process_ras_data_cb,
|
|
|
|
};
|
|
|
|
|
|
|
|
amdgpu_ras_late_fini(adev, ras_if, &ih_info);
|
|
|
|
kfree(ras_if);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-09-12 14:06:35 +08:00
|
|
|
int amdgpu_gfx_process_ras_data_cb(struct amdgpu_device *adev,
|
|
|
|
void *err_data,
|
|
|
|
struct amdgpu_iv_entry *entry)
|
|
|
|
{
|
2019-09-23 19:10:19 +08:00
|
|
|
/* TODO ue will trigger an interrupt.
|
|
|
|
*
|
|
|
|
* When “Full RAS” is enabled, the per-IP interrupt sources should
|
|
|
|
* be disabled and the driver should only look for the aggregated
|
|
|
|
* interrupt via sync flood
|
|
|
|
*/
|
2019-09-12 14:06:35 +08:00
|
|
|
if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) {
|
|
|
|
kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
|
|
|
|
if (adev->gfx.funcs->query_ras_error_count)
|
|
|
|
adev->gfx.funcs->query_ras_error_count(adev, err_data);
|
2019-12-13 16:46:05 +08:00
|
|
|
amdgpu_ras_reset_gpu(adev);
|
2019-09-12 14:06:35 +08:00
|
|
|
}
|
|
|
|
return AMDGPU_RAS_SUCCESS;
|
|
|
|
}
|
|
|
|
|
|
|
|
int amdgpu_gfx_cp_ecc_error_irq(struct amdgpu_device *adev,
|
|
|
|
struct amdgpu_irq_src *source,
|
|
|
|
struct amdgpu_iv_entry *entry)
|
|
|
|
{
|
|
|
|
struct ras_common_if *ras_if = adev->gfx.ras_if;
|
|
|
|
struct ras_dispatch_if ih_data = {
|
|
|
|
.entry = entry,
|
|
|
|
};
|
|
|
|
|
|
|
|
if (!ras_if)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
ih_data.head = *ras_if;
|
|
|
|
|
|
|
|
DRM_ERROR("CP ECC ERROR IRQ\n");
|
|
|
|
amdgpu_ras_interrupt_dispatch(adev, &ih_data);
|
|
|
|
return 0;
|
|
|
|
}
|
2020-01-15 14:32:00 +08:00
|
|
|
|
|
|
|
uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)
|
|
|
|
{
|
|
|
|
signed long r, cnt = 0;
|
|
|
|
unsigned long flags;
|
|
|
|
uint32_t seq;
|
|
|
|
struct amdgpu_kiq *kiq = &adev->gfx.kiq;
|
|
|
|
struct amdgpu_ring *ring = &kiq->ring;
|
|
|
|
|
|
|
|
BUG_ON(!ring->funcs->emit_rreg);
|
|
|
|
|
|
|
|
spin_lock_irqsave(&kiq->ring_lock, flags);
|
|
|
|
amdgpu_ring_alloc(ring, 32);
|
|
|
|
amdgpu_ring_emit_rreg(ring, reg);
|
|
|
|
amdgpu_fence_emit_polling(ring, &seq);
|
|
|
|
amdgpu_ring_commit(ring);
|
|
|
|
spin_unlock_irqrestore(&kiq->ring_lock, flags);
|
|
|
|
|
|
|
|
r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
|
|
|
|
|
|
|
|
/* don't wait anymore for gpu reset case because this way may
|
|
|
|
* block gpu_recover() routine forever, e.g. this virt_kiq_rreg
|
|
|
|
* is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
|
|
|
|
* never return if we keep waiting in virt_kiq_rreg, which cause
|
|
|
|
* gpu_recover() hang there.
|
|
|
|
*
|
|
|
|
* also don't wait anymore for IRQ context
|
|
|
|
* */
|
|
|
|
if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
|
|
|
|
goto failed_kiq_read;
|
|
|
|
|
|
|
|
might_sleep();
|
|
|
|
while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
|
|
|
|
msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
|
|
|
|
r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (cnt > MAX_KIQ_REG_TRY)
|
|
|
|
goto failed_kiq_read;
|
|
|
|
|
|
|
|
return adev->wb.wb[kiq->reg_val_offs];
|
|
|
|
|
|
|
|
failed_kiq_read:
|
|
|
|
pr_err("failed to read reg:%x\n", reg);
|
|
|
|
return ~0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
|
|
|
|
{
|
|
|
|
signed long r, cnt = 0;
|
|
|
|
unsigned long flags;
|
|
|
|
uint32_t seq;
|
|
|
|
struct amdgpu_kiq *kiq = &adev->gfx.kiq;
|
|
|
|
struct amdgpu_ring *ring = &kiq->ring;
|
|
|
|
|
|
|
|
BUG_ON(!ring->funcs->emit_wreg);
|
|
|
|
|
|
|
|
spin_lock_irqsave(&kiq->ring_lock, flags);
|
|
|
|
amdgpu_ring_alloc(ring, 32);
|
|
|
|
amdgpu_ring_emit_wreg(ring, reg, v);
|
|
|
|
amdgpu_fence_emit_polling(ring, &seq);
|
|
|
|
amdgpu_ring_commit(ring);
|
|
|
|
spin_unlock_irqrestore(&kiq->ring_lock, flags);
|
|
|
|
|
|
|
|
r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
|
|
|
|
|
|
|
|
/* don't wait anymore for gpu reset case because this way may
|
|
|
|
* block gpu_recover() routine forever, e.g. this virt_kiq_rreg
|
|
|
|
* is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
|
|
|
|
* never return if we keep waiting in virt_kiq_rreg, which cause
|
|
|
|
* gpu_recover() hang there.
|
|
|
|
*
|
|
|
|
* also don't wait anymore for IRQ context
|
|
|
|
* */
|
|
|
|
if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
|
|
|
|
goto failed_kiq_write;
|
|
|
|
|
|
|
|
might_sleep();
|
|
|
|
while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
|
|
|
|
|
|
|
|
msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
|
|
|
|
r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (cnt > MAX_KIQ_REG_TRY)
|
|
|
|
goto failed_kiq_write;
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
failed_kiq_write:
|
|
|
|
pr_err("failed to write reg:%x\n", reg);
|
|
|
|
}
|