drm/amdgpu: separate PASID mapping from VM flush v2
Stuffing the PASID mapping into the VM flush isn't flexible enough since the PASID mapping changes not as often as we need a VM flush. v2: add missing use of gmc_v7_0_emit_pasid_mapping Signed-off-by: Christian König <christian.koenig@amd.com> Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
This commit is contained in:
parent
ec47734a6d
commit
c633c00bf0
|
@ -1774,7 +1774,8 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
|
||||||
#define amdgpu_asic_flush_hdp(adev, r) (adev)->asic_funcs->flush_hdp((adev), (r))
|
#define amdgpu_asic_flush_hdp(adev, r) (adev)->asic_funcs->flush_hdp((adev), (r))
|
||||||
#define amdgpu_asic_invalidate_hdp(adev, r) (adev)->asic_funcs->invalidate_hdp((adev), (r))
|
#define amdgpu_asic_invalidate_hdp(adev, r) (adev)->asic_funcs->invalidate_hdp((adev), (r))
|
||||||
#define amdgpu_gmc_flush_gpu_tlb(adev, vmid) (adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid))
|
#define amdgpu_gmc_flush_gpu_tlb(adev, vmid) (adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid))
|
||||||
#define amdgpu_gmc_emit_flush_gpu_tlb(r, vmid, pasid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (pasid), (addr))
|
#define amdgpu_gmc_emit_flush_gpu_tlb(r, vmid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (addr))
|
||||||
|
#define amdgpu_gmc_emit_pasid_mapping(r, vmid, pasid) (r)->adev->gmc.gmc_funcs->emit_pasid_mapping((r), (vmid), (pasid))
|
||||||
#define amdgpu_gmc_set_pte_pde(adev, pt, idx, addr, flags) (adev)->gmc.gmc_funcs->set_pte_pde((adev), (pt), (idx), (addr), (flags))
|
#define amdgpu_gmc_set_pte_pde(adev, pt, idx, addr, flags) (adev)->gmc.gmc_funcs->set_pte_pde((adev), (pt), (idx), (addr), (flags))
|
||||||
#define amdgpu_gmc_get_vm_pde(adev, level, dst, flags) (adev)->gmc.gmc_funcs->get_vm_pde((adev), (level), (dst), (flags))
|
#define amdgpu_gmc_get_vm_pde(adev, level, dst, flags) (adev)->gmc.gmc_funcs->get_vm_pde((adev), (level), (dst), (flags))
|
||||||
#define amdgpu_gmc_get_pte_flags(adev, flags) (adev)->gmc.gmc_funcs->get_vm_pte_flags((adev),(flags))
|
#define amdgpu_gmc_get_pte_flags(adev, flags) (adev)->gmc.gmc_funcs->get_vm_pte_flags((adev),(flags))
|
||||||
|
@ -1789,7 +1790,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
|
||||||
#define amdgpu_ring_set_wptr(r) (r)->funcs->set_wptr((r))
|
#define amdgpu_ring_set_wptr(r) (r)->funcs->set_wptr((r))
|
||||||
#define amdgpu_ring_emit_ib(r, ib, vmid, c) (r)->funcs->emit_ib((r), (ib), (vmid), (c))
|
#define amdgpu_ring_emit_ib(r, ib, vmid, c) (r)->funcs->emit_ib((r), (ib), (vmid), (c))
|
||||||
#define amdgpu_ring_emit_pipeline_sync(r) (r)->funcs->emit_pipeline_sync((r))
|
#define amdgpu_ring_emit_pipeline_sync(r) (r)->funcs->emit_pipeline_sync((r))
|
||||||
#define amdgpu_ring_emit_vm_flush(r, vmid, pasid, addr) (r)->funcs->emit_vm_flush((r), (vmid), (pasid), (addr))
|
#define amdgpu_ring_emit_vm_flush(r, vmid, addr) (r)->funcs->emit_vm_flush((r), (vmid), (addr))
|
||||||
#define amdgpu_ring_emit_fence(r, addr, seq, flags) (r)->funcs->emit_fence((r), (addr), (seq), (flags))
|
#define amdgpu_ring_emit_fence(r, addr, seq, flags) (r)->funcs->emit_fence((r), (addr), (seq), (flags))
|
||||||
#define amdgpu_ring_emit_gds_switch(r, v, db, ds, wb, ws, ab, as) (r)->funcs->emit_gds_switch((r), (v), (db), (ds), (wb), (ws), (ab), (as))
|
#define amdgpu_ring_emit_gds_switch(r, v, db, ds, wb, ws, ab, as) (r)->funcs->emit_gds_switch((r), (v), (db), (ds), (wb), (ws), (ab), (as))
|
||||||
#define amdgpu_ring_emit_hdp_flush(r) (r)->funcs->emit_hdp_flush((r))
|
#define amdgpu_ring_emit_hdp_flush(r) (r)->funcs->emit_hdp_flush((r))
|
||||||
|
|
|
@ -54,7 +54,10 @@ struct amdgpu_gmc_funcs {
|
||||||
uint32_t vmid);
|
uint32_t vmid);
|
||||||
/* flush the vm tlb via ring */
|
/* flush the vm tlb via ring */
|
||||||
uint64_t (*emit_flush_gpu_tlb)(struct amdgpu_ring *ring, unsigned vmid,
|
uint64_t (*emit_flush_gpu_tlb)(struct amdgpu_ring *ring, unsigned vmid,
|
||||||
unsigned pasid, uint64_t pd_addr);
|
uint64_t pd_addr);
|
||||||
|
/* Change the VMID -> PASID mapping */
|
||||||
|
void (*emit_pasid_mapping)(struct amdgpu_ring *ring, unsigned vmid,
|
||||||
|
unsigned pasid);
|
||||||
/* write pte/pde updates using the cpu */
|
/* write pte/pde updates using the cpu */
|
||||||
int (*set_pte_pde)(struct amdgpu_device *adev,
|
int (*set_pte_pde)(struct amdgpu_device *adev,
|
||||||
void *cpu_pt_addr, /* cpu addr of page table */
|
void *cpu_pt_addr, /* cpu addr of page table */
|
||||||
|
|
|
@ -126,7 +126,7 @@ struct amdgpu_ring_funcs {
|
||||||
uint64_t seq, unsigned flags);
|
uint64_t seq, unsigned flags);
|
||||||
void (*emit_pipeline_sync)(struct amdgpu_ring *ring);
|
void (*emit_pipeline_sync)(struct amdgpu_ring *ring);
|
||||||
void (*emit_vm_flush)(struct amdgpu_ring *ring, unsigned vmid,
|
void (*emit_vm_flush)(struct amdgpu_ring *ring, unsigned vmid,
|
||||||
unsigned pasid, uint64_t pd_addr);
|
uint64_t pd_addr);
|
||||||
void (*emit_hdp_flush)(struct amdgpu_ring *ring);
|
void (*emit_hdp_flush)(struct amdgpu_ring *ring);
|
||||||
void (*emit_gds_switch)(struct amdgpu_ring *ring, uint32_t vmid,
|
void (*emit_gds_switch)(struct amdgpu_ring *ring, uint32_t vmid,
|
||||||
uint32_t gds_base, uint32_t gds_size,
|
uint32_t gds_base, uint32_t gds_size,
|
||||||
|
|
|
@ -612,8 +612,11 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_
|
||||||
struct dma_fence *fence;
|
struct dma_fence *fence;
|
||||||
|
|
||||||
trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
|
trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
|
||||||
amdgpu_ring_emit_vm_flush(ring, job->vmid, job->pasid,
|
amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);
|
||||||
job->vm_pd_addr);
|
if (adev->gmc.gmc_funcs->emit_pasid_mapping &&
|
||||||
|
ring->funcs->emit_wreg)
|
||||||
|
amdgpu_gmc_emit_pasid_mapping(ring, job->vmid,
|
||||||
|
job->pasid);
|
||||||
|
|
||||||
r = amdgpu_fence_emit(ring, &fence);
|
r = amdgpu_fence_emit(ring, &fence);
|
||||||
if (r)
|
if (r)
|
||||||
|
|
|
@ -873,13 +873,12 @@ static void cik_sdma_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
|
||||||
* using sDMA (CIK).
|
* using sDMA (CIK).
|
||||||
*/
|
*/
|
||||||
static void cik_sdma_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
static void cik_sdma_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
||||||
unsigned vmid, unsigned pasid,
|
unsigned vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) |
|
u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) |
|
||||||
SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */
|
SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */
|
||||||
|
|
||||||
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pasid, pd_addr);
|
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
|
||||||
|
|
||||||
amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
|
amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
|
||||||
amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2);
|
amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2);
|
||||||
|
|
|
@ -2326,12 +2326,11 @@ static void gfx_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
|
||||||
}
|
}
|
||||||
|
|
||||||
static void gfx_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
static void gfx_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
||||||
unsigned vmid, unsigned pasid,
|
unsigned vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
|
int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
|
||||||
|
|
||||||
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pasid, pd_addr);
|
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
|
||||||
|
|
||||||
/* wait for the invalidate to complete */
|
/* wait for the invalidate to complete */
|
||||||
amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
|
amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
|
||||||
|
|
|
@ -3219,12 +3219,11 @@ static void gfx_v7_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
|
||||||
* using the CP (CIK).
|
* using the CP (CIK).
|
||||||
*/
|
*/
|
||||||
static void gfx_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
static void gfx_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
||||||
unsigned vmid, unsigned pasid,
|
unsigned vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
|
int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
|
||||||
|
|
||||||
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pasid, pd_addr);
|
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
|
||||||
|
|
||||||
/* wait for the invalidate to complete */
|
/* wait for the invalidate to complete */
|
||||||
amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
|
amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
|
||||||
|
|
|
@ -6311,12 +6311,11 @@ static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
|
||||||
}
|
}
|
||||||
|
|
||||||
static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
||||||
unsigned vmid, unsigned pasid,
|
unsigned vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
|
int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
|
||||||
|
|
||||||
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pasid, pd_addr);
|
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
|
||||||
|
|
||||||
/* wait for the invalidate to complete */
|
/* wait for the invalidate to complete */
|
||||||
amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
|
amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
|
||||||
|
|
|
@ -3676,10 +3676,9 @@ static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
|
||||||
}
|
}
|
||||||
|
|
||||||
static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
||||||
unsigned vmid, unsigned pasid,
|
unsigned vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pasid, pd_addr);
|
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
|
||||||
|
|
||||||
/* compute doesn't have PFP */
|
/* compute doesn't have PFP */
|
||||||
if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
|
if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
|
||||||
|
|
|
@ -363,8 +363,7 @@ static void gmc_v6_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid)
|
||||||
}
|
}
|
||||||
|
|
||||||
static uint64_t gmc_v6_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
|
static uint64_t gmc_v6_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
|
||||||
unsigned vmid, unsigned pasid,
|
unsigned vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
uint32_t reg;
|
uint32_t reg;
|
||||||
|
|
||||||
|
|
|
@ -436,8 +436,7 @@ static void gmc_v7_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid)
|
||||||
}
|
}
|
||||||
|
|
||||||
static uint64_t gmc_v7_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
|
static uint64_t gmc_v7_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
|
||||||
unsigned vmid, unsigned pasid,
|
unsigned vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
uint32_t reg;
|
uint32_t reg;
|
||||||
|
|
||||||
|
@ -447,14 +446,18 @@ static uint64_t gmc_v7_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
|
||||||
reg = mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8;
|
reg = mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8;
|
||||||
amdgpu_ring_emit_wreg(ring, reg, pd_addr >> 12);
|
amdgpu_ring_emit_wreg(ring, reg, pd_addr >> 12);
|
||||||
|
|
||||||
amdgpu_ring_emit_wreg(ring, mmIH_VMID_0_LUT + vmid, pasid);
|
|
||||||
|
|
||||||
/* bits 0-15 are the VM contexts0-15 */
|
/* bits 0-15 are the VM contexts0-15 */
|
||||||
amdgpu_ring_emit_wreg(ring, mmVM_INVALIDATE_REQUEST, 1 << vmid);
|
amdgpu_ring_emit_wreg(ring, mmVM_INVALIDATE_REQUEST, 1 << vmid);
|
||||||
|
|
||||||
return pd_addr;
|
return pd_addr;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void gmc_v7_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
|
||||||
|
unsigned pasid)
|
||||||
|
{
|
||||||
|
amdgpu_ring_emit_wreg(ring, mmIH_VMID_0_LUT + vmid, pasid);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* gmc_v7_0_set_pte_pde - update the page tables using MMIO
|
* gmc_v7_0_set_pte_pde - update the page tables using MMIO
|
||||||
*
|
*
|
||||||
|
@ -1327,6 +1330,7 @@ static const struct amd_ip_funcs gmc_v7_0_ip_funcs = {
|
||||||
static const struct amdgpu_gmc_funcs gmc_v7_0_gmc_funcs = {
|
static const struct amdgpu_gmc_funcs gmc_v7_0_gmc_funcs = {
|
||||||
.flush_gpu_tlb = gmc_v7_0_flush_gpu_tlb,
|
.flush_gpu_tlb = gmc_v7_0_flush_gpu_tlb,
|
||||||
.emit_flush_gpu_tlb = gmc_v7_0_emit_flush_gpu_tlb,
|
.emit_flush_gpu_tlb = gmc_v7_0_emit_flush_gpu_tlb,
|
||||||
|
.emit_pasid_mapping = gmc_v7_0_emit_pasid_mapping,
|
||||||
.set_pte_pde = gmc_v7_0_set_pte_pde,
|
.set_pte_pde = gmc_v7_0_set_pte_pde,
|
||||||
.set_prt = gmc_v7_0_set_prt,
|
.set_prt = gmc_v7_0_set_prt,
|
||||||
.get_vm_pte_flags = gmc_v7_0_get_vm_pte_flags,
|
.get_vm_pte_flags = gmc_v7_0_get_vm_pte_flags,
|
||||||
|
|
|
@ -612,8 +612,7 @@ static void gmc_v8_0_flush_gpu_tlb(struct amdgpu_device *adev,
|
||||||
}
|
}
|
||||||
|
|
||||||
static uint64_t gmc_v8_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
|
static uint64_t gmc_v8_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
|
||||||
unsigned vmid, unsigned pasid,
|
unsigned vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
uint32_t reg;
|
uint32_t reg;
|
||||||
|
|
||||||
|
@ -623,14 +622,18 @@ static uint64_t gmc_v8_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
|
||||||
reg = mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8;
|
reg = mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8;
|
||||||
amdgpu_ring_emit_wreg(ring, reg, pd_addr >> 12);
|
amdgpu_ring_emit_wreg(ring, reg, pd_addr >> 12);
|
||||||
|
|
||||||
amdgpu_ring_emit_wreg(ring, mmIH_VMID_0_LUT + vmid, pasid);
|
|
||||||
|
|
||||||
/* bits 0-15 are the VM contexts0-15 */
|
/* bits 0-15 are the VM contexts0-15 */
|
||||||
amdgpu_ring_emit_wreg(ring, mmVM_INVALIDATE_REQUEST, 1 << vmid);
|
amdgpu_ring_emit_wreg(ring, mmVM_INVALIDATE_REQUEST, 1 << vmid);
|
||||||
|
|
||||||
return pd_addr;
|
return pd_addr;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void gmc_v8_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
|
||||||
|
unsigned pasid)
|
||||||
|
{
|
||||||
|
amdgpu_ring_emit_wreg(ring, mmIH_VMID_0_LUT + vmid, pasid);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* gmc_v8_0_set_pte_pde - update the page tables using MMIO
|
* gmc_v8_0_set_pte_pde - update the page tables using MMIO
|
||||||
*
|
*
|
||||||
|
@ -1662,6 +1665,7 @@ static const struct amd_ip_funcs gmc_v8_0_ip_funcs = {
|
||||||
static const struct amdgpu_gmc_funcs gmc_v8_0_gmc_funcs = {
|
static const struct amdgpu_gmc_funcs gmc_v8_0_gmc_funcs = {
|
||||||
.flush_gpu_tlb = gmc_v8_0_flush_gpu_tlb,
|
.flush_gpu_tlb = gmc_v8_0_flush_gpu_tlb,
|
||||||
.emit_flush_gpu_tlb = gmc_v8_0_emit_flush_gpu_tlb,
|
.emit_flush_gpu_tlb = gmc_v8_0_emit_flush_gpu_tlb,
|
||||||
|
.emit_pasid_mapping = gmc_v8_0_emit_pasid_mapping,
|
||||||
.set_pte_pde = gmc_v8_0_set_pte_pde,
|
.set_pte_pde = gmc_v8_0_set_pte_pde,
|
||||||
.set_prt = gmc_v8_0_set_prt,
|
.set_prt = gmc_v8_0_set_prt,
|
||||||
.get_vm_pte_flags = gmc_v8_0_get_vm_pte_flags,
|
.get_vm_pte_flags = gmc_v8_0_get_vm_pte_flags,
|
||||||
|
|
|
@ -368,17 +368,15 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
|
||||||
}
|
}
|
||||||
|
|
||||||
static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
|
static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
|
||||||
unsigned vmid, unsigned pasid,
|
unsigned vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
struct amdgpu_device *adev = ring->adev;
|
struct amdgpu_device *adev = ring->adev;
|
||||||
struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub];
|
struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub];
|
||||||
uint32_t req = gmc_v9_0_get_invalidate_req(vmid);
|
uint32_t req = gmc_v9_0_get_invalidate_req(vmid);
|
||||||
uint64_t flags = AMDGPU_PTE_VALID;
|
uint64_t flags = AMDGPU_PTE_VALID;
|
||||||
unsigned eng = ring->vm_inv_eng;
|
unsigned eng = ring->vm_inv_eng;
|
||||||
uint32_t reg;
|
|
||||||
|
|
||||||
amdgpu_gmc_get_vm_pde(ring->adev, -1, &pd_addr, &flags);
|
amdgpu_gmc_get_vm_pde(adev, -1, &pd_addr, &flags);
|
||||||
pd_addr |= flags;
|
pd_addr |= flags;
|
||||||
|
|
||||||
amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + (2 * vmid),
|
amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + (2 * vmid),
|
||||||
|
@ -387,13 +385,6 @@ static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
|
||||||
amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + (2 * vmid),
|
amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + (2 * vmid),
|
||||||
upper_32_bits(pd_addr));
|
upper_32_bits(pd_addr));
|
||||||
|
|
||||||
if (ring->funcs->vmhub == AMDGPU_GFXHUB)
|
|
||||||
reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
|
|
||||||
else
|
|
||||||
reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
|
|
||||||
|
|
||||||
amdgpu_ring_emit_wreg(ring, reg, pasid);
|
|
||||||
|
|
||||||
amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_req + eng, req);
|
amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_req + eng, req);
|
||||||
|
|
||||||
/* wait for the invalidate to complete */
|
/* wait for the invalidate to complete */
|
||||||
|
@ -403,6 +394,20 @@ static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
|
||||||
return pd_addr;
|
return pd_addr;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
|
||||||
|
unsigned pasid)
|
||||||
|
{
|
||||||
|
struct amdgpu_device *adev = ring->adev;
|
||||||
|
uint32_t reg;
|
||||||
|
|
||||||
|
if (ring->funcs->vmhub == AMDGPU_GFXHUB)
|
||||||
|
reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
|
||||||
|
else
|
||||||
|
reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
|
||||||
|
|
||||||
|
amdgpu_ring_emit_wreg(ring, reg, pasid);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* gmc_v9_0_set_pte_pde - update the page tables using MMIO
|
* gmc_v9_0_set_pte_pde - update the page tables using MMIO
|
||||||
*
|
*
|
||||||
|
@ -529,6 +534,7 @@ static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
|
||||||
static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
|
static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
|
||||||
.flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
|
.flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
|
||||||
.emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
|
.emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
|
||||||
|
.emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
|
||||||
.set_pte_pde = gmc_v9_0_set_pte_pde,
|
.set_pte_pde = gmc_v9_0_set_pte_pde,
|
||||||
.get_vm_pte_flags = gmc_v9_0_get_vm_pte_flags,
|
.get_vm_pte_flags = gmc_v9_0_get_vm_pte_flags,
|
||||||
.get_vm_pde = gmc_v9_0_get_vm_pde
|
.get_vm_pde = gmc_v9_0_get_vm_pde
|
||||||
|
|
|
@ -852,10 +852,9 @@ static void sdma_v2_4_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
|
||||||
* using sDMA (VI).
|
* using sDMA (VI).
|
||||||
*/
|
*/
|
||||||
static void sdma_v2_4_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
static void sdma_v2_4_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
||||||
unsigned vmid, unsigned pasid,
|
unsigned vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pasid, pd_addr);
|
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
|
||||||
|
|
||||||
/* wait for flush */
|
/* wait for flush */
|
||||||
amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
|
amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
|
||||||
|
|
|
@ -1117,10 +1117,9 @@ static void sdma_v3_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
|
||||||
* using sDMA (VI).
|
* using sDMA (VI).
|
||||||
*/
|
*/
|
||||||
static void sdma_v3_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
static void sdma_v3_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
||||||
unsigned vmid, unsigned pasid,
|
unsigned vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pasid, pd_addr);
|
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
|
||||||
|
|
||||||
/* wait for flush */
|
/* wait for flush */
|
||||||
amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
|
amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
|
||||||
|
|
|
@ -1123,10 +1123,9 @@ static void sdma_v4_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
|
||||||
* using sDMA (VEGA10).
|
* using sDMA (VEGA10).
|
||||||
*/
|
*/
|
||||||
static void sdma_v4_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
static void sdma_v4_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
||||||
unsigned vmid, unsigned pasid,
|
unsigned vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pasid, pd_addr);
|
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void sdma_v4_0_ring_emit_wreg(struct amdgpu_ring *ring,
|
static void sdma_v4_0_ring_emit_wreg(struct amdgpu_ring *ring,
|
||||||
|
|
|
@ -460,10 +460,9 @@ static void si_dma_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
|
||||||
* using sDMA (VI).
|
* using sDMA (VI).
|
||||||
*/
|
*/
|
||||||
static void si_dma_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
static void si_dma_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
||||||
unsigned vmid, unsigned pasid,
|
unsigned vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pasid, pd_addr);
|
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
|
||||||
|
|
||||||
/* wait for invalidate to complete */
|
/* wait for invalidate to complete */
|
||||||
amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_POLL_REG_MEM, 0, 0, 0, 0));
|
amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_POLL_REG_MEM, 0, 0, 0, 0));
|
||||||
|
|
|
@ -1058,10 +1058,9 @@ static void uvd_v6_0_ring_emit_wreg(struct amdgpu_ring *ring,
|
||||||
}
|
}
|
||||||
|
|
||||||
static void uvd_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
static void uvd_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
||||||
unsigned vmid, unsigned pasid,
|
unsigned vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pasid, pd_addr);
|
amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
|
||||||
|
|
||||||
amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0));
|
amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0));
|
||||||
amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2);
|
amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2);
|
||||||
|
@ -1107,8 +1106,7 @@ static void uvd_v6_0_enc_ring_insert_end(struct amdgpu_ring *ring)
|
||||||
}
|
}
|
||||||
|
|
||||||
static void uvd_v6_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
static void uvd_v6_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
||||||
unsigned int vmid, unsigned pasid,
|
unsigned int vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
amdgpu_ring_write(ring, HEVC_ENC_CMD_UPDATE_PTB);
|
amdgpu_ring_write(ring, HEVC_ENC_CMD_UPDATE_PTB);
|
||||||
amdgpu_ring_write(ring, vmid);
|
amdgpu_ring_write(ring, vmid);
|
||||||
|
|
|
@ -1261,13 +1261,12 @@ static void uvd_v7_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
|
||||||
}
|
}
|
||||||
|
|
||||||
static void uvd_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
static void uvd_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
||||||
unsigned vmid, unsigned pasid,
|
unsigned vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
|
struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
|
||||||
uint32_t data0, data1, mask;
|
uint32_t data0, data1, mask;
|
||||||
|
|
||||||
pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pasid, pd_addr);
|
pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
|
||||||
|
|
||||||
/* wait for reg writes */
|
/* wait for reg writes */
|
||||||
data0 = hub->ctx0_ptb_addr_lo32 + vmid * 2;
|
data0 = hub->ctx0_ptb_addr_lo32 + vmid * 2;
|
||||||
|
@ -1302,12 +1301,11 @@ static void uvd_v7_0_enc_ring_emit_reg_wait(struct amdgpu_ring *ring,
|
||||||
}
|
}
|
||||||
|
|
||||||
static void uvd_v7_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
static void uvd_v7_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
||||||
unsigned int vmid, unsigned pasid,
|
unsigned int vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
|
struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
|
||||||
|
|
||||||
pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pasid, pd_addr);
|
pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
|
||||||
|
|
||||||
/* wait for reg writes */
|
/* wait for reg writes */
|
||||||
uvd_v7_0_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + vmid * 2,
|
uvd_v7_0_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + vmid * 2,
|
||||||
|
|
|
@ -844,8 +844,7 @@ static void vce_v3_0_ring_emit_ib(struct amdgpu_ring *ring,
|
||||||
}
|
}
|
||||||
|
|
||||||
static void vce_v3_0_emit_vm_flush(struct amdgpu_ring *ring,
|
static void vce_v3_0_emit_vm_flush(struct amdgpu_ring *ring,
|
||||||
unsigned int vmid, unsigned pasid,
|
unsigned int vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
amdgpu_ring_write(ring, VCE_CMD_UPDATE_PTB);
|
amdgpu_ring_write(ring, VCE_CMD_UPDATE_PTB);
|
||||||
amdgpu_ring_write(ring, vmid);
|
amdgpu_ring_write(ring, vmid);
|
||||||
|
|
|
@ -975,12 +975,11 @@ static void vce_v4_0_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
|
||||||
}
|
}
|
||||||
|
|
||||||
static void vce_v4_0_emit_vm_flush(struct amdgpu_ring *ring,
|
static void vce_v4_0_emit_vm_flush(struct amdgpu_ring *ring,
|
||||||
unsigned int vmid, unsigned pasid,
|
unsigned int vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
|
struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
|
||||||
|
|
||||||
pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pasid, pd_addr);
|
pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
|
||||||
|
|
||||||
/* wait for reg writes */
|
/* wait for reg writes */
|
||||||
vce_v4_0_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + vmid * 2,
|
vce_v4_0_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + vmid * 2,
|
||||||
|
|
|
@ -859,13 +859,12 @@ static void vcn_v1_0_dec_ring_emit_reg_wait(struct amdgpu_ring *ring,
|
||||||
}
|
}
|
||||||
|
|
||||||
static void vcn_v1_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
static void vcn_v1_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
||||||
unsigned vmid, unsigned pasid,
|
unsigned vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
|
struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
|
||||||
uint32_t data0, data1, mask;
|
uint32_t data0, data1, mask;
|
||||||
|
|
||||||
pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pasid, pd_addr);
|
pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
|
||||||
|
|
||||||
/* wait for register write */
|
/* wait for register write */
|
||||||
data0 = hub->ctx0_ptb_addr_lo32 + vmid * 2;
|
data0 = hub->ctx0_ptb_addr_lo32 + vmid * 2;
|
||||||
|
@ -997,12 +996,11 @@ static void vcn_v1_0_enc_ring_emit_reg_wait(struct amdgpu_ring *ring,
|
||||||
}
|
}
|
||||||
|
|
||||||
static void vcn_v1_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
static void vcn_v1_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
|
||||||
unsigned int vmid, unsigned pasid,
|
unsigned int vmid, uint64_t pd_addr)
|
||||||
uint64_t pd_addr)
|
|
||||||
{
|
{
|
||||||
struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
|
struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
|
||||||
|
|
||||||
pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pasid, pd_addr);
|
pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
|
||||||
|
|
||||||
/* wait for reg writes */
|
/* wait for reg writes */
|
||||||
vcn_v1_0_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + vmid * 2,
|
vcn_v1_0_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + vmid * 2,
|
||||||
|
|
Loading…
Reference in New Issue