drm/amdgpu: add new unlocked flag for PTE updates
For HMM support we need the ability to invalidate PTEs from a MM callback where we can't lock the root PD. Add a new flag to better support this instead of assuming that all invalidation updates are unlocked. Signed-off-by: Christian König <christian.koenig@amd.com> Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
This commit is contained in:
parent
eaad0c3aa9
commit
9c466bcbda
|
@ -1446,7 +1446,7 @@ static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params,
|
|||
uint64_t incr, entry_end, pe_start;
|
||||
struct amdgpu_bo *pt;
|
||||
|
||||
if (flags & (AMDGPU_PTE_VALID | AMDGPU_PTE_PRT)) {
|
||||
if (!params->unlocked) {
|
||||
/* make sure that the page tables covering the
|
||||
* address range are actually allocated
|
||||
*/
|
||||
|
@ -1458,8 +1458,12 @@ static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params,
|
|||
|
||||
shift = amdgpu_vm_level_shift(adev, cursor.level);
|
||||
parent_shift = amdgpu_vm_level_shift(adev, cursor.level - 1);
|
||||
if (adev->asic_type < CHIP_VEGA10 &&
|
||||
(flags & AMDGPU_PTE_VALID)) {
|
||||
if (params->unlocked) {
|
||||
/* Unlocked updates are only allowed on the leaves */
|
||||
if (amdgpu_vm_pt_descendant(adev, &cursor))
|
||||
continue;
|
||||
} else if (adev->asic_type < CHIP_VEGA10 &&
|
||||
(flags & AMDGPU_PTE_VALID)) {
|
||||
/* No huge page support before GMC v9 */
|
||||
if (cursor.level != AMDGPU_VM_PTB) {
|
||||
if (!amdgpu_vm_pt_descendant(adev, &cursor))
|
||||
|
@ -1558,6 +1562,7 @@ static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params,
|
|||
* @adev: amdgpu_device pointer
|
||||
* @vm: requested vm
|
||||
* @immediate: immediate submission in a page fault
|
||||
* @unlocked: unlocked invalidation during MM callback
|
||||
* @resv: fences we need to sync to
|
||||
* @start: start of mapped range
|
||||
* @last: last mapped entry
|
||||
|
@ -1573,7 +1578,7 @@ static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params,
|
|||
*/
|
||||
static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
|
||||
struct amdgpu_vm *vm, bool immediate,
|
||||
struct dma_resv *resv,
|
||||
bool unlocked, struct dma_resv *resv,
|
||||
uint64_t start, uint64_t last,
|
||||
uint64_t flags, uint64_t addr,
|
||||
dma_addr_t *pages_addr,
|
||||
|
@ -1603,11 +1608,12 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
|
|||
goto error_unlock;
|
||||
}
|
||||
|
||||
if (flags & (AMDGPU_PTE_VALID | AMDGPU_PTE_PRT)) {
|
||||
struct amdgpu_bo *root = vm->root.base.bo;
|
||||
if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) {
|
||||
struct dma_fence *tmp = dma_fence_get_stub();
|
||||
|
||||
if (!dma_fence_is_signaled(vm->last_immediate))
|
||||
amdgpu_bo_fence(root, vm->last_immediate, true);
|
||||
amdgpu_bo_fence(vm->root.base.bo, vm->last_unlocked, true);
|
||||
swap(vm->last_unlocked, tmp);
|
||||
dma_fence_put(tmp);
|
||||
}
|
||||
|
||||
r = vm->update_funcs->prepare(¶ms, resv, sync_mode);
|
||||
|
@ -1721,7 +1727,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
|
|||
}
|
||||
|
||||
last = min((uint64_t)mapping->last, start + max_entries - 1);
|
||||
r = amdgpu_vm_bo_update_mapping(adev, vm, false, resv,
|
||||
r = amdgpu_vm_bo_update_mapping(adev, vm, false, false, resv,
|
||||
start, last, flags, addr,
|
||||
dma_addr, fence);
|
||||
if (r)
|
||||
|
@ -2018,7 +2024,7 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
|
|||
mapping->start < AMDGPU_GMC_HOLE_START)
|
||||
init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
|
||||
|
||||
r = amdgpu_vm_bo_update_mapping(adev, vm, false, resv,
|
||||
r = amdgpu_vm_bo_update_mapping(adev, vm, false, false, resv,
|
||||
mapping->start, mapping->last,
|
||||
init_pte_value, 0, NULL, &f);
|
||||
amdgpu_vm_free_mapping(adev, vm, mapping, f);
|
||||
|
@ -2582,7 +2588,7 @@ bool amdgpu_vm_evictable(struct amdgpu_bo *bo)
|
|||
return false;
|
||||
|
||||
/* Don't evict VM page tables while they are updated */
|
||||
if (!dma_fence_is_signaled(bo_base->vm->last_immediate)) {
|
||||
if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) {
|
||||
amdgpu_vm_eviction_unlock(bo_base->vm);
|
||||
return false;
|
||||
}
|
||||
|
@ -2759,7 +2765,7 @@ long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout)
|
|||
if (timeout <= 0)
|
||||
return timeout;
|
||||
|
||||
return dma_fence_wait_timeout(vm->last_immediate, true, timeout);
|
||||
return dma_fence_wait_timeout(vm->last_unlocked, true, timeout);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -2831,7 +2837,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
|
|||
else
|
||||
vm->update_funcs = &amdgpu_vm_sdma_funcs;
|
||||
vm->last_update = NULL;
|
||||
vm->last_immediate = dma_fence_get_stub();
|
||||
vm->last_unlocked = dma_fence_get_stub();
|
||||
|
||||
mutex_init(&vm->eviction_lock);
|
||||
vm->evicting = false;
|
||||
|
@ -2885,7 +2891,7 @@ error_free_root:
|
|||
vm->root.base.bo = NULL;
|
||||
|
||||
error_free_delayed:
|
||||
dma_fence_put(vm->last_immediate);
|
||||
dma_fence_put(vm->last_unlocked);
|
||||
drm_sched_entity_destroy(&vm->delayed);
|
||||
|
||||
error_free_immediate:
|
||||
|
@ -3086,8 +3092,8 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
|
|||
vm->pasid = 0;
|
||||
}
|
||||
|
||||
dma_fence_wait(vm->last_immediate, false);
|
||||
dma_fence_put(vm->last_immediate);
|
||||
dma_fence_wait(vm->last_unlocked, false);
|
||||
dma_fence_put(vm->last_unlocked);
|
||||
|
||||
list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
|
||||
if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
|
||||
|
@ -3337,8 +3343,8 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, unsigned int pasid,
|
|||
value = 0;
|
||||
}
|
||||
|
||||
r = amdgpu_vm_bo_update_mapping(adev, vm, true, NULL, addr, addr + 1,
|
||||
flags, value, NULL, NULL);
|
||||
r = amdgpu_vm_bo_update_mapping(adev, vm, true, false, NULL, addr,
|
||||
addr + 1, flags, value, NULL, NULL);
|
||||
if (r)
|
||||
goto error_unlock;
|
||||
|
||||
|
|
|
@ -210,6 +210,11 @@ struct amdgpu_vm_update_params {
|
|||
*/
|
||||
bool immediate;
|
||||
|
||||
/**
|
||||
* @unlocked: true if the root BO is not locked
|
||||
*/
|
||||
bool unlocked;
|
||||
|
||||
/**
|
||||
* @pages_addr:
|
||||
*
|
||||
|
@ -277,8 +282,8 @@ struct amdgpu_vm {
|
|||
struct drm_sched_entity immediate;
|
||||
struct drm_sched_entity delayed;
|
||||
|
||||
/* Last submission to the scheduler entities */
|
||||
struct dma_fence *last_immediate;
|
||||
/* Last unlocked submission to the scheduler entities */
|
||||
struct dma_fence *last_unlocked;
|
||||
|
||||
unsigned int pasid;
|
||||
/* dedicated to vm */
|
||||
|
|
|
@ -92,8 +92,8 @@ static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p,
|
|||
{
|
||||
struct amdgpu_ib *ib = p->job->ibs;
|
||||
struct drm_sched_entity *entity;
|
||||
struct dma_fence *f, *tmp;
|
||||
struct amdgpu_ring *ring;
|
||||
struct dma_fence *f;
|
||||
int r;
|
||||
|
||||
entity = p->immediate ? &p->vm->immediate : &p->vm->delayed;
|
||||
|
@ -106,13 +106,13 @@ static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p,
|
|||
if (r)
|
||||
goto error;
|
||||
|
||||
if (p->immediate) {
|
||||
tmp = dma_fence_get(f);
|
||||
swap(p->vm->last_immediate, f);
|
||||
if (p->unlocked) {
|
||||
struct dma_fence *tmp = dma_fence_get(f);
|
||||
|
||||
swap(p->vm->last_unlocked, f);
|
||||
dma_fence_put(tmp);
|
||||
} else {
|
||||
dma_resv_add_shared_fence(p->vm->root.base.bo->tbo.base.resv,
|
||||
f);
|
||||
amdgpu_bo_fence(p->vm->root.base.bo, f, true);
|
||||
}
|
||||
|
||||
if (fence && !p->immediate)
|
||||
|
|
Loading…
Reference in New Issue