From cbe12e74ee4e29b6cb4e63fa284e80b73ad57926 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Wed, 9 Oct 2013 03:18:07 -0700 Subject: [PATCH 1/5] drm/ttm: Allow vm fault retries Make use of the FAULT_FLAG_ALLOW_RETRY flag to allow dropping the mmap_sem while waiting for bo idle. FAULT_FLAG_ALLOW_RETRY appears to be primarily designed for disk waits but should work just as fine for GPU waits.. Signed-off-by: Thomas Hellstrom Reviewed-by: Jakob Bornecrantz --- drivers/gpu/drm/ttm/ttm_bo_vm.c | 62 ++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 1006c15445e9..c03514b93f9c 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -41,6 +41,51 @@ #define TTM_BO_VM_NUM_PREFAULT 16 +static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, + struct vm_area_struct *vma, + struct vm_fault *vmf) +{ + struct ttm_bo_device *bdev = bo->bdev; + int ret = 0; + + spin_lock(&bdev->fence_lock); + if (likely(!test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags))) + goto out_unlock; + + /* + * Quick non-stalling check for idle. + */ + ret = ttm_bo_wait(bo, false, false, true); + if (likely(ret == 0)) + goto out_unlock; + + /* + * If possible, avoid waiting for GPU with mmap_sem + * held. + */ + if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { + ret = VM_FAULT_RETRY; + if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) + goto out_unlock; + + up_read(&vma->vm_mm->mmap_sem); + (void) ttm_bo_wait(bo, false, true, false); + goto out_unlock; + } + + /* + * Ordinary wait. + */ + ret = ttm_bo_wait(bo, false, true, false); + if (unlikely(ret != 0)) + ret = (ret != -ERESTARTSYS) ? VM_FAULT_SIGBUS : + VM_FAULT_NOPAGE; + +out_unlock: + spin_unlock(&bdev->fence_lock); + return ret; +} + static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) { struct ttm_buffer_object *bo = (struct ttm_buffer_object *) @@ -91,18 +136,11 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) * Wait for buffer data in transit, due to a pipelined * move. */ - - spin_lock(&bdev->fence_lock); - if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) { - ret = ttm_bo_wait(bo, false, true, false); - spin_unlock(&bdev->fence_lock); - if (unlikely(ret != 0)) { - retval = (ret != -ERESTARTSYS) ? - VM_FAULT_SIGBUS : VM_FAULT_NOPAGE; - goto out_unlock; - } - } else - spin_unlock(&bdev->fence_lock); + ret = ttm_bo_vm_fault_idle(bo, vma, vmf); + if (unlikely(ret != 0)) { + retval = ret; + goto out_unlock; + } ret = ttm_mem_io_lock(man, true); if (unlikely(ret != 0)) { From 15205fbcbe5abf52d6b064c5e8d2f901518d14a2 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Thu, 10 Oct 2013 11:09:03 -0700 Subject: [PATCH 2/5] drm/ttm: Make NO_EVICT bos available to shrinkers pending destruction NO_EVICT bos that are not idle when all references are dropped are put on the delayed destroy list. However, since they are not on LRU lists, they are not available to shrinkers at that point, and buffers on the delayed destroy list are not checked very often for idle. So when these buffers are put on the delayed destroy list, clear the NO_EVICT flag and put them on the right LRU list. This way they are immediately available for eviction or shrinkers and will not cause false OOMS. Signed-off-by: Thomas Hellstrom Reviewed-by: Jakob Bornecrantz --- drivers/gpu/drm/ttm/ttm_bo.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index f1a857ec1021..6c1a38f53066 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -429,8 +429,20 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) sync_obj = driver->sync_obj_ref(bo->sync_obj); spin_unlock(&bdev->fence_lock); - if (!ret) + if (!ret) { + + /* + * Make NO_EVICT bos immediately available to + * shrinkers, now that they are queued for + * destruction. + */ + if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) { + bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; + ttm_bo_add_to_lru(bo); + } + ww_mutex_unlock(&bo->resv->lock); + } kref_get(&bo->list_kref); list_add_tail(&bo->ddestroy, &bdev->ddestroy); From 9a0599ddeae012a771bba5e23393fc52d8a59d89 Mon Sep 17 00:00:00 2001 From: Jakob Bornecrantz Date: Wed, 30 Oct 2013 02:46:56 -0700 Subject: [PATCH 3/5] drm/ttm: Handle in-memory region copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the case where the ttm pointer may be NULL causing a NULL pointer dereference. Signed-off-by: Jakob Bornecrantz Signed-off-by: Thomas Hellström Cc: stable@vger.kernel.org --- drivers/gpu/drm/ttm/ttm_bo_util.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 7cc904d3a4d1..8369e35c0dce 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -348,7 +348,9 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, if (old_iomap == NULL && ttm == NULL) goto out2; - if (ttm->state == tt_unpopulated) { + /* TTM might be null for moves within the same region. + */ + if (ttm && ttm->state == tt_unpopulated) { ret = ttm->bdev->driver->ttm_tt_populate(ttm); if (ret) { /* if we fail here don't nuke the mm node From da95c788ef0c645378ffccb7060a0df1a33aee38 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Wed, 30 Oct 2013 03:29:50 -0700 Subject: [PATCH 4/5] drm/ttm: Fix ttm_bo_move_memcpy All error paths will want to keep the mm node, so handle this at the function exit. This fixes an ioremap failure error path. Also add some comments to make the function a bit easier to understand. Signed-off-by: Thomas Hellstrom Reviewed-by: Jakob Bornecrantz Cc: stable@vger.kernel.org --- drivers/gpu/drm/ttm/ttm_bo_util.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 8369e35c0dce..4834c463c38b 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -343,21 +343,25 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, if (ret) goto out; + /* + * Single TTM move. NOP. + */ if (old_iomap == NULL && new_iomap == NULL) goto out2; + + /* + * Move nonexistent data. NOP. + */ if (old_iomap == NULL && ttm == NULL) goto out2; - /* TTM might be null for moves within the same region. + /* + * TTM might be null for moves within the same region. */ if (ttm && ttm->state == tt_unpopulated) { ret = ttm->bdev->driver->ttm_tt_populate(ttm); - if (ret) { - /* if we fail here don't nuke the mm node - * as the bo still owns it */ - old_copy.mm_node = NULL; + if (ret) goto out1; - } } add = 0; @@ -383,11 +387,8 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, prot); } else ret = ttm_copy_io_page(new_iomap, old_iomap, page); - if (ret) { - /* failing here, means keep old copy as-is */ - old_copy.mm_node = NULL; + if (ret) goto out1; - } } mb(); out2: @@ -405,7 +406,12 @@ out1: ttm_mem_reg_iounmap(bdev, old_mem, new_iomap); out: ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap); - ttm_bo_mem_put(bo, &old_copy); + + /* + * On error, keep the mm node! + */ + if (!ret) + ttm_bo_mem_put(bo, &old_copy); return ret; } EXPORT_SYMBOL(ttm_bo_move_memcpy); From 59c8e66378fb78adbcd05f0d09783dde6fef282b Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Mon, 28 Oct 2013 02:02:19 -0700 Subject: [PATCH 5/5] drm/ttm: Fix memory type compatibility check Also check the busy placements before deciding to move a buffer object. Failing to do this may result in a completely unneccessary move within a single memory type. Signed-off-by: Thomas Hellstrom Reviewed-by: Jakob Bornecrantz Cc: stable@vger.kernel.org --- drivers/gpu/drm/ttm/ttm_bo.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 6c1a38f53066..8d5a646ebe6a 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -998,24 +998,32 @@ out_unlock: return ret; } -static int ttm_bo_mem_compat(struct ttm_placement *placement, - struct ttm_mem_reg *mem) +static bool ttm_bo_mem_compat(struct ttm_placement *placement, + struct ttm_mem_reg *mem, + uint32_t *new_flags) { int i; if (mem->mm_node && placement->lpfn != 0 && (mem->start < placement->fpfn || mem->start + mem->num_pages > placement->lpfn)) - return -1; + return false; for (i = 0; i < placement->num_placement; i++) { - if ((placement->placement[i] & mem->placement & - TTM_PL_MASK_CACHING) && - (placement->placement[i] & mem->placement & - TTM_PL_MASK_MEM)) - return i; + *new_flags = placement->placement[i]; + if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && + (*new_flags & mem->placement & TTM_PL_MASK_MEM)) + return true; } - return -1; + + for (i = 0; i < placement->num_busy_placement; i++) { + *new_flags = placement->busy_placement[i]; + if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && + (*new_flags & mem->placement & TTM_PL_MASK_MEM)) + return true; + } + + return false; } int ttm_bo_validate(struct ttm_buffer_object *bo, @@ -1024,6 +1032,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, bool no_wait_gpu) { int ret; + uint32_t new_flags; lockdep_assert_held(&bo->resv->lock.base); /* Check that range is valid */ @@ -1034,8 +1043,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, /* * Check whether we need to move buffer. */ - ret = ttm_bo_mem_compat(placement, &bo->mem); - if (ret < 0) { + if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { ret = ttm_bo_move_buffer(bo, placement, interruptible, no_wait_gpu); if (ret) @@ -1045,7 +1053,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, * Use the access and other non-mapping-related flag bits from * the compatible memory placement flags to the active flags */ - ttm_flag_masked(&bo->mem.placement, placement->placement[ret], + ttm_flag_masked(&bo->mem.placement, new_flags, ~TTM_PL_MASK_MEMTYPE); } /*