2016-07-20 16:21:08 +08:00
|
|
|
/*
|
|
|
|
* Copyright © 2008-2015 Intel Corporation
|
|
|
|
*
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
|
|
* to deal in the Software without restriction, including without limitation
|
|
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
|
|
*
|
|
|
|
* The above copyright notice and this permission notice (including the next
|
|
|
|
* paragraph) shall be included in all copies or substantial portions of the
|
|
|
|
* Software.
|
|
|
|
*
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
|
|
* IN THE SOFTWARE.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
drm/i915: Refactor activity tracking for requests
With the introduction of requests, we amplified the number of atomic
refcounted objects we use and update every execbuffer; from none to
several references, and a set of references that need to be changed. We
also introduced interesting side-effects in the order of retiring
requests and objects.
Instead of independently tracking the last request for an object, track
the active objects for each request. The object will reside in the
buffer list of its most recent active request and so we reduce the kref
interchange to a list_move. Now retirements are entirely driven by the
request, dramatically simplifying activity tracking on the object
themselves, and removing the ambiguity between retiring objects and
retiring requests.
Furthermore with the consolidation of managing the activity tracking
centrally, we can look forward to using RCU to enable lockless lookup of
the current active requests for an object. In the future, we will be
able to query the status or wait upon rendering to an object without
even touching the struct_mutex BKL.
All told, less code, simpler and faster, and more extensible.
v2: Add a typedef for the function pointer for convenience later.
v3: Make the noop retirement callback explicit. Allow passing NULL to
the init_request_active() which is expanded to a common noop function.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1470293567-10811-16-git-send-email-chris@chris-wilson.co.uk
2016-08-04 14:52:35 +08:00
|
|
|
#include <linux/prefetch.h>
|
2016-10-28 20:58:24 +08:00
|
|
|
#include <linux/dma-fence-array.h>
|
drm/i915: Refactor activity tracking for requests
With the introduction of requests, we amplified the number of atomic
refcounted objects we use and update every execbuffer; from none to
several references, and a set of references that need to be changed. We
also introduced interesting side-effects in the order of retiring
requests and objects.
Instead of independently tracking the last request for an object, track
the active objects for each request. The object will reside in the
buffer list of its most recent active request and so we reduce the kref
interchange to a list_move. Now retirements are entirely driven by the
request, dramatically simplifying activity tracking on the object
themselves, and removing the ambiguity between retiring objects and
retiring requests.
Furthermore with the consolidation of managing the activity tracking
centrally, we can look forward to using RCU to enable lockless lookup of
the current active requests for an object. In the future, we will be
able to query the status or wait upon rendering to an object without
even touching the struct_mutex BKL.
All told, less code, simpler and faster, and more extensible.
v2: Add a typedef for the function pointer for convenience later.
v3: Make the noop retirement callback explicit. Allow passing NULL to
the init_request_active() which is expanded to a common noop function.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1470293567-10811-16-git-send-email-chris@chris-wilson.co.uk
2016-08-04 14:52:35 +08:00
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
#include "i915_drv.h"
|
|
|
|
|
2016-10-25 20:00:45 +08:00
|
|
|
static const char *i915_fence_get_driver_name(struct dma_fence *fence)
|
2016-07-20 16:21:11 +08:00
|
|
|
{
|
|
|
|
return "i915";
|
|
|
|
}
|
|
|
|
|
2016-10-25 20:00:45 +08:00
|
|
|
static const char *i915_fence_get_timeline_name(struct dma_fence *fence)
|
2016-07-20 16:21:11 +08:00
|
|
|
{
|
2016-10-28 20:58:46 +08:00
|
|
|
return to_request(fence)->timeline->common->name;
|
2016-07-20 16:21:11 +08:00
|
|
|
}
|
|
|
|
|
2016-10-25 20:00:45 +08:00
|
|
|
static bool i915_fence_signaled(struct dma_fence *fence)
|
2016-07-20 16:21:11 +08:00
|
|
|
{
|
|
|
|
return i915_gem_request_completed(to_request(fence));
|
|
|
|
}
|
|
|
|
|
2016-10-25 20:00:45 +08:00
|
|
|
static bool i915_fence_enable_signaling(struct dma_fence *fence)
|
2016-07-20 16:21:11 +08:00
|
|
|
{
|
|
|
|
if (i915_fence_signaled(fence))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
intel_engine_enable_signaling(to_request(fence));
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2016-10-25 20:00:45 +08:00
|
|
|
static signed long i915_fence_wait(struct dma_fence *fence,
|
2016-07-20 16:21:11 +08:00
|
|
|
bool interruptible,
|
2016-10-28 20:58:27 +08:00
|
|
|
signed long timeout)
|
2016-07-20 16:21:11 +08:00
|
|
|
{
|
2016-10-28 20:58:27 +08:00
|
|
|
return i915_wait_request(to_request(fence), interruptible, timeout);
|
2016-07-20 16:21:11 +08:00
|
|
|
}
|
|
|
|
|
2016-10-25 20:00:45 +08:00
|
|
|
static void i915_fence_release(struct dma_fence *fence)
|
2016-07-20 16:21:11 +08:00
|
|
|
{
|
|
|
|
struct drm_i915_gem_request *req = to_request(fence);
|
|
|
|
|
2016-11-25 21:17:18 +08:00
|
|
|
/* The request is put onto a RCU freelist (i.e. the address
|
|
|
|
* is immediately reused), mark the fences as being freed now.
|
|
|
|
* Otherwise the debugobjects for the fences are only marked as
|
|
|
|
* freed when the slab cache itself is freed, and so we would get
|
|
|
|
* caught trying to reuse dead objects.
|
|
|
|
*/
|
|
|
|
i915_sw_fence_fini(&req->submit);
|
|
|
|
i915_sw_fence_fini(&req->execute);
|
|
|
|
|
2016-07-20 16:21:11 +08:00
|
|
|
kmem_cache_free(req->i915->requests, req);
|
|
|
|
}
|
|
|
|
|
2016-10-25 20:00:45 +08:00
|
|
|
const struct dma_fence_ops i915_fence_ops = {
|
2016-07-20 16:21:11 +08:00
|
|
|
.get_driver_name = i915_fence_get_driver_name,
|
|
|
|
.get_timeline_name = i915_fence_get_timeline_name,
|
|
|
|
.enable_signaling = i915_fence_enable_signaling,
|
|
|
|
.signaled = i915_fence_signaled,
|
|
|
|
.wait = i915_fence_wait,
|
|
|
|
.release = i915_fence_release,
|
|
|
|
};
|
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
|
|
|
|
struct drm_file *file)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_private;
|
|
|
|
struct drm_i915_file_private *file_priv;
|
|
|
|
|
|
|
|
WARN_ON(!req || !file || req->file_priv);
|
|
|
|
|
|
|
|
if (!req || !file)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (req->file_priv)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
dev_private = req->i915;
|
|
|
|
file_priv = file->driver_priv;
|
|
|
|
|
|
|
|
spin_lock(&file_priv->mm.lock);
|
|
|
|
req->file_priv = file_priv;
|
|
|
|
list_add_tail(&req->client_list, &file_priv->mm.request_list);
|
|
|
|
spin_unlock(&file_priv->mm.lock);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
|
|
|
|
{
|
|
|
|
struct drm_i915_file_private *file_priv = request->file_priv;
|
|
|
|
|
|
|
|
if (!file_priv)
|
|
|
|
return;
|
|
|
|
|
|
|
|
spin_lock(&file_priv->mm.lock);
|
|
|
|
list_del(&request->client_list);
|
|
|
|
request->file_priv = NULL;
|
|
|
|
spin_unlock(&file_priv->mm.lock);
|
|
|
|
}
|
|
|
|
|
2016-11-15 04:41:02 +08:00
|
|
|
static struct i915_dependency *
|
|
|
|
i915_dependency_alloc(struct drm_i915_private *i915)
|
|
|
|
{
|
|
|
|
return kmem_cache_alloc(i915->dependencies, GFP_KERNEL);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
i915_dependency_free(struct drm_i915_private *i915,
|
|
|
|
struct i915_dependency *dep)
|
|
|
|
{
|
|
|
|
kmem_cache_free(i915->dependencies, dep);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
__i915_priotree_add_dependency(struct i915_priotree *pt,
|
|
|
|
struct i915_priotree *signal,
|
|
|
|
struct i915_dependency *dep,
|
|
|
|
unsigned long flags)
|
|
|
|
{
|
2016-11-15 04:41:03 +08:00
|
|
|
INIT_LIST_HEAD(&dep->dfs_link);
|
2016-11-15 04:41:02 +08:00
|
|
|
list_add(&dep->wait_link, &signal->waiters_list);
|
|
|
|
list_add(&dep->signal_link, &pt->signalers_list);
|
|
|
|
dep->signaler = signal;
|
|
|
|
dep->flags = flags;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
i915_priotree_add_dependency(struct drm_i915_private *i915,
|
|
|
|
struct i915_priotree *pt,
|
|
|
|
struct i915_priotree *signal)
|
|
|
|
{
|
|
|
|
struct i915_dependency *dep;
|
|
|
|
|
|
|
|
dep = i915_dependency_alloc(i915);
|
|
|
|
if (!dep)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
__i915_priotree_add_dependency(pt, signal, dep, I915_DEPENDENCY_ALLOC);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
i915_priotree_fini(struct drm_i915_private *i915, struct i915_priotree *pt)
|
|
|
|
{
|
|
|
|
struct i915_dependency *dep, *next;
|
|
|
|
|
2016-11-15 04:41:03 +08:00
|
|
|
GEM_BUG_ON(!RB_EMPTY_NODE(&pt->node));
|
|
|
|
|
2016-11-15 04:41:02 +08:00
|
|
|
/* Everyone we depended upon (the fences we wait to be signaled)
|
|
|
|
* should retire before us and remove themselves from our list.
|
|
|
|
* However, retirement is run independently on each timeline and
|
|
|
|
* so we may be called out-of-order.
|
|
|
|
*/
|
|
|
|
list_for_each_entry_safe(dep, next, &pt->signalers_list, signal_link) {
|
|
|
|
list_del(&dep->wait_link);
|
|
|
|
if (dep->flags & I915_DEPENDENCY_ALLOC)
|
|
|
|
i915_dependency_free(i915, dep);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Remove ourselves from everyone who depends upon us */
|
|
|
|
list_for_each_entry_safe(dep, next, &pt->waiters_list, wait_link) {
|
|
|
|
list_del(&dep->signal_link);
|
|
|
|
if (dep->flags & I915_DEPENDENCY_ALLOC)
|
|
|
|
i915_dependency_free(i915, dep);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
i915_priotree_init(struct i915_priotree *pt)
|
|
|
|
{
|
|
|
|
INIT_LIST_HEAD(&pt->signalers_list);
|
|
|
|
INIT_LIST_HEAD(&pt->waiters_list);
|
2016-11-15 04:41:03 +08:00
|
|
|
RB_CLEAR_NODE(&pt->node);
|
|
|
|
pt->priority = INT_MIN;
|
2016-11-15 04:41:02 +08:00
|
|
|
}
|
|
|
|
|
2017-02-23 15:44:09 +08:00
|
|
|
static int reset_all_global_seqno(struct drm_i915_private *i915, u32 seqno)
|
|
|
|
{
|
|
|
|
struct i915_gem_timeline *timeline = &i915->gt.global_timeline;
|
|
|
|
struct intel_engine_cs *engine;
|
|
|
|
enum intel_engine_id id;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
/* Carefully retire all requests without writing to the rings */
|
|
|
|
ret = i915_gem_wait_for_idle(i915,
|
|
|
|
I915_WAIT_INTERRUPTIBLE |
|
|
|
|
I915_WAIT_LOCKED);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
i915_gem_retire_requests(i915);
|
|
|
|
GEM_BUG_ON(i915->gt.active_requests > 1);
|
|
|
|
|
|
|
|
/* If the seqno wraps around, we need to clear the breadcrumb rbtree */
|
|
|
|
for_each_engine(engine, i915, id) {
|
|
|
|
struct intel_timeline *tl = &timeline->engine[id];
|
|
|
|
|
|
|
|
if (!i915_seqno_passed(seqno, tl->seqno)) {
|
|
|
|
/* spin until threads are complete */
|
|
|
|
while (intel_breadcrumbs_busy(engine))
|
|
|
|
cond_resched();
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Finally reset hw state */
|
|
|
|
tl->seqno = seqno;
|
|
|
|
intel_engine_init_global_seqno(engine, seqno);
|
|
|
|
}
|
|
|
|
|
|
|
|
list_for_each_entry(timeline, &i915->gt.timelines, link) {
|
|
|
|
for_each_engine(engine, i915, id) {
|
|
|
|
struct intel_timeline *tl = &timeline->engine[id];
|
|
|
|
|
|
|
|
memset(tl->sync_seqno, 0, sizeof(tl->sync_seqno));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = to_i915(dev);
|
|
|
|
|
|
|
|
lockdep_assert_held(&dev_priv->drm.struct_mutex);
|
|
|
|
|
|
|
|
if (seqno == 0)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/* HWS page needs to be set less than what we
|
|
|
|
* will inject to ring
|
|
|
|
*/
|
|
|
|
return reset_all_global_seqno(dev_priv, seqno - 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int reserve_seqno(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
u32 active = ++engine->timeline->inflight_seqnos;
|
|
|
|
u32 seqno = engine->timeline->seqno;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
/* Reservation is fine until we need to wrap around */
|
|
|
|
if (likely(!add_overflows(seqno, active)))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
ret = reset_all_global_seqno(engine->i915, 0);
|
|
|
|
if (ret) {
|
|
|
|
engine->timeline->inflight_seqnos--;
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-02-23 15:44:08 +08:00
|
|
|
static void unreserve_seqno(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
GEM_BUG_ON(!engine->timeline->inflight_seqnos);
|
|
|
|
engine->timeline->inflight_seqnos--;
|
|
|
|
}
|
|
|
|
|
drm/i915: Refactor activity tracking for requests
With the introduction of requests, we amplified the number of atomic
refcounted objects we use and update every execbuffer; from none to
several references, and a set of references that need to be changed. We
also introduced interesting side-effects in the order of retiring
requests and objects.
Instead of independently tracking the last request for an object, track
the active objects for each request. The object will reside in the
buffer list of its most recent active request and so we reduce the kref
interchange to a list_move. Now retirements are entirely driven by the
request, dramatically simplifying activity tracking on the object
themselves, and removing the ambiguity between retiring objects and
retiring requests.
Furthermore with the consolidation of managing the activity tracking
centrally, we can look forward to using RCU to enable lockless lookup of
the current active requests for an object. In the future, we will be
able to query the status or wait upon rendering to an object without
even touching the struct_mutex BKL.
All told, less code, simpler and faster, and more extensible.
v2: Add a typedef for the function pointer for convenience later.
v3: Make the noop retirement callback explicit. Allow passing NULL to
the init_request_active() which is expanded to a common noop function.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1470293567-10811-16-git-send-email-chris@chris-wilson.co.uk
2016-08-04 14:52:35 +08:00
|
|
|
void i915_gem_retire_noop(struct i915_gem_active *active,
|
|
|
|
struct drm_i915_gem_request *request)
|
|
|
|
{
|
|
|
|
/* Space left intentionally blank */
|
|
|
|
}
|
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
static void i915_gem_request_retire(struct drm_i915_gem_request *request)
|
|
|
|
{
|
drm/i915: Unify active context tracking between legacy/execlists/guc
The requests conversion introduced a nasty bug where we could generate a
new request in the middle of constructing a request if we needed to idle
the system in order to evict space for a context. The request to idle
would be executed (and waited upon) before the current one, creating a
minor havoc in the seqno accounting, as we will consider the current
request to already be completed (prior to deferred seqno assignment) but
ring->last_retired_head would have been updated and still could allow
us to overwrite the current request before execution.
We also employed two different mechanisms to track the active context
until it was switched out. The legacy method allowed for waiting upon an
active context (it could forcibly evict any vma, including context's),
but the execlists method took a step backwards by pinning the vma for
the entire active lifespan of the context (the only way to evict was to
idle the entire GPU, not individual contexts). However, to circumvent
the tricky issue of locking (i.e. we cannot take struct_mutex at the
time of i915_gem_request_submit(), where we would want to move the
previous context onto the active tracker and unpin it), we take the
execlists approach and keep the contexts pinned until retirement.
The benefit of the execlists approach, more important for execlists than
legacy, was the reduction in work in pinning the context for each
request - as the context was kept pinned until idle, it could short
circuit the pinning for all active contexts.
We introduce new engine vfuncs to pin and unpin the context
respectively. The context is pinned at the start of the request, and
only unpinned when the following request is retired (this ensures that
the context is idle and coherent in main memory before we unpin it). We
move the engine->last_context tracking into the retirement itself
(rather than during request submission) in order to allow the submission
to be reordered or unwound without undue difficultly.
And finally an ulterior motive for unifying context handling was to
prepare for mock requests.
v2: Rename to last_retired_context, split out legacy_context tracking
for MI_SET_CONTEXT.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161218153724.8439-3-chris@chris-wilson.co.uk
2016-12-18 23:37:20 +08:00
|
|
|
struct intel_engine_cs *engine = request->engine;
|
drm/i915: Refactor activity tracking for requests
With the introduction of requests, we amplified the number of atomic
refcounted objects we use and update every execbuffer; from none to
several references, and a set of references that need to be changed. We
also introduced interesting side-effects in the order of retiring
requests and objects.
Instead of independently tracking the last request for an object, track
the active objects for each request. The object will reside in the
buffer list of its most recent active request and so we reduce the kref
interchange to a list_move. Now retirements are entirely driven by the
request, dramatically simplifying activity tracking on the object
themselves, and removing the ambiguity between retiring objects and
retiring requests.
Furthermore with the consolidation of managing the activity tracking
centrally, we can look forward to using RCU to enable lockless lookup of
the current active requests for an object. In the future, we will be
able to query the status or wait upon rendering to an object without
even touching the struct_mutex BKL.
All told, less code, simpler and faster, and more extensible.
v2: Add a typedef for the function pointer for convenience later.
v3: Make the noop retirement callback explicit. Allow passing NULL to
the init_request_active() which is expanded to a common noop function.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1470293567-10811-16-git-send-email-chris@chris-wilson.co.uk
2016-08-04 14:52:35 +08:00
|
|
|
struct i915_gem_active *active, *next;
|
|
|
|
|
2016-10-28 20:58:32 +08:00
|
|
|
lockdep_assert_held(&request->i915->drm.struct_mutex);
|
2016-11-25 21:17:17 +08:00
|
|
|
GEM_BUG_ON(!i915_sw_fence_signaled(&request->submit));
|
|
|
|
GEM_BUG_ON(!i915_sw_fence_signaled(&request->execute));
|
2016-10-28 20:58:32 +08:00
|
|
|
GEM_BUG_ON(!i915_gem_request_completed(request));
|
2016-11-16 00:46:20 +08:00
|
|
|
GEM_BUG_ON(!request->i915->gt.active_requests);
|
2016-10-28 20:58:32 +08:00
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
trace_i915_gem_request_retire(request);
|
2016-10-28 20:58:58 +08:00
|
|
|
|
drm/i915: Unify active context tracking between legacy/execlists/guc
The requests conversion introduced a nasty bug where we could generate a
new request in the middle of constructing a request if we needed to idle
the system in order to evict space for a context. The request to idle
would be executed (and waited upon) before the current one, creating a
minor havoc in the seqno accounting, as we will consider the current
request to already be completed (prior to deferred seqno assignment) but
ring->last_retired_head would have been updated and still could allow
us to overwrite the current request before execution.
We also employed two different mechanisms to track the active context
until it was switched out. The legacy method allowed for waiting upon an
active context (it could forcibly evict any vma, including context's),
but the execlists method took a step backwards by pinning the vma for
the entire active lifespan of the context (the only way to evict was to
idle the entire GPU, not individual contexts). However, to circumvent
the tricky issue of locking (i.e. we cannot take struct_mutex at the
time of i915_gem_request_submit(), where we would want to move the
previous context onto the active tracker and unpin it), we take the
execlists approach and keep the contexts pinned until retirement.
The benefit of the execlists approach, more important for execlists than
legacy, was the reduction in work in pinning the context for each
request - as the context was kept pinned until idle, it could short
circuit the pinning for all active contexts.
We introduce new engine vfuncs to pin and unpin the context
respectively. The context is pinned at the start of the request, and
only unpinned when the following request is retired (this ensures that
the context is idle and coherent in main memory before we unpin it). We
move the engine->last_context tracking into the retirement itself
(rather than during request submission) in order to allow the submission
to be reordered or unwound without undue difficultly.
And finally an ulterior motive for unifying context handling was to
prepare for mock requests.
v2: Rename to last_retired_context, split out legacy_context tracking
for MI_SET_CONTEXT.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161218153724.8439-3-chris@chris-wilson.co.uk
2016-12-18 23:37:20 +08:00
|
|
|
spin_lock_irq(&engine->timeline->lock);
|
2016-10-28 20:58:27 +08:00
|
|
|
list_del_init(&request->link);
|
drm/i915: Unify active context tracking between legacy/execlists/guc
The requests conversion introduced a nasty bug where we could generate a
new request in the middle of constructing a request if we needed to idle
the system in order to evict space for a context. The request to idle
would be executed (and waited upon) before the current one, creating a
minor havoc in the seqno accounting, as we will consider the current
request to already be completed (prior to deferred seqno assignment) but
ring->last_retired_head would have been updated and still could allow
us to overwrite the current request before execution.
We also employed two different mechanisms to track the active context
until it was switched out. The legacy method allowed for waiting upon an
active context (it could forcibly evict any vma, including context's),
but the execlists method took a step backwards by pinning the vma for
the entire active lifespan of the context (the only way to evict was to
idle the entire GPU, not individual contexts). However, to circumvent
the tricky issue of locking (i.e. we cannot take struct_mutex at the
time of i915_gem_request_submit(), where we would want to move the
previous context onto the active tracker and unpin it), we take the
execlists approach and keep the contexts pinned until retirement.
The benefit of the execlists approach, more important for execlists than
legacy, was the reduction in work in pinning the context for each
request - as the context was kept pinned until idle, it could short
circuit the pinning for all active contexts.
We introduce new engine vfuncs to pin and unpin the context
respectively. The context is pinned at the start of the request, and
only unpinned when the following request is retired (this ensures that
the context is idle and coherent in main memory before we unpin it). We
move the engine->last_context tracking into the retirement itself
(rather than during request submission) in order to allow the submission
to be reordered or unwound without undue difficultly.
And finally an ulterior motive for unifying context handling was to
prepare for mock requests.
v2: Rename to last_retired_context, split out legacy_context tracking
for MI_SET_CONTEXT.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161218153724.8439-3-chris@chris-wilson.co.uk
2016-12-18 23:37:20 +08:00
|
|
|
spin_unlock_irq(&engine->timeline->lock);
|
2016-07-20 16:21:08 +08:00
|
|
|
|
|
|
|
/* We know the GPU must have read the request to have
|
|
|
|
* sent us the seqno + interrupt, so use the position
|
|
|
|
* of tail of the request to update the last known position
|
|
|
|
* of the GPU head.
|
|
|
|
*
|
|
|
|
* Note this requires that we are always called in request
|
|
|
|
* completion order.
|
|
|
|
*/
|
2016-08-04 14:52:36 +08:00
|
|
|
list_del(&request->ring_link);
|
2016-08-03 05:50:19 +08:00
|
|
|
request->ring->last_retired_head = request->postfix;
|
2016-11-16 00:46:20 +08:00
|
|
|
if (!--request->i915->gt.active_requests) {
|
|
|
|
GEM_BUG_ON(!request->i915->gt.awake);
|
|
|
|
mod_delayed_work(request->i915->wq,
|
|
|
|
&request->i915->gt.idle_work,
|
|
|
|
msecs_to_jiffies(100));
|
|
|
|
}
|
2017-02-23 15:44:08 +08:00
|
|
|
unreserve_seqno(request->engine);
|
2016-07-20 16:21:08 +08:00
|
|
|
|
drm/i915: Refactor activity tracking for requests
With the introduction of requests, we amplified the number of atomic
refcounted objects we use and update every execbuffer; from none to
several references, and a set of references that need to be changed. We
also introduced interesting side-effects in the order of retiring
requests and objects.
Instead of independently tracking the last request for an object, track
the active objects for each request. The object will reside in the
buffer list of its most recent active request and so we reduce the kref
interchange to a list_move. Now retirements are entirely driven by the
request, dramatically simplifying activity tracking on the object
themselves, and removing the ambiguity between retiring objects and
retiring requests.
Furthermore with the consolidation of managing the activity tracking
centrally, we can look forward to using RCU to enable lockless lookup of
the current active requests for an object. In the future, we will be
able to query the status or wait upon rendering to an object without
even touching the struct_mutex BKL.
All told, less code, simpler and faster, and more extensible.
v2: Add a typedef for the function pointer for convenience later.
v3: Make the noop retirement callback explicit. Allow passing NULL to
the init_request_active() which is expanded to a common noop function.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1470293567-10811-16-git-send-email-chris@chris-wilson.co.uk
2016-08-04 14:52:35 +08:00
|
|
|
/* Walk through the active list, calling retire on each. This allows
|
|
|
|
* objects to track their GPU activity and mark themselves as idle
|
|
|
|
* when their *last* active request is completed (updating state
|
|
|
|
* tracking lists for eviction, active references for GEM, etc).
|
|
|
|
*
|
|
|
|
* As the ->retire() may free the node, we decouple it first and
|
|
|
|
* pass along the auxiliary information (to avoid dereferencing
|
|
|
|
* the node after the callback).
|
|
|
|
*/
|
|
|
|
list_for_each_entry_safe(active, next, &request->active_list, link) {
|
|
|
|
/* In microbenchmarks or focusing upon time inside the kernel,
|
|
|
|
* we may spend an inordinate amount of time simply handling
|
|
|
|
* the retirement of requests and processing their callbacks.
|
|
|
|
* Of which, this loop itself is particularly hot due to the
|
|
|
|
* cache misses when jumping around the list of i915_gem_active.
|
|
|
|
* So we try to keep this loop as streamlined as possible and
|
|
|
|
* also prefetch the next i915_gem_active to try and hide
|
|
|
|
* the likely cache miss.
|
|
|
|
*/
|
|
|
|
prefetchw(next);
|
|
|
|
|
|
|
|
INIT_LIST_HEAD(&active->link);
|
drm/i915: Enable lockless lookup of request tracking via RCU
If we enable RCU for the requests (providing a grace period where we can
inspect a "dead" request before it is freed), we can allow callers to
carefully perform lockless lookup of an active request.
However, by enabling deferred freeing of requests, we can potentially
hog a lot of memory when dealing with tens of thousands of requests per
second - with a quick insertion of a synchronize_rcu() inside our
shrinker callback, that issue disappears.
v2: Currently, it is our responsibility to handle reclaim i.e. to avoid
hogging memory with the delayed slab frees. At the moment, we wait for a
grace period in the shrinker, and block for all RCU callbacks on oom.
Suggested alternatives focus on flushing our RCU callback when we have a
certain number of outstanding request frees, and blocking on that flush
after a second high watermark. (So rather than wait for the system to
run out of memory, we stop issuing requests - both are nondeterministic.)
Paul E. McKenney wrote:
Another approach is synchronize_rcu() after some largish number of
requests. The advantage of this approach is that it throttles the
production of callbacks at the source. The corresponding disadvantage
is that it slows things up.
Another approach is to use call_rcu(), but if the previous call_rcu()
is still in flight, block waiting for it. Yet another approach is
the get_state_synchronize_rcu() / cond_synchronize_rcu() pair. The
idea is to do something like this:
cond_synchronize_rcu(cookie);
cookie = get_state_synchronize_rcu();
You would of course do an initial get_state_synchronize_rcu() to
get things going. This would not block unless there was less than
one grace period's worth of time between invocations. But this
assumes a busy system, where there is almost always a grace period
in flight. But you can make that happen as follows:
cond_synchronize_rcu(cookie);
cookie = get_state_synchronize_rcu();
call_rcu(&my_rcu_head, noop_function);
Note that you need additional code to make sure that the old callback
has completed before doing a new one. Setting and clearing a flag
with appropriate memory ordering control suffices (e.g,. smp_load_acquire()
and smp_store_release()).
v3: More comments on compiler and processor order of operations within
the RCU lookup and discover we can use rcu_access_pointer() here instead.
v4: Wrap i915_gem_active_get_rcu() to take the rcu_read_lock itself.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Maarten Lankhorst <maarten.lankhorst@linux.intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Cc: Josh Triplett <josh@joshtriplett.org>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Link: http://patchwork.freedesktop.org/patch/msgid/1470324762-2545-25-git-send-email-chris@chris-wilson.co.uk
2016-08-04 23:32:41 +08:00
|
|
|
RCU_INIT_POINTER(active->request, NULL);
|
drm/i915: Refactor activity tracking for requests
With the introduction of requests, we amplified the number of atomic
refcounted objects we use and update every execbuffer; from none to
several references, and a set of references that need to be changed. We
also introduced interesting side-effects in the order of retiring
requests and objects.
Instead of independently tracking the last request for an object, track
the active objects for each request. The object will reside in the
buffer list of its most recent active request and so we reduce the kref
interchange to a list_move. Now retirements are entirely driven by the
request, dramatically simplifying activity tracking on the object
themselves, and removing the ambiguity between retiring objects and
retiring requests.
Furthermore with the consolidation of managing the activity tracking
centrally, we can look forward to using RCU to enable lockless lookup of
the current active requests for an object. In the future, we will be
able to query the status or wait upon rendering to an object without
even touching the struct_mutex BKL.
All told, less code, simpler and faster, and more extensible.
v2: Add a typedef for the function pointer for convenience later.
v3: Make the noop retirement callback explicit. Allow passing NULL to
the init_request_active() which is expanded to a common noop function.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1470293567-10811-16-git-send-email-chris@chris-wilson.co.uk
2016-08-04 14:52:35 +08:00
|
|
|
|
|
|
|
active->retire(active, request);
|
|
|
|
}
|
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
i915_gem_request_remove_from_client(request);
|
|
|
|
|
2016-11-16 23:20:31 +08:00
|
|
|
/* Retirement decays the ban score as it is a sign of ctx progress */
|
2016-11-16 23:20:34 +08:00
|
|
|
if (request->ctx->ban_score > 0)
|
|
|
|
request->ctx->ban_score--;
|
2016-11-16 23:20:31 +08:00
|
|
|
|
drm/i915: Unify active context tracking between legacy/execlists/guc
The requests conversion introduced a nasty bug where we could generate a
new request in the middle of constructing a request if we needed to idle
the system in order to evict space for a context. The request to idle
would be executed (and waited upon) before the current one, creating a
minor havoc in the seqno accounting, as we will consider the current
request to already be completed (prior to deferred seqno assignment) but
ring->last_retired_head would have been updated and still could allow
us to overwrite the current request before execution.
We also employed two different mechanisms to track the active context
until it was switched out. The legacy method allowed for waiting upon an
active context (it could forcibly evict any vma, including context's),
but the execlists method took a step backwards by pinning the vma for
the entire active lifespan of the context (the only way to evict was to
idle the entire GPU, not individual contexts). However, to circumvent
the tricky issue of locking (i.e. we cannot take struct_mutex at the
time of i915_gem_request_submit(), where we would want to move the
previous context onto the active tracker and unpin it), we take the
execlists approach and keep the contexts pinned until retirement.
The benefit of the execlists approach, more important for execlists than
legacy, was the reduction in work in pinning the context for each
request - as the context was kept pinned until idle, it could short
circuit the pinning for all active contexts.
We introduce new engine vfuncs to pin and unpin the context
respectively. The context is pinned at the start of the request, and
only unpinned when the following request is retired (this ensures that
the context is idle and coherent in main memory before we unpin it). We
move the engine->last_context tracking into the retirement itself
(rather than during request submission) in order to allow the submission
to be reordered or unwound without undue difficultly.
And finally an ulterior motive for unifying context handling was to
prepare for mock requests.
v2: Rename to last_retired_context, split out legacy_context tracking
for MI_SET_CONTEXT.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161218153724.8439-3-chris@chris-wilson.co.uk
2016-12-18 23:37:20 +08:00
|
|
|
/* The backing object for the context is done after switching to the
|
|
|
|
* *next* context. Therefore we cannot retire the previous context until
|
|
|
|
* the next context has already started running. However, since we
|
|
|
|
* cannot take the required locks at i915_gem_request_submit() we
|
|
|
|
* defer the unpinning of the active context to now, retirement of
|
|
|
|
* the subsequent request.
|
|
|
|
*/
|
|
|
|
if (engine->last_retired_context)
|
|
|
|
engine->context_unpin(engine, engine->last_retired_context);
|
|
|
|
engine->last_retired_context = request->ctx;
|
drm/i915: Move GEM activity tracking into a common struct reservation_object
In preparation to support many distinct timelines, we need to expand the
activity tracking on the GEM object to handle more than just a request
per engine. We already use the struct reservation_object on the dma-buf
to handle many fence contexts, so integrating that into the GEM object
itself is the preferred solution. (For example, we can now share the same
reservation_object between every consumer/producer using this buffer and
skip the manual import/export via dma-buf.)
v2: Reimplement busy-ioctl (by walking the reservation object), postpone
the ABI change for another day. Similarly use the reservation object to
find the last_write request (if active and from i915) for choosing
display CS flips.
Caveats:
* busy-ioctl: busy-ioctl only reports on the native fences, it will not
warn of stalls (in set-domain-ioctl, pread/pwrite etc) if the object is
being rendered to by external fences. It also will not report the same
busy state as wait-ioctl (or polling on the dma-buf) in the same
circumstances. On the plus side, it does retain reporting of which
*i915* engines are engaged with this object.
* non-blocking atomic modesets take a step backwards as the wait for
render completion blocks the ioctl. This is fixed in a subsequent
patch to use a fence instead for awaiting on the rendering, see
"drm/i915: Restore nonblocking awaits for modesetting"
* dynamic array manipulation for shared-fences in reservation is slower
than the previous lockless static assignment (e.g. gem_exec_lut_handle
runtime on ivb goes from 42s to 66s), mainly due to atomic operations
(maintaining the fence refcounts).
* loss of object-level retirement callbacks, emulated by VMA retirement
tracking.
* minor loss of object-level last activity information from debugfs,
could be replaced with per-vma information if desired
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161028125858.23563-21-chris@chris-wilson.co.uk
2016-10-28 20:58:44 +08:00
|
|
|
|
|
|
|
dma_fence_signal(&request->fence);
|
2016-11-15 04:41:02 +08:00
|
|
|
|
|
|
|
i915_priotree_fini(request->i915, &request->priotree);
|
2016-07-20 20:31:49 +08:00
|
|
|
i915_gem_request_put(request);
|
2016-07-20 16:21:08 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void i915_gem_request_retire_upto(struct drm_i915_gem_request *req)
|
|
|
|
{
|
|
|
|
struct intel_engine_cs *engine = req->engine;
|
|
|
|
struct drm_i915_gem_request *tmp;
|
|
|
|
|
|
|
|
lockdep_assert_held(&req->i915->drm.struct_mutex);
|
2016-11-25 21:17:15 +08:00
|
|
|
GEM_BUG_ON(!i915_gem_request_completed(req));
|
|
|
|
|
2016-10-28 20:58:27 +08:00
|
|
|
if (list_empty(&req->link))
|
|
|
|
return;
|
2016-07-20 16:21:08 +08:00
|
|
|
|
|
|
|
do {
|
2016-10-28 20:58:46 +08:00
|
|
|
tmp = list_first_entry(&engine->timeline->requests,
|
2016-08-04 14:52:33 +08:00
|
|
|
typeof(*tmp), link);
|
2016-07-20 16:21:08 +08:00
|
|
|
|
|
|
|
i915_gem_request_retire(tmp);
|
|
|
|
} while (tmp != req);
|
|
|
|
}
|
|
|
|
|
2017-02-23 15:44:08 +08:00
|
|
|
static u32 timeline_get_seqno(struct intel_timeline *tl)
|
2016-10-28 20:58:56 +08:00
|
|
|
{
|
2017-02-23 15:44:08 +08:00
|
|
|
return ++tl->seqno;
|
2016-10-28 20:58:56 +08:00
|
|
|
}
|
|
|
|
|
2016-11-15 04:40:59 +08:00
|
|
|
void __i915_gem_request_submit(struct drm_i915_gem_request *request)
|
2016-09-09 21:11:54 +08:00
|
|
|
{
|
2016-10-28 20:58:46 +08:00
|
|
|
struct intel_engine_cs *engine = request->engine;
|
2016-10-28 20:58:57 +08:00
|
|
|
struct intel_timeline *timeline;
|
|
|
|
u32 seqno;
|
2016-09-09 21:11:54 +08:00
|
|
|
|
2016-10-28 20:58:58 +08:00
|
|
|
/* Transfer from per-context onto the global per-engine timeline */
|
|
|
|
timeline = engine->timeline;
|
|
|
|
GEM_BUG_ON(timeline == request->timeline);
|
2016-11-15 04:40:59 +08:00
|
|
|
assert_spin_locked(&timeline->lock);
|
2016-09-09 21:11:54 +08:00
|
|
|
|
2017-02-23 15:44:08 +08:00
|
|
|
seqno = timeline_get_seqno(timeline);
|
2016-10-28 20:58:57 +08:00
|
|
|
GEM_BUG_ON(!seqno);
|
|
|
|
GEM_BUG_ON(i915_seqno_passed(intel_engine_get_seqno(engine), seqno));
|
|
|
|
|
|
|
|
/* We may be recursing from the signal callback of another i915 fence */
|
|
|
|
spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING);
|
|
|
|
request->global_seqno = seqno;
|
|
|
|
if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))
|
|
|
|
intel_engine_enable_signaling(request);
|
|
|
|
spin_unlock(&request->lock);
|
|
|
|
|
|
|
|
GEM_BUG_ON(!request->global_seqno);
|
2016-10-28 20:58:52 +08:00
|
|
|
engine->emit_breadcrumb(request,
|
|
|
|
request->ring->vaddr + request->postfix);
|
2016-09-09 21:11:54 +08:00
|
|
|
|
2016-11-15 04:40:57 +08:00
|
|
|
spin_lock(&request->timeline->lock);
|
2016-10-28 20:58:58 +08:00
|
|
|
list_move_tail(&request->link, &timeline->requests);
|
|
|
|
spin_unlock(&request->timeline->lock);
|
|
|
|
|
2016-11-15 04:40:58 +08:00
|
|
|
i915_sw_fence_commit(&request->execute);
|
2017-02-21 19:01:42 +08:00
|
|
|
trace_i915_gem_request_execute(request);
|
2016-11-15 04:40:59 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void i915_gem_request_submit(struct drm_i915_gem_request *request)
|
|
|
|
{
|
|
|
|
struct intel_engine_cs *engine = request->engine;
|
|
|
|
unsigned long flags;
|
2016-11-15 04:40:58 +08:00
|
|
|
|
2016-11-15 04:40:59 +08:00
|
|
|
/* Will be called from irq-context when using foreign fences. */
|
|
|
|
spin_lock_irqsave(&engine->timeline->lock, flags);
|
|
|
|
|
|
|
|
__i915_gem_request_submit(request);
|
|
|
|
|
|
|
|
spin_unlock_irqrestore(&engine->timeline->lock, flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __i915_sw_fence_call
|
|
|
|
submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
|
|
|
|
{
|
2016-11-25 21:17:17 +08:00
|
|
|
struct drm_i915_gem_request *request =
|
|
|
|
container_of(fence, typeof(*request), submit);
|
2016-11-15 04:40:59 +08:00
|
|
|
|
2016-11-25 21:17:17 +08:00
|
|
|
switch (state) {
|
|
|
|
case FENCE_COMPLETE:
|
2017-02-21 19:01:42 +08:00
|
|
|
trace_i915_gem_request_submit(request);
|
2016-11-15 04:40:59 +08:00
|
|
|
request->engine->submit_request(request);
|
2016-11-25 21:17:17 +08:00
|
|
|
break;
|
|
|
|
|
|
|
|
case FENCE_FREE:
|
|
|
|
i915_gem_request_put(request);
|
|
|
|
break;
|
2016-11-15 04:40:59 +08:00
|
|
|
}
|
2016-10-28 20:58:58 +08:00
|
|
|
|
2016-09-09 21:11:54 +08:00
|
|
|
return NOTIFY_DONE;
|
|
|
|
}
|
|
|
|
|
2016-11-15 04:40:58 +08:00
|
|
|
static int __i915_sw_fence_call
|
|
|
|
execute_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
|
|
|
|
{
|
2016-11-25 21:17:17 +08:00
|
|
|
struct drm_i915_gem_request *request =
|
|
|
|
container_of(fence, typeof(*request), execute);
|
|
|
|
|
|
|
|
switch (state) {
|
|
|
|
case FENCE_COMPLETE:
|
|
|
|
break;
|
|
|
|
|
|
|
|
case FENCE_FREE:
|
|
|
|
i915_gem_request_put(request);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2016-11-15 04:40:58 +08:00
|
|
|
return NOTIFY_DONE;
|
|
|
|
}
|
|
|
|
|
2016-08-03 05:50:26 +08:00
|
|
|
/**
|
|
|
|
* i915_gem_request_alloc - allocate a request structure
|
|
|
|
*
|
|
|
|
* @engine: engine that we wish to issue the request on.
|
|
|
|
* @ctx: context that the request will be associated with.
|
|
|
|
* This can be NULL if the request is not directly related to
|
|
|
|
* any specific user context, in which case this function will
|
|
|
|
* choose an appropriate context to use.
|
|
|
|
*
|
|
|
|
* Returns a pointer to the allocated request if successful,
|
|
|
|
* or an error code if not.
|
|
|
|
*/
|
|
|
|
struct drm_i915_gem_request *
|
|
|
|
i915_gem_request_alloc(struct intel_engine_cs *engine,
|
|
|
|
struct i915_gem_context *ctx)
|
2016-07-20 16:21:08 +08:00
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
struct drm_i915_gem_request *req;
|
|
|
|
int ret;
|
|
|
|
|
2016-10-28 20:58:56 +08:00
|
|
|
lockdep_assert_held(&dev_priv->drm.struct_mutex);
|
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
/* ABI: Before userspace accesses the GPU (e.g. execbuffer), report
|
2017-01-15 00:23:33 +08:00
|
|
|
* EIO if the GPU is already wedged.
|
2016-07-20 16:21:08 +08:00
|
|
|
*/
|
2017-01-15 00:23:33 +08:00
|
|
|
if (i915_terminally_wedged(&dev_priv->gpu_error))
|
|
|
|
return ERR_PTR(-EIO);
|
2016-07-20 16:21:08 +08:00
|
|
|
|
drm/i915: Unify active context tracking between legacy/execlists/guc
The requests conversion introduced a nasty bug where we could generate a
new request in the middle of constructing a request if we needed to idle
the system in order to evict space for a context. The request to idle
would be executed (and waited upon) before the current one, creating a
minor havoc in the seqno accounting, as we will consider the current
request to already be completed (prior to deferred seqno assignment) but
ring->last_retired_head would have been updated and still could allow
us to overwrite the current request before execution.
We also employed two different mechanisms to track the active context
until it was switched out. The legacy method allowed for waiting upon an
active context (it could forcibly evict any vma, including context's),
but the execlists method took a step backwards by pinning the vma for
the entire active lifespan of the context (the only way to evict was to
idle the entire GPU, not individual contexts). However, to circumvent
the tricky issue of locking (i.e. we cannot take struct_mutex at the
time of i915_gem_request_submit(), where we would want to move the
previous context onto the active tracker and unpin it), we take the
execlists approach and keep the contexts pinned until retirement.
The benefit of the execlists approach, more important for execlists than
legacy, was the reduction in work in pinning the context for each
request - as the context was kept pinned until idle, it could short
circuit the pinning for all active contexts.
We introduce new engine vfuncs to pin and unpin the context
respectively. The context is pinned at the start of the request, and
only unpinned when the following request is retired (this ensures that
the context is idle and coherent in main memory before we unpin it). We
move the engine->last_context tracking into the retirement itself
(rather than during request submission) in order to allow the submission
to be reordered or unwound without undue difficultly.
And finally an ulterior motive for unifying context handling was to
prepare for mock requests.
v2: Rename to last_retired_context, split out legacy_context tracking
for MI_SET_CONTEXT.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161218153724.8439-3-chris@chris-wilson.co.uk
2016-12-18 23:37:20 +08:00
|
|
|
/* Pinning the contexts may generate requests in order to acquire
|
|
|
|
* GGTT space, so do this first before we reserve a seqno for
|
|
|
|
* ourselves.
|
|
|
|
*/
|
|
|
|
ret = engine->context_pin(engine, ctx);
|
2016-10-28 20:58:56 +08:00
|
|
|
if (ret)
|
|
|
|
return ERR_PTR(ret);
|
|
|
|
|
2017-02-23 15:44:08 +08:00
|
|
|
ret = reserve_seqno(engine);
|
drm/i915: Unify active context tracking between legacy/execlists/guc
The requests conversion introduced a nasty bug where we could generate a
new request in the middle of constructing a request if we needed to idle
the system in order to evict space for a context. The request to idle
would be executed (and waited upon) before the current one, creating a
minor havoc in the seqno accounting, as we will consider the current
request to already be completed (prior to deferred seqno assignment) but
ring->last_retired_head would have been updated and still could allow
us to overwrite the current request before execution.
We also employed two different mechanisms to track the active context
until it was switched out. The legacy method allowed for waiting upon an
active context (it could forcibly evict any vma, including context's),
but the execlists method took a step backwards by pinning the vma for
the entire active lifespan of the context (the only way to evict was to
idle the entire GPU, not individual contexts). However, to circumvent
the tricky issue of locking (i.e. we cannot take struct_mutex at the
time of i915_gem_request_submit(), where we would want to move the
previous context onto the active tracker and unpin it), we take the
execlists approach and keep the contexts pinned until retirement.
The benefit of the execlists approach, more important for execlists than
legacy, was the reduction in work in pinning the context for each
request - as the context was kept pinned until idle, it could short
circuit the pinning for all active contexts.
We introduce new engine vfuncs to pin and unpin the context
respectively. The context is pinned at the start of the request, and
only unpinned when the following request is retired (this ensures that
the context is idle and coherent in main memory before we unpin it). We
move the engine->last_context tracking into the retirement itself
(rather than during request submission) in order to allow the submission
to be reordered or unwound without undue difficultly.
And finally an ulterior motive for unifying context handling was to
prepare for mock requests.
v2: Rename to last_retired_context, split out legacy_context tracking
for MI_SET_CONTEXT.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161218153724.8439-3-chris@chris-wilson.co.uk
2016-12-18 23:37:20 +08:00
|
|
|
if (ret)
|
|
|
|
goto err_unpin;
|
|
|
|
|
2016-07-20 16:21:09 +08:00
|
|
|
/* Move the oldest request to the slab-cache (if not in use!) */
|
2016-10-28 20:58:46 +08:00
|
|
|
req = list_first_entry_or_null(&engine->timeline->requests,
|
2016-08-04 14:52:33 +08:00
|
|
|
typeof(*req), link);
|
2016-10-28 20:58:58 +08:00
|
|
|
if (req && __i915_gem_request_completed(req))
|
2016-07-26 19:01:51 +08:00
|
|
|
i915_gem_request_retire(req);
|
2016-07-20 16:21:09 +08:00
|
|
|
|
2016-08-09 16:23:34 +08:00
|
|
|
/* Beware: Dragons be flying overhead.
|
|
|
|
*
|
|
|
|
* We use RCU to look up requests in flight. The lookups may
|
|
|
|
* race with the request being allocated from the slab freelist.
|
|
|
|
* That is the request we are writing to here, may be in the process
|
2016-08-10 00:03:22 +08:00
|
|
|
* of being read by __i915_gem_active_get_rcu(). As such,
|
2016-08-09 16:23:34 +08:00
|
|
|
* we have to be very careful when overwriting the contents. During
|
|
|
|
* the RCU lookup, we change chase the request->engine pointer,
|
2016-10-28 20:58:49 +08:00
|
|
|
* read the request->global_seqno and increment the reference count.
|
2016-08-09 16:23:34 +08:00
|
|
|
*
|
|
|
|
* The reference count is incremented atomically. If it is zero,
|
|
|
|
* the lookup knows the request is unallocated and complete. Otherwise,
|
|
|
|
* it is either still in use, or has been reallocated and reset
|
2016-10-25 20:00:45 +08:00
|
|
|
* with dma_fence_init(). This increment is safe for release as we
|
|
|
|
* check that the request we have a reference to and matches the active
|
2016-08-09 16:23:34 +08:00
|
|
|
* request.
|
|
|
|
*
|
|
|
|
* Before we increment the refcount, we chase the request->engine
|
|
|
|
* pointer. We must not call kmem_cache_zalloc() or else we set
|
|
|
|
* that pointer to NULL and cause a crash during the lookup. If
|
|
|
|
* we see the request is completed (based on the value of the
|
|
|
|
* old engine and seqno), the lookup is complete and reports NULL.
|
|
|
|
* If we decide the request is not completed (new engine or seqno),
|
|
|
|
* then we grab a reference and double check that it is still the
|
|
|
|
* active request - which it won't be and restart the lookup.
|
|
|
|
*
|
|
|
|
* Do not use kmem_cache_zalloc() here!
|
|
|
|
*/
|
|
|
|
req = kmem_cache_alloc(dev_priv->requests, GFP_KERNEL);
|
2016-10-28 20:58:56 +08:00
|
|
|
if (!req) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto err_unreserve;
|
|
|
|
}
|
2016-07-20 16:21:08 +08:00
|
|
|
|
2016-10-28 20:58:58 +08:00
|
|
|
req->timeline = i915_gem_context_lookup_timeline(ctx, engine);
|
|
|
|
GEM_BUG_ON(req->timeline == engine->timeline);
|
2016-10-28 20:58:46 +08:00
|
|
|
|
2016-07-20 16:21:11 +08:00
|
|
|
spin_lock_init(&req->lock);
|
2016-10-25 20:00:45 +08:00
|
|
|
dma_fence_init(&req->fence,
|
|
|
|
&i915_fence_ops,
|
|
|
|
&req->lock,
|
2016-10-28 20:58:46 +08:00
|
|
|
req->timeline->fence_context,
|
2017-02-23 15:44:08 +08:00
|
|
|
timeline_get_seqno(req->timeline));
|
2016-07-20 16:21:11 +08:00
|
|
|
|
2016-11-25 21:17:17 +08:00
|
|
|
/* We bump the ref for the fence chain */
|
|
|
|
i915_sw_fence_init(&i915_gem_request_get(req)->submit, submit_notify);
|
|
|
|
i915_sw_fence_init(&i915_gem_request_get(req)->execute, execute_notify);
|
|
|
|
|
2016-11-15 04:40:58 +08:00
|
|
|
/* Ensure that the execute fence completes after the submit fence -
|
|
|
|
* as we complete the execute fence from within the submit fence
|
|
|
|
* callback, its completion would otherwise be visible first.
|
|
|
|
*/
|
|
|
|
i915_sw_fence_await_sw_fence(&req->execute, &req->submit, &req->execq);
|
2016-09-09 21:11:54 +08:00
|
|
|
|
2016-11-15 04:41:02 +08:00
|
|
|
i915_priotree_init(&req->priotree);
|
|
|
|
|
drm/i915: Refactor activity tracking for requests
With the introduction of requests, we amplified the number of atomic
refcounted objects we use and update every execbuffer; from none to
several references, and a set of references that need to be changed. We
also introduced interesting side-effects in the order of retiring
requests and objects.
Instead of independently tracking the last request for an object, track
the active objects for each request. The object will reside in the
buffer list of its most recent active request and so we reduce the kref
interchange to a list_move. Now retirements are entirely driven by the
request, dramatically simplifying activity tracking on the object
themselves, and removing the ambiguity between retiring objects and
retiring requests.
Furthermore with the consolidation of managing the activity tracking
centrally, we can look forward to using RCU to enable lockless lookup of
the current active requests for an object. In the future, we will be
able to query the status or wait upon rendering to an object without
even touching the struct_mutex BKL.
All told, less code, simpler and faster, and more extensible.
v2: Add a typedef for the function pointer for convenience later.
v3: Make the noop retirement callback explicit. Allow passing NULL to
the init_request_active() which is expanded to a common noop function.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1470293567-10811-16-git-send-email-chris@chris-wilson.co.uk
2016-08-04 14:52:35 +08:00
|
|
|
INIT_LIST_HEAD(&req->active_list);
|
2016-07-20 16:21:08 +08:00
|
|
|
req->i915 = dev_priv;
|
|
|
|
req->engine = engine;
|
drm/i915: Unify active context tracking between legacy/execlists/guc
The requests conversion introduced a nasty bug where we could generate a
new request in the middle of constructing a request if we needed to idle
the system in order to evict space for a context. The request to idle
would be executed (and waited upon) before the current one, creating a
minor havoc in the seqno accounting, as we will consider the current
request to already be completed (prior to deferred seqno assignment) but
ring->last_retired_head would have been updated and still could allow
us to overwrite the current request before execution.
We also employed two different mechanisms to track the active context
until it was switched out. The legacy method allowed for waiting upon an
active context (it could forcibly evict any vma, including context's),
but the execlists method took a step backwards by pinning the vma for
the entire active lifespan of the context (the only way to evict was to
idle the entire GPU, not individual contexts). However, to circumvent
the tricky issue of locking (i.e. we cannot take struct_mutex at the
time of i915_gem_request_submit(), where we would want to move the
previous context onto the active tracker and unpin it), we take the
execlists approach and keep the contexts pinned until retirement.
The benefit of the execlists approach, more important for execlists than
legacy, was the reduction in work in pinning the context for each
request - as the context was kept pinned until idle, it could short
circuit the pinning for all active contexts.
We introduce new engine vfuncs to pin and unpin the context
respectively. The context is pinned at the start of the request, and
only unpinned when the following request is retired (this ensures that
the context is idle and coherent in main memory before we unpin it). We
move the engine->last_context tracking into the retirement itself
(rather than during request submission) in order to allow the submission
to be reordered or unwound without undue difficultly.
And finally an ulterior motive for unifying context handling was to
prepare for mock requests.
v2: Rename to last_retired_context, split out legacy_context tracking
for MI_SET_CONTEXT.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161218153724.8439-3-chris@chris-wilson.co.uk
2016-12-18 23:37:20 +08:00
|
|
|
req->ctx = ctx;
|
2016-07-20 16:21:08 +08:00
|
|
|
|
2016-08-09 16:23:34 +08:00
|
|
|
/* No zalloc, must clear what we need by hand */
|
2016-10-28 20:58:57 +08:00
|
|
|
req->global_seqno = 0;
|
2016-08-09 16:23:34 +08:00
|
|
|
req->file_priv = NULL;
|
2016-08-15 17:49:06 +08:00
|
|
|
req->batch = NULL;
|
2016-08-09 16:23:34 +08:00
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
/*
|
|
|
|
* Reserve space in the ring buffer for all the commands required to
|
|
|
|
* eventually emit this request. This is to guarantee that the
|
|
|
|
* i915_add_request() call can't fail. Note that the reserve may need
|
|
|
|
* to be redone if the request is not actually submitted straight
|
|
|
|
* away, e.g. because a GPU scheduler has deferred it.
|
|
|
|
*/
|
|
|
|
req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST;
|
2016-10-28 20:58:51 +08:00
|
|
|
GEM_BUG_ON(req->reserved_space < engine->emit_breadcrumb_sz);
|
2016-07-20 16:21:08 +08:00
|
|
|
|
2016-12-18 23:37:24 +08:00
|
|
|
ret = engine->request_alloc(req);
|
2016-07-20 16:21:08 +08:00
|
|
|
if (ret)
|
|
|
|
goto err_ctx;
|
|
|
|
|
2016-08-15 17:48:40 +08:00
|
|
|
/* Record the position of the start of the request so that
|
|
|
|
* should we detect the updated seqno part-way through the
|
|
|
|
* GPU processing the request, we never over-estimate the
|
|
|
|
* position of the head.
|
|
|
|
*/
|
|
|
|
req->head = req->ring->tail;
|
|
|
|
|
2017-02-23 15:44:08 +08:00
|
|
|
/* Check that we didn't interrupt ourselves with a new request */
|
|
|
|
GEM_BUG_ON(req->timeline->seqno != req->fence.seqno);
|
2016-08-03 05:50:26 +08:00
|
|
|
return req;
|
2016-07-20 16:21:08 +08:00
|
|
|
|
|
|
|
err_ctx:
|
2016-11-25 21:17:16 +08:00
|
|
|
/* Make sure we didn't add ourselves to external state before freeing */
|
|
|
|
GEM_BUG_ON(!list_empty(&req->active_list));
|
|
|
|
GEM_BUG_ON(!list_empty(&req->priotree.signalers_list));
|
|
|
|
GEM_BUG_ON(!list_empty(&req->priotree.waiters_list));
|
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
kmem_cache_free(dev_priv->requests, req);
|
2016-10-28 20:58:56 +08:00
|
|
|
err_unreserve:
|
2017-02-23 15:44:08 +08:00
|
|
|
unreserve_seqno(engine);
|
drm/i915: Unify active context tracking between legacy/execlists/guc
The requests conversion introduced a nasty bug where we could generate a
new request in the middle of constructing a request if we needed to idle
the system in order to evict space for a context. The request to idle
would be executed (and waited upon) before the current one, creating a
minor havoc in the seqno accounting, as we will consider the current
request to already be completed (prior to deferred seqno assignment) but
ring->last_retired_head would have been updated and still could allow
us to overwrite the current request before execution.
We also employed two different mechanisms to track the active context
until it was switched out. The legacy method allowed for waiting upon an
active context (it could forcibly evict any vma, including context's),
but the execlists method took a step backwards by pinning the vma for
the entire active lifespan of the context (the only way to evict was to
idle the entire GPU, not individual contexts). However, to circumvent
the tricky issue of locking (i.e. we cannot take struct_mutex at the
time of i915_gem_request_submit(), where we would want to move the
previous context onto the active tracker and unpin it), we take the
execlists approach and keep the contexts pinned until retirement.
The benefit of the execlists approach, more important for execlists than
legacy, was the reduction in work in pinning the context for each
request - as the context was kept pinned until idle, it could short
circuit the pinning for all active contexts.
We introduce new engine vfuncs to pin and unpin the context
respectively. The context is pinned at the start of the request, and
only unpinned when the following request is retired (this ensures that
the context is idle and coherent in main memory before we unpin it). We
move the engine->last_context tracking into the retirement itself
(rather than during request submission) in order to allow the submission
to be reordered or unwound without undue difficultly.
And finally an ulterior motive for unifying context handling was to
prepare for mock requests.
v2: Rename to last_retired_context, split out legacy_context tracking
for MI_SET_CONTEXT.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161218153724.8439-3-chris@chris-wilson.co.uk
2016-12-18 23:37:20 +08:00
|
|
|
err_unpin:
|
|
|
|
engine->context_unpin(engine, ctx);
|
2016-08-03 05:50:26 +08:00
|
|
|
return ERR_PTR(ret);
|
2016-07-20 16:21:08 +08:00
|
|
|
}
|
|
|
|
|
2016-09-09 21:11:56 +08:00
|
|
|
static int
|
|
|
|
i915_gem_request_await_request(struct drm_i915_gem_request *to,
|
|
|
|
struct drm_i915_gem_request *from)
|
|
|
|
{
|
2016-10-28 20:58:53 +08:00
|
|
|
int ret;
|
2016-09-09 21:11:56 +08:00
|
|
|
|
|
|
|
GEM_BUG_ON(to == from);
|
|
|
|
|
2016-11-15 04:41:02 +08:00
|
|
|
if (to->engine->schedule) {
|
|
|
|
ret = i915_priotree_add_dependency(to->i915,
|
|
|
|
&to->priotree,
|
|
|
|
&from->priotree);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2016-10-28 20:58:46 +08:00
|
|
|
if (to->timeline == from->timeline)
|
2016-09-09 21:11:56 +08:00
|
|
|
return 0;
|
|
|
|
|
2016-10-28 20:58:46 +08:00
|
|
|
if (to->engine == from->engine) {
|
|
|
|
ret = i915_sw_fence_await_sw_fence_gfp(&to->submit,
|
|
|
|
&from->submit,
|
|
|
|
GFP_KERNEL);
|
|
|
|
return ret < 0 ? ret : 0;
|
|
|
|
}
|
|
|
|
|
2016-10-28 20:58:49 +08:00
|
|
|
if (!from->global_seqno) {
|
|
|
|
ret = i915_sw_fence_await_dma_fence(&to->submit,
|
|
|
|
&from->fence, 0,
|
|
|
|
GFP_KERNEL);
|
|
|
|
return ret < 0 ? ret : 0;
|
|
|
|
}
|
|
|
|
|
2016-10-28 20:58:53 +08:00
|
|
|
if (from->global_seqno <= to->timeline->sync_seqno[from->engine->id])
|
2016-09-09 21:11:56 +08:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
trace_i915_gem_ring_sync_to(to, from);
|
|
|
|
if (!i915.semaphores) {
|
2016-09-09 21:12:00 +08:00
|
|
|
if (!i915_spin_request(from, TASK_INTERRUPTIBLE, 2)) {
|
|
|
|
ret = i915_sw_fence_await_dma_fence(&to->submit,
|
|
|
|
&from->fence, 0,
|
|
|
|
GFP_KERNEL);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
}
|
2016-09-09 21:11:56 +08:00
|
|
|
} else {
|
|
|
|
ret = to->engine->semaphore.sync_to(to, from);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2016-10-28 20:58:53 +08:00
|
|
|
to->timeline->sync_seqno[from->engine->id] = from->global_seqno;
|
2016-09-09 21:11:56 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-10-28 20:58:24 +08:00
|
|
|
int
|
|
|
|
i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req,
|
|
|
|
struct dma_fence *fence)
|
|
|
|
{
|
|
|
|
struct dma_fence_array *array;
|
|
|
|
int ret;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (dma_fence_is_i915(fence))
|
|
|
|
return i915_gem_request_await_request(req, to_request(fence));
|
|
|
|
|
|
|
|
if (!dma_fence_is_array(fence)) {
|
|
|
|
ret = i915_sw_fence_await_dma_fence(&req->submit,
|
|
|
|
fence, I915_FENCE_TIMEOUT,
|
|
|
|
GFP_KERNEL);
|
|
|
|
return ret < 0 ? ret : 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Note that if the fence-array was created in signal-on-any mode,
|
|
|
|
* we should *not* decompose it into its individual fences. However,
|
|
|
|
* we don't currently store which mode the fence-array is operating
|
|
|
|
* in. Fortunately, the only user of signal-on-any is private to
|
|
|
|
* amdgpu and we should not see any incoming fence-array from
|
|
|
|
* sync-file being in signal-on-any mode.
|
|
|
|
*/
|
|
|
|
|
|
|
|
array = to_dma_fence_array(fence);
|
|
|
|
for (i = 0; i < array->num_fences; i++) {
|
|
|
|
struct dma_fence *child = array->fences[i];
|
|
|
|
|
|
|
|
if (dma_fence_is_i915(child))
|
|
|
|
ret = i915_gem_request_await_request(req,
|
|
|
|
to_request(child));
|
|
|
|
else
|
|
|
|
ret = i915_sw_fence_await_dma_fence(&req->submit,
|
|
|
|
child, I915_FENCE_TIMEOUT,
|
|
|
|
GFP_KERNEL);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-09-09 21:11:56 +08:00
|
|
|
/**
|
|
|
|
* i915_gem_request_await_object - set this request to (async) wait upon a bo
|
|
|
|
*
|
|
|
|
* @to: request we are wishing to use
|
|
|
|
* @obj: object which may be in use on another ring.
|
|
|
|
*
|
|
|
|
* This code is meant to abstract object synchronization with the GPU.
|
|
|
|
* Conceptually we serialise writes between engines inside the GPU.
|
|
|
|
* We only allow one engine to write into a buffer at any time, but
|
|
|
|
* multiple readers. To ensure each has a coherent view of memory, we must:
|
|
|
|
*
|
|
|
|
* - If there is an outstanding write request to the object, the new
|
|
|
|
* request must wait for it to complete (either CPU or in hw, requests
|
|
|
|
* on the same ring will be naturally ordered).
|
|
|
|
*
|
|
|
|
* - If we are a write request (pending_write_domain is set), the new
|
|
|
|
* request must wait for outstanding read requests to complete.
|
|
|
|
*
|
|
|
|
* Returns 0 if successful, else propagates up the lower layer error.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
i915_gem_request_await_object(struct drm_i915_gem_request *to,
|
|
|
|
struct drm_i915_gem_object *obj,
|
|
|
|
bool write)
|
|
|
|
{
|
drm/i915: Move GEM activity tracking into a common struct reservation_object
In preparation to support many distinct timelines, we need to expand the
activity tracking on the GEM object to handle more than just a request
per engine. We already use the struct reservation_object on the dma-buf
to handle many fence contexts, so integrating that into the GEM object
itself is the preferred solution. (For example, we can now share the same
reservation_object between every consumer/producer using this buffer and
skip the manual import/export via dma-buf.)
v2: Reimplement busy-ioctl (by walking the reservation object), postpone
the ABI change for another day. Similarly use the reservation object to
find the last_write request (if active and from i915) for choosing
display CS flips.
Caveats:
* busy-ioctl: busy-ioctl only reports on the native fences, it will not
warn of stalls (in set-domain-ioctl, pread/pwrite etc) if the object is
being rendered to by external fences. It also will not report the same
busy state as wait-ioctl (or polling on the dma-buf) in the same
circumstances. On the plus side, it does retain reporting of which
*i915* engines are engaged with this object.
* non-blocking atomic modesets take a step backwards as the wait for
render completion blocks the ioctl. This is fixed in a subsequent
patch to use a fence instead for awaiting on the rendering, see
"drm/i915: Restore nonblocking awaits for modesetting"
* dynamic array manipulation for shared-fences in reservation is slower
than the previous lockless static assignment (e.g. gem_exec_lut_handle
runtime on ivb goes from 42s to 66s), mainly due to atomic operations
(maintaining the fence refcounts).
* loss of object-level retirement callbacks, emulated by VMA retirement
tracking.
* minor loss of object-level last activity information from debugfs,
could be replaced with per-vma information if desired
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161028125858.23563-21-chris@chris-wilson.co.uk
2016-10-28 20:58:44 +08:00
|
|
|
struct dma_fence *excl;
|
|
|
|
int ret = 0;
|
2016-09-09 21:11:56 +08:00
|
|
|
|
|
|
|
if (write) {
|
drm/i915: Move GEM activity tracking into a common struct reservation_object
In preparation to support many distinct timelines, we need to expand the
activity tracking on the GEM object to handle more than just a request
per engine. We already use the struct reservation_object on the dma-buf
to handle many fence contexts, so integrating that into the GEM object
itself is the preferred solution. (For example, we can now share the same
reservation_object between every consumer/producer using this buffer and
skip the manual import/export via dma-buf.)
v2: Reimplement busy-ioctl (by walking the reservation object), postpone
the ABI change for another day. Similarly use the reservation object to
find the last_write request (if active and from i915) for choosing
display CS flips.
Caveats:
* busy-ioctl: busy-ioctl only reports on the native fences, it will not
warn of stalls (in set-domain-ioctl, pread/pwrite etc) if the object is
being rendered to by external fences. It also will not report the same
busy state as wait-ioctl (or polling on the dma-buf) in the same
circumstances. On the plus side, it does retain reporting of which
*i915* engines are engaged with this object.
* non-blocking atomic modesets take a step backwards as the wait for
render completion blocks the ioctl. This is fixed in a subsequent
patch to use a fence instead for awaiting on the rendering, see
"drm/i915: Restore nonblocking awaits for modesetting"
* dynamic array manipulation for shared-fences in reservation is slower
than the previous lockless static assignment (e.g. gem_exec_lut_handle
runtime on ivb goes from 42s to 66s), mainly due to atomic operations
(maintaining the fence refcounts).
* loss of object-level retirement callbacks, emulated by VMA retirement
tracking.
* minor loss of object-level last activity information from debugfs,
could be replaced with per-vma information if desired
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161028125858.23563-21-chris@chris-wilson.co.uk
2016-10-28 20:58:44 +08:00
|
|
|
struct dma_fence **shared;
|
|
|
|
unsigned int count, i;
|
|
|
|
|
|
|
|
ret = reservation_object_get_fences_rcu(obj->resv,
|
|
|
|
&excl, &count, &shared);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
for (i = 0; i < count; i++) {
|
|
|
|
ret = i915_gem_request_await_dma_fence(to, shared[i]);
|
|
|
|
if (ret)
|
|
|
|
break;
|
|
|
|
|
|
|
|
dma_fence_put(shared[i]);
|
|
|
|
}
|
|
|
|
|
|
|
|
for (; i < count; i++)
|
|
|
|
dma_fence_put(shared[i]);
|
|
|
|
kfree(shared);
|
2016-09-09 21:11:56 +08:00
|
|
|
} else {
|
drm/i915: Move GEM activity tracking into a common struct reservation_object
In preparation to support many distinct timelines, we need to expand the
activity tracking on the GEM object to handle more than just a request
per engine. We already use the struct reservation_object on the dma-buf
to handle many fence contexts, so integrating that into the GEM object
itself is the preferred solution. (For example, we can now share the same
reservation_object between every consumer/producer using this buffer and
skip the manual import/export via dma-buf.)
v2: Reimplement busy-ioctl (by walking the reservation object), postpone
the ABI change for another day. Similarly use the reservation object to
find the last_write request (if active and from i915) for choosing
display CS flips.
Caveats:
* busy-ioctl: busy-ioctl only reports on the native fences, it will not
warn of stalls (in set-domain-ioctl, pread/pwrite etc) if the object is
being rendered to by external fences. It also will not report the same
busy state as wait-ioctl (or polling on the dma-buf) in the same
circumstances. On the plus side, it does retain reporting of which
*i915* engines are engaged with this object.
* non-blocking atomic modesets take a step backwards as the wait for
render completion blocks the ioctl. This is fixed in a subsequent
patch to use a fence instead for awaiting on the rendering, see
"drm/i915: Restore nonblocking awaits for modesetting"
* dynamic array manipulation for shared-fences in reservation is slower
than the previous lockless static assignment (e.g. gem_exec_lut_handle
runtime on ivb goes from 42s to 66s), mainly due to atomic operations
(maintaining the fence refcounts).
* loss of object-level retirement callbacks, emulated by VMA retirement
tracking.
* minor loss of object-level last activity information from debugfs,
could be replaced with per-vma information if desired
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161028125858.23563-21-chris@chris-wilson.co.uk
2016-10-28 20:58:44 +08:00
|
|
|
excl = reservation_object_get_excl_rcu(obj->resv);
|
2016-09-09 21:11:56 +08:00
|
|
|
}
|
|
|
|
|
drm/i915: Move GEM activity tracking into a common struct reservation_object
In preparation to support many distinct timelines, we need to expand the
activity tracking on the GEM object to handle more than just a request
per engine. We already use the struct reservation_object on the dma-buf
to handle many fence contexts, so integrating that into the GEM object
itself is the preferred solution. (For example, we can now share the same
reservation_object between every consumer/producer using this buffer and
skip the manual import/export via dma-buf.)
v2: Reimplement busy-ioctl (by walking the reservation object), postpone
the ABI change for another day. Similarly use the reservation object to
find the last_write request (if active and from i915) for choosing
display CS flips.
Caveats:
* busy-ioctl: busy-ioctl only reports on the native fences, it will not
warn of stalls (in set-domain-ioctl, pread/pwrite etc) if the object is
being rendered to by external fences. It also will not report the same
busy state as wait-ioctl (or polling on the dma-buf) in the same
circumstances. On the plus side, it does retain reporting of which
*i915* engines are engaged with this object.
* non-blocking atomic modesets take a step backwards as the wait for
render completion blocks the ioctl. This is fixed in a subsequent
patch to use a fence instead for awaiting on the rendering, see
"drm/i915: Restore nonblocking awaits for modesetting"
* dynamic array manipulation for shared-fences in reservation is slower
than the previous lockless static assignment (e.g. gem_exec_lut_handle
runtime on ivb goes from 42s to 66s), mainly due to atomic operations
(maintaining the fence refcounts).
* loss of object-level retirement callbacks, emulated by VMA retirement
tracking.
* minor loss of object-level last activity information from debugfs,
could be replaced with per-vma information if desired
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161028125858.23563-21-chris@chris-wilson.co.uk
2016-10-28 20:58:44 +08:00
|
|
|
if (excl) {
|
|
|
|
if (ret == 0)
|
|
|
|
ret = i915_gem_request_await_dma_fence(to, excl);
|
2016-09-09 21:11:56 +08:00
|
|
|
|
drm/i915: Move GEM activity tracking into a common struct reservation_object
In preparation to support many distinct timelines, we need to expand the
activity tracking on the GEM object to handle more than just a request
per engine. We already use the struct reservation_object on the dma-buf
to handle many fence contexts, so integrating that into the GEM object
itself is the preferred solution. (For example, we can now share the same
reservation_object between every consumer/producer using this buffer and
skip the manual import/export via dma-buf.)
v2: Reimplement busy-ioctl (by walking the reservation object), postpone
the ABI change for another day. Similarly use the reservation object to
find the last_write request (if active and from i915) for choosing
display CS flips.
Caveats:
* busy-ioctl: busy-ioctl only reports on the native fences, it will not
warn of stalls (in set-domain-ioctl, pread/pwrite etc) if the object is
being rendered to by external fences. It also will not report the same
busy state as wait-ioctl (or polling on the dma-buf) in the same
circumstances. On the plus side, it does retain reporting of which
*i915* engines are engaged with this object.
* non-blocking atomic modesets take a step backwards as the wait for
render completion blocks the ioctl. This is fixed in a subsequent
patch to use a fence instead for awaiting on the rendering, see
"drm/i915: Restore nonblocking awaits for modesetting"
* dynamic array manipulation for shared-fences in reservation is slower
than the previous lockless static assignment (e.g. gem_exec_lut_handle
runtime on ivb goes from 42s to 66s), mainly due to atomic operations
(maintaining the fence refcounts).
* loss of object-level retirement callbacks, emulated by VMA retirement
tracking.
* minor loss of object-level last activity information from debugfs,
could be replaced with per-vma information if desired
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161028125858.23563-21-chris@chris-wilson.co.uk
2016-10-28 20:58:44 +08:00
|
|
|
dma_fence_put(excl);
|
2016-09-09 21:11:56 +08:00
|
|
|
}
|
|
|
|
|
drm/i915: Move GEM activity tracking into a common struct reservation_object
In preparation to support many distinct timelines, we need to expand the
activity tracking on the GEM object to handle more than just a request
per engine. We already use the struct reservation_object on the dma-buf
to handle many fence contexts, so integrating that into the GEM object
itself is the preferred solution. (For example, we can now share the same
reservation_object between every consumer/producer using this buffer and
skip the manual import/export via dma-buf.)
v2: Reimplement busy-ioctl (by walking the reservation object), postpone
the ABI change for another day. Similarly use the reservation object to
find the last_write request (if active and from i915) for choosing
display CS flips.
Caveats:
* busy-ioctl: busy-ioctl only reports on the native fences, it will not
warn of stalls (in set-domain-ioctl, pread/pwrite etc) if the object is
being rendered to by external fences. It also will not report the same
busy state as wait-ioctl (or polling on the dma-buf) in the same
circumstances. On the plus side, it does retain reporting of which
*i915* engines are engaged with this object.
* non-blocking atomic modesets take a step backwards as the wait for
render completion blocks the ioctl. This is fixed in a subsequent
patch to use a fence instead for awaiting on the rendering, see
"drm/i915: Restore nonblocking awaits for modesetting"
* dynamic array manipulation for shared-fences in reservation is slower
than the previous lockless static assignment (e.g. gem_exec_lut_handle
runtime on ivb goes from 42s to 66s), mainly due to atomic operations
(maintaining the fence refcounts).
* loss of object-level retirement callbacks, emulated by VMA retirement
tracking.
* minor loss of object-level last activity information from debugfs,
could be replaced with per-vma information if desired
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161028125858.23563-21-chris@chris-wilson.co.uk
2016-10-28 20:58:44 +08:00
|
|
|
return ret;
|
2016-09-09 21:11:56 +08:00
|
|
|
}
|
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
static void i915_gem_mark_busy(const struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
|
|
|
|
if (dev_priv->gt.awake)
|
|
|
|
return;
|
|
|
|
|
2016-11-16 00:46:20 +08:00
|
|
|
GEM_BUG_ON(!dev_priv->gt.active_requests);
|
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
intel_runtime_pm_get_noresume(dev_priv);
|
|
|
|
dev_priv->gt.awake = true;
|
|
|
|
|
2016-07-22 04:16:19 +08:00
|
|
|
intel_enable_gt_powersave(dev_priv);
|
2016-07-20 16:21:08 +08:00
|
|
|
i915_update_gfx_val(dev_priv);
|
|
|
|
if (INTEL_GEN(dev_priv) >= 6)
|
|
|
|
gen6_rps_busy(dev_priv);
|
|
|
|
|
|
|
|
queue_delayed_work(dev_priv->wq,
|
|
|
|
&dev_priv->gt.retire_work,
|
|
|
|
round_jiffies_up_relative(HZ));
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* NB: This function is not allowed to fail. Doing so would mean the the
|
|
|
|
* request is not being tracked for completion but the work itself is
|
|
|
|
* going to happen on the hardware. This would be a Bad Thing(tm).
|
|
|
|
*/
|
2016-08-10 20:41:46 +08:00
|
|
|
void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
|
2016-07-20 16:21:08 +08:00
|
|
|
{
|
2016-08-15 17:48:46 +08:00
|
|
|
struct intel_engine_cs *engine = request->engine;
|
|
|
|
struct intel_ring *ring = request->ring;
|
2016-10-28 20:58:46 +08:00
|
|
|
struct intel_timeline *timeline = request->timeline;
|
2016-09-09 21:12:00 +08:00
|
|
|
struct drm_i915_gem_request *prev;
|
2017-02-14 19:32:42 +08:00
|
|
|
u32 *cs;
|
2016-10-28 20:58:52 +08:00
|
|
|
int err;
|
2016-07-20 16:21:08 +08:00
|
|
|
|
2016-10-28 20:58:32 +08:00
|
|
|
lockdep_assert_held(&request->i915->drm.struct_mutex);
|
2016-09-09 21:11:55 +08:00
|
|
|
trace_i915_gem_request_add(request);
|
|
|
|
|
2017-01-11 22:08:58 +08:00
|
|
|
/* Make sure that no request gazumped us - if it was allocated after
|
|
|
|
* our i915_gem_request_alloc() and called __i915_add_request() before
|
|
|
|
* us, the timeline will hold its seqno which is later than ours.
|
|
|
|
*/
|
2017-02-23 15:44:08 +08:00
|
|
|
GEM_BUG_ON(timeline->seqno != request->fence.seqno);
|
2017-01-11 22:08:58 +08:00
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
/*
|
|
|
|
* To ensure that this call will not fail, space for its emissions
|
|
|
|
* should already have been reserved in the ring buffer. Let the ring
|
|
|
|
* know that it is time to use that space up.
|
|
|
|
*/
|
|
|
|
request->reserved_space = 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Emit any outstanding flushes - execbuf can fail to emit the flush
|
|
|
|
* after having emitted the batchbuffer command. Hence we need to fix
|
|
|
|
* things up similar to emitting the lazy request. The difference here
|
|
|
|
* is that the flush _must_ happen before the next request, no matter
|
|
|
|
* what.
|
|
|
|
*/
|
|
|
|
if (flush_caches) {
|
2016-10-28 20:58:52 +08:00
|
|
|
err = engine->emit_flush(request, EMIT_FLUSH);
|
2016-08-03 05:50:24 +08:00
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
/* Not allowed to fail! */
|
2016-10-28 20:58:52 +08:00
|
|
|
WARN(err, "engine->emit_flush() failed: %d!\n", err);
|
2016-07-20 16:21:08 +08:00
|
|
|
}
|
|
|
|
|
2016-08-15 17:48:40 +08:00
|
|
|
/* Record the position of the start of the breadcrumb so that
|
2016-07-20 16:21:08 +08:00
|
|
|
* should we detect the updated seqno part-way through the
|
|
|
|
* GPU processing the request, we never over-estimate the
|
2016-08-15 17:48:40 +08:00
|
|
|
* position of the ring's HEAD.
|
2016-07-20 16:21:08 +08:00
|
|
|
*/
|
2017-02-14 19:32:42 +08:00
|
|
|
cs = intel_ring_begin(request, engine->emit_breadcrumb_sz);
|
|
|
|
GEM_BUG_ON(IS_ERR(cs));
|
|
|
|
request->postfix = intel_ring_offset(request, cs);
|
2016-07-20 16:21:08 +08:00
|
|
|
|
2016-09-09 21:11:55 +08:00
|
|
|
/* Seal the request and mark it as pending execution. Note that
|
|
|
|
* we may inspect this state, without holding any locks, during
|
|
|
|
* hangcheck. Hence we apply the barrier to ensure that we do not
|
|
|
|
* see a more recent value in the hws than we are tracking.
|
|
|
|
*/
|
2016-09-09 21:12:00 +08:00
|
|
|
|
2016-10-28 20:58:46 +08:00
|
|
|
prev = i915_gem_active_raw(&timeline->last_request,
|
2016-09-09 21:12:00 +08:00
|
|
|
&request->i915->drm.struct_mutex);
|
2016-11-15 04:41:02 +08:00
|
|
|
if (prev) {
|
2016-09-09 21:12:00 +08:00
|
|
|
i915_sw_fence_await_sw_fence(&request->submit, &prev->submit,
|
|
|
|
&request->submitq);
|
2016-11-15 04:41:02 +08:00
|
|
|
if (engine->schedule)
|
|
|
|
__i915_priotree_add_dependency(&request->priotree,
|
|
|
|
&prev->priotree,
|
|
|
|
&request->dep,
|
|
|
|
0);
|
|
|
|
}
|
2016-09-09 21:12:00 +08:00
|
|
|
|
2016-10-28 20:58:58 +08:00
|
|
|
spin_lock_irq(&timeline->lock);
|
2016-10-28 20:58:57 +08:00
|
|
|
list_add_tail(&request->link, &timeline->requests);
|
2016-10-28 20:58:58 +08:00
|
|
|
spin_unlock_irq(&timeline->lock);
|
|
|
|
|
2017-02-23 15:44:08 +08:00
|
|
|
GEM_BUG_ON(timeline->seqno != request->fence.seqno);
|
2016-10-28 20:58:46 +08:00
|
|
|
i915_gem_active_set(&timeline->last_request, request);
|
2016-10-28 20:58:57 +08:00
|
|
|
|
2016-09-09 21:11:55 +08:00
|
|
|
list_add_tail(&request->ring_link, &ring->request_list);
|
2016-10-28 20:58:57 +08:00
|
|
|
request->emitted_jiffies = jiffies;
|
2016-09-09 21:11:55 +08:00
|
|
|
|
2017-02-23 15:44:08 +08:00
|
|
|
if (!request->i915->gt.active_requests++)
|
|
|
|
i915_gem_mark_busy(engine);
|
2016-09-09 21:11:54 +08:00
|
|
|
|
2016-11-15 04:41:01 +08:00
|
|
|
/* Let the backend know a new request has arrived that may need
|
|
|
|
* to adjust the existing execution schedule due to a high priority
|
|
|
|
* request - i.e. we may want to preempt the current request in order
|
|
|
|
* to run a high priority dependency chain *before* we can execute this
|
|
|
|
* request.
|
|
|
|
*
|
|
|
|
* This is called before the request is ready to run so that we can
|
|
|
|
* decide whether to preempt the entire chain so that it is ready to
|
|
|
|
* run at the earliest possible convenience.
|
|
|
|
*/
|
|
|
|
if (engine->schedule)
|
2016-11-15 04:41:04 +08:00
|
|
|
engine->schedule(request, request->ctx->priority);
|
2016-11-15 04:41:01 +08:00
|
|
|
|
2016-09-09 21:11:54 +08:00
|
|
|
local_bh_disable();
|
|
|
|
i915_sw_fence_commit(&request->submit);
|
|
|
|
local_bh_enable(); /* Kick the execlists tasklet if just scheduled */
|
2016-07-20 16:21:08 +08:00
|
|
|
}
|
|
|
|
|
2016-09-09 21:11:51 +08:00
|
|
|
static void reset_wait_queue(wait_queue_head_t *q, wait_queue_t *wait)
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
spin_lock_irqsave(&q->lock, flags);
|
|
|
|
if (list_empty(&wait->task_list))
|
|
|
|
__add_wait_queue(q, wait);
|
|
|
|
spin_unlock_irqrestore(&q->lock, flags);
|
|
|
|
}
|
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
static unsigned long local_clock_us(unsigned int *cpu)
|
|
|
|
{
|
|
|
|
unsigned long t;
|
|
|
|
|
|
|
|
/* Cheaply and approximately convert from nanoseconds to microseconds.
|
|
|
|
* The result and subsequent calculations are also defined in the same
|
|
|
|
* approximate microseconds units. The principal source of timing
|
|
|
|
* error here is from the simple truncation.
|
|
|
|
*
|
|
|
|
* Note that local_clock() is only defined wrt to the current CPU;
|
|
|
|
* the comparisons are no longer valid if we switch CPUs. Instead of
|
|
|
|
* blocking preemption for the entire busywait, we can detect the CPU
|
|
|
|
* switch and use that as indicator of system load and a reason to
|
|
|
|
* stop busywaiting, see busywait_stop().
|
|
|
|
*/
|
|
|
|
*cpu = get_cpu();
|
|
|
|
t = local_clock() >> 10;
|
|
|
|
put_cpu();
|
|
|
|
|
|
|
|
return t;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool busywait_stop(unsigned long timeout, unsigned int cpu)
|
|
|
|
{
|
|
|
|
unsigned int this_cpu;
|
|
|
|
|
|
|
|
if (time_after(local_clock_us(&this_cpu), timeout))
|
|
|
|
return true;
|
|
|
|
|
|
|
|
return this_cpu != cpu;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool __i915_spin_request(const struct drm_i915_gem_request *req,
|
|
|
|
int state, unsigned long timeout_us)
|
|
|
|
{
|
2017-02-17 23:13:01 +08:00
|
|
|
struct intel_engine_cs *engine = req->engine;
|
|
|
|
unsigned int irq, cpu;
|
2016-07-20 16:21:08 +08:00
|
|
|
|
|
|
|
/* When waiting for high frequency requests, e.g. during synchronous
|
|
|
|
* rendering split between the CPU and GPU, the finite amount of time
|
|
|
|
* required to set up the irq and wait upon it limits the response
|
|
|
|
* rate. By busywaiting on the request completion for a short while we
|
|
|
|
* can service the high frequency waits as quick as possible. However,
|
|
|
|
* if it is a slow request, we want to sleep as quickly as possible.
|
|
|
|
* The tradeoff between waiting and sleeping is roughly the time it
|
|
|
|
* takes to sleep on a request, on the order of a microsecond.
|
|
|
|
*/
|
|
|
|
|
2017-02-17 23:13:01 +08:00
|
|
|
irq = atomic_read(&engine->irq_count);
|
2016-07-20 16:21:08 +08:00
|
|
|
timeout_us += local_clock_us(&cpu);
|
|
|
|
do {
|
2016-10-28 20:58:49 +08:00
|
|
|
if (__i915_gem_request_completed(req))
|
2016-07-20 16:21:08 +08:00
|
|
|
return true;
|
|
|
|
|
2017-02-17 23:13:01 +08:00
|
|
|
/* Seqno are meant to be ordered *before* the interrupt. If
|
|
|
|
* we see an interrupt without a corresponding seqno advance,
|
|
|
|
* assume we won't see one in the near future but require
|
|
|
|
* the engine->seqno_barrier() to fixup coherency.
|
|
|
|
*/
|
|
|
|
if (atomic_read(&engine->irq_count) != irq)
|
|
|
|
break;
|
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
if (signal_pending_state(state, current))
|
|
|
|
break;
|
|
|
|
|
|
|
|
if (busywait_stop(timeout_us, cpu))
|
|
|
|
break;
|
|
|
|
|
2016-10-25 17:03:14 +08:00
|
|
|
cpu_relax();
|
2016-07-20 16:21:08 +08:00
|
|
|
} while (!need_resched());
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2016-10-28 20:58:48 +08:00
|
|
|
static long
|
2016-11-15 04:40:58 +08:00
|
|
|
__i915_request_wait_for_execute(struct drm_i915_gem_request *request,
|
|
|
|
unsigned int flags,
|
|
|
|
long timeout)
|
2016-10-28 20:58:48 +08:00
|
|
|
{
|
|
|
|
const int state = flags & I915_WAIT_INTERRUPTIBLE ?
|
|
|
|
TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
|
|
|
|
wait_queue_head_t *q = &request->i915->gpu_error.wait_queue;
|
|
|
|
DEFINE_WAIT(reset);
|
|
|
|
DEFINE_WAIT(wait);
|
|
|
|
|
|
|
|
if (flags & I915_WAIT_LOCKED)
|
|
|
|
add_wait_queue(q, &reset);
|
|
|
|
|
|
|
|
do {
|
2016-11-15 04:40:58 +08:00
|
|
|
prepare_to_wait(&request->execute.wait, &wait, state);
|
2016-10-28 20:58:48 +08:00
|
|
|
|
2016-11-15 04:40:58 +08:00
|
|
|
if (i915_sw_fence_done(&request->execute))
|
2016-10-28 20:58:48 +08:00
|
|
|
break;
|
|
|
|
|
|
|
|
if (flags & I915_WAIT_LOCKED &&
|
|
|
|
i915_reset_in_progress(&request->i915->gpu_error)) {
|
|
|
|
__set_current_state(TASK_RUNNING);
|
|
|
|
i915_reset(request->i915);
|
|
|
|
reset_wait_queue(q, &reset);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (signal_pending_state(state, current)) {
|
|
|
|
timeout = -ERESTARTSYS;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2017-02-09 02:12:38 +08:00
|
|
|
if (!timeout) {
|
|
|
|
timeout = -ETIME;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2016-10-28 20:58:48 +08:00
|
|
|
timeout = io_schedule_timeout(timeout);
|
2017-02-09 02:12:38 +08:00
|
|
|
} while (1);
|
2016-11-15 04:40:58 +08:00
|
|
|
finish_wait(&request->execute.wait, &wait);
|
2016-10-28 20:58:48 +08:00
|
|
|
|
|
|
|
if (flags & I915_WAIT_LOCKED)
|
|
|
|
remove_wait_queue(q, &reset);
|
|
|
|
|
|
|
|
return timeout;
|
|
|
|
}
|
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
/**
|
2016-08-04 14:52:40 +08:00
|
|
|
* i915_wait_request - wait until execution of request has finished
|
2016-10-28 20:58:27 +08:00
|
|
|
* @req: the request to wait upon
|
2016-09-09 21:11:49 +08:00
|
|
|
* @flags: how to wait
|
2016-10-28 20:58:27 +08:00
|
|
|
* @timeout: how long to wait in jiffies
|
|
|
|
*
|
|
|
|
* i915_wait_request() waits for the request to be completed, for a
|
|
|
|
* maximum of @timeout jiffies (with MAX_SCHEDULE_TIMEOUT implying an
|
|
|
|
* unbounded wait).
|
2016-07-20 16:21:08 +08:00
|
|
|
*
|
2016-10-28 20:58:27 +08:00
|
|
|
* If the caller holds the struct_mutex, the caller must pass I915_WAIT_LOCKED
|
|
|
|
* in via the flags, and vice versa if the struct_mutex is not held, the caller
|
|
|
|
* must not specify that the wait is locked.
|
2016-07-20 16:21:08 +08:00
|
|
|
*
|
2016-10-28 20:58:27 +08:00
|
|
|
* Returns the remaining time (in jiffies) if the request completed, which may
|
|
|
|
* be zero or -ETIME if the request is unfinished after the timeout expires.
|
|
|
|
* May return -EINTR is called with I915_WAIT_INTERRUPTIBLE and a signal is
|
|
|
|
* pending before the request completes.
|
2016-07-20 16:21:08 +08:00
|
|
|
*/
|
2016-10-28 20:58:27 +08:00
|
|
|
long i915_wait_request(struct drm_i915_gem_request *req,
|
|
|
|
unsigned int flags,
|
|
|
|
long timeout)
|
2016-07-20 16:21:08 +08:00
|
|
|
{
|
2016-09-09 21:11:49 +08:00
|
|
|
const int state = flags & I915_WAIT_INTERRUPTIBLE ?
|
|
|
|
TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
|
2017-02-23 15:44:10 +08:00
|
|
|
wait_queue_head_t *errq = &req->i915->gpu_error.wait_queue;
|
2016-07-20 16:21:08 +08:00
|
|
|
DEFINE_WAIT(reset);
|
|
|
|
struct intel_wait wait;
|
|
|
|
|
|
|
|
might_sleep();
|
2016-09-09 21:11:50 +08:00
|
|
|
#if IS_ENABLED(CONFIG_LOCKDEP)
|
2016-10-28 20:58:27 +08:00
|
|
|
GEM_BUG_ON(debug_locks &&
|
|
|
|
!!lockdep_is_held(&req->i915->drm.struct_mutex) !=
|
2016-09-09 21:11:50 +08:00
|
|
|
!!(flags & I915_WAIT_LOCKED));
|
|
|
|
#endif
|
2016-10-28 20:58:27 +08:00
|
|
|
GEM_BUG_ON(timeout < 0);
|
2016-07-20 16:21:08 +08:00
|
|
|
|
|
|
|
if (i915_gem_request_completed(req))
|
2016-10-28 20:58:27 +08:00
|
|
|
return timeout;
|
2016-07-20 16:21:08 +08:00
|
|
|
|
2016-10-28 20:58:27 +08:00
|
|
|
if (!timeout)
|
|
|
|
return -ETIME;
|
2016-07-20 16:21:08 +08:00
|
|
|
|
2017-02-21 19:00:24 +08:00
|
|
|
trace_i915_gem_request_wait_begin(req, flags);
|
2016-07-20 16:21:08 +08:00
|
|
|
|
2017-02-23 15:44:11 +08:00
|
|
|
if (flags & I915_WAIT_LOCKED)
|
|
|
|
add_wait_queue(errq, &reset);
|
|
|
|
|
2016-11-15 04:40:58 +08:00
|
|
|
if (!i915_sw_fence_done(&req->execute)) {
|
|
|
|
timeout = __i915_request_wait_for_execute(req, flags, timeout);
|
2016-10-28 20:58:48 +08:00
|
|
|
if (timeout < 0)
|
|
|
|
goto complete;
|
|
|
|
|
2016-11-15 04:40:58 +08:00
|
|
|
GEM_BUG_ON(!i915_sw_fence_done(&req->execute));
|
2016-10-28 20:58:48 +08:00
|
|
|
}
|
2016-11-15 04:40:58 +08:00
|
|
|
GEM_BUG_ON(!i915_sw_fence_done(&req->submit));
|
2016-10-28 20:58:49 +08:00
|
|
|
GEM_BUG_ON(!req->global_seqno);
|
2016-10-28 20:58:48 +08:00
|
|
|
|
2016-08-06 00:11:24 +08:00
|
|
|
/* Optimistic short spin before touching IRQs */
|
2016-07-20 16:21:08 +08:00
|
|
|
if (i915_spin_request(req, state, 5))
|
|
|
|
goto complete;
|
|
|
|
|
|
|
|
set_current_state(state);
|
2016-10-28 20:58:49 +08:00
|
|
|
intel_wait_init(&wait, req->global_seqno);
|
2016-07-20 16:21:08 +08:00
|
|
|
if (intel_engine_add_wait(req->engine, &wait))
|
|
|
|
/* In order to check that we haven't missed the interrupt
|
|
|
|
* as we enabled it, we need to kick ourselves to do a
|
|
|
|
* coherent check on the seqno before we sleep.
|
|
|
|
*/
|
|
|
|
goto wakeup;
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
if (signal_pending_state(state, current)) {
|
2016-10-28 20:58:27 +08:00
|
|
|
timeout = -ERESTARTSYS;
|
2016-07-20 16:21:08 +08:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2016-10-28 20:58:27 +08:00
|
|
|
if (!timeout) {
|
|
|
|
timeout = -ETIME;
|
2016-07-20 16:21:08 +08:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2016-10-28 20:58:27 +08:00
|
|
|
timeout = io_schedule_timeout(timeout);
|
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
if (intel_wait_complete(&wait))
|
|
|
|
break;
|
|
|
|
|
|
|
|
set_current_state(state);
|
|
|
|
|
|
|
|
wakeup:
|
|
|
|
/* Carefully check if the request is complete, giving time
|
|
|
|
* for the seqno to be visible following the interrupt.
|
|
|
|
* We also have to check in case we are kicked by the GPU
|
|
|
|
* reset in order to drop the struct_mutex.
|
|
|
|
*/
|
|
|
|
if (__i915_request_irq_complete(req))
|
|
|
|
break;
|
|
|
|
|
2016-09-09 21:11:51 +08:00
|
|
|
/* If the GPU is hung, and we hold the lock, reset the GPU
|
|
|
|
* and then check for completion. On a full reset, the engine's
|
|
|
|
* HW seqno will be advanced passed us and we are complete.
|
|
|
|
* If we do a partial reset, we have to wait for the GPU to
|
|
|
|
* resume and update the breadcrumb.
|
|
|
|
*
|
|
|
|
* If we don't hold the mutex, we can just wait for the worker
|
|
|
|
* to come along and update the breadcrumb (either directly
|
|
|
|
* itself, or indirectly by recovering the GPU).
|
|
|
|
*/
|
|
|
|
if (flags & I915_WAIT_LOCKED &&
|
|
|
|
i915_reset_in_progress(&req->i915->gpu_error)) {
|
|
|
|
__set_current_state(TASK_RUNNING);
|
|
|
|
i915_reset(req->i915);
|
2017-02-23 15:44:10 +08:00
|
|
|
reset_wait_queue(errq, &reset);
|
2016-09-09 21:11:51 +08:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
/* Only spin if we know the GPU is processing this request */
|
|
|
|
if (i915_spin_request(req, state, 2))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
intel_engine_remove_wait(req->engine, &wait);
|
|
|
|
__set_current_state(TASK_RUNNING);
|
2016-09-09 21:11:50 +08:00
|
|
|
|
2016-07-20 16:21:08 +08:00
|
|
|
complete:
|
2017-02-23 15:44:11 +08:00
|
|
|
if (flags & I915_WAIT_LOCKED)
|
|
|
|
remove_wait_queue(errq, &reset);
|
2016-07-20 16:21:08 +08:00
|
|
|
trace_i915_gem_request_wait_end(req);
|
|
|
|
|
2016-10-28 20:58:27 +08:00
|
|
|
return timeout;
|
2016-07-20 16:21:08 +08:00
|
|
|
}
|
2016-08-04 14:52:42 +08:00
|
|
|
|
2016-10-28 20:58:56 +08:00
|
|
|
static void engine_retire_requests(struct intel_engine_cs *engine)
|
2016-08-04 14:52:42 +08:00
|
|
|
{
|
|
|
|
struct drm_i915_gem_request *request, *next;
|
|
|
|
|
2016-10-28 20:58:46 +08:00
|
|
|
list_for_each_entry_safe(request, next,
|
|
|
|
&engine->timeline->requests, link) {
|
2016-10-28 20:58:58 +08:00
|
|
|
if (!__i915_gem_request_completed(request))
|
2016-10-28 20:58:56 +08:00
|
|
|
return;
|
2016-08-04 14:52:42 +08:00
|
|
|
|
|
|
|
i915_gem_request_retire(request);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void i915_gem_retire_requests(struct drm_i915_private *dev_priv)
|
|
|
|
{
|
|
|
|
struct intel_engine_cs *engine;
|
2016-10-28 20:58:56 +08:00
|
|
|
enum intel_engine_id id;
|
2016-08-04 14:52:42 +08:00
|
|
|
|
|
|
|
lockdep_assert_held(&dev_priv->drm.struct_mutex);
|
|
|
|
|
2016-10-28 20:58:56 +08:00
|
|
|
if (!dev_priv->gt.active_requests)
|
2016-08-04 14:52:42 +08:00
|
|
|
return;
|
|
|
|
|
2016-10-28 20:58:56 +08:00
|
|
|
for_each_engine(engine, dev_priv, id)
|
|
|
|
engine_retire_requests(engine);
|
2016-08-04 14:52:42 +08:00
|
|
|
}
|
2017-02-14 01:15:21 +08:00
|
|
|
|
|
|
|
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
|
|
|
|
#include "selftests/mock_request.c"
|
|
|
|
#include "selftests/i915_gem_request.c"
|
|
|
|
#endif
|