2010-05-21 09:08:55 +08:00
|
|
|
#ifndef _INTEL_RINGBUFFER_H_
|
|
|
|
#define _INTEL_RINGBUFFER_H_
|
|
|
|
|
2014-05-11 05:10:43 +08:00
|
|
|
#include <linux/hashtable.h>
|
2015-04-07 23:20:36 +08:00
|
|
|
#include "i915_gem_batch_pool.h"
|
2016-08-05 17:14:11 +08:00
|
|
|
#include "i915_gem_request.h"
|
2014-05-11 05:10:43 +08:00
|
|
|
|
|
|
|
#define I915_CMD_HASH_ORDER 9
|
|
|
|
|
2014-07-25 00:04:28 +08:00
|
|
|
/* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
|
|
|
|
* but keeps the logic simple. Indeed, the whole purpose of this macro is just
|
|
|
|
* to give some inclination as to some of the magic values used in the various
|
|
|
|
* workarounds!
|
|
|
|
*/
|
|
|
|
#define CACHELINE_BYTES 64
|
2015-06-20 02:07:01 +08:00
|
|
|
#define CACHELINE_DWORDS (CACHELINE_BYTES / sizeof(uint32_t))
|
2014-07-25 00:04:28 +08:00
|
|
|
|
2012-12-04 00:43:32 +08:00
|
|
|
/*
|
|
|
|
* Gen2 BSpec "1. Programming Environment" / 1.4.4.6 "Ring Buffer Use"
|
|
|
|
* Gen3 BSpec "vol1c Memory Interface Functions" / 2.3.4.5 "Ring Buffer Use"
|
|
|
|
* Gen4+ BSpec "vol1c Memory Interface and Command Stream" / 5.3.4.5 "Ring Buffer Use"
|
|
|
|
*
|
|
|
|
* "If the Ring Buffer Head Pointer and the Tail Pointer are on the same
|
|
|
|
* cacheline, the Head Pointer must not be greater than the Tail
|
|
|
|
* Pointer."
|
|
|
|
*/
|
|
|
|
#define I915_RING_FREE_SPACE 64
|
|
|
|
|
2016-08-15 17:48:57 +08:00
|
|
|
struct intel_hw_status_page {
|
|
|
|
struct i915_vma *vma;
|
|
|
|
u32 *page_addr;
|
|
|
|
u32 ggtt_offset;
|
2010-05-21 09:08:55 +08:00
|
|
|
};
|
|
|
|
|
2016-07-21 01:16:05 +08:00
|
|
|
#define I915_READ_TAIL(engine) I915_READ(RING_TAIL((engine)->mmio_base))
|
|
|
|
#define I915_WRITE_TAIL(engine, val) I915_WRITE(RING_TAIL((engine)->mmio_base), val)
|
2010-11-09 17:17:32 +08:00
|
|
|
|
2016-07-21 01:16:05 +08:00
|
|
|
#define I915_READ_START(engine) I915_READ(RING_START((engine)->mmio_base))
|
|
|
|
#define I915_WRITE_START(engine, val) I915_WRITE(RING_START((engine)->mmio_base), val)
|
2010-11-09 17:17:32 +08:00
|
|
|
|
2016-07-21 01:16:05 +08:00
|
|
|
#define I915_READ_HEAD(engine) I915_READ(RING_HEAD((engine)->mmio_base))
|
|
|
|
#define I915_WRITE_HEAD(engine, val) I915_WRITE(RING_HEAD((engine)->mmio_base), val)
|
2010-11-09 17:17:32 +08:00
|
|
|
|
2016-07-21 01:16:05 +08:00
|
|
|
#define I915_READ_CTL(engine) I915_READ(RING_CTL((engine)->mmio_base))
|
|
|
|
#define I915_WRITE_CTL(engine, val) I915_WRITE(RING_CTL((engine)->mmio_base), val)
|
2010-11-09 17:17:32 +08:00
|
|
|
|
2016-07-21 01:16:05 +08:00
|
|
|
#define I915_READ_IMR(engine) I915_READ(RING_IMR((engine)->mmio_base))
|
|
|
|
#define I915_WRITE_IMR(engine, val) I915_WRITE(RING_IMR((engine)->mmio_base), val)
|
2010-08-02 22:29:44 +08:00
|
|
|
|
2016-07-21 01:16:05 +08:00
|
|
|
#define I915_READ_MODE(engine) I915_READ(RING_MI_MODE((engine)->mmio_base))
|
|
|
|
#define I915_WRITE_MODE(engine, val) I915_WRITE(RING_MI_MODE((engine)->mmio_base), val)
|
2014-03-12 19:09:41 +08:00
|
|
|
|
2014-07-01 00:53:37 +08:00
|
|
|
/* seqno size is actually only a uint32, but since we plan to use MI_FLUSH_DW to
|
|
|
|
* do the writes, and that must have qw aligned offsets, simply pretend it's 8b.
|
|
|
|
*/
|
2016-04-07 14:29:14 +08:00
|
|
|
#define gen8_semaphore_seqno_size sizeof(uint64_t)
|
|
|
|
#define GEN8_SEMAPHORE_OFFSET(__from, __to) \
|
|
|
|
(((__from) * I915_NUM_ENGINES + (__to)) * gen8_semaphore_seqno_size)
|
2014-07-01 00:53:37 +08:00
|
|
|
#define GEN8_SIGNAL_OFFSET(__ring, to) \
|
|
|
|
(i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \
|
2016-04-07 14:29:14 +08:00
|
|
|
GEN8_SEMAPHORE_OFFSET((__ring)->id, (to)))
|
2014-07-01 00:53:37 +08:00
|
|
|
#define GEN8_WAIT_OFFSET(__ring, from) \
|
|
|
|
(i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \
|
2016-04-07 14:29:14 +08:00
|
|
|
GEN8_SEMAPHORE_OFFSET(from, (__ring)->id))
|
2014-07-01 00:53:37 +08:00
|
|
|
|
2016-08-03 05:50:21 +08:00
|
|
|
enum intel_engine_hangcheck_action {
|
2013-09-06 21:03:28 +08:00
|
|
|
HANGCHECK_IDLE = 0,
|
2013-08-11 17:44:01 +08:00
|
|
|
HANGCHECK_WAIT,
|
|
|
|
HANGCHECK_ACTIVE,
|
|
|
|
HANGCHECK_KICK,
|
|
|
|
HANGCHECK_HUNG,
|
|
|
|
};
|
2013-06-12 17:35:32 +08:00
|
|
|
|
2014-01-31 01:04:43 +08:00
|
|
|
#define HANGCHECK_SCORE_RING_HUNG 31
|
|
|
|
|
2016-08-03 05:50:21 +08:00
|
|
|
struct intel_engine_hangcheck {
|
2014-03-21 20:41:53 +08:00
|
|
|
u64 acthd;
|
2013-05-24 22:16:07 +08:00
|
|
|
u32 seqno;
|
2013-05-30 14:04:29 +08:00
|
|
|
int score;
|
2016-08-03 05:50:21 +08:00
|
|
|
enum intel_engine_hangcheck_action action;
|
2014-06-06 17:22:29 +08:00
|
|
|
int deadlock;
|
2015-12-01 23:56:12 +08:00
|
|
|
u32 instdone[I915_NUM_INSTDONE_REG];
|
2013-05-24 22:16:07 +08:00
|
|
|
};
|
|
|
|
|
2016-08-03 05:50:21 +08:00
|
|
|
struct intel_ring {
|
2016-01-15 23:10:28 +08:00
|
|
|
struct i915_vma *vma;
|
2016-08-15 17:48:57 +08:00
|
|
|
void *vaddr;
|
2014-05-22 21:13:34 +08:00
|
|
|
|
2016-03-16 19:00:38 +08:00
|
|
|
struct intel_engine_cs *engine;
|
2015-09-03 20:01:40 +08:00
|
|
|
struct list_head link;
|
2014-08-11 22:17:44 +08:00
|
|
|
|
2016-08-04 14:52:36 +08:00
|
|
|
struct list_head request_list;
|
|
|
|
|
2014-05-22 21:13:34 +08:00
|
|
|
u32 head;
|
|
|
|
u32 tail;
|
|
|
|
int space;
|
|
|
|
int size;
|
|
|
|
int effective_size;
|
2016-08-15 17:48:57 +08:00
|
|
|
bool needs_iomap;
|
2014-05-22 21:13:34 +08:00
|
|
|
|
|
|
|
/** We track the position of the requests in the ring buffer, and
|
|
|
|
* when each is retired we increment last_retired_head as the GPU
|
|
|
|
* must have finished processing the request and so we know we
|
|
|
|
* can advance the ringbuffer up to that position.
|
|
|
|
*
|
|
|
|
* last_retired_head is set to -1 after the value is consumed so
|
|
|
|
* we can detect new retirements.
|
|
|
|
*/
|
|
|
|
u32 last_retired_head;
|
|
|
|
};
|
|
|
|
|
2016-05-24 21:53:34 +08:00
|
|
|
struct i915_gem_context;
|
2016-03-07 15:30:27 +08:00
|
|
|
struct drm_i915_reg_table;
|
2015-01-15 21:10:38 +08:00
|
|
|
|
2015-06-20 02:07:01 +08:00
|
|
|
/*
|
|
|
|
* we use a single page to load ctx workarounds so all of these
|
|
|
|
* values are referred in terms of dwords
|
|
|
|
*
|
|
|
|
* struct i915_wa_ctx_bb:
|
|
|
|
* offset: specifies batch starting position, also helpful in case
|
|
|
|
* if we want to have multiple batches at different offsets based on
|
|
|
|
* some criteria. It is not a requirement at the moment but provides
|
|
|
|
* an option for future use.
|
|
|
|
* size: size of the batch in DWORDS
|
|
|
|
*/
|
|
|
|
struct i915_ctx_workarounds {
|
|
|
|
struct i915_wa_ctx_bb {
|
|
|
|
u32 offset;
|
|
|
|
u32 size;
|
|
|
|
} indirect_ctx, per_ctx;
|
|
|
|
struct drm_i915_gem_object *obj;
|
|
|
|
};
|
|
|
|
|
2016-07-02 00:23:25 +08:00
|
|
|
struct drm_i915_gem_request;
|
|
|
|
|
2016-05-06 22:40:21 +08:00
|
|
|
struct intel_engine_cs {
|
|
|
|
struct drm_i915_private *i915;
|
2010-05-21 09:08:55 +08:00
|
|
|
const char *name;
|
2016-03-16 19:00:40 +08:00
|
|
|
enum intel_engine_id {
|
2016-01-15 23:12:50 +08:00
|
|
|
RCS = 0,
|
2011-12-14 20:57:00 +08:00
|
|
|
BCS,
|
2016-01-15 23:12:50 +08:00
|
|
|
VCS,
|
|
|
|
VCS2, /* Keep instances of the same type engine together. */
|
|
|
|
VECS
|
2010-09-18 18:02:01 +08:00
|
|
|
} id;
|
2016-03-16 19:00:39 +08:00
|
|
|
#define I915_NUM_ENGINES 5
|
2016-01-15 23:12:50 +08:00
|
|
|
#define _VCS(n) (VCS + (n))
|
2016-01-16 00:51:46 +08:00
|
|
|
unsigned int exec_id;
|
2016-04-29 20:18:23 +08:00
|
|
|
unsigned int hw_id;
|
|
|
|
unsigned int guc_id; /* XXX same as hw_id? */
|
2016-07-20 16:21:11 +08:00
|
|
|
u64 fence_context;
|
2010-08-02 22:24:01 +08:00
|
|
|
u32 mmio_base;
|
2016-07-13 23:03:35 +08:00
|
|
|
unsigned int irq_shift;
|
2016-08-03 05:50:21 +08:00
|
|
|
struct intel_ring *buffer;
|
2015-09-03 20:01:40 +08:00
|
|
|
struct list_head buffers;
|
2010-05-21 09:08:55 +08:00
|
|
|
|
drm/i915: Slaughter the thundering i915_wait_request herd
One particularly stressful scenario consists of many independent tasks
all competing for GPU time and waiting upon the results (e.g. realtime
transcoding of many, many streams). One bottleneck in particular is that
each client waits on its own results, but every client is woken up after
every batchbuffer - hence the thunder of hooves as then every client must
do its heavyweight dance to read a coherent seqno to see if it is the
lucky one.
Ideally, we only want one client to wake up after the interrupt and
check its request for completion. Since the requests must retire in
order, we can select the first client on the oldest request to be woken.
Once that client has completed his wait, we can then wake up the
next client and so on. However, all clients then incur latency as every
process in the chain may be delayed for scheduling - this may also then
cause some priority inversion. To reduce the latency, when a client
is added or removed from the list, we scan the tree for completed
seqno and wake up all the completed waiters in parallel.
Using igt/benchmarks/gem_latency, we can demonstrate this effect. The
benchmark measures the number of GPU cycles between completion of a
batch and the client waking up from a call to wait-ioctl. With many
concurrent waiters, with each on a different request, we observe that
the wakeup latency before the patch scales nearly linearly with the
number of waiters (before external factors kick in making the scaling much
worse). After applying the patch, we can see that only the single waiter
for the request is being woken up, providing a constant wakeup latency
for every operation. However, the situation is not quite as rosy for
many waiters on the same request, though to the best of my knowledge this
is much less likely in practice. Here, we can observe that the
concurrent waiters incur extra latency from being woken up by the
solitary bottom-half, rather than directly by the interrupt. This
appears to be scheduler induced (having discounted adverse effects from
having a rbtree walk/erase in the wakeup path), each additional
wake_up_process() costs approximately 1us on big core. Another effect of
performing the secondary wakeups from the first bottom-half is the
incurred delay this imposes on high priority threads - rather than
immediately returning to userspace and leaving the interrupt handler to
wake the others.
To offset the delay incurred with additional waiters on a request, we
could use a hybrid scheme that did a quick read in the interrupt handler
and dequeued all the completed waiters (incurring the overhead in the
interrupt handler, not the best plan either as we then incur GPU
submission latency) but we would still have to wake up the bottom-half
every time to do the heavyweight slow read. Or we could only kick the
waiters on the seqno with the same priority as the current task (i.e. in
the realtime waiter scenario, only it is woken up immediately by the
interrupt and simply queues the next waiter before returning to userspace,
minimising its delay at the expense of the chain, and also reducing
contention on its scheduler runqueue). This is effective at avoid long
pauses in the interrupt handler and at avoiding the extra latency in
realtime/high-priority waiters.
v2: Convert from a kworker per engine into a dedicated kthread for the
bottom-half.
v3: Rename request members and tweak comments.
v4: Use a per-engine spinlock in the breadcrumbs bottom-half.
v5: Fix race in locklessly checking waiter status and kicking the task on
adding a new waiter.
v6: Fix deciding when to force the timer to hide missing interrupts.
v7: Move the bottom-half from the kthread to the first client process.
v8: Reword a few comments
v9: Break the busy loop when the interrupt is unmasked or has fired.
v10: Comments, unnecessary churn, better debugging from Tvrtko
v11: Wake all completed waiters on removing the current bottom-half to
reduce the latency of waking up a herd of clients all waiting on the
same request.
v12: Rearrange missed-interrupt fault injection so that it works with
igt/drv_missed_irq_hang
v13: Rename intel_breadcrumb and friends to intel_wait in preparation
for signal handling.
v14: RCU commentary, assert_spin_locked
v15: Hide BUG_ON behind the compiler; report on gem_latency findings.
v16: Sort seqno-groups by priority so that first-waiter has the highest
task priority (and so avoid priority inversion).
v17: Add waiters to post-mortem GPU hang state.
v18: Return early for a completed wait after acquiring the spinlock.
Avoids adding ourselves to the tree if the is already complete, and
skips the awkward question of why we don't do completion wakeups for
waits earlier than or equal to ourselves.
v19: Prepare for init_breadcrumbs to fail. Later patches may want to
allocate during init, so be prepared to propagate back the error code.
Testcase: igt/gem_concurrent_blit
Testcase: igt/benchmarks/gem_latency
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: "Gong, Zhipeng" <zhipeng.gong@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Dave Gordon <david.s.gordon@intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com> #v18
Link: http://patchwork.freedesktop.org/patch/msgid/1467390209-3576-6-git-send-email-chris@chris-wilson.co.uk
2016-07-02 00:23:15 +08:00
|
|
|
/* Rather than have every client wait upon all user interrupts,
|
|
|
|
* with the herd waking after every interrupt and each doing the
|
|
|
|
* heavyweight seqno dance, we delegate the task (of being the
|
|
|
|
* bottom-half of the user interrupt) to the first client. After
|
|
|
|
* every interrupt, we wake up one client, who does the heavyweight
|
|
|
|
* coherent seqno read and either goes back to sleep (if incomplete),
|
|
|
|
* or wakes up all the completed clients in parallel, before then
|
|
|
|
* transferring the bottom-half status to the next client in the queue.
|
|
|
|
*
|
|
|
|
* Compared to walking the entire list of waiters in a single dedicated
|
|
|
|
* bottom-half, we reduce the latency of the first waiter by avoiding
|
|
|
|
* a context switch, but incur additional coherent seqno reads when
|
|
|
|
* following the chain of request breadcrumbs. Since it is most likely
|
|
|
|
* that we have a single client waiting on each seqno, then reducing
|
|
|
|
* the overhead of waking that client is much preferred.
|
|
|
|
*/
|
|
|
|
struct intel_breadcrumbs {
|
2016-08-10 00:47:52 +08:00
|
|
|
struct task_struct __rcu *irq_seqno_bh; /* bh for interrupts */
|
2016-07-06 19:39:02 +08:00
|
|
|
bool irq_posted;
|
|
|
|
|
drm/i915: Slaughter the thundering i915_wait_request herd
One particularly stressful scenario consists of many independent tasks
all competing for GPU time and waiting upon the results (e.g. realtime
transcoding of many, many streams). One bottleneck in particular is that
each client waits on its own results, but every client is woken up after
every batchbuffer - hence the thunder of hooves as then every client must
do its heavyweight dance to read a coherent seqno to see if it is the
lucky one.
Ideally, we only want one client to wake up after the interrupt and
check its request for completion. Since the requests must retire in
order, we can select the first client on the oldest request to be woken.
Once that client has completed his wait, we can then wake up the
next client and so on. However, all clients then incur latency as every
process in the chain may be delayed for scheduling - this may also then
cause some priority inversion. To reduce the latency, when a client
is added or removed from the list, we scan the tree for completed
seqno and wake up all the completed waiters in parallel.
Using igt/benchmarks/gem_latency, we can demonstrate this effect. The
benchmark measures the number of GPU cycles between completion of a
batch and the client waking up from a call to wait-ioctl. With many
concurrent waiters, with each on a different request, we observe that
the wakeup latency before the patch scales nearly linearly with the
number of waiters (before external factors kick in making the scaling much
worse). After applying the patch, we can see that only the single waiter
for the request is being woken up, providing a constant wakeup latency
for every operation. However, the situation is not quite as rosy for
many waiters on the same request, though to the best of my knowledge this
is much less likely in practice. Here, we can observe that the
concurrent waiters incur extra latency from being woken up by the
solitary bottom-half, rather than directly by the interrupt. This
appears to be scheduler induced (having discounted adverse effects from
having a rbtree walk/erase in the wakeup path), each additional
wake_up_process() costs approximately 1us on big core. Another effect of
performing the secondary wakeups from the first bottom-half is the
incurred delay this imposes on high priority threads - rather than
immediately returning to userspace and leaving the interrupt handler to
wake the others.
To offset the delay incurred with additional waiters on a request, we
could use a hybrid scheme that did a quick read in the interrupt handler
and dequeued all the completed waiters (incurring the overhead in the
interrupt handler, not the best plan either as we then incur GPU
submission latency) but we would still have to wake up the bottom-half
every time to do the heavyweight slow read. Or we could only kick the
waiters on the seqno with the same priority as the current task (i.e. in
the realtime waiter scenario, only it is woken up immediately by the
interrupt and simply queues the next waiter before returning to userspace,
minimising its delay at the expense of the chain, and also reducing
contention on its scheduler runqueue). This is effective at avoid long
pauses in the interrupt handler and at avoiding the extra latency in
realtime/high-priority waiters.
v2: Convert from a kworker per engine into a dedicated kthread for the
bottom-half.
v3: Rename request members and tweak comments.
v4: Use a per-engine spinlock in the breadcrumbs bottom-half.
v5: Fix race in locklessly checking waiter status and kicking the task on
adding a new waiter.
v6: Fix deciding when to force the timer to hide missing interrupts.
v7: Move the bottom-half from the kthread to the first client process.
v8: Reword a few comments
v9: Break the busy loop when the interrupt is unmasked or has fired.
v10: Comments, unnecessary churn, better debugging from Tvrtko
v11: Wake all completed waiters on removing the current bottom-half to
reduce the latency of waking up a herd of clients all waiting on the
same request.
v12: Rearrange missed-interrupt fault injection so that it works with
igt/drv_missed_irq_hang
v13: Rename intel_breadcrumb and friends to intel_wait in preparation
for signal handling.
v14: RCU commentary, assert_spin_locked
v15: Hide BUG_ON behind the compiler; report on gem_latency findings.
v16: Sort seqno-groups by priority so that first-waiter has the highest
task priority (and so avoid priority inversion).
v17: Add waiters to post-mortem GPU hang state.
v18: Return early for a completed wait after acquiring the spinlock.
Avoids adding ourselves to the tree if the is already complete, and
skips the awkward question of why we don't do completion wakeups for
waits earlier than or equal to ourselves.
v19: Prepare for init_breadcrumbs to fail. Later patches may want to
allocate during init, so be prepared to propagate back the error code.
Testcase: igt/gem_concurrent_blit
Testcase: igt/benchmarks/gem_latency
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: "Gong, Zhipeng" <zhipeng.gong@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Dave Gordon <david.s.gordon@intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com> #v18
Link: http://patchwork.freedesktop.org/patch/msgid/1467390209-3576-6-git-send-email-chris@chris-wilson.co.uk
2016-07-02 00:23:15 +08:00
|
|
|
spinlock_t lock; /* protects the lists of requests */
|
|
|
|
struct rb_root waiters; /* sorted by retirement, priority */
|
2016-07-02 00:23:25 +08:00
|
|
|
struct rb_root signals; /* sorted by retirement */
|
drm/i915: Slaughter the thundering i915_wait_request herd
One particularly stressful scenario consists of many independent tasks
all competing for GPU time and waiting upon the results (e.g. realtime
transcoding of many, many streams). One bottleneck in particular is that
each client waits on its own results, but every client is woken up after
every batchbuffer - hence the thunder of hooves as then every client must
do its heavyweight dance to read a coherent seqno to see if it is the
lucky one.
Ideally, we only want one client to wake up after the interrupt and
check its request for completion. Since the requests must retire in
order, we can select the first client on the oldest request to be woken.
Once that client has completed his wait, we can then wake up the
next client and so on. However, all clients then incur latency as every
process in the chain may be delayed for scheduling - this may also then
cause some priority inversion. To reduce the latency, when a client
is added or removed from the list, we scan the tree for completed
seqno and wake up all the completed waiters in parallel.
Using igt/benchmarks/gem_latency, we can demonstrate this effect. The
benchmark measures the number of GPU cycles between completion of a
batch and the client waking up from a call to wait-ioctl. With many
concurrent waiters, with each on a different request, we observe that
the wakeup latency before the patch scales nearly linearly with the
number of waiters (before external factors kick in making the scaling much
worse). After applying the patch, we can see that only the single waiter
for the request is being woken up, providing a constant wakeup latency
for every operation. However, the situation is not quite as rosy for
many waiters on the same request, though to the best of my knowledge this
is much less likely in practice. Here, we can observe that the
concurrent waiters incur extra latency from being woken up by the
solitary bottom-half, rather than directly by the interrupt. This
appears to be scheduler induced (having discounted adverse effects from
having a rbtree walk/erase in the wakeup path), each additional
wake_up_process() costs approximately 1us on big core. Another effect of
performing the secondary wakeups from the first bottom-half is the
incurred delay this imposes on high priority threads - rather than
immediately returning to userspace and leaving the interrupt handler to
wake the others.
To offset the delay incurred with additional waiters on a request, we
could use a hybrid scheme that did a quick read in the interrupt handler
and dequeued all the completed waiters (incurring the overhead in the
interrupt handler, not the best plan either as we then incur GPU
submission latency) but we would still have to wake up the bottom-half
every time to do the heavyweight slow read. Or we could only kick the
waiters on the seqno with the same priority as the current task (i.e. in
the realtime waiter scenario, only it is woken up immediately by the
interrupt and simply queues the next waiter before returning to userspace,
minimising its delay at the expense of the chain, and also reducing
contention on its scheduler runqueue). This is effective at avoid long
pauses in the interrupt handler and at avoiding the extra latency in
realtime/high-priority waiters.
v2: Convert from a kworker per engine into a dedicated kthread for the
bottom-half.
v3: Rename request members and tweak comments.
v4: Use a per-engine spinlock in the breadcrumbs bottom-half.
v5: Fix race in locklessly checking waiter status and kicking the task on
adding a new waiter.
v6: Fix deciding when to force the timer to hide missing interrupts.
v7: Move the bottom-half from the kthread to the first client process.
v8: Reword a few comments
v9: Break the busy loop when the interrupt is unmasked or has fired.
v10: Comments, unnecessary churn, better debugging from Tvrtko
v11: Wake all completed waiters on removing the current bottom-half to
reduce the latency of waking up a herd of clients all waiting on the
same request.
v12: Rearrange missed-interrupt fault injection so that it works with
igt/drv_missed_irq_hang
v13: Rename intel_breadcrumb and friends to intel_wait in preparation
for signal handling.
v14: RCU commentary, assert_spin_locked
v15: Hide BUG_ON behind the compiler; report on gem_latency findings.
v16: Sort seqno-groups by priority so that first-waiter has the highest
task priority (and so avoid priority inversion).
v17: Add waiters to post-mortem GPU hang state.
v18: Return early for a completed wait after acquiring the spinlock.
Avoids adding ourselves to the tree if the is already complete, and
skips the awkward question of why we don't do completion wakeups for
waits earlier than or equal to ourselves.
v19: Prepare for init_breadcrumbs to fail. Later patches may want to
allocate during init, so be prepared to propagate back the error code.
Testcase: igt/gem_concurrent_blit
Testcase: igt/benchmarks/gem_latency
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: "Gong, Zhipeng" <zhipeng.gong@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Dave Gordon <david.s.gordon@intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com> #v18
Link: http://patchwork.freedesktop.org/patch/msgid/1467390209-3576-6-git-send-email-chris@chris-wilson.co.uk
2016-07-02 00:23:15 +08:00
|
|
|
struct intel_wait *first_wait; /* oldest waiter by retirement */
|
2016-07-02 00:23:25 +08:00
|
|
|
struct task_struct *signaler; /* used for fence signalling */
|
2016-07-02 00:23:26 +08:00
|
|
|
struct drm_i915_gem_request *first_signal;
|
drm/i915: Slaughter the thundering i915_wait_request herd
One particularly stressful scenario consists of many independent tasks
all competing for GPU time and waiting upon the results (e.g. realtime
transcoding of many, many streams). One bottleneck in particular is that
each client waits on its own results, but every client is woken up after
every batchbuffer - hence the thunder of hooves as then every client must
do its heavyweight dance to read a coherent seqno to see if it is the
lucky one.
Ideally, we only want one client to wake up after the interrupt and
check its request for completion. Since the requests must retire in
order, we can select the first client on the oldest request to be woken.
Once that client has completed his wait, we can then wake up the
next client and so on. However, all clients then incur latency as every
process in the chain may be delayed for scheduling - this may also then
cause some priority inversion. To reduce the latency, when a client
is added or removed from the list, we scan the tree for completed
seqno and wake up all the completed waiters in parallel.
Using igt/benchmarks/gem_latency, we can demonstrate this effect. The
benchmark measures the number of GPU cycles between completion of a
batch and the client waking up from a call to wait-ioctl. With many
concurrent waiters, with each on a different request, we observe that
the wakeup latency before the patch scales nearly linearly with the
number of waiters (before external factors kick in making the scaling much
worse). After applying the patch, we can see that only the single waiter
for the request is being woken up, providing a constant wakeup latency
for every operation. However, the situation is not quite as rosy for
many waiters on the same request, though to the best of my knowledge this
is much less likely in practice. Here, we can observe that the
concurrent waiters incur extra latency from being woken up by the
solitary bottom-half, rather than directly by the interrupt. This
appears to be scheduler induced (having discounted adverse effects from
having a rbtree walk/erase in the wakeup path), each additional
wake_up_process() costs approximately 1us on big core. Another effect of
performing the secondary wakeups from the first bottom-half is the
incurred delay this imposes on high priority threads - rather than
immediately returning to userspace and leaving the interrupt handler to
wake the others.
To offset the delay incurred with additional waiters on a request, we
could use a hybrid scheme that did a quick read in the interrupt handler
and dequeued all the completed waiters (incurring the overhead in the
interrupt handler, not the best plan either as we then incur GPU
submission latency) but we would still have to wake up the bottom-half
every time to do the heavyweight slow read. Or we could only kick the
waiters on the seqno with the same priority as the current task (i.e. in
the realtime waiter scenario, only it is woken up immediately by the
interrupt and simply queues the next waiter before returning to userspace,
minimising its delay at the expense of the chain, and also reducing
contention on its scheduler runqueue). This is effective at avoid long
pauses in the interrupt handler and at avoiding the extra latency in
realtime/high-priority waiters.
v2: Convert from a kworker per engine into a dedicated kthread for the
bottom-half.
v3: Rename request members and tweak comments.
v4: Use a per-engine spinlock in the breadcrumbs bottom-half.
v5: Fix race in locklessly checking waiter status and kicking the task on
adding a new waiter.
v6: Fix deciding when to force the timer to hide missing interrupts.
v7: Move the bottom-half from the kthread to the first client process.
v8: Reword a few comments
v9: Break the busy loop when the interrupt is unmasked or has fired.
v10: Comments, unnecessary churn, better debugging from Tvrtko
v11: Wake all completed waiters on removing the current bottom-half to
reduce the latency of waking up a herd of clients all waiting on the
same request.
v12: Rearrange missed-interrupt fault injection so that it works with
igt/drv_missed_irq_hang
v13: Rename intel_breadcrumb and friends to intel_wait in preparation
for signal handling.
v14: RCU commentary, assert_spin_locked
v15: Hide BUG_ON behind the compiler; report on gem_latency findings.
v16: Sort seqno-groups by priority so that first-waiter has the highest
task priority (and so avoid priority inversion).
v17: Add waiters to post-mortem GPU hang state.
v18: Return early for a completed wait after acquiring the spinlock.
Avoids adding ourselves to the tree if the is already complete, and
skips the awkward question of why we don't do completion wakeups for
waits earlier than or equal to ourselves.
v19: Prepare for init_breadcrumbs to fail. Later patches may want to
allocate during init, so be prepared to propagate back the error code.
Testcase: igt/gem_concurrent_blit
Testcase: igt/benchmarks/gem_latency
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: "Gong, Zhipeng" <zhipeng.gong@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Dave Gordon <david.s.gordon@intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com> #v18
Link: http://patchwork.freedesktop.org/patch/msgid/1467390209-3576-6-git-send-email-chris@chris-wilson.co.uk
2016-07-02 00:23:15 +08:00
|
|
|
struct timer_list fake_irq; /* used after a missed interrupt */
|
2016-08-10 00:47:51 +08:00
|
|
|
struct timer_list hangcheck; /* detect missed interrupts */
|
|
|
|
|
|
|
|
unsigned long timeout;
|
2016-07-06 19:39:02 +08:00
|
|
|
|
|
|
|
bool irq_enabled : 1;
|
|
|
|
bool rpm_wakelock : 1;
|
drm/i915: Slaughter the thundering i915_wait_request herd
One particularly stressful scenario consists of many independent tasks
all competing for GPU time and waiting upon the results (e.g. realtime
transcoding of many, many streams). One bottleneck in particular is that
each client waits on its own results, but every client is woken up after
every batchbuffer - hence the thunder of hooves as then every client must
do its heavyweight dance to read a coherent seqno to see if it is the
lucky one.
Ideally, we only want one client to wake up after the interrupt and
check its request for completion. Since the requests must retire in
order, we can select the first client on the oldest request to be woken.
Once that client has completed his wait, we can then wake up the
next client and so on. However, all clients then incur latency as every
process in the chain may be delayed for scheduling - this may also then
cause some priority inversion. To reduce the latency, when a client
is added or removed from the list, we scan the tree for completed
seqno and wake up all the completed waiters in parallel.
Using igt/benchmarks/gem_latency, we can demonstrate this effect. The
benchmark measures the number of GPU cycles between completion of a
batch and the client waking up from a call to wait-ioctl. With many
concurrent waiters, with each on a different request, we observe that
the wakeup latency before the patch scales nearly linearly with the
number of waiters (before external factors kick in making the scaling much
worse). After applying the patch, we can see that only the single waiter
for the request is being woken up, providing a constant wakeup latency
for every operation. However, the situation is not quite as rosy for
many waiters on the same request, though to the best of my knowledge this
is much less likely in practice. Here, we can observe that the
concurrent waiters incur extra latency from being woken up by the
solitary bottom-half, rather than directly by the interrupt. This
appears to be scheduler induced (having discounted adverse effects from
having a rbtree walk/erase in the wakeup path), each additional
wake_up_process() costs approximately 1us on big core. Another effect of
performing the secondary wakeups from the first bottom-half is the
incurred delay this imposes on high priority threads - rather than
immediately returning to userspace and leaving the interrupt handler to
wake the others.
To offset the delay incurred with additional waiters on a request, we
could use a hybrid scheme that did a quick read in the interrupt handler
and dequeued all the completed waiters (incurring the overhead in the
interrupt handler, not the best plan either as we then incur GPU
submission latency) but we would still have to wake up the bottom-half
every time to do the heavyweight slow read. Or we could only kick the
waiters on the seqno with the same priority as the current task (i.e. in
the realtime waiter scenario, only it is woken up immediately by the
interrupt and simply queues the next waiter before returning to userspace,
minimising its delay at the expense of the chain, and also reducing
contention on its scheduler runqueue). This is effective at avoid long
pauses in the interrupt handler and at avoiding the extra latency in
realtime/high-priority waiters.
v2: Convert from a kworker per engine into a dedicated kthread for the
bottom-half.
v3: Rename request members and tweak comments.
v4: Use a per-engine spinlock in the breadcrumbs bottom-half.
v5: Fix race in locklessly checking waiter status and kicking the task on
adding a new waiter.
v6: Fix deciding when to force the timer to hide missing interrupts.
v7: Move the bottom-half from the kthread to the first client process.
v8: Reword a few comments
v9: Break the busy loop when the interrupt is unmasked or has fired.
v10: Comments, unnecessary churn, better debugging from Tvrtko
v11: Wake all completed waiters on removing the current bottom-half to
reduce the latency of waking up a herd of clients all waiting on the
same request.
v12: Rearrange missed-interrupt fault injection so that it works with
igt/drv_missed_irq_hang
v13: Rename intel_breadcrumb and friends to intel_wait in preparation
for signal handling.
v14: RCU commentary, assert_spin_locked
v15: Hide BUG_ON behind the compiler; report on gem_latency findings.
v16: Sort seqno-groups by priority so that first-waiter has the highest
task priority (and so avoid priority inversion).
v17: Add waiters to post-mortem GPU hang state.
v18: Return early for a completed wait after acquiring the spinlock.
Avoids adding ourselves to the tree if the is already complete, and
skips the awkward question of why we don't do completion wakeups for
waits earlier than or equal to ourselves.
v19: Prepare for init_breadcrumbs to fail. Later patches may want to
allocate during init, so be prepared to propagate back the error code.
Testcase: igt/gem_concurrent_blit
Testcase: igt/benchmarks/gem_latency
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: "Gong, Zhipeng" <zhipeng.gong@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Dave Gordon <david.s.gordon@intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com> #v18
Link: http://patchwork.freedesktop.org/patch/msgid/1467390209-3576-6-git-send-email-chris@chris-wilson.co.uk
2016-07-02 00:23:15 +08:00
|
|
|
} breadcrumbs;
|
|
|
|
|
2015-04-07 23:20:36 +08:00
|
|
|
/*
|
|
|
|
* A pool of objects to use as shadow copies of client batch buffers
|
|
|
|
* when the command parser is enabled. Prevents the client from
|
|
|
|
* modifying the batch contents after software parsing.
|
|
|
|
*/
|
|
|
|
struct i915_gem_batch_pool batch_pool;
|
|
|
|
|
2010-05-21 09:08:55 +08:00
|
|
|
struct intel_hw_status_page status_page;
|
2015-06-20 02:07:01 +08:00
|
|
|
struct i915_ctx_workarounds wa_ctx;
|
2010-05-21 09:08:55 +08:00
|
|
|
|
2016-07-02 00:23:28 +08:00
|
|
|
u32 irq_keep_mask; /* always keep these interrupts */
|
|
|
|
u32 irq_enable_mask; /* bitmask to enable ring interrupt */
|
2016-07-21 01:16:06 +08:00
|
|
|
void (*irq_enable)(struct intel_engine_cs *engine);
|
|
|
|
void (*irq_disable)(struct intel_engine_cs *engine);
|
2010-05-21 09:08:55 +08:00
|
|
|
|
2016-07-21 01:16:06 +08:00
|
|
|
int (*init_hw)(struct intel_engine_cs *engine);
|
2010-05-21 09:08:55 +08:00
|
|
|
|
2015-05-30 00:43:44 +08:00
|
|
|
int (*init_context)(struct drm_i915_gem_request *req);
|
2014-08-26 21:44:50 +08:00
|
|
|
|
2016-08-03 05:50:31 +08:00
|
|
|
int (*emit_flush)(struct drm_i915_gem_request *request,
|
|
|
|
u32 mode);
|
|
|
|
#define EMIT_INVALIDATE BIT(0)
|
|
|
|
#define EMIT_FLUSH BIT(1)
|
|
|
|
#define EMIT_BARRIER (EMIT_INVALIDATE | EMIT_FLUSH)
|
|
|
|
int (*emit_bb_start)(struct drm_i915_gem_request *req,
|
|
|
|
u64 offset, u32 length,
|
|
|
|
unsigned int dispatch_flags);
|
|
|
|
#define I915_DISPATCH_SECURE BIT(0)
|
|
|
|
#define I915_DISPATCH_PINNED BIT(1)
|
|
|
|
#define I915_DISPATCH_RS BIT(2)
|
|
|
|
int (*emit_request)(struct drm_i915_gem_request *req);
|
|
|
|
void (*submit_request)(struct drm_i915_gem_request *req);
|
2012-08-09 17:58:30 +08:00
|
|
|
/* Some chipsets are not quite as coherent as advertised and need
|
|
|
|
* an expensive kick to force a true read of the up-to-date seqno.
|
|
|
|
* However, the up-to-date seqno is not always required and the last
|
|
|
|
* seen value is good enough. Note that the seqno will always be
|
|
|
|
* monotonic, even if not coherent.
|
|
|
|
*/
|
2016-07-21 01:16:06 +08:00
|
|
|
void (*irq_seqno_barrier)(struct intel_engine_cs *engine);
|
|
|
|
void (*cleanup)(struct intel_engine_cs *engine);
|
2014-04-30 05:52:28 +08:00
|
|
|
|
2014-07-01 00:53:37 +08:00
|
|
|
/* GEN8 signal/wait table - never trust comments!
|
|
|
|
* signal to signal to signal to signal to signal to
|
|
|
|
* RCS VCS BCS VECS VCS2
|
|
|
|
* --------------------------------------------------------------------
|
|
|
|
* RCS | NOP (0x00) | VCS (0x08) | BCS (0x10) | VECS (0x18) | VCS2 (0x20) |
|
|
|
|
* |-------------------------------------------------------------------
|
|
|
|
* VCS | RCS (0x28) | NOP (0x30) | BCS (0x38) | VECS (0x40) | VCS2 (0x48) |
|
|
|
|
* |-------------------------------------------------------------------
|
|
|
|
* BCS | RCS (0x50) | VCS (0x58) | NOP (0x60) | VECS (0x68) | VCS2 (0x70) |
|
|
|
|
* |-------------------------------------------------------------------
|
|
|
|
* VECS | RCS (0x78) | VCS (0x80) | BCS (0x88) | NOP (0x90) | VCS2 (0x98) |
|
|
|
|
* |-------------------------------------------------------------------
|
|
|
|
* VCS2 | RCS (0xa0) | VCS (0xa8) | BCS (0xb0) | VECS (0xb8) | NOP (0xc0) |
|
|
|
|
* |-------------------------------------------------------------------
|
|
|
|
*
|
|
|
|
* Generalization:
|
|
|
|
* f(x, y) := (x->id * NUM_RINGS * seqno_size) + (seqno_size * y->id)
|
|
|
|
* ie. transpose of g(x, y)
|
|
|
|
*
|
|
|
|
* sync from sync from sync from sync from sync from
|
|
|
|
* RCS VCS BCS VECS VCS2
|
|
|
|
* --------------------------------------------------------------------
|
|
|
|
* RCS | NOP (0x00) | VCS (0x28) | BCS (0x50) | VECS (0x78) | VCS2 (0xa0) |
|
|
|
|
* |-------------------------------------------------------------------
|
|
|
|
* VCS | RCS (0x08) | NOP (0x30) | BCS (0x58) | VECS (0x80) | VCS2 (0xa8) |
|
|
|
|
* |-------------------------------------------------------------------
|
|
|
|
* BCS | RCS (0x10) | VCS (0x38) | NOP (0x60) | VECS (0x88) | VCS2 (0xb0) |
|
|
|
|
* |-------------------------------------------------------------------
|
|
|
|
* VECS | RCS (0x18) | VCS (0x40) | BCS (0x68) | NOP (0x90) | VCS2 (0xb8) |
|
|
|
|
* |-------------------------------------------------------------------
|
|
|
|
* VCS2 | RCS (0x20) | VCS (0x48) | BCS (0x70) | VECS (0x98) | NOP (0xc0) |
|
|
|
|
* |-------------------------------------------------------------------
|
|
|
|
*
|
|
|
|
* Generalization:
|
|
|
|
* g(x, y) := (y->id * NUM_RINGS * seqno_size) + (seqno_size * x->id)
|
|
|
|
* ie. transpose of f(x, y)
|
|
|
|
*/
|
2014-04-30 05:52:28 +08:00
|
|
|
struct {
|
2016-03-16 19:00:39 +08:00
|
|
|
u32 sync_seqno[I915_NUM_ENGINES-1];
|
2014-04-30 05:52:29 +08:00
|
|
|
|
2014-07-01 00:53:37 +08:00
|
|
|
union {
|
|
|
|
struct {
|
|
|
|
/* our mbox written by others */
|
2016-03-16 19:00:39 +08:00
|
|
|
u32 wait[I915_NUM_ENGINES];
|
2014-07-01 00:53:37 +08:00
|
|
|
/* mboxes this ring signals to */
|
2016-03-16 19:00:39 +08:00
|
|
|
i915_reg_t signal[I915_NUM_ENGINES];
|
2014-07-01 00:53:37 +08:00
|
|
|
} mbox;
|
2016-03-16 19:00:39 +08:00
|
|
|
u64 signal_ggtt[I915_NUM_ENGINES];
|
2014-07-01 00:53:37 +08:00
|
|
|
};
|
2014-04-30 05:52:29 +08:00
|
|
|
|
|
|
|
/* AKA wait() */
|
2016-08-03 05:50:40 +08:00
|
|
|
int (*sync_to)(struct drm_i915_gem_request *req,
|
|
|
|
struct drm_i915_gem_request *signal);
|
|
|
|
int (*signal)(struct drm_i915_gem_request *req);
|
2014-04-30 05:52:28 +08:00
|
|
|
} semaphore;
|
2013-05-29 10:22:18 +08:00
|
|
|
|
2014-07-25 00:04:27 +08:00
|
|
|
/* Execlists */
|
drm/i915: Move execlists irq handler to a bottom half
Doing a lot of work in the interrupt handler introduces huge
latencies to the system as a whole.
Most dramatic effect can be seen by running an all engine
stress test like igt/gem_exec_nop/all where, when the kernel
config is lean enough, the whole system can be brought into
multi-second periods of complete non-interactivty. That can
look for example like this:
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [kworker/u8:3:143]
Modules linked in: [redacted for brevity]
CPU: 0 PID: 143 Comm: kworker/u8:3 Tainted: G U L 4.5.0-160321+ #183
Hardware name: Intel Corporation Broadwell Client platform/WhiteTip Mountain 1
Workqueue: i915 gen6_pm_rps_work [i915]
task: ffff8800aae88000 ti: ffff8800aae90000 task.ti: ffff8800aae90000
RIP: 0010:[<ffffffff8104a3c2>] [<ffffffff8104a3c2>] __do_softirq+0x72/0x1d0
RSP: 0000:ffff88014f403f38 EFLAGS: 00000206
RAX: ffff8800aae94000 RBX: 0000000000000000 RCX: 00000000000006e0
RDX: 0000000000000020 RSI: 0000000004208060 RDI: 0000000000215d80
RBP: ffff88014f403f80 R08: 0000000b1b42c180 R09: 0000000000000022
R10: 0000000000000004 R11: 00000000ffffffff R12: 000000000000a030
R13: 0000000000000082 R14: ffff8800aa4d0080 R15: 0000000000000082
FS: 0000000000000000(0000) GS:ffff88014f400000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa53b90c000 CR3: 0000000001a0a000 CR4: 00000000001406f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
042080601b33869f ffff8800aae94000 00000000fffc2678 ffff88010000000a
0000000000000000 000000000000a030 0000000000005302 ffff8800aa4d0080
0000000000000206 ffff88014f403f90 ffffffff8104a716 ffff88014f403fa8
Call Trace:
<IRQ>
[<ffffffff8104a716>] irq_exit+0x86/0x90
[<ffffffff81031e7d>] smp_apic_timer_interrupt+0x3d/0x50
[<ffffffff814f3eac>] apic_timer_interrupt+0x7c/0x90
<EOI>
[<ffffffffa01c5b40>] ? gen8_write64+0x1a0/0x1a0 [i915]
[<ffffffff814f2b39>] ? _raw_spin_unlock_irqrestore+0x9/0x20
[<ffffffffa01c5c44>] gen8_write32+0x104/0x1a0 [i915]
[<ffffffff8132c6a2>] ? n_tty_receive_buf_common+0x372/0xae0
[<ffffffffa017cc9e>] gen6_set_rps_thresholds+0x1be/0x330 [i915]
[<ffffffffa017eaf0>] gen6_set_rps+0x70/0x200 [i915]
[<ffffffffa0185375>] intel_set_rps+0x25/0x30 [i915]
[<ffffffffa01768fd>] gen6_pm_rps_work+0x10d/0x2e0 [i915]
[<ffffffff81063852>] ? finish_task_switch+0x72/0x1c0
[<ffffffff8105ab29>] process_one_work+0x139/0x350
[<ffffffff8105b186>] worker_thread+0x126/0x490
[<ffffffff8105b060>] ? rescuer_thread+0x320/0x320
[<ffffffff8105fa64>] kthread+0xc4/0xe0
[<ffffffff8105f9a0>] ? kthread_create_on_node+0x170/0x170
[<ffffffff814f351f>] ret_from_fork+0x3f/0x70
[<ffffffff8105f9a0>] ? kthread_create_on_node+0x170/0x170
I could not explain, or find a code path, which would explain
a +20 second lockup, but from some instrumentation it was
apparent the interrupts off proportion of time was between
10-25% under heavy load which is quite bad.
When a interrupt "cliff" is reached, which was >~320k irq/s on
my machine, the whole system goes into a terrible state of the
above described multi-second lockups.
By moving the GT interrupt handling to a tasklet in a most
simple way, the problem above disappears completely.
Testing the effect on sytem-wide latencies using
igt/gem_syslatency shows the following before this patch:
gem_syslatency: cycles=1532739, latency mean=416531.829us max=2499237us
gem_syslatency: cycles=1839434, latency mean=1458099.157us max=4998944us
gem_syslatency: cycles=1432570, latency mean=2688.451us max=1201185us
gem_syslatency: cycles=1533543, latency mean=416520.499us max=2498886us
This shows that the unrelated process is experiencing huge
delays in its wake-up latency. After the patch the results
look like this:
gem_syslatency: cycles=808907, latency mean=53.133us max=1640us
gem_syslatency: cycles=862154, latency mean=62.778us max=2117us
gem_syslatency: cycles=856039, latency mean=58.079us max=2123us
gem_syslatency: cycles=841683, latency mean=56.914us max=1667us
Showing a huge improvement in the unrelated process wake-up
latency. It also shows an approximate halving in the number
of total empty batches submitted during the test. This may
not be worrying since the test puts the driver under
a very unrealistic load with ncpu threads doing empty batch
submission to all GPU engines each.
Another benefit compared to the hard-irq handling is that now
work on all engines can be dispatched in parallel since we can
have up to number of CPUs active tasklets. (While previously
a single hard-irq would serially dispatch on one engine after
another.)
More interesting scenario with regards to throughput is
"gem_latency -n 100" which shows 25% better throughput and
CPU usage, and 14% better dispatch latencies.
I did not find any gains or regressions with Synmark2 or
GLbench under light testing. More benchmarking is certainly
required.
v2:
* execlists_lock should be taken as spin_lock_bh when
queuing work from userspace now. (Chris Wilson)
* uncore.lock must be taken with spin_lock_irq when
submitting requests since that now runs from either
softirq or process context.
v3:
* Expanded commit message with more testing data;
* converted missed locking sites to _bh;
* added execlist_lock comment. (Chris Wilson)
v4:
* Mention dispatch parallelism in commit. (Chris Wilson)
* Do not hold uncore.lock over MMIO reads since the block
is already serialised per-engine via the tasklet itself.
(Chris Wilson)
* intel_lrc_irq_handler should be static. (Chris Wilson)
* Cancel/sync the tasklet on GPU reset. (Chris Wilson)
* Document and WARN that tasklet cannot be active/pending
on engine cleanup. (Chris Wilson/Imre Deak)
Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Imre Deak <imre.deak@intel.com>
Testcase: igt/gem_exec_nop/all
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=94350
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: http://patchwork.freedesktop.org/patch/msgid/1459768316-6670-1-git-send-email-tvrtko.ursulin@linux.intel.com
2016-04-04 19:11:56 +08:00
|
|
|
struct tasklet_struct irq_tasklet;
|
|
|
|
spinlock_t execlist_lock; /* used inside tasklet, use spin_lock_bh */
|
2014-07-25 00:04:38 +08:00
|
|
|
struct list_head execlist_queue;
|
2016-04-12 21:37:31 +08:00
|
|
|
unsigned int fw_domains;
|
2016-02-27 00:58:32 +08:00
|
|
|
unsigned int next_context_status_buffer;
|
|
|
|
unsigned int idle_lite_restore_wa;
|
2016-01-15 23:10:27 +08:00
|
|
|
bool disable_lite_restore_wa;
|
|
|
|
u32 ctx_desc_template;
|
2014-07-25 00:04:27 +08:00
|
|
|
|
2010-05-21 09:08:55 +08:00
|
|
|
/**
|
|
|
|
* List of breadcrumbs associated with GPU requests currently
|
|
|
|
* outstanding.
|
|
|
|
*/
|
|
|
|
struct list_head request_list;
|
|
|
|
|
2015-07-09 22:30:57 +08:00
|
|
|
/**
|
|
|
|
* Seqno of request most recently submitted to request_list.
|
|
|
|
* Used exclusively by hang checker to avoid grabbing lock while
|
|
|
|
* inspecting request list.
|
|
|
|
*/
|
|
|
|
u32 last_submitted_seqno;
|
|
|
|
|
2016-08-05 17:14:11 +08:00
|
|
|
/* An RCU guarded pointer to the last request. No reference is
|
|
|
|
* held to the request, users must carefully acquire a reference to
|
2016-08-10 00:03:22 +08:00
|
|
|
* the request using i915_gem_active_get_rcu(), or hold the
|
2016-08-05 17:14:11 +08:00
|
|
|
* struct_mutex.
|
|
|
|
*/
|
|
|
|
struct i915_gem_active last_request;
|
|
|
|
|
2016-05-24 21:53:34 +08:00
|
|
|
struct i915_gem_context *last_context;
|
2012-06-05 05:42:43 +08:00
|
|
|
|
2016-08-03 05:50:21 +08:00
|
|
|
struct intel_engine_hangcheck hangcheck;
|
2013-05-24 22:16:07 +08:00
|
|
|
|
2013-08-27 03:58:11 +08:00
|
|
|
struct {
|
|
|
|
struct drm_i915_gem_object *obj;
|
|
|
|
u32 gtt_offset;
|
|
|
|
} scratch;
|
2014-02-19 02:15:46 +08:00
|
|
|
|
2014-05-11 05:10:43 +08:00
|
|
|
bool needs_cmd_parser;
|
|
|
|
|
2014-02-19 02:15:46 +08:00
|
|
|
/*
|
2014-05-11 05:10:43 +08:00
|
|
|
* Table of commands the command parser needs to know about
|
2016-07-27 16:07:26 +08:00
|
|
|
* for this engine.
|
2014-02-19 02:15:46 +08:00
|
|
|
*/
|
2014-05-11 05:10:43 +08:00
|
|
|
DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER);
|
2014-02-19 02:15:46 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Table of registers allowed in commands that read/write registers.
|
|
|
|
*/
|
2016-03-07 15:30:27 +08:00
|
|
|
const struct drm_i915_reg_table *reg_tables;
|
|
|
|
int reg_table_count;
|
2014-02-19 02:15:46 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Returns the bitmask for the length field of the specified command.
|
|
|
|
* Return 0 for an unrecognized/invalid command.
|
|
|
|
*
|
2016-07-27 16:07:26 +08:00
|
|
|
* If the command parser finds an entry for a command in the engine's
|
2014-02-19 02:15:46 +08:00
|
|
|
* cmd_tables, it gets the command's length based on the table entry.
|
2016-07-27 16:07:26 +08:00
|
|
|
* If not, it calls this function to determine the per-engine length
|
|
|
|
* field encoding for the command (i.e. different opcode ranges use
|
|
|
|
* certain bits to encode the command length in the header).
|
2014-02-19 02:15:46 +08:00
|
|
|
*/
|
|
|
|
u32 (*get_cmd_length_mask)(u32 cmd_header);
|
2010-05-21 09:08:55 +08:00
|
|
|
};
|
|
|
|
|
2015-12-08 23:02:36 +08:00
|
|
|
static inline bool
|
2016-07-04 15:08:31 +08:00
|
|
|
intel_engine_initialized(const struct intel_engine_cs *engine)
|
2015-12-08 23:02:36 +08:00
|
|
|
{
|
2016-05-06 22:40:21 +08:00
|
|
|
return engine->i915 != NULL;
|
2015-12-08 23:02:36 +08:00
|
|
|
}
|
2012-05-11 21:29:30 +08:00
|
|
|
|
2011-12-14 20:57:00 +08:00
|
|
|
static inline unsigned
|
2016-07-04 15:08:31 +08:00
|
|
|
intel_engine_flag(const struct intel_engine_cs *engine)
|
2011-12-14 20:57:00 +08:00
|
|
|
{
|
2016-03-16 19:00:37 +08:00
|
|
|
return 1 << engine->id;
|
2011-12-14 20:57:00 +08:00
|
|
|
}
|
|
|
|
|
2010-12-04 19:30:53 +08:00
|
|
|
static inline u32
|
2016-08-03 05:50:21 +08:00
|
|
|
intel_engine_sync_index(struct intel_engine_cs *engine,
|
|
|
|
struct intel_engine_cs *other)
|
2010-12-04 19:30:53 +08:00
|
|
|
{
|
|
|
|
int idx;
|
|
|
|
|
|
|
|
/*
|
2014-07-01 00:51:11 +08:00
|
|
|
* rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2;
|
|
|
|
* vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs;
|
|
|
|
* bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs;
|
|
|
|
* vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs;
|
|
|
|
* vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs;
|
2010-12-04 19:30:53 +08:00
|
|
|
*/
|
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
idx = (other - engine) - 1;
|
2010-12-04 19:30:53 +08:00
|
|
|
if (idx < 0)
|
2016-03-16 19:00:39 +08:00
|
|
|
idx += I915_NUM_ENGINES;
|
2010-12-04 19:30:53 +08:00
|
|
|
|
|
|
|
return idx;
|
|
|
|
}
|
|
|
|
|
2015-08-14 23:35:27 +08:00
|
|
|
static inline void
|
2016-03-16 19:00:37 +08:00
|
|
|
intel_flush_status_page(struct intel_engine_cs *engine, int reg)
|
2015-08-14 23:35:27 +08:00
|
|
|
{
|
2016-04-09 17:57:56 +08:00
|
|
|
mb();
|
|
|
|
clflush(&engine->status_page.page_addr[reg]);
|
|
|
|
mb();
|
2015-08-14 23:35:27 +08:00
|
|
|
}
|
|
|
|
|
2010-05-21 09:08:55 +08:00
|
|
|
static inline u32
|
2016-04-09 17:57:57 +08:00
|
|
|
intel_read_status_page(struct intel_engine_cs *engine, int reg)
|
2010-05-21 09:08:55 +08:00
|
|
|
{
|
2012-04-27 05:28:16 +08:00
|
|
|
/* Ensure that the compiler doesn't optimize away the load. */
|
2016-04-09 17:57:57 +08:00
|
|
|
return READ_ONCE(engine->status_page.page_addr[reg]);
|
2010-05-21 09:08:55 +08:00
|
|
|
}
|
|
|
|
|
2012-12-19 17:13:05 +08:00
|
|
|
static inline void
|
2016-03-16 19:00:37 +08:00
|
|
|
intel_write_status_page(struct intel_engine_cs *engine,
|
2012-12-19 17:13:05 +08:00
|
|
|
int reg, u32 value)
|
|
|
|
{
|
2016-03-16 19:00:37 +08:00
|
|
|
engine->status_page.page_addr[reg] = value;
|
2012-12-19 17:13:05 +08:00
|
|
|
}
|
|
|
|
|
2016-01-18 15:19:47 +08:00
|
|
|
/*
|
2011-01-14 03:06:50 +08:00
|
|
|
* Reads a dword out of the status page, which is written to from the command
|
|
|
|
* queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
|
|
|
|
* MI_STORE_DATA_IMM.
|
|
|
|
*
|
|
|
|
* The following dwords have a reserved meaning:
|
|
|
|
* 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes.
|
|
|
|
* 0x04: ring 0 head pointer
|
|
|
|
* 0x05: ring 1 head pointer (915-class)
|
|
|
|
* 0x06: ring 2 head pointer (915-class)
|
|
|
|
* 0x10-0x1b: Context status DWords (GM45)
|
|
|
|
* 0x1f: Last written status offset. (GM45)
|
2015-02-18 19:48:21 +08:00
|
|
|
* 0x20-0x2f: Reserved (Gen6+)
|
2011-01-14 03:06:50 +08:00
|
|
|
*
|
2015-02-18 19:48:21 +08:00
|
|
|
* The area from dword 0x30 to 0x3ff is available for driver usage.
|
2011-01-14 03:06:50 +08:00
|
|
|
*/
|
2015-02-18 19:48:21 +08:00
|
|
|
#define I915_GEM_HWS_INDEX 0x30
|
2016-01-20 21:43:35 +08:00
|
|
|
#define I915_GEM_HWS_INDEX_ADDR (I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
|
2015-02-18 19:48:21 +08:00
|
|
|
#define I915_GEM_HWS_SCRATCH_INDEX 0x40
|
2012-10-27 00:42:42 +08:00
|
|
|
#define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
|
2011-01-14 03:06:50 +08:00
|
|
|
|
2016-08-03 05:50:21 +08:00
|
|
|
struct intel_ring *
|
|
|
|
intel_engine_create_ring(struct intel_engine_cs *engine, int size);
|
2016-08-03 05:50:23 +08:00
|
|
|
int intel_ring_pin(struct intel_ring *ring);
|
|
|
|
void intel_ring_unpin(struct intel_ring *ring);
|
2016-08-03 05:50:21 +08:00
|
|
|
void intel_ring_free(struct intel_ring *ring);
|
2014-07-25 00:04:15 +08:00
|
|
|
|
2016-08-03 05:50:21 +08:00
|
|
|
void intel_engine_stop(struct intel_engine_cs *engine);
|
|
|
|
void intel_engine_cleanup(struct intel_engine_cs *engine);
|
2011-03-20 09:14:27 +08:00
|
|
|
|
2015-03-19 20:30:08 +08:00
|
|
|
int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request);
|
|
|
|
|
2015-05-30 00:44:07 +08:00
|
|
|
int __must_check intel_ring_begin(struct drm_i915_gem_request *req, int n);
|
2015-05-30 00:44:06 +08:00
|
|
|
int __must_check intel_ring_cacheline_align(struct drm_i915_gem_request *req);
|
2016-07-20 20:31:55 +08:00
|
|
|
|
2016-08-03 05:50:21 +08:00
|
|
|
static inline void intel_ring_emit(struct intel_ring *ring, u32 data)
|
2016-07-20 20:31:55 +08:00
|
|
|
{
|
2016-08-03 05:50:18 +08:00
|
|
|
*(uint32_t *)(ring->vaddr + ring->tail) = data;
|
|
|
|
ring->tail += 4;
|
2016-07-20 20:31:55 +08:00
|
|
|
}
|
|
|
|
|
2016-08-03 05:50:21 +08:00
|
|
|
static inline void intel_ring_emit_reg(struct intel_ring *ring, i915_reg_t reg)
|
2015-11-05 05:20:07 +08:00
|
|
|
{
|
2016-08-03 05:50:18 +08:00
|
|
|
intel_ring_emit(ring, i915_mmio_reg_offset(reg));
|
2015-11-05 05:20:07 +08:00
|
|
|
}
|
2016-07-20 20:31:55 +08:00
|
|
|
|
2016-08-03 05:50:21 +08:00
|
|
|
static inline void intel_ring_advance(struct intel_ring *ring)
|
2013-08-11 05:16:32 +08:00
|
|
|
{
|
2016-08-03 05:50:30 +08:00
|
|
|
/* Dummy function.
|
|
|
|
*
|
|
|
|
* This serves as a placeholder in the code so that the reader
|
|
|
|
* can compare against the preceding intel_ring_begin() and
|
|
|
|
* check that the number of dwords emitted matches the space
|
|
|
|
* reserved for the command packet (i.e. the value passed to
|
|
|
|
* intel_ring_begin()).
|
2016-08-03 05:50:29 +08:00
|
|
|
*/
|
2016-08-03 05:50:30 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline u32 intel_ring_offset(struct intel_ring *ring, u32 value)
|
|
|
|
{
|
|
|
|
/* Don't write ring->size (equivalent to 0) as that hangs some GPUs. */
|
|
|
|
return value & (ring->size - 1);
|
2013-08-11 05:16:32 +08:00
|
|
|
}
|
2016-07-20 20:31:55 +08:00
|
|
|
|
2014-07-25 00:04:26 +08:00
|
|
|
int __intel_ring_space(int head, int tail, int size);
|
2016-08-03 05:50:22 +08:00
|
|
|
void intel_ring_update_space(struct intel_ring *ring);
|
2013-08-11 05:16:32 +08:00
|
|
|
|
2016-08-03 05:50:21 +08:00
|
|
|
void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno);
|
2010-05-21 09:08:55 +08:00
|
|
|
|
2016-07-02 00:23:20 +08:00
|
|
|
int intel_init_pipe_control(struct intel_engine_cs *engine, int size);
|
2016-03-16 19:00:37 +08:00
|
|
|
void intel_fini_pipe_control(struct intel_engine_cs *engine);
|
2014-07-25 00:04:24 +08:00
|
|
|
|
2016-07-13 23:03:41 +08:00
|
|
|
void intel_engine_setup_common(struct intel_engine_cs *engine);
|
|
|
|
int intel_engine_init_common(struct intel_engine_cs *engine);
|
2016-08-03 20:19:16 +08:00
|
|
|
void intel_engine_cleanup_common(struct intel_engine_cs *engine);
|
2016-07-13 23:03:41 +08:00
|
|
|
|
2016-08-05 17:14:11 +08:00
|
|
|
static inline int intel_engine_idle(struct intel_engine_cs *engine,
|
|
|
|
bool interruptible)
|
|
|
|
{
|
|
|
|
/* Wait upon the last request to be completed */
|
|
|
|
return i915_gem_active_wait_unlocked(&engine->last_request,
|
|
|
|
interruptible, NULL, NULL);
|
|
|
|
}
|
|
|
|
|
2016-07-13 23:03:37 +08:00
|
|
|
int intel_init_render_ring_buffer(struct intel_engine_cs *engine);
|
|
|
|
int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine);
|
|
|
|
int intel_init_bsd2_ring_buffer(struct intel_engine_cs *engine);
|
|
|
|
int intel_init_blt_ring_buffer(struct intel_engine_cs *engine);
|
|
|
|
int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine);
|
2010-05-21 09:08:55 +08:00
|
|
|
|
2016-08-03 05:50:21 +08:00
|
|
|
u64 intel_engine_get_active_head(struct intel_engine_cs *engine);
|
2016-07-02 00:23:17 +08:00
|
|
|
static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
|
|
|
|
}
|
2010-09-25 03:20:10 +08:00
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
int init_workarounds_ring(struct intel_engine_cs *engine);
|
2014-11-12 00:47:33 +08:00
|
|
|
|
drm/i915: Reserve ring buffer space for i915_add_request() commands
It is a bad idea for i915_add_request() to fail. The work will already have been
send to the ring and will be processed, but there will not be any tracking or
management of that work.
The only way the add request call can fail is if it can't write its epilogue
commands to the ring (cache flushing, seqno updates, interrupt signalling). The
reasons for that are mostly down to running out of ring buffer space and the
problems associated with trying to get some more. This patch prevents that
situation from happening in the first place.
When a request is created, it marks sufficient space as reserved for the
epilogue commands. Thus guaranteeing that by the time the epilogue is written,
there will be plenty of space for it. Note that a ring_begin() call is required
to actually reserve the space (and do any potential waiting). However, that is
not currently done at request creation time. This is because the ring_begin()
code can allocate a request. Hence calling begin() from the request allocation
code would lead to infinite recursion! Later patches in this series remove the
need for begin() to do the allocate. At that point, it becomes safe for the
allocate to call begin() and really reserve the space.
Until then, there is a potential for insufficient space to be available at the
point of calling i915_add_request(). However, that would only be in the case
where the request was created and immediately submitted without ever calling
ring_begin() and adding any work to that request. Which should never happen. And
even if it does, and if that request happens to fall down the tiny window of
opportunity for failing due to being out of ring space then does it really
matter because the request wasn't doing anything in the first place?
v2: Updated the 'reserved space too small' warning to include the offending
sizes. Added a 'cancel' operation to clean up when a request is abandoned. Added
re-initialisation of tracking state after a buffer wrap to keep the sanity
checks accurate.
v3: Incremented the reserved size to accommodate Ironlake (after finally
managing to run on an ILK system). Also fixed missing wrap code in LRC mode.
v4: Added extra comment and removed duplicate WARN (feedback from Tomas).
For: VIZ-5115
CC: Tomas Elf <tomas.elf@intel.com>
Signed-off-by: John Harrison <John.C.Harrison@Intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2015-06-18 20:10:09 +08:00
|
|
|
/*
|
|
|
|
* Arbitrary size for largest possible 'add request' sequence. The code paths
|
|
|
|
* are complex and variable. Empirical measurement shows that the worst case
|
2016-04-29 16:07:04 +08:00
|
|
|
* is BDW at 192 bytes (6 + 6 + 36 dwords), then ILK at 136 bytes. However,
|
|
|
|
* we need to allocate double the largest single packet within that emission
|
|
|
|
* to account for tail wraparound (so 6 + 6 + 72 dwords for BDW).
|
drm/i915: Reserve ring buffer space for i915_add_request() commands
It is a bad idea for i915_add_request() to fail. The work will already have been
send to the ring and will be processed, but there will not be any tracking or
management of that work.
The only way the add request call can fail is if it can't write its epilogue
commands to the ring (cache flushing, seqno updates, interrupt signalling). The
reasons for that are mostly down to running out of ring buffer space and the
problems associated with trying to get some more. This patch prevents that
situation from happening in the first place.
When a request is created, it marks sufficient space as reserved for the
epilogue commands. Thus guaranteeing that by the time the epilogue is written,
there will be plenty of space for it. Note that a ring_begin() call is required
to actually reserve the space (and do any potential waiting). However, that is
not currently done at request creation time. This is because the ring_begin()
code can allocate a request. Hence calling begin() from the request allocation
code would lead to infinite recursion! Later patches in this series remove the
need for begin() to do the allocate. At that point, it becomes safe for the
allocate to call begin() and really reserve the space.
Until then, there is a potential for insufficient space to be available at the
point of calling i915_add_request(). However, that would only be in the case
where the request was created and immediately submitted without ever calling
ring_begin() and adding any work to that request. Which should never happen. And
even if it does, and if that request happens to fall down the tiny window of
opportunity for failing due to being out of ring space then does it really
matter because the request wasn't doing anything in the first place?
v2: Updated the 'reserved space too small' warning to include the offending
sizes. Added a 'cancel' operation to clean up when a request is abandoned. Added
re-initialisation of tracking state after a buffer wrap to keep the sanity
checks accurate.
v3: Incremented the reserved size to accommodate Ironlake (after finally
managing to run on an ILK system). Also fixed missing wrap code in LRC mode.
v4: Added extra comment and removed duplicate WARN (feedback from Tomas).
For: VIZ-5115
CC: Tomas Elf <tomas.elf@intel.com>
Signed-off-by: John Harrison <John.C.Harrison@Intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2015-06-18 20:10:09 +08:00
|
|
|
*/
|
2016-04-29 16:07:04 +08:00
|
|
|
#define MIN_SPACE_FOR_ADD_REQUEST 336
|
drm/i915: Reserve ring buffer space for i915_add_request() commands
It is a bad idea for i915_add_request() to fail. The work will already have been
send to the ring and will be processed, but there will not be any tracking or
management of that work.
The only way the add request call can fail is if it can't write its epilogue
commands to the ring (cache flushing, seqno updates, interrupt signalling). The
reasons for that are mostly down to running out of ring buffer space and the
problems associated with trying to get some more. This patch prevents that
situation from happening in the first place.
When a request is created, it marks sufficient space as reserved for the
epilogue commands. Thus guaranteeing that by the time the epilogue is written,
there will be plenty of space for it. Note that a ring_begin() call is required
to actually reserve the space (and do any potential waiting). However, that is
not currently done at request creation time. This is because the ring_begin()
code can allocate a request. Hence calling begin() from the request allocation
code would lead to infinite recursion! Later patches in this series remove the
need for begin() to do the allocate. At that point, it becomes safe for the
allocate to call begin() and really reserve the space.
Until then, there is a potential for insufficient space to be available at the
point of calling i915_add_request(). However, that would only be in the case
where the request was created and immediately submitted without ever calling
ring_begin() and adding any work to that request. Which should never happen. And
even if it does, and if that request happens to fall down the tiny window of
opportunity for failing due to being out of ring space then does it really
matter because the request wasn't doing anything in the first place?
v2: Updated the 'reserved space too small' warning to include the offending
sizes. Added a 'cancel' operation to clean up when a request is abandoned. Added
re-initialisation of tracking state after a buffer wrap to keep the sanity
checks accurate.
v3: Incremented the reserved size to accommodate Ironlake (after finally
managing to run on an ILK system). Also fixed missing wrap code in LRC mode.
v4: Added extra comment and removed duplicate WARN (feedback from Tomas).
For: VIZ-5115
CC: Tomas Elf <tomas.elf@intel.com>
Signed-off-by: John Harrison <John.C.Harrison@Intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2015-06-18 20:10:09 +08:00
|
|
|
|
2016-04-29 20:18:21 +08:00
|
|
|
static inline u32 intel_hws_seqno_address(struct intel_engine_cs *engine)
|
|
|
|
{
|
2016-08-15 17:48:57 +08:00
|
|
|
return engine->status_page.ggtt_offset + I915_GEM_HWS_INDEX_ADDR;
|
2016-04-29 20:18:21 +08:00
|
|
|
}
|
|
|
|
|
drm/i915: Slaughter the thundering i915_wait_request herd
One particularly stressful scenario consists of many independent tasks
all competing for GPU time and waiting upon the results (e.g. realtime
transcoding of many, many streams). One bottleneck in particular is that
each client waits on its own results, but every client is woken up after
every batchbuffer - hence the thunder of hooves as then every client must
do its heavyweight dance to read a coherent seqno to see if it is the
lucky one.
Ideally, we only want one client to wake up after the interrupt and
check its request for completion. Since the requests must retire in
order, we can select the first client on the oldest request to be woken.
Once that client has completed his wait, we can then wake up the
next client and so on. However, all clients then incur latency as every
process in the chain may be delayed for scheduling - this may also then
cause some priority inversion. To reduce the latency, when a client
is added or removed from the list, we scan the tree for completed
seqno and wake up all the completed waiters in parallel.
Using igt/benchmarks/gem_latency, we can demonstrate this effect. The
benchmark measures the number of GPU cycles between completion of a
batch and the client waking up from a call to wait-ioctl. With many
concurrent waiters, with each on a different request, we observe that
the wakeup latency before the patch scales nearly linearly with the
number of waiters (before external factors kick in making the scaling much
worse). After applying the patch, we can see that only the single waiter
for the request is being woken up, providing a constant wakeup latency
for every operation. However, the situation is not quite as rosy for
many waiters on the same request, though to the best of my knowledge this
is much less likely in practice. Here, we can observe that the
concurrent waiters incur extra latency from being woken up by the
solitary bottom-half, rather than directly by the interrupt. This
appears to be scheduler induced (having discounted adverse effects from
having a rbtree walk/erase in the wakeup path), each additional
wake_up_process() costs approximately 1us on big core. Another effect of
performing the secondary wakeups from the first bottom-half is the
incurred delay this imposes on high priority threads - rather than
immediately returning to userspace and leaving the interrupt handler to
wake the others.
To offset the delay incurred with additional waiters on a request, we
could use a hybrid scheme that did a quick read in the interrupt handler
and dequeued all the completed waiters (incurring the overhead in the
interrupt handler, not the best plan either as we then incur GPU
submission latency) but we would still have to wake up the bottom-half
every time to do the heavyweight slow read. Or we could only kick the
waiters on the seqno with the same priority as the current task (i.e. in
the realtime waiter scenario, only it is woken up immediately by the
interrupt and simply queues the next waiter before returning to userspace,
minimising its delay at the expense of the chain, and also reducing
contention on its scheduler runqueue). This is effective at avoid long
pauses in the interrupt handler and at avoiding the extra latency in
realtime/high-priority waiters.
v2: Convert from a kworker per engine into a dedicated kthread for the
bottom-half.
v3: Rename request members and tweak comments.
v4: Use a per-engine spinlock in the breadcrumbs bottom-half.
v5: Fix race in locklessly checking waiter status and kicking the task on
adding a new waiter.
v6: Fix deciding when to force the timer to hide missing interrupts.
v7: Move the bottom-half from the kthread to the first client process.
v8: Reword a few comments
v9: Break the busy loop when the interrupt is unmasked or has fired.
v10: Comments, unnecessary churn, better debugging from Tvrtko
v11: Wake all completed waiters on removing the current bottom-half to
reduce the latency of waking up a herd of clients all waiting on the
same request.
v12: Rearrange missed-interrupt fault injection so that it works with
igt/drv_missed_irq_hang
v13: Rename intel_breadcrumb and friends to intel_wait in preparation
for signal handling.
v14: RCU commentary, assert_spin_locked
v15: Hide BUG_ON behind the compiler; report on gem_latency findings.
v16: Sort seqno-groups by priority so that first-waiter has the highest
task priority (and so avoid priority inversion).
v17: Add waiters to post-mortem GPU hang state.
v18: Return early for a completed wait after acquiring the spinlock.
Avoids adding ourselves to the tree if the is already complete, and
skips the awkward question of why we don't do completion wakeups for
waits earlier than or equal to ourselves.
v19: Prepare for init_breadcrumbs to fail. Later patches may want to
allocate during init, so be prepared to propagate back the error code.
Testcase: igt/gem_concurrent_blit
Testcase: igt/benchmarks/gem_latency
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: "Gong, Zhipeng" <zhipeng.gong@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Dave Gordon <david.s.gordon@intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com> #v18
Link: http://patchwork.freedesktop.org/patch/msgid/1467390209-3576-6-git-send-email-chris@chris-wilson.co.uk
2016-07-02 00:23:15 +08:00
|
|
|
/* intel_breadcrumbs.c -- user interrupt bottom-half for waiters */
|
|
|
|
int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine);
|
|
|
|
|
|
|
|
static inline void intel_wait_init(struct intel_wait *wait, u32 seqno)
|
|
|
|
{
|
|
|
|
wait->tsk = current;
|
|
|
|
wait->seqno = seqno;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool intel_wait_complete(const struct intel_wait *wait)
|
|
|
|
{
|
|
|
|
return RB_EMPTY_NODE(&wait->node);
|
|
|
|
}
|
|
|
|
|
|
|
|
bool intel_engine_add_wait(struct intel_engine_cs *engine,
|
|
|
|
struct intel_wait *wait);
|
|
|
|
void intel_engine_remove_wait(struct intel_engine_cs *engine,
|
|
|
|
struct intel_wait *wait);
|
2016-07-02 00:23:26 +08:00
|
|
|
void intel_engine_enable_signaling(struct drm_i915_gem_request *request);
|
drm/i915: Slaughter the thundering i915_wait_request herd
One particularly stressful scenario consists of many independent tasks
all competing for GPU time and waiting upon the results (e.g. realtime
transcoding of many, many streams). One bottleneck in particular is that
each client waits on its own results, but every client is woken up after
every batchbuffer - hence the thunder of hooves as then every client must
do its heavyweight dance to read a coherent seqno to see if it is the
lucky one.
Ideally, we only want one client to wake up after the interrupt and
check its request for completion. Since the requests must retire in
order, we can select the first client on the oldest request to be woken.
Once that client has completed his wait, we can then wake up the
next client and so on. However, all clients then incur latency as every
process in the chain may be delayed for scheduling - this may also then
cause some priority inversion. To reduce the latency, when a client
is added or removed from the list, we scan the tree for completed
seqno and wake up all the completed waiters in parallel.
Using igt/benchmarks/gem_latency, we can demonstrate this effect. The
benchmark measures the number of GPU cycles between completion of a
batch and the client waking up from a call to wait-ioctl. With many
concurrent waiters, with each on a different request, we observe that
the wakeup latency before the patch scales nearly linearly with the
number of waiters (before external factors kick in making the scaling much
worse). After applying the patch, we can see that only the single waiter
for the request is being woken up, providing a constant wakeup latency
for every operation. However, the situation is not quite as rosy for
many waiters on the same request, though to the best of my knowledge this
is much less likely in practice. Here, we can observe that the
concurrent waiters incur extra latency from being woken up by the
solitary bottom-half, rather than directly by the interrupt. This
appears to be scheduler induced (having discounted adverse effects from
having a rbtree walk/erase in the wakeup path), each additional
wake_up_process() costs approximately 1us on big core. Another effect of
performing the secondary wakeups from the first bottom-half is the
incurred delay this imposes on high priority threads - rather than
immediately returning to userspace and leaving the interrupt handler to
wake the others.
To offset the delay incurred with additional waiters on a request, we
could use a hybrid scheme that did a quick read in the interrupt handler
and dequeued all the completed waiters (incurring the overhead in the
interrupt handler, not the best plan either as we then incur GPU
submission latency) but we would still have to wake up the bottom-half
every time to do the heavyweight slow read. Or we could only kick the
waiters on the seqno with the same priority as the current task (i.e. in
the realtime waiter scenario, only it is woken up immediately by the
interrupt and simply queues the next waiter before returning to userspace,
minimising its delay at the expense of the chain, and also reducing
contention on its scheduler runqueue). This is effective at avoid long
pauses in the interrupt handler and at avoiding the extra latency in
realtime/high-priority waiters.
v2: Convert from a kworker per engine into a dedicated kthread for the
bottom-half.
v3: Rename request members and tweak comments.
v4: Use a per-engine spinlock in the breadcrumbs bottom-half.
v5: Fix race in locklessly checking waiter status and kicking the task on
adding a new waiter.
v6: Fix deciding when to force the timer to hide missing interrupts.
v7: Move the bottom-half from the kthread to the first client process.
v8: Reword a few comments
v9: Break the busy loop when the interrupt is unmasked or has fired.
v10: Comments, unnecessary churn, better debugging from Tvrtko
v11: Wake all completed waiters on removing the current bottom-half to
reduce the latency of waking up a herd of clients all waiting on the
same request.
v12: Rearrange missed-interrupt fault injection so that it works with
igt/drv_missed_irq_hang
v13: Rename intel_breadcrumb and friends to intel_wait in preparation
for signal handling.
v14: RCU commentary, assert_spin_locked
v15: Hide BUG_ON behind the compiler; report on gem_latency findings.
v16: Sort seqno-groups by priority so that first-waiter has the highest
task priority (and so avoid priority inversion).
v17: Add waiters to post-mortem GPU hang state.
v18: Return early for a completed wait after acquiring the spinlock.
Avoids adding ourselves to the tree if the is already complete, and
skips the awkward question of why we don't do completion wakeups for
waits earlier than or equal to ourselves.
v19: Prepare for init_breadcrumbs to fail. Later patches may want to
allocate during init, so be prepared to propagate back the error code.
Testcase: igt/gem_concurrent_blit
Testcase: igt/benchmarks/gem_latency
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: "Gong, Zhipeng" <zhipeng.gong@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Dave Gordon <david.s.gordon@intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com> #v18
Link: http://patchwork.freedesktop.org/patch/msgid/1467390209-3576-6-git-send-email-chris@chris-wilson.co.uk
2016-07-02 00:23:15 +08:00
|
|
|
|
2016-08-10 00:47:52 +08:00
|
|
|
static inline bool intel_engine_has_waiter(const struct intel_engine_cs *engine)
|
drm/i915: Slaughter the thundering i915_wait_request herd
One particularly stressful scenario consists of many independent tasks
all competing for GPU time and waiting upon the results (e.g. realtime
transcoding of many, many streams). One bottleneck in particular is that
each client waits on its own results, but every client is woken up after
every batchbuffer - hence the thunder of hooves as then every client must
do its heavyweight dance to read a coherent seqno to see if it is the
lucky one.
Ideally, we only want one client to wake up after the interrupt and
check its request for completion. Since the requests must retire in
order, we can select the first client on the oldest request to be woken.
Once that client has completed his wait, we can then wake up the
next client and so on. However, all clients then incur latency as every
process in the chain may be delayed for scheduling - this may also then
cause some priority inversion. To reduce the latency, when a client
is added or removed from the list, we scan the tree for completed
seqno and wake up all the completed waiters in parallel.
Using igt/benchmarks/gem_latency, we can demonstrate this effect. The
benchmark measures the number of GPU cycles between completion of a
batch and the client waking up from a call to wait-ioctl. With many
concurrent waiters, with each on a different request, we observe that
the wakeup latency before the patch scales nearly linearly with the
number of waiters (before external factors kick in making the scaling much
worse). After applying the patch, we can see that only the single waiter
for the request is being woken up, providing a constant wakeup latency
for every operation. However, the situation is not quite as rosy for
many waiters on the same request, though to the best of my knowledge this
is much less likely in practice. Here, we can observe that the
concurrent waiters incur extra latency from being woken up by the
solitary bottom-half, rather than directly by the interrupt. This
appears to be scheduler induced (having discounted adverse effects from
having a rbtree walk/erase in the wakeup path), each additional
wake_up_process() costs approximately 1us on big core. Another effect of
performing the secondary wakeups from the first bottom-half is the
incurred delay this imposes on high priority threads - rather than
immediately returning to userspace and leaving the interrupt handler to
wake the others.
To offset the delay incurred with additional waiters on a request, we
could use a hybrid scheme that did a quick read in the interrupt handler
and dequeued all the completed waiters (incurring the overhead in the
interrupt handler, not the best plan either as we then incur GPU
submission latency) but we would still have to wake up the bottom-half
every time to do the heavyweight slow read. Or we could only kick the
waiters on the seqno with the same priority as the current task (i.e. in
the realtime waiter scenario, only it is woken up immediately by the
interrupt and simply queues the next waiter before returning to userspace,
minimising its delay at the expense of the chain, and also reducing
contention on its scheduler runqueue). This is effective at avoid long
pauses in the interrupt handler and at avoiding the extra latency in
realtime/high-priority waiters.
v2: Convert from a kworker per engine into a dedicated kthread for the
bottom-half.
v3: Rename request members and tweak comments.
v4: Use a per-engine spinlock in the breadcrumbs bottom-half.
v5: Fix race in locklessly checking waiter status and kicking the task on
adding a new waiter.
v6: Fix deciding when to force the timer to hide missing interrupts.
v7: Move the bottom-half from the kthread to the first client process.
v8: Reword a few comments
v9: Break the busy loop when the interrupt is unmasked or has fired.
v10: Comments, unnecessary churn, better debugging from Tvrtko
v11: Wake all completed waiters on removing the current bottom-half to
reduce the latency of waking up a herd of clients all waiting on the
same request.
v12: Rearrange missed-interrupt fault injection so that it works with
igt/drv_missed_irq_hang
v13: Rename intel_breadcrumb and friends to intel_wait in preparation
for signal handling.
v14: RCU commentary, assert_spin_locked
v15: Hide BUG_ON behind the compiler; report on gem_latency findings.
v16: Sort seqno-groups by priority so that first-waiter has the highest
task priority (and so avoid priority inversion).
v17: Add waiters to post-mortem GPU hang state.
v18: Return early for a completed wait after acquiring the spinlock.
Avoids adding ourselves to the tree if the is already complete, and
skips the awkward question of why we don't do completion wakeups for
waits earlier than or equal to ourselves.
v19: Prepare for init_breadcrumbs to fail. Later patches may want to
allocate during init, so be prepared to propagate back the error code.
Testcase: igt/gem_concurrent_blit
Testcase: igt/benchmarks/gem_latency
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: "Gong, Zhipeng" <zhipeng.gong@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Dave Gordon <david.s.gordon@intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com> #v18
Link: http://patchwork.freedesktop.org/patch/msgid/1467390209-3576-6-git-send-email-chris@chris-wilson.co.uk
2016-07-02 00:23:15 +08:00
|
|
|
{
|
2016-08-10 00:47:52 +08:00
|
|
|
return rcu_access_pointer(engine->breadcrumbs.irq_seqno_bh);
|
drm/i915: Slaughter the thundering i915_wait_request herd
One particularly stressful scenario consists of many independent tasks
all competing for GPU time and waiting upon the results (e.g. realtime
transcoding of many, many streams). One bottleneck in particular is that
each client waits on its own results, but every client is woken up after
every batchbuffer - hence the thunder of hooves as then every client must
do its heavyweight dance to read a coherent seqno to see if it is the
lucky one.
Ideally, we only want one client to wake up after the interrupt and
check its request for completion. Since the requests must retire in
order, we can select the first client on the oldest request to be woken.
Once that client has completed his wait, we can then wake up the
next client and so on. However, all clients then incur latency as every
process in the chain may be delayed for scheduling - this may also then
cause some priority inversion. To reduce the latency, when a client
is added or removed from the list, we scan the tree for completed
seqno and wake up all the completed waiters in parallel.
Using igt/benchmarks/gem_latency, we can demonstrate this effect. The
benchmark measures the number of GPU cycles between completion of a
batch and the client waking up from a call to wait-ioctl. With many
concurrent waiters, with each on a different request, we observe that
the wakeup latency before the patch scales nearly linearly with the
number of waiters (before external factors kick in making the scaling much
worse). After applying the patch, we can see that only the single waiter
for the request is being woken up, providing a constant wakeup latency
for every operation. However, the situation is not quite as rosy for
many waiters on the same request, though to the best of my knowledge this
is much less likely in practice. Here, we can observe that the
concurrent waiters incur extra latency from being woken up by the
solitary bottom-half, rather than directly by the interrupt. This
appears to be scheduler induced (having discounted adverse effects from
having a rbtree walk/erase in the wakeup path), each additional
wake_up_process() costs approximately 1us on big core. Another effect of
performing the secondary wakeups from the first bottom-half is the
incurred delay this imposes on high priority threads - rather than
immediately returning to userspace and leaving the interrupt handler to
wake the others.
To offset the delay incurred with additional waiters on a request, we
could use a hybrid scheme that did a quick read in the interrupt handler
and dequeued all the completed waiters (incurring the overhead in the
interrupt handler, not the best plan either as we then incur GPU
submission latency) but we would still have to wake up the bottom-half
every time to do the heavyweight slow read. Or we could only kick the
waiters on the seqno with the same priority as the current task (i.e. in
the realtime waiter scenario, only it is woken up immediately by the
interrupt and simply queues the next waiter before returning to userspace,
minimising its delay at the expense of the chain, and also reducing
contention on its scheduler runqueue). This is effective at avoid long
pauses in the interrupt handler and at avoiding the extra latency in
realtime/high-priority waiters.
v2: Convert from a kworker per engine into a dedicated kthread for the
bottom-half.
v3: Rename request members and tweak comments.
v4: Use a per-engine spinlock in the breadcrumbs bottom-half.
v5: Fix race in locklessly checking waiter status and kicking the task on
adding a new waiter.
v6: Fix deciding when to force the timer to hide missing interrupts.
v7: Move the bottom-half from the kthread to the first client process.
v8: Reword a few comments
v9: Break the busy loop when the interrupt is unmasked or has fired.
v10: Comments, unnecessary churn, better debugging from Tvrtko
v11: Wake all completed waiters on removing the current bottom-half to
reduce the latency of waking up a herd of clients all waiting on the
same request.
v12: Rearrange missed-interrupt fault injection so that it works with
igt/drv_missed_irq_hang
v13: Rename intel_breadcrumb and friends to intel_wait in preparation
for signal handling.
v14: RCU commentary, assert_spin_locked
v15: Hide BUG_ON behind the compiler; report on gem_latency findings.
v16: Sort seqno-groups by priority so that first-waiter has the highest
task priority (and so avoid priority inversion).
v17: Add waiters to post-mortem GPU hang state.
v18: Return early for a completed wait after acquiring the spinlock.
Avoids adding ourselves to the tree if the is already complete, and
skips the awkward question of why we don't do completion wakeups for
waits earlier than or equal to ourselves.
v19: Prepare for init_breadcrumbs to fail. Later patches may want to
allocate during init, so be prepared to propagate back the error code.
Testcase: igt/gem_concurrent_blit
Testcase: igt/benchmarks/gem_latency
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: "Gong, Zhipeng" <zhipeng.gong@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Dave Gordon <david.s.gordon@intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com> #v18
Link: http://patchwork.freedesktop.org/patch/msgid/1467390209-3576-6-git-send-email-chris@chris-wilson.co.uk
2016-07-02 00:23:15 +08:00
|
|
|
}
|
|
|
|
|
2016-08-10 00:47:52 +08:00
|
|
|
static inline bool intel_engine_wakeup(const struct intel_engine_cs *engine)
|
drm/i915: Slaughter the thundering i915_wait_request herd
One particularly stressful scenario consists of many independent tasks
all competing for GPU time and waiting upon the results (e.g. realtime
transcoding of many, many streams). One bottleneck in particular is that
each client waits on its own results, but every client is woken up after
every batchbuffer - hence the thunder of hooves as then every client must
do its heavyweight dance to read a coherent seqno to see if it is the
lucky one.
Ideally, we only want one client to wake up after the interrupt and
check its request for completion. Since the requests must retire in
order, we can select the first client on the oldest request to be woken.
Once that client has completed his wait, we can then wake up the
next client and so on. However, all clients then incur latency as every
process in the chain may be delayed for scheduling - this may also then
cause some priority inversion. To reduce the latency, when a client
is added or removed from the list, we scan the tree for completed
seqno and wake up all the completed waiters in parallel.
Using igt/benchmarks/gem_latency, we can demonstrate this effect. The
benchmark measures the number of GPU cycles between completion of a
batch and the client waking up from a call to wait-ioctl. With many
concurrent waiters, with each on a different request, we observe that
the wakeup latency before the patch scales nearly linearly with the
number of waiters (before external factors kick in making the scaling much
worse). After applying the patch, we can see that only the single waiter
for the request is being woken up, providing a constant wakeup latency
for every operation. However, the situation is not quite as rosy for
many waiters on the same request, though to the best of my knowledge this
is much less likely in practice. Here, we can observe that the
concurrent waiters incur extra latency from being woken up by the
solitary bottom-half, rather than directly by the interrupt. This
appears to be scheduler induced (having discounted adverse effects from
having a rbtree walk/erase in the wakeup path), each additional
wake_up_process() costs approximately 1us on big core. Another effect of
performing the secondary wakeups from the first bottom-half is the
incurred delay this imposes on high priority threads - rather than
immediately returning to userspace and leaving the interrupt handler to
wake the others.
To offset the delay incurred with additional waiters on a request, we
could use a hybrid scheme that did a quick read in the interrupt handler
and dequeued all the completed waiters (incurring the overhead in the
interrupt handler, not the best plan either as we then incur GPU
submission latency) but we would still have to wake up the bottom-half
every time to do the heavyweight slow read. Or we could only kick the
waiters on the seqno with the same priority as the current task (i.e. in
the realtime waiter scenario, only it is woken up immediately by the
interrupt and simply queues the next waiter before returning to userspace,
minimising its delay at the expense of the chain, and also reducing
contention on its scheduler runqueue). This is effective at avoid long
pauses in the interrupt handler and at avoiding the extra latency in
realtime/high-priority waiters.
v2: Convert from a kworker per engine into a dedicated kthread for the
bottom-half.
v3: Rename request members and tweak comments.
v4: Use a per-engine spinlock in the breadcrumbs bottom-half.
v5: Fix race in locklessly checking waiter status and kicking the task on
adding a new waiter.
v6: Fix deciding when to force the timer to hide missing interrupts.
v7: Move the bottom-half from the kthread to the first client process.
v8: Reword a few comments
v9: Break the busy loop when the interrupt is unmasked or has fired.
v10: Comments, unnecessary churn, better debugging from Tvrtko
v11: Wake all completed waiters on removing the current bottom-half to
reduce the latency of waking up a herd of clients all waiting on the
same request.
v12: Rearrange missed-interrupt fault injection so that it works with
igt/drv_missed_irq_hang
v13: Rename intel_breadcrumb and friends to intel_wait in preparation
for signal handling.
v14: RCU commentary, assert_spin_locked
v15: Hide BUG_ON behind the compiler; report on gem_latency findings.
v16: Sort seqno-groups by priority so that first-waiter has the highest
task priority (and so avoid priority inversion).
v17: Add waiters to post-mortem GPU hang state.
v18: Return early for a completed wait after acquiring the spinlock.
Avoids adding ourselves to the tree if the is already complete, and
skips the awkward question of why we don't do completion wakeups for
waits earlier than or equal to ourselves.
v19: Prepare for init_breadcrumbs to fail. Later patches may want to
allocate during init, so be prepared to propagate back the error code.
Testcase: igt/gem_concurrent_blit
Testcase: igt/benchmarks/gem_latency
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: "Gong, Zhipeng" <zhipeng.gong@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Dave Gordon <david.s.gordon@intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com> #v18
Link: http://patchwork.freedesktop.org/patch/msgid/1467390209-3576-6-git-send-email-chris@chris-wilson.co.uk
2016-07-02 00:23:15 +08:00
|
|
|
{
|
|
|
|
bool wakeup = false;
|
2016-08-10 00:47:52 +08:00
|
|
|
|
drm/i915: Slaughter the thundering i915_wait_request herd
One particularly stressful scenario consists of many independent tasks
all competing for GPU time and waiting upon the results (e.g. realtime
transcoding of many, many streams). One bottleneck in particular is that
each client waits on its own results, but every client is woken up after
every batchbuffer - hence the thunder of hooves as then every client must
do its heavyweight dance to read a coherent seqno to see if it is the
lucky one.
Ideally, we only want one client to wake up after the interrupt and
check its request for completion. Since the requests must retire in
order, we can select the first client on the oldest request to be woken.
Once that client has completed his wait, we can then wake up the
next client and so on. However, all clients then incur latency as every
process in the chain may be delayed for scheduling - this may also then
cause some priority inversion. To reduce the latency, when a client
is added or removed from the list, we scan the tree for completed
seqno and wake up all the completed waiters in parallel.
Using igt/benchmarks/gem_latency, we can demonstrate this effect. The
benchmark measures the number of GPU cycles between completion of a
batch and the client waking up from a call to wait-ioctl. With many
concurrent waiters, with each on a different request, we observe that
the wakeup latency before the patch scales nearly linearly with the
number of waiters (before external factors kick in making the scaling much
worse). After applying the patch, we can see that only the single waiter
for the request is being woken up, providing a constant wakeup latency
for every operation. However, the situation is not quite as rosy for
many waiters on the same request, though to the best of my knowledge this
is much less likely in practice. Here, we can observe that the
concurrent waiters incur extra latency from being woken up by the
solitary bottom-half, rather than directly by the interrupt. This
appears to be scheduler induced (having discounted adverse effects from
having a rbtree walk/erase in the wakeup path), each additional
wake_up_process() costs approximately 1us on big core. Another effect of
performing the secondary wakeups from the first bottom-half is the
incurred delay this imposes on high priority threads - rather than
immediately returning to userspace and leaving the interrupt handler to
wake the others.
To offset the delay incurred with additional waiters on a request, we
could use a hybrid scheme that did a quick read in the interrupt handler
and dequeued all the completed waiters (incurring the overhead in the
interrupt handler, not the best plan either as we then incur GPU
submission latency) but we would still have to wake up the bottom-half
every time to do the heavyweight slow read. Or we could only kick the
waiters on the seqno with the same priority as the current task (i.e. in
the realtime waiter scenario, only it is woken up immediately by the
interrupt and simply queues the next waiter before returning to userspace,
minimising its delay at the expense of the chain, and also reducing
contention on its scheduler runqueue). This is effective at avoid long
pauses in the interrupt handler and at avoiding the extra latency in
realtime/high-priority waiters.
v2: Convert from a kworker per engine into a dedicated kthread for the
bottom-half.
v3: Rename request members and tweak comments.
v4: Use a per-engine spinlock in the breadcrumbs bottom-half.
v5: Fix race in locklessly checking waiter status and kicking the task on
adding a new waiter.
v6: Fix deciding when to force the timer to hide missing interrupts.
v7: Move the bottom-half from the kthread to the first client process.
v8: Reword a few comments
v9: Break the busy loop when the interrupt is unmasked or has fired.
v10: Comments, unnecessary churn, better debugging from Tvrtko
v11: Wake all completed waiters on removing the current bottom-half to
reduce the latency of waking up a herd of clients all waiting on the
same request.
v12: Rearrange missed-interrupt fault injection so that it works with
igt/drv_missed_irq_hang
v13: Rename intel_breadcrumb and friends to intel_wait in preparation
for signal handling.
v14: RCU commentary, assert_spin_locked
v15: Hide BUG_ON behind the compiler; report on gem_latency findings.
v16: Sort seqno-groups by priority so that first-waiter has the highest
task priority (and so avoid priority inversion).
v17: Add waiters to post-mortem GPU hang state.
v18: Return early for a completed wait after acquiring the spinlock.
Avoids adding ourselves to the tree if the is already complete, and
skips the awkward question of why we don't do completion wakeups for
waits earlier than or equal to ourselves.
v19: Prepare for init_breadcrumbs to fail. Later patches may want to
allocate during init, so be prepared to propagate back the error code.
Testcase: igt/gem_concurrent_blit
Testcase: igt/benchmarks/gem_latency
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: "Gong, Zhipeng" <zhipeng.gong@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Dave Gordon <david.s.gordon@intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com> #v18
Link: http://patchwork.freedesktop.org/patch/msgid/1467390209-3576-6-git-send-email-chris@chris-wilson.co.uk
2016-07-02 00:23:15 +08:00
|
|
|
/* Note that for this not to dangerously chase a dangling pointer,
|
2016-08-10 00:47:52 +08:00
|
|
|
* we must hold the rcu_read_lock here.
|
drm/i915: Slaughter the thundering i915_wait_request herd
One particularly stressful scenario consists of many independent tasks
all competing for GPU time and waiting upon the results (e.g. realtime
transcoding of many, many streams). One bottleneck in particular is that
each client waits on its own results, but every client is woken up after
every batchbuffer - hence the thunder of hooves as then every client must
do its heavyweight dance to read a coherent seqno to see if it is the
lucky one.
Ideally, we only want one client to wake up after the interrupt and
check its request for completion. Since the requests must retire in
order, we can select the first client on the oldest request to be woken.
Once that client has completed his wait, we can then wake up the
next client and so on. However, all clients then incur latency as every
process in the chain may be delayed for scheduling - this may also then
cause some priority inversion. To reduce the latency, when a client
is added or removed from the list, we scan the tree for completed
seqno and wake up all the completed waiters in parallel.
Using igt/benchmarks/gem_latency, we can demonstrate this effect. The
benchmark measures the number of GPU cycles between completion of a
batch and the client waking up from a call to wait-ioctl. With many
concurrent waiters, with each on a different request, we observe that
the wakeup latency before the patch scales nearly linearly with the
number of waiters (before external factors kick in making the scaling much
worse). After applying the patch, we can see that only the single waiter
for the request is being woken up, providing a constant wakeup latency
for every operation. However, the situation is not quite as rosy for
many waiters on the same request, though to the best of my knowledge this
is much less likely in practice. Here, we can observe that the
concurrent waiters incur extra latency from being woken up by the
solitary bottom-half, rather than directly by the interrupt. This
appears to be scheduler induced (having discounted adverse effects from
having a rbtree walk/erase in the wakeup path), each additional
wake_up_process() costs approximately 1us on big core. Another effect of
performing the secondary wakeups from the first bottom-half is the
incurred delay this imposes on high priority threads - rather than
immediately returning to userspace and leaving the interrupt handler to
wake the others.
To offset the delay incurred with additional waiters on a request, we
could use a hybrid scheme that did a quick read in the interrupt handler
and dequeued all the completed waiters (incurring the overhead in the
interrupt handler, not the best plan either as we then incur GPU
submission latency) but we would still have to wake up the bottom-half
every time to do the heavyweight slow read. Or we could only kick the
waiters on the seqno with the same priority as the current task (i.e. in
the realtime waiter scenario, only it is woken up immediately by the
interrupt and simply queues the next waiter before returning to userspace,
minimising its delay at the expense of the chain, and also reducing
contention on its scheduler runqueue). This is effective at avoid long
pauses in the interrupt handler and at avoiding the extra latency in
realtime/high-priority waiters.
v2: Convert from a kworker per engine into a dedicated kthread for the
bottom-half.
v3: Rename request members and tweak comments.
v4: Use a per-engine spinlock in the breadcrumbs bottom-half.
v5: Fix race in locklessly checking waiter status and kicking the task on
adding a new waiter.
v6: Fix deciding when to force the timer to hide missing interrupts.
v7: Move the bottom-half from the kthread to the first client process.
v8: Reword a few comments
v9: Break the busy loop when the interrupt is unmasked or has fired.
v10: Comments, unnecessary churn, better debugging from Tvrtko
v11: Wake all completed waiters on removing the current bottom-half to
reduce the latency of waking up a herd of clients all waiting on the
same request.
v12: Rearrange missed-interrupt fault injection so that it works with
igt/drv_missed_irq_hang
v13: Rename intel_breadcrumb and friends to intel_wait in preparation
for signal handling.
v14: RCU commentary, assert_spin_locked
v15: Hide BUG_ON behind the compiler; report on gem_latency findings.
v16: Sort seqno-groups by priority so that first-waiter has the highest
task priority (and so avoid priority inversion).
v17: Add waiters to post-mortem GPU hang state.
v18: Return early for a completed wait after acquiring the spinlock.
Avoids adding ourselves to the tree if the is already complete, and
skips the awkward question of why we don't do completion wakeups for
waits earlier than or equal to ourselves.
v19: Prepare for init_breadcrumbs to fail. Later patches may want to
allocate during init, so be prepared to propagate back the error code.
Testcase: igt/gem_concurrent_blit
Testcase: igt/benchmarks/gem_latency
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: "Gong, Zhipeng" <zhipeng.gong@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Dave Gordon <david.s.gordon@intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com> #v18
Link: http://patchwork.freedesktop.org/patch/msgid/1467390209-3576-6-git-send-email-chris@chris-wilson.co.uk
2016-07-02 00:23:15 +08:00
|
|
|
*
|
|
|
|
* Also note that tsk is likely to be in !TASK_RUNNING state so an
|
|
|
|
* early test for tsk->state != TASK_RUNNING before wake_up_process()
|
|
|
|
* is unlikely to be beneficial.
|
|
|
|
*/
|
2016-08-10 00:47:52 +08:00
|
|
|
if (intel_engine_has_waiter(engine)) {
|
|
|
|
struct task_struct *tsk;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
tsk = rcu_dereference(engine->breadcrumbs.irq_seqno_bh);
|
|
|
|
if (tsk)
|
|
|
|
wakeup = wake_up_process(tsk);
|
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
|
|
|
|
drm/i915: Slaughter the thundering i915_wait_request herd
One particularly stressful scenario consists of many independent tasks
all competing for GPU time and waiting upon the results (e.g. realtime
transcoding of many, many streams). One bottleneck in particular is that
each client waits on its own results, but every client is woken up after
every batchbuffer - hence the thunder of hooves as then every client must
do its heavyweight dance to read a coherent seqno to see if it is the
lucky one.
Ideally, we only want one client to wake up after the interrupt and
check its request for completion. Since the requests must retire in
order, we can select the first client on the oldest request to be woken.
Once that client has completed his wait, we can then wake up the
next client and so on. However, all clients then incur latency as every
process in the chain may be delayed for scheduling - this may also then
cause some priority inversion. To reduce the latency, when a client
is added or removed from the list, we scan the tree for completed
seqno and wake up all the completed waiters in parallel.
Using igt/benchmarks/gem_latency, we can demonstrate this effect. The
benchmark measures the number of GPU cycles between completion of a
batch and the client waking up from a call to wait-ioctl. With many
concurrent waiters, with each on a different request, we observe that
the wakeup latency before the patch scales nearly linearly with the
number of waiters (before external factors kick in making the scaling much
worse). After applying the patch, we can see that only the single waiter
for the request is being woken up, providing a constant wakeup latency
for every operation. However, the situation is not quite as rosy for
many waiters on the same request, though to the best of my knowledge this
is much less likely in practice. Here, we can observe that the
concurrent waiters incur extra latency from being woken up by the
solitary bottom-half, rather than directly by the interrupt. This
appears to be scheduler induced (having discounted adverse effects from
having a rbtree walk/erase in the wakeup path), each additional
wake_up_process() costs approximately 1us on big core. Another effect of
performing the secondary wakeups from the first bottom-half is the
incurred delay this imposes on high priority threads - rather than
immediately returning to userspace and leaving the interrupt handler to
wake the others.
To offset the delay incurred with additional waiters on a request, we
could use a hybrid scheme that did a quick read in the interrupt handler
and dequeued all the completed waiters (incurring the overhead in the
interrupt handler, not the best plan either as we then incur GPU
submission latency) but we would still have to wake up the bottom-half
every time to do the heavyweight slow read. Or we could only kick the
waiters on the seqno with the same priority as the current task (i.e. in
the realtime waiter scenario, only it is woken up immediately by the
interrupt and simply queues the next waiter before returning to userspace,
minimising its delay at the expense of the chain, and also reducing
contention on its scheduler runqueue). This is effective at avoid long
pauses in the interrupt handler and at avoiding the extra latency in
realtime/high-priority waiters.
v2: Convert from a kworker per engine into a dedicated kthread for the
bottom-half.
v3: Rename request members and tweak comments.
v4: Use a per-engine spinlock in the breadcrumbs bottom-half.
v5: Fix race in locklessly checking waiter status and kicking the task on
adding a new waiter.
v6: Fix deciding when to force the timer to hide missing interrupts.
v7: Move the bottom-half from the kthread to the first client process.
v8: Reword a few comments
v9: Break the busy loop when the interrupt is unmasked or has fired.
v10: Comments, unnecessary churn, better debugging from Tvrtko
v11: Wake all completed waiters on removing the current bottom-half to
reduce the latency of waking up a herd of clients all waiting on the
same request.
v12: Rearrange missed-interrupt fault injection so that it works with
igt/drv_missed_irq_hang
v13: Rename intel_breadcrumb and friends to intel_wait in preparation
for signal handling.
v14: RCU commentary, assert_spin_locked
v15: Hide BUG_ON behind the compiler; report on gem_latency findings.
v16: Sort seqno-groups by priority so that first-waiter has the highest
task priority (and so avoid priority inversion).
v17: Add waiters to post-mortem GPU hang state.
v18: Return early for a completed wait after acquiring the spinlock.
Avoids adding ourselves to the tree if the is already complete, and
skips the awkward question of why we don't do completion wakeups for
waits earlier than or equal to ourselves.
v19: Prepare for init_breadcrumbs to fail. Later patches may want to
allocate during init, so be prepared to propagate back the error code.
Testcase: igt/gem_concurrent_blit
Testcase: igt/benchmarks/gem_latency
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: "Gong, Zhipeng" <zhipeng.gong@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Dave Gordon <david.s.gordon@intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com> #v18
Link: http://patchwork.freedesktop.org/patch/msgid/1467390209-3576-6-git-send-email-chris@chris-wilson.co.uk
2016-07-02 00:23:15 +08:00
|
|
|
return wakeup;
|
|
|
|
}
|
|
|
|
|
|
|
|
void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine);
|
|
|
|
unsigned int intel_kick_waiters(struct drm_i915_private *i915);
|
2016-07-02 00:23:25 +08:00
|
|
|
unsigned int intel_kick_signalers(struct drm_i915_private *i915);
|
drm/i915: Slaughter the thundering i915_wait_request herd
One particularly stressful scenario consists of many independent tasks
all competing for GPU time and waiting upon the results (e.g. realtime
transcoding of many, many streams). One bottleneck in particular is that
each client waits on its own results, but every client is woken up after
every batchbuffer - hence the thunder of hooves as then every client must
do its heavyweight dance to read a coherent seqno to see if it is the
lucky one.
Ideally, we only want one client to wake up after the interrupt and
check its request for completion. Since the requests must retire in
order, we can select the first client on the oldest request to be woken.
Once that client has completed his wait, we can then wake up the
next client and so on. However, all clients then incur latency as every
process in the chain may be delayed for scheduling - this may also then
cause some priority inversion. To reduce the latency, when a client
is added or removed from the list, we scan the tree for completed
seqno and wake up all the completed waiters in parallel.
Using igt/benchmarks/gem_latency, we can demonstrate this effect. The
benchmark measures the number of GPU cycles between completion of a
batch and the client waking up from a call to wait-ioctl. With many
concurrent waiters, with each on a different request, we observe that
the wakeup latency before the patch scales nearly linearly with the
number of waiters (before external factors kick in making the scaling much
worse). After applying the patch, we can see that only the single waiter
for the request is being woken up, providing a constant wakeup latency
for every operation. However, the situation is not quite as rosy for
many waiters on the same request, though to the best of my knowledge this
is much less likely in practice. Here, we can observe that the
concurrent waiters incur extra latency from being woken up by the
solitary bottom-half, rather than directly by the interrupt. This
appears to be scheduler induced (having discounted adverse effects from
having a rbtree walk/erase in the wakeup path), each additional
wake_up_process() costs approximately 1us on big core. Another effect of
performing the secondary wakeups from the first bottom-half is the
incurred delay this imposes on high priority threads - rather than
immediately returning to userspace and leaving the interrupt handler to
wake the others.
To offset the delay incurred with additional waiters on a request, we
could use a hybrid scheme that did a quick read in the interrupt handler
and dequeued all the completed waiters (incurring the overhead in the
interrupt handler, not the best plan either as we then incur GPU
submission latency) but we would still have to wake up the bottom-half
every time to do the heavyweight slow read. Or we could only kick the
waiters on the seqno with the same priority as the current task (i.e. in
the realtime waiter scenario, only it is woken up immediately by the
interrupt and simply queues the next waiter before returning to userspace,
minimising its delay at the expense of the chain, and also reducing
contention on its scheduler runqueue). This is effective at avoid long
pauses in the interrupt handler and at avoiding the extra latency in
realtime/high-priority waiters.
v2: Convert from a kworker per engine into a dedicated kthread for the
bottom-half.
v3: Rename request members and tweak comments.
v4: Use a per-engine spinlock in the breadcrumbs bottom-half.
v5: Fix race in locklessly checking waiter status and kicking the task on
adding a new waiter.
v6: Fix deciding when to force the timer to hide missing interrupts.
v7: Move the bottom-half from the kthread to the first client process.
v8: Reword a few comments
v9: Break the busy loop when the interrupt is unmasked or has fired.
v10: Comments, unnecessary churn, better debugging from Tvrtko
v11: Wake all completed waiters on removing the current bottom-half to
reduce the latency of waking up a herd of clients all waiting on the
same request.
v12: Rearrange missed-interrupt fault injection so that it works with
igt/drv_missed_irq_hang
v13: Rename intel_breadcrumb and friends to intel_wait in preparation
for signal handling.
v14: RCU commentary, assert_spin_locked
v15: Hide BUG_ON behind the compiler; report on gem_latency findings.
v16: Sort seqno-groups by priority so that first-waiter has the highest
task priority (and so avoid priority inversion).
v17: Add waiters to post-mortem GPU hang state.
v18: Return early for a completed wait after acquiring the spinlock.
Avoids adding ourselves to the tree if the is already complete, and
skips the awkward question of why we don't do completion wakeups for
waits earlier than or equal to ourselves.
v19: Prepare for init_breadcrumbs to fail. Later patches may want to
allocate during init, so be prepared to propagate back the error code.
Testcase: igt/gem_concurrent_blit
Testcase: igt/benchmarks/gem_latency
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: "Gong, Zhipeng" <zhipeng.gong@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Dave Gordon <david.s.gordon@intel.com>
Cc: "Goel, Akash" <akash.goel@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com> #v18
Link: http://patchwork.freedesktop.org/patch/msgid/1467390209-3576-6-git-send-email-chris@chris-wilson.co.uk
2016-07-02 00:23:15 +08:00
|
|
|
|
2016-08-05 17:14:11 +08:00
|
|
|
static inline bool intel_engine_is_active(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
return i915_gem_active_isset(&engine->last_request);
|
|
|
|
}
|
|
|
|
|
2010-05-21 09:08:55 +08:00
|
|
|
#endif /* _INTEL_RINGBUFFER_H_ */
|