2008-07-31 03:06:12 +08:00
|
|
|
/*
|
2015-03-18 17:46:04 +08:00
|
|
|
* Copyright © 2008-2015 Intel Corporation
|
2008-07-31 03:06:12 +08:00
|
|
|
*
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
|
|
* to deal in the Software without restriction, including without limitation
|
|
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
|
|
*
|
|
|
|
* The above copyright notice and this permission notice (including the next
|
|
|
|
* paragraph) shall be included in all copies or substantial portions of the
|
|
|
|
* Software.
|
|
|
|
*
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
|
|
* IN THE SOFTWARE.
|
|
|
|
*
|
|
|
|
* Authors:
|
|
|
|
* Eric Anholt <eric@anholt.net>
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
2012-10-03 01:01:07 +08:00
|
|
|
#include <drm/drmP.h>
|
2013-07-25 03:07:52 +08:00
|
|
|
#include <drm/drm_vma_manager.h>
|
2012-10-03 01:01:07 +08:00
|
|
|
#include <drm/i915_drm.h>
|
2008-07-31 03:06:12 +08:00
|
|
|
#include "i915_drv.h"
|
2015-02-10 19:05:49 +08:00
|
|
|
#include "i915_vgpu.h"
|
2009-08-25 18:15:50 +08:00
|
|
|
#include "i915_trace.h"
|
2009-08-18 04:31:43 +08:00
|
|
|
#include "intel_drv.h"
|
2011-06-28 07:18:18 +08:00
|
|
|
#include <linux/shmem_fs.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 16:04:11 +08:00
|
|
|
#include <linux/slab.h>
|
2008-07-31 03:06:12 +08:00
|
|
|
#include <linux/swap.h>
|
DRM: i915: add mode setting support
This commit adds i915 driver support for the DRM mode setting APIs.
Currently, VGA, LVDS, SDVO DVI & VGA, TV and DVO LVDS outputs are
supported. HDMI, DisplayPort and additional SDVO output support will
follow.
Support for the mode setting code is controlled by the new 'modeset'
module option. A new config option, CONFIG_DRM_I915_KMS controls the
default behavior, and whether a PCI ID list is built into the module for
use by user level module utilities.
Note that if mode setting is enabled, user level drivers that access
display registers directly or that don't use the kernel graphics memory
manager will likely corrupt kernel graphics memory, disrupt output
configuration (possibly leading to hangs and/or blank displays), and
prevent panic/oops messages from appearing. So use caution when
enabling this code; be sure your user level code supports the new
interfaces.
A new SysRq key, 'g', provides emergency support for switching back to
the kernel's framebuffer console; which is useful for testing.
Co-authors: Dave Airlie <airlied@linux.ie>, Hong Liu <hong.liu@intel.com>
Signed-off-by: Jesse Barnes <jbarnes@virtuousgeek.org>
Signed-off-by: Eric Anholt <eric@anholt.net>
Signed-off-by: Dave Airlie <airlied@redhat.com>
2008-11-08 06:24:08 +08:00
|
|
|
#include <linux/pci.h>
|
2012-05-10 21:25:09 +08:00
|
|
|
#include <linux/dma-buf.h>
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
|
2015-01-21 21:53:48 +08:00
|
|
|
static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
|
2014-03-17 20:21:55 +08:00
|
|
|
static void
|
2015-04-27 20:41:17 +08:00
|
|
|
i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
|
|
|
|
static void
|
|
|
|
i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring);
|
2012-04-17 22:31:31 +08:00
|
|
|
|
2013-08-08 21:41:03 +08:00
|
|
|
static bool cpu_cache_is_coherent(struct drm_device *dev,
|
|
|
|
enum i915_cache_level level)
|
|
|
|
{
|
|
|
|
return HAS_LLC(dev) || level != I915_CACHE_NONE;
|
|
|
|
}
|
|
|
|
|
2013-08-09 19:26:45 +08:00
|
|
|
static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
|
|
|
|
{
|
|
|
|
if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
|
|
|
|
return true;
|
|
|
|
|
|
|
|
return obj->pin_display;
|
|
|
|
}
|
|
|
|
|
2010-09-30 18:46:12 +08:00
|
|
|
/* some bookkeeping */
|
|
|
|
static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
|
|
|
|
size_t size)
|
|
|
|
{
|
2013-07-25 04:40:23 +08:00
|
|
|
spin_lock(&dev_priv->mm.object_stat_lock);
|
2010-09-30 18:46:12 +08:00
|
|
|
dev_priv->mm.object_count++;
|
|
|
|
dev_priv->mm.object_memory += size;
|
2013-07-25 04:40:23 +08:00
|
|
|
spin_unlock(&dev_priv->mm.object_stat_lock);
|
2010-09-30 18:46:12 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
|
|
|
|
size_t size)
|
|
|
|
{
|
2013-07-25 04:40:23 +08:00
|
|
|
spin_lock(&dev_priv->mm.object_stat_lock);
|
2010-09-30 18:46:12 +08:00
|
|
|
dev_priv->mm.object_count--;
|
|
|
|
dev_priv->mm.object_memory -= size;
|
2013-07-25 04:40:23 +08:00
|
|
|
spin_unlock(&dev_priv->mm.object_stat_lock);
|
2010-09-30 18:46:12 +08:00
|
|
|
}
|
|
|
|
|
2011-01-26 23:55:56 +08:00
|
|
|
static int
|
2012-11-15 00:14:05 +08:00
|
|
|
i915_gem_wait_for_error(struct i915_gpu_error *error)
|
2010-09-25 17:19:17 +08:00
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
2016-04-14 00:35:05 +08:00
|
|
|
if (!i915_reset_in_progress(error))
|
2010-09-25 17:19:17 +08:00
|
|
|
return 0;
|
|
|
|
|
2012-07-05 04:18:41 +08:00
|
|
|
/*
|
|
|
|
* Only wait 10 seconds for the gpu reset to complete to avoid hanging
|
|
|
|
* userspace. If it takes that long something really bad is going on and
|
|
|
|
* we should simply try to bail out and fail as gracefully as possible.
|
|
|
|
*/
|
2012-11-16 00:17:22 +08:00
|
|
|
ret = wait_event_interruptible_timeout(error->reset_queue,
|
2016-04-14 00:35:05 +08:00
|
|
|
!i915_reset_in_progress(error),
|
2012-11-16 00:17:22 +08:00
|
|
|
10*HZ);
|
2012-07-05 04:18:41 +08:00
|
|
|
if (ret == 0) {
|
|
|
|
DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
|
|
|
|
return -EIO;
|
|
|
|
} else if (ret < 0) {
|
2010-09-25 17:19:17 +08:00
|
|
|
return ret;
|
2016-04-14 00:35:05 +08:00
|
|
|
} else {
|
|
|
|
return 0;
|
2012-07-05 04:18:41 +08:00
|
|
|
}
|
2010-09-25 17:19:17 +08:00
|
|
|
}
|
|
|
|
|
2010-11-26 02:00:26 +08:00
|
|
|
int i915_mutex_lock_interruptible(struct drm_device *dev)
|
2010-09-25 18:22:51 +08:00
|
|
|
{
|
2012-11-15 00:14:05 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2010-09-25 18:22:51 +08:00
|
|
|
int ret;
|
|
|
|
|
2012-11-15 00:14:05 +08:00
|
|
|
ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
|
2010-09-25 18:22:51 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
ret = mutex_lock_interruptible(&dev->struct_mutex);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2010-09-29 23:10:57 +08:00
|
|
|
WARN_ON(i915_verify_lists(dev));
|
2010-09-25 18:22:51 +08:00
|
|
|
return 0;
|
|
|
|
}
|
2010-09-25 17:19:17 +08:00
|
|
|
|
2008-10-23 12:40:13 +08:00
|
|
|
int
|
|
|
|
i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_file *file)
|
2008-10-23 12:40:13 +08:00
|
|
|
{
|
2016-03-30 21:57:10 +08:00
|
|
|
struct drm_i915_private *dev_priv = to_i915(dev);
|
2016-03-18 16:42:57 +08:00
|
|
|
struct i915_ggtt *ggtt = &dev_priv->ggtt;
|
2016-03-30 21:57:10 +08:00
|
|
|
struct drm_i915_gem_get_aperture *args = data;
|
2015-07-01 18:51:10 +08:00
|
|
|
struct i915_vma *vma;
|
2010-11-24 20:23:44 +08:00
|
|
|
size_t pinned;
|
2008-10-23 12:40:13 +08:00
|
|
|
|
2010-11-24 20:23:44 +08:00
|
|
|
pinned = 0;
|
2010-09-30 18:46:12 +08:00
|
|
|
mutex_lock(&dev->struct_mutex);
|
2016-02-26 19:03:19 +08:00
|
|
|
list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
|
2015-07-01 18:51:10 +08:00
|
|
|
if (vma->pin_count)
|
|
|
|
pinned += vma->node.size;
|
2016-02-26 19:03:19 +08:00
|
|
|
list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
|
2015-07-01 18:51:10 +08:00
|
|
|
if (vma->pin_count)
|
|
|
|
pinned += vma->node.size;
|
2010-09-30 18:46:12 +08:00
|
|
|
mutex_unlock(&dev->struct_mutex);
|
2008-10-23 12:40:13 +08:00
|
|
|
|
2016-03-30 21:57:10 +08:00
|
|
|
args->aper_size = ggtt->base.total;
|
2011-08-17 03:34:10 +08:00
|
|
|
args->aper_available_size = args->aper_size - pinned;
|
2010-11-24 20:23:44 +08:00
|
|
|
|
2008-10-23 12:40:13 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2014-11-04 20:51:40 +08:00
|
|
|
static int
|
|
|
|
i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
|
2014-05-21 19:42:56 +08:00
|
|
|
{
|
2014-11-04 20:51:40 +08:00
|
|
|
struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
|
|
|
|
char *vaddr = obj->phys_handle->vaddr;
|
|
|
|
struct sg_table *st;
|
|
|
|
struct scatterlist *sg;
|
|
|
|
int i;
|
2014-05-21 19:42:56 +08:00
|
|
|
|
2014-11-04 20:51:40 +08:00
|
|
|
if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
|
|
|
|
struct page *page;
|
|
|
|
char *src;
|
|
|
|
|
|
|
|
page = shmem_read_mapping_page(mapping, i);
|
|
|
|
if (IS_ERR(page))
|
|
|
|
return PTR_ERR(page);
|
|
|
|
|
|
|
|
src = kmap_atomic(page);
|
|
|
|
memcpy(vaddr, src, PAGE_SIZE);
|
|
|
|
drm_clflush_virt_range(vaddr, PAGE_SIZE);
|
|
|
|
kunmap_atomic(src);
|
|
|
|
|
mm, fs: get rid of PAGE_CACHE_* and page_cache_{get,release} macros
PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} macros were introduced *long* time
ago with promise that one day it will be possible to implement page
cache with bigger chunks than PAGE_SIZE.
This promise never materialized. And unlikely will.
We have many places where PAGE_CACHE_SIZE assumed to be equal to
PAGE_SIZE. And it's constant source of confusion on whether
PAGE_CACHE_* or PAGE_* constant should be used in a particular case,
especially on the border between fs and mm.
Global switching to PAGE_CACHE_SIZE != PAGE_SIZE would cause to much
breakage to be doable.
Let's stop pretending that pages in page cache are special. They are
not.
The changes are pretty straight-forward:
- <foo> << (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- <foo> >> (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} -> PAGE_{SIZE,SHIFT,MASK,ALIGN};
- page_cache_get() -> get_page();
- page_cache_release() -> put_page();
This patch contains automated changes generated with coccinelle using
script below. For some reason, coccinelle doesn't patch header files.
I've called spatch for them manually.
The only adjustment after coccinelle is revert of changes to
PAGE_CAHCE_ALIGN definition: we are going to drop it later.
There are few places in the code where coccinelle didn't reach. I'll
fix them manually in a separate patch. Comments and documentation also
will be addressed with the separate patch.
virtual patch
@@
expression E;
@@
- E << (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
expression E;
@@
- E >> (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
@@
- PAGE_CACHE_SHIFT
+ PAGE_SHIFT
@@
@@
- PAGE_CACHE_SIZE
+ PAGE_SIZE
@@
@@
- PAGE_CACHE_MASK
+ PAGE_MASK
@@
expression E;
@@
- PAGE_CACHE_ALIGN(E)
+ PAGE_ALIGN(E)
@@
expression E;
@@
- page_cache_get(E)
+ get_page(E)
@@
expression E;
@@
- page_cache_release(E)
+ put_page(E)
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-04-01 20:29:47 +08:00
|
|
|
put_page(page);
|
2014-11-04 20:51:40 +08:00
|
|
|
vaddr += PAGE_SIZE;
|
|
|
|
}
|
|
|
|
|
|
|
|
i915_gem_chipset_flush(obj->base.dev);
|
|
|
|
|
|
|
|
st = kmalloc(sizeof(*st), GFP_KERNEL);
|
|
|
|
if (st == NULL)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
if (sg_alloc_table(st, 1, GFP_KERNEL)) {
|
|
|
|
kfree(st);
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
sg = st->sgl;
|
|
|
|
sg->offset = 0;
|
|
|
|
sg->length = obj->base.size;
|
2014-05-21 19:42:56 +08:00
|
|
|
|
2014-11-04 20:51:40 +08:00
|
|
|
sg_dma_address(sg) = obj->phys_handle->busaddr;
|
|
|
|
sg_dma_len(sg) = obj->base.size;
|
|
|
|
|
|
|
|
obj->pages = st;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
BUG_ON(obj->madv == __I915_MADV_PURGED);
|
2014-05-21 19:42:56 +08:00
|
|
|
|
2014-11-04 20:51:40 +08:00
|
|
|
ret = i915_gem_object_set_to_cpu_domain(obj, true);
|
|
|
|
if (ret) {
|
|
|
|
/* In the event of a disaster, abandon all caches and
|
|
|
|
* hope for the best.
|
|
|
|
*/
|
|
|
|
WARN_ON(ret != -EIO);
|
|
|
|
obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (obj->madv == I915_MADV_DONTNEED)
|
|
|
|
obj->dirty = 0;
|
|
|
|
|
|
|
|
if (obj->dirty) {
|
2014-05-21 19:42:56 +08:00
|
|
|
struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
|
2014-11-04 20:51:40 +08:00
|
|
|
char *vaddr = obj->phys_handle->vaddr;
|
2014-05-21 19:42:56 +08:00
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
|
2014-11-04 20:51:40 +08:00
|
|
|
struct page *page;
|
|
|
|
char *dst;
|
|
|
|
|
|
|
|
page = shmem_read_mapping_page(mapping, i);
|
|
|
|
if (IS_ERR(page))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
dst = kmap_atomic(page);
|
|
|
|
drm_clflush_virt_range(vaddr, PAGE_SIZE);
|
|
|
|
memcpy(dst, vaddr, PAGE_SIZE);
|
|
|
|
kunmap_atomic(dst);
|
|
|
|
|
|
|
|
set_page_dirty(page);
|
|
|
|
if (obj->madv == I915_MADV_WILLNEED)
|
2014-05-21 19:42:56 +08:00
|
|
|
mark_page_accessed(page);
|
mm, fs: get rid of PAGE_CACHE_* and page_cache_{get,release} macros
PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} macros were introduced *long* time
ago with promise that one day it will be possible to implement page
cache with bigger chunks than PAGE_SIZE.
This promise never materialized. And unlikely will.
We have many places where PAGE_CACHE_SIZE assumed to be equal to
PAGE_SIZE. And it's constant source of confusion on whether
PAGE_CACHE_* or PAGE_* constant should be used in a particular case,
especially on the border between fs and mm.
Global switching to PAGE_CACHE_SIZE != PAGE_SIZE would cause to much
breakage to be doable.
Let's stop pretending that pages in page cache are special. They are
not.
The changes are pretty straight-forward:
- <foo> << (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- <foo> >> (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} -> PAGE_{SIZE,SHIFT,MASK,ALIGN};
- page_cache_get() -> get_page();
- page_cache_release() -> put_page();
This patch contains automated changes generated with coccinelle using
script below. For some reason, coccinelle doesn't patch header files.
I've called spatch for them manually.
The only adjustment after coccinelle is revert of changes to
PAGE_CAHCE_ALIGN definition: we are going to drop it later.
There are few places in the code where coccinelle didn't reach. I'll
fix them manually in a separate patch. Comments and documentation also
will be addressed with the separate patch.
virtual patch
@@
expression E;
@@
- E << (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
expression E;
@@
- E >> (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
@@
- PAGE_CACHE_SHIFT
+ PAGE_SHIFT
@@
@@
- PAGE_CACHE_SIZE
+ PAGE_SIZE
@@
@@
- PAGE_CACHE_MASK
+ PAGE_MASK
@@
expression E;
@@
- PAGE_CACHE_ALIGN(E)
+ PAGE_ALIGN(E)
@@
expression E;
@@
- page_cache_get(E)
+ get_page(E)
@@
expression E;
@@
- page_cache_release(E)
+ put_page(E)
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-04-01 20:29:47 +08:00
|
|
|
put_page(page);
|
2014-05-21 19:42:56 +08:00
|
|
|
vaddr += PAGE_SIZE;
|
|
|
|
}
|
2014-11-04 20:51:40 +08:00
|
|
|
obj->dirty = 0;
|
2014-05-21 19:42:56 +08:00
|
|
|
}
|
|
|
|
|
2014-11-04 20:51:40 +08:00
|
|
|
sg_free_table(obj->pages);
|
|
|
|
kfree(obj->pages);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
|
|
|
|
{
|
|
|
|
drm_pci_free(obj->base.dev, obj->phys_handle);
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
|
|
|
|
.get_pages = i915_gem_object_get_pages_phys,
|
|
|
|
.put_pages = i915_gem_object_put_pages_phys,
|
|
|
|
.release = i915_gem_object_release_phys,
|
|
|
|
};
|
|
|
|
|
|
|
|
static int
|
|
|
|
drop_pages(struct drm_i915_gem_object *obj)
|
|
|
|
{
|
|
|
|
struct i915_vma *vma, *next;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
drm_gem_object_reference(&obj->base);
|
2016-02-26 19:03:19 +08:00
|
|
|
list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link)
|
2014-11-04 20:51:40 +08:00
|
|
|
if (i915_vma_unbind(vma))
|
|
|
|
break;
|
|
|
|
|
|
|
|
ret = i915_gem_object_put_pages(obj);
|
|
|
|
drm_gem_object_unreference(&obj->base);
|
|
|
|
|
|
|
|
return ret;
|
2014-05-21 19:42:56 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
|
|
|
|
int align)
|
|
|
|
{
|
|
|
|
drm_dma_handle_t *phys;
|
2014-11-04 20:51:40 +08:00
|
|
|
int ret;
|
2014-05-21 19:42:56 +08:00
|
|
|
|
|
|
|
if (obj->phys_handle) {
|
|
|
|
if ((unsigned long)obj->phys_handle->vaddr & (align -1))
|
|
|
|
return -EBUSY;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (obj->madv != I915_MADV_WILLNEED)
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
if (obj->base.filp == NULL)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2014-11-04 20:51:40 +08:00
|
|
|
ret = drop_pages(obj);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2014-05-21 19:42:56 +08:00
|
|
|
/* create a new object */
|
|
|
|
phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
|
|
|
|
if (!phys)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
obj->phys_handle = phys;
|
2014-11-04 20:51:40 +08:00
|
|
|
obj->ops = &i915_gem_phys_ops;
|
|
|
|
|
|
|
|
return i915_gem_object_get_pages(obj);
|
2014-05-21 19:42:56 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
|
|
|
|
struct drm_i915_gem_pwrite *args,
|
|
|
|
struct drm_file *file_priv)
|
|
|
|
{
|
|
|
|
struct drm_device *dev = obj->base.dev;
|
|
|
|
void *vaddr = obj->phys_handle->vaddr + args->offset;
|
|
|
|
char __user *user_data = to_user_ptr(args->data_ptr);
|
2015-02-14 03:23:45 +08:00
|
|
|
int ret = 0;
|
2014-11-04 20:51:40 +08:00
|
|
|
|
|
|
|
/* We manually control the domain here and pretend that it
|
|
|
|
* remains coherent i.e. in the GTT domain, like shmem_pwrite.
|
|
|
|
*/
|
|
|
|
ret = i915_gem_object_wait_rendering(obj, false);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2014-05-21 19:42:56 +08:00
|
|
|
|
2015-06-19 02:43:24 +08:00
|
|
|
intel_fb_obj_invalidate(obj, ORIGIN_CPU);
|
2014-05-21 19:42:56 +08:00
|
|
|
if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
|
|
|
|
unsigned long unwritten;
|
|
|
|
|
|
|
|
/* The physical object once assigned is fixed for the lifetime
|
|
|
|
* of the obj, so we can safely drop the lock and continue
|
|
|
|
* to access vaddr.
|
|
|
|
*/
|
|
|
|
mutex_unlock(&dev->struct_mutex);
|
|
|
|
unwritten = copy_from_user(vaddr, user_data, args->size);
|
|
|
|
mutex_lock(&dev->struct_mutex);
|
2015-02-14 03:23:45 +08:00
|
|
|
if (unwritten) {
|
|
|
|
ret = -EFAULT;
|
|
|
|
goto out;
|
|
|
|
}
|
2014-05-21 19:42:56 +08:00
|
|
|
}
|
|
|
|
|
2014-11-04 20:51:40 +08:00
|
|
|
drm_clflush_virt_range(vaddr, args->size);
|
2014-05-21 19:42:56 +08:00
|
|
|
i915_gem_chipset_flush(dev);
|
2015-02-14 03:23:45 +08:00
|
|
|
|
|
|
|
out:
|
2015-07-08 07:28:51 +08:00
|
|
|
intel_fb_obj_flush(obj, false, ORIGIN_CPU);
|
2015-02-14 03:23:45 +08:00
|
|
|
return ret;
|
2014-05-21 19:42:56 +08:00
|
|
|
}
|
|
|
|
|
2012-11-15 19:32:30 +08:00
|
|
|
void *i915_gem_object_alloc(struct drm_device *dev)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2015-04-07 23:20:57 +08:00
|
|
|
return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
|
2012-11-15 19:32:30 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void i915_gem_object_free(struct drm_i915_gem_object *obj)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
|
2015-04-07 23:20:57 +08:00
|
|
|
kmem_cache_free(dev_priv->objects, obj);
|
2012-11-15 19:32:30 +08:00
|
|
|
}
|
|
|
|
|
2011-02-07 10:16:14 +08:00
|
|
|
static int
|
|
|
|
i915_gem_create(struct drm_file *file,
|
|
|
|
struct drm_device *dev,
|
|
|
|
uint64_t size,
|
|
|
|
uint32_t *handle_p)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_i915_gem_object *obj;
|
2009-08-23 17:40:55 +08:00
|
|
|
int ret;
|
|
|
|
u32 handle;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2011-02-07 10:16:14 +08:00
|
|
|
size = roundup(size, PAGE_SIZE);
|
2011-09-14 20:14:28 +08:00
|
|
|
if (size == 0)
|
|
|
|
return -EINVAL;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
|
|
|
/* Allocate the new object */
|
2011-02-07 10:16:14 +08:00
|
|
|
obj = i915_gem_alloc_object(dev, size);
|
2008-07-31 03:06:12 +08:00
|
|
|
if (obj == NULL)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
ret = drm_gem_handle_create(file, &obj->base, &handle);
|
2010-10-14 20:20:40 +08:00
|
|
|
/* drop reference from allocate - handle holds it now */
|
2013-07-25 05:25:03 +08:00
|
|
|
drm_gem_object_unreference_unlocked(&obj->base);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2010-10-14 20:20:40 +08:00
|
|
|
|
2011-02-07 10:16:14 +08:00
|
|
|
*handle_p = handle;
|
2008-07-31 03:06:12 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-02-07 10:16:14 +08:00
|
|
|
int
|
|
|
|
i915_gem_dumb_create(struct drm_file *file,
|
|
|
|
struct drm_device *dev,
|
|
|
|
struct drm_mode_create_dumb *args)
|
|
|
|
{
|
|
|
|
/* have to work out size/pitch and return them */
|
2013-10-19 05:48:24 +08:00
|
|
|
args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
|
2011-02-07 10:16:14 +08:00
|
|
|
args->size = args->pitch * args->height;
|
|
|
|
return i915_gem_create(file, dev,
|
2014-12-24 11:11:17 +08:00
|
|
|
args->size, &args->handle);
|
2011-02-07 10:16:14 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Creates a new mm object and returns a handle to it.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
i915_gem_create_ioctl(struct drm_device *dev, void *data,
|
|
|
|
struct drm_file *file)
|
|
|
|
{
|
|
|
|
struct drm_i915_gem_create *args = data;
|
2012-04-23 22:50:50 +08:00
|
|
|
|
2011-02-07 10:16:14 +08:00
|
|
|
return i915_gem_create(file, dev,
|
2014-12-24 11:11:17 +08:00
|
|
|
args->size, &args->handle);
|
2011-02-07 10:16:14 +08:00
|
|
|
}
|
|
|
|
|
2011-12-14 20:57:32 +08:00
|
|
|
static inline int
|
|
|
|
__copy_to_user_swizzled(char __user *cpu_vaddr,
|
|
|
|
const char *gpu_vaddr, int gpu_offset,
|
|
|
|
int length)
|
|
|
|
{
|
|
|
|
int ret, cpu_offset = 0;
|
|
|
|
|
|
|
|
while (length > 0) {
|
|
|
|
int cacheline_end = ALIGN(gpu_offset + 1, 64);
|
|
|
|
int this_length = min(cacheline_end - gpu_offset, length);
|
|
|
|
int swizzled_gpu_offset = gpu_offset ^ 64;
|
|
|
|
|
|
|
|
ret = __copy_to_user(cpu_vaddr + cpu_offset,
|
|
|
|
gpu_vaddr + swizzled_gpu_offset,
|
|
|
|
this_length);
|
|
|
|
if (ret)
|
|
|
|
return ret + length;
|
|
|
|
|
|
|
|
cpu_offset += this_length;
|
|
|
|
gpu_offset += this_length;
|
|
|
|
length -= this_length;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
drm/i915: rewrite shmem_pwrite_slow to use copy_from_user
... instead of get_user_pages, because that fails on non page-backed
user addresses like e.g. a gtt mapping of a bo.
To get there essentially copy the vfs read path into pagecache. We
can't call that right away because we have to take care of bit17
swizzling. To not deadlock with our own pagefault handler we need
to completely drop struct_mutex, reducing the atomicty-guarantees
of our userspace abi. Implications for racing with other gem ioctl:
- execbuf, pwrite, pread: Due to -EFAULT fallback to slow paths there's
already the risk of the pwrite call not being atomic, no degration.
- read/write access to mmaps: already fully racy, no degration.
- set_tiling: Calling set_tiling while reading/writing is already
pretty much undefined, now it just got a bit worse. set_tiling is
only called by libdrm on unused/new bos, so no problem.
- set_domain: When changing to the gtt domain while copying (without any
read/write access, e.g. for synchronization), we might leave unflushed
data in the cpu caches. The clflush_object at the end of pwrite_slow
takes care of this problem.
- truncating of purgeable objects: the shmem_read_mapping_page call could
reinstate backing storage for truncated objects. The check at the end
of pwrite_slow takes care of this.
v2:
- add missing intel_gtt_chipset_flush
- add __ to copy_from_user_swizzled as suggest by Chris Wilson.
v3: Fixup bit17 swizzling, it swizzled the wrong pages.
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2011-12-14 20:57:31 +08:00
|
|
|
static inline int
|
2012-04-17 05:07:47 +08:00
|
|
|
__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
|
|
|
|
const char __user *cpu_vaddr,
|
drm/i915: rewrite shmem_pwrite_slow to use copy_from_user
... instead of get_user_pages, because that fails on non page-backed
user addresses like e.g. a gtt mapping of a bo.
To get there essentially copy the vfs read path into pagecache. We
can't call that right away because we have to take care of bit17
swizzling. To not deadlock with our own pagefault handler we need
to completely drop struct_mutex, reducing the atomicty-guarantees
of our userspace abi. Implications for racing with other gem ioctl:
- execbuf, pwrite, pread: Due to -EFAULT fallback to slow paths there's
already the risk of the pwrite call not being atomic, no degration.
- read/write access to mmaps: already fully racy, no degration.
- set_tiling: Calling set_tiling while reading/writing is already
pretty much undefined, now it just got a bit worse. set_tiling is
only called by libdrm on unused/new bos, so no problem.
- set_domain: When changing to the gtt domain while copying (without any
read/write access, e.g. for synchronization), we might leave unflushed
data in the cpu caches. The clflush_object at the end of pwrite_slow
takes care of this problem.
- truncating of purgeable objects: the shmem_read_mapping_page call could
reinstate backing storage for truncated objects. The check at the end
of pwrite_slow takes care of this.
v2:
- add missing intel_gtt_chipset_flush
- add __ to copy_from_user_swizzled as suggest by Chris Wilson.
v3: Fixup bit17 swizzling, it swizzled the wrong pages.
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2011-12-14 20:57:31 +08:00
|
|
|
int length)
|
|
|
|
{
|
|
|
|
int ret, cpu_offset = 0;
|
|
|
|
|
|
|
|
while (length > 0) {
|
|
|
|
int cacheline_end = ALIGN(gpu_offset + 1, 64);
|
|
|
|
int this_length = min(cacheline_end - gpu_offset, length);
|
|
|
|
int swizzled_gpu_offset = gpu_offset ^ 64;
|
|
|
|
|
|
|
|
ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
|
|
|
|
cpu_vaddr + cpu_offset,
|
|
|
|
this_length);
|
|
|
|
if (ret)
|
|
|
|
return ret + length;
|
|
|
|
|
|
|
|
cpu_offset += this_length;
|
|
|
|
gpu_offset += this_length;
|
|
|
|
length -= this_length;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2014-02-19 02:15:45 +08:00
|
|
|
/*
|
|
|
|
* Pins the specified object's pages and synchronizes the object with
|
|
|
|
* GPU accesses. Sets needs_clflush to non-zero if the caller should
|
|
|
|
* flush the object from the CPU cache.
|
|
|
|
*/
|
|
|
|
int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
|
|
|
|
int *needs_clflush)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
*needs_clflush = 0;
|
|
|
|
|
2016-02-10 03:44:12 +08:00
|
|
|
if (WARN_ON((obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE) == 0))
|
2014-02-19 02:15:45 +08:00
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
|
|
|
|
/* If we're not in the cpu read domain, set ourself into the gtt
|
|
|
|
* read domain and manually flush cachelines (if required). This
|
|
|
|
* optimizes for the case when the gpu will dirty the data
|
|
|
|
* anyway again before the next pread happens. */
|
|
|
|
*needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
|
|
|
|
obj->cache_level);
|
|
|
|
ret = i915_gem_object_wait_rendering(obj, true);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = i915_gem_object_get_pages(obj);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
i915_gem_object_pin_pages(obj);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2012-03-26 01:47:40 +08:00
|
|
|
/* Per-page copy function for the shmem pread fastpath.
|
|
|
|
* Flushes invalid cachelines before reading the target if
|
|
|
|
* needs_clflush is set. */
|
2009-03-11 02:44:52 +08:00
|
|
|
static int
|
2012-03-26 01:47:40 +08:00
|
|
|
shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
|
|
|
|
char __user *user_data,
|
|
|
|
bool page_do_bit17_swizzling, bool needs_clflush)
|
|
|
|
{
|
|
|
|
char *vaddr;
|
|
|
|
int ret;
|
|
|
|
|
2012-03-26 01:47:43 +08:00
|
|
|
if (unlikely(page_do_bit17_swizzling))
|
2012-03-26 01:47:40 +08:00
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
vaddr = kmap_atomic(page);
|
|
|
|
if (needs_clflush)
|
|
|
|
drm_clflush_virt_range(vaddr + shmem_page_offset,
|
|
|
|
page_length);
|
|
|
|
ret = __copy_to_user_inatomic(user_data,
|
|
|
|
vaddr + shmem_page_offset,
|
|
|
|
page_length);
|
|
|
|
kunmap_atomic(vaddr);
|
|
|
|
|
2012-09-05 04:02:56 +08:00
|
|
|
return ret ? -EFAULT : 0;
|
2012-03-26 01:47:40 +08:00
|
|
|
}
|
|
|
|
|
2012-03-26 01:47:42 +08:00
|
|
|
static void
|
|
|
|
shmem_clflush_swizzled_range(char *addr, unsigned long length,
|
|
|
|
bool swizzled)
|
|
|
|
{
|
2012-03-26 01:47:43 +08:00
|
|
|
if (unlikely(swizzled)) {
|
2012-03-26 01:47:42 +08:00
|
|
|
unsigned long start = (unsigned long) addr;
|
|
|
|
unsigned long end = (unsigned long) addr + length;
|
|
|
|
|
|
|
|
/* For swizzling simply ensure that we always flush both
|
|
|
|
* channels. Lame, but simple and it works. Swizzled
|
|
|
|
* pwrite/pread is far from a hotpath - current userspace
|
|
|
|
* doesn't use it at all. */
|
|
|
|
start = round_down(start, 128);
|
|
|
|
end = round_up(end, 128);
|
|
|
|
|
|
|
|
drm_clflush_virt_range((void *)start, end - start);
|
|
|
|
} else {
|
|
|
|
drm_clflush_virt_range(addr, length);
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2012-03-26 01:47:40 +08:00
|
|
|
/* Only difference to the fast-path function is that this can handle bit17
|
|
|
|
* and uses non-atomic copy and kmap functions. */
|
|
|
|
static int
|
|
|
|
shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
|
|
|
|
char __user *user_data,
|
|
|
|
bool page_do_bit17_swizzling, bool needs_clflush)
|
|
|
|
{
|
|
|
|
char *vaddr;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
vaddr = kmap(page);
|
|
|
|
if (needs_clflush)
|
2012-03-26 01:47:42 +08:00
|
|
|
shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
|
|
|
|
page_length,
|
|
|
|
page_do_bit17_swizzling);
|
2012-03-26 01:47:40 +08:00
|
|
|
|
|
|
|
if (page_do_bit17_swizzling)
|
|
|
|
ret = __copy_to_user_swizzled(user_data,
|
|
|
|
vaddr, shmem_page_offset,
|
|
|
|
page_length);
|
|
|
|
else
|
|
|
|
ret = __copy_to_user(user_data,
|
|
|
|
vaddr + shmem_page_offset,
|
|
|
|
page_length);
|
|
|
|
kunmap(page);
|
|
|
|
|
2012-09-05 04:02:56 +08:00
|
|
|
return ret ? - EFAULT : 0;
|
2012-03-26 01:47:40 +08:00
|
|
|
}
|
|
|
|
|
2009-03-11 02:44:52 +08:00
|
|
|
static int
|
2012-03-26 01:47:29 +08:00
|
|
|
i915_gem_shmem_pread(struct drm_device *dev,
|
|
|
|
struct drm_i915_gem_object *obj,
|
|
|
|
struct drm_i915_gem_pread *args,
|
|
|
|
struct drm_file *file)
|
2009-03-11 02:44:52 +08:00
|
|
|
{
|
2011-12-14 20:57:32 +08:00
|
|
|
char __user *user_data;
|
2009-03-11 02:44:52 +08:00
|
|
|
ssize_t remain;
|
2011-12-14 20:57:32 +08:00
|
|
|
loff_t offset;
|
2012-02-15 21:42:43 +08:00
|
|
|
int shmem_page_offset, page_length, ret = 0;
|
2011-12-14 20:57:32 +08:00
|
|
|
int obj_do_bit17_swizzling, page_do_bit17_swizzling;
|
2012-03-26 01:47:36 +08:00
|
|
|
int prefaulted = 0;
|
2012-03-26 01:47:31 +08:00
|
|
|
int needs_clflush = 0;
|
2013-02-19 01:28:02 +08:00
|
|
|
struct sg_page_iter sg_iter;
|
2009-03-11 02:44:52 +08:00
|
|
|
|
2013-02-22 22:12:51 +08:00
|
|
|
user_data = to_user_ptr(args->data_ptr);
|
2009-03-11 02:44:52 +08:00
|
|
|
remain = args->size;
|
|
|
|
|
2011-12-14 20:57:32 +08:00
|
|
|
obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
|
2009-03-11 02:44:52 +08:00
|
|
|
|
2014-02-19 02:15:45 +08:00
|
|
|
ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
|
2012-09-05 04:02:56 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2011-12-14 20:57:32 +08:00
|
|
|
offset = args->offset;
|
2009-03-11 02:44:52 +08:00
|
|
|
|
2013-02-19 01:28:02 +08:00
|
|
|
for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
|
|
|
|
offset >> PAGE_SHIFT) {
|
2013-03-26 21:14:18 +08:00
|
|
|
struct page *page = sg_page_iter_page(&sg_iter);
|
2012-06-01 22:20:22 +08:00
|
|
|
|
|
|
|
if (remain <= 0)
|
|
|
|
break;
|
|
|
|
|
2009-03-11 02:44:52 +08:00
|
|
|
/* Operation in this page
|
|
|
|
*
|
|
|
|
* shmem_page_offset = offset within page in shmem file
|
|
|
|
* page_length = bytes to copy for this page
|
|
|
|
*/
|
2011-05-13 05:17:11 +08:00
|
|
|
shmem_page_offset = offset_in_page(offset);
|
2009-03-11 02:44:52 +08:00
|
|
|
page_length = remain;
|
|
|
|
if ((shmem_page_offset + page_length) > PAGE_SIZE)
|
|
|
|
page_length = PAGE_SIZE - shmem_page_offset;
|
|
|
|
|
2011-12-14 20:57:32 +08:00
|
|
|
page_do_bit17_swizzling = obj_do_bit17_swizzling &&
|
|
|
|
(page_to_phys(page) & (1 << 17)) != 0;
|
|
|
|
|
2012-03-26 01:47:40 +08:00
|
|
|
ret = shmem_pread_fast(page, shmem_page_offset, page_length,
|
|
|
|
user_data, page_do_bit17_swizzling,
|
|
|
|
needs_clflush);
|
|
|
|
if (ret == 0)
|
|
|
|
goto next_page;
|
2012-03-26 01:47:29 +08:00
|
|
|
|
|
|
|
mutex_unlock(&dev->struct_mutex);
|
|
|
|
|
2014-01-21 17:24:25 +08:00
|
|
|
if (likely(!i915.prefault_disable) && !prefaulted) {
|
2012-03-26 01:47:41 +08:00
|
|
|
ret = fault_in_multipages_writeable(user_data, remain);
|
2012-03-26 01:47:36 +08:00
|
|
|
/* Userspace is tricking us, but we've already clobbered
|
|
|
|
* its pages with the prefault and promised to write the
|
|
|
|
* data up to the first fault. Hence ignore any errors
|
|
|
|
* and just continue. */
|
|
|
|
(void)ret;
|
|
|
|
prefaulted = 1;
|
|
|
|
}
|
2009-03-11 02:44:52 +08:00
|
|
|
|
2012-03-26 01:47:40 +08:00
|
|
|
ret = shmem_pread_slow(page, shmem_page_offset, page_length,
|
|
|
|
user_data, page_do_bit17_swizzling,
|
|
|
|
needs_clflush);
|
2009-03-11 02:44:52 +08:00
|
|
|
|
2012-03-26 01:47:29 +08:00
|
|
|
mutex_lock(&dev->struct_mutex);
|
2012-09-05 04:02:56 +08:00
|
|
|
|
|
|
|
if (ret)
|
2011-12-14 20:57:32 +08:00
|
|
|
goto out;
|
|
|
|
|
2014-03-07 16:30:36 +08:00
|
|
|
next_page:
|
2009-03-11 02:44:52 +08:00
|
|
|
remain -= page_length;
|
2011-12-14 20:57:32 +08:00
|
|
|
user_data += page_length;
|
2009-03-11 02:44:52 +08:00
|
|
|
offset += page_length;
|
|
|
|
}
|
|
|
|
|
2010-10-14 22:26:45 +08:00
|
|
|
out:
|
2012-09-05 04:02:56 +08:00
|
|
|
i915_gem_object_unpin_pages(obj);
|
|
|
|
|
2009-03-11 02:44:52 +08:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2008-07-31 03:06:12 +08:00
|
|
|
/**
|
|
|
|
* Reads data from the object referenced by handle.
|
|
|
|
*
|
|
|
|
* On error, the contents of *data are undefined.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
i915_gem_pread_ioctl(struct drm_device *dev, void *data,
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_file *file)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
|
|
|
struct drm_i915_gem_pread *args = data;
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_i915_gem_object *obj;
|
2010-09-27 03:23:38 +08:00
|
|
|
int ret = 0;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2010-11-17 17:10:42 +08:00
|
|
|
if (args->size == 0)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (!access_ok(VERIFY_WRITE,
|
2013-02-22 22:12:51 +08:00
|
|
|
to_user_ptr(args->data_ptr),
|
2010-11-17 17:10:42 +08:00
|
|
|
args->size))
|
|
|
|
return -EFAULT;
|
|
|
|
|
2010-10-14 22:26:45 +08:00
|
|
|
ret = i915_mutex_lock_interruptible(dev);
|
2010-10-17 16:45:41 +08:00
|
|
|
if (ret)
|
2010-10-14 22:26:45 +08:00
|
|
|
return ret;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
|
2011-02-19 19:31:06 +08:00
|
|
|
if (&obj->base == NULL) {
|
2010-10-17 16:45:41 +08:00
|
|
|
ret = -ENOENT;
|
|
|
|
goto unlock;
|
2010-10-14 22:26:45 +08:00
|
|
|
}
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2010-09-27 03:21:44 +08:00
|
|
|
/* Bounds check source. */
|
2010-11-09 03:18:58 +08:00
|
|
|
if (args->offset > obj->base.size ||
|
|
|
|
args->size > obj->base.size - args->offset) {
|
2010-09-27 03:50:05 +08:00
|
|
|
ret = -EINVAL;
|
2010-09-27 03:23:38 +08:00
|
|
|
goto out;
|
2010-09-27 03:50:05 +08:00
|
|
|
}
|
|
|
|
|
2012-05-10 21:25:09 +08:00
|
|
|
/* prime objects have no backing filp to GEM pread/pwrite
|
|
|
|
* pages from.
|
|
|
|
*/
|
|
|
|
if (!obj->base.filp) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2011-02-03 19:57:46 +08:00
|
|
|
trace_i915_gem_object_pread(obj, args->offset, args->size);
|
|
|
|
|
2012-03-26 01:47:29 +08:00
|
|
|
ret = i915_gem_shmem_pread(dev, obj, args, file);
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2010-09-27 03:23:38 +08:00
|
|
|
out:
|
2010-11-09 03:18:58 +08:00
|
|
|
drm_gem_object_unreference(&obj->base);
|
2010-10-17 16:45:41 +08:00
|
|
|
unlock:
|
2010-10-14 22:26:45 +08:00
|
|
|
mutex_unlock(&dev->struct_mutex);
|
2009-03-11 02:44:52 +08:00
|
|
|
return ret;
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
2008-10-31 10:38:48 +08:00
|
|
|
/* This is the fast write path which cannot handle
|
|
|
|
* page faults in the source data
|
2008-10-21 05:16:43 +08:00
|
|
|
*/
|
2008-10-31 10:38:48 +08:00
|
|
|
|
|
|
|
static inline int
|
|
|
|
fast_user_write(struct io_mapping *mapping,
|
|
|
|
loff_t page_base, int page_offset,
|
|
|
|
char __user *user_data,
|
|
|
|
int length)
|
2008-10-21 05:16:43 +08:00
|
|
|
{
|
2012-04-17 05:07:47 +08:00
|
|
|
void __iomem *vaddr_atomic;
|
|
|
|
void *vaddr;
|
2008-10-31 10:38:48 +08:00
|
|
|
unsigned long unwritten;
|
2008-10-21 05:16:43 +08:00
|
|
|
|
2010-10-27 05:21:51 +08:00
|
|
|
vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
|
2012-04-17 05:07:47 +08:00
|
|
|
/* We can use the cpu mem copy function because this is X86. */
|
|
|
|
vaddr = (void __force*)vaddr_atomic + page_offset;
|
|
|
|
unwritten = __copy_from_user_inatomic_nocache(vaddr,
|
2008-10-31 10:38:48 +08:00
|
|
|
user_data, length);
|
2010-10-27 05:21:51 +08:00
|
|
|
io_mapping_unmap_atomic(vaddr_atomic);
|
2010-10-14 22:03:58 +08:00
|
|
|
return unwritten;
|
2008-10-31 10:38:48 +08:00
|
|
|
}
|
|
|
|
|
2009-03-10 00:42:23 +08:00
|
|
|
/**
|
|
|
|
* This is the fast pwrite path, where we copy the data directly from the
|
|
|
|
* user into the GTT, uncached.
|
|
|
|
*/
|
2008-07-31 03:06:12 +08:00
|
|
|
static int
|
2010-11-09 03:18:58 +08:00
|
|
|
i915_gem_gtt_pwrite_fast(struct drm_device *dev,
|
|
|
|
struct drm_i915_gem_object *obj,
|
2009-03-10 00:42:23 +08:00
|
|
|
struct drm_i915_gem_pwrite *args,
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_file *file)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
2016-03-30 21:57:10 +08:00
|
|
|
struct drm_i915_private *dev_priv = to_i915(dev);
|
|
|
|
struct i915_ggtt *ggtt = &dev_priv->ggtt;
|
2008-07-31 03:06:12 +08:00
|
|
|
ssize_t remain;
|
2008-10-31 10:38:48 +08:00
|
|
|
loff_t offset, page_base;
|
2008-07-31 03:06:12 +08:00
|
|
|
char __user *user_data;
|
2012-03-26 01:47:35 +08:00
|
|
|
int page_offset, page_length, ret;
|
|
|
|
|
2014-02-14 21:01:11 +08:00
|
|
|
ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
|
2012-03-26 01:47:35 +08:00
|
|
|
if (ret)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
ret = i915_gem_object_set_to_gtt_domain(obj, true);
|
|
|
|
if (ret)
|
|
|
|
goto out_unpin;
|
|
|
|
|
|
|
|
ret = i915_gem_object_put_fence(obj);
|
|
|
|
if (ret)
|
|
|
|
goto out_unpin;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2013-02-22 22:12:51 +08:00
|
|
|
user_data = to_user_ptr(args->data_ptr);
|
2008-07-31 03:06:12 +08:00
|
|
|
remain = args->size;
|
|
|
|
|
2013-07-06 05:41:04 +08:00
|
|
|
offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2015-06-19 02:43:24 +08:00
|
|
|
intel_fb_obj_invalidate(obj, ORIGIN_GTT);
|
2015-02-14 03:23:45 +08:00
|
|
|
|
2008-07-31 03:06:12 +08:00
|
|
|
while (remain > 0) {
|
|
|
|
/* Operation in this page
|
|
|
|
*
|
2008-10-31 10:38:48 +08:00
|
|
|
* page_base = page offset within aperture
|
|
|
|
* page_offset = offset within page
|
|
|
|
* page_length = bytes to copy for this page
|
2008-07-31 03:06:12 +08:00
|
|
|
*/
|
2011-05-13 05:17:11 +08:00
|
|
|
page_base = offset & PAGE_MASK;
|
|
|
|
page_offset = offset_in_page(offset);
|
2008-10-31 10:38:48 +08:00
|
|
|
page_length = remain;
|
|
|
|
if ((page_offset + remain) > PAGE_SIZE)
|
|
|
|
page_length = PAGE_SIZE - page_offset;
|
|
|
|
|
|
|
|
/* If we get a fault while copying data, then (presumably) our
|
2009-03-10 00:42:23 +08:00
|
|
|
* source page isn't available. Return the error and we'll
|
|
|
|
* retry in the slow path.
|
2008-10-31 10:38:48 +08:00
|
|
|
*/
|
2016-03-30 21:57:10 +08:00
|
|
|
if (fast_user_write(ggtt->mappable, page_base,
|
2012-03-26 01:47:35 +08:00
|
|
|
page_offset, user_data, page_length)) {
|
|
|
|
ret = -EFAULT;
|
2015-02-14 03:23:45 +08:00
|
|
|
goto out_flush;
|
2012-03-26 01:47:35 +08:00
|
|
|
}
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2008-10-31 10:38:48 +08:00
|
|
|
remain -= page_length;
|
|
|
|
user_data += page_length;
|
|
|
|
offset += page_length;
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
2015-02-14 03:23:45 +08:00
|
|
|
out_flush:
|
2015-07-08 07:28:51 +08:00
|
|
|
intel_fb_obj_flush(obj, false, ORIGIN_GTT);
|
2012-03-26 01:47:35 +08:00
|
|
|
out_unpin:
|
2013-12-07 06:10:55 +08:00
|
|
|
i915_gem_object_ggtt_unpin(obj);
|
2012-03-26 01:47:35 +08:00
|
|
|
out:
|
2009-03-10 00:42:23 +08:00
|
|
|
return ret;
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
2012-03-26 01:47:40 +08:00
|
|
|
/* Per-page copy function for the shmem pwrite fastpath.
|
|
|
|
* Flushes invalid cachelines before writing to the target if
|
|
|
|
* needs_clflush_before is set and flushes out any written cachelines after
|
|
|
|
* writing if needs_clflush is set. */
|
2008-10-03 03:24:47 +08:00
|
|
|
static int
|
2012-03-26 01:47:40 +08:00
|
|
|
shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
|
|
|
|
char __user *user_data,
|
|
|
|
bool page_do_bit17_swizzling,
|
|
|
|
bool needs_clflush_before,
|
|
|
|
bool needs_clflush_after)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
2012-03-26 01:47:40 +08:00
|
|
|
char *vaddr;
|
2008-07-31 03:06:12 +08:00
|
|
|
int ret;
|
2009-03-10 00:42:23 +08:00
|
|
|
|
2012-03-26 01:47:43 +08:00
|
|
|
if (unlikely(page_do_bit17_swizzling))
|
2012-03-26 01:47:40 +08:00
|
|
|
return -EINVAL;
|
2009-03-10 00:42:23 +08:00
|
|
|
|
2012-03-26 01:47:40 +08:00
|
|
|
vaddr = kmap_atomic(page);
|
|
|
|
if (needs_clflush_before)
|
|
|
|
drm_clflush_virt_range(vaddr + shmem_page_offset,
|
|
|
|
page_length);
|
2014-03-07 16:30:37 +08:00
|
|
|
ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
|
|
|
|
user_data, page_length);
|
2012-03-26 01:47:40 +08:00
|
|
|
if (needs_clflush_after)
|
|
|
|
drm_clflush_virt_range(vaddr + shmem_page_offset,
|
|
|
|
page_length);
|
|
|
|
kunmap_atomic(vaddr);
|
2009-03-10 00:42:23 +08:00
|
|
|
|
2012-09-05 04:02:55 +08:00
|
|
|
return ret ? -EFAULT : 0;
|
2009-03-10 00:42:23 +08:00
|
|
|
}
|
|
|
|
|
2012-03-26 01:47:40 +08:00
|
|
|
/* Only difference to the fast-path function is that this can handle bit17
|
|
|
|
* and uses non-atomic copy and kmap functions. */
|
2008-10-03 03:24:47 +08:00
|
|
|
static int
|
2012-03-26 01:47:40 +08:00
|
|
|
shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
|
|
|
|
char __user *user_data,
|
|
|
|
bool page_do_bit17_swizzling,
|
|
|
|
bool needs_clflush_before,
|
|
|
|
bool needs_clflush_after)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
2012-03-26 01:47:40 +08:00
|
|
|
char *vaddr;
|
|
|
|
int ret;
|
2010-10-28 20:45:36 +08:00
|
|
|
|
2012-03-26 01:47:40 +08:00
|
|
|
vaddr = kmap(page);
|
2012-03-26 01:47:43 +08:00
|
|
|
if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
|
2012-03-26 01:47:42 +08:00
|
|
|
shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
|
|
|
|
page_length,
|
|
|
|
page_do_bit17_swizzling);
|
2012-03-26 01:47:40 +08:00
|
|
|
if (page_do_bit17_swizzling)
|
|
|
|
ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
|
2010-10-28 20:45:36 +08:00
|
|
|
user_data,
|
|
|
|
page_length);
|
2012-03-26 01:47:40 +08:00
|
|
|
else
|
|
|
|
ret = __copy_from_user(vaddr + shmem_page_offset,
|
|
|
|
user_data,
|
|
|
|
page_length);
|
|
|
|
if (needs_clflush_after)
|
2012-03-26 01:47:42 +08:00
|
|
|
shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
|
|
|
|
page_length,
|
|
|
|
page_do_bit17_swizzling);
|
2012-03-26 01:47:40 +08:00
|
|
|
kunmap(page);
|
2009-03-10 04:42:30 +08:00
|
|
|
|
2012-09-05 04:02:55 +08:00
|
|
|
return ret ? -EFAULT : 0;
|
2009-03-10 04:42:30 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
2012-03-26 01:47:28 +08:00
|
|
|
i915_gem_shmem_pwrite(struct drm_device *dev,
|
|
|
|
struct drm_i915_gem_object *obj,
|
|
|
|
struct drm_i915_gem_pwrite *args,
|
|
|
|
struct drm_file *file)
|
2009-03-10 04:42:30 +08:00
|
|
|
{
|
|
|
|
ssize_t remain;
|
drm/i915: rewrite shmem_pwrite_slow to use copy_from_user
... instead of get_user_pages, because that fails on non page-backed
user addresses like e.g. a gtt mapping of a bo.
To get there essentially copy the vfs read path into pagecache. We
can't call that right away because we have to take care of bit17
swizzling. To not deadlock with our own pagefault handler we need
to completely drop struct_mutex, reducing the atomicty-guarantees
of our userspace abi. Implications for racing with other gem ioctl:
- execbuf, pwrite, pread: Due to -EFAULT fallback to slow paths there's
already the risk of the pwrite call not being atomic, no degration.
- read/write access to mmaps: already fully racy, no degration.
- set_tiling: Calling set_tiling while reading/writing is already
pretty much undefined, now it just got a bit worse. set_tiling is
only called by libdrm on unused/new bos, so no problem.
- set_domain: When changing to the gtt domain while copying (without any
read/write access, e.g. for synchronization), we might leave unflushed
data in the cpu caches. The clflush_object at the end of pwrite_slow
takes care of this problem.
- truncating of purgeable objects: the shmem_read_mapping_page call could
reinstate backing storage for truncated objects. The check at the end
of pwrite_slow takes care of this.
v2:
- add missing intel_gtt_chipset_flush
- add __ to copy_from_user_swizzled as suggest by Chris Wilson.
v3: Fixup bit17 swizzling, it swizzled the wrong pages.
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2011-12-14 20:57:31 +08:00
|
|
|
loff_t offset;
|
|
|
|
char __user *user_data;
|
2012-02-15 21:42:43 +08:00
|
|
|
int shmem_page_offset, page_length, ret = 0;
|
drm/i915: rewrite shmem_pwrite_slow to use copy_from_user
... instead of get_user_pages, because that fails on non page-backed
user addresses like e.g. a gtt mapping of a bo.
To get there essentially copy the vfs read path into pagecache. We
can't call that right away because we have to take care of bit17
swizzling. To not deadlock with our own pagefault handler we need
to completely drop struct_mutex, reducing the atomicty-guarantees
of our userspace abi. Implications for racing with other gem ioctl:
- execbuf, pwrite, pread: Due to -EFAULT fallback to slow paths there's
already the risk of the pwrite call not being atomic, no degration.
- read/write access to mmaps: already fully racy, no degration.
- set_tiling: Calling set_tiling while reading/writing is already
pretty much undefined, now it just got a bit worse. set_tiling is
only called by libdrm on unused/new bos, so no problem.
- set_domain: When changing to the gtt domain while copying (without any
read/write access, e.g. for synchronization), we might leave unflushed
data in the cpu caches. The clflush_object at the end of pwrite_slow
takes care of this problem.
- truncating of purgeable objects: the shmem_read_mapping_page call could
reinstate backing storage for truncated objects. The check at the end
of pwrite_slow takes care of this.
v2:
- add missing intel_gtt_chipset_flush
- add __ to copy_from_user_swizzled as suggest by Chris Wilson.
v3: Fixup bit17 swizzling, it swizzled the wrong pages.
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2011-12-14 20:57:31 +08:00
|
|
|
int obj_do_bit17_swizzling, page_do_bit17_swizzling;
|
2012-03-26 01:47:28 +08:00
|
|
|
int hit_slowpath = 0;
|
2012-03-26 01:47:37 +08:00
|
|
|
int needs_clflush_after = 0;
|
|
|
|
int needs_clflush_before = 0;
|
2013-02-19 01:28:02 +08:00
|
|
|
struct sg_page_iter sg_iter;
|
2009-03-10 04:42:30 +08:00
|
|
|
|
2013-02-22 22:12:51 +08:00
|
|
|
user_data = to_user_ptr(args->data_ptr);
|
2009-03-10 04:42:30 +08:00
|
|
|
remain = args->size;
|
|
|
|
|
drm/i915: rewrite shmem_pwrite_slow to use copy_from_user
... instead of get_user_pages, because that fails on non page-backed
user addresses like e.g. a gtt mapping of a bo.
To get there essentially copy the vfs read path into pagecache. We
can't call that right away because we have to take care of bit17
swizzling. To not deadlock with our own pagefault handler we need
to completely drop struct_mutex, reducing the atomicty-guarantees
of our userspace abi. Implications for racing with other gem ioctl:
- execbuf, pwrite, pread: Due to -EFAULT fallback to slow paths there's
already the risk of the pwrite call not being atomic, no degration.
- read/write access to mmaps: already fully racy, no degration.
- set_tiling: Calling set_tiling while reading/writing is already
pretty much undefined, now it just got a bit worse. set_tiling is
only called by libdrm on unused/new bos, so no problem.
- set_domain: When changing to the gtt domain while copying (without any
read/write access, e.g. for synchronization), we might leave unflushed
data in the cpu caches. The clflush_object at the end of pwrite_slow
takes care of this problem.
- truncating of purgeable objects: the shmem_read_mapping_page call could
reinstate backing storage for truncated objects. The check at the end
of pwrite_slow takes care of this.
v2:
- add missing intel_gtt_chipset_flush
- add __ to copy_from_user_swizzled as suggest by Chris Wilson.
v3: Fixup bit17 swizzling, it swizzled the wrong pages.
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2011-12-14 20:57:31 +08:00
|
|
|
obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
|
2009-03-10 04:42:30 +08:00
|
|
|
|
2012-03-26 01:47:37 +08:00
|
|
|
if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
|
|
|
|
/* If we're not in the cpu write domain, set ourself into the gtt
|
|
|
|
* write domain and manually flush cachelines (if required). This
|
|
|
|
* optimizes for the case when the gpu will use the data
|
|
|
|
* right away and we therefore have to clflush anyway. */
|
2013-08-09 19:26:45 +08:00
|
|
|
needs_clflush_after = cpu_write_needs_clflush(obj);
|
2013-09-12 05:57:48 +08:00
|
|
|
ret = i915_gem_object_wait_rendering(obj, false);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2012-03-26 01:47:37 +08:00
|
|
|
}
|
2013-08-08 21:41:03 +08:00
|
|
|
/* Same trick applies to invalidate partially written cachelines read
|
|
|
|
* before writing. */
|
|
|
|
if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
|
|
|
|
needs_clflush_before =
|
|
|
|
!cpu_cache_is_coherent(dev, obj->cache_level);
|
2012-03-26 01:47:37 +08:00
|
|
|
|
2012-09-05 04:02:55 +08:00
|
|
|
ret = i915_gem_object_get_pages(obj);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2015-06-19 02:43:24 +08:00
|
|
|
intel_fb_obj_invalidate(obj, ORIGIN_CPU);
|
2015-02-14 03:23:45 +08:00
|
|
|
|
2012-09-05 04:02:55 +08:00
|
|
|
i915_gem_object_pin_pages(obj);
|
|
|
|
|
2008-07-31 03:06:12 +08:00
|
|
|
offset = args->offset;
|
2010-11-09 03:18:58 +08:00
|
|
|
obj->dirty = 1;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2013-02-19 01:28:02 +08:00
|
|
|
for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
|
|
|
|
offset >> PAGE_SHIFT) {
|
2013-03-26 21:14:18 +08:00
|
|
|
struct page *page = sg_page_iter_page(&sg_iter);
|
2012-03-26 01:47:37 +08:00
|
|
|
int partial_cacheline_write;
|
2010-10-28 20:45:36 +08:00
|
|
|
|
2012-06-01 22:20:22 +08:00
|
|
|
if (remain <= 0)
|
|
|
|
break;
|
|
|
|
|
2009-03-10 04:42:30 +08:00
|
|
|
/* Operation in this page
|
|
|
|
*
|
|
|
|
* shmem_page_offset = offset within page in shmem file
|
|
|
|
* page_length = bytes to copy for this page
|
|
|
|
*/
|
2011-05-13 05:17:11 +08:00
|
|
|
shmem_page_offset = offset_in_page(offset);
|
2009-03-10 04:42:30 +08:00
|
|
|
|
|
|
|
page_length = remain;
|
|
|
|
if ((shmem_page_offset + page_length) > PAGE_SIZE)
|
|
|
|
page_length = PAGE_SIZE - shmem_page_offset;
|
|
|
|
|
2012-03-26 01:47:37 +08:00
|
|
|
/* If we don't overwrite a cacheline completely we need to be
|
|
|
|
* careful to have up-to-date data by first clflushing. Don't
|
|
|
|
* overcomplicate things and flush the entire patch. */
|
|
|
|
partial_cacheline_write = needs_clflush_before &&
|
|
|
|
((shmem_page_offset | page_length)
|
|
|
|
& (boot_cpu_data.x86_clflush_size - 1));
|
|
|
|
|
drm/i915: rewrite shmem_pwrite_slow to use copy_from_user
... instead of get_user_pages, because that fails on non page-backed
user addresses like e.g. a gtt mapping of a bo.
To get there essentially copy the vfs read path into pagecache. We
can't call that right away because we have to take care of bit17
swizzling. To not deadlock with our own pagefault handler we need
to completely drop struct_mutex, reducing the atomicty-guarantees
of our userspace abi. Implications for racing with other gem ioctl:
- execbuf, pwrite, pread: Due to -EFAULT fallback to slow paths there's
already the risk of the pwrite call not being atomic, no degration.
- read/write access to mmaps: already fully racy, no degration.
- set_tiling: Calling set_tiling while reading/writing is already
pretty much undefined, now it just got a bit worse. set_tiling is
only called by libdrm on unused/new bos, so no problem.
- set_domain: When changing to the gtt domain while copying (without any
read/write access, e.g. for synchronization), we might leave unflushed
data in the cpu caches. The clflush_object at the end of pwrite_slow
takes care of this problem.
- truncating of purgeable objects: the shmem_read_mapping_page call could
reinstate backing storage for truncated objects. The check at the end
of pwrite_slow takes care of this.
v2:
- add missing intel_gtt_chipset_flush
- add __ to copy_from_user_swizzled as suggest by Chris Wilson.
v3: Fixup bit17 swizzling, it swizzled the wrong pages.
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2011-12-14 20:57:31 +08:00
|
|
|
page_do_bit17_swizzling = obj_do_bit17_swizzling &&
|
|
|
|
(page_to_phys(page) & (1 << 17)) != 0;
|
|
|
|
|
2012-03-26 01:47:40 +08:00
|
|
|
ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
|
|
|
|
user_data, page_do_bit17_swizzling,
|
|
|
|
partial_cacheline_write,
|
|
|
|
needs_clflush_after);
|
|
|
|
if (ret == 0)
|
|
|
|
goto next_page;
|
2012-03-26 01:47:28 +08:00
|
|
|
|
|
|
|
hit_slowpath = 1;
|
|
|
|
mutex_unlock(&dev->struct_mutex);
|
2012-03-26 01:47:40 +08:00
|
|
|
ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
|
|
|
|
user_data, page_do_bit17_swizzling,
|
|
|
|
partial_cacheline_write,
|
|
|
|
needs_clflush_after);
|
2009-03-10 04:42:30 +08:00
|
|
|
|
2012-03-26 01:47:28 +08:00
|
|
|
mutex_lock(&dev->struct_mutex);
|
2012-09-05 04:02:55 +08:00
|
|
|
|
|
|
|
if (ret)
|
drm/i915: rewrite shmem_pwrite_slow to use copy_from_user
... instead of get_user_pages, because that fails on non page-backed
user addresses like e.g. a gtt mapping of a bo.
To get there essentially copy the vfs read path into pagecache. We
can't call that right away because we have to take care of bit17
swizzling. To not deadlock with our own pagefault handler we need
to completely drop struct_mutex, reducing the atomicty-guarantees
of our userspace abi. Implications for racing with other gem ioctl:
- execbuf, pwrite, pread: Due to -EFAULT fallback to slow paths there's
already the risk of the pwrite call not being atomic, no degration.
- read/write access to mmaps: already fully racy, no degration.
- set_tiling: Calling set_tiling while reading/writing is already
pretty much undefined, now it just got a bit worse. set_tiling is
only called by libdrm on unused/new bos, so no problem.
- set_domain: When changing to the gtt domain while copying (without any
read/write access, e.g. for synchronization), we might leave unflushed
data in the cpu caches. The clflush_object at the end of pwrite_slow
takes care of this problem.
- truncating of purgeable objects: the shmem_read_mapping_page call could
reinstate backing storage for truncated objects. The check at the end
of pwrite_slow takes care of this.
v2:
- add missing intel_gtt_chipset_flush
- add __ to copy_from_user_swizzled as suggest by Chris Wilson.
v3: Fixup bit17 swizzling, it swizzled the wrong pages.
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2011-12-14 20:57:31 +08:00
|
|
|
goto out;
|
|
|
|
|
2014-03-07 16:30:36 +08:00
|
|
|
next_page:
|
2009-03-10 04:42:30 +08:00
|
|
|
remain -= page_length;
|
drm/i915: rewrite shmem_pwrite_slow to use copy_from_user
... instead of get_user_pages, because that fails on non page-backed
user addresses like e.g. a gtt mapping of a bo.
To get there essentially copy the vfs read path into pagecache. We
can't call that right away because we have to take care of bit17
swizzling. To not deadlock with our own pagefault handler we need
to completely drop struct_mutex, reducing the atomicty-guarantees
of our userspace abi. Implications for racing with other gem ioctl:
- execbuf, pwrite, pread: Due to -EFAULT fallback to slow paths there's
already the risk of the pwrite call not being atomic, no degration.
- read/write access to mmaps: already fully racy, no degration.
- set_tiling: Calling set_tiling while reading/writing is already
pretty much undefined, now it just got a bit worse. set_tiling is
only called by libdrm on unused/new bos, so no problem.
- set_domain: When changing to the gtt domain while copying (without any
read/write access, e.g. for synchronization), we might leave unflushed
data in the cpu caches. The clflush_object at the end of pwrite_slow
takes care of this problem.
- truncating of purgeable objects: the shmem_read_mapping_page call could
reinstate backing storage for truncated objects. The check at the end
of pwrite_slow takes care of this.
v2:
- add missing intel_gtt_chipset_flush
- add __ to copy_from_user_swizzled as suggest by Chris Wilson.
v3: Fixup bit17 swizzling, it swizzled the wrong pages.
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2011-12-14 20:57:31 +08:00
|
|
|
user_data += page_length;
|
2009-03-10 04:42:30 +08:00
|
|
|
offset += page_length;
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
2010-10-14 22:03:58 +08:00
|
|
|
out:
|
2012-09-05 04:02:55 +08:00
|
|
|
i915_gem_object_unpin_pages(obj);
|
|
|
|
|
2012-03-26 01:47:28 +08:00
|
|
|
if (hit_slowpath) {
|
2012-11-15 23:53:58 +08:00
|
|
|
/*
|
|
|
|
* Fixup: Flush cpu caches in case we didn't flush the dirty
|
|
|
|
* cachelines in-line while writing and the object moved
|
|
|
|
* out of the cpu write domain while we've dropped the lock.
|
|
|
|
*/
|
|
|
|
if (!needs_clflush_after &&
|
|
|
|
obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
|
2013-08-08 21:41:09 +08:00
|
|
|
if (i915_gem_clflush_object(obj, obj->pin_display))
|
2015-08-12 00:47:10 +08:00
|
|
|
needs_clflush_after = true;
|
2012-03-26 01:47:28 +08:00
|
|
|
}
|
drm/i915: rewrite shmem_pwrite_slow to use copy_from_user
... instead of get_user_pages, because that fails on non page-backed
user addresses like e.g. a gtt mapping of a bo.
To get there essentially copy the vfs read path into pagecache. We
can't call that right away because we have to take care of bit17
swizzling. To not deadlock with our own pagefault handler we need
to completely drop struct_mutex, reducing the atomicty-guarantees
of our userspace abi. Implications for racing with other gem ioctl:
- execbuf, pwrite, pread: Due to -EFAULT fallback to slow paths there's
already the risk of the pwrite call not being atomic, no degration.
- read/write access to mmaps: already fully racy, no degration.
- set_tiling: Calling set_tiling while reading/writing is already
pretty much undefined, now it just got a bit worse. set_tiling is
only called by libdrm on unused/new bos, so no problem.
- set_domain: When changing to the gtt domain while copying (without any
read/write access, e.g. for synchronization), we might leave unflushed
data in the cpu caches. The clflush_object at the end of pwrite_slow
takes care of this problem.
- truncating of purgeable objects: the shmem_read_mapping_page call could
reinstate backing storage for truncated objects. The check at the end
of pwrite_slow takes care of this.
v2:
- add missing intel_gtt_chipset_flush
- add __ to copy_from_user_swizzled as suggest by Chris Wilson.
v3: Fixup bit17 swizzling, it swizzled the wrong pages.
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2011-12-14 20:57:31 +08:00
|
|
|
}
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2012-03-26 01:47:37 +08:00
|
|
|
if (needs_clflush_after)
|
2012-11-05 01:21:27 +08:00
|
|
|
i915_gem_chipset_flush(dev);
|
2015-08-12 00:47:10 +08:00
|
|
|
else
|
|
|
|
obj->cache_dirty = true;
|
2012-03-26 01:47:37 +08:00
|
|
|
|
2015-07-08 07:28:51 +08:00
|
|
|
intel_fb_obj_flush(obj, false, ORIGIN_CPU);
|
2009-03-10 04:42:30 +08:00
|
|
|
return ret;
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Writes data to the object referenced by handle.
|
|
|
|
*
|
|
|
|
* On error, the contents of the buffer that were to be modified are undefined.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
|
2010-10-14 22:03:58 +08:00
|
|
|
struct drm_file *file)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
2014-11-12 22:40:35 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2008-07-31 03:06:12 +08:00
|
|
|
struct drm_i915_gem_pwrite *args = data;
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_i915_gem_object *obj;
|
2010-11-17 17:10:42 +08:00
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (args->size == 0)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (!access_ok(VERIFY_READ,
|
2013-02-22 22:12:51 +08:00
|
|
|
to_user_ptr(args->data_ptr),
|
2010-11-17 17:10:42 +08:00
|
|
|
args->size))
|
|
|
|
return -EFAULT;
|
|
|
|
|
2014-01-21 17:24:25 +08:00
|
|
|
if (likely(!i915.prefault_disable)) {
|
2013-07-19 13:51:24 +08:00
|
|
|
ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
|
|
|
|
args->size);
|
|
|
|
if (ret)
|
|
|
|
return -EFAULT;
|
|
|
|
}
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2014-11-12 22:40:35 +08:00
|
|
|
intel_runtime_pm_get(dev_priv);
|
|
|
|
|
2010-10-14 22:03:58 +08:00
|
|
|
ret = i915_mutex_lock_interruptible(dev);
|
2010-10-17 16:45:41 +08:00
|
|
|
if (ret)
|
2014-11-12 22:40:35 +08:00
|
|
|
goto put_rpm;
|
2010-10-17 16:45:41 +08:00
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
|
2011-02-19 19:31:06 +08:00
|
|
|
if (&obj->base == NULL) {
|
2010-10-17 16:45:41 +08:00
|
|
|
ret = -ENOENT;
|
|
|
|
goto unlock;
|
2010-10-14 22:03:58 +08:00
|
|
|
}
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2010-09-27 03:21:44 +08:00
|
|
|
/* Bounds check destination. */
|
2010-11-09 03:18:58 +08:00
|
|
|
if (args->offset > obj->base.size ||
|
|
|
|
args->size > obj->base.size - args->offset) {
|
2010-09-27 03:50:05 +08:00
|
|
|
ret = -EINVAL;
|
2010-09-27 03:23:38 +08:00
|
|
|
goto out;
|
2010-09-27 03:50:05 +08:00
|
|
|
}
|
|
|
|
|
2012-05-10 21:25:09 +08:00
|
|
|
/* prime objects have no backing filp to GEM pread/pwrite
|
|
|
|
* pages from.
|
|
|
|
*/
|
|
|
|
if (!obj->base.filp) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2011-02-03 19:57:46 +08:00
|
|
|
trace_i915_gem_object_pwrite(obj, args->offset, args->size);
|
|
|
|
|
2012-03-26 01:47:35 +08:00
|
|
|
ret = -EFAULT;
|
2008-07-31 03:06:12 +08:00
|
|
|
/* We can only do the GTT pwrite on untiled buffers, as otherwise
|
|
|
|
* it would end up going through the fenced access, and we'll get
|
|
|
|
* different detiling behavior between reading and writing.
|
|
|
|
* pread/pwrite currently are reading and writing from the CPU
|
|
|
|
* perspective, requiring manual detiling by the client.
|
|
|
|
*/
|
2013-08-09 19:26:45 +08:00
|
|
|
if (obj->tiling_mode == I915_TILING_NONE &&
|
|
|
|
obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
|
|
|
|
cpu_write_needs_clflush(obj)) {
|
2010-10-14 22:03:58 +08:00
|
|
|
ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
|
2012-03-26 01:47:35 +08:00
|
|
|
/* Note that the gtt paths might fail with non-page-backed user
|
|
|
|
* pointers (e.g. gtt mappings when moving data between
|
|
|
|
* textures). Fallback to the shmem path in that case. */
|
2010-10-14 22:03:58 +08:00
|
|
|
}
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2014-11-04 20:51:40 +08:00
|
|
|
if (ret == -EFAULT || ret == -ENOSPC) {
|
|
|
|
if (obj->phys_handle)
|
|
|
|
ret = i915_gem_phys_pwrite(obj, args, file);
|
|
|
|
else
|
|
|
|
ret = i915_gem_shmem_pwrite(dev, obj, args, file);
|
|
|
|
}
|
2011-12-14 20:57:30 +08:00
|
|
|
|
2010-09-27 03:23:38 +08:00
|
|
|
out:
|
2010-11-09 03:18:58 +08:00
|
|
|
drm_gem_object_unreference(&obj->base);
|
2010-10-17 16:45:41 +08:00
|
|
|
unlock:
|
2010-10-14 22:03:58 +08:00
|
|
|
mutex_unlock(&dev->struct_mutex);
|
2014-11-12 22:40:35 +08:00
|
|
|
put_rpm:
|
|
|
|
intel_runtime_pm_put(dev_priv);
|
|
|
|
|
2008-07-31 03:06:12 +08:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2012-08-24 16:35:08 +08:00
|
|
|
int
|
2012-11-15 00:14:05 +08:00
|
|
|
i915_gem_check_wedge(struct i915_gpu_error *error,
|
2012-08-24 16:35:08 +08:00
|
|
|
bool interruptible)
|
|
|
|
{
|
2016-04-14 00:35:03 +08:00
|
|
|
if (i915_reset_in_progress_or_wedged(error)) {
|
2016-04-14 00:35:05 +08:00
|
|
|
/* Recovery complete, but the reset failed ... */
|
|
|
|
if (i915_terminally_wedged(error))
|
|
|
|
return -EIO;
|
|
|
|
|
2012-08-24 16:35:08 +08:00
|
|
|
/* Non-interruptible callers can't handle -EAGAIN, hence return
|
|
|
|
* -EIO unconditionally for these. */
|
|
|
|
if (!interruptible)
|
|
|
|
return -EIO;
|
|
|
|
|
2016-04-14 00:35:05 +08:00
|
|
|
return -EAGAIN;
|
2012-08-24 16:35:08 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2013-09-26 00:34:55 +08:00
|
|
|
static void fake_irq(unsigned long data)
|
|
|
|
{
|
|
|
|
wake_up_process((struct task_struct *)data);
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool missed_irq(struct drm_i915_private *dev_priv,
|
2016-03-16 19:00:37 +08:00
|
|
|
struct intel_engine_cs *engine)
|
2013-09-26 00:34:55 +08:00
|
|
|
{
|
2016-03-16 19:00:37 +08:00
|
|
|
return test_bit(engine->id, &dev_priv->gpu_error.missed_irq_rings);
|
2013-09-26 00:34:55 +08:00
|
|
|
}
|
|
|
|
|
drm/i915: Limit the busy wait on requests to 5us not 10ms!
When waiting for high frequency requests, the finite amount of time
required to set up the irq and wait upon it limits the response rate. By
busywaiting on the request completion for a short while we can service
the high frequency waits as quick as possible. However, if it is a slow
request, we want to sleep as quickly as possible. The tradeoff between
waiting and sleeping is roughly the time it takes to sleep on a request,
on the order of a microsecond. Based on measurements of synchronous
workloads from across big core and little atom, I have set the limit for
busywaiting as 10 microseconds. In most of the synchronous cases, we can
reduce the limit down to as little as 2 miscroseconds, but that leaves
quite a few test cases regressing by factors of 3 and more.
The code currently uses the jiffie clock, but that is far too coarse (on
the order of 10 milliseconds) and results in poor interactivity as the
CPU ends up being hogged by slow requests. To get microsecond resolution
we need to use a high resolution timer. The cheapest of which is polling
local_clock(), but that is only valid on the same CPU. If we switch CPUs
because the task was preempted, we can also use that as an indicator that
the system is too busy to waste cycles on spinning and we should sleep
instead.
__i915_spin_request was introduced in
commit 2def4ad99befa25775dd2f714fdd4d92faec6e34 [v4.2]
Author: Chris Wilson <chris@chris-wilson.co.uk>
Date: Tue Apr 7 16:20:41 2015 +0100
drm/i915: Optimistically spin for the request completion
v2: Drop full u64 for unsigned long - the timer is 32bit wraparound safe,
so we can use native register sizes on smaller architectures. Mention
the approximate microseconds units for elapsed time and add some extra
comments describing the reason for busywaiting.
v3: Raise the limit to 10us
v4: Now 5us.
Reported-by: Jens Axboe <axboe@kernel.dk>
Link: https://lkml.org/lkml/2015/11/12/621
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Eero Tamminen <eero.t.tamminen@intel.com>
Cc: "Rantala, Valtteri" <valtteri.rantala@intel.com>
Cc: stable@vger.kernel.org
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Link: http://patchwork.freedesktop.org/patch/msgid/1449833608-22125-3-git-send-email-chris@chris-wilson.co.uk
2015-12-11 19:32:58 +08:00
|
|
|
static unsigned long local_clock_us(unsigned *cpu)
|
|
|
|
{
|
|
|
|
unsigned long t;
|
|
|
|
|
|
|
|
/* Cheaply and approximately convert from nanoseconds to microseconds.
|
|
|
|
* The result and subsequent calculations are also defined in the same
|
|
|
|
* approximate microseconds units. The principal source of timing
|
|
|
|
* error here is from the simple truncation.
|
|
|
|
*
|
|
|
|
* Note that local_clock() is only defined wrt to the current CPU;
|
|
|
|
* the comparisons are no longer valid if we switch CPUs. Instead of
|
|
|
|
* blocking preemption for the entire busywait, we can detect the CPU
|
|
|
|
* switch and use that as indicator of system load and a reason to
|
|
|
|
* stop busywaiting, see busywait_stop().
|
|
|
|
*/
|
|
|
|
*cpu = get_cpu();
|
|
|
|
t = local_clock() >> 10;
|
|
|
|
put_cpu();
|
|
|
|
|
|
|
|
return t;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool busywait_stop(unsigned long timeout, unsigned cpu)
|
|
|
|
{
|
|
|
|
unsigned this_cpu;
|
|
|
|
|
|
|
|
if (time_after(local_clock_us(&this_cpu), timeout))
|
|
|
|
return true;
|
|
|
|
|
|
|
|
return this_cpu != cpu;
|
|
|
|
}
|
|
|
|
|
2015-12-11 19:32:57 +08:00
|
|
|
static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
{
|
2015-04-07 23:20:41 +08:00
|
|
|
unsigned long timeout;
|
drm/i915: Limit the busy wait on requests to 5us not 10ms!
When waiting for high frequency requests, the finite amount of time
required to set up the irq and wait upon it limits the response rate. By
busywaiting on the request completion for a short while we can service
the high frequency waits as quick as possible. However, if it is a slow
request, we want to sleep as quickly as possible. The tradeoff between
waiting and sleeping is roughly the time it takes to sleep on a request,
on the order of a microsecond. Based on measurements of synchronous
workloads from across big core and little atom, I have set the limit for
busywaiting as 10 microseconds. In most of the synchronous cases, we can
reduce the limit down to as little as 2 miscroseconds, but that leaves
quite a few test cases regressing by factors of 3 and more.
The code currently uses the jiffie clock, but that is far too coarse (on
the order of 10 milliseconds) and results in poor interactivity as the
CPU ends up being hogged by slow requests. To get microsecond resolution
we need to use a high resolution timer. The cheapest of which is polling
local_clock(), but that is only valid on the same CPU. If we switch CPUs
because the task was preempted, we can also use that as an indicator that
the system is too busy to waste cycles on spinning and we should sleep
instead.
__i915_spin_request was introduced in
commit 2def4ad99befa25775dd2f714fdd4d92faec6e34 [v4.2]
Author: Chris Wilson <chris@chris-wilson.co.uk>
Date: Tue Apr 7 16:20:41 2015 +0100
drm/i915: Optimistically spin for the request completion
v2: Drop full u64 for unsigned long - the timer is 32bit wraparound safe,
so we can use native register sizes on smaller architectures. Mention
the approximate microseconds units for elapsed time and add some extra
comments describing the reason for busywaiting.
v3: Raise the limit to 10us
v4: Now 5us.
Reported-by: Jens Axboe <axboe@kernel.dk>
Link: https://lkml.org/lkml/2015/11/12/621
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Eero Tamminen <eero.t.tamminen@intel.com>
Cc: "Rantala, Valtteri" <valtteri.rantala@intel.com>
Cc: stable@vger.kernel.org
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Link: http://patchwork.freedesktop.org/patch/msgid/1449833608-22125-3-git-send-email-chris@chris-wilson.co.uk
2015-12-11 19:32:58 +08:00
|
|
|
unsigned cpu;
|
|
|
|
|
|
|
|
/* When waiting for high frequency requests, e.g. during synchronous
|
|
|
|
* rendering split between the CPU and GPU, the finite amount of time
|
|
|
|
* required to set up the irq and wait upon it limits the response
|
|
|
|
* rate. By busywaiting on the request completion for a short while we
|
|
|
|
* can service the high frequency waits as quick as possible. However,
|
|
|
|
* if it is a slow request, we want to sleep as quickly as possible.
|
|
|
|
* The tradeoff between waiting and sleeping is roughly the time it
|
|
|
|
* takes to sleep on a request, on the order of a microsecond.
|
|
|
|
*/
|
2015-04-07 23:20:41 +08:00
|
|
|
|
2016-03-16 19:00:38 +08:00
|
|
|
if (req->engine->irq_refcount)
|
2015-04-07 23:20:41 +08:00
|
|
|
return -EBUSY;
|
|
|
|
|
2015-12-11 19:32:59 +08:00
|
|
|
/* Only spin if we know the GPU is processing this request */
|
|
|
|
if (!i915_gem_request_started(req, true))
|
|
|
|
return -EAGAIN;
|
|
|
|
|
drm/i915: Limit the busy wait on requests to 5us not 10ms!
When waiting for high frequency requests, the finite amount of time
required to set up the irq and wait upon it limits the response rate. By
busywaiting on the request completion for a short while we can service
the high frequency waits as quick as possible. However, if it is a slow
request, we want to sleep as quickly as possible. The tradeoff between
waiting and sleeping is roughly the time it takes to sleep on a request,
on the order of a microsecond. Based on measurements of synchronous
workloads from across big core and little atom, I have set the limit for
busywaiting as 10 microseconds. In most of the synchronous cases, we can
reduce the limit down to as little as 2 miscroseconds, but that leaves
quite a few test cases regressing by factors of 3 and more.
The code currently uses the jiffie clock, but that is far too coarse (on
the order of 10 milliseconds) and results in poor interactivity as the
CPU ends up being hogged by slow requests. To get microsecond resolution
we need to use a high resolution timer. The cheapest of which is polling
local_clock(), but that is only valid on the same CPU. If we switch CPUs
because the task was preempted, we can also use that as an indicator that
the system is too busy to waste cycles on spinning and we should sleep
instead.
__i915_spin_request was introduced in
commit 2def4ad99befa25775dd2f714fdd4d92faec6e34 [v4.2]
Author: Chris Wilson <chris@chris-wilson.co.uk>
Date: Tue Apr 7 16:20:41 2015 +0100
drm/i915: Optimistically spin for the request completion
v2: Drop full u64 for unsigned long - the timer is 32bit wraparound safe,
so we can use native register sizes on smaller architectures. Mention
the approximate microseconds units for elapsed time and add some extra
comments describing the reason for busywaiting.
v3: Raise the limit to 10us
v4: Now 5us.
Reported-by: Jens Axboe <axboe@kernel.dk>
Link: https://lkml.org/lkml/2015/11/12/621
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Eero Tamminen <eero.t.tamminen@intel.com>
Cc: "Rantala, Valtteri" <valtteri.rantala@intel.com>
Cc: stable@vger.kernel.org
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Link: http://patchwork.freedesktop.org/patch/msgid/1449833608-22125-3-git-send-email-chris@chris-wilson.co.uk
2015-12-11 19:32:58 +08:00
|
|
|
timeout = local_clock_us(&cpu) + 5;
|
2015-04-07 23:20:41 +08:00
|
|
|
while (!need_resched()) {
|
2015-05-21 20:21:25 +08:00
|
|
|
if (i915_gem_request_completed(req, true))
|
2015-04-07 23:20:41 +08:00
|
|
|
return 0;
|
|
|
|
|
2015-12-11 19:32:57 +08:00
|
|
|
if (signal_pending_state(state, current))
|
|
|
|
break;
|
|
|
|
|
drm/i915: Limit the busy wait on requests to 5us not 10ms!
When waiting for high frequency requests, the finite amount of time
required to set up the irq and wait upon it limits the response rate. By
busywaiting on the request completion for a short while we can service
the high frequency waits as quick as possible. However, if it is a slow
request, we want to sleep as quickly as possible. The tradeoff between
waiting and sleeping is roughly the time it takes to sleep on a request,
on the order of a microsecond. Based on measurements of synchronous
workloads from across big core and little atom, I have set the limit for
busywaiting as 10 microseconds. In most of the synchronous cases, we can
reduce the limit down to as little as 2 miscroseconds, but that leaves
quite a few test cases regressing by factors of 3 and more.
The code currently uses the jiffie clock, but that is far too coarse (on
the order of 10 milliseconds) and results in poor interactivity as the
CPU ends up being hogged by slow requests. To get microsecond resolution
we need to use a high resolution timer. The cheapest of which is polling
local_clock(), but that is only valid on the same CPU. If we switch CPUs
because the task was preempted, we can also use that as an indicator that
the system is too busy to waste cycles on spinning and we should sleep
instead.
__i915_spin_request was introduced in
commit 2def4ad99befa25775dd2f714fdd4d92faec6e34 [v4.2]
Author: Chris Wilson <chris@chris-wilson.co.uk>
Date: Tue Apr 7 16:20:41 2015 +0100
drm/i915: Optimistically spin for the request completion
v2: Drop full u64 for unsigned long - the timer is 32bit wraparound safe,
so we can use native register sizes on smaller architectures. Mention
the approximate microseconds units for elapsed time and add some extra
comments describing the reason for busywaiting.
v3: Raise the limit to 10us
v4: Now 5us.
Reported-by: Jens Axboe <axboe@kernel.dk>
Link: https://lkml.org/lkml/2015/11/12/621
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: "Rogozhkin, Dmitry V" <dmitry.v.rogozhkin@intel.com>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: Eero Tamminen <eero.t.tamminen@intel.com>
Cc: "Rantala, Valtteri" <valtteri.rantala@intel.com>
Cc: stable@vger.kernel.org
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Link: http://patchwork.freedesktop.org/patch/msgid/1449833608-22125-3-git-send-email-chris@chris-wilson.co.uk
2015-12-11 19:32:58 +08:00
|
|
|
if (busywait_stop(timeout, cpu))
|
2015-04-07 23:20:41 +08:00
|
|
|
break;
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
|
2015-04-07 23:20:41 +08:00
|
|
|
cpu_relax_lowlatency();
|
|
|
|
}
|
2015-12-11 19:32:59 +08:00
|
|
|
|
2015-05-21 20:21:25 +08:00
|
|
|
if (i915_gem_request_completed(req, false))
|
2015-04-07 23:20:41 +08:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
return -EAGAIN;
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
}
|
|
|
|
|
2012-08-24 16:35:08 +08:00
|
|
|
/**
|
2014-11-25 02:49:35 +08:00
|
|
|
* __i915_wait_request - wait until execution of request has finished
|
|
|
|
* @req: duh!
|
|
|
|
* @reset_counter: reset sequence associated with the given request
|
2012-08-24 16:35:08 +08:00
|
|
|
* @interruptible: do an interruptible wait (normally yes)
|
|
|
|
* @timeout: in - how long to wait (NULL forever); out - how much time remaining
|
|
|
|
*
|
drm/i915: create a race-free reset detection
With the previous patch the state transition handling of the reset
code itself is now (hopefully) race free and solid. But that still
leaves out everyone else - with the various lock-free wait paths
we have there's the possibility that the reset happens between the
point where we read the seqno we should wait on and the actual wait.
And if __wait_seqno then never sees the RESET_IN_PROGRESS state, we'll
happily wait for a seqno which will in all likelyhood never signal.
In practice this is not a big problem since the X server gets
constantly interrupted, and can then submit more work (hopefully) to
unblock everyone else: As soon as a new seqno write lands, all waiters
will unblock. But running the i-g-t reset testcase ZZ_hangman can
expose this race, especially on slower hw with fewer cpu cores.
Now looking forward to ARB_robustness and friends that's not the best
possible behaviour, hence this patch adds a reset_counter to be able
to detect any reset, even if a given thread never observed the
in-progress state.
The important part is to correctly order things:
- The write side needs to increment the counter after any seqno gets
reset. Hence we need to do that at the end of the reset work, and
again wake everyone up. We also need to place a barrier in between
any possible seqno changes and the counter increment, since any
unlock operations only guarantee that nothing leaks out, but not
that at later load operation gets moved ahead.
- On the read side we need to ensure that no reset can sneak in and
invalidate the seqno. In all cases we can use the one-sided barrier
that unlock operations guarantee (of the lock protecting the
respective seqno/ring pair) to ensure correct ordering. Hence it is
sufficient to place the atomic read before the mutex/spin_unlock and
no additional barriers are required.
The end-result of all this is that we need to wake up everyone twice
in a reset operation:
- First, before the reset starts, to get any lockholders of the locks,
so that the reset can proceed.
- Second, after the reset is completed, to allow waiters to properly
and reliably detect the reset condition and bail out.
I admit that this entire reset_counter thing smells a bit like
overkill, but I think it's justified since it makes it really explicit
what the bail-out condition is. And we need a reset counter anyway to
implement ARB_robustness, and imo with finer-grained locking on the
horizont this is the most resilient scheme I could think of.
v2: Drop spurious change in the wait_for_error EXIT_COND - we only
need to wait until we leave the reset-in-progress wedged state.
v3: Don't play tricks with barriers in the throttle ioctl, the
spin_unlock is barrier enough.
I've also considered using a little helper to grab the current
reset_counter, but then decided that hiding the atomic_read isn't a
great idea, since having it explicitly show up in the code is a nice
remainder to reviews to check the memory barriers.
v4: Add a comment to explain why we need to fall through in
__wait_seqno in the end variable assignments.
v5: Review from Damien:
- s/smb/smp/ in a comment
- don't increment the reset counter after we've set it to WEDGED. Now
we (again) properly wedge the gpu when the reset fails.
Reviewed-by: Damien Lespiau <damien.lespiau@intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-12-06 16:01:42 +08:00
|
|
|
* Note: It is of utmost importance that the passed in seqno and reset_counter
|
|
|
|
* values have been read by the caller in an smp safe manner. Where read-side
|
|
|
|
* locks are involved, it is sufficient to read the reset_counter before
|
|
|
|
* unlocking the lock that protects the seqno. For lockless tricks, the
|
|
|
|
* reset_counter _must_ be read before, and an appropriate smp_rmb must be
|
|
|
|
* inserted.
|
|
|
|
*
|
2014-11-25 02:49:35 +08:00
|
|
|
* Returns 0 if the request was found within the alloted time. Else returns the
|
2012-08-24 16:35:08 +08:00
|
|
|
* errno with remaining time filled in timeout argument.
|
|
|
|
*/
|
2014-11-25 02:49:35 +08:00
|
|
|
int __i915_wait_request(struct drm_i915_gem_request *req,
|
drm/i915: create a race-free reset detection
With the previous patch the state transition handling of the reset
code itself is now (hopefully) race free and solid. But that still
leaves out everyone else - with the various lock-free wait paths
we have there's the possibility that the reset happens between the
point where we read the seqno we should wait on and the actual wait.
And if __wait_seqno then never sees the RESET_IN_PROGRESS state, we'll
happily wait for a seqno which will in all likelyhood never signal.
In practice this is not a big problem since the X server gets
constantly interrupted, and can then submit more work (hopefully) to
unblock everyone else: As soon as a new seqno write lands, all waiters
will unblock. But running the i-g-t reset testcase ZZ_hangman can
expose this race, especially on slower hw with fewer cpu cores.
Now looking forward to ARB_robustness and friends that's not the best
possible behaviour, hence this patch adds a reset_counter to be able
to detect any reset, even if a given thread never observed the
in-progress state.
The important part is to correctly order things:
- The write side needs to increment the counter after any seqno gets
reset. Hence we need to do that at the end of the reset work, and
again wake everyone up. We also need to place a barrier in between
any possible seqno changes and the counter increment, since any
unlock operations only guarantee that nothing leaks out, but not
that at later load operation gets moved ahead.
- On the read side we need to ensure that no reset can sneak in and
invalidate the seqno. In all cases we can use the one-sided barrier
that unlock operations guarantee (of the lock protecting the
respective seqno/ring pair) to ensure correct ordering. Hence it is
sufficient to place the atomic read before the mutex/spin_unlock and
no additional barriers are required.
The end-result of all this is that we need to wake up everyone twice
in a reset operation:
- First, before the reset starts, to get any lockholders of the locks,
so that the reset can proceed.
- Second, after the reset is completed, to allow waiters to properly
and reliably detect the reset condition and bail out.
I admit that this entire reset_counter thing smells a bit like
overkill, but I think it's justified since it makes it really explicit
what the bail-out condition is. And we need a reset counter anyway to
implement ARB_robustness, and imo with finer-grained locking on the
horizont this is the most resilient scheme I could think of.
v2: Drop spurious change in the wait_for_error EXIT_COND - we only
need to wait until we leave the reset-in-progress wedged state.
v3: Don't play tricks with barriers in the throttle ioctl, the
spin_unlock is barrier enough.
I've also considered using a little helper to grab the current
reset_counter, but then decided that hiding the atomic_read isn't a
great idea, since having it explicitly show up in the code is a nice
remainder to reviews to check the memory barriers.
v4: Add a comment to explain why we need to fall through in
__wait_seqno in the end variable assignments.
v5: Review from Damien:
- s/smb/smp/ in a comment
- don't increment the reset counter after we've set it to WEDGED. Now
we (again) properly wedge the gpu when the reset fails.
Reviewed-by: Damien Lespiau <damien.lespiau@intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-12-06 16:01:42 +08:00
|
|
|
unsigned reset_counter,
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
bool interruptible,
|
2014-07-17 05:05:06 +08:00
|
|
|
s64 *timeout,
|
2015-04-27 20:41:22 +08:00
|
|
|
struct intel_rps_client *rps)
|
2012-08-24 16:35:08 +08:00
|
|
|
{
|
2016-03-16 19:00:39 +08:00
|
|
|
struct intel_engine_cs *engine = i915_gem_request_get_engine(req);
|
2016-03-16 19:00:36 +08:00
|
|
|
struct drm_device *dev = engine->dev;
|
2014-03-31 19:27:16 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2013-12-12 23:54:42 +08:00
|
|
|
const bool irq_test_in_progress =
|
2016-03-16 19:00:39 +08:00
|
|
|
ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_engine_flag(engine);
|
2015-12-11 19:32:57 +08:00
|
|
|
int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
|
2013-09-26 00:34:55 +08:00
|
|
|
DEFINE_WAIT(wait);
|
2013-12-10 23:02:43 +08:00
|
|
|
unsigned long timeout_expire;
|
2016-01-15 23:11:12 +08:00
|
|
|
s64 before = 0; /* Only to silence a compiler warning. */
|
2012-08-24 16:35:08 +08:00
|
|
|
int ret;
|
|
|
|
|
2014-06-21 00:29:20 +08:00
|
|
|
WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
|
2013-08-20 00:18:09 +08:00
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
if (list_empty(&req->list))
|
|
|
|
return 0;
|
|
|
|
|
2014-11-25 02:49:42 +08:00
|
|
|
if (i915_gem_request_completed(req, true))
|
2012-08-24 16:35:08 +08:00
|
|
|
return 0;
|
|
|
|
|
2015-11-26 21:31:42 +08:00
|
|
|
timeout_expire = 0;
|
|
|
|
if (timeout) {
|
|
|
|
if (WARN_ON(*timeout < 0))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (*timeout == 0)
|
|
|
|
return -ETIME;
|
|
|
|
|
|
|
|
timeout_expire = jiffies + nsecs_to_jiffies_timeout(*timeout);
|
2016-01-15 23:11:12 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Record current time in case interrupted by signal, or wedged.
|
|
|
|
*/
|
|
|
|
before = ktime_get_raw_ns();
|
2015-11-26 21:31:42 +08:00
|
|
|
}
|
2012-08-24 16:35:08 +08:00
|
|
|
|
2015-04-27 20:41:22 +08:00
|
|
|
if (INTEL_INFO(dev_priv)->gen >= 6)
|
2015-04-27 20:41:24 +08:00
|
|
|
gen6_rps_boost(dev_priv, rps, req->emitted_jiffies);
|
2012-08-24 16:35:08 +08:00
|
|
|
|
2014-11-25 02:49:38 +08:00
|
|
|
trace_i915_gem_request_wait_begin(req);
|
2015-04-07 23:20:41 +08:00
|
|
|
|
|
|
|
/* Optimistic spin for the next jiffie before touching IRQs */
|
2015-12-11 19:32:57 +08:00
|
|
|
ret = __i915_spin_request(req, state);
|
2015-04-07 23:20:41 +08:00
|
|
|
if (ret == 0)
|
|
|
|
goto out;
|
|
|
|
|
2016-03-16 19:00:36 +08:00
|
|
|
if (!irq_test_in_progress && WARN_ON(!engine->irq_get(engine))) {
|
2015-04-07 23:20:41 +08:00
|
|
|
ret = -ENODEV;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2013-09-26 00:34:55 +08:00
|
|
|
for (;;) {
|
|
|
|
struct timer_list timer;
|
2012-08-24 16:35:08 +08:00
|
|
|
|
2016-03-16 19:00:36 +08:00
|
|
|
prepare_to_wait(&engine->irq_queue, &wait, state);
|
2012-08-24 16:35:08 +08:00
|
|
|
|
drm/i915: create a race-free reset detection
With the previous patch the state transition handling of the reset
code itself is now (hopefully) race free and solid. But that still
leaves out everyone else - with the various lock-free wait paths
we have there's the possibility that the reset happens between the
point where we read the seqno we should wait on and the actual wait.
And if __wait_seqno then never sees the RESET_IN_PROGRESS state, we'll
happily wait for a seqno which will in all likelyhood never signal.
In practice this is not a big problem since the X server gets
constantly interrupted, and can then submit more work (hopefully) to
unblock everyone else: As soon as a new seqno write lands, all waiters
will unblock. But running the i-g-t reset testcase ZZ_hangman can
expose this race, especially on slower hw with fewer cpu cores.
Now looking forward to ARB_robustness and friends that's not the best
possible behaviour, hence this patch adds a reset_counter to be able
to detect any reset, even if a given thread never observed the
in-progress state.
The important part is to correctly order things:
- The write side needs to increment the counter after any seqno gets
reset. Hence we need to do that at the end of the reset work, and
again wake everyone up. We also need to place a barrier in between
any possible seqno changes and the counter increment, since any
unlock operations only guarantee that nothing leaks out, but not
that at later load operation gets moved ahead.
- On the read side we need to ensure that no reset can sneak in and
invalidate the seqno. In all cases we can use the one-sided barrier
that unlock operations guarantee (of the lock protecting the
respective seqno/ring pair) to ensure correct ordering. Hence it is
sufficient to place the atomic read before the mutex/spin_unlock and
no additional barriers are required.
The end-result of all this is that we need to wake up everyone twice
in a reset operation:
- First, before the reset starts, to get any lockholders of the locks,
so that the reset can proceed.
- Second, after the reset is completed, to allow waiters to properly
and reliably detect the reset condition and bail out.
I admit that this entire reset_counter thing smells a bit like
overkill, but I think it's justified since it makes it really explicit
what the bail-out condition is. And we need a reset counter anyway to
implement ARB_robustness, and imo with finer-grained locking on the
horizont this is the most resilient scheme I could think of.
v2: Drop spurious change in the wait_for_error EXIT_COND - we only
need to wait until we leave the reset-in-progress wedged state.
v3: Don't play tricks with barriers in the throttle ioctl, the
spin_unlock is barrier enough.
I've also considered using a little helper to grab the current
reset_counter, but then decided that hiding the atomic_read isn't a
great idea, since having it explicitly show up in the code is a nice
remainder to reviews to check the memory barriers.
v4: Add a comment to explain why we need to fall through in
__wait_seqno in the end variable assignments.
v5: Review from Damien:
- s/smb/smp/ in a comment
- don't increment the reset counter after we've set it to WEDGED. Now
we (again) properly wedge the gpu when the reset fails.
Reviewed-by: Damien Lespiau <damien.lespiau@intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-12-06 16:01:42 +08:00
|
|
|
/* We need to check whether any gpu reset happened in between
|
|
|
|
* the caller grabbing the seqno and now ... */
|
2016-04-14 00:35:03 +08:00
|
|
|
if (reset_counter != i915_reset_counter(&dev_priv->gpu_error)) {
|
2013-09-26 00:34:55 +08:00
|
|
|
/* ... but upgrade the -EAGAIN to an -EIO if the gpu
|
|
|
|
* is truely gone. */
|
|
|
|
ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
|
|
|
|
if (ret == 0)
|
|
|
|
ret = -EAGAIN;
|
|
|
|
break;
|
|
|
|
}
|
drm/i915: create a race-free reset detection
With the previous patch the state transition handling of the reset
code itself is now (hopefully) race free and solid. But that still
leaves out everyone else - with the various lock-free wait paths
we have there's the possibility that the reset happens between the
point where we read the seqno we should wait on and the actual wait.
And if __wait_seqno then never sees the RESET_IN_PROGRESS state, we'll
happily wait for a seqno which will in all likelyhood never signal.
In practice this is not a big problem since the X server gets
constantly interrupted, and can then submit more work (hopefully) to
unblock everyone else: As soon as a new seqno write lands, all waiters
will unblock. But running the i-g-t reset testcase ZZ_hangman can
expose this race, especially on slower hw with fewer cpu cores.
Now looking forward to ARB_robustness and friends that's not the best
possible behaviour, hence this patch adds a reset_counter to be able
to detect any reset, even if a given thread never observed the
in-progress state.
The important part is to correctly order things:
- The write side needs to increment the counter after any seqno gets
reset. Hence we need to do that at the end of the reset work, and
again wake everyone up. We also need to place a barrier in between
any possible seqno changes and the counter increment, since any
unlock operations only guarantee that nothing leaks out, but not
that at later load operation gets moved ahead.
- On the read side we need to ensure that no reset can sneak in and
invalidate the seqno. In all cases we can use the one-sided barrier
that unlock operations guarantee (of the lock protecting the
respective seqno/ring pair) to ensure correct ordering. Hence it is
sufficient to place the atomic read before the mutex/spin_unlock and
no additional barriers are required.
The end-result of all this is that we need to wake up everyone twice
in a reset operation:
- First, before the reset starts, to get any lockholders of the locks,
so that the reset can proceed.
- Second, after the reset is completed, to allow waiters to properly
and reliably detect the reset condition and bail out.
I admit that this entire reset_counter thing smells a bit like
overkill, but I think it's justified since it makes it really explicit
what the bail-out condition is. And we need a reset counter anyway to
implement ARB_robustness, and imo with finer-grained locking on the
horizont this is the most resilient scheme I could think of.
v2: Drop spurious change in the wait_for_error EXIT_COND - we only
need to wait until we leave the reset-in-progress wedged state.
v3: Don't play tricks with barriers in the throttle ioctl, the
spin_unlock is barrier enough.
I've also considered using a little helper to grab the current
reset_counter, but then decided that hiding the atomic_read isn't a
great idea, since having it explicitly show up in the code is a nice
remainder to reviews to check the memory barriers.
v4: Add a comment to explain why we need to fall through in
__wait_seqno in the end variable assignments.
v5: Review from Damien:
- s/smb/smp/ in a comment
- don't increment the reset counter after we've set it to WEDGED. Now
we (again) properly wedge the gpu when the reset fails.
Reviewed-by: Damien Lespiau <damien.lespiau@intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-12-06 16:01:42 +08:00
|
|
|
|
2014-11-25 02:49:42 +08:00
|
|
|
if (i915_gem_request_completed(req, false)) {
|
2013-09-26 00:34:55 +08:00
|
|
|
ret = 0;
|
|
|
|
break;
|
|
|
|
}
|
2012-08-24 16:35:08 +08:00
|
|
|
|
2015-12-11 19:32:57 +08:00
|
|
|
if (signal_pending_state(state, current)) {
|
2013-09-26 00:34:55 +08:00
|
|
|
ret = -ERESTARTSYS;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2013-12-10 23:02:43 +08:00
|
|
|
if (timeout && time_after_eq(jiffies, timeout_expire)) {
|
2013-09-26 00:34:55 +08:00
|
|
|
ret = -ETIME;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
timer.function = NULL;
|
2016-03-16 19:00:36 +08:00
|
|
|
if (timeout || missed_irq(dev_priv, engine)) {
|
2013-12-10 23:02:43 +08:00
|
|
|
unsigned long expire;
|
|
|
|
|
2013-09-26 00:34:55 +08:00
|
|
|
setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
|
2016-03-16 19:00:36 +08:00
|
|
|
expire = missed_irq(dev_priv, engine) ? jiffies + 1 : timeout_expire;
|
2013-09-26 00:34:55 +08:00
|
|
|
mod_timer(&timer, expire);
|
|
|
|
}
|
|
|
|
|
2013-10-04 16:58:46 +08:00
|
|
|
io_schedule();
|
2013-09-26 00:34:55 +08:00
|
|
|
|
|
|
|
if (timer.function) {
|
|
|
|
del_singleshot_timer_sync(&timer);
|
|
|
|
destroy_timer_on_stack(&timer);
|
|
|
|
}
|
|
|
|
}
|
2013-12-12 23:54:42 +08:00
|
|
|
if (!irq_test_in_progress)
|
2016-03-16 19:00:36 +08:00
|
|
|
engine->irq_put(engine);
|
2013-09-26 00:34:55 +08:00
|
|
|
|
2016-03-16 19:00:36 +08:00
|
|
|
finish_wait(&engine->irq_queue, &wait);
|
2012-08-24 16:35:08 +08:00
|
|
|
|
2015-04-07 23:20:41 +08:00
|
|
|
out:
|
|
|
|
trace_i915_gem_request_wait_end(req);
|
|
|
|
|
2012-08-24 16:35:08 +08:00
|
|
|
if (timeout) {
|
2016-01-15 23:11:12 +08:00
|
|
|
s64 tres = *timeout - (ktime_get_raw_ns() - before);
|
2014-07-17 05:05:06 +08:00
|
|
|
|
|
|
|
*timeout = tres < 0 ? 0 : tres;
|
2014-11-28 17:29:55 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Apparently ktime isn't accurate enough and occasionally has a
|
|
|
|
* bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
|
|
|
|
* things up to make the test happy. We allow up to 1 jiffy.
|
|
|
|
*
|
|
|
|
* This is a regrssion from the timespec->ktime conversion.
|
|
|
|
*/
|
|
|
|
if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
|
|
|
|
*timeout = 0;
|
2012-08-24 16:35:08 +08:00
|
|
|
}
|
|
|
|
|
2013-09-26 00:34:55 +08:00
|
|
|
return ret;
|
2012-08-24 16:35:08 +08:00
|
|
|
}
|
|
|
|
|
2015-05-30 00:44:12 +08:00
|
|
|
int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
|
|
|
|
struct drm_file *file)
|
|
|
|
{
|
|
|
|
struct drm_i915_file_private *file_priv;
|
|
|
|
|
|
|
|
WARN_ON(!req || !file || req->file_priv);
|
|
|
|
|
|
|
|
if (!req || !file)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (req->file_priv)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
file_priv = file->driver_priv;
|
|
|
|
|
|
|
|
spin_lock(&file_priv->mm.lock);
|
|
|
|
req->file_priv = file_priv;
|
|
|
|
list_add_tail(&req->client_list, &file_priv->mm.request_list);
|
|
|
|
spin_unlock(&file_priv->mm.lock);
|
|
|
|
|
|
|
|
req->pid = get_pid(task_pid(current));
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
static inline void
|
|
|
|
i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
|
|
|
|
{
|
|
|
|
struct drm_i915_file_private *file_priv = request->file_priv;
|
|
|
|
|
|
|
|
if (!file_priv)
|
|
|
|
return;
|
|
|
|
|
|
|
|
spin_lock(&file_priv->mm.lock);
|
|
|
|
list_del(&request->client_list);
|
|
|
|
request->file_priv = NULL;
|
|
|
|
spin_unlock(&file_priv->mm.lock);
|
2015-05-30 00:44:12 +08:00
|
|
|
|
|
|
|
put_pid(request->pid);
|
|
|
|
request->pid = NULL;
|
2015-04-27 20:41:17 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static void i915_gem_request_retire(struct drm_i915_gem_request *request)
|
|
|
|
{
|
|
|
|
trace_i915_gem_request_retire(request);
|
|
|
|
|
|
|
|
/* We know the GPU must have read the request to have
|
|
|
|
* sent us the seqno + interrupt, so use the position
|
|
|
|
* of tail of the request to update the last known position
|
|
|
|
* of the GPU head.
|
|
|
|
*
|
|
|
|
* Note this requires that we are always called in request
|
|
|
|
* completion order.
|
|
|
|
*/
|
|
|
|
request->ringbuf->last_retired_head = request->postfix;
|
|
|
|
|
|
|
|
list_del_init(&request->list);
|
|
|
|
i915_gem_request_remove_from_client(request);
|
|
|
|
|
|
|
|
i915_gem_request_unreference(request);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
__i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
|
|
|
|
{
|
2016-03-16 19:00:38 +08:00
|
|
|
struct intel_engine_cs *engine = req->engine;
|
2015-04-27 20:41:17 +08:00
|
|
|
struct drm_i915_gem_request *tmp;
|
|
|
|
|
|
|
|
lockdep_assert_held(&engine->dev->struct_mutex);
|
|
|
|
|
|
|
|
if (list_empty(&req->list))
|
|
|
|
return;
|
|
|
|
|
|
|
|
do {
|
|
|
|
tmp = list_first_entry(&engine->request_list,
|
|
|
|
typeof(*tmp), list);
|
|
|
|
|
|
|
|
i915_gem_request_retire(tmp);
|
|
|
|
} while (tmp != req);
|
|
|
|
|
|
|
|
WARN_ON(i915_verify_lists(engine->dev));
|
|
|
|
}
|
|
|
|
|
2012-08-24 16:35:08 +08:00
|
|
|
/**
|
2014-11-26 21:17:05 +08:00
|
|
|
* Waits for a request to be signaled, and cleans up the
|
2012-08-24 16:35:08 +08:00
|
|
|
* request and object lists appropriately for that event.
|
|
|
|
*/
|
|
|
|
int
|
2014-11-26 21:17:05 +08:00
|
|
|
i915_wait_request(struct drm_i915_gem_request *req)
|
2012-08-24 16:35:08 +08:00
|
|
|
{
|
2014-11-26 21:17:05 +08:00
|
|
|
struct drm_device *dev;
|
|
|
|
struct drm_i915_private *dev_priv;
|
|
|
|
bool interruptible;
|
2012-08-24 16:35:08 +08:00
|
|
|
int ret;
|
|
|
|
|
2014-11-26 21:17:05 +08:00
|
|
|
BUG_ON(req == NULL);
|
|
|
|
|
2016-03-16 19:00:38 +08:00
|
|
|
dev = req->engine->dev;
|
2014-11-26 21:17:05 +08:00
|
|
|
dev_priv = dev->dev_private;
|
|
|
|
interruptible = dev_priv->mm.interruptible;
|
|
|
|
|
2012-08-24 16:35:08 +08:00
|
|
|
BUG_ON(!mutex_is_locked(&dev->struct_mutex));
|
|
|
|
|
2012-11-15 00:14:05 +08:00
|
|
|
ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
|
2012-08-24 16:35:08 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
ret = __i915_wait_request(req,
|
2016-04-14 00:35:03 +08:00
|
|
|
i915_reset_counter(&dev_priv->gpu_error),
|
2014-11-25 02:49:35 +08:00
|
|
|
interruptible, NULL, NULL);
|
2015-04-27 20:41:17 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
2013-06-30 05:05:26 +08:00
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
__i915_gem_request_retire__upto(req);
|
2013-06-30 05:05:26 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2012-08-24 16:35:08 +08:00
|
|
|
/**
|
|
|
|
* Ensures that all rendering to the object has completed and the object is
|
|
|
|
* safe to unbind from the GTT or access from the CPU.
|
|
|
|
*/
|
2015-04-27 20:41:14 +08:00
|
|
|
int
|
2012-08-24 16:35:08 +08:00
|
|
|
i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
|
|
|
|
bool readonly)
|
|
|
|
{
|
2015-04-27 20:41:17 +08:00
|
|
|
int ret, i;
|
2012-08-24 16:35:08 +08:00
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
if (!obj->active)
|
2012-08-24 16:35:08 +08:00
|
|
|
return 0;
|
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
if (readonly) {
|
|
|
|
if (obj->last_write_req != NULL) {
|
|
|
|
ret = i915_wait_request(obj->last_write_req);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2012-08-24 16:35:08 +08:00
|
|
|
|
2016-03-16 19:00:38 +08:00
|
|
|
i = obj->last_write_req->engine->id;
|
2015-04-27 20:41:17 +08:00
|
|
|
if (obj->last_read_req[i] == obj->last_write_req)
|
|
|
|
i915_gem_object_retire__read(obj, i);
|
|
|
|
else
|
|
|
|
i915_gem_object_retire__write(obj);
|
|
|
|
}
|
|
|
|
} else {
|
2016-03-16 19:00:39 +08:00
|
|
|
for (i = 0; i < I915_NUM_ENGINES; i++) {
|
2015-04-27 20:41:17 +08:00
|
|
|
if (obj->last_read_req[i] == NULL)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
ret = i915_wait_request(obj->last_read_req[i]);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
i915_gem_object_retire__read(obj, i);
|
|
|
|
}
|
2016-04-14 00:35:02 +08:00
|
|
|
GEM_BUG_ON(obj->active);
|
2015-04-27 20:41:17 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
|
|
|
|
struct drm_i915_gem_request *req)
|
|
|
|
{
|
2016-03-16 19:00:38 +08:00
|
|
|
int ring = req->engine->id;
|
2015-04-27 20:41:17 +08:00
|
|
|
|
|
|
|
if (obj->last_read_req[ring] == req)
|
|
|
|
i915_gem_object_retire__read(obj, ring);
|
|
|
|
else if (obj->last_write_req == req)
|
|
|
|
i915_gem_object_retire__write(obj);
|
|
|
|
|
|
|
|
__i915_gem_request_retire__upto(req);
|
2012-08-24 16:35:08 +08:00
|
|
|
}
|
|
|
|
|
2012-08-24 16:35:09 +08:00
|
|
|
/* A nonblocking variant of the above wait. This is a highly dangerous routine
|
|
|
|
* as the object state may change during this call.
|
|
|
|
*/
|
|
|
|
static __must_check int
|
|
|
|
i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
|
2015-04-27 20:41:22 +08:00
|
|
|
struct intel_rps_client *rps,
|
2012-08-24 16:35:09 +08:00
|
|
|
bool readonly)
|
|
|
|
{
|
|
|
|
struct drm_device *dev = obj->base.dev;
|
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2016-03-16 19:00:39 +08:00
|
|
|
struct drm_i915_gem_request *requests[I915_NUM_ENGINES];
|
drm/i915: create a race-free reset detection
With the previous patch the state transition handling of the reset
code itself is now (hopefully) race free and solid. But that still
leaves out everyone else - with the various lock-free wait paths
we have there's the possibility that the reset happens between the
point where we read the seqno we should wait on and the actual wait.
And if __wait_seqno then never sees the RESET_IN_PROGRESS state, we'll
happily wait for a seqno which will in all likelyhood never signal.
In practice this is not a big problem since the X server gets
constantly interrupted, and can then submit more work (hopefully) to
unblock everyone else: As soon as a new seqno write lands, all waiters
will unblock. But running the i-g-t reset testcase ZZ_hangman can
expose this race, especially on slower hw with fewer cpu cores.
Now looking forward to ARB_robustness and friends that's not the best
possible behaviour, hence this patch adds a reset_counter to be able
to detect any reset, even if a given thread never observed the
in-progress state.
The important part is to correctly order things:
- The write side needs to increment the counter after any seqno gets
reset. Hence we need to do that at the end of the reset work, and
again wake everyone up. We also need to place a barrier in between
any possible seqno changes and the counter increment, since any
unlock operations only guarantee that nothing leaks out, but not
that at later load operation gets moved ahead.
- On the read side we need to ensure that no reset can sneak in and
invalidate the seqno. In all cases we can use the one-sided barrier
that unlock operations guarantee (of the lock protecting the
respective seqno/ring pair) to ensure correct ordering. Hence it is
sufficient to place the atomic read before the mutex/spin_unlock and
no additional barriers are required.
The end-result of all this is that we need to wake up everyone twice
in a reset operation:
- First, before the reset starts, to get any lockholders of the locks,
so that the reset can proceed.
- Second, after the reset is completed, to allow waiters to properly
and reliably detect the reset condition and bail out.
I admit that this entire reset_counter thing smells a bit like
overkill, but I think it's justified since it makes it really explicit
what the bail-out condition is. And we need a reset counter anyway to
implement ARB_robustness, and imo with finer-grained locking on the
horizont this is the most resilient scheme I could think of.
v2: Drop spurious change in the wait_for_error EXIT_COND - we only
need to wait until we leave the reset-in-progress wedged state.
v3: Don't play tricks with barriers in the throttle ioctl, the
spin_unlock is barrier enough.
I've also considered using a little helper to grab the current
reset_counter, but then decided that hiding the atomic_read isn't a
great idea, since having it explicitly show up in the code is a nice
remainder to reviews to check the memory barriers.
v4: Add a comment to explain why we need to fall through in
__wait_seqno in the end variable assignments.
v5: Review from Damien:
- s/smb/smp/ in a comment
- don't increment the reset counter after we've set it to WEDGED. Now
we (again) properly wedge the gpu when the reset fails.
Reviewed-by: Damien Lespiau <damien.lespiau@intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-12-06 16:01:42 +08:00
|
|
|
unsigned reset_counter;
|
2015-04-27 20:41:17 +08:00
|
|
|
int ret, i, n = 0;
|
2012-08-24 16:35:09 +08:00
|
|
|
|
|
|
|
BUG_ON(!mutex_is_locked(&dev->struct_mutex));
|
|
|
|
BUG_ON(!dev_priv->mm.interruptible);
|
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
if (!obj->active)
|
2012-08-24 16:35:09 +08:00
|
|
|
return 0;
|
|
|
|
|
2012-11-15 00:14:05 +08:00
|
|
|
ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
|
2012-08-24 16:35:09 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2016-04-14 00:35:03 +08:00
|
|
|
reset_counter = i915_reset_counter(&dev_priv->gpu_error);
|
2015-04-27 20:41:17 +08:00
|
|
|
|
|
|
|
if (readonly) {
|
|
|
|
struct drm_i915_gem_request *req;
|
|
|
|
|
|
|
|
req = obj->last_write_req;
|
|
|
|
if (req == NULL)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
requests[n++] = i915_gem_request_reference(req);
|
|
|
|
} else {
|
2016-03-16 19:00:39 +08:00
|
|
|
for (i = 0; i < I915_NUM_ENGINES; i++) {
|
2015-04-27 20:41:17 +08:00
|
|
|
struct drm_i915_gem_request *req;
|
|
|
|
|
|
|
|
req = obj->last_read_req[i];
|
|
|
|
if (req == NULL)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
requests[n++] = i915_gem_request_reference(req);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-08-24 16:35:09 +08:00
|
|
|
mutex_unlock(&dev->struct_mutex);
|
2015-04-27 20:41:17 +08:00
|
|
|
for (i = 0; ret == 0 && i < n; i++)
|
|
|
|
ret = __i915_wait_request(requests[i], reset_counter, true,
|
2015-04-27 20:41:22 +08:00
|
|
|
NULL, rps);
|
2012-08-24 16:35:09 +08:00
|
|
|
mutex_lock(&dev->struct_mutex);
|
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
for (i = 0; i < n; i++) {
|
|
|
|
if (ret == 0)
|
|
|
|
i915_gem_object_retire_request(obj, requests[i]);
|
|
|
|
i915_gem_request_unreference(requests[i]);
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret;
|
2012-08-24 16:35:09 +08:00
|
|
|
}
|
|
|
|
|
2015-04-27 20:41:22 +08:00
|
|
|
static struct intel_rps_client *to_rps_client(struct drm_file *file)
|
|
|
|
{
|
|
|
|
struct drm_i915_file_private *fpriv = file->driver_priv;
|
|
|
|
return &fpriv->rps;
|
|
|
|
}
|
|
|
|
|
2008-07-31 03:06:12 +08:00
|
|
|
/**
|
2008-11-11 02:53:25 +08:00
|
|
|
* Called when user space prepares to use an object with the CPU, either
|
|
|
|
* through the mmap ioctl's mapping or a GTT mapping.
|
2008-07-31 03:06:12 +08:00
|
|
|
*/
|
|
|
|
int
|
|
|
|
i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_file *file)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
|
|
|
struct drm_i915_gem_set_domain *args = data;
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_i915_gem_object *obj;
|
2008-11-11 02:53:25 +08:00
|
|
|
uint32_t read_domains = args->read_domains;
|
|
|
|
uint32_t write_domain = args->write_domain;
|
2008-07-31 03:06:12 +08:00
|
|
|
int ret;
|
|
|
|
|
2008-11-11 02:53:25 +08:00
|
|
|
/* Only handle setting domains to types used by the CPU. */
|
2009-06-06 16:46:02 +08:00
|
|
|
if (write_domain & I915_GEM_GPU_DOMAINS)
|
2008-11-11 02:53:25 +08:00
|
|
|
return -EINVAL;
|
|
|
|
|
2009-06-06 16:46:02 +08:00
|
|
|
if (read_domains & I915_GEM_GPU_DOMAINS)
|
2008-11-11 02:53:25 +08:00
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/* Having something in the write domain implies it's in the read
|
|
|
|
* domain, and only that read domain. Enforce that in the request.
|
|
|
|
*/
|
|
|
|
if (write_domain != 0 && read_domains != write_domain)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2010-09-25 18:22:51 +08:00
|
|
|
ret = i915_mutex_lock_interruptible(dev);
|
2010-10-17 16:45:41 +08:00
|
|
|
if (ret)
|
2010-09-25 18:22:51 +08:00
|
|
|
return ret;
|
2010-10-17 16:45:41 +08:00
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
|
2011-02-19 19:31:06 +08:00
|
|
|
if (&obj->base == NULL) {
|
2010-10-17 16:45:41 +08:00
|
|
|
ret = -ENOENT;
|
|
|
|
goto unlock;
|
2010-09-25 18:22:51 +08:00
|
|
|
}
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2012-08-24 16:35:09 +08:00
|
|
|
/* Try to flush the object off the GPU without holding the lock.
|
|
|
|
* We will repeat the flush holding the lock in the normal manner
|
|
|
|
* to catch cases where we are gazumped.
|
|
|
|
*/
|
2014-02-08 04:37:06 +08:00
|
|
|
ret = i915_gem_object_wait_rendering__nonblocking(obj,
|
2015-04-27 20:41:22 +08:00
|
|
|
to_rps_client(file),
|
2014-02-08 04:37:06 +08:00
|
|
|
!write_domain);
|
2012-08-24 16:35:09 +08:00
|
|
|
if (ret)
|
|
|
|
goto unref;
|
|
|
|
|
2015-01-02 18:59:29 +08:00
|
|
|
if (read_domains & I915_GEM_DOMAIN_GTT)
|
2008-11-11 02:53:25 +08:00
|
|
|
ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
|
2015-01-02 18:59:29 +08:00
|
|
|
else
|
2008-11-15 05:35:19 +08:00
|
|
|
ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
|
2008-11-11 02:53:25 +08:00
|
|
|
|
2015-06-27 01:35:16 +08:00
|
|
|
if (write_domain != 0)
|
|
|
|
intel_fb_obj_invalidate(obj,
|
|
|
|
write_domain == I915_GEM_DOMAIN_GTT ?
|
|
|
|
ORIGIN_GTT : ORIGIN_CPU);
|
|
|
|
|
2012-08-24 16:35:09 +08:00
|
|
|
unref:
|
2010-11-09 03:18:58 +08:00
|
|
|
drm_gem_object_unreference(&obj->base);
|
2010-10-17 16:45:41 +08:00
|
|
|
unlock:
|
2008-07-31 03:06:12 +08:00
|
|
|
mutex_unlock(&dev->struct_mutex);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Called when user space has done writes to this buffer
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_file *file)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
|
|
|
struct drm_i915_gem_sw_finish *args = data;
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_i915_gem_object *obj;
|
2008-07-31 03:06:12 +08:00
|
|
|
int ret = 0;
|
|
|
|
|
2010-09-25 18:22:51 +08:00
|
|
|
ret = i915_mutex_lock_interruptible(dev);
|
2010-10-17 16:45:41 +08:00
|
|
|
if (ret)
|
2010-09-25 18:22:51 +08:00
|
|
|
return ret;
|
2010-10-17 16:45:41 +08:00
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
|
2011-02-19 19:31:06 +08:00
|
|
|
if (&obj->base == NULL) {
|
2010-10-17 16:45:41 +08:00
|
|
|
ret = -ENOENT;
|
|
|
|
goto unlock;
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Pinned buffers may be scanout, so flush the cache */
|
2013-08-09 19:26:45 +08:00
|
|
|
if (obj->pin_display)
|
2015-01-21 21:53:48 +08:00
|
|
|
i915_gem_object_flush_cpu_write_domain(obj);
|
2008-11-15 05:35:19 +08:00
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
drm_gem_object_unreference(&obj->base);
|
2010-10-17 16:45:41 +08:00
|
|
|
unlock:
|
2008-07-31 03:06:12 +08:00
|
|
|
mutex_unlock(&dev->struct_mutex);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Maps the contents of an object, returning the address it is mapped
|
|
|
|
* into.
|
|
|
|
*
|
|
|
|
* While the mapping holds a reference on the contents of the object, it doesn't
|
|
|
|
* imply a ref on the object itself.
|
2014-10-16 18:28:18 +08:00
|
|
|
*
|
|
|
|
* IMPORTANT:
|
|
|
|
*
|
|
|
|
* DRM driver writers who look a this function as an example for how to do GEM
|
|
|
|
* mmap support, please don't implement mmap support like here. The modern way
|
|
|
|
* to implement DRM mmap support is with an mmap offset ioctl (like
|
|
|
|
* i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
|
|
|
|
* That way debug tooling like valgrind will understand what's going on, hiding
|
|
|
|
* the mmap call in a driver private ioctl will break that. The i915 driver only
|
|
|
|
* does cpu mmaps this way because we didn't know better.
|
2008-07-31 03:06:12 +08:00
|
|
|
*/
|
|
|
|
int
|
|
|
|
i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_file *file)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
|
|
|
struct drm_i915_gem_mmap *args = data;
|
|
|
|
struct drm_gem_object *obj;
|
|
|
|
unsigned long addr;
|
|
|
|
|
drm/i915: Support creation of unbound wc user mappings for objects
This patch provides support to create write-combining virtual mappings of
GEM object. It intends to provide the same funtionality of 'mmap_gtt'
interface without the constraints and contention of a limited aperture
space, but requires clients handles the linear to tile conversion on their
own. This is for improving the CPU write operation performance, as with such
mapping, writes and reads are almost 50% faster than with mmap_gtt. Similar
to the GTT mmapping, unlike the regular CPU mmapping, it avoids the cache
flush after update from CPU side, when object is passed onto GPU. This
type of mapping is specially useful in case of sub-region update,
i.e. when only a portion of the object is to be updated. Using a CPU mmap
in such cases would normally incur a clflush of the whole object, and
using a GTT mmapping would likely require eviction of an active object or
fence and thus stall. The write-combining CPU mmap avoids both.
To ensure the cache coherency, before using this mapping, the GTT domain
has been reused here. This provides the required cache flush if the object
is in CPU domain or synchronization against the concurrent rendering.
Although the access through an uncached mmap should automatically
invalidate the cache lines, this may not be true for non-temporal write
instructions and also not all pages of the object may be updated at any
given point of time through this mapping. Having a call to get_pages in
set_to_gtt_domain function, as added in the earlier patch 'drm/i915:
Broaden application of set-domain(GTT)', would guarantee the clflush and
so there will be no cachelines holding the data for the object before it
is accessed through this map.
The drm_i915_gem_mmap structure (for the DRM_I915_GEM_MMAP_IOCTL) has been
extended with a new flags field (defaulting to 0 for existent users). In
order for userspace to detect the extended ioctl, a new parameter
I915_PARAM_MMAP_VERSION has been added for versioning the ioctl interface.
v2: Fix error handling, invalid flag detection, renaming (ickle)
v3: Rebase to latest drm-intel-nightly codebase
The new mmapping is exercised by igt/gem_mmap_wc,
igt/gem_concurrent_blit and igt/gem_gtt_speed.
Change-Id: Ie883942f9e689525f72fe9a8d3780c3a9faa769a
Signed-off-by: Akash Goel <akash.goel@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2015-01-02 18:59:30 +08:00
|
|
|
if (args->flags & ~(I915_MMAP_WC))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (args->flags & I915_MMAP_WC && !cpu_has_pat)
|
|
|
|
return -ENODEV;
|
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
obj = drm_gem_object_lookup(dev, file, args->handle);
|
2008-07-31 03:06:12 +08:00
|
|
|
if (obj == NULL)
|
2010-08-04 21:19:46 +08:00
|
|
|
return -ENOENT;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2012-05-10 21:25:09 +08:00
|
|
|
/* prime objects have no backing filp to GEM mmap
|
|
|
|
* pages from.
|
|
|
|
*/
|
|
|
|
if (!obj->filp) {
|
|
|
|
drm_gem_object_unreference_unlocked(obj);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2012-04-21 08:13:58 +08:00
|
|
|
addr = vm_mmap(obj->filp, 0, args->size,
|
2008-07-31 03:06:12 +08:00
|
|
|
PROT_READ | PROT_WRITE, MAP_SHARED,
|
|
|
|
args->offset);
|
drm/i915: Support creation of unbound wc user mappings for objects
This patch provides support to create write-combining virtual mappings of
GEM object. It intends to provide the same funtionality of 'mmap_gtt'
interface without the constraints and contention of a limited aperture
space, but requires clients handles the linear to tile conversion on their
own. This is for improving the CPU write operation performance, as with such
mapping, writes and reads are almost 50% faster than with mmap_gtt. Similar
to the GTT mmapping, unlike the regular CPU mmapping, it avoids the cache
flush after update from CPU side, when object is passed onto GPU. This
type of mapping is specially useful in case of sub-region update,
i.e. when only a portion of the object is to be updated. Using a CPU mmap
in such cases would normally incur a clflush of the whole object, and
using a GTT mmapping would likely require eviction of an active object or
fence and thus stall. The write-combining CPU mmap avoids both.
To ensure the cache coherency, before using this mapping, the GTT domain
has been reused here. This provides the required cache flush if the object
is in CPU domain or synchronization against the concurrent rendering.
Although the access through an uncached mmap should automatically
invalidate the cache lines, this may not be true for non-temporal write
instructions and also not all pages of the object may be updated at any
given point of time through this mapping. Having a call to get_pages in
set_to_gtt_domain function, as added in the earlier patch 'drm/i915:
Broaden application of set-domain(GTT)', would guarantee the clflush and
so there will be no cachelines holding the data for the object before it
is accessed through this map.
The drm_i915_gem_mmap structure (for the DRM_I915_GEM_MMAP_IOCTL) has been
extended with a new flags field (defaulting to 0 for existent users). In
order for userspace to detect the extended ioctl, a new parameter
I915_PARAM_MMAP_VERSION has been added for versioning the ioctl interface.
v2: Fix error handling, invalid flag detection, renaming (ickle)
v3: Rebase to latest drm-intel-nightly codebase
The new mmapping is exercised by igt/gem_mmap_wc,
igt/gem_concurrent_blit and igt/gem_gtt_speed.
Change-Id: Ie883942f9e689525f72fe9a8d3780c3a9faa769a
Signed-off-by: Akash Goel <akash.goel@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2015-01-02 18:59:30 +08:00
|
|
|
if (args->flags & I915_MMAP_WC) {
|
|
|
|
struct mm_struct *mm = current->mm;
|
|
|
|
struct vm_area_struct *vma;
|
|
|
|
|
|
|
|
down_write(&mm->mmap_sem);
|
|
|
|
vma = find_vma(mm, addr);
|
|
|
|
if (vma)
|
|
|
|
vma->vm_page_prot =
|
|
|
|
pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
|
|
|
|
else
|
|
|
|
addr = -ENOMEM;
|
|
|
|
up_write(&mm->mmap_sem);
|
|
|
|
}
|
2010-02-09 13:49:12 +08:00
|
|
|
drm_gem_object_unreference_unlocked(obj);
|
2008-07-31 03:06:12 +08:00
|
|
|
if (IS_ERR((void *)addr))
|
|
|
|
return addr;
|
|
|
|
|
|
|
|
args->addr_ptr = (uint64_t) addr;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-11-13 02:03:55 +08:00
|
|
|
/**
|
|
|
|
* i915_gem_fault - fault a page into the GTT
|
2015-09-15 20:58:44 +08:00
|
|
|
* @vma: VMA in question
|
|
|
|
* @vmf: fault info
|
2008-11-13 02:03:55 +08:00
|
|
|
*
|
|
|
|
* The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
|
|
|
|
* from userspace. The fault handler takes care of binding the object to
|
|
|
|
* the GTT (if needed), allocating and programming a fence register (again,
|
|
|
|
* only if needed based on whether the old reg is still valid or the object
|
|
|
|
* is tiled) and inserting a new PTE into the faulting process.
|
|
|
|
*
|
|
|
|
* Note that the faulting process may involve evicting existing objects
|
|
|
|
* from the GTT and/or fence registers to make room. So performance may
|
|
|
|
* suffer if the GTT working set is large or there are few fence registers
|
|
|
|
* left.
|
|
|
|
*/
|
|
|
|
int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
|
|
|
|
{
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
|
|
|
|
struct drm_device *dev = obj->base.dev;
|
2016-03-30 21:57:10 +08:00
|
|
|
struct drm_i915_private *dev_priv = to_i915(dev);
|
|
|
|
struct i915_ggtt *ggtt = &dev_priv->ggtt;
|
2015-05-06 19:36:09 +08:00
|
|
|
struct i915_ggtt_view view = i915_ggtt_view_normal;
|
2008-11-13 02:03:55 +08:00
|
|
|
pgoff_t page_offset;
|
|
|
|
unsigned long pfn;
|
|
|
|
int ret = 0;
|
2009-01-27 09:10:45 +08:00
|
|
|
bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
|
2008-11-13 02:03:55 +08:00
|
|
|
|
2013-11-28 04:20:34 +08:00
|
|
|
intel_runtime_pm_get(dev_priv);
|
|
|
|
|
2008-11-13 02:03:55 +08:00
|
|
|
/* We don't use vmf->pgoff since that has the fake offset */
|
|
|
|
page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
|
|
|
|
PAGE_SHIFT;
|
|
|
|
|
2011-02-07 21:09:31 +08:00
|
|
|
ret = i915_mutex_lock_interruptible(dev);
|
|
|
|
if (ret)
|
|
|
|
goto out;
|
2010-09-25 04:15:47 +08:00
|
|
|
|
2011-02-03 19:57:46 +08:00
|
|
|
trace_i915_gem_object_fault(obj, page_offset, true, write);
|
|
|
|
|
2014-02-08 04:37:06 +08:00
|
|
|
/* Try to flush the object off the GPU first without holding the lock.
|
|
|
|
* Upon reacquiring the lock, we will perform our sanity checks and then
|
|
|
|
* repeat the flush holding the lock in the normal manner to catch cases
|
|
|
|
* where we are gazumped.
|
|
|
|
*/
|
|
|
|
ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
|
|
|
|
if (ret)
|
|
|
|
goto unlock;
|
|
|
|
|
2012-12-16 20:43:36 +08:00
|
|
|
/* Access to snoopable pages through the GTT is incoherent. */
|
|
|
|
if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
|
2014-05-28 23:16:41 +08:00
|
|
|
ret = -EFAULT;
|
2012-12-16 20:43:36 +08:00
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
|
2015-05-06 19:36:09 +08:00
|
|
|
/* Use a partial view if the object is bigger than the aperture. */
|
2016-03-30 21:57:10 +08:00
|
|
|
if (obj->base.size >= ggtt->mappable_end &&
|
2015-05-08 19:37:39 +08:00
|
|
|
obj->tiling_mode == I915_TILING_NONE) {
|
2015-05-06 19:36:09 +08:00
|
|
|
static const unsigned int chunk_size = 256; // 1 MiB
|
2015-05-08 19:37:39 +08:00
|
|
|
|
2015-05-06 19:36:09 +08:00
|
|
|
memset(&view, 0, sizeof(view));
|
|
|
|
view.type = I915_GGTT_VIEW_PARTIAL;
|
|
|
|
view.params.partial.offset = rounddown(page_offset, chunk_size);
|
|
|
|
view.params.partial.size =
|
|
|
|
min_t(unsigned int,
|
|
|
|
chunk_size,
|
|
|
|
(vma->vm_end - vma->vm_start)/PAGE_SIZE -
|
|
|
|
view.params.partial.offset);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Now pin it into the GTT if needed */
|
|
|
|
ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE);
|
2012-11-20 18:45:17 +08:00
|
|
|
if (ret)
|
|
|
|
goto unlock;
|
2010-10-28 21:44:08 +08:00
|
|
|
|
2012-11-20 18:45:17 +08:00
|
|
|
ret = i915_gem_object_set_to_gtt_domain(obj, write);
|
|
|
|
if (ret)
|
|
|
|
goto unpin;
|
2012-02-16 06:50:22 +08:00
|
|
|
|
2012-04-17 22:31:24 +08:00
|
|
|
ret = i915_gem_object_get_fence(obj);
|
2010-11-11 00:40:20 +08:00
|
|
|
if (ret)
|
2012-11-20 18:45:17 +08:00
|
|
|
goto unpin;
|
2010-08-08 04:45:03 +08:00
|
|
|
|
2014-06-10 19:14:40 +08:00
|
|
|
/* Finally, remap it using the new GTT offset */
|
2016-03-30 21:57:10 +08:00
|
|
|
pfn = ggtt->mappable_base +
|
2015-05-06 19:36:09 +08:00
|
|
|
i915_gem_obj_ggtt_offset_view(obj, &view);
|
2013-07-06 05:41:04 +08:00
|
|
|
pfn >>= PAGE_SHIFT;
|
2008-11-13 02:03:55 +08:00
|
|
|
|
2015-05-06 19:36:09 +08:00
|
|
|
if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
|
|
|
|
/* Overriding existing pages in partial view does not cause
|
|
|
|
* us any trouble as TLBs are still valid because the fault
|
|
|
|
* is due to userspace losing part of the mapping or never
|
|
|
|
* having accessed it before (at this partials' range).
|
|
|
|
*/
|
|
|
|
unsigned long base = vma->vm_start +
|
|
|
|
(view.params.partial.offset << PAGE_SHIFT);
|
|
|
|
unsigned int i;
|
2014-06-10 19:14:40 +08:00
|
|
|
|
2015-05-06 19:36:09 +08:00
|
|
|
for (i = 0; i < view.params.partial.size; i++) {
|
|
|
|
ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
|
2014-06-10 19:14:40 +08:00
|
|
|
if (ret)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
obj->fault_mappable = true;
|
2015-05-06 19:36:09 +08:00
|
|
|
} else {
|
|
|
|
if (!obj->fault_mappable) {
|
|
|
|
unsigned long size = min_t(unsigned long,
|
|
|
|
vma->vm_end - vma->vm_start,
|
|
|
|
obj->base.size);
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < size >> PAGE_SHIFT; i++) {
|
|
|
|
ret = vm_insert_pfn(vma,
|
|
|
|
(unsigned long)vma->vm_start + i * PAGE_SIZE,
|
|
|
|
pfn + i);
|
|
|
|
if (ret)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
obj->fault_mappable = true;
|
|
|
|
} else
|
|
|
|
ret = vm_insert_pfn(vma,
|
|
|
|
(unsigned long)vmf->virtual_address,
|
|
|
|
pfn + page_offset);
|
|
|
|
}
|
2012-11-20 18:45:17 +08:00
|
|
|
unpin:
|
2015-05-06 19:36:09 +08:00
|
|
|
i915_gem_object_ggtt_unpin_view(obj, &view);
|
2009-09-23 07:43:56 +08:00
|
|
|
unlock:
|
2008-11-13 02:03:55 +08:00
|
|
|
mutex_unlock(&dev->struct_mutex);
|
2011-02-07 21:09:31 +08:00
|
|
|
out:
|
2008-11-13 02:03:55 +08:00
|
|
|
switch (ret) {
|
2011-02-07 21:09:31 +08:00
|
|
|
case -EIO:
|
2014-09-04 15:36:18 +08:00
|
|
|
/*
|
|
|
|
* We eat errors when the gpu is terminally wedged to avoid
|
|
|
|
* userspace unduly crashing (gl has no provisions for mmaps to
|
|
|
|
* fail). But any other -EIO isn't ours (e.g. swap in failure)
|
|
|
|
* and so needs to be reported.
|
|
|
|
*/
|
|
|
|
if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
|
2013-11-28 04:20:34 +08:00
|
|
|
ret = VM_FAULT_SIGBUS;
|
|
|
|
break;
|
|
|
|
}
|
2010-11-07 17:18:22 +08:00
|
|
|
case -EAGAIN:
|
2013-09-12 23:57:28 +08:00
|
|
|
/*
|
|
|
|
* EAGAIN means the gpu is hung and we'll wait for the error
|
|
|
|
* handler to reset everything when re-faulting in
|
|
|
|
* i915_mutex_lock_interruptible.
|
2011-02-07 21:09:31 +08:00
|
|
|
*/
|
2009-09-23 07:43:56 +08:00
|
|
|
case 0:
|
|
|
|
case -ERESTARTSYS:
|
2011-02-12 04:31:19 +08:00
|
|
|
case -EINTR:
|
2012-10-03 22:15:26 +08:00
|
|
|
case -EBUSY:
|
|
|
|
/*
|
|
|
|
* EBUSY is ok: this just means that another thread
|
|
|
|
* already did the job.
|
|
|
|
*/
|
2013-11-28 04:20:34 +08:00
|
|
|
ret = VM_FAULT_NOPAGE;
|
|
|
|
break;
|
2008-11-13 02:03:55 +08:00
|
|
|
case -ENOMEM:
|
2013-11-28 04:20:34 +08:00
|
|
|
ret = VM_FAULT_OOM;
|
|
|
|
break;
|
2012-10-17 17:17:16 +08:00
|
|
|
case -ENOSPC:
|
2014-01-31 19:34:57 +08:00
|
|
|
case -EFAULT:
|
2013-11-28 04:20:34 +08:00
|
|
|
ret = VM_FAULT_SIGBUS;
|
|
|
|
break;
|
2008-11-13 02:03:55 +08:00
|
|
|
default:
|
2012-10-17 17:17:16 +08:00
|
|
|
WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
|
2013-11-28 04:20:34 +08:00
|
|
|
ret = VM_FAULT_SIGBUS;
|
|
|
|
break;
|
2008-11-13 02:03:55 +08:00
|
|
|
}
|
2013-11-28 04:20:34 +08:00
|
|
|
|
|
|
|
intel_runtime_pm_put(dev_priv);
|
|
|
|
return ret;
|
2008-11-13 02:03:55 +08:00
|
|
|
}
|
|
|
|
|
2009-07-10 15:18:50 +08:00
|
|
|
/**
|
|
|
|
* i915_gem_release_mmap - remove physical page mappings
|
|
|
|
* @obj: obj in question
|
|
|
|
*
|
tree-wide: fix assorted typos all over the place
That is "success", "unknown", "through", "performance", "[re|un]mapping"
, "access", "default", "reasonable", "[con]currently", "temperature"
, "channel", "[un]used", "application", "example","hierarchy", "therefore"
, "[over|under]flow", "contiguous", "threshold", "enough" and others.
Signed-off-by: André Goddard Rosa <andre.goddard@gmail.com>
Signed-off-by: Jiri Kosina <jkosina@suse.cz>
2009-11-14 23:09:05 +08:00
|
|
|
* Preserve the reservation of the mmapping with the DRM core code, but
|
2009-07-10 15:18:50 +08:00
|
|
|
* relinquish ownership of the pages back to the system.
|
|
|
|
*
|
|
|
|
* It is vital that we remove the page mapping if we have mapped a tiled
|
|
|
|
* object through the GTT and then lose the fence register due to
|
|
|
|
* resource pressure. Similarly if the object has been moved out of the
|
|
|
|
* aperture, than pages mapped into userspace must be revoked. Removing the
|
|
|
|
* mapping will then trigger a page fault on the next user access, allowing
|
|
|
|
* fixup by i915_gem_fault().
|
|
|
|
*/
|
2009-07-11 04:02:26 +08:00
|
|
|
void
|
2010-11-09 03:18:58 +08:00
|
|
|
i915_gem_release_mmap(struct drm_i915_gem_object *obj)
|
2009-07-10 15:18:50 +08:00
|
|
|
{
|
2010-11-24 20:23:44 +08:00
|
|
|
if (!obj->fault_mappable)
|
|
|
|
return;
|
2009-07-10 15:18:50 +08:00
|
|
|
|
2014-01-03 21:24:19 +08:00
|
|
|
drm_vma_node_unmap(&obj->base.vma_node,
|
|
|
|
obj->base.dev->anon_inode->i_mapping);
|
2010-11-24 20:23:44 +08:00
|
|
|
obj->fault_mappable = false;
|
2009-07-10 15:18:50 +08:00
|
|
|
}
|
|
|
|
|
2014-06-16 15:57:44 +08:00
|
|
|
void
|
|
|
|
i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
|
|
|
|
{
|
|
|
|
struct drm_i915_gem_object *obj;
|
|
|
|
|
|
|
|
list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
|
|
|
|
i915_gem_release_mmap(obj);
|
|
|
|
}
|
|
|
|
|
2013-01-08 03:47:35 +08:00
|
|
|
uint32_t
|
2011-07-19 04:11:49 +08:00
|
|
|
i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
|
2010-11-09 19:47:32 +08:00
|
|
|
{
|
2011-07-19 04:11:49 +08:00
|
|
|
uint32_t gtt_size;
|
2010-11-09 19:47:32 +08:00
|
|
|
|
|
|
|
if (INTEL_INFO(dev)->gen >= 4 ||
|
2011-07-19 04:11:49 +08:00
|
|
|
tiling_mode == I915_TILING_NONE)
|
|
|
|
return size;
|
2010-11-09 19:47:32 +08:00
|
|
|
|
|
|
|
/* Previous chips need a power-of-two fence region when tiling */
|
|
|
|
if (INTEL_INFO(dev)->gen == 3)
|
2011-07-19 04:11:49 +08:00
|
|
|
gtt_size = 1024*1024;
|
2010-11-09 19:47:32 +08:00
|
|
|
else
|
2011-07-19 04:11:49 +08:00
|
|
|
gtt_size = 512*1024;
|
2010-11-09 19:47:32 +08:00
|
|
|
|
2011-07-19 04:11:49 +08:00
|
|
|
while (gtt_size < size)
|
|
|
|
gtt_size <<= 1;
|
2010-11-09 19:47:32 +08:00
|
|
|
|
2011-07-19 04:11:49 +08:00
|
|
|
return gtt_size;
|
2010-11-09 19:47:32 +08:00
|
|
|
}
|
|
|
|
|
2008-11-13 02:03:55 +08:00
|
|
|
/**
|
|
|
|
* i915_gem_get_gtt_alignment - return required GTT alignment for an object
|
|
|
|
* @obj: object to check
|
|
|
|
*
|
|
|
|
* Return the required GTT alignment for an object, taking into account
|
2010-11-15 05:32:36 +08:00
|
|
|
* potential fence register mapping.
|
2008-11-13 02:03:55 +08:00
|
|
|
*/
|
2013-01-08 03:47:33 +08:00
|
|
|
uint32_t
|
|
|
|
i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
|
|
|
|
int tiling_mode, bool fenced)
|
2008-11-13 02:03:55 +08:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Minimum alignment is 4k (GTT page size), but might be greater
|
|
|
|
* if a fence register is needed for the object.
|
|
|
|
*/
|
2013-01-08 03:47:33 +08:00
|
|
|
if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
|
2011-07-19 04:11:49 +08:00
|
|
|
tiling_mode == I915_TILING_NONE)
|
2008-11-13 02:03:55 +08:00
|
|
|
return 4096;
|
|
|
|
|
2010-09-25 04:15:47 +08:00
|
|
|
/*
|
|
|
|
* Previous chips need to be aligned to the size of the smallest
|
|
|
|
* fence register that can contain the object.
|
|
|
|
*/
|
2011-07-19 04:11:49 +08:00
|
|
|
return i915_gem_get_gtt_size(dev, size, tiling_mode);
|
2010-09-25 04:15:47 +08:00
|
|
|
}
|
|
|
|
|
2012-08-11 22:41:03 +08:00
|
|
|
static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
|
|
|
|
int ret;
|
|
|
|
|
2013-07-25 03:07:52 +08:00
|
|
|
if (drm_vma_node_has_offset(&obj->base.vma_node))
|
2012-08-11 22:41:03 +08:00
|
|
|
return 0;
|
|
|
|
|
2012-12-20 22:11:16 +08:00
|
|
|
dev_priv->mm.shrinker_no_lock_stealing = true;
|
|
|
|
|
2012-08-11 22:41:03 +08:00
|
|
|
ret = drm_gem_create_mmap_offset(&obj->base);
|
|
|
|
if (ret != -ENOSPC)
|
2012-12-20 22:11:16 +08:00
|
|
|
goto out;
|
2012-08-11 22:41:03 +08:00
|
|
|
|
|
|
|
/* Badly fragmented mmap space? The only way we can recover
|
|
|
|
* space is by destroying unwanted objects. We can't randomly release
|
|
|
|
* mmap_offsets as userspace expects them to be persistent for the
|
|
|
|
* lifetime of the objects. The closest we can is to release the
|
|
|
|
* offsets on purgeable objects by truncating it and marking it purged,
|
|
|
|
* which prevents userspace from ever using that object again.
|
|
|
|
*/
|
2014-09-09 18:16:08 +08:00
|
|
|
i915_gem_shrink(dev_priv,
|
|
|
|
obj->base.size >> PAGE_SHIFT,
|
|
|
|
I915_SHRINK_BOUND |
|
|
|
|
I915_SHRINK_UNBOUND |
|
|
|
|
I915_SHRINK_PURGEABLE);
|
2012-08-11 22:41:03 +08:00
|
|
|
ret = drm_gem_create_mmap_offset(&obj->base);
|
|
|
|
if (ret != -ENOSPC)
|
2012-12-20 22:11:16 +08:00
|
|
|
goto out;
|
2012-08-11 22:41:03 +08:00
|
|
|
|
|
|
|
i915_gem_shrink_all(dev_priv);
|
2012-12-20 22:11:16 +08:00
|
|
|
ret = drm_gem_create_mmap_offset(&obj->base);
|
|
|
|
out:
|
|
|
|
dev_priv->mm.shrinker_no_lock_stealing = false;
|
|
|
|
|
|
|
|
return ret;
|
2012-08-11 22:41:03 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
|
|
|
|
{
|
|
|
|
drm_gem_free_mmap_offset(&obj->base);
|
|
|
|
}
|
|
|
|
|
2014-12-24 11:11:17 +08:00
|
|
|
int
|
2011-02-07 10:16:14 +08:00
|
|
|
i915_gem_mmap_gtt(struct drm_file *file,
|
|
|
|
struct drm_device *dev,
|
2014-12-24 11:11:17 +08:00
|
|
|
uint32_t handle,
|
2011-02-07 10:16:14 +08:00
|
|
|
uint64_t *offset)
|
2008-11-13 02:03:55 +08:00
|
|
|
{
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_i915_gem_object *obj;
|
2008-11-13 02:03:55 +08:00
|
|
|
int ret;
|
|
|
|
|
2010-09-25 18:22:51 +08:00
|
|
|
ret = i915_mutex_lock_interruptible(dev);
|
2010-10-17 16:45:41 +08:00
|
|
|
if (ret)
|
2010-09-25 18:22:51 +08:00
|
|
|
return ret;
|
2008-11-13 02:03:55 +08:00
|
|
|
|
2011-02-07 10:16:14 +08:00
|
|
|
obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
|
2011-02-19 19:31:06 +08:00
|
|
|
if (&obj->base == NULL) {
|
2010-10-17 16:45:41 +08:00
|
|
|
ret = -ENOENT;
|
|
|
|
goto unlock;
|
|
|
|
}
|
2008-11-13 02:03:55 +08:00
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
if (obj->madv != I915_MADV_WILLNEED) {
|
2014-02-10 17:03:50 +08:00
|
|
|
DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
|
2014-01-31 19:34:58 +08:00
|
|
|
ret = -EFAULT;
|
2010-10-17 16:45:41 +08:00
|
|
|
goto out;
|
2009-09-23 01:46:17 +08:00
|
|
|
}
|
|
|
|
|
2012-08-11 22:41:03 +08:00
|
|
|
ret = i915_gem_object_create_mmap_offset(obj);
|
|
|
|
if (ret)
|
|
|
|
goto out;
|
2008-11-13 02:03:55 +08:00
|
|
|
|
2013-07-25 03:07:52 +08:00
|
|
|
*offset = drm_vma_node_offset_addr(&obj->base.vma_node);
|
2008-11-13 02:03:55 +08:00
|
|
|
|
2010-10-17 16:45:41 +08:00
|
|
|
out:
|
2010-11-09 03:18:58 +08:00
|
|
|
drm_gem_object_unreference(&obj->base);
|
2010-10-17 16:45:41 +08:00
|
|
|
unlock:
|
2008-11-13 02:03:55 +08:00
|
|
|
mutex_unlock(&dev->struct_mutex);
|
2010-10-17 16:45:41 +08:00
|
|
|
return ret;
|
2008-11-13 02:03:55 +08:00
|
|
|
}
|
|
|
|
|
2011-02-07 10:16:14 +08:00
|
|
|
/**
|
|
|
|
* i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
|
|
|
|
* @dev: DRM device
|
|
|
|
* @data: GTT mapping ioctl data
|
|
|
|
* @file: GEM object info
|
|
|
|
*
|
|
|
|
* Simply returns the fake offset to userspace so it can mmap it.
|
|
|
|
* The mmap call will end up in drm_gem_mmap(), which will set things
|
|
|
|
* up so we can get faults in the handler above.
|
|
|
|
*
|
|
|
|
* The fault handler will take care of binding the object into the GTT
|
|
|
|
* (since it may have been evicted to make room for something), allocating
|
|
|
|
* a fence register, and mapping the appropriate aperture address into
|
|
|
|
* userspace.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
|
|
|
|
struct drm_file *file)
|
|
|
|
{
|
|
|
|
struct drm_i915_gem_mmap_gtt *args = data;
|
|
|
|
|
2014-12-24 11:11:17 +08:00
|
|
|
return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
|
2011-02-07 10:16:14 +08:00
|
|
|
}
|
|
|
|
|
2012-08-20 16:23:20 +08:00
|
|
|
/* Immediately discard the backing storage */
|
|
|
|
static void
|
|
|
|
i915_gem_object_truncate(struct drm_i915_gem_object *obj)
|
2010-10-28 20:45:36 +08:00
|
|
|
{
|
2012-08-11 22:41:05 +08:00
|
|
|
i915_gem_object_free_mmap_offset(obj);
|
2012-05-10 21:25:09 +08:00
|
|
|
|
2012-08-11 22:41:05 +08:00
|
|
|
if (obj->base.filp == NULL)
|
|
|
|
return;
|
2010-10-28 20:45:36 +08:00
|
|
|
|
2012-08-20 16:23:20 +08:00
|
|
|
/* Our goal here is to return as much of the memory as
|
|
|
|
* is possible back to the system as we are called from OOM.
|
|
|
|
* To do this we must instruct the shmfs to drop all of its
|
|
|
|
* backing pages, *now*.
|
|
|
|
*/
|
2014-03-25 21:23:06 +08:00
|
|
|
shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
|
2012-08-20 16:23:20 +08:00
|
|
|
obj->madv = __I915_MADV_PURGED;
|
|
|
|
}
|
2010-10-28 20:45:36 +08:00
|
|
|
|
2014-03-25 21:23:06 +08:00
|
|
|
/* Try to discard unwanted pages */
|
|
|
|
static void
|
|
|
|
i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
|
2012-08-20 16:23:20 +08:00
|
|
|
{
|
2014-03-25 21:23:06 +08:00
|
|
|
struct address_space *mapping;
|
|
|
|
|
|
|
|
switch (obj->madv) {
|
|
|
|
case I915_MADV_DONTNEED:
|
|
|
|
i915_gem_object_truncate(obj);
|
|
|
|
case __I915_MADV_PURGED:
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (obj->base.filp == NULL)
|
|
|
|
return;
|
|
|
|
|
|
|
|
mapping = file_inode(obj->base.filp)->i_mapping,
|
|
|
|
invalidate_mapping_pages(mapping, 0, (loff_t)-1);
|
2010-10-28 20:45:36 +08:00
|
|
|
}
|
|
|
|
|
2010-09-27 22:51:07 +08:00
|
|
|
static void
|
2010-11-09 03:18:58 +08:00
|
|
|
i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
2013-02-19 01:28:03 +08:00
|
|
|
struct sg_page_iter sg_iter;
|
|
|
|
int ret;
|
2012-05-10 21:25:09 +08:00
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
BUG_ON(obj->madv == __I915_MADV_PURGED);
|
2008-07-31 03:06:12 +08:00
|
|
|
|
drm/i915: Track unbound pages
When dealing with a working set larger than the GATT, or even the
mappable aperture when touching through the GTT, we end up with evicting
objects only to rebind them at a new offset again later. Moving an
object into and out of the GTT requires clflushing the pages, thus
causing a double-clflush penalty for rebinding.
To avoid having to clflush on rebinding, we can track the pages as they
are evicted from the GTT and only relinquish those pages on memory
pressure.
As usual, if it were not for the handling of out-of-memory condition and
having to manually shrink our own bo caches, it would be a net reduction
of code. Alas.
Note: The patch also contains a few changes to the last-hope
evict_everything logic in i916_gem_execbuffer.c - we no longer try to
only evict the purgeable stuff in a first try (since that's superflous
and only helps in OOM corner-cases, not fragmented-gtt trashing
situations).
Also, the extraction of the get_pages retry loop from bind_to_gtt (and
other callsites) to get_pages should imo have been a separate patch.
v2: Ditch the newly added put_pages (for unbound objects only) in
i915_gem_reset. A quick irc discussion hasn't revealed any important
reason for this, so if we need this, I'd like to have a git blame'able
explanation for it.
v3: Undo the s/drm_malloc_ab/kmalloc/ in get_pages that Chris noticed.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
[danvet: Split out code movements and rant a bit in the commit message
with a few Notes. Done v2]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-08-20 17:40:46 +08:00
|
|
|
ret = i915_gem_object_set_to_cpu_domain(obj, true);
|
|
|
|
if (ret) {
|
|
|
|
/* In the event of a disaster, abandon all caches and
|
|
|
|
* hope for the best.
|
|
|
|
*/
|
|
|
|
WARN_ON(ret != -EIO);
|
2013-08-09 19:26:45 +08:00
|
|
|
i915_gem_clflush_object(obj, true);
|
drm/i915: Track unbound pages
When dealing with a working set larger than the GATT, or even the
mappable aperture when touching through the GTT, we end up with evicting
objects only to rebind them at a new offset again later. Moving an
object into and out of the GTT requires clflushing the pages, thus
causing a double-clflush penalty for rebinding.
To avoid having to clflush on rebinding, we can track the pages as they
are evicted from the GTT and only relinquish those pages on memory
pressure.
As usual, if it were not for the handling of out-of-memory condition and
having to manually shrink our own bo caches, it would be a net reduction
of code. Alas.
Note: The patch also contains a few changes to the last-hope
evict_everything logic in i916_gem_execbuffer.c - we no longer try to
only evict the purgeable stuff in a first try (since that's superflous
and only helps in OOM corner-cases, not fragmented-gtt trashing
situations).
Also, the extraction of the get_pages retry loop from bind_to_gtt (and
other callsites) to get_pages should imo have been a separate patch.
v2: Ditch the newly added put_pages (for unbound objects only) in
i915_gem_reset. A quick irc discussion hasn't revealed any important
reason for this, so if we need this, I'd like to have a git blame'able
explanation for it.
v3: Undo the s/drm_malloc_ab/kmalloc/ in get_pages that Chris noticed.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
[danvet: Split out code movements and rant a bit in the commit message
with a few Notes. Done v2]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-08-20 17:40:46 +08:00
|
|
|
obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
|
|
|
|
}
|
|
|
|
|
2015-07-09 17:59:05 +08:00
|
|
|
i915_gem_gtt_finish_object(obj);
|
|
|
|
|
2011-09-13 03:30:02 +08:00
|
|
|
if (i915_gem_object_needs_bit17_swizzle(obj))
|
2009-03-13 07:56:27 +08:00
|
|
|
i915_gem_object_save_bit_17_swizzle(obj);
|
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
if (obj->madv == I915_MADV_DONTNEED)
|
|
|
|
obj->dirty = 0;
|
2009-09-14 23:50:29 +08:00
|
|
|
|
2013-02-19 01:28:03 +08:00
|
|
|
for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
|
2013-03-26 21:14:18 +08:00
|
|
|
struct page *page = sg_page_iter_page(&sg_iter);
|
2012-06-01 22:20:22 +08:00
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
if (obj->dirty)
|
2012-06-01 22:20:22 +08:00
|
|
|
set_page_dirty(page);
|
2009-09-14 23:50:29 +08:00
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
if (obj->madv == I915_MADV_WILLNEED)
|
2012-06-01 22:20:22 +08:00
|
|
|
mark_page_accessed(page);
|
2009-09-14 23:50:29 +08:00
|
|
|
|
mm, fs: get rid of PAGE_CACHE_* and page_cache_{get,release} macros
PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} macros were introduced *long* time
ago with promise that one day it will be possible to implement page
cache with bigger chunks than PAGE_SIZE.
This promise never materialized. And unlikely will.
We have many places where PAGE_CACHE_SIZE assumed to be equal to
PAGE_SIZE. And it's constant source of confusion on whether
PAGE_CACHE_* or PAGE_* constant should be used in a particular case,
especially on the border between fs and mm.
Global switching to PAGE_CACHE_SIZE != PAGE_SIZE would cause to much
breakage to be doable.
Let's stop pretending that pages in page cache are special. They are
not.
The changes are pretty straight-forward:
- <foo> << (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- <foo> >> (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} -> PAGE_{SIZE,SHIFT,MASK,ALIGN};
- page_cache_get() -> get_page();
- page_cache_release() -> put_page();
This patch contains automated changes generated with coccinelle using
script below. For some reason, coccinelle doesn't patch header files.
I've called spatch for them manually.
The only adjustment after coccinelle is revert of changes to
PAGE_CAHCE_ALIGN definition: we are going to drop it later.
There are few places in the code where coccinelle didn't reach. I'll
fix them manually in a separate patch. Comments and documentation also
will be addressed with the separate patch.
virtual patch
@@
expression E;
@@
- E << (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
expression E;
@@
- E >> (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
@@
- PAGE_CACHE_SHIFT
+ PAGE_SHIFT
@@
@@
- PAGE_CACHE_SIZE
+ PAGE_SIZE
@@
@@
- PAGE_CACHE_MASK
+ PAGE_MASK
@@
expression E;
@@
- PAGE_CACHE_ALIGN(E)
+ PAGE_ALIGN(E)
@@
expression E;
@@
- page_cache_get(E)
+ get_page(E)
@@
expression E;
@@
- page_cache_release(E)
+ put_page(E)
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-04-01 20:29:47 +08:00
|
|
|
put_page(page);
|
2009-09-14 23:50:29 +08:00
|
|
|
}
|
2010-11-09 03:18:58 +08:00
|
|
|
obj->dirty = 0;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2012-06-01 22:20:22 +08:00
|
|
|
sg_free_table(obj->pages);
|
|
|
|
kfree(obj->pages);
|
2012-06-07 22:38:42 +08:00
|
|
|
}
|
drm/i915: Track unbound pages
When dealing with a working set larger than the GATT, or even the
mappable aperture when touching through the GTT, we end up with evicting
objects only to rebind them at a new offset again later. Moving an
object into and out of the GTT requires clflushing the pages, thus
causing a double-clflush penalty for rebinding.
To avoid having to clflush on rebinding, we can track the pages as they
are evicted from the GTT and only relinquish those pages on memory
pressure.
As usual, if it were not for the handling of out-of-memory condition and
having to manually shrink our own bo caches, it would be a net reduction
of code. Alas.
Note: The patch also contains a few changes to the last-hope
evict_everything logic in i916_gem_execbuffer.c - we no longer try to
only evict the purgeable stuff in a first try (since that's superflous
and only helps in OOM corner-cases, not fragmented-gtt trashing
situations).
Also, the extraction of the get_pages retry loop from bind_to_gtt (and
other callsites) to get_pages should imo have been a separate patch.
v2: Ditch the newly added put_pages (for unbound objects only) in
i915_gem_reset. A quick irc discussion hasn't revealed any important
reason for this, so if we need this, I'd like to have a git blame'able
explanation for it.
v3: Undo the s/drm_malloc_ab/kmalloc/ in get_pages that Chris noticed.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
[danvet: Split out code movements and rant a bit in the commit message
with a few Notes. Done v2]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-08-20 17:40:46 +08:00
|
|
|
|
2013-01-15 20:39:35 +08:00
|
|
|
int
|
2012-06-07 22:38:42 +08:00
|
|
|
i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
|
|
|
|
{
|
|
|
|
const struct drm_i915_gem_object_ops *ops = obj->ops;
|
|
|
|
|
2012-09-05 04:02:58 +08:00
|
|
|
if (obj->pages == NULL)
|
2012-06-07 22:38:42 +08:00
|
|
|
return 0;
|
|
|
|
|
2012-09-05 04:02:54 +08:00
|
|
|
if (obj->pages_pin_count)
|
|
|
|
return -EBUSY;
|
|
|
|
|
2013-08-01 08:00:12 +08:00
|
|
|
BUG_ON(i915_gem_obj_bound_any(obj));
|
2013-08-01 08:00:04 +08:00
|
|
|
|
2012-12-03 19:49:00 +08:00
|
|
|
/* ->put_pages might need to allocate memory for the bit17 swizzle
|
|
|
|
* array, hence protect them from being reaped by removing them from gtt
|
|
|
|
* lists early. */
|
2013-06-01 02:28:48 +08:00
|
|
|
list_del(&obj->global_list);
|
2012-12-03 19:49:00 +08:00
|
|
|
|
2016-04-08 19:11:11 +08:00
|
|
|
if (obj->mapping) {
|
2016-04-08 19:11:14 +08:00
|
|
|
if (is_vmalloc_addr(obj->mapping))
|
|
|
|
vunmap(obj->mapping);
|
|
|
|
else
|
|
|
|
kunmap(kmap_to_page(obj->mapping));
|
2016-04-08 19:11:11 +08:00
|
|
|
obj->mapping = NULL;
|
|
|
|
}
|
|
|
|
|
2012-06-07 22:38:42 +08:00
|
|
|
ops->put_pages(obj);
|
2010-11-09 03:18:58 +08:00
|
|
|
obj->pages = NULL;
|
2012-06-07 22:38:42 +08:00
|
|
|
|
2014-03-25 21:23:06 +08:00
|
|
|
i915_gem_object_invalidate(obj);
|
drm/i915: Track unbound pages
When dealing with a working set larger than the GATT, or even the
mappable aperture when touching through the GTT, we end up with evicting
objects only to rebind them at a new offset again later. Moving an
object into and out of the GTT requires clflushing the pages, thus
causing a double-clflush penalty for rebinding.
To avoid having to clflush on rebinding, we can track the pages as they
are evicted from the GTT and only relinquish those pages on memory
pressure.
As usual, if it were not for the handling of out-of-memory condition and
having to manually shrink our own bo caches, it would be a net reduction
of code. Alas.
Note: The patch also contains a few changes to the last-hope
evict_everything logic in i916_gem_execbuffer.c - we no longer try to
only evict the purgeable stuff in a first try (since that's superflous
and only helps in OOM corner-cases, not fragmented-gtt trashing
situations).
Also, the extraction of the get_pages retry loop from bind_to_gtt (and
other callsites) to get_pages should imo have been a separate patch.
v2: Ditch the newly added put_pages (for unbound objects only) in
i915_gem_reset. A quick irc discussion hasn't revealed any important
reason for this, so if we need this, I'd like to have a git blame'able
explanation for it.
v3: Undo the s/drm_malloc_ab/kmalloc/ in get_pages that Chris noticed.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
[danvet: Split out code movements and rant a bit in the commit message
with a few Notes. Done v2]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-08-20 17:40:46 +08:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2012-06-07 22:38:42 +08:00
|
|
|
static int
|
drm/i915: Track unbound pages
When dealing with a working set larger than the GATT, or even the
mappable aperture when touching through the GTT, we end up with evicting
objects only to rebind them at a new offset again later. Moving an
object into and out of the GTT requires clflushing the pages, thus
causing a double-clflush penalty for rebinding.
To avoid having to clflush on rebinding, we can track the pages as they
are evicted from the GTT and only relinquish those pages on memory
pressure.
As usual, if it were not for the handling of out-of-memory condition and
having to manually shrink our own bo caches, it would be a net reduction
of code. Alas.
Note: The patch also contains a few changes to the last-hope
evict_everything logic in i916_gem_execbuffer.c - we no longer try to
only evict the purgeable stuff in a first try (since that's superflous
and only helps in OOM corner-cases, not fragmented-gtt trashing
situations).
Also, the extraction of the get_pages retry loop from bind_to_gtt (and
other callsites) to get_pages should imo have been a separate patch.
v2: Ditch the newly added put_pages (for unbound objects only) in
i915_gem_reset. A quick irc discussion hasn't revealed any important
reason for this, so if we need this, I'd like to have a git blame'able
explanation for it.
v3: Undo the s/drm_malloc_ab/kmalloc/ in get_pages that Chris noticed.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
[danvet: Split out code movements and rant a bit in the commit message
with a few Notes. Done v2]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-08-20 17:40:46 +08:00
|
|
|
i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
|
2010-10-28 20:45:36 +08:00
|
|
|
{
|
drm/i915: Track unbound pages
When dealing with a working set larger than the GATT, or even the
mappable aperture when touching through the GTT, we end up with evicting
objects only to rebind them at a new offset again later. Moving an
object into and out of the GTT requires clflushing the pages, thus
causing a double-clflush penalty for rebinding.
To avoid having to clflush on rebinding, we can track the pages as they
are evicted from the GTT and only relinquish those pages on memory
pressure.
As usual, if it were not for the handling of out-of-memory condition and
having to manually shrink our own bo caches, it would be a net reduction
of code. Alas.
Note: The patch also contains a few changes to the last-hope
evict_everything logic in i916_gem_execbuffer.c - we no longer try to
only evict the purgeable stuff in a first try (since that's superflous
and only helps in OOM corner-cases, not fragmented-gtt trashing
situations).
Also, the extraction of the get_pages retry loop from bind_to_gtt (and
other callsites) to get_pages should imo have been a separate patch.
v2: Ditch the newly added put_pages (for unbound objects only) in
i915_gem_reset. A quick irc discussion hasn't revealed any important
reason for this, so if we need this, I'd like to have a git blame'able
explanation for it.
v3: Undo the s/drm_malloc_ab/kmalloc/ in get_pages that Chris noticed.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
[danvet: Split out code movements and rant a bit in the commit message
with a few Notes. Done v2]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-08-20 17:40:46 +08:00
|
|
|
struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
|
2010-10-28 20:45:36 +08:00
|
|
|
int page_count, i;
|
|
|
|
struct address_space *mapping;
|
2012-06-01 22:20:22 +08:00
|
|
|
struct sg_table *st;
|
|
|
|
struct scatterlist *sg;
|
2013-02-19 01:28:03 +08:00
|
|
|
struct sg_page_iter sg_iter;
|
2010-10-28 20:45:36 +08:00
|
|
|
struct page *page;
|
2013-02-19 01:28:03 +08:00
|
|
|
unsigned long last_pfn = 0; /* suppress gcc warning */
|
2015-07-09 17:59:05 +08:00
|
|
|
int ret;
|
drm/i915: Track unbound pages
When dealing with a working set larger than the GATT, or even the
mappable aperture when touching through the GTT, we end up with evicting
objects only to rebind them at a new offset again later. Moving an
object into and out of the GTT requires clflushing the pages, thus
causing a double-clflush penalty for rebinding.
To avoid having to clflush on rebinding, we can track the pages as they
are evicted from the GTT and only relinquish those pages on memory
pressure.
As usual, if it were not for the handling of out-of-memory condition and
having to manually shrink our own bo caches, it would be a net reduction
of code. Alas.
Note: The patch also contains a few changes to the last-hope
evict_everything logic in i916_gem_execbuffer.c - we no longer try to
only evict the purgeable stuff in a first try (since that's superflous
and only helps in OOM corner-cases, not fragmented-gtt trashing
situations).
Also, the extraction of the get_pages retry loop from bind_to_gtt (and
other callsites) to get_pages should imo have been a separate patch.
v2: Ditch the newly added put_pages (for unbound objects only) in
i915_gem_reset. A quick irc discussion hasn't revealed any important
reason for this, so if we need this, I'd like to have a git blame'able
explanation for it.
v3: Undo the s/drm_malloc_ab/kmalloc/ in get_pages that Chris noticed.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
[danvet: Split out code movements and rant a bit in the commit message
with a few Notes. Done v2]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-08-20 17:40:46 +08:00
|
|
|
gfp_t gfp;
|
2010-10-28 20:45:36 +08:00
|
|
|
|
drm/i915: Track unbound pages
When dealing with a working set larger than the GATT, or even the
mappable aperture when touching through the GTT, we end up with evicting
objects only to rebind them at a new offset again later. Moving an
object into and out of the GTT requires clflushing the pages, thus
causing a double-clflush penalty for rebinding.
To avoid having to clflush on rebinding, we can track the pages as they
are evicted from the GTT and only relinquish those pages on memory
pressure.
As usual, if it were not for the handling of out-of-memory condition and
having to manually shrink our own bo caches, it would be a net reduction
of code. Alas.
Note: The patch also contains a few changes to the last-hope
evict_everything logic in i916_gem_execbuffer.c - we no longer try to
only evict the purgeable stuff in a first try (since that's superflous
and only helps in OOM corner-cases, not fragmented-gtt trashing
situations).
Also, the extraction of the get_pages retry loop from bind_to_gtt (and
other callsites) to get_pages should imo have been a separate patch.
v2: Ditch the newly added put_pages (for unbound objects only) in
i915_gem_reset. A quick irc discussion hasn't revealed any important
reason for this, so if we need this, I'd like to have a git blame'able
explanation for it.
v3: Undo the s/drm_malloc_ab/kmalloc/ in get_pages that Chris noticed.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
[danvet: Split out code movements and rant a bit in the commit message
with a few Notes. Done v2]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-08-20 17:40:46 +08:00
|
|
|
/* Assert that the object is not currently in any GPU domain. As it
|
|
|
|
* wasn't in the GTT, there shouldn't be any way it could have been in
|
|
|
|
* a GPU cache
|
|
|
|
*/
|
|
|
|
BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
|
|
|
|
BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
|
|
|
|
|
2012-06-01 22:20:22 +08:00
|
|
|
st = kmalloc(sizeof(*st), GFP_KERNEL);
|
|
|
|
if (st == NULL)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
page_count = obj->base.size / PAGE_SIZE;
|
2012-06-01 22:20:22 +08:00
|
|
|
if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
|
|
|
|
kfree(st);
|
2010-10-28 20:45:36 +08:00
|
|
|
return -ENOMEM;
|
2012-06-01 22:20:22 +08:00
|
|
|
}
|
2010-10-28 20:45:36 +08:00
|
|
|
|
2012-06-01 22:20:22 +08:00
|
|
|
/* Get the list of pages out of our struct file. They'll be pinned
|
|
|
|
* at this point until we release them.
|
|
|
|
*
|
|
|
|
* Fail silently without starting the shrinker
|
|
|
|
*/
|
2013-01-24 06:07:38 +08:00
|
|
|
mapping = file_inode(obj->base.filp)->i_mapping;
|
2015-11-07 08:28:49 +08:00
|
|
|
gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM));
|
2015-11-07 08:28:21 +08:00
|
|
|
gfp |= __GFP_NORETRY | __GFP_NOWARN;
|
2013-02-19 01:28:03 +08:00
|
|
|
sg = st->sgl;
|
|
|
|
st->nents = 0;
|
|
|
|
for (i = 0; i < page_count; i++) {
|
drm/i915: Track unbound pages
When dealing with a working set larger than the GATT, or even the
mappable aperture when touching through the GTT, we end up with evicting
objects only to rebind them at a new offset again later. Moving an
object into and out of the GTT requires clflushing the pages, thus
causing a double-clflush penalty for rebinding.
To avoid having to clflush on rebinding, we can track the pages as they
are evicted from the GTT and only relinquish those pages on memory
pressure.
As usual, if it were not for the handling of out-of-memory condition and
having to manually shrink our own bo caches, it would be a net reduction
of code. Alas.
Note: The patch also contains a few changes to the last-hope
evict_everything logic in i916_gem_execbuffer.c - we no longer try to
only evict the purgeable stuff in a first try (since that's superflous
and only helps in OOM corner-cases, not fragmented-gtt trashing
situations).
Also, the extraction of the get_pages retry loop from bind_to_gtt (and
other callsites) to get_pages should imo have been a separate patch.
v2: Ditch the newly added put_pages (for unbound objects only) in
i915_gem_reset. A quick irc discussion hasn't revealed any important
reason for this, so if we need this, I'd like to have a git blame'able
explanation for it.
v3: Undo the s/drm_malloc_ab/kmalloc/ in get_pages that Chris noticed.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
[danvet: Split out code movements and rant a bit in the commit message
with a few Notes. Done v2]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-08-20 17:40:46 +08:00
|
|
|
page = shmem_read_mapping_page_gfp(mapping, i, gfp);
|
|
|
|
if (IS_ERR(page)) {
|
2014-09-09 18:16:08 +08:00
|
|
|
i915_gem_shrink(dev_priv,
|
|
|
|
page_count,
|
|
|
|
I915_SHRINK_BOUND |
|
|
|
|
I915_SHRINK_UNBOUND |
|
|
|
|
I915_SHRINK_PURGEABLE);
|
drm/i915: Track unbound pages
When dealing with a working set larger than the GATT, or even the
mappable aperture when touching through the GTT, we end up with evicting
objects only to rebind them at a new offset again later. Moving an
object into and out of the GTT requires clflushing the pages, thus
causing a double-clflush penalty for rebinding.
To avoid having to clflush on rebinding, we can track the pages as they
are evicted from the GTT and only relinquish those pages on memory
pressure.
As usual, if it were not for the handling of out-of-memory condition and
having to manually shrink our own bo caches, it would be a net reduction
of code. Alas.
Note: The patch also contains a few changes to the last-hope
evict_everything logic in i916_gem_execbuffer.c - we no longer try to
only evict the purgeable stuff in a first try (since that's superflous
and only helps in OOM corner-cases, not fragmented-gtt trashing
situations).
Also, the extraction of the get_pages retry loop from bind_to_gtt (and
other callsites) to get_pages should imo have been a separate patch.
v2: Ditch the newly added put_pages (for unbound objects only) in
i915_gem_reset. A quick irc discussion hasn't revealed any important
reason for this, so if we need this, I'd like to have a git blame'able
explanation for it.
v3: Undo the s/drm_malloc_ab/kmalloc/ in get_pages that Chris noticed.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
[danvet: Split out code movements and rant a bit in the commit message
with a few Notes. Done v2]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-08-20 17:40:46 +08:00
|
|
|
page = shmem_read_mapping_page_gfp(mapping, i, gfp);
|
|
|
|
}
|
|
|
|
if (IS_ERR(page)) {
|
|
|
|
/* We've tried hard to allocate the memory by reaping
|
|
|
|
* our own buffer, now let the real VM do its job and
|
|
|
|
* go down in flames if truly OOM.
|
|
|
|
*/
|
|
|
|
i915_gem_shrink_all(dev_priv);
|
2014-05-25 20:34:10 +08:00
|
|
|
page = shmem_read_mapping_page(mapping, i);
|
2015-07-09 17:59:05 +08:00
|
|
|
if (IS_ERR(page)) {
|
|
|
|
ret = PTR_ERR(page);
|
drm/i915: Track unbound pages
When dealing with a working set larger than the GATT, or even the
mappable aperture when touching through the GTT, we end up with evicting
objects only to rebind them at a new offset again later. Moving an
object into and out of the GTT requires clflushing the pages, thus
causing a double-clflush penalty for rebinding.
To avoid having to clflush on rebinding, we can track the pages as they
are evicted from the GTT and only relinquish those pages on memory
pressure.
As usual, if it were not for the handling of out-of-memory condition and
having to manually shrink our own bo caches, it would be a net reduction
of code. Alas.
Note: The patch also contains a few changes to the last-hope
evict_everything logic in i916_gem_execbuffer.c - we no longer try to
only evict the purgeable stuff in a first try (since that's superflous
and only helps in OOM corner-cases, not fragmented-gtt trashing
situations).
Also, the extraction of the get_pages retry loop from bind_to_gtt (and
other callsites) to get_pages should imo have been a separate patch.
v2: Ditch the newly added put_pages (for unbound objects only) in
i915_gem_reset. A quick irc discussion hasn't revealed any important
reason for this, so if we need this, I'd like to have a git blame'able
explanation for it.
v3: Undo the s/drm_malloc_ab/kmalloc/ in get_pages that Chris noticed.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
[danvet: Split out code movements and rant a bit in the commit message
with a few Notes. Done v2]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-08-20 17:40:46 +08:00
|
|
|
goto err_pages;
|
2015-07-09 17:59:05 +08:00
|
|
|
}
|
drm/i915: Track unbound pages
When dealing with a working set larger than the GATT, or even the
mappable aperture when touching through the GTT, we end up with evicting
objects only to rebind them at a new offset again later. Moving an
object into and out of the GTT requires clflushing the pages, thus
causing a double-clflush penalty for rebinding.
To avoid having to clflush on rebinding, we can track the pages as they
are evicted from the GTT and only relinquish those pages on memory
pressure.
As usual, if it were not for the handling of out-of-memory condition and
having to manually shrink our own bo caches, it would be a net reduction
of code. Alas.
Note: The patch also contains a few changes to the last-hope
evict_everything logic in i916_gem_execbuffer.c - we no longer try to
only evict the purgeable stuff in a first try (since that's superflous
and only helps in OOM corner-cases, not fragmented-gtt trashing
situations).
Also, the extraction of the get_pages retry loop from bind_to_gtt (and
other callsites) to get_pages should imo have been a separate patch.
v2: Ditch the newly added put_pages (for unbound objects only) in
i915_gem_reset. A quick irc discussion hasn't revealed any important
reason for this, so if we need this, I'd like to have a git blame'able
explanation for it.
v3: Undo the s/drm_malloc_ab/kmalloc/ in get_pages that Chris noticed.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
[danvet: Split out code movements and rant a bit in the commit message
with a few Notes. Done v2]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-08-20 17:40:46 +08:00
|
|
|
}
|
2013-06-24 23:47:48 +08:00
|
|
|
#ifdef CONFIG_SWIOTLB
|
|
|
|
if (swiotlb_nr_tbl()) {
|
|
|
|
st->nents++;
|
|
|
|
sg_set_page(sg, page, PAGE_SIZE, 0);
|
|
|
|
sg = sg_next(sg);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
#endif
|
2013-02-19 01:28:03 +08:00
|
|
|
if (!i || page_to_pfn(page) != last_pfn + 1) {
|
|
|
|
if (i)
|
|
|
|
sg = sg_next(sg);
|
|
|
|
st->nents++;
|
|
|
|
sg_set_page(sg, page, PAGE_SIZE, 0);
|
|
|
|
} else {
|
|
|
|
sg->length += PAGE_SIZE;
|
|
|
|
}
|
|
|
|
last_pfn = page_to_pfn(page);
|
2013-10-08 04:15:45 +08:00
|
|
|
|
|
|
|
/* Check that the i965g/gm workaround works. */
|
|
|
|
WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
|
2010-10-28 20:45:36 +08:00
|
|
|
}
|
2013-06-24 23:47:48 +08:00
|
|
|
#ifdef CONFIG_SWIOTLB
|
|
|
|
if (!swiotlb_nr_tbl())
|
|
|
|
#endif
|
|
|
|
sg_mark_end(sg);
|
2012-10-19 22:51:06 +08:00
|
|
|
obj->pages = st;
|
|
|
|
|
2015-07-09 17:59:05 +08:00
|
|
|
ret = i915_gem_gtt_prepare_object(obj);
|
|
|
|
if (ret)
|
|
|
|
goto err_pages;
|
|
|
|
|
2011-09-13 03:30:02 +08:00
|
|
|
if (i915_gem_object_needs_bit17_swizzle(obj))
|
2010-10-28 20:45:36 +08:00
|
|
|
i915_gem_object_do_bit_17_swizzle(obj);
|
|
|
|
|
2014-11-20 16:26:30 +08:00
|
|
|
if (obj->tiling_mode != I915_TILING_NONE &&
|
|
|
|
dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
|
|
|
|
i915_gem_object_pin_pages(obj);
|
|
|
|
|
2010-10-28 20:45:36 +08:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
err_pages:
|
2013-02-19 01:28:03 +08:00
|
|
|
sg_mark_end(sg);
|
|
|
|
for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
|
mm, fs: get rid of PAGE_CACHE_* and page_cache_{get,release} macros
PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} macros were introduced *long* time
ago with promise that one day it will be possible to implement page
cache with bigger chunks than PAGE_SIZE.
This promise never materialized. And unlikely will.
We have many places where PAGE_CACHE_SIZE assumed to be equal to
PAGE_SIZE. And it's constant source of confusion on whether
PAGE_CACHE_* or PAGE_* constant should be used in a particular case,
especially on the border between fs and mm.
Global switching to PAGE_CACHE_SIZE != PAGE_SIZE would cause to much
breakage to be doable.
Let's stop pretending that pages in page cache are special. They are
not.
The changes are pretty straight-forward:
- <foo> << (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- <foo> >> (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} -> PAGE_{SIZE,SHIFT,MASK,ALIGN};
- page_cache_get() -> get_page();
- page_cache_release() -> put_page();
This patch contains automated changes generated with coccinelle using
script below. For some reason, coccinelle doesn't patch header files.
I've called spatch for them manually.
The only adjustment after coccinelle is revert of changes to
PAGE_CAHCE_ALIGN definition: we are going to drop it later.
There are few places in the code where coccinelle didn't reach. I'll
fix them manually in a separate patch. Comments and documentation also
will be addressed with the separate patch.
virtual patch
@@
expression E;
@@
- E << (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
expression E;
@@
- E >> (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
@@
- PAGE_CACHE_SHIFT
+ PAGE_SHIFT
@@
@@
- PAGE_CACHE_SIZE
+ PAGE_SIZE
@@
@@
- PAGE_CACHE_MASK
+ PAGE_MASK
@@
expression E;
@@
- PAGE_CACHE_ALIGN(E)
+ PAGE_ALIGN(E)
@@
expression E;
@@
- page_cache_get(E)
+ get_page(E)
@@
expression E;
@@
- page_cache_release(E)
+ put_page(E)
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-04-01 20:29:47 +08:00
|
|
|
put_page(sg_page_iter_page(&sg_iter));
|
2012-06-01 22:20:22 +08:00
|
|
|
sg_free_table(st);
|
|
|
|
kfree(st);
|
2014-03-25 21:23:03 +08:00
|
|
|
|
|
|
|
/* shmemfs first checks if there is enough memory to allocate the page
|
|
|
|
* and reports ENOSPC should there be insufficient, along with the usual
|
|
|
|
* ENOMEM for a genuine allocation failure.
|
|
|
|
*
|
|
|
|
* We use ENOSPC in our driver to mean that we have run out of aperture
|
|
|
|
* space and so want to translate the error from shmemfs back to our
|
|
|
|
* usual understanding of ENOMEM.
|
|
|
|
*/
|
2015-07-09 17:59:05 +08:00
|
|
|
if (ret == -ENOSPC)
|
|
|
|
ret = -ENOMEM;
|
|
|
|
|
|
|
|
return ret;
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
2012-06-07 22:38:42 +08:00
|
|
|
/* Ensure that the associated pages are gathered from the backing storage
|
|
|
|
* and pinned into our object. i915_gem_object_get_pages() may be called
|
|
|
|
* multiple times before they are released by a single call to
|
|
|
|
* i915_gem_object_put_pages() - once the pages are no longer referenced
|
|
|
|
* either as a result of memory pressure (reaping pages under the shrinker)
|
|
|
|
* or as the object is itself released.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
|
|
|
|
const struct drm_i915_gem_object_ops *ops = obj->ops;
|
|
|
|
int ret;
|
|
|
|
|
2012-09-05 04:02:58 +08:00
|
|
|
if (obj->pages)
|
2012-06-07 22:38:42 +08:00
|
|
|
return 0;
|
|
|
|
|
2013-01-08 18:53:09 +08:00
|
|
|
if (obj->madv != I915_MADV_WILLNEED) {
|
2014-02-10 17:03:50 +08:00
|
|
|
DRM_DEBUG("Attempting to obtain a purgeable object\n");
|
2014-01-31 19:34:58 +08:00
|
|
|
return -EFAULT;
|
2013-01-08 18:53:09 +08:00
|
|
|
}
|
|
|
|
|
2012-09-05 04:02:54 +08:00
|
|
|
BUG_ON(obj->pages_pin_count);
|
|
|
|
|
2012-06-07 22:38:42 +08:00
|
|
|
ret = ops->get_pages(obj);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2013-06-01 02:28:48 +08:00
|
|
|
list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
|
2015-04-07 23:20:25 +08:00
|
|
|
|
|
|
|
obj->get_page.sg = obj->pages->sgl;
|
|
|
|
obj->get_page.last = 0;
|
|
|
|
|
2012-06-07 22:38:42 +08:00
|
|
|
return 0;
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
2016-04-08 19:11:11 +08:00
|
|
|
void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
lockdep_assert_held(&obj->base.dev->struct_mutex);
|
|
|
|
|
|
|
|
ret = i915_gem_object_get_pages(obj);
|
|
|
|
if (ret)
|
|
|
|
return ERR_PTR(ret);
|
|
|
|
|
|
|
|
i915_gem_object_pin_pages(obj);
|
|
|
|
|
|
|
|
if (obj->mapping == NULL) {
|
|
|
|
struct page **pages;
|
|
|
|
|
2016-04-08 19:11:14 +08:00
|
|
|
pages = NULL;
|
|
|
|
if (obj->base.size == PAGE_SIZE)
|
|
|
|
obj->mapping = kmap(sg_page(obj->pages->sgl));
|
|
|
|
else
|
|
|
|
pages = drm_malloc_gfp(obj->base.size >> PAGE_SHIFT,
|
|
|
|
sizeof(*pages),
|
|
|
|
GFP_TEMPORARY);
|
2016-04-08 19:11:11 +08:00
|
|
|
if (pages != NULL) {
|
2016-04-08 19:11:14 +08:00
|
|
|
struct sg_page_iter sg_iter;
|
|
|
|
int n;
|
|
|
|
|
2016-04-08 19:11:11 +08:00
|
|
|
n = 0;
|
|
|
|
for_each_sg_page(obj->pages->sgl, &sg_iter,
|
|
|
|
obj->pages->nents, 0)
|
|
|
|
pages[n++] = sg_page_iter_page(&sg_iter);
|
|
|
|
|
|
|
|
obj->mapping = vmap(pages, n, 0, PAGE_KERNEL);
|
|
|
|
drm_free_large(pages);
|
|
|
|
}
|
|
|
|
if (obj->mapping == NULL) {
|
|
|
|
i915_gem_object_unpin_pages(obj);
|
|
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return obj->mapping;
|
|
|
|
}
|
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
void i915_vma_move_to_active(struct i915_vma *vma,
|
2015-05-30 00:43:50 +08:00
|
|
|
struct drm_i915_gem_request *req)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
2015-04-27 20:41:17 +08:00
|
|
|
struct drm_i915_gem_object *obj = vma->obj;
|
2016-03-16 19:00:36 +08:00
|
|
|
struct intel_engine_cs *engine;
|
2015-05-30 00:43:50 +08:00
|
|
|
|
2016-03-16 19:00:39 +08:00
|
|
|
engine = i915_gem_request_get_engine(req);
|
2008-07-31 03:06:12 +08:00
|
|
|
|
|
|
|
/* Add a reference if we're newly entering the active list. */
|
2015-04-27 20:41:17 +08:00
|
|
|
if (obj->active == 0)
|
2010-11-09 03:18:58 +08:00
|
|
|
drm_gem_object_reference(&obj->base);
|
2016-03-16 19:00:39 +08:00
|
|
|
obj->active |= intel_engine_flag(engine);
|
drm/i915: allow lazy emitting of requests
Sometimes (like when flushing in preparation of batchbuffer execution)
we know that we'll emit a request but haven't yet done so. Allow this
case by simply taking the next seqno by default. Ensure that a request
is eventually emitted before waiting for an request by issuing it
in i915_wait_request iff this is not yet done.
Also replace one open-coded version of i915_gem_object_wait_rendering,
to prevent future code-diversion.
Chris Wilson asked me to explain and clarify what this patch does and why.
Here it goes:
Old way of moving objects onto the active list and associating them with a
reques:
1. i915_add_request + store the returned seqno somewhere
2. i915_gem_object_move_to_active (with the stored seqno as parameter)
For the current users, this is all fine. But I'd like to associate objects
(and fence regs) with the batchbuffer request deep down in the execbuf
call-chain. I thought about three ways of implementing this.
a) Don't care, just emit request when we need a new seqno. When heavily
pipelining fence reg changes, this would have caused tons of superflous
request (and corresponding irqs).
b) Thread all changed fences, objects, whatever through the execbuf-maze,
so that when we emit a request, we can store the new seqno at all the right
places.
c) Kill that seqno-threading-around business by simply storing the next
seqno, i.e. allow 2. to be done before 1. in the above sequence.
I've decided to implement c) (in this patch). The following patches are
just fall-out that resulted from this small conceptual change.
* We can handle the flushing list processing where we actually emit a flush
(i915_gem_flush and i915_retire_commands) instead of in i915_add_request.
The code makes IMHO more sense this way (and i915_add_request looses the
flush_domains parameter, obviously).
* We can avoid emitting unnecessary requests. IMHO there's no point in
emitting more than one request per batchbuffer (with or without an
corresponding irq).
* By enforcing 2. before 1. ordering in the above sequence the seqno
argument of i915_gem_object_move_to_active is redundant and can be
dropped.
v2: Now i915_wait_request issues request if it is not yet emitted.
Also introduce i915_gem_next_request_seqno(dev) just in case we ever
need to do some prep work before using a new seqno.
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
[ickle: Keep i915_gem_object_set_to_display_plane() uninterruptible.]
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
2010-02-12 05:13:59 +08:00
|
|
|
|
2016-03-16 19:00:40 +08:00
|
|
|
list_move_tail(&obj->engine_list[engine->id], &engine->active_list);
|
2016-03-16 19:00:36 +08:00
|
|
|
i915_gem_request_assign(&obj->last_read_req[engine->id], req);
|
2010-11-12 21:53:37 +08:00
|
|
|
|
2016-02-26 19:03:19 +08:00
|
|
|
list_move_tail(&vma->vm_link, &vma->vm->active_list);
|
2010-11-12 21:53:37 +08:00
|
|
|
}
|
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
static void
|
|
|
|
i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
|
2013-09-25 00:57:58 +08:00
|
|
|
{
|
2016-04-14 00:35:02 +08:00
|
|
|
GEM_BUG_ON(obj->last_write_req == NULL);
|
|
|
|
GEM_BUG_ON(!(obj->active & intel_engine_flag(obj->last_write_req->engine)));
|
2015-04-27 20:41:17 +08:00
|
|
|
|
|
|
|
i915_gem_request_assign(&obj->last_write_req, NULL);
|
2015-07-08 07:28:51 +08:00
|
|
|
intel_fb_obj_flush(obj, true, ORIGIN_CS);
|
2013-09-25 00:57:58 +08:00
|
|
|
}
|
|
|
|
|
2010-11-12 21:53:37 +08:00
|
|
|
static void
|
2015-04-27 20:41:17 +08:00
|
|
|
i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
|
2008-11-07 08:00:31 +08:00
|
|
|
{
|
2013-12-07 06:10:51 +08:00
|
|
|
struct i915_vma *vma;
|
2008-11-07 08:00:31 +08:00
|
|
|
|
2016-04-14 00:35:02 +08:00
|
|
|
GEM_BUG_ON(obj->last_read_req[ring] == NULL);
|
|
|
|
GEM_BUG_ON(!(obj->active & (1 << ring)));
|
2015-04-27 20:41:17 +08:00
|
|
|
|
2016-03-16 19:00:40 +08:00
|
|
|
list_del_init(&obj->engine_list[ring]);
|
2015-04-27 20:41:17 +08:00
|
|
|
i915_gem_request_assign(&obj->last_read_req[ring], NULL);
|
|
|
|
|
2016-03-16 19:00:38 +08:00
|
|
|
if (obj->last_write_req && obj->last_write_req->engine->id == ring)
|
2015-04-27 20:41:17 +08:00
|
|
|
i915_gem_object_retire__write(obj);
|
|
|
|
|
|
|
|
obj->active &= ~(1 << ring);
|
|
|
|
if (obj->active)
|
|
|
|
return;
|
2010-11-12 21:53:37 +08:00
|
|
|
|
2015-07-27 17:26:26 +08:00
|
|
|
/* Bump our place on the bound list to keep it roughly in LRU order
|
|
|
|
* so that we don't steal from recently used but inactive objects
|
|
|
|
* (unless we are forced to ofc!)
|
|
|
|
*/
|
|
|
|
list_move_tail(&obj->global_list,
|
|
|
|
&to_i915(obj->base.dev)->mm.bound_list);
|
|
|
|
|
2016-02-26 19:03:19 +08:00
|
|
|
list_for_each_entry(vma, &obj->vma_list, obj_link) {
|
|
|
|
if (!list_empty(&vma->vm_link))
|
|
|
|
list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
|
2013-12-07 06:10:51 +08:00
|
|
|
}
|
2010-11-12 21:53:37 +08:00
|
|
|
|
2014-11-25 02:49:26 +08:00
|
|
|
i915_gem_request_assign(&obj->last_fenced_req, NULL);
|
2010-11-12 21:53:37 +08:00
|
|
|
drm_gem_object_unreference(&obj->base);
|
2014-03-17 20:21:55 +08:00
|
|
|
}
|
|
|
|
|
2012-11-28 00:22:52 +08:00
|
|
|
static int
|
2012-12-19 17:13:08 +08:00
|
|
|
i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
|
2012-01-25 23:32:49 +08:00
|
|
|
{
|
2012-11-28 00:22:52 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2016-03-16 19:00:36 +08:00
|
|
|
struct intel_engine_cs *engine;
|
2016-04-07 14:29:13 +08:00
|
|
|
int ret;
|
2012-01-25 23:32:49 +08:00
|
|
|
|
2012-12-10 19:56:17 +08:00
|
|
|
/* Carefully retire all requests without writing to the rings */
|
2016-03-24 19:20:38 +08:00
|
|
|
for_each_engine(engine, dev_priv) {
|
2016-03-16 19:00:39 +08:00
|
|
|
ret = intel_engine_idle(engine);
|
2012-12-10 19:56:17 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
2012-11-28 00:22:52 +08:00
|
|
|
}
|
|
|
|
i915_gem_retire_requests(dev);
|
2012-12-10 19:56:17 +08:00
|
|
|
|
|
|
|
/* Finally reset hw state */
|
2016-04-07 14:29:13 +08:00
|
|
|
for_each_engine(engine, dev_priv)
|
2016-03-16 19:00:36 +08:00
|
|
|
intel_ring_init_seqno(engine, seqno);
|
2012-12-04 21:12:04 +08:00
|
|
|
|
2012-11-28 00:22:52 +08:00
|
|
|
return 0;
|
2012-01-25 23:32:49 +08:00
|
|
|
}
|
|
|
|
|
2012-12-19 17:13:08 +08:00
|
|
|
int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (seqno == 0)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/* HWS page needs to be set less than what we
|
|
|
|
* will inject to ring
|
|
|
|
*/
|
|
|
|
ret = i915_gem_init_seqno(dev, seqno - 1);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
/* Carefully set the last_seqno value so that wrap
|
|
|
|
* detection still works
|
|
|
|
*/
|
|
|
|
dev_priv->next_seqno = seqno;
|
|
|
|
dev_priv->last_seqno = seqno - 1;
|
|
|
|
if (dev_priv->last_seqno == 0)
|
|
|
|
dev_priv->last_seqno--;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2012-11-28 00:22:52 +08:00
|
|
|
int
|
|
|
|
i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
|
2012-01-25 23:32:49 +08:00
|
|
|
{
|
2012-11-28 00:22:52 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
|
|
|
|
|
|
|
/* reserve 0 for non-seqno */
|
|
|
|
if (dev_priv->next_seqno == 0) {
|
2012-12-19 17:13:08 +08:00
|
|
|
int ret = i915_gem_init_seqno(dev, 0);
|
2012-11-28 00:22:52 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
2012-01-25 23:32:49 +08:00
|
|
|
|
2012-11-28 00:22:52 +08:00
|
|
|
dev_priv->next_seqno = 1;
|
|
|
|
}
|
2012-01-25 23:32:49 +08:00
|
|
|
|
2012-12-10 21:41:48 +08:00
|
|
|
*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
|
2012-11-28 00:22:52 +08:00
|
|
|
return 0;
|
2012-01-25 23:32:49 +08:00
|
|
|
}
|
|
|
|
|
2015-05-30 00:43:24 +08:00
|
|
|
/*
|
|
|
|
* NB: This function is not allowed to fail. Doing so would mean the the
|
|
|
|
* request is not being tracked for completion but the work itself is
|
|
|
|
* going to happen on the hardware. This would be a Bad Thing(tm).
|
|
|
|
*/
|
2015-05-30 00:43:49 +08:00
|
|
|
void __i915_add_request(struct drm_i915_gem_request *request,
|
2015-05-30 00:43:34 +08:00
|
|
|
struct drm_i915_gem_object *obj,
|
|
|
|
bool flush_caches)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
2016-03-16 19:00:36 +08:00
|
|
|
struct intel_engine_cs *engine;
|
2015-05-30 00:43:49 +08:00
|
|
|
struct drm_i915_private *dev_priv;
|
2014-07-25 00:04:29 +08:00
|
|
|
struct intel_ringbuffer *ringbuf;
|
2015-01-15 21:10:39 +08:00
|
|
|
u32 request_start;
|
2010-10-27 23:11:02 +08:00
|
|
|
int ret;
|
|
|
|
|
2014-07-25 00:04:29 +08:00
|
|
|
if (WARN_ON(request == NULL))
|
2015-05-30 00:43:24 +08:00
|
|
|
return;
|
2014-07-25 00:04:29 +08:00
|
|
|
|
2016-03-16 19:00:38 +08:00
|
|
|
engine = request->engine;
|
2016-03-17 21:04:10 +08:00
|
|
|
dev_priv = request->i915;
|
2015-05-30 00:43:49 +08:00
|
|
|
ringbuf = request->ringbuf;
|
|
|
|
|
drm/i915: Reserve ring buffer space for i915_add_request() commands
It is a bad idea for i915_add_request() to fail. The work will already have been
send to the ring and will be processed, but there will not be any tracking or
management of that work.
The only way the add request call can fail is if it can't write its epilogue
commands to the ring (cache flushing, seqno updates, interrupt signalling). The
reasons for that are mostly down to running out of ring buffer space and the
problems associated with trying to get some more. This patch prevents that
situation from happening in the first place.
When a request is created, it marks sufficient space as reserved for the
epilogue commands. Thus guaranteeing that by the time the epilogue is written,
there will be plenty of space for it. Note that a ring_begin() call is required
to actually reserve the space (and do any potential waiting). However, that is
not currently done at request creation time. This is because the ring_begin()
code can allocate a request. Hence calling begin() from the request allocation
code would lead to infinite recursion! Later patches in this series remove the
need for begin() to do the allocate. At that point, it becomes safe for the
allocate to call begin() and really reserve the space.
Until then, there is a potential for insufficient space to be available at the
point of calling i915_add_request(). However, that would only be in the case
where the request was created and immediately submitted without ever calling
ring_begin() and adding any work to that request. Which should never happen. And
even if it does, and if that request happens to fall down the tiny window of
opportunity for failing due to being out of ring space then does it really
matter because the request wasn't doing anything in the first place?
v2: Updated the 'reserved space too small' warning to include the offending
sizes. Added a 'cancel' operation to clean up when a request is abandoned. Added
re-initialisation of tracking state after a buffer wrap to keep the sanity
checks accurate.
v3: Incremented the reserved size to accommodate Ironlake (after finally
managing to run on an ILK system). Also fixed missing wrap code in LRC mode.
v4: Added extra comment and removed duplicate WARN (feedback from Tomas).
For: VIZ-5115
CC: Tomas Elf <tomas.elf@intel.com>
Signed-off-by: John Harrison <John.C.Harrison@Intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2015-06-18 20:10:09 +08:00
|
|
|
/*
|
|
|
|
* To ensure that this call will not fail, space for its emissions
|
|
|
|
* should already have been reserved in the ring buffer. Let the ring
|
|
|
|
* know that it is time to use that space up.
|
|
|
|
*/
|
|
|
|
intel_ring_reserved_space_use(ringbuf);
|
|
|
|
|
2014-07-25 00:04:29 +08:00
|
|
|
request_start = intel_ring_get_tail(ringbuf);
|
2012-06-14 02:45:19 +08:00
|
|
|
/*
|
|
|
|
* Emit any outstanding flushes - execbuf can fail to emit the flush
|
|
|
|
* after having emitted the batchbuffer command. Hence we need to fix
|
|
|
|
* things up similar to emitting the lazy request. The difference here
|
|
|
|
* is that the flush _must_ happen before the next request, no matter
|
|
|
|
* what.
|
|
|
|
*/
|
2015-05-30 00:43:34 +08:00
|
|
|
if (flush_caches) {
|
|
|
|
if (i915.enable_execlists)
|
2015-05-30 00:43:55 +08:00
|
|
|
ret = logical_ring_flush_all_caches(request);
|
2015-05-30 00:43:34 +08:00
|
|
|
else
|
2015-05-30 00:43:55 +08:00
|
|
|
ret = intel_ring_flush_all_caches(request);
|
2015-05-30 00:43:34 +08:00
|
|
|
/* Not allowed to fail! */
|
|
|
|
WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
|
|
|
|
}
|
2012-06-14 02:45:19 +08:00
|
|
|
|
2016-04-07 14:29:17 +08:00
|
|
|
trace_i915_gem_request_add(request);
|
|
|
|
|
|
|
|
request->head = request_start;
|
|
|
|
|
|
|
|
/* Whilst this request exists, batch_obj will be on the
|
|
|
|
* active_list, and so will hold the active reference. Only when this
|
|
|
|
* request is retired will the the batch_obj be moved onto the
|
|
|
|
* inactive_list and lose its active reference. Hence we do not need
|
|
|
|
* to explicitly hold another reference here.
|
|
|
|
*/
|
|
|
|
request->batch_obj = obj;
|
|
|
|
|
|
|
|
/* Seal the request and mark it as pending execution. Note that
|
|
|
|
* we may inspect this state, without holding any locks, during
|
|
|
|
* hangcheck. Hence we apply the barrier to ensure that we do not
|
|
|
|
* see a more recent value in the hws than we are tracking.
|
|
|
|
*/
|
|
|
|
request->emitted_jiffies = jiffies;
|
|
|
|
request->previous_seqno = engine->last_submitted_seqno;
|
|
|
|
smp_store_mb(engine->last_submitted_seqno, request->seqno);
|
|
|
|
list_add_tail(&request->list, &engine->request_list);
|
|
|
|
|
2012-02-15 19:25:36 +08:00
|
|
|
/* Record the position of the start of the request so that
|
|
|
|
* should we detect the updated seqno part-way through the
|
|
|
|
* GPU processing the request, we never over-estimate the
|
|
|
|
* position of the head.
|
|
|
|
*/
|
2015-01-15 21:10:39 +08:00
|
|
|
request->postfix = intel_ring_get_tail(ringbuf);
|
2012-02-15 19:25:36 +08:00
|
|
|
|
2015-05-30 00:43:24 +08:00
|
|
|
if (i915.enable_execlists)
|
2016-03-16 19:00:36 +08:00
|
|
|
ret = engine->emit_request(request);
|
2015-05-30 00:43:24 +08:00
|
|
|
else {
|
2016-03-16 19:00:36 +08:00
|
|
|
ret = engine->add_request(request);
|
2015-04-16 01:11:33 +08:00
|
|
|
|
|
|
|
request->tail = intel_ring_get_tail(ringbuf);
|
2014-07-25 00:04:29 +08:00
|
|
|
}
|
2015-05-30 00:43:24 +08:00
|
|
|
/* Not allowed to fail! */
|
|
|
|
WARN(ret, "emit|add_request failed: %d!\n", ret);
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2016-03-16 19:00:36 +08:00
|
|
|
i915_queue_hangcheck(engine->dev);
|
2013-07-03 22:22:08 +08:00
|
|
|
|
2014-11-20 03:36:48 +08:00
|
|
|
queue_delayed_work(dev_priv->wq,
|
|
|
|
&dev_priv->mm.retire_work,
|
|
|
|
round_jiffies_up_relative(HZ));
|
|
|
|
intel_mark_busy(dev_priv->dev);
|
2012-06-14 02:45:19 +08:00
|
|
|
|
drm/i915: Reserve ring buffer space for i915_add_request() commands
It is a bad idea for i915_add_request() to fail. The work will already have been
send to the ring and will be processed, but there will not be any tracking or
management of that work.
The only way the add request call can fail is if it can't write its epilogue
commands to the ring (cache flushing, seqno updates, interrupt signalling). The
reasons for that are mostly down to running out of ring buffer space and the
problems associated with trying to get some more. This patch prevents that
situation from happening in the first place.
When a request is created, it marks sufficient space as reserved for the
epilogue commands. Thus guaranteeing that by the time the epilogue is written,
there will be plenty of space for it. Note that a ring_begin() call is required
to actually reserve the space (and do any potential waiting). However, that is
not currently done at request creation time. This is because the ring_begin()
code can allocate a request. Hence calling begin() from the request allocation
code would lead to infinite recursion! Later patches in this series remove the
need for begin() to do the allocate. At that point, it becomes safe for the
allocate to call begin() and really reserve the space.
Until then, there is a potential for insufficient space to be available at the
point of calling i915_add_request(). However, that would only be in the case
where the request was created and immediately submitted without ever calling
ring_begin() and adding any work to that request. Which should never happen. And
even if it does, and if that request happens to fall down the tiny window of
opportunity for failing due to being out of ring space then does it really
matter because the request wasn't doing anything in the first place?
v2: Updated the 'reserved space too small' warning to include the offending
sizes. Added a 'cancel' operation to clean up when a request is abandoned. Added
re-initialisation of tracking state after a buffer wrap to keep the sanity
checks accurate.
v3: Incremented the reserved size to accommodate Ironlake (after finally
managing to run on an ILK system). Also fixed missing wrap code in LRC mode.
v4: Added extra comment and removed duplicate WARN (feedback from Tomas).
For: VIZ-5115
CC: Tomas Elf <tomas.elf@intel.com>
Signed-off-by: John Harrison <John.C.Harrison@Intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2015-06-18 20:10:09 +08:00
|
|
|
/* Sanity check that the reserved size was large enough. */
|
|
|
|
intel_ring_reserved_space_end(ringbuf);
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
2014-01-31 01:04:44 +08:00
|
|
|
static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
|
2014-05-22 21:13:37 +08:00
|
|
|
const struct intel_context *ctx)
|
2013-08-30 21:19:28 +08:00
|
|
|
{
|
2014-01-30 22:01:15 +08:00
|
|
|
unsigned long elapsed;
|
2013-08-30 21:19:28 +08:00
|
|
|
|
2014-01-30 22:01:15 +08:00
|
|
|
elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
|
|
|
|
|
|
|
|
if (ctx->hang_stats.banned)
|
2013-08-30 21:19:28 +08:00
|
|
|
return true;
|
|
|
|
|
2014-12-25 00:13:39 +08:00
|
|
|
if (ctx->hang_stats.ban_period_seconds &&
|
|
|
|
elapsed <= ctx->hang_stats.ban_period_seconds) {
|
2014-02-21 22:26:47 +08:00
|
|
|
if (!i915_gem_context_is_default(ctx)) {
|
2014-01-30 22:05:48 +08:00
|
|
|
DRM_DEBUG("context hanging too fast, banning!\n");
|
2014-02-21 22:26:47 +08:00
|
|
|
return true;
|
2014-03-29 00:18:18 +08:00
|
|
|
} else if (i915_stop_ring_allow_ban(dev_priv)) {
|
|
|
|
if (i915_stop_ring_allow_warn(dev_priv))
|
|
|
|
DRM_ERROR("gpu hanging too fast, banning!\n");
|
2014-02-21 22:26:47 +08:00
|
|
|
return true;
|
2014-01-30 22:05:48 +08:00
|
|
|
}
|
2013-08-30 21:19:28 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2014-01-31 01:04:44 +08:00
|
|
|
static void i915_set_reset_status(struct drm_i915_private *dev_priv,
|
2014-05-22 21:13:37 +08:00
|
|
|
struct intel_context *ctx,
|
2014-01-31 01:04:43 +08:00
|
|
|
const bool guilty)
|
2013-06-12 20:13:20 +08:00
|
|
|
{
|
2014-01-30 22:01:15 +08:00
|
|
|
struct i915_ctx_hang_stats *hs;
|
|
|
|
|
|
|
|
if (WARN_ON(!ctx))
|
|
|
|
return;
|
2013-06-12 20:13:20 +08:00
|
|
|
|
2014-01-30 22:01:15 +08:00
|
|
|
hs = &ctx->hang_stats;
|
|
|
|
|
|
|
|
if (guilty) {
|
2014-01-31 01:04:44 +08:00
|
|
|
hs->banned = i915_context_is_banned(dev_priv, ctx);
|
2014-01-30 22:01:15 +08:00
|
|
|
hs->batch_active++;
|
|
|
|
hs->guilty_ts = get_seconds();
|
|
|
|
} else {
|
|
|
|
hs->batch_pending++;
|
2013-06-12 20:13:20 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-11-25 02:49:24 +08:00
|
|
|
void i915_gem_request_free(struct kref *req_ref)
|
|
|
|
{
|
|
|
|
struct drm_i915_gem_request *req = container_of(req_ref,
|
|
|
|
typeof(*req), ref);
|
|
|
|
struct intel_context *ctx = req->ctx;
|
|
|
|
|
2015-05-30 00:44:12 +08:00
|
|
|
if (req->file_priv)
|
|
|
|
i915_gem_request_remove_from_client(req);
|
|
|
|
|
2014-11-25 18:39:25 +08:00
|
|
|
if (ctx) {
|
2016-01-20 03:02:55 +08:00
|
|
|
if (i915.enable_execlists && ctx != req->i915->kernel_context)
|
2016-03-16 19:00:38 +08:00
|
|
|
intel_lr_context_unpin(ctx, req->engine);
|
2014-11-25 02:49:24 +08:00
|
|
|
|
drm/i915/bdw: Pin the context backing objects to GGTT on-demand
Up until now, we have pinned every logical ring context backing object
during creation, and left it pinned until destruction. This made my life
easier, but it's a harmful thing to do, because we cause fragmentation
of the GGTT (and, eventually, we would run out of space).
This patch makes the pinning on-demand: the backing objects of the two
contexts that are written to the ELSP are pinned right before submission
and unpinned once the hardware is done with them. The only context that
is still pinned regardless is the global default one, so that the HWS can
still be accessed in the same way (ring->status_page).
v2: In the early version of this patch, we were pinning the context as
we put it into the ELSP: on the one hand, this is very efficient because
only a maximum two contexts are pinned at any given time, but on the other
hand, we cannot really pin in interrupt time :(
v3: Use a mutex rather than atomic_t to protect pin count to avoid races.
Do not unpin default context in free_request.
v4: Break out pin and unpin into functions. Fix style problems reported
by checkpatch
v5: Remove unpin_lock as all pinning and unpinning is done with the struct
mutex already locked. Add WARN_ONs to make sure this is the case in future.
Issue: VIZ-4277
Signed-off-by: Oscar Mateo <oscar.mateo@intel.com>
Signed-off-by: Thomas Daniel <thomas.daniel@intel.com>
Reviewed-by: Akash Goel <akash.goels@gmail.com>
Reviewed-by: Deepak S<deepak.s@linux.intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2014-11-13 18:28:10 +08:00
|
|
|
i915_gem_context_unreference(ctx);
|
|
|
|
}
|
2014-11-25 02:49:24 +08:00
|
|
|
|
2015-04-07 23:20:57 +08:00
|
|
|
kmem_cache_free(req->i915->requests, req);
|
2013-05-02 21:48:08 +08:00
|
|
|
}
|
|
|
|
|
drm/i915: simplify allocation of driver-internal requests
There are a number of places where the driver needs a request, but isn't
working on behalf of any specific user or in a specific context. At
present, we associate them with the per-engine default context. A future
patch will abolish those per-engine context pointers; but we can already
eliminate a lot of the references to them, just by making the allocator
allow NULL as a shorthand for "an appropriate context for this ring",
which will mean that the callers don't need to know anything about how
the "appropriate context" is found (e.g. per-ring vs per-device, etc).
So this patch renames the existing i915_gem_request_alloc(), and makes
it local (static inline), and replaces it with a wrapper that provides
a default if the context is NULL, and also has a nicer calling
convention (doesn't require a pointer to an output parameter). Then we
change all callers to use the new convention:
OLD:
err = i915_gem_request_alloc(ring, user_ctx, &req);
if (err) ...
NEW:
req = i915_gem_request_alloc(ring, user_ctx);
if (IS_ERR(req)) ...
OLD:
err = i915_gem_request_alloc(ring, ring->default_context, &req);
if (err) ...
NEW:
req = i915_gem_request_alloc(ring, NULL);
if (IS_ERR(req)) ...
v4: Rebased
Signed-off-by: Dave Gordon <david.s.gordon@intel.com>
Reviewed-by: Nick Hoath <nicholas.hoath@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1453230175-19330-2-git-send-email-david.s.gordon@intel.com
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2016-01-20 03:02:53 +08:00
|
|
|
static inline int
|
2016-03-16 19:00:37 +08:00
|
|
|
__i915_gem_request_alloc(struct intel_engine_cs *engine,
|
drm/i915: simplify allocation of driver-internal requests
There are a number of places where the driver needs a request, but isn't
working on behalf of any specific user or in a specific context. At
present, we associate them with the per-engine default context. A future
patch will abolish those per-engine context pointers; but we can already
eliminate a lot of the references to them, just by making the allocator
allow NULL as a shorthand for "an appropriate context for this ring",
which will mean that the callers don't need to know anything about how
the "appropriate context" is found (e.g. per-ring vs per-device, etc).
So this patch renames the existing i915_gem_request_alloc(), and makes
it local (static inline), and replaces it with a wrapper that provides
a default if the context is NULL, and also has a nicer calling
convention (doesn't require a pointer to an output parameter). Then we
change all callers to use the new convention:
OLD:
err = i915_gem_request_alloc(ring, user_ctx, &req);
if (err) ...
NEW:
req = i915_gem_request_alloc(ring, user_ctx);
if (IS_ERR(req)) ...
OLD:
err = i915_gem_request_alloc(ring, ring->default_context, &req);
if (err) ...
NEW:
req = i915_gem_request_alloc(ring, NULL);
if (IS_ERR(req)) ...
v4: Rebased
Signed-off-by: Dave Gordon <david.s.gordon@intel.com>
Reviewed-by: Nick Hoath <nicholas.hoath@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1453230175-19330-2-git-send-email-david.s.gordon@intel.com
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2016-01-20 03:02:53 +08:00
|
|
|
struct intel_context *ctx,
|
|
|
|
struct drm_i915_gem_request **req_out)
|
2015-03-19 20:30:08 +08:00
|
|
|
{
|
2016-03-16 19:00:37 +08:00
|
|
|
struct drm_i915_private *dev_priv = to_i915(engine->dev);
|
2015-05-21 20:21:25 +08:00
|
|
|
struct drm_i915_gem_request *req;
|
2015-03-19 20:30:08 +08:00
|
|
|
int ret;
|
|
|
|
|
2015-05-30 00:43:29 +08:00
|
|
|
if (!req_out)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2015-05-30 00:44:11 +08:00
|
|
|
*req_out = NULL;
|
2015-03-19 20:30:08 +08:00
|
|
|
|
2015-05-21 20:21:25 +08:00
|
|
|
req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
|
|
|
|
if (req == NULL)
|
2015-03-19 20:30:08 +08:00
|
|
|
return -ENOMEM;
|
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
ret = i915_gem_get_seqno(engine->dev, &req->seqno);
|
2015-05-22 04:01:45 +08:00
|
|
|
if (ret)
|
|
|
|
goto err;
|
2015-03-19 20:30:08 +08:00
|
|
|
|
2015-05-30 00:43:26 +08:00
|
|
|
kref_init(&req->ref);
|
|
|
|
req->i915 = dev_priv;
|
2016-03-16 19:00:38 +08:00
|
|
|
req->engine = engine;
|
2015-05-30 00:43:26 +08:00
|
|
|
req->ctx = ctx;
|
|
|
|
i915_gem_context_reference(req->ctx);
|
2015-03-19 20:30:08 +08:00
|
|
|
|
|
|
|
if (i915.enable_execlists)
|
2015-05-30 00:43:26 +08:00
|
|
|
ret = intel_logical_ring_alloc_request_extras(req);
|
2015-03-19 20:30:08 +08:00
|
|
|
else
|
2015-05-21 20:21:25 +08:00
|
|
|
ret = intel_ring_alloc_request_extras(req);
|
2015-05-30 00:43:26 +08:00
|
|
|
if (ret) {
|
|
|
|
i915_gem_context_unreference(req->ctx);
|
2015-05-22 04:01:45 +08:00
|
|
|
goto err;
|
2015-05-30 00:43:26 +08:00
|
|
|
}
|
2015-03-19 20:30:08 +08:00
|
|
|
|
drm/i915: Reserve ring buffer space for i915_add_request() commands
It is a bad idea for i915_add_request() to fail. The work will already have been
send to the ring and will be processed, but there will not be any tracking or
management of that work.
The only way the add request call can fail is if it can't write its epilogue
commands to the ring (cache flushing, seqno updates, interrupt signalling). The
reasons for that are mostly down to running out of ring buffer space and the
problems associated with trying to get some more. This patch prevents that
situation from happening in the first place.
When a request is created, it marks sufficient space as reserved for the
epilogue commands. Thus guaranteeing that by the time the epilogue is written,
there will be plenty of space for it. Note that a ring_begin() call is required
to actually reserve the space (and do any potential waiting). However, that is
not currently done at request creation time. This is because the ring_begin()
code can allocate a request. Hence calling begin() from the request allocation
code would lead to infinite recursion! Later patches in this series remove the
need for begin() to do the allocate. At that point, it becomes safe for the
allocate to call begin() and really reserve the space.
Until then, there is a potential for insufficient space to be available at the
point of calling i915_add_request(). However, that would only be in the case
where the request was created and immediately submitted without ever calling
ring_begin() and adding any work to that request. Which should never happen. And
even if it does, and if that request happens to fall down the tiny window of
opportunity for failing due to being out of ring space then does it really
matter because the request wasn't doing anything in the first place?
v2: Updated the 'reserved space too small' warning to include the offending
sizes. Added a 'cancel' operation to clean up when a request is abandoned. Added
re-initialisation of tracking state after a buffer wrap to keep the sanity
checks accurate.
v3: Incremented the reserved size to accommodate Ironlake (after finally
managing to run on an ILK system). Also fixed missing wrap code in LRC mode.
v4: Added extra comment and removed duplicate WARN (feedback from Tomas).
For: VIZ-5115
CC: Tomas Elf <tomas.elf@intel.com>
Signed-off-by: John Harrison <John.C.Harrison@Intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2015-06-18 20:10:09 +08:00
|
|
|
/*
|
|
|
|
* Reserve space in the ring buffer for all the commands required to
|
|
|
|
* eventually emit this request. This is to guarantee that the
|
|
|
|
* i915_add_request() call can't fail. Note that the reserve may need
|
|
|
|
* to be redone if the request is not actually submitted straight
|
|
|
|
* away, e.g. because a GPU scheduler has deferred it.
|
|
|
|
*/
|
2015-05-30 00:44:09 +08:00
|
|
|
if (i915.enable_execlists)
|
|
|
|
ret = intel_logical_ring_reserve_space(req);
|
|
|
|
else
|
|
|
|
ret = intel_ring_reserve_space(req);
|
|
|
|
if (ret) {
|
|
|
|
/*
|
|
|
|
* At this point, the request is fully allocated even if not
|
|
|
|
* fully prepared. Thus it can be cleaned up using the proper
|
|
|
|
* free code.
|
|
|
|
*/
|
|
|
|
i915_gem_request_cancel(req);
|
|
|
|
return ret;
|
|
|
|
}
|
drm/i915: Reserve ring buffer space for i915_add_request() commands
It is a bad idea for i915_add_request() to fail. The work will already have been
send to the ring and will be processed, but there will not be any tracking or
management of that work.
The only way the add request call can fail is if it can't write its epilogue
commands to the ring (cache flushing, seqno updates, interrupt signalling). The
reasons for that are mostly down to running out of ring buffer space and the
problems associated with trying to get some more. This patch prevents that
situation from happening in the first place.
When a request is created, it marks sufficient space as reserved for the
epilogue commands. Thus guaranteeing that by the time the epilogue is written,
there will be plenty of space for it. Note that a ring_begin() call is required
to actually reserve the space (and do any potential waiting). However, that is
not currently done at request creation time. This is because the ring_begin()
code can allocate a request. Hence calling begin() from the request allocation
code would lead to infinite recursion! Later patches in this series remove the
need for begin() to do the allocate. At that point, it becomes safe for the
allocate to call begin() and really reserve the space.
Until then, there is a potential for insufficient space to be available at the
point of calling i915_add_request(). However, that would only be in the case
where the request was created and immediately submitted without ever calling
ring_begin() and adding any work to that request. Which should never happen. And
even if it does, and if that request happens to fall down the tiny window of
opportunity for failing due to being out of ring space then does it really
matter because the request wasn't doing anything in the first place?
v2: Updated the 'reserved space too small' warning to include the offending
sizes. Added a 'cancel' operation to clean up when a request is abandoned. Added
re-initialisation of tracking state after a buffer wrap to keep the sanity
checks accurate.
v3: Incremented the reserved size to accommodate Ironlake (after finally
managing to run on an ILK system). Also fixed missing wrap code in LRC mode.
v4: Added extra comment and removed duplicate WARN (feedback from Tomas).
For: VIZ-5115
CC: Tomas Elf <tomas.elf@intel.com>
Signed-off-by: John Harrison <John.C.Harrison@Intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2015-06-18 20:10:09 +08:00
|
|
|
|
2015-05-30 00:44:11 +08:00
|
|
|
*req_out = req;
|
2015-03-19 20:30:08 +08:00
|
|
|
return 0;
|
2015-05-22 04:01:45 +08:00
|
|
|
|
|
|
|
err:
|
|
|
|
kmem_cache_free(dev_priv->requests, req);
|
|
|
|
return ret;
|
2013-05-02 21:48:08 +08:00
|
|
|
}
|
|
|
|
|
drm/i915: simplify allocation of driver-internal requests
There are a number of places where the driver needs a request, but isn't
working on behalf of any specific user or in a specific context. At
present, we associate them with the per-engine default context. A future
patch will abolish those per-engine context pointers; but we can already
eliminate a lot of the references to them, just by making the allocator
allow NULL as a shorthand for "an appropriate context for this ring",
which will mean that the callers don't need to know anything about how
the "appropriate context" is found (e.g. per-ring vs per-device, etc).
So this patch renames the existing i915_gem_request_alloc(), and makes
it local (static inline), and replaces it with a wrapper that provides
a default if the context is NULL, and also has a nicer calling
convention (doesn't require a pointer to an output parameter). Then we
change all callers to use the new convention:
OLD:
err = i915_gem_request_alloc(ring, user_ctx, &req);
if (err) ...
NEW:
req = i915_gem_request_alloc(ring, user_ctx);
if (IS_ERR(req)) ...
OLD:
err = i915_gem_request_alloc(ring, ring->default_context, &req);
if (err) ...
NEW:
req = i915_gem_request_alloc(ring, NULL);
if (IS_ERR(req)) ...
v4: Rebased
Signed-off-by: Dave Gordon <david.s.gordon@intel.com>
Reviewed-by: Nick Hoath <nicholas.hoath@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1453230175-19330-2-git-send-email-david.s.gordon@intel.com
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2016-01-20 03:02:53 +08:00
|
|
|
/**
|
|
|
|
* i915_gem_request_alloc - allocate a request structure
|
|
|
|
*
|
|
|
|
* @engine: engine that we wish to issue the request on.
|
|
|
|
* @ctx: context that the request will be associated with.
|
|
|
|
* This can be NULL if the request is not directly related to
|
|
|
|
* any specific user context, in which case this function will
|
|
|
|
* choose an appropriate context to use.
|
|
|
|
*
|
|
|
|
* Returns a pointer to the allocated request if successful,
|
|
|
|
* or an error code if not.
|
|
|
|
*/
|
|
|
|
struct drm_i915_gem_request *
|
|
|
|
i915_gem_request_alloc(struct intel_engine_cs *engine,
|
|
|
|
struct intel_context *ctx)
|
|
|
|
{
|
|
|
|
struct drm_i915_gem_request *req;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
if (ctx == NULL)
|
2016-01-20 03:02:54 +08:00
|
|
|
ctx = to_i915(engine->dev)->kernel_context;
|
drm/i915: simplify allocation of driver-internal requests
There are a number of places where the driver needs a request, but isn't
working on behalf of any specific user or in a specific context. At
present, we associate them with the per-engine default context. A future
patch will abolish those per-engine context pointers; but we can already
eliminate a lot of the references to them, just by making the allocator
allow NULL as a shorthand for "an appropriate context for this ring",
which will mean that the callers don't need to know anything about how
the "appropriate context" is found (e.g. per-ring vs per-device, etc).
So this patch renames the existing i915_gem_request_alloc(), and makes
it local (static inline), and replaces it with a wrapper that provides
a default if the context is NULL, and also has a nicer calling
convention (doesn't require a pointer to an output parameter). Then we
change all callers to use the new convention:
OLD:
err = i915_gem_request_alloc(ring, user_ctx, &req);
if (err) ...
NEW:
req = i915_gem_request_alloc(ring, user_ctx);
if (IS_ERR(req)) ...
OLD:
err = i915_gem_request_alloc(ring, ring->default_context, &req);
if (err) ...
NEW:
req = i915_gem_request_alloc(ring, NULL);
if (IS_ERR(req)) ...
v4: Rebased
Signed-off-by: Dave Gordon <david.s.gordon@intel.com>
Reviewed-by: Nick Hoath <nicholas.hoath@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1453230175-19330-2-git-send-email-david.s.gordon@intel.com
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2016-01-20 03:02:53 +08:00
|
|
|
err = __i915_gem_request_alloc(engine, ctx, &req);
|
|
|
|
return err ? ERR_PTR(err) : req;
|
|
|
|
}
|
|
|
|
|
drm/i915: Reserve ring buffer space for i915_add_request() commands
It is a bad idea for i915_add_request() to fail. The work will already have been
send to the ring and will be processed, but there will not be any tracking or
management of that work.
The only way the add request call can fail is if it can't write its epilogue
commands to the ring (cache flushing, seqno updates, interrupt signalling). The
reasons for that are mostly down to running out of ring buffer space and the
problems associated with trying to get some more. This patch prevents that
situation from happening in the first place.
When a request is created, it marks sufficient space as reserved for the
epilogue commands. Thus guaranteeing that by the time the epilogue is written,
there will be plenty of space for it. Note that a ring_begin() call is required
to actually reserve the space (and do any potential waiting). However, that is
not currently done at request creation time. This is because the ring_begin()
code can allocate a request. Hence calling begin() from the request allocation
code would lead to infinite recursion! Later patches in this series remove the
need for begin() to do the allocate. At that point, it becomes safe for the
allocate to call begin() and really reserve the space.
Until then, there is a potential for insufficient space to be available at the
point of calling i915_add_request(). However, that would only be in the case
where the request was created and immediately submitted without ever calling
ring_begin() and adding any work to that request. Which should never happen. And
even if it does, and if that request happens to fall down the tiny window of
opportunity for failing due to being out of ring space then does it really
matter because the request wasn't doing anything in the first place?
v2: Updated the 'reserved space too small' warning to include the offending
sizes. Added a 'cancel' operation to clean up when a request is abandoned. Added
re-initialisation of tracking state after a buffer wrap to keep the sanity
checks accurate.
v3: Incremented the reserved size to accommodate Ironlake (after finally
managing to run on an ILK system). Also fixed missing wrap code in LRC mode.
v4: Added extra comment and removed duplicate WARN (feedback from Tomas).
For: VIZ-5115
CC: Tomas Elf <tomas.elf@intel.com>
Signed-off-by: John Harrison <John.C.Harrison@Intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2015-06-18 20:10:09 +08:00
|
|
|
void i915_gem_request_cancel(struct drm_i915_gem_request *req)
|
|
|
|
{
|
|
|
|
intel_ring_reserved_space_cancel(req->ringbuf);
|
|
|
|
|
|
|
|
i915_gem_request_unreference(req);
|
|
|
|
}
|
|
|
|
|
2014-02-25 23:11:23 +08:00
|
|
|
struct drm_i915_gem_request *
|
2016-03-16 19:00:37 +08:00
|
|
|
i915_gem_find_active_request(struct intel_engine_cs *engine)
|
2010-09-19 19:21:28 +08:00
|
|
|
{
|
2013-12-04 19:37:09 +08:00
|
|
|
struct drm_i915_gem_request *request;
|
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
list_for_each_entry(request, &engine->request_list, list) {
|
2014-11-25 02:49:42 +08:00
|
|
|
if (i915_gem_request_completed(request, false))
|
2013-12-04 19:37:09 +08:00
|
|
|
continue;
|
2013-06-12 20:13:20 +08:00
|
|
|
|
2014-01-31 01:04:43 +08:00
|
|
|
return request;
|
2013-12-04 19:37:09 +08:00
|
|
|
}
|
2014-01-31 01:04:43 +08:00
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2016-03-16 19:00:39 +08:00
|
|
|
static void i915_gem_reset_engine_status(struct drm_i915_private *dev_priv,
|
2016-03-16 19:00:37 +08:00
|
|
|
struct intel_engine_cs *engine)
|
2014-01-31 01:04:43 +08:00
|
|
|
{
|
|
|
|
struct drm_i915_gem_request *request;
|
|
|
|
bool ring_hung;
|
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
request = i915_gem_find_active_request(engine);
|
2014-01-31 01:04:43 +08:00
|
|
|
|
|
|
|
if (request == NULL)
|
|
|
|
return;
|
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
|
2014-01-31 01:04:43 +08:00
|
|
|
|
2014-01-31 01:04:44 +08:00
|
|
|
i915_set_reset_status(dev_priv, request->ctx, ring_hung);
|
2014-01-31 01:04:43 +08:00
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
list_for_each_entry_continue(request, &engine->request_list, list)
|
2014-01-31 01:04:44 +08:00
|
|
|
i915_set_reset_status(dev_priv, request->ctx, false);
|
2013-12-04 19:37:09 +08:00
|
|
|
}
|
2013-06-12 20:13:20 +08:00
|
|
|
|
2016-03-16 19:00:39 +08:00
|
|
|
static void i915_gem_reset_engine_cleanup(struct drm_i915_private *dev_priv,
|
2016-03-16 19:00:37 +08:00
|
|
|
struct intel_engine_cs *engine)
|
2013-12-04 19:37:09 +08:00
|
|
|
{
|
2015-09-03 20:01:40 +08:00
|
|
|
struct intel_ringbuffer *buffer;
|
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
while (!list_empty(&engine->active_list)) {
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_i915_gem_object *obj;
|
2010-09-19 19:21:28 +08:00
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
obj = list_first_entry(&engine->active_list,
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_i915_gem_object,
|
2016-03-16 19:00:40 +08:00
|
|
|
engine_list[engine->id]);
|
2010-09-19 19:21:28 +08:00
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
i915_gem_object_retire__read(obj, engine->id);
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
2014-01-02 02:15:13 +08:00
|
|
|
|
drm/i915/bdw: Pin the context backing objects to GGTT on-demand
Up until now, we have pinned every logical ring context backing object
during creation, and left it pinned until destruction. This made my life
easier, but it's a harmful thing to do, because we cause fragmentation
of the GGTT (and, eventually, we would run out of space).
This patch makes the pinning on-demand: the backing objects of the two
contexts that are written to the ELSP are pinned right before submission
and unpinned once the hardware is done with them. The only context that
is still pinned regardless is the global default one, so that the HWS can
still be accessed in the same way (ring->status_page).
v2: In the early version of this patch, we were pinning the context as
we put it into the ELSP: on the one hand, this is very efficient because
only a maximum two contexts are pinned at any given time, but on the other
hand, we cannot really pin in interrupt time :(
v3: Use a mutex rather than atomic_t to protect pin count to avoid races.
Do not unpin default context in free_request.
v4: Break out pin and unpin into functions. Fix style problems reported
by checkpatch
v5: Remove unpin_lock as all pinning and unpinning is done with the struct
mutex already locked. Add WARN_ONs to make sure this is the case in future.
Issue: VIZ-4277
Signed-off-by: Oscar Mateo <oscar.mateo@intel.com>
Signed-off-by: Thomas Daniel <thomas.daniel@intel.com>
Reviewed-by: Akash Goel <akash.goels@gmail.com>
Reviewed-by: Deepak S<deepak.s@linux.intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2014-11-13 18:28:10 +08:00
|
|
|
/*
|
|
|
|
* Clear the execlists queue up before freeing the requests, as those
|
|
|
|
* are the ones that keep the context and ringbuffer backing objects
|
|
|
|
* pinned in place.
|
|
|
|
*/
|
|
|
|
|
2015-10-19 23:32:32 +08:00
|
|
|
if (i915.enable_execlists) {
|
drm/i915: Move execlists irq handler to a bottom half
Doing a lot of work in the interrupt handler introduces huge
latencies to the system as a whole.
Most dramatic effect can be seen by running an all engine
stress test like igt/gem_exec_nop/all where, when the kernel
config is lean enough, the whole system can be brought into
multi-second periods of complete non-interactivty. That can
look for example like this:
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [kworker/u8:3:143]
Modules linked in: [redacted for brevity]
CPU: 0 PID: 143 Comm: kworker/u8:3 Tainted: G U L 4.5.0-160321+ #183
Hardware name: Intel Corporation Broadwell Client platform/WhiteTip Mountain 1
Workqueue: i915 gen6_pm_rps_work [i915]
task: ffff8800aae88000 ti: ffff8800aae90000 task.ti: ffff8800aae90000
RIP: 0010:[<ffffffff8104a3c2>] [<ffffffff8104a3c2>] __do_softirq+0x72/0x1d0
RSP: 0000:ffff88014f403f38 EFLAGS: 00000206
RAX: ffff8800aae94000 RBX: 0000000000000000 RCX: 00000000000006e0
RDX: 0000000000000020 RSI: 0000000004208060 RDI: 0000000000215d80
RBP: ffff88014f403f80 R08: 0000000b1b42c180 R09: 0000000000000022
R10: 0000000000000004 R11: 00000000ffffffff R12: 000000000000a030
R13: 0000000000000082 R14: ffff8800aa4d0080 R15: 0000000000000082
FS: 0000000000000000(0000) GS:ffff88014f400000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa53b90c000 CR3: 0000000001a0a000 CR4: 00000000001406f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
042080601b33869f ffff8800aae94000 00000000fffc2678 ffff88010000000a
0000000000000000 000000000000a030 0000000000005302 ffff8800aa4d0080
0000000000000206 ffff88014f403f90 ffffffff8104a716 ffff88014f403fa8
Call Trace:
<IRQ>
[<ffffffff8104a716>] irq_exit+0x86/0x90
[<ffffffff81031e7d>] smp_apic_timer_interrupt+0x3d/0x50
[<ffffffff814f3eac>] apic_timer_interrupt+0x7c/0x90
<EOI>
[<ffffffffa01c5b40>] ? gen8_write64+0x1a0/0x1a0 [i915]
[<ffffffff814f2b39>] ? _raw_spin_unlock_irqrestore+0x9/0x20
[<ffffffffa01c5c44>] gen8_write32+0x104/0x1a0 [i915]
[<ffffffff8132c6a2>] ? n_tty_receive_buf_common+0x372/0xae0
[<ffffffffa017cc9e>] gen6_set_rps_thresholds+0x1be/0x330 [i915]
[<ffffffffa017eaf0>] gen6_set_rps+0x70/0x200 [i915]
[<ffffffffa0185375>] intel_set_rps+0x25/0x30 [i915]
[<ffffffffa01768fd>] gen6_pm_rps_work+0x10d/0x2e0 [i915]
[<ffffffff81063852>] ? finish_task_switch+0x72/0x1c0
[<ffffffff8105ab29>] process_one_work+0x139/0x350
[<ffffffff8105b186>] worker_thread+0x126/0x490
[<ffffffff8105b060>] ? rescuer_thread+0x320/0x320
[<ffffffff8105fa64>] kthread+0xc4/0xe0
[<ffffffff8105f9a0>] ? kthread_create_on_node+0x170/0x170
[<ffffffff814f351f>] ret_from_fork+0x3f/0x70
[<ffffffff8105f9a0>] ? kthread_create_on_node+0x170/0x170
I could not explain, or find a code path, which would explain
a +20 second lockup, but from some instrumentation it was
apparent the interrupts off proportion of time was between
10-25% under heavy load which is quite bad.
When a interrupt "cliff" is reached, which was >~320k irq/s on
my machine, the whole system goes into a terrible state of the
above described multi-second lockups.
By moving the GT interrupt handling to a tasklet in a most
simple way, the problem above disappears completely.
Testing the effect on sytem-wide latencies using
igt/gem_syslatency shows the following before this patch:
gem_syslatency: cycles=1532739, latency mean=416531.829us max=2499237us
gem_syslatency: cycles=1839434, latency mean=1458099.157us max=4998944us
gem_syslatency: cycles=1432570, latency mean=2688.451us max=1201185us
gem_syslatency: cycles=1533543, latency mean=416520.499us max=2498886us
This shows that the unrelated process is experiencing huge
delays in its wake-up latency. After the patch the results
look like this:
gem_syslatency: cycles=808907, latency mean=53.133us max=1640us
gem_syslatency: cycles=862154, latency mean=62.778us max=2117us
gem_syslatency: cycles=856039, latency mean=58.079us max=2123us
gem_syslatency: cycles=841683, latency mean=56.914us max=1667us
Showing a huge improvement in the unrelated process wake-up
latency. It also shows an approximate halving in the number
of total empty batches submitted during the test. This may
not be worrying since the test puts the driver under
a very unrealistic load with ncpu threads doing empty batch
submission to all GPU engines each.
Another benefit compared to the hard-irq handling is that now
work on all engines can be dispatched in parallel since we can
have up to number of CPUs active tasklets. (While previously
a single hard-irq would serially dispatch on one engine after
another.)
More interesting scenario with regards to throughput is
"gem_latency -n 100" which shows 25% better throughput and
CPU usage, and 14% better dispatch latencies.
I did not find any gains or regressions with Synmark2 or
GLbench under light testing. More benchmarking is certainly
required.
v2:
* execlists_lock should be taken as spin_lock_bh when
queuing work from userspace now. (Chris Wilson)
* uncore.lock must be taken with spin_lock_irq when
submitting requests since that now runs from either
softirq or process context.
v3:
* Expanded commit message with more testing data;
* converted missed locking sites to _bh;
* added execlist_lock comment. (Chris Wilson)
v4:
* Mention dispatch parallelism in commit. (Chris Wilson)
* Do not hold uncore.lock over MMIO reads since the block
is already serialised per-engine via the tasklet itself.
(Chris Wilson)
* intel_lrc_irq_handler should be static. (Chris Wilson)
* Cancel/sync the tasklet on GPU reset. (Chris Wilson)
* Document and WARN that tasklet cannot be active/pending
on engine cleanup. (Chris Wilson/Imre Deak)
Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Imre Deak <imre.deak@intel.com>
Testcase: igt/gem_exec_nop/all
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=94350
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: http://patchwork.freedesktop.org/patch/msgid/1459768316-6670-1-git-send-email-tvrtko.ursulin@linux.intel.com
2016-04-04 19:11:56 +08:00
|
|
|
/* Ensure irq handler finishes or is cancelled. */
|
|
|
|
tasklet_kill(&engine->irq_tasklet);
|
2015-01-13 17:32:24 +08:00
|
|
|
|
drm/i915: Move execlists irq handler to a bottom half
Doing a lot of work in the interrupt handler introduces huge
latencies to the system as a whole.
Most dramatic effect can be seen by running an all engine
stress test like igt/gem_exec_nop/all where, when the kernel
config is lean enough, the whole system can be brought into
multi-second periods of complete non-interactivty. That can
look for example like this:
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [kworker/u8:3:143]
Modules linked in: [redacted for brevity]
CPU: 0 PID: 143 Comm: kworker/u8:3 Tainted: G U L 4.5.0-160321+ #183
Hardware name: Intel Corporation Broadwell Client platform/WhiteTip Mountain 1
Workqueue: i915 gen6_pm_rps_work [i915]
task: ffff8800aae88000 ti: ffff8800aae90000 task.ti: ffff8800aae90000
RIP: 0010:[<ffffffff8104a3c2>] [<ffffffff8104a3c2>] __do_softirq+0x72/0x1d0
RSP: 0000:ffff88014f403f38 EFLAGS: 00000206
RAX: ffff8800aae94000 RBX: 0000000000000000 RCX: 00000000000006e0
RDX: 0000000000000020 RSI: 0000000004208060 RDI: 0000000000215d80
RBP: ffff88014f403f80 R08: 0000000b1b42c180 R09: 0000000000000022
R10: 0000000000000004 R11: 00000000ffffffff R12: 000000000000a030
R13: 0000000000000082 R14: ffff8800aa4d0080 R15: 0000000000000082
FS: 0000000000000000(0000) GS:ffff88014f400000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa53b90c000 CR3: 0000000001a0a000 CR4: 00000000001406f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
042080601b33869f ffff8800aae94000 00000000fffc2678 ffff88010000000a
0000000000000000 000000000000a030 0000000000005302 ffff8800aa4d0080
0000000000000206 ffff88014f403f90 ffffffff8104a716 ffff88014f403fa8
Call Trace:
<IRQ>
[<ffffffff8104a716>] irq_exit+0x86/0x90
[<ffffffff81031e7d>] smp_apic_timer_interrupt+0x3d/0x50
[<ffffffff814f3eac>] apic_timer_interrupt+0x7c/0x90
<EOI>
[<ffffffffa01c5b40>] ? gen8_write64+0x1a0/0x1a0 [i915]
[<ffffffff814f2b39>] ? _raw_spin_unlock_irqrestore+0x9/0x20
[<ffffffffa01c5c44>] gen8_write32+0x104/0x1a0 [i915]
[<ffffffff8132c6a2>] ? n_tty_receive_buf_common+0x372/0xae0
[<ffffffffa017cc9e>] gen6_set_rps_thresholds+0x1be/0x330 [i915]
[<ffffffffa017eaf0>] gen6_set_rps+0x70/0x200 [i915]
[<ffffffffa0185375>] intel_set_rps+0x25/0x30 [i915]
[<ffffffffa01768fd>] gen6_pm_rps_work+0x10d/0x2e0 [i915]
[<ffffffff81063852>] ? finish_task_switch+0x72/0x1c0
[<ffffffff8105ab29>] process_one_work+0x139/0x350
[<ffffffff8105b186>] worker_thread+0x126/0x490
[<ffffffff8105b060>] ? rescuer_thread+0x320/0x320
[<ffffffff8105fa64>] kthread+0xc4/0xe0
[<ffffffff8105f9a0>] ? kthread_create_on_node+0x170/0x170
[<ffffffff814f351f>] ret_from_fork+0x3f/0x70
[<ffffffff8105f9a0>] ? kthread_create_on_node+0x170/0x170
I could not explain, or find a code path, which would explain
a +20 second lockup, but from some instrumentation it was
apparent the interrupts off proportion of time was between
10-25% under heavy load which is quite bad.
When a interrupt "cliff" is reached, which was >~320k irq/s on
my machine, the whole system goes into a terrible state of the
above described multi-second lockups.
By moving the GT interrupt handling to a tasklet in a most
simple way, the problem above disappears completely.
Testing the effect on sytem-wide latencies using
igt/gem_syslatency shows the following before this patch:
gem_syslatency: cycles=1532739, latency mean=416531.829us max=2499237us
gem_syslatency: cycles=1839434, latency mean=1458099.157us max=4998944us
gem_syslatency: cycles=1432570, latency mean=2688.451us max=1201185us
gem_syslatency: cycles=1533543, latency mean=416520.499us max=2498886us
This shows that the unrelated process is experiencing huge
delays in its wake-up latency. After the patch the results
look like this:
gem_syslatency: cycles=808907, latency mean=53.133us max=1640us
gem_syslatency: cycles=862154, latency mean=62.778us max=2117us
gem_syslatency: cycles=856039, latency mean=58.079us max=2123us
gem_syslatency: cycles=841683, latency mean=56.914us max=1667us
Showing a huge improvement in the unrelated process wake-up
latency. It also shows an approximate halving in the number
of total empty batches submitted during the test. This may
not be worrying since the test puts the driver under
a very unrealistic load with ncpu threads doing empty batch
submission to all GPU engines each.
Another benefit compared to the hard-irq handling is that now
work on all engines can be dispatched in parallel since we can
have up to number of CPUs active tasklets. (While previously
a single hard-irq would serially dispatch on one engine after
another.)
More interesting scenario with regards to throughput is
"gem_latency -n 100" which shows 25% better throughput and
CPU usage, and 14% better dispatch latencies.
I did not find any gains or regressions with Synmark2 or
GLbench under light testing. More benchmarking is certainly
required.
v2:
* execlists_lock should be taken as spin_lock_bh when
queuing work from userspace now. (Chris Wilson)
* uncore.lock must be taken with spin_lock_irq when
submitting requests since that now runs from either
softirq or process context.
v3:
* Expanded commit message with more testing data;
* converted missed locking sites to _bh;
* added execlist_lock comment. (Chris Wilson)
v4:
* Mention dispatch parallelism in commit. (Chris Wilson)
* Do not hold uncore.lock over MMIO reads since the block
is already serialised per-engine via the tasklet itself.
(Chris Wilson)
* intel_lrc_irq_handler should be static. (Chris Wilson)
* Cancel/sync the tasklet on GPU reset. (Chris Wilson)
* Document and WARN that tasklet cannot be active/pending
on engine cleanup. (Chris Wilson/Imre Deak)
Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Imre Deak <imre.deak@intel.com>
Testcase: igt/gem_exec_nop/all
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=94350
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: http://patchwork.freedesktop.org/patch/msgid/1459768316-6670-1-git-send-email-tvrtko.ursulin@linux.intel.com
2016-04-04 19:11:56 +08:00
|
|
|
spin_lock_bh(&engine->execlist_lock);
|
2015-10-24 01:02:37 +08:00
|
|
|
/* list_splice_tail_init checks for empty lists */
|
2016-03-16 19:00:37 +08:00
|
|
|
list_splice_tail_init(&engine->execlist_queue,
|
|
|
|
&engine->execlist_retired_req_list);
|
drm/i915: Move execlists irq handler to a bottom half
Doing a lot of work in the interrupt handler introduces huge
latencies to the system as a whole.
Most dramatic effect can be seen by running an all engine
stress test like igt/gem_exec_nop/all where, when the kernel
config is lean enough, the whole system can be brought into
multi-second periods of complete non-interactivty. That can
look for example like this:
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [kworker/u8:3:143]
Modules linked in: [redacted for brevity]
CPU: 0 PID: 143 Comm: kworker/u8:3 Tainted: G U L 4.5.0-160321+ #183
Hardware name: Intel Corporation Broadwell Client platform/WhiteTip Mountain 1
Workqueue: i915 gen6_pm_rps_work [i915]
task: ffff8800aae88000 ti: ffff8800aae90000 task.ti: ffff8800aae90000
RIP: 0010:[<ffffffff8104a3c2>] [<ffffffff8104a3c2>] __do_softirq+0x72/0x1d0
RSP: 0000:ffff88014f403f38 EFLAGS: 00000206
RAX: ffff8800aae94000 RBX: 0000000000000000 RCX: 00000000000006e0
RDX: 0000000000000020 RSI: 0000000004208060 RDI: 0000000000215d80
RBP: ffff88014f403f80 R08: 0000000b1b42c180 R09: 0000000000000022
R10: 0000000000000004 R11: 00000000ffffffff R12: 000000000000a030
R13: 0000000000000082 R14: ffff8800aa4d0080 R15: 0000000000000082
FS: 0000000000000000(0000) GS:ffff88014f400000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa53b90c000 CR3: 0000000001a0a000 CR4: 00000000001406f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
042080601b33869f ffff8800aae94000 00000000fffc2678 ffff88010000000a
0000000000000000 000000000000a030 0000000000005302 ffff8800aa4d0080
0000000000000206 ffff88014f403f90 ffffffff8104a716 ffff88014f403fa8
Call Trace:
<IRQ>
[<ffffffff8104a716>] irq_exit+0x86/0x90
[<ffffffff81031e7d>] smp_apic_timer_interrupt+0x3d/0x50
[<ffffffff814f3eac>] apic_timer_interrupt+0x7c/0x90
<EOI>
[<ffffffffa01c5b40>] ? gen8_write64+0x1a0/0x1a0 [i915]
[<ffffffff814f2b39>] ? _raw_spin_unlock_irqrestore+0x9/0x20
[<ffffffffa01c5c44>] gen8_write32+0x104/0x1a0 [i915]
[<ffffffff8132c6a2>] ? n_tty_receive_buf_common+0x372/0xae0
[<ffffffffa017cc9e>] gen6_set_rps_thresholds+0x1be/0x330 [i915]
[<ffffffffa017eaf0>] gen6_set_rps+0x70/0x200 [i915]
[<ffffffffa0185375>] intel_set_rps+0x25/0x30 [i915]
[<ffffffffa01768fd>] gen6_pm_rps_work+0x10d/0x2e0 [i915]
[<ffffffff81063852>] ? finish_task_switch+0x72/0x1c0
[<ffffffff8105ab29>] process_one_work+0x139/0x350
[<ffffffff8105b186>] worker_thread+0x126/0x490
[<ffffffff8105b060>] ? rescuer_thread+0x320/0x320
[<ffffffff8105fa64>] kthread+0xc4/0xe0
[<ffffffff8105f9a0>] ? kthread_create_on_node+0x170/0x170
[<ffffffff814f351f>] ret_from_fork+0x3f/0x70
[<ffffffff8105f9a0>] ? kthread_create_on_node+0x170/0x170
I could not explain, or find a code path, which would explain
a +20 second lockup, but from some instrumentation it was
apparent the interrupts off proportion of time was between
10-25% under heavy load which is quite bad.
When a interrupt "cliff" is reached, which was >~320k irq/s on
my machine, the whole system goes into a terrible state of the
above described multi-second lockups.
By moving the GT interrupt handling to a tasklet in a most
simple way, the problem above disappears completely.
Testing the effect on sytem-wide latencies using
igt/gem_syslatency shows the following before this patch:
gem_syslatency: cycles=1532739, latency mean=416531.829us max=2499237us
gem_syslatency: cycles=1839434, latency mean=1458099.157us max=4998944us
gem_syslatency: cycles=1432570, latency mean=2688.451us max=1201185us
gem_syslatency: cycles=1533543, latency mean=416520.499us max=2498886us
This shows that the unrelated process is experiencing huge
delays in its wake-up latency. After the patch the results
look like this:
gem_syslatency: cycles=808907, latency mean=53.133us max=1640us
gem_syslatency: cycles=862154, latency mean=62.778us max=2117us
gem_syslatency: cycles=856039, latency mean=58.079us max=2123us
gem_syslatency: cycles=841683, latency mean=56.914us max=1667us
Showing a huge improvement in the unrelated process wake-up
latency. It also shows an approximate halving in the number
of total empty batches submitted during the test. This may
not be worrying since the test puts the driver under
a very unrealistic load with ncpu threads doing empty batch
submission to all GPU engines each.
Another benefit compared to the hard-irq handling is that now
work on all engines can be dispatched in parallel since we can
have up to number of CPUs active tasklets. (While previously
a single hard-irq would serially dispatch on one engine after
another.)
More interesting scenario with regards to throughput is
"gem_latency -n 100" which shows 25% better throughput and
CPU usage, and 14% better dispatch latencies.
I did not find any gains or regressions with Synmark2 or
GLbench under light testing. More benchmarking is certainly
required.
v2:
* execlists_lock should be taken as spin_lock_bh when
queuing work from userspace now. (Chris Wilson)
* uncore.lock must be taken with spin_lock_irq when
submitting requests since that now runs from either
softirq or process context.
v3:
* Expanded commit message with more testing data;
* converted missed locking sites to _bh;
* added execlist_lock comment. (Chris Wilson)
v4:
* Mention dispatch parallelism in commit. (Chris Wilson)
* Do not hold uncore.lock over MMIO reads since the block
is already serialised per-engine via the tasklet itself.
(Chris Wilson)
* intel_lrc_irq_handler should be static. (Chris Wilson)
* Cancel/sync the tasklet on GPU reset. (Chris Wilson)
* Document and WARN that tasklet cannot be active/pending
on engine cleanup. (Chris Wilson/Imre Deak)
Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Imre Deak <imre.deak@intel.com>
Testcase: igt/gem_exec_nop/all
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=94350
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: http://patchwork.freedesktop.org/patch/msgid/1459768316-6670-1-git-send-email-tvrtko.ursulin@linux.intel.com
2016-04-04 19:11:56 +08:00
|
|
|
spin_unlock_bh(&engine->execlist_lock);
|
2015-01-13 17:32:24 +08:00
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
intel_execlists_retire_requests(engine);
|
drm/i915/bdw: Pin the context backing objects to GGTT on-demand
Up until now, we have pinned every logical ring context backing object
during creation, and left it pinned until destruction. This made my life
easier, but it's a harmful thing to do, because we cause fragmentation
of the GGTT (and, eventually, we would run out of space).
This patch makes the pinning on-demand: the backing objects of the two
contexts that are written to the ELSP are pinned right before submission
and unpinned once the hardware is done with them. The only context that
is still pinned regardless is the global default one, so that the HWS can
still be accessed in the same way (ring->status_page).
v2: In the early version of this patch, we were pinning the context as
we put it into the ELSP: on the one hand, this is very efficient because
only a maximum two contexts are pinned at any given time, but on the other
hand, we cannot really pin in interrupt time :(
v3: Use a mutex rather than atomic_t to protect pin count to avoid races.
Do not unpin default context in free_request.
v4: Break out pin and unpin into functions. Fix style problems reported
by checkpatch
v5: Remove unpin_lock as all pinning and unpinning is done with the struct
mutex already locked. Add WARN_ONs to make sure this is the case in future.
Issue: VIZ-4277
Signed-off-by: Oscar Mateo <oscar.mateo@intel.com>
Signed-off-by: Thomas Daniel <thomas.daniel@intel.com>
Reviewed-by: Akash Goel <akash.goels@gmail.com>
Reviewed-by: Deepak S<deepak.s@linux.intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2014-11-13 18:28:10 +08:00
|
|
|
}
|
|
|
|
|
2014-01-02 02:15:13 +08:00
|
|
|
/*
|
|
|
|
* We must free the requests after all the corresponding objects have
|
|
|
|
* been moved off active lists. Which is the same order as the normal
|
|
|
|
* retire_requests function does. This is important if object hold
|
|
|
|
* implicit references on things like e.g. ppgtt address spaces through
|
|
|
|
* the request.
|
|
|
|
*/
|
2016-03-16 19:00:37 +08:00
|
|
|
while (!list_empty(&engine->request_list)) {
|
2014-01-02 02:15:13 +08:00
|
|
|
struct drm_i915_gem_request *request;
|
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
request = list_first_entry(&engine->request_list,
|
2014-01-02 02:15:13 +08:00
|
|
|
struct drm_i915_gem_request,
|
|
|
|
list);
|
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
i915_gem_request_retire(request);
|
2014-01-02 02:15:13 +08:00
|
|
|
}
|
2015-09-03 20:01:40 +08:00
|
|
|
|
|
|
|
/* Having flushed all requests from all queues, we know that all
|
|
|
|
* ringbuffers must now be empty. However, since we do not reclaim
|
|
|
|
* all space when retiring the request (to prevent HEADs colliding
|
|
|
|
* with rapid ringbuffer wraparound) the amount of available space
|
|
|
|
* upon reset is less than when we start. Do one more pass over
|
|
|
|
* all the ringbuffers to reset last_retired_head.
|
|
|
|
*/
|
2016-03-16 19:00:37 +08:00
|
|
|
list_for_each_entry(buffer, &engine->buffers, link) {
|
2015-09-03 20:01:40 +08:00
|
|
|
buffer->last_retired_head = buffer->tail;
|
|
|
|
intel_ring_update_space(buffer);
|
|
|
|
}
|
2016-04-07 14:29:11 +08:00
|
|
|
|
|
|
|
intel_ring_init_seqno(engine, engine->last_submitted_seqno);
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
2010-09-30 23:53:18 +08:00
|
|
|
void i915_gem_reset(struct drm_device *dev)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
2010-09-19 19:31:36 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2016-03-16 19:00:36 +08:00
|
|
|
struct intel_engine_cs *engine;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2013-12-04 19:37:09 +08:00
|
|
|
/*
|
|
|
|
* Before we free the objects from the requests, we need to inspect
|
|
|
|
* them for finding the guilty party. As the requests only borrow
|
|
|
|
* their reference to the objects, the inspection must be done first.
|
|
|
|
*/
|
2016-03-24 19:20:38 +08:00
|
|
|
for_each_engine(engine, dev_priv)
|
2016-03-16 19:00:39 +08:00
|
|
|
i915_gem_reset_engine_status(dev_priv, engine);
|
2013-12-04 19:37:09 +08:00
|
|
|
|
2016-03-24 19:20:38 +08:00
|
|
|
for_each_engine(engine, dev_priv)
|
2016-03-16 19:00:39 +08:00
|
|
|
i915_gem_reset_engine_cleanup(dev_priv, engine);
|
2010-09-22 17:31:52 +08:00
|
|
|
|
2013-12-07 06:11:03 +08:00
|
|
|
i915_gem_context_reset(dev);
|
|
|
|
|
2013-06-12 17:15:12 +08:00
|
|
|
i915_gem_restore_fences(dev);
|
2015-04-27 20:41:17 +08:00
|
|
|
|
|
|
|
WARN_ON(i915_verify_lists(dev));
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* This function clears the request list as sequence numbers are passed.
|
|
|
|
*/
|
2014-05-05 16:07:33 +08:00
|
|
|
void
|
2016-03-16 19:00:37 +08:00
|
|
|
i915_gem_retire_requests_ring(struct intel_engine_cs *engine)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
2016-03-16 19:00:37 +08:00
|
|
|
WARN_ON(i915_verify_lists(engine->dev));
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2015-03-19 02:19:22 +08:00
|
|
|
/* Retire requests first as we use it above for the early return.
|
|
|
|
* If we retire requests last, we may use a later seqno and so clear
|
|
|
|
* the requests lists without clearing the active list, leading to
|
|
|
|
* confusion.
|
2014-01-07 19:45:14 +08:00
|
|
|
*/
|
2016-03-16 19:00:37 +08:00
|
|
|
while (!list_empty(&engine->request_list)) {
|
2008-07-31 03:06:12 +08:00
|
|
|
struct drm_i915_gem_request *request;
|
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
request = list_first_entry(&engine->request_list,
|
2008-07-31 03:06:12 +08:00
|
|
|
struct drm_i915_gem_request,
|
|
|
|
list);
|
|
|
|
|
2014-11-25 02:49:42 +08:00
|
|
|
if (!i915_gem_request_completed(request, true))
|
2010-09-18 08:38:04 +08:00
|
|
|
break;
|
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
i915_gem_request_retire(request);
|
2010-09-18 08:38:04 +08:00
|
|
|
}
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2015-03-19 02:19:22 +08:00
|
|
|
/* Move any buffers on the active list that are no longer referenced
|
|
|
|
* by the ringbuffer to the flushing/inactive lists as appropriate,
|
|
|
|
* before we free the context associated with the requests.
|
|
|
|
*/
|
2016-03-16 19:00:37 +08:00
|
|
|
while (!list_empty(&engine->active_list)) {
|
2015-03-19 02:19:22 +08:00
|
|
|
struct drm_i915_gem_object *obj;
|
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
obj = list_first_entry(&engine->active_list,
|
|
|
|
struct drm_i915_gem_object,
|
2016-03-16 19:00:40 +08:00
|
|
|
engine_list[engine->id]);
|
2015-03-19 02:19:22 +08:00
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
if (!list_empty(&obj->last_read_req[engine->id]->list))
|
2015-03-19 02:19:22 +08:00
|
|
|
break;
|
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
i915_gem_object_retire__read(obj, engine->id);
|
2015-03-19 02:19:22 +08:00
|
|
|
}
|
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
if (unlikely(engine->trace_irq_req &&
|
|
|
|
i915_gem_request_completed(engine->trace_irq_req, true))) {
|
|
|
|
engine->irq_put(engine);
|
|
|
|
i915_gem_request_assign(&engine->trace_irq_req, NULL);
|
2009-09-24 12:26:06 +08:00
|
|
|
}
|
2010-09-29 23:10:57 +08:00
|
|
|
|
2016-03-16 19:00:37 +08:00
|
|
|
WARN_ON(i915_verify_lists(engine->dev));
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
bool
|
2010-07-24 06:18:49 +08:00
|
|
|
i915_gem_retire_requests(struct drm_device *dev)
|
|
|
|
{
|
2014-03-31 19:27:16 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2016-03-16 19:00:36 +08:00
|
|
|
struct intel_engine_cs *engine;
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
bool idle = true;
|
2010-07-24 06:18:49 +08:00
|
|
|
|
2016-03-24 19:20:38 +08:00
|
|
|
for_each_engine(engine, dev_priv) {
|
2016-03-16 19:00:36 +08:00
|
|
|
i915_gem_retire_requests_ring(engine);
|
|
|
|
idle &= list_empty(&engine->request_list);
|
2014-11-13 18:27:05 +08:00
|
|
|
if (i915.enable_execlists) {
|
drm/i915: Move execlists irq handler to a bottom half
Doing a lot of work in the interrupt handler introduces huge
latencies to the system as a whole.
Most dramatic effect can be seen by running an all engine
stress test like igt/gem_exec_nop/all where, when the kernel
config is lean enough, the whole system can be brought into
multi-second periods of complete non-interactivty. That can
look for example like this:
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [kworker/u8:3:143]
Modules linked in: [redacted for brevity]
CPU: 0 PID: 143 Comm: kworker/u8:3 Tainted: G U L 4.5.0-160321+ #183
Hardware name: Intel Corporation Broadwell Client platform/WhiteTip Mountain 1
Workqueue: i915 gen6_pm_rps_work [i915]
task: ffff8800aae88000 ti: ffff8800aae90000 task.ti: ffff8800aae90000
RIP: 0010:[<ffffffff8104a3c2>] [<ffffffff8104a3c2>] __do_softirq+0x72/0x1d0
RSP: 0000:ffff88014f403f38 EFLAGS: 00000206
RAX: ffff8800aae94000 RBX: 0000000000000000 RCX: 00000000000006e0
RDX: 0000000000000020 RSI: 0000000004208060 RDI: 0000000000215d80
RBP: ffff88014f403f80 R08: 0000000b1b42c180 R09: 0000000000000022
R10: 0000000000000004 R11: 00000000ffffffff R12: 000000000000a030
R13: 0000000000000082 R14: ffff8800aa4d0080 R15: 0000000000000082
FS: 0000000000000000(0000) GS:ffff88014f400000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa53b90c000 CR3: 0000000001a0a000 CR4: 00000000001406f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
042080601b33869f ffff8800aae94000 00000000fffc2678 ffff88010000000a
0000000000000000 000000000000a030 0000000000005302 ffff8800aa4d0080
0000000000000206 ffff88014f403f90 ffffffff8104a716 ffff88014f403fa8
Call Trace:
<IRQ>
[<ffffffff8104a716>] irq_exit+0x86/0x90
[<ffffffff81031e7d>] smp_apic_timer_interrupt+0x3d/0x50
[<ffffffff814f3eac>] apic_timer_interrupt+0x7c/0x90
<EOI>
[<ffffffffa01c5b40>] ? gen8_write64+0x1a0/0x1a0 [i915]
[<ffffffff814f2b39>] ? _raw_spin_unlock_irqrestore+0x9/0x20
[<ffffffffa01c5c44>] gen8_write32+0x104/0x1a0 [i915]
[<ffffffff8132c6a2>] ? n_tty_receive_buf_common+0x372/0xae0
[<ffffffffa017cc9e>] gen6_set_rps_thresholds+0x1be/0x330 [i915]
[<ffffffffa017eaf0>] gen6_set_rps+0x70/0x200 [i915]
[<ffffffffa0185375>] intel_set_rps+0x25/0x30 [i915]
[<ffffffffa01768fd>] gen6_pm_rps_work+0x10d/0x2e0 [i915]
[<ffffffff81063852>] ? finish_task_switch+0x72/0x1c0
[<ffffffff8105ab29>] process_one_work+0x139/0x350
[<ffffffff8105b186>] worker_thread+0x126/0x490
[<ffffffff8105b060>] ? rescuer_thread+0x320/0x320
[<ffffffff8105fa64>] kthread+0xc4/0xe0
[<ffffffff8105f9a0>] ? kthread_create_on_node+0x170/0x170
[<ffffffff814f351f>] ret_from_fork+0x3f/0x70
[<ffffffff8105f9a0>] ? kthread_create_on_node+0x170/0x170
I could not explain, or find a code path, which would explain
a +20 second lockup, but from some instrumentation it was
apparent the interrupts off proportion of time was between
10-25% under heavy load which is quite bad.
When a interrupt "cliff" is reached, which was >~320k irq/s on
my machine, the whole system goes into a terrible state of the
above described multi-second lockups.
By moving the GT interrupt handling to a tasklet in a most
simple way, the problem above disappears completely.
Testing the effect on sytem-wide latencies using
igt/gem_syslatency shows the following before this patch:
gem_syslatency: cycles=1532739, latency mean=416531.829us max=2499237us
gem_syslatency: cycles=1839434, latency mean=1458099.157us max=4998944us
gem_syslatency: cycles=1432570, latency mean=2688.451us max=1201185us
gem_syslatency: cycles=1533543, latency mean=416520.499us max=2498886us
This shows that the unrelated process is experiencing huge
delays in its wake-up latency. After the patch the results
look like this:
gem_syslatency: cycles=808907, latency mean=53.133us max=1640us
gem_syslatency: cycles=862154, latency mean=62.778us max=2117us
gem_syslatency: cycles=856039, latency mean=58.079us max=2123us
gem_syslatency: cycles=841683, latency mean=56.914us max=1667us
Showing a huge improvement in the unrelated process wake-up
latency. It also shows an approximate halving in the number
of total empty batches submitted during the test. This may
not be worrying since the test puts the driver under
a very unrealistic load with ncpu threads doing empty batch
submission to all GPU engines each.
Another benefit compared to the hard-irq handling is that now
work on all engines can be dispatched in parallel since we can
have up to number of CPUs active tasklets. (While previously
a single hard-irq would serially dispatch on one engine after
another.)
More interesting scenario with regards to throughput is
"gem_latency -n 100" which shows 25% better throughput and
CPU usage, and 14% better dispatch latencies.
I did not find any gains or regressions with Synmark2 or
GLbench under light testing. More benchmarking is certainly
required.
v2:
* execlists_lock should be taken as spin_lock_bh when
queuing work from userspace now. (Chris Wilson)
* uncore.lock must be taken with spin_lock_irq when
submitting requests since that now runs from either
softirq or process context.
v3:
* Expanded commit message with more testing data;
* converted missed locking sites to _bh;
* added execlist_lock comment. (Chris Wilson)
v4:
* Mention dispatch parallelism in commit. (Chris Wilson)
* Do not hold uncore.lock over MMIO reads since the block
is already serialised per-engine via the tasklet itself.
(Chris Wilson)
* intel_lrc_irq_handler should be static. (Chris Wilson)
* Cancel/sync the tasklet on GPU reset. (Chris Wilson)
* Document and WARN that tasklet cannot be active/pending
on engine cleanup. (Chris Wilson/Imre Deak)
Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Imre Deak <imre.deak@intel.com>
Testcase: igt/gem_exec_nop/all
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=94350
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: http://patchwork.freedesktop.org/patch/msgid/1459768316-6670-1-git-send-email-tvrtko.ursulin@linux.intel.com
2016-04-04 19:11:56 +08:00
|
|
|
spin_lock_bh(&engine->execlist_lock);
|
2016-03-16 19:00:36 +08:00
|
|
|
idle &= list_empty(&engine->execlist_queue);
|
drm/i915: Move execlists irq handler to a bottom half
Doing a lot of work in the interrupt handler introduces huge
latencies to the system as a whole.
Most dramatic effect can be seen by running an all engine
stress test like igt/gem_exec_nop/all where, when the kernel
config is lean enough, the whole system can be brought into
multi-second periods of complete non-interactivty. That can
look for example like this:
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [kworker/u8:3:143]
Modules linked in: [redacted for brevity]
CPU: 0 PID: 143 Comm: kworker/u8:3 Tainted: G U L 4.5.0-160321+ #183
Hardware name: Intel Corporation Broadwell Client platform/WhiteTip Mountain 1
Workqueue: i915 gen6_pm_rps_work [i915]
task: ffff8800aae88000 ti: ffff8800aae90000 task.ti: ffff8800aae90000
RIP: 0010:[<ffffffff8104a3c2>] [<ffffffff8104a3c2>] __do_softirq+0x72/0x1d0
RSP: 0000:ffff88014f403f38 EFLAGS: 00000206
RAX: ffff8800aae94000 RBX: 0000000000000000 RCX: 00000000000006e0
RDX: 0000000000000020 RSI: 0000000004208060 RDI: 0000000000215d80
RBP: ffff88014f403f80 R08: 0000000b1b42c180 R09: 0000000000000022
R10: 0000000000000004 R11: 00000000ffffffff R12: 000000000000a030
R13: 0000000000000082 R14: ffff8800aa4d0080 R15: 0000000000000082
FS: 0000000000000000(0000) GS:ffff88014f400000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa53b90c000 CR3: 0000000001a0a000 CR4: 00000000001406f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
042080601b33869f ffff8800aae94000 00000000fffc2678 ffff88010000000a
0000000000000000 000000000000a030 0000000000005302 ffff8800aa4d0080
0000000000000206 ffff88014f403f90 ffffffff8104a716 ffff88014f403fa8
Call Trace:
<IRQ>
[<ffffffff8104a716>] irq_exit+0x86/0x90
[<ffffffff81031e7d>] smp_apic_timer_interrupt+0x3d/0x50
[<ffffffff814f3eac>] apic_timer_interrupt+0x7c/0x90
<EOI>
[<ffffffffa01c5b40>] ? gen8_write64+0x1a0/0x1a0 [i915]
[<ffffffff814f2b39>] ? _raw_spin_unlock_irqrestore+0x9/0x20
[<ffffffffa01c5c44>] gen8_write32+0x104/0x1a0 [i915]
[<ffffffff8132c6a2>] ? n_tty_receive_buf_common+0x372/0xae0
[<ffffffffa017cc9e>] gen6_set_rps_thresholds+0x1be/0x330 [i915]
[<ffffffffa017eaf0>] gen6_set_rps+0x70/0x200 [i915]
[<ffffffffa0185375>] intel_set_rps+0x25/0x30 [i915]
[<ffffffffa01768fd>] gen6_pm_rps_work+0x10d/0x2e0 [i915]
[<ffffffff81063852>] ? finish_task_switch+0x72/0x1c0
[<ffffffff8105ab29>] process_one_work+0x139/0x350
[<ffffffff8105b186>] worker_thread+0x126/0x490
[<ffffffff8105b060>] ? rescuer_thread+0x320/0x320
[<ffffffff8105fa64>] kthread+0xc4/0xe0
[<ffffffff8105f9a0>] ? kthread_create_on_node+0x170/0x170
[<ffffffff814f351f>] ret_from_fork+0x3f/0x70
[<ffffffff8105f9a0>] ? kthread_create_on_node+0x170/0x170
I could not explain, or find a code path, which would explain
a +20 second lockup, but from some instrumentation it was
apparent the interrupts off proportion of time was between
10-25% under heavy load which is quite bad.
When a interrupt "cliff" is reached, which was >~320k irq/s on
my machine, the whole system goes into a terrible state of the
above described multi-second lockups.
By moving the GT interrupt handling to a tasklet in a most
simple way, the problem above disappears completely.
Testing the effect on sytem-wide latencies using
igt/gem_syslatency shows the following before this patch:
gem_syslatency: cycles=1532739, latency mean=416531.829us max=2499237us
gem_syslatency: cycles=1839434, latency mean=1458099.157us max=4998944us
gem_syslatency: cycles=1432570, latency mean=2688.451us max=1201185us
gem_syslatency: cycles=1533543, latency mean=416520.499us max=2498886us
This shows that the unrelated process is experiencing huge
delays in its wake-up latency. After the patch the results
look like this:
gem_syslatency: cycles=808907, latency mean=53.133us max=1640us
gem_syslatency: cycles=862154, latency mean=62.778us max=2117us
gem_syslatency: cycles=856039, latency mean=58.079us max=2123us
gem_syslatency: cycles=841683, latency mean=56.914us max=1667us
Showing a huge improvement in the unrelated process wake-up
latency. It also shows an approximate halving in the number
of total empty batches submitted during the test. This may
not be worrying since the test puts the driver under
a very unrealistic load with ncpu threads doing empty batch
submission to all GPU engines each.
Another benefit compared to the hard-irq handling is that now
work on all engines can be dispatched in parallel since we can
have up to number of CPUs active tasklets. (While previously
a single hard-irq would serially dispatch on one engine after
another.)
More interesting scenario with regards to throughput is
"gem_latency -n 100" which shows 25% better throughput and
CPU usage, and 14% better dispatch latencies.
I did not find any gains or regressions with Synmark2 or
GLbench under light testing. More benchmarking is certainly
required.
v2:
* execlists_lock should be taken as spin_lock_bh when
queuing work from userspace now. (Chris Wilson)
* uncore.lock must be taken with spin_lock_irq when
submitting requests since that now runs from either
softirq or process context.
v3:
* Expanded commit message with more testing data;
* converted missed locking sites to _bh;
* added execlist_lock comment. (Chris Wilson)
v4:
* Mention dispatch parallelism in commit. (Chris Wilson)
* Do not hold uncore.lock over MMIO reads since the block
is already serialised per-engine via the tasklet itself.
(Chris Wilson)
* intel_lrc_irq_handler should be static. (Chris Wilson)
* Cancel/sync the tasklet on GPU reset. (Chris Wilson)
* Document and WARN that tasklet cannot be active/pending
on engine cleanup. (Chris Wilson/Imre Deak)
Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Imre Deak <imre.deak@intel.com>
Testcase: igt/gem_exec_nop/all
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=94350
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: http://patchwork.freedesktop.org/patch/msgid/1459768316-6670-1-git-send-email-tvrtko.ursulin@linux.intel.com
2016-04-04 19:11:56 +08:00
|
|
|
spin_unlock_bh(&engine->execlist_lock);
|
2014-11-13 18:27:05 +08:00
|
|
|
|
2016-03-16 19:00:36 +08:00
|
|
|
intel_execlists_retire_requests(engine);
|
2014-11-13 18:27:05 +08:00
|
|
|
}
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
if (idle)
|
|
|
|
mod_delayed_work(dev_priv->wq,
|
|
|
|
&dev_priv->mm.idle_work,
|
|
|
|
msecs_to_jiffies(100));
|
|
|
|
|
|
|
|
return idle;
|
2010-07-24 06:18:49 +08:00
|
|
|
}
|
|
|
|
|
2010-08-21 06:25:16 +08:00
|
|
|
static void
|
2008-07-31 03:06:12 +08:00
|
|
|
i915_gem_retire_work_handler(struct work_struct *work)
|
|
|
|
{
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
struct drm_i915_private *dev_priv =
|
|
|
|
container_of(work, typeof(*dev_priv), mm.retire_work.work);
|
|
|
|
struct drm_device *dev = dev_priv->dev;
|
2011-01-10 05:05:44 +08:00
|
|
|
bool idle;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2010-09-29 19:26:37 +08:00
|
|
|
/* Come back later if the device is busy... */
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
idle = false;
|
|
|
|
if (mutex_trylock(&dev->struct_mutex)) {
|
|
|
|
idle = i915_gem_retire_requests(dev);
|
|
|
|
mutex_unlock(&dev->struct_mutex);
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
if (!idle)
|
2012-10-06 00:02:57 +08:00
|
|
|
queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
|
|
|
|
round_jiffies_up_relative(HZ));
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
}
|
2011-01-10 05:05:44 +08:00
|
|
|
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
static void
|
|
|
|
i915_gem_idle_work_handler(struct work_struct *work)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv =
|
|
|
|
container_of(work, typeof(*dev_priv), mm.idle_work.work);
|
2015-04-07 23:20:37 +08:00
|
|
|
struct drm_device *dev = dev_priv->dev;
|
2016-03-24 19:20:38 +08:00
|
|
|
struct intel_engine_cs *engine;
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
|
2016-03-24 19:20:38 +08:00
|
|
|
for_each_engine(engine, dev_priv)
|
|
|
|
if (!list_empty(&engine->request_list))
|
2015-04-07 23:21:08 +08:00
|
|
|
return;
|
2015-04-07 23:20:37 +08:00
|
|
|
|
2015-12-09 16:29:36 +08:00
|
|
|
/* we probably should sync with hangcheck here, using cancel_work_sync.
|
2016-03-24 19:20:38 +08:00
|
|
|
* Also locking seems to be fubar here, engine->request_list is protected
|
2015-12-09 16:29:36 +08:00
|
|
|
* by dev->struct_mutex. */
|
|
|
|
|
2015-04-07 23:20:37 +08:00
|
|
|
intel_mark_idle(dev);
|
|
|
|
|
|
|
|
if (mutex_trylock(&dev->struct_mutex)) {
|
2016-03-24 19:20:38 +08:00
|
|
|
for_each_engine(engine, dev_priv)
|
2016-03-16 19:00:36 +08:00
|
|
|
i915_gem_batch_pool_fini(&engine->batch_pool);
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
|
2015-04-07 23:20:37 +08:00
|
|
|
mutex_unlock(&dev->struct_mutex);
|
|
|
|
}
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
2012-06-01 21:21:23 +08:00
|
|
|
/**
|
|
|
|
* Ensures that an object will eventually get non-busy by flushing any required
|
|
|
|
* write domains, emitting any outstanding lazy request and retiring and
|
|
|
|
* completed requests.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
|
|
|
|
{
|
2015-05-30 00:44:15 +08:00
|
|
|
int i;
|
2015-04-27 20:41:17 +08:00
|
|
|
|
|
|
|
if (!obj->active)
|
|
|
|
return 0;
|
2012-06-01 21:21:23 +08:00
|
|
|
|
2016-03-16 19:00:39 +08:00
|
|
|
for (i = 0; i < I915_NUM_ENGINES; i++) {
|
2015-04-27 20:41:17 +08:00
|
|
|
struct drm_i915_gem_request *req;
|
2014-11-25 02:49:43 +08:00
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
req = obj->last_read_req[i];
|
|
|
|
if (req == NULL)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (list_empty(&req->list))
|
|
|
|
goto retire;
|
|
|
|
|
|
|
|
if (i915_gem_request_completed(req, true)) {
|
|
|
|
__i915_gem_request_retire__upto(req);
|
|
|
|
retire:
|
|
|
|
i915_gem_object_retire__read(obj, i);
|
|
|
|
}
|
2012-06-01 21:21:23 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2012-05-25 06:03:10 +08:00
|
|
|
/**
|
|
|
|
* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
|
|
|
|
* @DRM_IOCTL_ARGS: standard ioctl arguments
|
|
|
|
*
|
|
|
|
* Returns 0 if successful, else an error is returned with the remaining time in
|
|
|
|
* the timeout parameter.
|
|
|
|
* -ETIME: object is still busy after timeout
|
|
|
|
* -ERESTARTSYS: signal interrupted the wait
|
|
|
|
* -ENONENT: object doesn't exist
|
|
|
|
* Also possible, but rare:
|
|
|
|
* -EAGAIN: GPU wedged
|
|
|
|
* -ENOMEM: damn
|
|
|
|
* -ENODEV: Internal IRQ fail
|
|
|
|
* -E?: The add request failed
|
|
|
|
*
|
|
|
|
* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
|
|
|
|
* non-zero timeout parameter the wait ioctl will wait for the given number of
|
|
|
|
* nanoseconds on an object becoming unbusy. Since the wait itself does so
|
|
|
|
* without holding struct_mutex the object may become re-busied before this
|
|
|
|
* function completes. A similar but shorter * race condition exists in the busy
|
|
|
|
* ioctl
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
|
|
|
|
{
|
2014-03-31 19:27:16 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2012-05-25 06:03:10 +08:00
|
|
|
struct drm_i915_gem_wait *args = data;
|
|
|
|
struct drm_i915_gem_object *obj;
|
2016-03-16 19:00:39 +08:00
|
|
|
struct drm_i915_gem_request *req[I915_NUM_ENGINES];
|
drm/i915: create a race-free reset detection
With the previous patch the state transition handling of the reset
code itself is now (hopefully) race free and solid. But that still
leaves out everyone else - with the various lock-free wait paths
we have there's the possibility that the reset happens between the
point where we read the seqno we should wait on and the actual wait.
And if __wait_seqno then never sees the RESET_IN_PROGRESS state, we'll
happily wait for a seqno which will in all likelyhood never signal.
In practice this is not a big problem since the X server gets
constantly interrupted, and can then submit more work (hopefully) to
unblock everyone else: As soon as a new seqno write lands, all waiters
will unblock. But running the i-g-t reset testcase ZZ_hangman can
expose this race, especially on slower hw with fewer cpu cores.
Now looking forward to ARB_robustness and friends that's not the best
possible behaviour, hence this patch adds a reset_counter to be able
to detect any reset, even if a given thread never observed the
in-progress state.
The important part is to correctly order things:
- The write side needs to increment the counter after any seqno gets
reset. Hence we need to do that at the end of the reset work, and
again wake everyone up. We also need to place a barrier in between
any possible seqno changes and the counter increment, since any
unlock operations only guarantee that nothing leaks out, but not
that at later load operation gets moved ahead.
- On the read side we need to ensure that no reset can sneak in and
invalidate the seqno. In all cases we can use the one-sided barrier
that unlock operations guarantee (of the lock protecting the
respective seqno/ring pair) to ensure correct ordering. Hence it is
sufficient to place the atomic read before the mutex/spin_unlock and
no additional barriers are required.
The end-result of all this is that we need to wake up everyone twice
in a reset operation:
- First, before the reset starts, to get any lockholders of the locks,
so that the reset can proceed.
- Second, after the reset is completed, to allow waiters to properly
and reliably detect the reset condition and bail out.
I admit that this entire reset_counter thing smells a bit like
overkill, but I think it's justified since it makes it really explicit
what the bail-out condition is. And we need a reset counter anyway to
implement ARB_robustness, and imo with finer-grained locking on the
horizont this is the most resilient scheme I could think of.
v2: Drop spurious change in the wait_for_error EXIT_COND - we only
need to wait until we leave the reset-in-progress wedged state.
v3: Don't play tricks with barriers in the throttle ioctl, the
spin_unlock is barrier enough.
I've also considered using a little helper to grab the current
reset_counter, but then decided that hiding the atomic_read isn't a
great idea, since having it explicitly show up in the code is a nice
remainder to reviews to check the memory barriers.
v4: Add a comment to explain why we need to fall through in
__wait_seqno in the end variable assignments.
v5: Review from Damien:
- s/smb/smp/ in a comment
- don't increment the reset counter after we've set it to WEDGED. Now
we (again) properly wedge the gpu when the reset fails.
Reviewed-by: Damien Lespiau <damien.lespiau@intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-12-06 16:01:42 +08:00
|
|
|
unsigned reset_counter;
|
2015-04-27 20:41:17 +08:00
|
|
|
int i, n = 0;
|
|
|
|
int ret;
|
2012-05-25 06:03:10 +08:00
|
|
|
|
2014-09-29 21:31:26 +08:00
|
|
|
if (args->flags != 0)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2012-05-25 06:03:10 +08:00
|
|
|
ret = i915_mutex_lock_interruptible(dev);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
|
|
|
|
if (&obj->base == NULL) {
|
|
|
|
mutex_unlock(&dev->struct_mutex);
|
|
|
|
return -ENOENT;
|
|
|
|
}
|
|
|
|
|
2012-06-01 21:21:23 +08:00
|
|
|
/* Need to make sure the object gets inactive eventually. */
|
|
|
|
ret = i915_gem_object_flush_active(obj);
|
2012-05-25 06:03:10 +08:00
|
|
|
if (ret)
|
|
|
|
goto out;
|
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
if (!obj->active)
|
2014-11-25 02:49:26 +08:00
|
|
|
goto out;
|
2012-05-25 06:03:10 +08:00
|
|
|
|
|
|
|
/* Do this after OLR check to make sure we make forward progress polling
|
2015-03-05 02:09:26 +08:00
|
|
|
* on this IOCTL with a timeout == 0 (like busy ioctl)
|
2012-05-25 06:03:10 +08:00
|
|
|
*/
|
2015-03-05 02:09:26 +08:00
|
|
|
if (args->timeout_ns == 0) {
|
2012-05-25 06:03:10 +08:00
|
|
|
ret = -ETIME;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
drm_gem_object_unreference(&obj->base);
|
2016-04-14 00:35:03 +08:00
|
|
|
reset_counter = i915_reset_counter(&dev_priv->gpu_error);
|
2015-04-27 20:41:17 +08:00
|
|
|
|
2016-03-16 19:00:39 +08:00
|
|
|
for (i = 0; i < I915_NUM_ENGINES; i++) {
|
2015-04-27 20:41:17 +08:00
|
|
|
if (obj->last_read_req[i] == NULL)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
req[n++] = i915_gem_request_reference(obj->last_read_req[i]);
|
|
|
|
}
|
|
|
|
|
2012-05-25 06:03:10 +08:00
|
|
|
mutex_unlock(&dev->struct_mutex);
|
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
for (i = 0; i < n; i++) {
|
|
|
|
if (ret == 0)
|
|
|
|
ret = __i915_wait_request(req[i], reset_counter, true,
|
|
|
|
args->timeout_ns > 0 ? &args->timeout_ns : NULL,
|
2015-12-02 17:13:46 +08:00
|
|
|
to_rps_client(file));
|
2015-04-27 20:41:17 +08:00
|
|
|
i915_gem_request_unreference__unlocked(req[i]);
|
|
|
|
}
|
2014-11-25 02:49:28 +08:00
|
|
|
return ret;
|
2012-05-25 06:03:10 +08:00
|
|
|
|
|
|
|
out:
|
|
|
|
drm_gem_object_unreference(&obj->base);
|
|
|
|
mutex_unlock(&dev->struct_mutex);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
static int
|
|
|
|
__i915_gem_object_sync(struct drm_i915_gem_object *obj,
|
|
|
|
struct intel_engine_cs *to,
|
2015-06-18 20:14:56 +08:00
|
|
|
struct drm_i915_gem_request *from_req,
|
|
|
|
struct drm_i915_gem_request **to_req)
|
2015-04-27 20:41:17 +08:00
|
|
|
{
|
|
|
|
struct intel_engine_cs *from;
|
|
|
|
int ret;
|
|
|
|
|
2016-03-16 19:00:39 +08:00
|
|
|
from = i915_gem_request_get_engine(from_req);
|
2015-04-27 20:41:17 +08:00
|
|
|
if (to == from)
|
|
|
|
return 0;
|
|
|
|
|
2015-06-18 20:14:56 +08:00
|
|
|
if (i915_gem_request_completed(from_req, true))
|
2015-04-27 20:41:17 +08:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (!i915_semaphore_is_enabled(obj->base.dev)) {
|
2015-04-27 20:41:20 +08:00
|
|
|
struct drm_i915_private *i915 = to_i915(obj->base.dev);
|
2015-06-18 20:14:56 +08:00
|
|
|
ret = __i915_wait_request(from_req,
|
2016-04-14 00:35:03 +08:00
|
|
|
i915_reset_counter(&i915->gpu_error),
|
2015-04-27 20:41:20 +08:00
|
|
|
i915->mm.interruptible,
|
|
|
|
NULL,
|
|
|
|
&i915->rps.semaphores);
|
2015-04-27 20:41:17 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2015-06-18 20:14:56 +08:00
|
|
|
i915_gem_object_retire_request(obj, from_req);
|
2015-04-27 20:41:17 +08:00
|
|
|
} else {
|
|
|
|
int idx = intel_ring_sync_index(from, to);
|
2015-06-18 20:14:56 +08:00
|
|
|
u32 seqno = i915_gem_request_get_seqno(from_req);
|
|
|
|
|
|
|
|
WARN_ON(!to_req);
|
2015-04-27 20:41:17 +08:00
|
|
|
|
|
|
|
if (seqno <= from->semaphore.sync_seqno[idx])
|
|
|
|
return 0;
|
|
|
|
|
2015-06-18 20:14:56 +08:00
|
|
|
if (*to_req == NULL) {
|
drm/i915: simplify allocation of driver-internal requests
There are a number of places where the driver needs a request, but isn't
working on behalf of any specific user or in a specific context. At
present, we associate them with the per-engine default context. A future
patch will abolish those per-engine context pointers; but we can already
eliminate a lot of the references to them, just by making the allocator
allow NULL as a shorthand for "an appropriate context for this ring",
which will mean that the callers don't need to know anything about how
the "appropriate context" is found (e.g. per-ring vs per-device, etc).
So this patch renames the existing i915_gem_request_alloc(), and makes
it local (static inline), and replaces it with a wrapper that provides
a default if the context is NULL, and also has a nicer calling
convention (doesn't require a pointer to an output parameter). Then we
change all callers to use the new convention:
OLD:
err = i915_gem_request_alloc(ring, user_ctx, &req);
if (err) ...
NEW:
req = i915_gem_request_alloc(ring, user_ctx);
if (IS_ERR(req)) ...
OLD:
err = i915_gem_request_alloc(ring, ring->default_context, &req);
if (err) ...
NEW:
req = i915_gem_request_alloc(ring, NULL);
if (IS_ERR(req)) ...
v4: Rebased
Signed-off-by: Dave Gordon <david.s.gordon@intel.com>
Reviewed-by: Nick Hoath <nicholas.hoath@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1453230175-19330-2-git-send-email-david.s.gordon@intel.com
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2016-01-20 03:02:53 +08:00
|
|
|
struct drm_i915_gem_request *req;
|
|
|
|
|
|
|
|
req = i915_gem_request_alloc(to, NULL);
|
|
|
|
if (IS_ERR(req))
|
|
|
|
return PTR_ERR(req);
|
|
|
|
|
|
|
|
*to_req = req;
|
2015-06-18 20:14:56 +08:00
|
|
|
}
|
|
|
|
|
2015-05-30 00:44:04 +08:00
|
|
|
trace_i915_gem_ring_sync_to(*to_req, from, from_req);
|
|
|
|
ret = to->semaphore.sync_to(*to_req, from, seqno);
|
2015-04-27 20:41:17 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
/* We use last_read_req because sync_to()
|
|
|
|
* might have just caused seqno wrap under
|
|
|
|
* the radar.
|
|
|
|
*/
|
|
|
|
from->semaphore.sync_seqno[idx] =
|
|
|
|
i915_gem_request_get_seqno(obj->last_read_req[from->id]);
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2012-04-12 02:18:19 +08:00
|
|
|
/**
|
|
|
|
* i915_gem_object_sync - sync an object to a ring.
|
|
|
|
*
|
|
|
|
* @obj: object which may be in use on another ring.
|
|
|
|
* @to: ring we wish to use the object on. May be NULL.
|
2015-06-18 20:14:56 +08:00
|
|
|
* @to_req: request we wish to use the object for. See below.
|
|
|
|
* This will be allocated and returned if a request is
|
|
|
|
* required but not passed in.
|
2012-04-12 02:18:19 +08:00
|
|
|
*
|
|
|
|
* This code is meant to abstract object synchronization with the GPU.
|
|
|
|
* Calling with NULL implies synchronizing the object with the CPU
|
2015-04-27 20:41:17 +08:00
|
|
|
* rather than a particular GPU ring. Conceptually we serialise writes
|
2015-06-18 20:14:56 +08:00
|
|
|
* between engines inside the GPU. We only allow one engine to write
|
2015-04-27 20:41:17 +08:00
|
|
|
* into a buffer at any time, but multiple readers. To ensure each has
|
|
|
|
* a coherent view of memory, we must:
|
|
|
|
*
|
|
|
|
* - If there is an outstanding write request to the object, the new
|
|
|
|
* request must wait for it to complete (either CPU or in hw, requests
|
|
|
|
* on the same ring will be naturally ordered).
|
|
|
|
*
|
|
|
|
* - If we are a write request (pending_write_domain is set), the new
|
|
|
|
* request must wait for outstanding read requests to complete.
|
2012-04-12 02:18:19 +08:00
|
|
|
*
|
2015-06-18 20:14:56 +08:00
|
|
|
* For CPU synchronisation (NULL to) no request is required. For syncing with
|
|
|
|
* rings to_req must be non-NULL. However, a request does not have to be
|
|
|
|
* pre-allocated. If *to_req is NULL and sync commands will be emitted then a
|
|
|
|
* request will be allocated automatically and returned through *to_req. Note
|
|
|
|
* that it is not guaranteed that commands will be emitted (because the system
|
|
|
|
* might already be idle). Hence there is no need to create a request that
|
|
|
|
* might never have any work submitted. Note further that if a request is
|
|
|
|
* returned in *to_req, it is the responsibility of the caller to submit
|
|
|
|
* that request (after potentially adding more work to it).
|
|
|
|
*
|
2012-04-12 02:18:19 +08:00
|
|
|
* Returns 0 if successful, else propagates up the lower layer error.
|
|
|
|
*/
|
2012-04-06 05:47:36 +08:00
|
|
|
int
|
|
|
|
i915_gem_object_sync(struct drm_i915_gem_object *obj,
|
2015-06-18 20:14:56 +08:00
|
|
|
struct intel_engine_cs *to,
|
|
|
|
struct drm_i915_gem_request **to_req)
|
2012-04-06 05:47:36 +08:00
|
|
|
{
|
2015-04-27 20:41:17 +08:00
|
|
|
const bool readonly = obj->base.pending_write_domain == 0;
|
2016-03-16 19:00:39 +08:00
|
|
|
struct drm_i915_gem_request *req[I915_NUM_ENGINES];
|
2015-04-27 20:41:17 +08:00
|
|
|
int ret, i, n;
|
2014-11-25 02:49:43 +08:00
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
if (!obj->active)
|
2012-04-06 05:47:36 +08:00
|
|
|
return 0;
|
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
if (to == NULL)
|
|
|
|
return i915_gem_object_wait_rendering(obj, readonly);
|
2012-04-06 05:47:36 +08:00
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
n = 0;
|
|
|
|
if (readonly) {
|
|
|
|
if (obj->last_write_req)
|
|
|
|
req[n++] = obj->last_write_req;
|
|
|
|
} else {
|
2016-03-16 19:00:39 +08:00
|
|
|
for (i = 0; i < I915_NUM_ENGINES; i++)
|
2015-04-27 20:41:17 +08:00
|
|
|
if (obj->last_read_req[i])
|
|
|
|
req[n++] = obj->last_read_req[i];
|
|
|
|
}
|
|
|
|
for (i = 0; i < n; i++) {
|
2015-06-18 20:14:56 +08:00
|
|
|
ret = __i915_gem_object_sync(obj, to, req[i], to_req);
|
2015-04-27 20:41:17 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
}
|
2012-04-06 05:47:36 +08:00
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
return 0;
|
2012-04-06 05:47:36 +08:00
|
|
|
}
|
|
|
|
|
2011-04-14 05:06:03 +08:00
|
|
|
static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
|
|
|
|
{
|
|
|
|
u32 old_write_domain, old_read_domains;
|
|
|
|
|
|
|
|
/* Force a pagefault for domain tracking on next user access */
|
|
|
|
i915_gem_release_mmap(obj);
|
|
|
|
|
2011-06-25 12:02:59 +08:00
|
|
|
if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
|
|
|
|
return;
|
|
|
|
|
2012-10-10 02:24:38 +08:00
|
|
|
/* Wait for any direct GTT access to complete */
|
|
|
|
mb();
|
|
|
|
|
2011-04-14 05:06:03 +08:00
|
|
|
old_read_domains = obj->base.read_domains;
|
|
|
|
old_write_domain = obj->base.write_domain;
|
|
|
|
|
|
|
|
obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
|
|
|
|
obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
|
|
|
|
|
|
|
|
trace_i915_gem_object_change_domain(obj,
|
|
|
|
old_read_domains,
|
|
|
|
old_write_domain);
|
|
|
|
}
|
|
|
|
|
2015-10-05 20:26:36 +08:00
|
|
|
static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
drm/i915: plumb VM into bind/unbind code
As alluded to in several patches, and it will be reiterated later... A
VMA is an abstraction for a GEM BO bound into an address space.
Therefore it stands to reason, that the existing bind, and unbind are
the ones which will be the most impacted. This patch implements this,
and updates all callers which weren't already updated in the series
(because it was too messy).
This patch represents the bulk of an earlier, larger patch. I've pulled
out a bunch of things by the request of Daniel. The history is preserved
for posterity with the email convention of ">" One big change from the
original patch aside from a bunch of cropping is I've created an
i915_vma_unbind() function. That is because we always have the VMA
anyway, and doing an extra lookup is useful. There is a caveat, we
retain an i915_gem_object_ggtt_unbind, for the global cases which might
not talk in VMAs.
> drm/i915: plumb VM into object operations
>
> This patch was formerly known as:
> "drm/i915: Create VMAs (part 3) - plumbing"
>
> This patch adds a VM argument, bind/unbind, and the object
> offset/size/color getters/setters. It preserves the old ggtt helper
> functions because things still need, and will continue to need them.
>
> Some code will still need to be ported over after this.
>
> v2: Fix purge to pick an object and unbind all vmas
> This was doable because of the global bound list change.
>
> v3: With the commit to actually pin/unpin pages in place, there is no
> longer a need to check if unbind succeeded before calling put_pages().
> Make put_pages only BUG() after checking pin count.
>
> v4: Rebased on top of the new hangcheck work by Mika
> plumbed eb_destroy also
> Many checkpatch related fixes
>
> v5: Very large rebase
>
> v6:
> Change BUG_ON to WARN_ON (Daniel)
> Rename vm to ggtt in preallocate stolen, since it is always ggtt when
> dealing with stolen memory. (Daniel)
> list_for_each will short-circuit already (Daniel)
> remove superflous space (Daniel)
> Use per object list of vmas (Daniel)
> Make obj_bound_any() use obj_bound for each vm (Ben)
> s/bind_to_gtt/bind_to_vm/ (Ben)
>
> Fixed up the inactive shrinker. As Daniel noticed the code could
> potentially count the same object multiple times. While it's not
> possible in the current case, since 1 object can only ever be bound into
> 1 address space thus far - we may as well try to get something more
> future proof in place now. With a prep patch before this to switch over
> to using the bound list + inactive check, we're now able to carry that
> forward for every address space an object is bound into.
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
[danvet: Rebase on top of the loss of "drm/i915: Cleanup more of VMA
in destroy".]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-08-01 08:00:10 +08:00
|
|
|
struct drm_i915_gem_object *obj = vma->obj;
|
2014-03-31 19:27:16 +08:00
|
|
|
struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
|
2013-01-08 18:53:09 +08:00
|
|
|
int ret;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2016-02-26 19:03:19 +08:00
|
|
|
if (list_empty(&vma->obj_link))
|
2008-07-31 03:06:12 +08:00
|
|
|
return 0;
|
|
|
|
|
2013-08-30 01:50:31 +08:00
|
|
|
if (!drm_mm_node_allocated(&vma->node)) {
|
|
|
|
i915_gem_vma_destroy(vma);
|
|
|
|
return 0;
|
|
|
|
}
|
2013-08-14 09:09:06 +08:00
|
|
|
|
2013-12-07 06:10:55 +08:00
|
|
|
if (vma->pin_count)
|
2012-05-25 02:11:20 +08:00
|
|
|
return -EBUSY;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2012-08-20 17:23:27 +08:00
|
|
|
BUG_ON(obj->pages == NULL);
|
|
|
|
|
2015-10-05 20:26:36 +08:00
|
|
|
if (wait) {
|
|
|
|
ret = i915_gem_object_wait_rendering(obj, false);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
}
|
2011-04-14 05:04:09 +08:00
|
|
|
|
2016-02-26 19:03:20 +08:00
|
|
|
if (vma->is_ggtt && vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
|
2014-02-14 21:06:07 +08:00
|
|
|
i915_gem_object_finish_gtt(obj);
|
2009-09-10 02:50:45 +08:00
|
|
|
|
2014-02-14 21:06:07 +08:00
|
|
|
/* release the fence reg _after_ flushing */
|
|
|
|
ret = i915_gem_object_put_fence(obj);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
}
|
2009-12-16 00:50:00 +08:00
|
|
|
|
drm/i915: plumb VM into bind/unbind code
As alluded to in several patches, and it will be reiterated later... A
VMA is an abstraction for a GEM BO bound into an address space.
Therefore it stands to reason, that the existing bind, and unbind are
the ones which will be the most impacted. This patch implements this,
and updates all callers which weren't already updated in the series
(because it was too messy).
This patch represents the bulk of an earlier, larger patch. I've pulled
out a bunch of things by the request of Daniel. The history is preserved
for posterity with the email convention of ">" One big change from the
original patch aside from a bunch of cropping is I've created an
i915_vma_unbind() function. That is because we always have the VMA
anyway, and doing an extra lookup is useful. There is a caveat, we
retain an i915_gem_object_ggtt_unbind, for the global cases which might
not talk in VMAs.
> drm/i915: plumb VM into object operations
>
> This patch was formerly known as:
> "drm/i915: Create VMAs (part 3) - plumbing"
>
> This patch adds a VM argument, bind/unbind, and the object
> offset/size/color getters/setters. It preserves the old ggtt helper
> functions because things still need, and will continue to need them.
>
> Some code will still need to be ported over after this.
>
> v2: Fix purge to pick an object and unbind all vmas
> This was doable because of the global bound list change.
>
> v3: With the commit to actually pin/unpin pages in place, there is no
> longer a need to check if unbind succeeded before calling put_pages().
> Make put_pages only BUG() after checking pin count.
>
> v4: Rebased on top of the new hangcheck work by Mika
> plumbed eb_destroy also
> Many checkpatch related fixes
>
> v5: Very large rebase
>
> v6:
> Change BUG_ON to WARN_ON (Daniel)
> Rename vm to ggtt in preallocate stolen, since it is always ggtt when
> dealing with stolen memory. (Daniel)
> list_for_each will short-circuit already (Daniel)
> remove superflous space (Daniel)
> Use per object list of vmas (Daniel)
> Make obj_bound_any() use obj_bound for each vm (Ben)
> s/bind_to_gtt/bind_to_vm/ (Ben)
>
> Fixed up the inactive shrinker. As Daniel noticed the code could
> potentially count the same object multiple times. While it's not
> possible in the current case, since 1 object can only ever be bound into
> 1 address space thus far - we may as well try to get something more
> future proof in place now. With a prep patch before this to switch over
> to using the bound list + inactive check, we're now able to carry that
> forward for every address space an object is bound into.
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
[danvet: Rebase on top of the loss of "drm/i915: Cleanup more of VMA
in destroy".]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-08-01 08:00:10 +08:00
|
|
|
trace_i915_vma_unbind(vma);
|
2011-02-03 19:57:46 +08:00
|
|
|
|
2015-04-14 23:35:12 +08:00
|
|
|
vma->vm->unbind_vma(vma);
|
2015-04-30 16:02:31 +08:00
|
|
|
vma->bound = 0;
|
drm/i915: Create bind/unbind abstraction for VMAs
To sum up what goes on here, we abstract the vma binding, similarly to
the previous object binding. This helps for distinguishing legacy
binding, versus modern binding. To keep the code churn as minimal as
possible, I am leaving in insert_entries(). It serves as the per
platform pte writing basically. bind_vma and insert_entries do share a
lot of similarities, and I did have designs to combine the two, but as
mentioned already... too much churn in an already massive patchset.
What follows are the 3 commits which existed discretely in the original
submissions. Upon rebasing on Broadwell support, it became clear that
separation was not good, and only made for more error prone code. Below
are the 3 commit messages with all their history.
drm/i915: Add bind/unbind object functions to VMA
drm/i915: Use the new vm [un]bind functions
drm/i915: reduce vm->insert_entries() usage
drm/i915: Add bind/unbind object functions to VMA
As we plumb the code with more VM information, it has become more
obvious that the easiest way to deal with bind and unbind is to simply
put the function pointers in the vm, and let those choose the correct
way to handle the page table updates. This change allows many places in
the code to simply be vm->bind, and not have to worry about
distinguishing PPGTT vs GGTT.
Notice that this patch has no impact on functionality. I've decided to
save the actual change until the next patch because I think it's easier
to review that way. I'm happy to squash the two, or let Daniel do it on
merge.
v2:
Make ggtt handle the quirky aliasing ppgtt
Add flags to bind object to support above
Don't ever call bind/unbind directly for PPGTT until we have real, full
PPGTT (use NULLs to assert this)
Make sure we rebind the ggtt if there already is a ggtt binding. This
happens on set cache levels.
Use VMA for bind/unbind (Daniel, Ben)
v3: Reorganize ggtt_vma_bind to be more concise and easier to read
(Ville). Change logic in unbind to only unbind ggtt when there is a
global mapping, and to remove a redundant check if the aliasing ppgtt
exists.
v4: Make the bind function a bit smarter about the cache levels to avoid
unnecessary multiple remaps. "I accept it is a wart, I think unifying
the pin_vma / bind_vma could be unified later" (Chris)
Removed the git notes, and put version info here. (Daniel)
v5: Update the comment to not suck (Chris)
v6:
Move bind/unbind to the VMA. It makes more sense in the VMA structure
(always has, but I was previously lazy). With this change, it will allow
us to keep a distinct insert_entries.
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
drm/i915: Use the new vm [un]bind functions
Building on the last patch which created the new function pointers in
the VM for bind/unbind, here we actually put those new function pointers
to use.
Split out as a separate patch to aid in review. I'm fine with squashing
into the previous patch if people request it.
v2: Updated to address the smart ggtt which can do aliasing as needed
Make sure we bind to global gtt when mappable and fenceable. I thought
we could get away without this initialy, but we cannot.
v3: Make the global GTT binding explicitly use the ggtt VM for
bind_vma(). While at it, use the new ggtt_vma helper (Chris)
At this point the original mailing list thread diverges. ie.
v4^:
use target_obj instead of obj for gen6 relocate_entry
vma->bind_vma() can be called safely during pin. So simply do that
instead of the complicated conditionals.
Don't restore PPGTT bound objects on resume path
Bug fix in resume path for globally bound Bos
Properly handle secure dispatch
Rebased on vma bind/unbind conversion
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
drm/i915: reduce vm->insert_entries() usage
FKA: drm/i915: eliminate vm->insert_entries()
With bind/unbind function pointers in place, we no longer need
insert_entries. We could, and want, to remove clear_range, however it's
not totally easy at this point. Since it's used in a couple of place
still that don't only deal in objects: setup, ppgtt init, and restore
gtt mappings.
v2: Don't actually remove insert_entries, just limit its usage. It will
be useful when we introduce gen8. It will always be called from the vma
bind/unbind.
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk> (v1)
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-12-07 06:10:56 +08:00
|
|
|
|
2016-02-26 19:03:19 +08:00
|
|
|
list_del_init(&vma->vm_link);
|
2016-02-26 19:03:20 +08:00
|
|
|
if (vma->is_ggtt) {
|
2014-12-11 01:27:58 +08:00
|
|
|
if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
|
|
|
|
obj->map_and_fenceable = false;
|
|
|
|
} else if (vma->ggtt_view.pages) {
|
|
|
|
sg_free_table(vma->ggtt_view.pages);
|
|
|
|
kfree(vma->ggtt_view.pages);
|
|
|
|
}
|
2015-06-11 15:06:08 +08:00
|
|
|
vma->ggtt_view.pages = NULL;
|
2014-12-11 01:27:58 +08:00
|
|
|
}
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2013-07-18 03:19:03 +08:00
|
|
|
drm_mm_remove_node(&vma->node);
|
|
|
|
i915_gem_vma_destroy(vma);
|
|
|
|
|
|
|
|
/* Since the unbound list is global, only move to that list if
|
2013-08-26 17:23:47 +08:00
|
|
|
* no more VMAs exist. */
|
2015-07-09 17:59:05 +08:00
|
|
|
if (list_empty(&obj->vma_list))
|
2013-07-18 03:19:03 +08:00
|
|
|
list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2013-12-04 17:59:09 +08:00
|
|
|
/* And finally now the object is completely decoupled from this vma,
|
|
|
|
* we can drop its hold on the backing storage and allow it to be
|
|
|
|
* reaped by the shrinker.
|
|
|
|
*/
|
|
|
|
i915_gem_object_unpin_pages(obj);
|
|
|
|
|
2011-01-08 01:09:48 +08:00
|
|
|
return 0;
|
2010-11-26 02:00:26 +08:00
|
|
|
}
|
|
|
|
|
2015-10-05 20:26:36 +08:00
|
|
|
int i915_vma_unbind(struct i915_vma *vma)
|
|
|
|
{
|
|
|
|
return __i915_vma_unbind(vma, true);
|
|
|
|
}
|
|
|
|
|
|
|
|
int __i915_vma_unbind_no_wait(struct i915_vma *vma)
|
|
|
|
{
|
|
|
|
return __i915_vma_unbind(vma, false);
|
|
|
|
}
|
|
|
|
|
2012-04-27 07:02:58 +08:00
|
|
|
int i915_gpu_idle(struct drm_device *dev)
|
2010-02-19 18:52:00 +08:00
|
|
|
{
|
2014-03-31 19:27:16 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2016-03-16 19:00:36 +08:00
|
|
|
struct intel_engine_cs *engine;
|
2016-03-24 19:20:38 +08:00
|
|
|
int ret;
|
2010-02-19 18:52:00 +08:00
|
|
|
|
|
|
|
/* Flush everything onto the inactive list. */
|
2016-03-24 19:20:38 +08:00
|
|
|
for_each_engine(engine, dev_priv) {
|
2014-08-20 23:29:24 +08:00
|
|
|
if (!i915.enable_execlists) {
|
2015-05-30 00:43:35 +08:00
|
|
|
struct drm_i915_gem_request *req;
|
|
|
|
|
2016-03-16 19:00:36 +08:00
|
|
|
req = i915_gem_request_alloc(engine, NULL);
|
drm/i915: simplify allocation of driver-internal requests
There are a number of places where the driver needs a request, but isn't
working on behalf of any specific user or in a specific context. At
present, we associate them with the per-engine default context. A future
patch will abolish those per-engine context pointers; but we can already
eliminate a lot of the references to them, just by making the allocator
allow NULL as a shorthand for "an appropriate context for this ring",
which will mean that the callers don't need to know anything about how
the "appropriate context" is found (e.g. per-ring vs per-device, etc).
So this patch renames the existing i915_gem_request_alloc(), and makes
it local (static inline), and replaces it with a wrapper that provides
a default if the context is NULL, and also has a nicer calling
convention (doesn't require a pointer to an output parameter). Then we
change all callers to use the new convention:
OLD:
err = i915_gem_request_alloc(ring, user_ctx, &req);
if (err) ...
NEW:
req = i915_gem_request_alloc(ring, user_ctx);
if (IS_ERR(req)) ...
OLD:
err = i915_gem_request_alloc(ring, ring->default_context, &req);
if (err) ...
NEW:
req = i915_gem_request_alloc(ring, NULL);
if (IS_ERR(req)) ...
v4: Rebased
Signed-off-by: Dave Gordon <david.s.gordon@intel.com>
Reviewed-by: Nick Hoath <nicholas.hoath@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1453230175-19330-2-git-send-email-david.s.gordon@intel.com
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2016-01-20 03:02:53 +08:00
|
|
|
if (IS_ERR(req))
|
|
|
|
return PTR_ERR(req);
|
2015-05-30 00:43:35 +08:00
|
|
|
|
2015-05-30 00:43:41 +08:00
|
|
|
ret = i915_switch_context(req);
|
2015-05-30 00:43:35 +08:00
|
|
|
if (ret) {
|
|
|
|
i915_gem_request_cancel(req);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2015-05-30 00:43:49 +08:00
|
|
|
i915_add_request_no_flush(req);
|
2014-08-20 23:29:24 +08:00
|
|
|
}
|
2012-08-15 05:35:14 +08:00
|
|
|
|
2016-03-16 19:00:39 +08:00
|
|
|
ret = intel_engine_idle(engine);
|
2010-12-04 19:30:53 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
}
|
2010-02-19 18:52:00 +08:00
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
WARN_ON(i915_verify_lists(dev));
|
2010-02-12 05:29:04 +08:00
|
|
|
return 0;
|
2010-02-19 18:52:00 +08:00
|
|
|
}
|
|
|
|
|
2014-09-11 15:43:48 +08:00
|
|
|
static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
|
2012-07-26 18:49:32 +08:00
|
|
|
unsigned long cache_level)
|
|
|
|
{
|
2014-09-11 15:43:48 +08:00
|
|
|
struct drm_mm_node *gtt_space = &vma->node;
|
2012-07-26 18:49:32 +08:00
|
|
|
struct drm_mm_node *other;
|
|
|
|
|
2014-09-11 15:43:48 +08:00
|
|
|
/*
|
|
|
|
* On some machines we have to be careful when putting differing types
|
|
|
|
* of snoopable memory together to avoid the prefetcher crossing memory
|
|
|
|
* domains and dying. During vm initialisation, we decide whether or not
|
|
|
|
* these constraints apply and set the drm_mm.color_adjust
|
|
|
|
* appropriately.
|
2012-07-26 18:49:32 +08:00
|
|
|
*/
|
2014-09-11 15:43:48 +08:00
|
|
|
if (vma->vm->mm.color_adjust == NULL)
|
2012-07-26 18:49:32 +08:00
|
|
|
return true;
|
|
|
|
|
2013-07-06 05:41:06 +08:00
|
|
|
if (!drm_mm_node_allocated(gtt_space))
|
2012-07-26 18:49:32 +08:00
|
|
|
return true;
|
|
|
|
|
|
|
|
if (list_empty(>t_space->node_list))
|
|
|
|
return true;
|
|
|
|
|
|
|
|
other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
|
|
|
|
if (other->allocated && !other->hole_follows && other->color != cache_level)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
|
|
|
|
if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2008-07-31 03:06:12 +08:00
|
|
|
/**
|
2015-05-06 19:33:58 +08:00
|
|
|
* Finds free space in the GTT aperture and binds the object or a view of it
|
|
|
|
* there.
|
2008-07-31 03:06:12 +08:00
|
|
|
*/
|
2014-02-14 21:01:20 +08:00
|
|
|
static struct i915_vma *
|
drm/i915: plumb VM into bind/unbind code
As alluded to in several patches, and it will be reiterated later... A
VMA is an abstraction for a GEM BO bound into an address space.
Therefore it stands to reason, that the existing bind, and unbind are
the ones which will be the most impacted. This patch implements this,
and updates all callers which weren't already updated in the series
(because it was too messy).
This patch represents the bulk of an earlier, larger patch. I've pulled
out a bunch of things by the request of Daniel. The history is preserved
for posterity with the email convention of ">" One big change from the
original patch aside from a bunch of cropping is I've created an
i915_vma_unbind() function. That is because we always have the VMA
anyway, and doing an extra lookup is useful. There is a caveat, we
retain an i915_gem_object_ggtt_unbind, for the global cases which might
not talk in VMAs.
> drm/i915: plumb VM into object operations
>
> This patch was formerly known as:
> "drm/i915: Create VMAs (part 3) - plumbing"
>
> This patch adds a VM argument, bind/unbind, and the object
> offset/size/color getters/setters. It preserves the old ggtt helper
> functions because things still need, and will continue to need them.
>
> Some code will still need to be ported over after this.
>
> v2: Fix purge to pick an object and unbind all vmas
> This was doable because of the global bound list change.
>
> v3: With the commit to actually pin/unpin pages in place, there is no
> longer a need to check if unbind succeeded before calling put_pages().
> Make put_pages only BUG() after checking pin count.
>
> v4: Rebased on top of the new hangcheck work by Mika
> plumbed eb_destroy also
> Many checkpatch related fixes
>
> v5: Very large rebase
>
> v6:
> Change BUG_ON to WARN_ON (Daniel)
> Rename vm to ggtt in preallocate stolen, since it is always ggtt when
> dealing with stolen memory. (Daniel)
> list_for_each will short-circuit already (Daniel)
> remove superflous space (Daniel)
> Use per object list of vmas (Daniel)
> Make obj_bound_any() use obj_bound for each vm (Ben)
> s/bind_to_gtt/bind_to_vm/ (Ben)
>
> Fixed up the inactive shrinker. As Daniel noticed the code could
> potentially count the same object multiple times. While it's not
> possible in the current case, since 1 object can only ever be bound into
> 1 address space thus far - we may as well try to get something more
> future proof in place now. With a prep patch before this to switch over
> to using the bound list + inactive check, we're now able to carry that
> forward for every address space an object is bound into.
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
[danvet: Rebase on top of the loss of "drm/i915: Cleanup more of VMA
in destroy".]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-08-01 08:00:10 +08:00
|
|
|
i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
|
|
|
|
struct i915_address_space *vm,
|
2015-03-16 20:11:13 +08:00
|
|
|
const struct i915_ggtt_view *ggtt_view,
|
drm/i915: plumb VM into bind/unbind code
As alluded to in several patches, and it will be reiterated later... A
VMA is an abstraction for a GEM BO bound into an address space.
Therefore it stands to reason, that the existing bind, and unbind are
the ones which will be the most impacted. This patch implements this,
and updates all callers which weren't already updated in the series
(because it was too messy).
This patch represents the bulk of an earlier, larger patch. I've pulled
out a bunch of things by the request of Daniel. The history is preserved
for posterity with the email convention of ">" One big change from the
original patch aside from a bunch of cropping is I've created an
i915_vma_unbind() function. That is because we always have the VMA
anyway, and doing an extra lookup is useful. There is a caveat, we
retain an i915_gem_object_ggtt_unbind, for the global cases which might
not talk in VMAs.
> drm/i915: plumb VM into object operations
>
> This patch was formerly known as:
> "drm/i915: Create VMAs (part 3) - plumbing"
>
> This patch adds a VM argument, bind/unbind, and the object
> offset/size/color getters/setters. It preserves the old ggtt helper
> functions because things still need, and will continue to need them.
>
> Some code will still need to be ported over after this.
>
> v2: Fix purge to pick an object and unbind all vmas
> This was doable because of the global bound list change.
>
> v3: With the commit to actually pin/unpin pages in place, there is no
> longer a need to check if unbind succeeded before calling put_pages().
> Make put_pages only BUG() after checking pin count.
>
> v4: Rebased on top of the new hangcheck work by Mika
> plumbed eb_destroy also
> Many checkpatch related fixes
>
> v5: Very large rebase
>
> v6:
> Change BUG_ON to WARN_ON (Daniel)
> Rename vm to ggtt in preallocate stolen, since it is always ggtt when
> dealing with stolen memory. (Daniel)
> list_for_each will short-circuit already (Daniel)
> remove superflous space (Daniel)
> Use per object list of vmas (Daniel)
> Make obj_bound_any() use obj_bound for each vm (Ben)
> s/bind_to_gtt/bind_to_vm/ (Ben)
>
> Fixed up the inactive shrinker. As Daniel noticed the code could
> potentially count the same object multiple times. While it's not
> possible in the current case, since 1 object can only ever be bound into
> 1 address space thus far - we may as well try to get something more
> future proof in place now. With a prep patch before this to switch over
> to using the bound list + inactive check, we're now able to carry that
> forward for every address space an object is bound into.
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
[danvet: Rebase on top of the loss of "drm/i915: Cleanup more of VMA
in destroy".]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-08-01 08:00:10 +08:00
|
|
|
unsigned alignment,
|
2015-03-16 20:11:13 +08:00
|
|
|
uint64_t flags)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_device *dev = obj->base.dev;
|
2016-03-30 21:57:10 +08:00
|
|
|
struct drm_i915_private *dev_priv = to_i915(dev);
|
|
|
|
struct i915_ggtt *ggtt = &dev_priv->ggtt;
|
2015-07-30 00:23:58 +08:00
|
|
|
u32 fence_alignment, unfenced_alignment;
|
2015-10-01 20:33:57 +08:00
|
|
|
u32 search_flag, alloc_flag;
|
|
|
|
u64 start, end;
|
2015-07-30 00:23:58 +08:00
|
|
|
u64 size, fence_size;
|
2013-07-18 03:19:03 +08:00
|
|
|
struct i915_vma *vma;
|
2009-09-14 23:50:30 +08:00
|
|
|
int ret;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2015-05-06 19:33:58 +08:00
|
|
|
if (i915_is_ggtt(vm)) {
|
|
|
|
u32 view_size;
|
|
|
|
|
|
|
|
if (WARN_ON(!ggtt_view))
|
|
|
|
return ERR_PTR(-EINVAL);
|
2015-03-16 20:11:13 +08:00
|
|
|
|
2015-05-06 19:33:58 +08:00
|
|
|
view_size = i915_ggtt_view_size(obj, ggtt_view);
|
|
|
|
|
|
|
|
fence_size = i915_gem_get_gtt_size(dev,
|
|
|
|
view_size,
|
|
|
|
obj->tiling_mode);
|
|
|
|
fence_alignment = i915_gem_get_gtt_alignment(dev,
|
|
|
|
view_size,
|
|
|
|
obj->tiling_mode,
|
|
|
|
true);
|
|
|
|
unfenced_alignment = i915_gem_get_gtt_alignment(dev,
|
|
|
|
view_size,
|
|
|
|
obj->tiling_mode,
|
|
|
|
false);
|
|
|
|
size = flags & PIN_MAPPABLE ? fence_size : view_size;
|
|
|
|
} else {
|
|
|
|
fence_size = i915_gem_get_gtt_size(dev,
|
|
|
|
obj->base.size,
|
|
|
|
obj->tiling_mode);
|
|
|
|
fence_alignment = i915_gem_get_gtt_alignment(dev,
|
|
|
|
obj->base.size,
|
|
|
|
obj->tiling_mode,
|
|
|
|
true);
|
|
|
|
unfenced_alignment =
|
|
|
|
i915_gem_get_gtt_alignment(dev,
|
|
|
|
obj->base.size,
|
|
|
|
obj->tiling_mode,
|
|
|
|
false);
|
|
|
|
size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
|
|
|
|
}
|
2010-09-25 04:15:47 +08:00
|
|
|
|
2015-10-01 20:33:57 +08:00
|
|
|
start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
|
|
|
|
end = vm->total;
|
|
|
|
if (flags & PIN_MAPPABLE)
|
2016-03-30 21:57:10 +08:00
|
|
|
end = min_t(u64, end, ggtt->mappable_end);
|
2015-10-01 20:33:57 +08:00
|
|
|
if (flags & PIN_ZONE_4G)
|
2016-01-11 19:39:27 +08:00
|
|
|
end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
|
2015-10-01 20:33:57 +08:00
|
|
|
|
2008-07-31 03:06:12 +08:00
|
|
|
if (alignment == 0)
|
2014-02-14 21:01:11 +08:00
|
|
|
alignment = flags & PIN_MAPPABLE ? fence_alignment :
|
2010-11-15 05:32:36 +08:00
|
|
|
unfenced_alignment;
|
2014-02-14 21:01:11 +08:00
|
|
|
if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
|
2015-05-06 19:33:58 +08:00
|
|
|
DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
|
|
|
|
ggtt_view ? ggtt_view->type : 0,
|
|
|
|
alignment);
|
2014-02-14 21:01:20 +08:00
|
|
|
return ERR_PTR(-EINVAL);
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
2015-05-06 19:33:58 +08:00
|
|
|
/* If binding the object/GGTT view requires more space than the entire
|
|
|
|
* aperture has, reject it early before evicting everything in a vain
|
|
|
|
* attempt to find space.
|
2010-05-27 20:18:21 +08:00
|
|
|
*/
|
2015-05-06 19:33:58 +08:00
|
|
|
if (size > end) {
|
2015-07-30 00:23:58 +08:00
|
|
|
DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n",
|
2015-05-06 19:33:58 +08:00
|
|
|
ggtt_view ? ggtt_view->type : 0,
|
|
|
|
size,
|
2014-02-14 21:01:11 +08:00
|
|
|
flags & PIN_MAPPABLE ? "mappable" : "total",
|
drm/i915: Prevent negative relocation deltas from wrapping
This is pure evil. Userspace, I'm looking at you SNA, repacks batch
buffers on the fly after generation as they are being passed to the
kernel for execution. These batches also contain self-referenced
relocations as a single buffer encompasses the state commands, kernels,
vertices and sampler. During generation the buffers are placed at known
offsets within the full batch, and then the relocation deltas (as passed
to the kernel) are tweaked as the batch is repacked into a smaller buffer.
This means that userspace is passing negative relocations deltas, which
subsequently wrap to large values if the batch is at a low address. The
GPU hangs when it then tries to use the large value as a base for its
address offsets, rather than wrapping back to the real value (as one
would hope). As the GPU uses positive offsets from the base, we can
treat the relocation address as the minimum address read by the GPU.
For the upper bound, we trust that userspace will not read beyond the
end of the buffer.
So, how do we fix negative relocations from wrapping? We can either
check that every relocation looks valid when we write it, and then
position each object such that we prevent the offset wraparound, or we
just special-case the self-referential behaviour of SNA and force all
batches to be above 256k. Daniel prefers the latter approach.
This fixes a GPU hang when it tries to use an address (relocation +
offset) greater than the GTT size. The issue would occur quite easily
with full-ppgtt as each fd gets its own VM space, so low offsets would
often be handed out. However, with the rearrangement of the low GTT due
to capturing the BIOS framebuffer, it is already affecting kernels 3.15
onwards. I think only IVB+ is susceptible to this bug, but the workaround
should only kick in rarely, so it seems sensible to always apply it.
v3: Use a bias for batch buffers to prevent small negative delta relocations
from wrapping.
v4 from Daniel:
- s/BIAS/BATCH_OFFSET_BIAS/
- Extract eb_vma_misplaced/i915_vma_misplaced since the conditions
were growing rather cumbersome.
- Add a comment to eb_get_batch explaining why we do this.
- Apply the batch offset bias everywhere but mention that we've only
observed it on gen7 gpus.
- Drop PIN_OFFSET_FIX for now, that slipped in from a feature patch.
v5: Add static to eb_get_batch, spotted by 0-day tester.
Testcase: igt/gem_bad_reloc
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=78533
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk> (v3)
Cc: stable@vger.kernel.org
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2014-05-23 14:48:08 +08:00
|
|
|
end);
|
2014-02-14 21:01:20 +08:00
|
|
|
return ERR_PTR(-E2BIG);
|
2010-05-27 20:18:21 +08:00
|
|
|
}
|
|
|
|
|
2012-06-07 22:38:42 +08:00
|
|
|
ret = i915_gem_object_get_pages(obj);
|
drm/i915: Track unbound pages
When dealing with a working set larger than the GATT, or even the
mappable aperture when touching through the GTT, we end up with evicting
objects only to rebind them at a new offset again later. Moving an
object into and out of the GTT requires clflushing the pages, thus
causing a double-clflush penalty for rebinding.
To avoid having to clflush on rebinding, we can track the pages as they
are evicted from the GTT and only relinquish those pages on memory
pressure.
As usual, if it were not for the handling of out-of-memory condition and
having to manually shrink our own bo caches, it would be a net reduction
of code. Alas.
Note: The patch also contains a few changes to the last-hope
evict_everything logic in i916_gem_execbuffer.c - we no longer try to
only evict the purgeable stuff in a first try (since that's superflous
and only helps in OOM corner-cases, not fragmented-gtt trashing
situations).
Also, the extraction of the get_pages retry loop from bind_to_gtt (and
other callsites) to get_pages should imo have been a separate patch.
v2: Ditch the newly added put_pages (for unbound objects only) in
i915_gem_reset. A quick irc discussion hasn't revealed any important
reason for this, so if we need this, I'd like to have a git blame'able
explanation for it.
v3: Undo the s/drm_malloc_ab/kmalloc/ in get_pages that Chris noticed.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
[danvet: Split out code movements and rant a bit in the commit message
with a few Notes. Done v2]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-08-20 17:40:46 +08:00
|
|
|
if (ret)
|
2014-02-14 21:01:20 +08:00
|
|
|
return ERR_PTR(ret);
|
drm/i915: Track unbound pages
When dealing with a working set larger than the GATT, or even the
mappable aperture when touching through the GTT, we end up with evicting
objects only to rebind them at a new offset again later. Moving an
object into and out of the GTT requires clflushing the pages, thus
causing a double-clflush penalty for rebinding.
To avoid having to clflush on rebinding, we can track the pages as they
are evicted from the GTT and only relinquish those pages on memory
pressure.
As usual, if it were not for the handling of out-of-memory condition and
having to manually shrink our own bo caches, it would be a net reduction
of code. Alas.
Note: The patch also contains a few changes to the last-hope
evict_everything logic in i916_gem_execbuffer.c - we no longer try to
only evict the purgeable stuff in a first try (since that's superflous
and only helps in OOM corner-cases, not fragmented-gtt trashing
situations).
Also, the extraction of the get_pages retry loop from bind_to_gtt (and
other callsites) to get_pages should imo have been a separate patch.
v2: Ditch the newly added put_pages (for unbound objects only) in
i915_gem_reset. A quick irc discussion hasn't revealed any important
reason for this, so if we need this, I'd like to have a git blame'able
explanation for it.
v3: Undo the s/drm_malloc_ab/kmalloc/ in get_pages that Chris noticed.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
[danvet: Split out code movements and rant a bit in the commit message
with a few Notes. Done v2]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-08-20 17:40:46 +08:00
|
|
|
|
2012-11-20 18:45:16 +08:00
|
|
|
i915_gem_object_pin_pages(obj);
|
|
|
|
|
2015-03-16 20:11:13 +08:00
|
|
|
vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :
|
|
|
|
i915_gem_obj_lookup_or_create_vma(obj, vm);
|
|
|
|
|
2014-02-14 21:01:20 +08:00
|
|
|
if (IS_ERR(vma))
|
2013-07-22 18:12:38 +08:00
|
|
|
goto err_unpin;
|
2013-07-18 03:19:03 +08:00
|
|
|
|
2015-12-08 19:55:07 +08:00
|
|
|
if (flags & PIN_OFFSET_FIXED) {
|
|
|
|
uint64_t offset = flags & PIN_OFFSET_MASK;
|
|
|
|
|
|
|
|
if (offset & (alignment - 1) || offset + size > end) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto err_free_vma;
|
|
|
|
}
|
|
|
|
vma->node.start = offset;
|
|
|
|
vma->node.size = size;
|
|
|
|
vma->node.color = obj->cache_level;
|
|
|
|
ret = drm_mm_reserve_node(&vm->mm, &vma->node);
|
|
|
|
if (ret) {
|
|
|
|
ret = i915_gem_evict_for_vma(vma);
|
|
|
|
if (ret == 0)
|
|
|
|
ret = drm_mm_reserve_node(&vm->mm, &vma->node);
|
|
|
|
}
|
|
|
|
if (ret)
|
|
|
|
goto err_free_vma;
|
2015-10-01 20:33:57 +08:00
|
|
|
} else {
|
2015-12-08 19:55:07 +08:00
|
|
|
if (flags & PIN_HIGH) {
|
|
|
|
search_flag = DRM_MM_SEARCH_BELOW;
|
|
|
|
alloc_flag = DRM_MM_CREATE_TOP;
|
|
|
|
} else {
|
|
|
|
search_flag = DRM_MM_SEARCH_DEFAULT;
|
|
|
|
alloc_flag = DRM_MM_CREATE_DEFAULT;
|
|
|
|
}
|
2015-10-01 20:33:57 +08:00
|
|
|
|
2013-05-26 03:26:35 +08:00
|
|
|
search_free:
|
2015-12-08 19:55:07 +08:00
|
|
|
ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
|
|
|
|
size, alignment,
|
|
|
|
obj->cache_level,
|
|
|
|
start, end,
|
|
|
|
search_flag,
|
|
|
|
alloc_flag);
|
|
|
|
if (ret) {
|
|
|
|
ret = i915_gem_evict_something(dev, vm, size, alignment,
|
|
|
|
obj->cache_level,
|
|
|
|
start, end,
|
|
|
|
flags);
|
|
|
|
if (ret == 0)
|
|
|
|
goto search_free;
|
2009-09-21 07:22:34 +08:00
|
|
|
|
2015-12-08 19:55:07 +08:00
|
|
|
goto err_free_vma;
|
|
|
|
}
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
2014-09-11 15:43:48 +08:00
|
|
|
if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
|
2013-07-18 03:19:03 +08:00
|
|
|
ret = -EINVAL;
|
2013-07-22 18:12:38 +08:00
|
|
|
goto err_remove_node;
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
2014-12-11 01:27:58 +08:00
|
|
|
trace_i915_vma_bind(vma, flags);
|
2015-04-21 00:04:05 +08:00
|
|
|
ret = i915_vma_bind(vma, obj->cache_level, flags);
|
2014-12-11 01:27:58 +08:00
|
|
|
if (ret)
|
2015-07-09 17:59:05 +08:00
|
|
|
goto err_remove_node;
|
2014-12-11 01:27:58 +08:00
|
|
|
|
2013-06-01 02:28:48 +08:00
|
|
|
list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
|
2016-02-26 19:03:19 +08:00
|
|
|
list_add_tail(&vma->vm_link, &vm->inactive_list);
|
2010-08-07 18:01:20 +08:00
|
|
|
|
2014-02-14 21:01:20 +08:00
|
|
|
return vma;
|
2013-07-18 03:19:03 +08:00
|
|
|
|
2013-07-22 18:12:38 +08:00
|
|
|
err_remove_node:
|
2013-07-19 13:46:27 +08:00
|
|
|
drm_mm_remove_node(&vma->node);
|
2013-07-22 18:12:38 +08:00
|
|
|
err_free_vma:
|
2013-07-18 03:19:03 +08:00
|
|
|
i915_gem_vma_destroy(vma);
|
2014-02-14 21:01:20 +08:00
|
|
|
vma = ERR_PTR(ret);
|
2013-07-22 18:12:38 +08:00
|
|
|
err_unpin:
|
2013-07-18 03:19:03 +08:00
|
|
|
i915_gem_object_unpin_pages(obj);
|
2014-02-14 21:01:20 +08:00
|
|
|
return vma;
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
2013-08-08 21:41:09 +08:00
|
|
|
bool
|
2013-08-09 19:26:45 +08:00
|
|
|
i915_gem_clflush_object(struct drm_i915_gem_object *obj,
|
|
|
|
bool force)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
|
|
|
/* If we don't have a page list set up, then we're not pinned
|
|
|
|
* to GPU, and we can ignore the cache flush because it'll happen
|
|
|
|
* again at bind time.
|
|
|
|
*/
|
2010-11-09 03:18:58 +08:00
|
|
|
if (obj->pages == NULL)
|
2013-08-08 21:41:09 +08:00
|
|
|
return false;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2013-02-14 03:56:05 +08:00
|
|
|
/*
|
|
|
|
* Stolen memory is always coherent with the GPU as it is explicitly
|
|
|
|
* marked as wc by the system, or the system is cache-coherent.
|
|
|
|
*/
|
2014-11-04 20:51:40 +08:00
|
|
|
if (obj->stolen || obj->phys_handle)
|
2013-08-08 21:41:09 +08:00
|
|
|
return false;
|
2013-02-14 03:56:05 +08:00
|
|
|
|
2011-03-30 07:59:52 +08:00
|
|
|
/* If the GPU is snooping the contents of the CPU cache,
|
|
|
|
* we do not need to manually clear the CPU cache lines. However,
|
|
|
|
* the caches are only snooped when the render cache is
|
|
|
|
* flushed/invalidated. As we always have to emit invalidations
|
|
|
|
* and flushes when moving into and out of the RENDER domain, correct
|
|
|
|
* snooping behaviour occurs naturally as the result of our domain
|
|
|
|
* tracking.
|
|
|
|
*/
|
2015-01-13 21:32:52 +08:00
|
|
|
if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
|
|
|
|
obj->cache_dirty = true;
|
2013-08-08 21:41:09 +08:00
|
|
|
return false;
|
2015-01-13 21:32:52 +08:00
|
|
|
}
|
2011-03-30 07:59:52 +08:00
|
|
|
|
2009-08-25 18:15:50 +08:00
|
|
|
trace_i915_gem_object_clflush(obj);
|
2012-06-01 22:20:22 +08:00
|
|
|
drm_clflush_sg(obj->pages);
|
2015-01-13 21:32:52 +08:00
|
|
|
obj->cache_dirty = false;
|
2013-08-08 21:41:09 +08:00
|
|
|
|
|
|
|
return true;
|
2008-11-15 05:35:19 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/** Flushes the GTT write domain for the object if it's dirty. */
|
|
|
|
static void
|
2010-11-09 03:18:58 +08:00
|
|
|
i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
|
2008-11-15 05:35:19 +08:00
|
|
|
{
|
2009-08-25 18:15:50 +08:00
|
|
|
uint32_t old_write_domain;
|
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
|
2008-11-15 05:35:19 +08:00
|
|
|
return;
|
|
|
|
|
2011-01-05 02:42:07 +08:00
|
|
|
/* No actual flushing is required for the GTT write domain. Writes
|
2008-11-15 05:35:19 +08:00
|
|
|
* to it immediately go to main memory as far as we know, so there's
|
|
|
|
* no chipset flush. It also doesn't land in render cache.
|
2011-01-05 02:42:07 +08:00
|
|
|
*
|
|
|
|
* However, we do have to enforce the order so that all writes through
|
|
|
|
* the GTT land before any writes to the device, such as updates to
|
|
|
|
* the GATT itself.
|
2008-11-15 05:35:19 +08:00
|
|
|
*/
|
2011-01-05 02:42:07 +08:00
|
|
|
wmb();
|
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
old_write_domain = obj->base.write_domain;
|
|
|
|
obj->base.write_domain = 0;
|
2009-08-25 18:15:50 +08:00
|
|
|
|
2015-07-08 07:28:51 +08:00
|
|
|
intel_fb_obj_flush(obj, false, ORIGIN_GTT);
|
drm/i915: Track frontbuffer invalidation/flushing
So these are the guts of the new beast. This tracks when a frontbuffer
gets invalidated (due to frontbuffer rendering) and hence should be
constantly scaned out, and when it's flushed again and can be
compressed/one-shot-upload.
Rules for flushing are simple: The frontbuffer needs one more full
upload starting from the next vblank. Which means that the flushing
can _only_ be called once the frontbuffer update has been latched.
But this poses a problem for pageflips: We can't just delay the
flushing until the pageflip is latched, since that would pose the risk
that we override frontbuffer rendering that has been scheduled
in-between the pageflip ioctl and the actual latching.
To handle this track asynchronous invalidations (and also pageflip)
state per-ring and delay any in-between flushing until the rendering
has completed. And also cancel any delayed flushing if we get a new
invalidation request (whether delayed or not).
Also call intel_mark_fb_busy in both cases in all cases to make sure
that we keep the screen at the highest refresh rate both on flips,
synchronous plane updates and for frontbuffer rendering.
v2: Lots of improvements
Suggestions from Chris:
- Move invalidate/flush in flush_*_domain and set_to_*_domain.
- Drop the flush in busy_ioctl since it's redundant. Was a leftover
from an earlier concept to track flips/delayed flushes.
- Don't forget about the initial modeset enable/final disable.
Suggested by Chris.
Track flips accurately, too. Since flips complete independently of
rendering we need to track pending flips in a separate mask. Again if
an invalidate happens we need to cancel the evenutal flush to avoid
races.
v3:
Provide correct header declarations for flip functions. Currently not
needed outside of intel_display.c, but part of the proper interface.
v4: Add proper domain management to fbcon so that the fbcon buffer is
also tracked correctly.
v5: Fixup locking around the fbcon set_to_gtt_domain call.
v6: More comments from Chris:
- Split out fbcon changes.
- Drop superflous checks for potential scanout before calling intel_fb
functions - we can micro-optimize this later.
- s/intel_fb_/intel_fb_obj_/ to make it clear that this deals in gem
object. We already have precedence for fb_obj in the pin_and_fence
functions.
v7: Clarify the semantics of the flip flush handling by renaming
things a bit:
- Don't go through a gem object but take the relevant frontbuffer bits
directly. These functions center on the plane, the actual object is
irrelevant - even a flip to the same object as already active should
cause a flush.
- Add a new intel_frontbuffer_flip for synchronous plane updates. It
currently just calls intel_frontbuffer_flush since the implemenation
differs.
This way we achieve a clear split between one-shot update events on
one side and frontbuffer rendering with potentially a very long delay
between the invalidate and flush.
Chris and I also had some discussions about mark_busy and whether it
is appropriate to call from flush. But mark busy is a state which
should be derived from the 3 events (invalidate, flush, flip) we now
have by the users, like psr does by tracking relevant information in
psr.busy_frontbuffer_bits. DRRS (the only real use of mark_busy for
frontbuffer) needs to have similar logic. With that the overall
mark_busy in the core could be removed.
v8: Only when retiring gpu buffers only flush frontbuffer bits we
actually invalidated in a batch. Just for safety since before any
additional usage/invalidate we should always retire current rendering.
Suggested by Chris Wilson.
v9: Actually use intel_frontbuffer_flip in all appropriate places.
Spotted by Chris.
v10: Address more comments from Chris:
- Don't call _flip in set_base when the crtc is inactive, avoids redunancy
in the modeset case with the initial enabling of all planes.
- Add comments explaining that the initial/final plane enable/disable
still has work left to do before it's fully generic.
v11: Only invalidate for gtt/cpu access when writing. Spotted by Chris.
v12: s/_flush/_flip/ in intel_overlay.c per Chris' comment.
Cc: Rodrigo Vivi <rodrigo.vivi@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2014-06-19 22:01:59 +08:00
|
|
|
|
2009-08-25 18:15:50 +08:00
|
|
|
trace_i915_gem_object_change_domain(obj,
|
2010-11-09 03:18:58 +08:00
|
|
|
obj->base.read_domains,
|
2009-08-25 18:15:50 +08:00
|
|
|
old_write_domain);
|
2008-11-15 05:35:19 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/** Flushes the CPU write domain for the object if it's dirty. */
|
|
|
|
static void
|
2015-01-21 21:53:48 +08:00
|
|
|
i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
|
2008-11-15 05:35:19 +08:00
|
|
|
{
|
2009-08-25 18:15:50 +08:00
|
|
|
uint32_t old_write_domain;
|
2008-11-15 05:35:19 +08:00
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
|
2008-11-15 05:35:19 +08:00
|
|
|
return;
|
|
|
|
|
2015-01-21 21:53:48 +08:00
|
|
|
if (i915_gem_clflush_object(obj, obj->pin_display))
|
2013-08-08 21:41:09 +08:00
|
|
|
i915_gem_chipset_flush(obj->base.dev);
|
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
old_write_domain = obj->base.write_domain;
|
|
|
|
obj->base.write_domain = 0;
|
2009-08-25 18:15:50 +08:00
|
|
|
|
2015-07-08 07:28:51 +08:00
|
|
|
intel_fb_obj_flush(obj, false, ORIGIN_CPU);
|
drm/i915: Track frontbuffer invalidation/flushing
So these are the guts of the new beast. This tracks when a frontbuffer
gets invalidated (due to frontbuffer rendering) and hence should be
constantly scaned out, and when it's flushed again and can be
compressed/one-shot-upload.
Rules for flushing are simple: The frontbuffer needs one more full
upload starting from the next vblank. Which means that the flushing
can _only_ be called once the frontbuffer update has been latched.
But this poses a problem for pageflips: We can't just delay the
flushing until the pageflip is latched, since that would pose the risk
that we override frontbuffer rendering that has been scheduled
in-between the pageflip ioctl and the actual latching.
To handle this track asynchronous invalidations (and also pageflip)
state per-ring and delay any in-between flushing until the rendering
has completed. And also cancel any delayed flushing if we get a new
invalidation request (whether delayed or not).
Also call intel_mark_fb_busy in both cases in all cases to make sure
that we keep the screen at the highest refresh rate both on flips,
synchronous plane updates and for frontbuffer rendering.
v2: Lots of improvements
Suggestions from Chris:
- Move invalidate/flush in flush_*_domain and set_to_*_domain.
- Drop the flush in busy_ioctl since it's redundant. Was a leftover
from an earlier concept to track flips/delayed flushes.
- Don't forget about the initial modeset enable/final disable.
Suggested by Chris.
Track flips accurately, too. Since flips complete independently of
rendering we need to track pending flips in a separate mask. Again if
an invalidate happens we need to cancel the evenutal flush to avoid
races.
v3:
Provide correct header declarations for flip functions. Currently not
needed outside of intel_display.c, but part of the proper interface.
v4: Add proper domain management to fbcon so that the fbcon buffer is
also tracked correctly.
v5: Fixup locking around the fbcon set_to_gtt_domain call.
v6: More comments from Chris:
- Split out fbcon changes.
- Drop superflous checks for potential scanout before calling intel_fb
functions - we can micro-optimize this later.
- s/intel_fb_/intel_fb_obj_/ to make it clear that this deals in gem
object. We already have precedence for fb_obj in the pin_and_fence
functions.
v7: Clarify the semantics of the flip flush handling by renaming
things a bit:
- Don't go through a gem object but take the relevant frontbuffer bits
directly. These functions center on the plane, the actual object is
irrelevant - even a flip to the same object as already active should
cause a flush.
- Add a new intel_frontbuffer_flip for synchronous plane updates. It
currently just calls intel_frontbuffer_flush since the implemenation
differs.
This way we achieve a clear split between one-shot update events on
one side and frontbuffer rendering with potentially a very long delay
between the invalidate and flush.
Chris and I also had some discussions about mark_busy and whether it
is appropriate to call from flush. But mark busy is a state which
should be derived from the 3 events (invalidate, flush, flip) we now
have by the users, like psr does by tracking relevant information in
psr.busy_frontbuffer_bits. DRRS (the only real use of mark_busy for
frontbuffer) needs to have similar logic. With that the overall
mark_busy in the core could be removed.
v8: Only when retiring gpu buffers only flush frontbuffer bits we
actually invalidated in a batch. Just for safety since before any
additional usage/invalidate we should always retire current rendering.
Suggested by Chris Wilson.
v9: Actually use intel_frontbuffer_flip in all appropriate places.
Spotted by Chris.
v10: Address more comments from Chris:
- Don't call _flip in set_base when the crtc is inactive, avoids redunancy
in the modeset case with the initial enabling of all planes.
- Add comments explaining that the initial/final plane enable/disable
still has work left to do before it's fully generic.
v11: Only invalidate for gtt/cpu access when writing. Spotted by Chris.
v12: s/_flush/_flip/ in intel_overlay.c per Chris' comment.
Cc: Rodrigo Vivi <rodrigo.vivi@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2014-06-19 22:01:59 +08:00
|
|
|
|
2009-08-25 18:15:50 +08:00
|
|
|
trace_i915_gem_object_change_domain(obj,
|
2010-11-09 03:18:58 +08:00
|
|
|
obj->base.read_domains,
|
2009-08-25 18:15:50 +08:00
|
|
|
old_write_domain);
|
2008-11-15 05:35:19 +08:00
|
|
|
}
|
|
|
|
|
2008-11-11 02:53:25 +08:00
|
|
|
/**
|
|
|
|
* Moves a single object to the GTT read, and possibly write domain.
|
|
|
|
*
|
|
|
|
* This function returns when the move is complete, including waiting on
|
|
|
|
* flushes to occur.
|
|
|
|
*/
|
DRM: i915: add mode setting support
This commit adds i915 driver support for the DRM mode setting APIs.
Currently, VGA, LVDS, SDVO DVI & VGA, TV and DVO LVDS outputs are
supported. HDMI, DisplayPort and additional SDVO output support will
follow.
Support for the mode setting code is controlled by the new 'modeset'
module option. A new config option, CONFIG_DRM_I915_KMS controls the
default behavior, and whether a PCI ID list is built into the module for
use by user level module utilities.
Note that if mode setting is enabled, user level drivers that access
display registers directly or that don't use the kernel graphics memory
manager will likely corrupt kernel graphics memory, disrupt output
configuration (possibly leading to hangs and/or blank displays), and
prevent panic/oops messages from appearing. So use caution when
enabling this code; be sure your user level code supports the new
interfaces.
A new SysRq key, 'g', provides emergency support for switching back to
the kernel's framebuffer console; which is useful for testing.
Co-authors: Dave Airlie <airlied@linux.ie>, Hong Liu <hong.liu@intel.com>
Signed-off-by: Jesse Barnes <jbarnes@virtuousgeek.org>
Signed-off-by: Eric Anholt <eric@anholt.net>
Signed-off-by: Dave Airlie <airlied@redhat.com>
2008-11-08 06:24:08 +08:00
|
|
|
int
|
2010-11-23 23:26:33 +08:00
|
|
|
i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
|
2008-11-11 02:53:25 +08:00
|
|
|
{
|
2016-03-30 21:57:10 +08:00
|
|
|
struct drm_device *dev = obj->base.dev;
|
|
|
|
struct drm_i915_private *dev_priv = to_i915(dev);
|
|
|
|
struct i915_ggtt *ggtt = &dev_priv->ggtt;
|
2009-08-25 18:15:50 +08:00
|
|
|
uint32_t old_write_domain, old_read_domains;
|
2015-01-02 18:59:29 +08:00
|
|
|
struct i915_vma *vma;
|
2008-11-15 05:35:19 +08:00
|
|
|
int ret;
|
2008-11-11 02:53:25 +08:00
|
|
|
|
2011-02-07 23:23:02 +08:00
|
|
|
if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
|
|
|
|
return 0;
|
|
|
|
|
2012-07-20 19:41:01 +08:00
|
|
|
ret = i915_gem_object_wait_rendering(obj, !write);
|
2011-01-08 01:09:48 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2015-01-02 18:59:29 +08:00
|
|
|
/* Flush and acquire obj->pages so that we are coherent through
|
|
|
|
* direct access in memory with previous cached writes through
|
|
|
|
* shmemfs and that our cache domain tracking remains valid.
|
|
|
|
* For example, if the obj->filp was moved to swap without us
|
|
|
|
* being notified and releasing the pages, we would mistakenly
|
|
|
|
* continue to assume that the obj remained out of the CPU cached
|
|
|
|
* domain.
|
|
|
|
*/
|
|
|
|
ret = i915_gem_object_get_pages(obj);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2015-01-21 21:53:48 +08:00
|
|
|
i915_gem_object_flush_cpu_write_domain(obj);
|
2009-08-25 18:15:50 +08:00
|
|
|
|
2012-10-10 02:24:37 +08:00
|
|
|
/* Serialise direct access to this object with the barriers for
|
|
|
|
* coherent writes from the GPU, by effectively invalidating the
|
|
|
|
* GTT domain upon first access.
|
|
|
|
*/
|
|
|
|
if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
|
|
|
|
mb();
|
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
old_write_domain = obj->base.write_domain;
|
|
|
|
old_read_domains = obj->base.read_domains;
|
2009-08-25 18:15:50 +08:00
|
|
|
|
2008-11-15 05:35:19 +08:00
|
|
|
/* It should now be out of any other write domains, and we can update
|
|
|
|
* the domain values for our changes.
|
|
|
|
*/
|
2010-11-09 03:18:58 +08:00
|
|
|
BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
|
|
|
|
obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
|
2008-11-15 05:35:19 +08:00
|
|
|
if (write) {
|
2010-11-09 03:18:58 +08:00
|
|
|
obj->base.read_domains = I915_GEM_DOMAIN_GTT;
|
|
|
|
obj->base.write_domain = I915_GEM_DOMAIN_GTT;
|
|
|
|
obj->dirty = 1;
|
2008-11-11 02:53:25 +08:00
|
|
|
}
|
|
|
|
|
2009-08-25 18:15:50 +08:00
|
|
|
trace_i915_gem_object_change_domain(obj,
|
|
|
|
old_read_domains,
|
|
|
|
old_write_domain);
|
|
|
|
|
2012-04-24 22:52:35 +08:00
|
|
|
/* And bump the LRU for this access */
|
2015-01-02 18:59:29 +08:00
|
|
|
vma = i915_gem_obj_to_ggtt(obj);
|
|
|
|
if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
|
2016-02-26 19:03:19 +08:00
|
|
|
list_move_tail(&vma->vm_link,
|
2016-03-30 21:57:10 +08:00
|
|
|
&ggtt->base.inactive_list);
|
2012-04-24 22:52:35 +08:00
|
|
|
|
2008-11-15 05:35:19 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-10-09 21:11:27 +08:00
|
|
|
/**
|
|
|
|
* Changes the cache-level of an object across all VMA.
|
|
|
|
*
|
|
|
|
* After this function returns, the object will be in the new cache-level
|
|
|
|
* across all GTT and the contents of the backing storage will be coherent,
|
|
|
|
* with respect to the new cache-level. In order to keep the backing storage
|
|
|
|
* coherent for all users, we only allow a single cache level to be set
|
|
|
|
* globally on the object and prevent it from being changed whilst the
|
|
|
|
* hardware is reading from the object. That is if the object is currently
|
|
|
|
* on the scanout it will be set to uncached (or equivalent display
|
|
|
|
* cache coherency) and all non-MOCS GPU access will also be uncached so
|
|
|
|
* that all direct access to the scanout remains coherent.
|
|
|
|
*/
|
2011-04-04 16:44:39 +08:00
|
|
|
int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
|
|
|
|
enum i915_cache_level cache_level)
|
|
|
|
{
|
2012-02-10 00:15:47 +08:00
|
|
|
struct drm_device *dev = obj->base.dev;
|
2014-03-21 15:40:56 +08:00
|
|
|
struct i915_vma *vma, *next;
|
2015-10-09 21:11:27 +08:00
|
|
|
bool bound = false;
|
2015-08-12 00:47:10 +08:00
|
|
|
int ret = 0;
|
2011-04-04 16:44:39 +08:00
|
|
|
|
|
|
|
if (obj->cache_level == cache_level)
|
2015-08-12 00:47:10 +08:00
|
|
|
goto out;
|
2011-04-04 16:44:39 +08:00
|
|
|
|
2015-10-09 21:11:27 +08:00
|
|
|
/* Inspect the list of currently bound VMA and unbind any that would
|
|
|
|
* be invalid given the new cache-level. This is principally to
|
|
|
|
* catch the issue of the CS prefetch crossing page boundaries and
|
|
|
|
* reading an invalid PTE on older architectures.
|
|
|
|
*/
|
2016-02-26 19:03:19 +08:00
|
|
|
list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
|
2015-10-09 21:11:27 +08:00
|
|
|
if (!drm_mm_node_allocated(&vma->node))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (vma->pin_count) {
|
|
|
|
DRM_DEBUG("can not change the cache level of pinned objects\n");
|
|
|
|
return -EBUSY;
|
|
|
|
}
|
|
|
|
|
2014-09-11 15:43:48 +08:00
|
|
|
if (!i915_gem_valid_gtt_space(vma, cache_level)) {
|
drm/i915: plumb VM into bind/unbind code
As alluded to in several patches, and it will be reiterated later... A
VMA is an abstraction for a GEM BO bound into an address space.
Therefore it stands to reason, that the existing bind, and unbind are
the ones which will be the most impacted. This patch implements this,
and updates all callers which weren't already updated in the series
(because it was too messy).
This patch represents the bulk of an earlier, larger patch. I've pulled
out a bunch of things by the request of Daniel. The history is preserved
for posterity with the email convention of ">" One big change from the
original patch aside from a bunch of cropping is I've created an
i915_vma_unbind() function. That is because we always have the VMA
anyway, and doing an extra lookup is useful. There is a caveat, we
retain an i915_gem_object_ggtt_unbind, for the global cases which might
not talk in VMAs.
> drm/i915: plumb VM into object operations
>
> This patch was formerly known as:
> "drm/i915: Create VMAs (part 3) - plumbing"
>
> This patch adds a VM argument, bind/unbind, and the object
> offset/size/color getters/setters. It preserves the old ggtt helper
> functions because things still need, and will continue to need them.
>
> Some code will still need to be ported over after this.
>
> v2: Fix purge to pick an object and unbind all vmas
> This was doable because of the global bound list change.
>
> v3: With the commit to actually pin/unpin pages in place, there is no
> longer a need to check if unbind succeeded before calling put_pages().
> Make put_pages only BUG() after checking pin count.
>
> v4: Rebased on top of the new hangcheck work by Mika
> plumbed eb_destroy also
> Many checkpatch related fixes
>
> v5: Very large rebase
>
> v6:
> Change BUG_ON to WARN_ON (Daniel)
> Rename vm to ggtt in preallocate stolen, since it is always ggtt when
> dealing with stolen memory. (Daniel)
> list_for_each will short-circuit already (Daniel)
> remove superflous space (Daniel)
> Use per object list of vmas (Daniel)
> Make obj_bound_any() use obj_bound for each vm (Ben)
> s/bind_to_gtt/bind_to_vm/ (Ben)
>
> Fixed up the inactive shrinker. As Daniel noticed the code could
> potentially count the same object multiple times. While it's not
> possible in the current case, since 1 object can only ever be bound into
> 1 address space thus far - we may as well try to get something more
> future proof in place now. With a prep patch before this to switch over
> to using the bound list + inactive check, we're now able to carry that
> forward for every address space an object is bound into.
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
[danvet: Rebase on top of the loss of "drm/i915: Cleanup more of VMA
in destroy".]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-08-01 08:00:10 +08:00
|
|
|
ret = i915_vma_unbind(vma);
|
2013-08-01 08:00:03 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
2015-10-09 21:11:27 +08:00
|
|
|
} else
|
|
|
|
bound = true;
|
2012-07-26 18:49:32 +08:00
|
|
|
}
|
|
|
|
|
2015-10-09 21:11:27 +08:00
|
|
|
/* We can reuse the existing drm_mm nodes but need to change the
|
|
|
|
* cache-level on the PTE. We could simply unbind them all and
|
|
|
|
* rebind with the correct cache-level on next use. However since
|
|
|
|
* we already have a valid slot, dma mapping, pages etc, we may as
|
|
|
|
* rewrite the PTE in the belief that doing so tramples upon less
|
|
|
|
* state and so involves less work.
|
|
|
|
*/
|
|
|
|
if (bound) {
|
|
|
|
/* Before we change the PTE, the GPU must not be accessing it.
|
|
|
|
* If we wait upon the object, we know that all the bound
|
|
|
|
* VMA are no longer active.
|
|
|
|
*/
|
2015-04-27 20:41:14 +08:00
|
|
|
ret = i915_gem_object_wait_rendering(obj, false);
|
2011-04-04 16:44:39 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2015-10-09 21:11:27 +08:00
|
|
|
if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) {
|
|
|
|
/* Access to snoopable pages through the GTT is
|
|
|
|
* incoherent and on some machines causes a hard
|
|
|
|
* lockup. Relinquish the CPU mmaping to force
|
|
|
|
* userspace to refault in the pages and we can
|
|
|
|
* then double check if the GTT mapping is still
|
|
|
|
* valid for that pointer access.
|
|
|
|
*/
|
|
|
|
i915_gem_release_mmap(obj);
|
|
|
|
|
|
|
|
/* As we no longer need a fence for GTT access,
|
|
|
|
* we can relinquish it now (and so prevent having
|
|
|
|
* to steal a fence from someone else on the next
|
|
|
|
* fence request). Note GPU activity would have
|
|
|
|
* dropped the fence as all snoopable access is
|
|
|
|
* supposed to be linear.
|
|
|
|
*/
|
2011-04-04 16:44:39 +08:00
|
|
|
ret = i915_gem_object_put_fence(obj);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2015-10-09 21:11:27 +08:00
|
|
|
} else {
|
|
|
|
/* We either have incoherent backing store and
|
|
|
|
* so no GTT access or the architecture is fully
|
|
|
|
* coherent. In such cases, existing GTT mmaps
|
|
|
|
* ignore the cache bit in the PTE and we can
|
|
|
|
* rewrite it without confusing the GPU or having
|
|
|
|
* to force userspace to fault back in its mmaps.
|
|
|
|
*/
|
2011-04-04 16:44:39 +08:00
|
|
|
}
|
|
|
|
|
2016-02-26 19:03:19 +08:00
|
|
|
list_for_each_entry(vma, &obj->vma_list, obj_link) {
|
2015-10-09 21:11:27 +08:00
|
|
|
if (!drm_mm_node_allocated(&vma->node))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
}
|
2011-04-04 16:44:39 +08:00
|
|
|
}
|
|
|
|
|
2016-02-26 19:03:19 +08:00
|
|
|
list_for_each_entry(vma, &obj->vma_list, obj_link)
|
2013-08-09 19:26:45 +08:00
|
|
|
vma->node.color = cache_level;
|
|
|
|
obj->cache_level = cache_level;
|
|
|
|
|
2015-08-12 00:47:10 +08:00
|
|
|
out:
|
2015-10-09 21:11:27 +08:00
|
|
|
/* Flush the dirty CPU caches to the backing storage so that the
|
|
|
|
* object is now coherent at its new cache level (with respect
|
|
|
|
* to the access domain).
|
|
|
|
*/
|
2015-01-13 21:32:52 +08:00
|
|
|
if (obj->cache_dirty &&
|
|
|
|
obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
|
|
|
|
cpu_write_needs_clflush(obj)) {
|
|
|
|
if (i915_gem_clflush_object(obj, true))
|
|
|
|
i915_gem_chipset_flush(obj->base.dev);
|
2011-04-04 16:44:39 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2012-09-22 08:01:20 +08:00
|
|
|
int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
|
|
|
|
struct drm_file *file)
|
2012-07-10 17:27:08 +08:00
|
|
|
{
|
2012-09-22 08:01:20 +08:00
|
|
|
struct drm_i915_gem_caching *args = data;
|
2012-07-10 17:27:08 +08:00
|
|
|
struct drm_i915_gem_object *obj;
|
|
|
|
|
|
|
|
obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
|
2015-05-07 19:14:55 +08:00
|
|
|
if (&obj->base == NULL)
|
|
|
|
return -ENOENT;
|
2012-07-10 17:27:08 +08:00
|
|
|
|
2013-08-08 21:41:10 +08:00
|
|
|
switch (obj->cache_level) {
|
|
|
|
case I915_CACHE_LLC:
|
|
|
|
case I915_CACHE_L3_LLC:
|
|
|
|
args->caching = I915_CACHING_CACHED;
|
|
|
|
break;
|
|
|
|
|
2013-08-08 21:41:11 +08:00
|
|
|
case I915_CACHE_WT:
|
|
|
|
args->caching = I915_CACHING_DISPLAY;
|
|
|
|
break;
|
|
|
|
|
2013-08-08 21:41:10 +08:00
|
|
|
default:
|
|
|
|
args->caching = I915_CACHING_NONE;
|
|
|
|
break;
|
|
|
|
}
|
2012-07-10 17:27:08 +08:00
|
|
|
|
2015-05-07 19:14:55 +08:00
|
|
|
drm_gem_object_unreference_unlocked(&obj->base);
|
|
|
|
return 0;
|
2012-07-10 17:27:08 +08:00
|
|
|
}
|
|
|
|
|
2012-09-22 08:01:20 +08:00
|
|
|
int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
|
|
|
|
struct drm_file *file)
|
2012-07-10 17:27:08 +08:00
|
|
|
{
|
2015-11-05 03:25:32 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2012-09-22 08:01:20 +08:00
|
|
|
struct drm_i915_gem_caching *args = data;
|
2012-07-10 17:27:08 +08:00
|
|
|
struct drm_i915_gem_object *obj;
|
|
|
|
enum i915_cache_level level;
|
|
|
|
int ret;
|
|
|
|
|
2012-09-22 08:01:20 +08:00
|
|
|
switch (args->caching) {
|
|
|
|
case I915_CACHING_NONE:
|
2012-07-10 17:27:08 +08:00
|
|
|
level = I915_CACHE_NONE;
|
|
|
|
break;
|
2012-09-22 08:01:20 +08:00
|
|
|
case I915_CACHING_CACHED:
|
2015-08-14 23:43:30 +08:00
|
|
|
/*
|
|
|
|
* Due to a HW issue on BXT A stepping, GPU stores via a
|
|
|
|
* snooped mapping may leave stale data in a corresponding CPU
|
|
|
|
* cacheline, whereas normally such cachelines would get
|
|
|
|
* invalidated.
|
|
|
|
*/
|
2016-03-02 20:10:31 +08:00
|
|
|
if (!HAS_LLC(dev) && !HAS_SNOOP(dev))
|
2015-08-14 23:43:30 +08:00
|
|
|
return -ENODEV;
|
|
|
|
|
2012-07-10 17:27:08 +08:00
|
|
|
level = I915_CACHE_LLC;
|
|
|
|
break;
|
2013-08-08 21:41:11 +08:00
|
|
|
case I915_CACHING_DISPLAY:
|
|
|
|
level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
|
|
|
|
break;
|
2012-07-10 17:27:08 +08:00
|
|
|
default:
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2015-11-05 03:25:32 +08:00
|
|
|
intel_runtime_pm_get(dev_priv);
|
|
|
|
|
2012-09-27 07:15:20 +08:00
|
|
|
ret = i915_mutex_lock_interruptible(dev);
|
|
|
|
if (ret)
|
2015-11-05 03:25:32 +08:00
|
|
|
goto rpm_put;
|
2012-09-27 07:15:20 +08:00
|
|
|
|
2012-07-10 17:27:08 +08:00
|
|
|
obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
|
|
|
|
if (&obj->base == NULL) {
|
|
|
|
ret = -ENOENT;
|
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = i915_gem_object_set_cache_level(obj, level);
|
|
|
|
|
|
|
|
drm_gem_object_unreference(&obj->base);
|
|
|
|
unlock:
|
|
|
|
mutex_unlock(&dev->struct_mutex);
|
2015-11-05 03:25:32 +08:00
|
|
|
rpm_put:
|
|
|
|
intel_runtime_pm_put(dev_priv);
|
|
|
|
|
2012-07-10 17:27:08 +08:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2009-11-25 13:09:39 +08:00
|
|
|
/*
|
2011-04-14 16:41:17 +08:00
|
|
|
* Prepare buffer for display plane (scanout, cursors, etc).
|
|
|
|
* Can be called from an uninterruptible phase (modesetting) and allows
|
|
|
|
* any flushes to be pipelined (for pageflips).
|
2009-11-25 13:09:39 +08:00
|
|
|
*/
|
|
|
|
int
|
2011-04-14 16:41:17 +08:00
|
|
|
i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
|
|
|
|
u32 alignment,
|
2015-03-23 19:10:33 +08:00
|
|
|
const struct i915_ggtt_view *view)
|
2009-11-25 13:09:39 +08:00
|
|
|
{
|
2011-04-14 16:41:17 +08:00
|
|
|
u32 old_read_domains, old_write_domain;
|
2009-11-25 13:09:39 +08:00
|
|
|
int ret;
|
|
|
|
|
2013-08-09 19:25:09 +08:00
|
|
|
/* Mark the pin_display early so that we account for the
|
|
|
|
* display coherency whilst setting up the cache domains.
|
|
|
|
*/
|
2015-04-13 18:50:09 +08:00
|
|
|
obj->pin_display++;
|
2013-08-09 19:25:09 +08:00
|
|
|
|
2011-03-30 07:59:54 +08:00
|
|
|
/* The display engine is not coherent with the LLC cache on gen6. As
|
|
|
|
* a result, we make sure that the pinning that is about to occur is
|
|
|
|
* done with uncached PTEs. This is lowest common denominator for all
|
|
|
|
* chipsets.
|
|
|
|
*
|
|
|
|
* However for gen6+, we could do better by using the GFDT bit instead
|
|
|
|
* of uncaching, which would allow us to flush all the LLC-cached data
|
|
|
|
* with that bit in the PTE to main memory with just one PIPE_CONTROL.
|
|
|
|
*/
|
2013-08-08 21:41:10 +08:00
|
|
|
ret = i915_gem_object_set_cache_level(obj,
|
|
|
|
HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
|
2011-03-30 07:59:54 +08:00
|
|
|
if (ret)
|
2013-08-09 19:25:09 +08:00
|
|
|
goto err_unpin_display;
|
2011-03-30 07:59:54 +08:00
|
|
|
|
2011-04-14 16:41:17 +08:00
|
|
|
/* As the user may map the buffer once pinned in the display plane
|
|
|
|
* (e.g. libkms for the bootup splash), we have to ensure that we
|
|
|
|
* always use map_and_fenceable for all scanout buffers.
|
|
|
|
*/
|
2015-03-23 19:10:36 +08:00
|
|
|
ret = i915_gem_object_ggtt_pin(obj, view, alignment,
|
|
|
|
view->type == I915_GGTT_VIEW_NORMAL ?
|
|
|
|
PIN_MAPPABLE : 0);
|
2011-04-14 16:41:17 +08:00
|
|
|
if (ret)
|
2013-08-09 19:25:09 +08:00
|
|
|
goto err_unpin_display;
|
2011-04-14 16:41:17 +08:00
|
|
|
|
2015-01-21 21:53:48 +08:00
|
|
|
i915_gem_object_flush_cpu_write_domain(obj);
|
2010-05-27 20:18:14 +08:00
|
|
|
|
2011-04-14 16:41:17 +08:00
|
|
|
old_write_domain = obj->base.write_domain;
|
2010-11-09 03:18:58 +08:00
|
|
|
old_read_domains = obj->base.read_domains;
|
2011-04-14 16:41:17 +08:00
|
|
|
|
|
|
|
/* It should now be out of any other write domains, and we can update
|
|
|
|
* the domain values for our changes.
|
|
|
|
*/
|
2012-07-20 19:41:00 +08:00
|
|
|
obj->base.write_domain = 0;
|
2010-11-09 03:18:58 +08:00
|
|
|
obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
|
2009-11-25 13:09:39 +08:00
|
|
|
|
|
|
|
trace_i915_gem_object_change_domain(obj,
|
|
|
|
old_read_domains,
|
2011-04-14 16:41:17 +08:00
|
|
|
old_write_domain);
|
2009-11-25 13:09:39 +08:00
|
|
|
|
|
|
|
return 0;
|
2013-08-09 19:25:09 +08:00
|
|
|
|
|
|
|
err_unpin_display:
|
2015-04-13 18:50:09 +08:00
|
|
|
obj->pin_display--;
|
2013-08-09 19:25:09 +08:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2015-03-23 19:10:33 +08:00
|
|
|
i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
|
|
|
|
const struct i915_ggtt_view *view)
|
2013-08-09 19:25:09 +08:00
|
|
|
{
|
2015-04-13 18:50:09 +08:00
|
|
|
if (WARN_ON(obj->pin_display == 0))
|
|
|
|
return;
|
|
|
|
|
2015-03-23 19:10:33 +08:00
|
|
|
i915_gem_object_ggtt_unpin_view(obj, view);
|
|
|
|
|
2015-04-13 18:50:09 +08:00
|
|
|
obj->pin_display--;
|
2009-11-25 13:09:39 +08:00
|
|
|
}
|
|
|
|
|
2008-11-15 05:35:19 +08:00
|
|
|
/**
|
|
|
|
* Moves a single object to the CPU read, and possibly write domain.
|
|
|
|
*
|
|
|
|
* This function returns when the move is complete, including waiting on
|
|
|
|
* flushes to occur.
|
|
|
|
*/
|
2012-03-26 16:10:27 +08:00
|
|
|
int
|
2010-11-12 21:42:53 +08:00
|
|
|
i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
|
2008-11-15 05:35:19 +08:00
|
|
|
{
|
2009-08-25 18:15:50 +08:00
|
|
|
uint32_t old_write_domain, old_read_domains;
|
2008-11-15 05:35:19 +08:00
|
|
|
int ret;
|
|
|
|
|
2011-02-07 23:23:02 +08:00
|
|
|
if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
|
|
|
|
return 0;
|
|
|
|
|
2012-07-20 19:41:01 +08:00
|
|
|
ret = i915_gem_object_wait_rendering(obj, !write);
|
2011-01-08 01:09:48 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2008-11-15 05:35:19 +08:00
|
|
|
i915_gem_object_flush_gtt_write_domain(obj);
|
2008-11-11 02:53:25 +08:00
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
old_write_domain = obj->base.write_domain;
|
|
|
|
old_read_domains = obj->base.read_domains;
|
2009-08-25 18:15:50 +08:00
|
|
|
|
2008-11-15 05:35:19 +08:00
|
|
|
/* Flush the CPU cache if it's still invalid. */
|
2010-11-09 03:18:58 +08:00
|
|
|
if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
|
2013-08-09 19:26:45 +08:00
|
|
|
i915_gem_clflush_object(obj, false);
|
2008-11-11 02:53:25 +08:00
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
|
2008-11-11 02:53:25 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/* It should now be out of any other write domains, and we can update
|
|
|
|
* the domain values for our changes.
|
|
|
|
*/
|
2010-11-09 03:18:58 +08:00
|
|
|
BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
|
2008-11-15 05:35:19 +08:00
|
|
|
|
|
|
|
/* If we're writing through the CPU, then the GPU read domains will
|
|
|
|
* need to be invalidated at next use.
|
|
|
|
*/
|
|
|
|
if (write) {
|
2010-11-09 03:18:58 +08:00
|
|
|
obj->base.read_domains = I915_GEM_DOMAIN_CPU;
|
|
|
|
obj->base.write_domain = I915_GEM_DOMAIN_CPU;
|
2008-11-15 05:35:19 +08:00
|
|
|
}
|
2008-11-11 02:53:25 +08:00
|
|
|
|
2009-08-25 18:15:50 +08:00
|
|
|
trace_i915_gem_object_change_domain(obj,
|
|
|
|
old_read_domains,
|
|
|
|
old_write_domain);
|
|
|
|
|
2008-11-11 02:53:25 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-07-31 03:06:12 +08:00
|
|
|
/* Throttle our rendering by waiting until the ring has completed our requests
|
|
|
|
* emitted over 20 msec ago.
|
|
|
|
*
|
2009-06-03 15:27:35 +08:00
|
|
|
* Note that if we were to use the current jiffies each time around the loop,
|
|
|
|
* we wouldn't escape the function with any frames outstanding if the time to
|
|
|
|
* render a frame was over 20ms.
|
|
|
|
*
|
2008-07-31 03:06:12 +08:00
|
|
|
* This should get us reasonable parallelism between CPU and GPU but also
|
|
|
|
* relatively low latency when blocking on a particular request to finish.
|
|
|
|
*/
|
2009-03-13 02:23:52 +08:00
|
|
|
static int
|
2010-09-24 23:02:42 +08:00
|
|
|
i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
|
2009-03-13 02:23:52 +08:00
|
|
|
{
|
2010-09-24 23:02:42 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
|
|
|
struct drm_i915_file_private *file_priv = file->driver_priv;
|
2015-05-22 04:01:48 +08:00
|
|
|
unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
|
2014-11-25 02:49:27 +08:00
|
|
|
struct drm_i915_gem_request *request, *target = NULL;
|
drm/i915: create a race-free reset detection
With the previous patch the state transition handling of the reset
code itself is now (hopefully) race free and solid. But that still
leaves out everyone else - with the various lock-free wait paths
we have there's the possibility that the reset happens between the
point where we read the seqno we should wait on and the actual wait.
And if __wait_seqno then never sees the RESET_IN_PROGRESS state, we'll
happily wait for a seqno which will in all likelyhood never signal.
In practice this is not a big problem since the X server gets
constantly interrupted, and can then submit more work (hopefully) to
unblock everyone else: As soon as a new seqno write lands, all waiters
will unblock. But running the i-g-t reset testcase ZZ_hangman can
expose this race, especially on slower hw with fewer cpu cores.
Now looking forward to ARB_robustness and friends that's not the best
possible behaviour, hence this patch adds a reset_counter to be able
to detect any reset, even if a given thread never observed the
in-progress state.
The important part is to correctly order things:
- The write side needs to increment the counter after any seqno gets
reset. Hence we need to do that at the end of the reset work, and
again wake everyone up. We also need to place a barrier in between
any possible seqno changes and the counter increment, since any
unlock operations only guarantee that nothing leaks out, but not
that at later load operation gets moved ahead.
- On the read side we need to ensure that no reset can sneak in and
invalidate the seqno. In all cases we can use the one-sided barrier
that unlock operations guarantee (of the lock protecting the
respective seqno/ring pair) to ensure correct ordering. Hence it is
sufficient to place the atomic read before the mutex/spin_unlock and
no additional barriers are required.
The end-result of all this is that we need to wake up everyone twice
in a reset operation:
- First, before the reset starts, to get any lockholders of the locks,
so that the reset can proceed.
- Second, after the reset is completed, to allow waiters to properly
and reliably detect the reset condition and bail out.
I admit that this entire reset_counter thing smells a bit like
overkill, but I think it's justified since it makes it really explicit
what the bail-out condition is. And we need a reset counter anyway to
implement ARB_robustness, and imo with finer-grained locking on the
horizont this is the most resilient scheme I could think of.
v2: Drop spurious change in the wait_for_error EXIT_COND - we only
need to wait until we leave the reset-in-progress wedged state.
v3: Don't play tricks with barriers in the throttle ioctl, the
spin_unlock is barrier enough.
I've also considered using a little helper to grab the current
reset_counter, but then decided that hiding the atomic_read isn't a
great idea, since having it explicitly show up in the code is a nice
remainder to reviews to check the memory barriers.
v4: Add a comment to explain why we need to fall through in
__wait_seqno in the end variable assignments.
v5: Review from Damien:
- s/smb/smp/ in a comment
- don't increment the reset counter after we've set it to WEDGED. Now
we (again) properly wedge the gpu when the reset fails.
Reviewed-by: Damien Lespiau <damien.lespiau@intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-12-06 16:01:42 +08:00
|
|
|
unsigned reset_counter;
|
2010-09-24 23:02:42 +08:00
|
|
|
int ret;
|
2010-01-31 18:40:48 +08:00
|
|
|
|
2012-11-15 00:14:06 +08:00
|
|
|
ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2011-01-26 23:39:14 +08:00
|
|
|
|
2010-09-26 18:03:27 +08:00
|
|
|
spin_lock(&file_priv->mm.lock);
|
2010-09-24 23:02:42 +08:00
|
|
|
list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
|
2009-06-03 15:27:35 +08:00
|
|
|
if (time_after_eq(request->emitted_jiffies, recent_enough))
|
|
|
|
break;
|
2009-03-13 02:23:52 +08:00
|
|
|
|
2015-05-30 00:44:12 +08:00
|
|
|
/*
|
|
|
|
* Note that the request might not have been submitted yet.
|
|
|
|
* In which case emitted_jiffies will be zero.
|
|
|
|
*/
|
|
|
|
if (!request->emitted_jiffies)
|
|
|
|
continue;
|
|
|
|
|
2014-11-25 02:49:27 +08:00
|
|
|
target = request;
|
2009-06-03 15:27:35 +08:00
|
|
|
}
|
2016-04-14 00:35:03 +08:00
|
|
|
reset_counter = i915_reset_counter(&dev_priv->gpu_error);
|
2014-11-25 02:49:28 +08:00
|
|
|
if (target)
|
|
|
|
i915_gem_request_reference(target);
|
2010-09-26 18:03:27 +08:00
|
|
|
spin_unlock(&file_priv->mm.lock);
|
2009-03-13 02:23:52 +08:00
|
|
|
|
2014-11-25 02:49:27 +08:00
|
|
|
if (target == NULL)
|
2010-09-24 23:02:42 +08:00
|
|
|
return 0;
|
2009-04-07 04:55:41 +08:00
|
|
|
|
2014-11-25 02:49:35 +08:00
|
|
|
ret = __i915_wait_request(target, reset_counter, true, NULL, NULL);
|
2010-09-24 23:02:42 +08:00
|
|
|
if (ret == 0)
|
|
|
|
queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
|
2009-03-13 02:23:52 +08:00
|
|
|
|
2015-03-27 19:01:36 +08:00
|
|
|
i915_gem_request_unreference__unlocked(target);
|
2014-11-25 02:49:28 +08:00
|
|
|
|
2009-03-13 02:23:52 +08:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
drm/i915: Prevent negative relocation deltas from wrapping
This is pure evil. Userspace, I'm looking at you SNA, repacks batch
buffers on the fly after generation as they are being passed to the
kernel for execution. These batches also contain self-referenced
relocations as a single buffer encompasses the state commands, kernels,
vertices and sampler. During generation the buffers are placed at known
offsets within the full batch, and then the relocation deltas (as passed
to the kernel) are tweaked as the batch is repacked into a smaller buffer.
This means that userspace is passing negative relocations deltas, which
subsequently wrap to large values if the batch is at a low address. The
GPU hangs when it then tries to use the large value as a base for its
address offsets, rather than wrapping back to the real value (as one
would hope). As the GPU uses positive offsets from the base, we can
treat the relocation address as the minimum address read by the GPU.
For the upper bound, we trust that userspace will not read beyond the
end of the buffer.
So, how do we fix negative relocations from wrapping? We can either
check that every relocation looks valid when we write it, and then
position each object such that we prevent the offset wraparound, or we
just special-case the self-referential behaviour of SNA and force all
batches to be above 256k. Daniel prefers the latter approach.
This fixes a GPU hang when it tries to use an address (relocation +
offset) greater than the GTT size. The issue would occur quite easily
with full-ppgtt as each fd gets its own VM space, so low offsets would
often be handed out. However, with the rearrangement of the low GTT due
to capturing the BIOS framebuffer, it is already affecting kernels 3.15
onwards. I think only IVB+ is susceptible to this bug, but the workaround
should only kick in rarely, so it seems sensible to always apply it.
v3: Use a bias for batch buffers to prevent small negative delta relocations
from wrapping.
v4 from Daniel:
- s/BIAS/BATCH_OFFSET_BIAS/
- Extract eb_vma_misplaced/i915_vma_misplaced since the conditions
were growing rather cumbersome.
- Add a comment to eb_get_batch explaining why we do this.
- Apply the batch offset bias everywhere but mention that we've only
observed it on gen7 gpus.
- Drop PIN_OFFSET_FIX for now, that slipped in from a feature patch.
v5: Add static to eb_get_batch, spotted by 0-day tester.
Testcase: igt/gem_bad_reloc
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=78533
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk> (v3)
Cc: stable@vger.kernel.org
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2014-05-23 14:48:08 +08:00
|
|
|
static bool
|
|
|
|
i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
|
|
|
|
{
|
|
|
|
struct drm_i915_gem_object *obj = vma->obj;
|
|
|
|
|
|
|
|
if (alignment &&
|
|
|
|
vma->node.start & (alignment - 1))
|
|
|
|
return true;
|
|
|
|
|
|
|
|
if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
if (flags & PIN_OFFSET_BIAS &&
|
|
|
|
vma->node.start < (flags & PIN_OFFSET_MASK))
|
|
|
|
return true;
|
|
|
|
|
2015-12-08 19:55:07 +08:00
|
|
|
if (flags & PIN_OFFSET_FIXED &&
|
|
|
|
vma->node.start != (flags & PIN_OFFSET_MASK))
|
|
|
|
return true;
|
|
|
|
|
drm/i915: Prevent negative relocation deltas from wrapping
This is pure evil. Userspace, I'm looking at you SNA, repacks batch
buffers on the fly after generation as they are being passed to the
kernel for execution. These batches also contain self-referenced
relocations as a single buffer encompasses the state commands, kernels,
vertices and sampler. During generation the buffers are placed at known
offsets within the full batch, and then the relocation deltas (as passed
to the kernel) are tweaked as the batch is repacked into a smaller buffer.
This means that userspace is passing negative relocations deltas, which
subsequently wrap to large values if the batch is at a low address. The
GPU hangs when it then tries to use the large value as a base for its
address offsets, rather than wrapping back to the real value (as one
would hope). As the GPU uses positive offsets from the base, we can
treat the relocation address as the minimum address read by the GPU.
For the upper bound, we trust that userspace will not read beyond the
end of the buffer.
So, how do we fix negative relocations from wrapping? We can either
check that every relocation looks valid when we write it, and then
position each object such that we prevent the offset wraparound, or we
just special-case the self-referential behaviour of SNA and force all
batches to be above 256k. Daniel prefers the latter approach.
This fixes a GPU hang when it tries to use an address (relocation +
offset) greater than the GTT size. The issue would occur quite easily
with full-ppgtt as each fd gets its own VM space, so low offsets would
often be handed out. However, with the rearrangement of the low GTT due
to capturing the BIOS framebuffer, it is already affecting kernels 3.15
onwards. I think only IVB+ is susceptible to this bug, but the workaround
should only kick in rarely, so it seems sensible to always apply it.
v3: Use a bias for batch buffers to prevent small negative delta relocations
from wrapping.
v4 from Daniel:
- s/BIAS/BATCH_OFFSET_BIAS/
- Extract eb_vma_misplaced/i915_vma_misplaced since the conditions
were growing rather cumbersome.
- Add a comment to eb_get_batch explaining why we do this.
- Apply the batch offset bias everywhere but mention that we've only
observed it on gen7 gpus.
- Drop PIN_OFFSET_FIX for now, that slipped in from a feature patch.
v5: Add static to eb_get_batch, spotted by 0-day tester.
Testcase: igt/gem_bad_reloc
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=78533
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk> (v3)
Cc: stable@vger.kernel.org
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2014-05-23 14:48:08 +08:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2015-11-20 22:16:39 +08:00
|
|
|
void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
|
|
|
|
{
|
|
|
|
struct drm_i915_gem_object *obj = vma->obj;
|
|
|
|
bool mappable, fenceable;
|
|
|
|
u32 fence_size, fence_alignment;
|
|
|
|
|
|
|
|
fence_size = i915_gem_get_gtt_size(obj->base.dev,
|
|
|
|
obj->base.size,
|
|
|
|
obj->tiling_mode);
|
|
|
|
fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
|
|
|
|
obj->base.size,
|
|
|
|
obj->tiling_mode,
|
|
|
|
true);
|
|
|
|
|
|
|
|
fenceable = (vma->node.size == fence_size &&
|
|
|
|
(vma->node.start & (fence_alignment - 1)) == 0);
|
|
|
|
|
|
|
|
mappable = (vma->node.start + fence_size <=
|
2016-03-18 16:42:57 +08:00
|
|
|
to_i915(obj->base.dev)->ggtt.mappable_end);
|
2015-11-20 22:16:39 +08:00
|
|
|
|
|
|
|
obj->map_and_fenceable = mappable && fenceable;
|
|
|
|
}
|
|
|
|
|
2015-03-16 20:11:13 +08:00
|
|
|
static int
|
|
|
|
i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
|
|
|
|
struct i915_address_space *vm,
|
|
|
|
const struct i915_ggtt_view *ggtt_view,
|
|
|
|
uint32_t alignment,
|
|
|
|
uint64_t flags)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
2014-05-07 13:21:36 +08:00
|
|
|
struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
|
drm/i915: plumb VM into bind/unbind code
As alluded to in several patches, and it will be reiterated later... A
VMA is an abstraction for a GEM BO bound into an address space.
Therefore it stands to reason, that the existing bind, and unbind are
the ones which will be the most impacted. This patch implements this,
and updates all callers which weren't already updated in the series
(because it was too messy).
This patch represents the bulk of an earlier, larger patch. I've pulled
out a bunch of things by the request of Daniel. The history is preserved
for posterity with the email convention of ">" One big change from the
original patch aside from a bunch of cropping is I've created an
i915_vma_unbind() function. That is because we always have the VMA
anyway, and doing an extra lookup is useful. There is a caveat, we
retain an i915_gem_object_ggtt_unbind, for the global cases which might
not talk in VMAs.
> drm/i915: plumb VM into object operations
>
> This patch was formerly known as:
> "drm/i915: Create VMAs (part 3) - plumbing"
>
> This patch adds a VM argument, bind/unbind, and the object
> offset/size/color getters/setters. It preserves the old ggtt helper
> functions because things still need, and will continue to need them.
>
> Some code will still need to be ported over after this.
>
> v2: Fix purge to pick an object and unbind all vmas
> This was doable because of the global bound list change.
>
> v3: With the commit to actually pin/unpin pages in place, there is no
> longer a need to check if unbind succeeded before calling put_pages().
> Make put_pages only BUG() after checking pin count.
>
> v4: Rebased on top of the new hangcheck work by Mika
> plumbed eb_destroy also
> Many checkpatch related fixes
>
> v5: Very large rebase
>
> v6:
> Change BUG_ON to WARN_ON (Daniel)
> Rename vm to ggtt in preallocate stolen, since it is always ggtt when
> dealing with stolen memory. (Daniel)
> list_for_each will short-circuit already (Daniel)
> remove superflous space (Daniel)
> Use per object list of vmas (Daniel)
> Make obj_bound_any() use obj_bound for each vm (Ben)
> s/bind_to_gtt/bind_to_vm/ (Ben)
>
> Fixed up the inactive shrinker. As Daniel noticed the code could
> potentially count the same object multiple times. While it's not
> possible in the current case, since 1 object can only ever be bound into
> 1 address space thus far - we may as well try to get something more
> future proof in place now. With a prep patch before this to switch over
> to using the bound list + inactive check, we're now able to carry that
> forward for every address space an object is bound into.
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
[danvet: Rebase on top of the loss of "drm/i915: Cleanup more of VMA
in destroy".]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-08-01 08:00:10 +08:00
|
|
|
struct i915_vma *vma;
|
2014-10-31 21:53:52 +08:00
|
|
|
unsigned bound;
|
2008-07-31 03:06:12 +08:00
|
|
|
int ret;
|
|
|
|
|
2014-05-07 13:21:36 +08:00
|
|
|
if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
|
|
|
|
return -ENODEV;
|
|
|
|
|
2014-02-14 21:01:12 +08:00
|
|
|
if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
|
2014-02-14 21:01:11 +08:00
|
|
|
return -EINVAL;
|
drm/i915: plumb VM into bind/unbind code
As alluded to in several patches, and it will be reiterated later... A
VMA is an abstraction for a GEM BO bound into an address space.
Therefore it stands to reason, that the existing bind, and unbind are
the ones which will be the most impacted. This patch implements this,
and updates all callers which weren't already updated in the series
(because it was too messy).
This patch represents the bulk of an earlier, larger patch. I've pulled
out a bunch of things by the request of Daniel. The history is preserved
for posterity with the email convention of ">" One big change from the
original patch aside from a bunch of cropping is I've created an
i915_vma_unbind() function. That is because we always have the VMA
anyway, and doing an extra lookup is useful. There is a caveat, we
retain an i915_gem_object_ggtt_unbind, for the global cases which might
not talk in VMAs.
> drm/i915: plumb VM into object operations
>
> This patch was formerly known as:
> "drm/i915: Create VMAs (part 3) - plumbing"
>
> This patch adds a VM argument, bind/unbind, and the object
> offset/size/color getters/setters. It preserves the old ggtt helper
> functions because things still need, and will continue to need them.
>
> Some code will still need to be ported over after this.
>
> v2: Fix purge to pick an object and unbind all vmas
> This was doable because of the global bound list change.
>
> v3: With the commit to actually pin/unpin pages in place, there is no
> longer a need to check if unbind succeeded before calling put_pages().
> Make put_pages only BUG() after checking pin count.
>
> v4: Rebased on top of the new hangcheck work by Mika
> plumbed eb_destroy also
> Many checkpatch related fixes
>
> v5: Very large rebase
>
> v6:
> Change BUG_ON to WARN_ON (Daniel)
> Rename vm to ggtt in preallocate stolen, since it is always ggtt when
> dealing with stolen memory. (Daniel)
> list_for_each will short-circuit already (Daniel)
> remove superflous space (Daniel)
> Use per object list of vmas (Daniel)
> Make obj_bound_any() use obj_bound for each vm (Ben)
> s/bind_to_gtt/bind_to_vm/ (Ben)
>
> Fixed up the inactive shrinker. As Daniel noticed the code could
> potentially count the same object multiple times. While it's not
> possible in the current case, since 1 object can only ever be bound into
> 1 address space thus far - we may as well try to get something more
> future proof in place now. With a prep patch before this to switch over
> to using the bound list + inactive check, we're now able to carry that
> forward for every address space an object is bound into.
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
[danvet: Rebase on top of the loss of "drm/i915: Cleanup more of VMA
in destroy".]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-08-01 08:00:10 +08:00
|
|
|
|
2014-10-31 21:53:53 +08:00
|
|
|
if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
|
|
|
|
return -EINVAL;
|
|
|
|
|
2015-03-16 20:11:13 +08:00
|
|
|
if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
|
|
|
|
i915_gem_obj_to_vma(obj, vm);
|
|
|
|
|
drm/i915: plumb VM into bind/unbind code
As alluded to in several patches, and it will be reiterated later... A
VMA is an abstraction for a GEM BO bound into an address space.
Therefore it stands to reason, that the existing bind, and unbind are
the ones which will be the most impacted. This patch implements this,
and updates all callers which weren't already updated in the series
(because it was too messy).
This patch represents the bulk of an earlier, larger patch. I've pulled
out a bunch of things by the request of Daniel. The history is preserved
for posterity with the email convention of ">" One big change from the
original patch aside from a bunch of cropping is I've created an
i915_vma_unbind() function. That is because we always have the VMA
anyway, and doing an extra lookup is useful. There is a caveat, we
retain an i915_gem_object_ggtt_unbind, for the global cases which might
not talk in VMAs.
> drm/i915: plumb VM into object operations
>
> This patch was formerly known as:
> "drm/i915: Create VMAs (part 3) - plumbing"
>
> This patch adds a VM argument, bind/unbind, and the object
> offset/size/color getters/setters. It preserves the old ggtt helper
> functions because things still need, and will continue to need them.
>
> Some code will still need to be ported over after this.
>
> v2: Fix purge to pick an object and unbind all vmas
> This was doable because of the global bound list change.
>
> v3: With the commit to actually pin/unpin pages in place, there is no
> longer a need to check if unbind succeeded before calling put_pages().
> Make put_pages only BUG() after checking pin count.
>
> v4: Rebased on top of the new hangcheck work by Mika
> plumbed eb_destroy also
> Many checkpatch related fixes
>
> v5: Very large rebase
>
> v6:
> Change BUG_ON to WARN_ON (Daniel)
> Rename vm to ggtt in preallocate stolen, since it is always ggtt when
> dealing with stolen memory. (Daniel)
> list_for_each will short-circuit already (Daniel)
> remove superflous space (Daniel)
> Use per object list of vmas (Daniel)
> Make obj_bound_any() use obj_bound for each vm (Ben)
> s/bind_to_gtt/bind_to_vm/ (Ben)
>
> Fixed up the inactive shrinker. As Daniel noticed the code could
> potentially count the same object multiple times. While it's not
> possible in the current case, since 1 object can only ever be bound into
> 1 address space thus far - we may as well try to get something more
> future proof in place now. With a prep patch before this to switch over
> to using the bound list + inactive check, we're now able to carry that
> forward for every address space an object is bound into.
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
[danvet: Rebase on top of the loss of "drm/i915: Cleanup more of VMA
in destroy".]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-08-01 08:00:10 +08:00
|
|
|
if (vma) {
|
2013-12-07 06:10:55 +08:00
|
|
|
if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
|
|
|
|
return -EBUSY;
|
|
|
|
|
drm/i915: Prevent negative relocation deltas from wrapping
This is pure evil. Userspace, I'm looking at you SNA, repacks batch
buffers on the fly after generation as they are being passed to the
kernel for execution. These batches also contain self-referenced
relocations as a single buffer encompasses the state commands, kernels,
vertices and sampler. During generation the buffers are placed at known
offsets within the full batch, and then the relocation deltas (as passed
to the kernel) are tweaked as the batch is repacked into a smaller buffer.
This means that userspace is passing negative relocations deltas, which
subsequently wrap to large values if the batch is at a low address. The
GPU hangs when it then tries to use the large value as a base for its
address offsets, rather than wrapping back to the real value (as one
would hope). As the GPU uses positive offsets from the base, we can
treat the relocation address as the minimum address read by the GPU.
For the upper bound, we trust that userspace will not read beyond the
end of the buffer.
So, how do we fix negative relocations from wrapping? We can either
check that every relocation looks valid when we write it, and then
position each object such that we prevent the offset wraparound, or we
just special-case the self-referential behaviour of SNA and force all
batches to be above 256k. Daniel prefers the latter approach.
This fixes a GPU hang when it tries to use an address (relocation +
offset) greater than the GTT size. The issue would occur quite easily
with full-ppgtt as each fd gets its own VM space, so low offsets would
often be handed out. However, with the rearrangement of the low GTT due
to capturing the BIOS framebuffer, it is already affecting kernels 3.15
onwards. I think only IVB+ is susceptible to this bug, but the workaround
should only kick in rarely, so it seems sensible to always apply it.
v3: Use a bias for batch buffers to prevent small negative delta relocations
from wrapping.
v4 from Daniel:
- s/BIAS/BATCH_OFFSET_BIAS/
- Extract eb_vma_misplaced/i915_vma_misplaced since the conditions
were growing rather cumbersome.
- Add a comment to eb_get_batch explaining why we do this.
- Apply the batch offset bias everywhere but mention that we've only
observed it on gen7 gpus.
- Drop PIN_OFFSET_FIX for now, that slipped in from a feature patch.
v5: Add static to eb_get_batch, spotted by 0-day tester.
Testcase: igt/gem_bad_reloc
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=78533
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk> (v3)
Cc: stable@vger.kernel.org
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2014-05-23 14:48:08 +08:00
|
|
|
if (i915_vma_misplaced(vma, alignment, flags)) {
|
2013-12-07 06:10:55 +08:00
|
|
|
WARN(vma->pin_count,
|
2015-03-16 20:11:13 +08:00
|
|
|
"bo is already pinned in %s with incorrect alignment:"
|
2015-08-08 00:40:17 +08:00
|
|
|
" offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d,"
|
2010-11-05 00:11:09 +08:00
|
|
|
" obj->map_and_fenceable=%d\n",
|
2015-03-16 20:11:13 +08:00
|
|
|
ggtt_view ? "ggtt" : "ppgtt",
|
2015-08-08 00:40:17 +08:00
|
|
|
upper_32_bits(vma->node.start),
|
|
|
|
lower_32_bits(vma->node.start),
|
2014-12-11 01:27:58 +08:00
|
|
|
alignment,
|
drm/i915: Prevent negative relocation deltas from wrapping
This is pure evil. Userspace, I'm looking at you SNA, repacks batch
buffers on the fly after generation as they are being passed to the
kernel for execution. These batches also contain self-referenced
relocations as a single buffer encompasses the state commands, kernels,
vertices and sampler. During generation the buffers are placed at known
offsets within the full batch, and then the relocation deltas (as passed
to the kernel) are tweaked as the batch is repacked into a smaller buffer.
This means that userspace is passing negative relocations deltas, which
subsequently wrap to large values if the batch is at a low address. The
GPU hangs when it then tries to use the large value as a base for its
address offsets, rather than wrapping back to the real value (as one
would hope). As the GPU uses positive offsets from the base, we can
treat the relocation address as the minimum address read by the GPU.
For the upper bound, we trust that userspace will not read beyond the
end of the buffer.
So, how do we fix negative relocations from wrapping? We can either
check that every relocation looks valid when we write it, and then
position each object such that we prevent the offset wraparound, or we
just special-case the self-referential behaviour of SNA and force all
batches to be above 256k. Daniel prefers the latter approach.
This fixes a GPU hang when it tries to use an address (relocation +
offset) greater than the GTT size. The issue would occur quite easily
with full-ppgtt as each fd gets its own VM space, so low offsets would
often be handed out. However, with the rearrangement of the low GTT due
to capturing the BIOS framebuffer, it is already affecting kernels 3.15
onwards. I think only IVB+ is susceptible to this bug, but the workaround
should only kick in rarely, so it seems sensible to always apply it.
v3: Use a bias for batch buffers to prevent small negative delta relocations
from wrapping.
v4 from Daniel:
- s/BIAS/BATCH_OFFSET_BIAS/
- Extract eb_vma_misplaced/i915_vma_misplaced since the conditions
were growing rather cumbersome.
- Add a comment to eb_get_batch explaining why we do this.
- Apply the batch offset bias everywhere but mention that we've only
observed it on gen7 gpus.
- Drop PIN_OFFSET_FIX for now, that slipped in from a feature patch.
v5: Add static to eb_get_batch, spotted by 0-day tester.
Testcase: igt/gem_bad_reloc
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=78533
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk> (v3)
Cc: stable@vger.kernel.org
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2014-05-23 14:48:08 +08:00
|
|
|
!!(flags & PIN_MAPPABLE),
|
2010-11-09 03:18:58 +08:00
|
|
|
obj->map_and_fenceable);
|
drm/i915: plumb VM into bind/unbind code
As alluded to in several patches, and it will be reiterated later... A
VMA is an abstraction for a GEM BO bound into an address space.
Therefore it stands to reason, that the existing bind, and unbind are
the ones which will be the most impacted. This patch implements this,
and updates all callers which weren't already updated in the series
(because it was too messy).
This patch represents the bulk of an earlier, larger patch. I've pulled
out a bunch of things by the request of Daniel. The history is preserved
for posterity with the email convention of ">" One big change from the
original patch aside from a bunch of cropping is I've created an
i915_vma_unbind() function. That is because we always have the VMA
anyway, and doing an extra lookup is useful. There is a caveat, we
retain an i915_gem_object_ggtt_unbind, for the global cases which might
not talk in VMAs.
> drm/i915: plumb VM into object operations
>
> This patch was formerly known as:
> "drm/i915: Create VMAs (part 3) - plumbing"
>
> This patch adds a VM argument, bind/unbind, and the object
> offset/size/color getters/setters. It preserves the old ggtt helper
> functions because things still need, and will continue to need them.
>
> Some code will still need to be ported over after this.
>
> v2: Fix purge to pick an object and unbind all vmas
> This was doable because of the global bound list change.
>
> v3: With the commit to actually pin/unpin pages in place, there is no
> longer a need to check if unbind succeeded before calling put_pages().
> Make put_pages only BUG() after checking pin count.
>
> v4: Rebased on top of the new hangcheck work by Mika
> plumbed eb_destroy also
> Many checkpatch related fixes
>
> v5: Very large rebase
>
> v6:
> Change BUG_ON to WARN_ON (Daniel)
> Rename vm to ggtt in preallocate stolen, since it is always ggtt when
> dealing with stolen memory. (Daniel)
> list_for_each will short-circuit already (Daniel)
> remove superflous space (Daniel)
> Use per object list of vmas (Daniel)
> Make obj_bound_any() use obj_bound for each vm (Ben)
> s/bind_to_gtt/bind_to_vm/ (Ben)
>
> Fixed up the inactive shrinker. As Daniel noticed the code could
> potentially count the same object multiple times. While it's not
> possible in the current case, since 1 object can only ever be bound into
> 1 address space thus far - we may as well try to get something more
> future proof in place now. With a prep patch before this to switch over
> to using the bound list + inactive check, we're now able to carry that
> forward for every address space an object is bound into.
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
[danvet: Rebase on top of the loss of "drm/i915: Cleanup more of VMA
in destroy".]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-08-01 08:00:10 +08:00
|
|
|
ret = i915_vma_unbind(vma);
|
2010-05-27 20:18:18 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
2014-02-14 21:01:21 +08:00
|
|
|
|
|
|
|
vma = NULL;
|
2010-05-27 20:18:18 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-10-31 21:53:52 +08:00
|
|
|
bound = vma ? vma->bound : 0;
|
2014-02-14 21:01:21 +08:00
|
|
|
if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
|
2015-03-16 20:11:13 +08:00
|
|
|
vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,
|
|
|
|
flags);
|
2014-02-14 21:01:20 +08:00
|
|
|
if (IS_ERR(vma))
|
|
|
|
return PTR_ERR(vma);
|
2015-04-21 00:04:05 +08:00
|
|
|
} else {
|
|
|
|
ret = i915_vma_bind(vma, obj->cache_level, flags);
|
2014-12-11 01:27:58 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
}
|
2012-02-16 06:50:22 +08:00
|
|
|
|
2015-05-06 19:33:58 +08:00
|
|
|
if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
|
|
|
|
(bound ^ vma->bound) & GLOBAL_BIND) {
|
2015-11-20 22:16:39 +08:00
|
|
|
__i915_vma_set_map_and_fenceable(vma);
|
2015-05-06 19:33:58 +08:00
|
|
|
WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
|
|
|
|
}
|
2014-10-31 21:53:52 +08:00
|
|
|
|
2014-02-14 21:01:21 +08:00
|
|
|
vma->pin_count++;
|
2008-07-31 03:06:12 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-03-16 20:11:13 +08:00
|
|
|
int
|
|
|
|
i915_gem_object_pin(struct drm_i915_gem_object *obj,
|
|
|
|
struct i915_address_space *vm,
|
|
|
|
uint32_t alignment,
|
|
|
|
uint64_t flags)
|
|
|
|
{
|
|
|
|
return i915_gem_object_do_pin(obj, vm,
|
|
|
|
i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL,
|
|
|
|
alignment, flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
|
|
|
|
const struct i915_ggtt_view *view,
|
|
|
|
uint32_t alignment,
|
|
|
|
uint64_t flags)
|
|
|
|
{
|
2016-03-30 21:57:10 +08:00
|
|
|
struct drm_device *dev = obj->base.dev;
|
|
|
|
struct drm_i915_private *dev_priv = to_i915(dev);
|
|
|
|
struct i915_ggtt *ggtt = &dev_priv->ggtt;
|
|
|
|
|
2016-03-24 23:54:20 +08:00
|
|
|
BUG_ON(!view);
|
2015-03-16 20:11:13 +08:00
|
|
|
|
2016-03-30 21:57:10 +08:00
|
|
|
return i915_gem_object_do_pin(obj, &ggtt->base, view,
|
2015-03-17 23:36:51 +08:00
|
|
|
alignment, flags | PIN_GLOBAL);
|
2015-03-16 20:11:13 +08:00
|
|
|
}
|
|
|
|
|
2008-07-31 03:06:12 +08:00
|
|
|
void
|
2015-03-23 19:10:33 +08:00
|
|
|
i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
|
|
|
|
const struct i915_ggtt_view *view)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
2015-03-23 19:10:33 +08:00
|
|
|
struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2013-12-07 06:10:55 +08:00
|
|
|
BUG_ON(!vma);
|
2015-03-23 19:10:33 +08:00
|
|
|
WARN_ON(vma->pin_count == 0);
|
2015-03-27 19:09:22 +08:00
|
|
|
WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
|
2013-12-07 06:10:55 +08:00
|
|
|
|
2015-04-08 00:28:24 +08:00
|
|
|
--vma->pin_count;
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
i915_gem_busy_ioctl(struct drm_device *dev, void *data,
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_file *file)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
|
|
|
struct drm_i915_gem_busy *args = data;
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_i915_gem_object *obj;
|
2010-09-25 17:19:17 +08:00
|
|
|
int ret;
|
|
|
|
|
2010-09-25 18:22:51 +08:00
|
|
|
ret = i915_mutex_lock_interruptible(dev);
|
2010-10-17 16:45:41 +08:00
|
|
|
if (ret)
|
2010-09-25 18:22:51 +08:00
|
|
|
return ret;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
|
2011-02-19 19:31:06 +08:00
|
|
|
if (&obj->base == NULL) {
|
2010-10-17 16:45:41 +08:00
|
|
|
ret = -ENOENT;
|
|
|
|
goto unlock;
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
2010-05-21 09:08:57 +08:00
|
|
|
|
2010-08-04 22:36:30 +08:00
|
|
|
/* Count all active objects as busy, even if they are currently not used
|
|
|
|
* by the gpu. Users of this interface expect objects to eventually
|
|
|
|
* become non-busy without any further actions, therefore emit any
|
|
|
|
* necessary flushes here.
|
2008-12-15 11:05:04 +08:00
|
|
|
*/
|
2012-06-01 21:21:23 +08:00
|
|
|
ret = i915_gem_object_flush_active(obj);
|
2015-04-27 20:41:17 +08:00
|
|
|
if (ret)
|
|
|
|
goto unref;
|
2010-08-04 22:36:30 +08:00
|
|
|
|
2016-01-16 00:51:46 +08:00
|
|
|
args->busy = 0;
|
|
|
|
if (obj->active) {
|
|
|
|
int i;
|
|
|
|
|
2016-03-16 19:00:39 +08:00
|
|
|
for (i = 0; i < I915_NUM_ENGINES; i++) {
|
2016-01-16 00:51:46 +08:00
|
|
|
struct drm_i915_gem_request *req;
|
|
|
|
|
|
|
|
req = obj->last_read_req[i];
|
|
|
|
if (req)
|
2016-03-16 19:00:38 +08:00
|
|
|
args->busy |= 1 << (16 + req->engine->exec_id);
|
2016-01-16 00:51:46 +08:00
|
|
|
}
|
|
|
|
if (obj->last_write_req)
|
2016-03-16 19:00:38 +08:00
|
|
|
args->busy |= obj->last_write_req->engine->exec_id;
|
2016-01-16 00:51:46 +08:00
|
|
|
}
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2015-04-27 20:41:17 +08:00
|
|
|
unref:
|
2010-11-09 03:18:58 +08:00
|
|
|
drm_gem_object_unreference(&obj->base);
|
2010-10-17 16:45:41 +08:00
|
|
|
unlock:
|
2008-07-31 03:06:12 +08:00
|
|
|
mutex_unlock(&dev->struct_mutex);
|
2010-10-17 16:45:41 +08:00
|
|
|
return ret;
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
|
|
|
|
struct drm_file *file_priv)
|
|
|
|
{
|
2011-08-17 03:34:10 +08:00
|
|
|
return i915_gem_ring_throttle(dev, file_priv);
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
2009-09-14 23:50:29 +08:00
|
|
|
int
|
|
|
|
i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
|
|
|
|
struct drm_file *file_priv)
|
|
|
|
{
|
2014-11-20 16:26:30 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2009-09-14 23:50:29 +08:00
|
|
|
struct drm_i915_gem_madvise *args = data;
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_i915_gem_object *obj;
|
2010-09-25 18:22:51 +08:00
|
|
|
int ret;
|
2009-09-14 23:50:29 +08:00
|
|
|
|
|
|
|
switch (args->madv) {
|
|
|
|
case I915_MADV_DONTNEED:
|
|
|
|
case I915_MADV_WILLNEED:
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2010-10-17 16:45:41 +08:00
|
|
|
ret = i915_mutex_lock_interruptible(dev);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
|
2011-02-19 19:31:06 +08:00
|
|
|
if (&obj->base == NULL) {
|
2010-10-17 16:45:41 +08:00
|
|
|
ret = -ENOENT;
|
|
|
|
goto unlock;
|
2009-09-14 23:50:29 +08:00
|
|
|
}
|
|
|
|
|
2013-12-07 06:10:55 +08:00
|
|
|
if (i915_gem_obj_is_pinned(obj)) {
|
2010-10-17 16:45:41 +08:00
|
|
|
ret = -EINVAL;
|
|
|
|
goto out;
|
2009-09-14 23:50:29 +08:00
|
|
|
}
|
|
|
|
|
2014-11-20 16:26:30 +08:00
|
|
|
if (obj->pages &&
|
|
|
|
obj->tiling_mode != I915_TILING_NONE &&
|
|
|
|
dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
|
|
|
|
if (obj->madv == I915_MADV_WILLNEED)
|
|
|
|
i915_gem_object_unpin_pages(obj);
|
|
|
|
if (args->madv == I915_MADV_WILLNEED)
|
|
|
|
i915_gem_object_pin_pages(obj);
|
|
|
|
}
|
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
if (obj->madv != __I915_MADV_PURGED)
|
|
|
|
obj->madv = args->madv;
|
2009-09-14 23:50:29 +08:00
|
|
|
|
drm/i915: Track unbound pages
When dealing with a working set larger than the GATT, or even the
mappable aperture when touching through the GTT, we end up with evicting
objects only to rebind them at a new offset again later. Moving an
object into and out of the GTT requires clflushing the pages, thus
causing a double-clflush penalty for rebinding.
To avoid having to clflush on rebinding, we can track the pages as they
are evicted from the GTT and only relinquish those pages on memory
pressure.
As usual, if it were not for the handling of out-of-memory condition and
having to manually shrink our own bo caches, it would be a net reduction
of code. Alas.
Note: The patch also contains a few changes to the last-hope
evict_everything logic in i916_gem_execbuffer.c - we no longer try to
only evict the purgeable stuff in a first try (since that's superflous
and only helps in OOM corner-cases, not fragmented-gtt trashing
situations).
Also, the extraction of the get_pages retry loop from bind_to_gtt (and
other callsites) to get_pages should imo have been a separate patch.
v2: Ditch the newly added put_pages (for unbound objects only) in
i915_gem_reset. A quick irc discussion hasn't revealed any important
reason for this, so if we need this, I'd like to have a git blame'able
explanation for it.
v3: Undo the s/drm_malloc_ab/kmalloc/ in get_pages that Chris noticed.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
[danvet: Split out code movements and rant a bit in the commit message
with a few Notes. Done v2]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-08-20 17:40:46 +08:00
|
|
|
/* if the object is no longer attached, discard its backing storage */
|
2015-03-18 17:46:04 +08:00
|
|
|
if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
|
2009-09-21 06:13:10 +08:00
|
|
|
i915_gem_object_truncate(obj);
|
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
args->retained = obj->madv != __I915_MADV_PURGED;
|
2009-09-22 21:24:13 +08:00
|
|
|
|
2010-10-17 16:45:41 +08:00
|
|
|
out:
|
2010-11-09 03:18:58 +08:00
|
|
|
drm_gem_object_unreference(&obj->base);
|
2010-10-17 16:45:41 +08:00
|
|
|
unlock:
|
2009-09-14 23:50:29 +08:00
|
|
|
mutex_unlock(&dev->struct_mutex);
|
2010-10-17 16:45:41 +08:00
|
|
|
return ret;
|
2009-09-14 23:50:29 +08:00
|
|
|
}
|
|
|
|
|
2012-06-07 22:38:42 +08:00
|
|
|
void i915_gem_object_init(struct drm_i915_gem_object *obj,
|
|
|
|
const struct drm_i915_gem_object_ops *ops)
|
2012-08-11 22:41:06 +08:00
|
|
|
{
|
2015-04-27 20:41:17 +08:00
|
|
|
int i;
|
|
|
|
|
2013-06-01 02:28:48 +08:00
|
|
|
INIT_LIST_HEAD(&obj->global_list);
|
2016-03-16 19:00:39 +08:00
|
|
|
for (i = 0; i < I915_NUM_ENGINES; i++)
|
2016-03-16 19:00:40 +08:00
|
|
|
INIT_LIST_HEAD(&obj->engine_list[i]);
|
2013-08-14 17:38:33 +08:00
|
|
|
INIT_LIST_HEAD(&obj->obj_exec_link);
|
2013-07-18 03:19:03 +08:00
|
|
|
INIT_LIST_HEAD(&obj->vma_list);
|
2015-04-07 23:20:38 +08:00
|
|
|
INIT_LIST_HEAD(&obj->batch_pool_link);
|
2012-08-11 22:41:06 +08:00
|
|
|
|
2012-06-07 22:38:42 +08:00
|
|
|
obj->ops = ops;
|
|
|
|
|
2012-08-11 22:41:06 +08:00
|
|
|
obj->fence_reg = I915_FENCE_REG_NONE;
|
|
|
|
obj->madv = I915_MADV_WILLNEED;
|
|
|
|
|
|
|
|
i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
|
|
|
|
}
|
|
|
|
|
2012-06-07 22:38:42 +08:00
|
|
|
static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
|
2016-01-23 02:32:31 +08:00
|
|
|
.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
|
2012-06-07 22:38:42 +08:00
|
|
|
.get_pages = i915_gem_object_get_pages_gtt,
|
|
|
|
.put_pages = i915_gem_object_put_pages_gtt,
|
|
|
|
};
|
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
|
|
|
|
size_t size)
|
2010-04-10 03:05:06 +08:00
|
|
|
{
|
2010-04-10 03:05:07 +08:00
|
|
|
struct drm_i915_gem_object *obj;
|
2011-06-28 07:18:18 +08:00
|
|
|
struct address_space *mapping;
|
2012-11-30 05:18:51 +08:00
|
|
|
gfp_t mask;
|
2010-04-10 03:05:06 +08:00
|
|
|
|
2012-11-15 19:32:30 +08:00
|
|
|
obj = i915_gem_object_alloc(dev);
|
2010-04-10 03:05:07 +08:00
|
|
|
if (obj == NULL)
|
|
|
|
return NULL;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2010-04-10 03:05:07 +08:00
|
|
|
if (drm_gem_object_init(dev, &obj->base, size) != 0) {
|
2012-11-15 19:32:30 +08:00
|
|
|
i915_gem_object_free(obj);
|
2010-04-10 03:05:07 +08:00
|
|
|
return NULL;
|
|
|
|
}
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2012-05-25 03:48:12 +08:00
|
|
|
mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
|
|
|
|
if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
|
|
|
|
/* 965gm cannot relocate objects above 4GiB. */
|
|
|
|
mask &= ~__GFP_HIGHMEM;
|
|
|
|
mask |= __GFP_DMA32;
|
|
|
|
}
|
|
|
|
|
2013-01-24 06:07:38 +08:00
|
|
|
mapping = file_inode(obj->base.filp)->i_mapping;
|
2012-05-25 03:48:12 +08:00
|
|
|
mapping_set_gfp_mask(mapping, mask);
|
2011-06-28 07:18:18 +08:00
|
|
|
|
2012-06-07 22:38:42 +08:00
|
|
|
i915_gem_object_init(obj, &i915_gem_object_ops);
|
2010-09-30 18:46:12 +08:00
|
|
|
|
2010-04-10 03:05:07 +08:00
|
|
|
obj->base.write_domain = I915_GEM_DOMAIN_CPU;
|
|
|
|
obj->base.read_domains = I915_GEM_DOMAIN_CPU;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2012-01-18 00:43:53 +08:00
|
|
|
if (HAS_LLC(dev)) {
|
|
|
|
/* On some devices, we can have the GPU use the LLC (the CPU
|
2011-03-30 07:59:55 +08:00
|
|
|
* cache) for about a 10% performance improvement
|
|
|
|
* compared to uncached. Graphics requests other than
|
|
|
|
* display scanout are coherent with the CPU in
|
|
|
|
* accessing this cache. This means in this mode we
|
|
|
|
* don't need to clflush on the CPU side, and on the
|
|
|
|
* GPU side we only need to flush internal caches to
|
|
|
|
* get data visible to the CPU.
|
|
|
|
*
|
|
|
|
* However, we maintain the display planes as UC, and so
|
|
|
|
* need to rebind when first used as such.
|
|
|
|
*/
|
|
|
|
obj->cache_level = I915_CACHE_LLC;
|
|
|
|
} else
|
|
|
|
obj->cache_level = I915_CACHE_NONE;
|
|
|
|
|
2013-07-25 05:25:03 +08:00
|
|
|
trace_i915_gem_object_create(obj);
|
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
return obj;
|
2010-04-10 03:05:07 +08:00
|
|
|
}
|
|
|
|
|
2014-05-22 16:16:52 +08:00
|
|
|
static bool discard_backing_storage(struct drm_i915_gem_object *obj)
|
|
|
|
{
|
|
|
|
/* If we are the last user of the backing storage (be it shmemfs
|
|
|
|
* pages or stolen etc), we know that the pages are going to be
|
|
|
|
* immediately released. In this case, we can then skip copying
|
|
|
|
* back the contents from the GPU.
|
|
|
|
*/
|
|
|
|
|
|
|
|
if (obj->madv != I915_MADV_WILLNEED)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (obj->base.filp == NULL)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
/* At first glance, this looks racy, but then again so would be
|
|
|
|
* userspace racing mmap against close. However, the first external
|
|
|
|
* reference to the filp can only be obtained through the
|
|
|
|
* i915_gem_mmap_ioctl() which safeguards us against the user
|
|
|
|
* acquiring such a reference whilst we are in the middle of
|
|
|
|
* freeing the object.
|
|
|
|
*/
|
|
|
|
return atomic_long_read(&obj->base.filp->f_count) == 1;
|
|
|
|
}
|
|
|
|
|
2012-04-24 22:47:31 +08:00
|
|
|
void i915_gem_free_object(struct drm_gem_object *gem_obj)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
2012-04-24 22:47:31 +08:00
|
|
|
struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
|
2010-11-09 03:18:58 +08:00
|
|
|
struct drm_device *dev = obj->base.dev;
|
2014-03-31 19:27:16 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
drm/i915: plumb VM into bind/unbind code
As alluded to in several patches, and it will be reiterated later... A
VMA is an abstraction for a GEM BO bound into an address space.
Therefore it stands to reason, that the existing bind, and unbind are
the ones which will be the most impacted. This patch implements this,
and updates all callers which weren't already updated in the series
(because it was too messy).
This patch represents the bulk of an earlier, larger patch. I've pulled
out a bunch of things by the request of Daniel. The history is preserved
for posterity with the email convention of ">" One big change from the
original patch aside from a bunch of cropping is I've created an
i915_vma_unbind() function. That is because we always have the VMA
anyway, and doing an extra lookup is useful. There is a caveat, we
retain an i915_gem_object_ggtt_unbind, for the global cases which might
not talk in VMAs.
> drm/i915: plumb VM into object operations
>
> This patch was formerly known as:
> "drm/i915: Create VMAs (part 3) - plumbing"
>
> This patch adds a VM argument, bind/unbind, and the object
> offset/size/color getters/setters. It preserves the old ggtt helper
> functions because things still need, and will continue to need them.
>
> Some code will still need to be ported over after this.
>
> v2: Fix purge to pick an object and unbind all vmas
> This was doable because of the global bound list change.
>
> v3: With the commit to actually pin/unpin pages in place, there is no
> longer a need to check if unbind succeeded before calling put_pages().
> Make put_pages only BUG() after checking pin count.
>
> v4: Rebased on top of the new hangcheck work by Mika
> plumbed eb_destroy also
> Many checkpatch related fixes
>
> v5: Very large rebase
>
> v6:
> Change BUG_ON to WARN_ON (Daniel)
> Rename vm to ggtt in preallocate stolen, since it is always ggtt when
> dealing with stolen memory. (Daniel)
> list_for_each will short-circuit already (Daniel)
> remove superflous space (Daniel)
> Use per object list of vmas (Daniel)
> Make obj_bound_any() use obj_bound for each vm (Ben)
> s/bind_to_gtt/bind_to_vm/ (Ben)
>
> Fixed up the inactive shrinker. As Daniel noticed the code could
> potentially count the same object multiple times. While it's not
> possible in the current case, since 1 object can only ever be bound into
> 1 address space thus far - we may as well try to get something more
> future proof in place now. With a prep patch before this to switch over
> to using the bound list + inactive check, we're now able to carry that
> forward for every address space an object is bound into.
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
[danvet: Rebase on top of the loss of "drm/i915: Cleanup more of VMA
in destroy".]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-08-01 08:00:10 +08:00
|
|
|
struct i915_vma *vma, *next;
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2013-11-28 04:20:34 +08:00
|
|
|
intel_runtime_pm_get(dev_priv);
|
|
|
|
|
2011-03-20 19:20:19 +08:00
|
|
|
trace_i915_gem_object_destroy(obj);
|
|
|
|
|
2016-02-26 19:03:19 +08:00
|
|
|
list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
|
2013-12-07 06:10:55 +08:00
|
|
|
int ret;
|
|
|
|
|
|
|
|
vma->pin_count = 0;
|
|
|
|
ret = i915_vma_unbind(vma);
|
drm/i915: plumb VM into bind/unbind code
As alluded to in several patches, and it will be reiterated later... A
VMA is an abstraction for a GEM BO bound into an address space.
Therefore it stands to reason, that the existing bind, and unbind are
the ones which will be the most impacted. This patch implements this,
and updates all callers which weren't already updated in the series
(because it was too messy).
This patch represents the bulk of an earlier, larger patch. I've pulled
out a bunch of things by the request of Daniel. The history is preserved
for posterity with the email convention of ">" One big change from the
original patch aside from a bunch of cropping is I've created an
i915_vma_unbind() function. That is because we always have the VMA
anyway, and doing an extra lookup is useful. There is a caveat, we
retain an i915_gem_object_ggtt_unbind, for the global cases which might
not talk in VMAs.
> drm/i915: plumb VM into object operations
>
> This patch was formerly known as:
> "drm/i915: Create VMAs (part 3) - plumbing"
>
> This patch adds a VM argument, bind/unbind, and the object
> offset/size/color getters/setters. It preserves the old ggtt helper
> functions because things still need, and will continue to need them.
>
> Some code will still need to be ported over after this.
>
> v2: Fix purge to pick an object and unbind all vmas
> This was doable because of the global bound list change.
>
> v3: With the commit to actually pin/unpin pages in place, there is no
> longer a need to check if unbind succeeded before calling put_pages().
> Make put_pages only BUG() after checking pin count.
>
> v4: Rebased on top of the new hangcheck work by Mika
> plumbed eb_destroy also
> Many checkpatch related fixes
>
> v5: Very large rebase
>
> v6:
> Change BUG_ON to WARN_ON (Daniel)
> Rename vm to ggtt in preallocate stolen, since it is always ggtt when
> dealing with stolen memory. (Daniel)
> list_for_each will short-circuit already (Daniel)
> remove superflous space (Daniel)
> Use per object list of vmas (Daniel)
> Make obj_bound_any() use obj_bound for each vm (Ben)
> s/bind_to_gtt/bind_to_vm/ (Ben)
>
> Fixed up the inactive shrinker. As Daniel noticed the code could
> potentially count the same object multiple times. While it's not
> possible in the current case, since 1 object can only ever be bound into
> 1 address space thus far - we may as well try to get something more
> future proof in place now. With a prep patch before this to switch over
> to using the bound list + inactive check, we're now able to carry that
> forward for every address space an object is bound into.
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
[danvet: Rebase on top of the loss of "drm/i915: Cleanup more of VMA
in destroy".]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-08-01 08:00:10 +08:00
|
|
|
if (WARN_ON(ret == -ERESTARTSYS)) {
|
|
|
|
bool was_interruptible;
|
2012-04-24 22:47:31 +08:00
|
|
|
|
drm/i915: plumb VM into bind/unbind code
As alluded to in several patches, and it will be reiterated later... A
VMA is an abstraction for a GEM BO bound into an address space.
Therefore it stands to reason, that the existing bind, and unbind are
the ones which will be the most impacted. This patch implements this,
and updates all callers which weren't already updated in the series
(because it was too messy).
This patch represents the bulk of an earlier, larger patch. I've pulled
out a bunch of things by the request of Daniel. The history is preserved
for posterity with the email convention of ">" One big change from the
original patch aside from a bunch of cropping is I've created an
i915_vma_unbind() function. That is because we always have the VMA
anyway, and doing an extra lookup is useful. There is a caveat, we
retain an i915_gem_object_ggtt_unbind, for the global cases which might
not talk in VMAs.
> drm/i915: plumb VM into object operations
>
> This patch was formerly known as:
> "drm/i915: Create VMAs (part 3) - plumbing"
>
> This patch adds a VM argument, bind/unbind, and the object
> offset/size/color getters/setters. It preserves the old ggtt helper
> functions because things still need, and will continue to need them.
>
> Some code will still need to be ported over after this.
>
> v2: Fix purge to pick an object and unbind all vmas
> This was doable because of the global bound list change.
>
> v3: With the commit to actually pin/unpin pages in place, there is no
> longer a need to check if unbind succeeded before calling put_pages().
> Make put_pages only BUG() after checking pin count.
>
> v4: Rebased on top of the new hangcheck work by Mika
> plumbed eb_destroy also
> Many checkpatch related fixes
>
> v5: Very large rebase
>
> v6:
> Change BUG_ON to WARN_ON (Daniel)
> Rename vm to ggtt in preallocate stolen, since it is always ggtt when
> dealing with stolen memory. (Daniel)
> list_for_each will short-circuit already (Daniel)
> remove superflous space (Daniel)
> Use per object list of vmas (Daniel)
> Make obj_bound_any() use obj_bound for each vm (Ben)
> s/bind_to_gtt/bind_to_vm/ (Ben)
>
> Fixed up the inactive shrinker. As Daniel noticed the code could
> potentially count the same object multiple times. While it's not
> possible in the current case, since 1 object can only ever be bound into
> 1 address space thus far - we may as well try to get something more
> future proof in place now. With a prep patch before this to switch over
> to using the bound list + inactive check, we're now able to carry that
> forward for every address space an object is bound into.
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
[danvet: Rebase on top of the loss of "drm/i915: Cleanup more of VMA
in destroy".]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-08-01 08:00:10 +08:00
|
|
|
was_interruptible = dev_priv->mm.interruptible;
|
|
|
|
dev_priv->mm.interruptible = false;
|
2012-04-24 22:47:31 +08:00
|
|
|
|
drm/i915: plumb VM into bind/unbind code
As alluded to in several patches, and it will be reiterated later... A
VMA is an abstraction for a GEM BO bound into an address space.
Therefore it stands to reason, that the existing bind, and unbind are
the ones which will be the most impacted. This patch implements this,
and updates all callers which weren't already updated in the series
(because it was too messy).
This patch represents the bulk of an earlier, larger patch. I've pulled
out a bunch of things by the request of Daniel. The history is preserved
for posterity with the email convention of ">" One big change from the
original patch aside from a bunch of cropping is I've created an
i915_vma_unbind() function. That is because we always have the VMA
anyway, and doing an extra lookup is useful. There is a caveat, we
retain an i915_gem_object_ggtt_unbind, for the global cases which might
not talk in VMAs.
> drm/i915: plumb VM into object operations
>
> This patch was formerly known as:
> "drm/i915: Create VMAs (part 3) - plumbing"
>
> This patch adds a VM argument, bind/unbind, and the object
> offset/size/color getters/setters. It preserves the old ggtt helper
> functions because things still need, and will continue to need them.
>
> Some code will still need to be ported over after this.
>
> v2: Fix purge to pick an object and unbind all vmas
> This was doable because of the global bound list change.
>
> v3: With the commit to actually pin/unpin pages in place, there is no
> longer a need to check if unbind succeeded before calling put_pages().
> Make put_pages only BUG() after checking pin count.
>
> v4: Rebased on top of the new hangcheck work by Mika
> plumbed eb_destroy also
> Many checkpatch related fixes
>
> v5: Very large rebase
>
> v6:
> Change BUG_ON to WARN_ON (Daniel)
> Rename vm to ggtt in preallocate stolen, since it is always ggtt when
> dealing with stolen memory. (Daniel)
> list_for_each will short-circuit already (Daniel)
> remove superflous space (Daniel)
> Use per object list of vmas (Daniel)
> Make obj_bound_any() use obj_bound for each vm (Ben)
> s/bind_to_gtt/bind_to_vm/ (Ben)
>
> Fixed up the inactive shrinker. As Daniel noticed the code could
> potentially count the same object multiple times. While it's not
> possible in the current case, since 1 object can only ever be bound into
> 1 address space thus far - we may as well try to get something more
> future proof in place now. With a prep patch before this to switch over
> to using the bound list + inactive check, we're now able to carry that
> forward for every address space an object is bound into.
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
[danvet: Rebase on top of the loss of "drm/i915: Cleanup more of VMA
in destroy".]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-08-01 08:00:10 +08:00
|
|
|
WARN_ON(i915_vma_unbind(vma));
|
2012-04-24 22:47:31 +08:00
|
|
|
|
drm/i915: plumb VM into bind/unbind code
As alluded to in several patches, and it will be reiterated later... A
VMA is an abstraction for a GEM BO bound into an address space.
Therefore it stands to reason, that the existing bind, and unbind are
the ones which will be the most impacted. This patch implements this,
and updates all callers which weren't already updated in the series
(because it was too messy).
This patch represents the bulk of an earlier, larger patch. I've pulled
out a bunch of things by the request of Daniel. The history is preserved
for posterity with the email convention of ">" One big change from the
original patch aside from a bunch of cropping is I've created an
i915_vma_unbind() function. That is because we always have the VMA
anyway, and doing an extra lookup is useful. There is a caveat, we
retain an i915_gem_object_ggtt_unbind, for the global cases which might
not talk in VMAs.
> drm/i915: plumb VM into object operations
>
> This patch was formerly known as:
> "drm/i915: Create VMAs (part 3) - plumbing"
>
> This patch adds a VM argument, bind/unbind, and the object
> offset/size/color getters/setters. It preserves the old ggtt helper
> functions because things still need, and will continue to need them.
>
> Some code will still need to be ported over after this.
>
> v2: Fix purge to pick an object and unbind all vmas
> This was doable because of the global bound list change.
>
> v3: With the commit to actually pin/unpin pages in place, there is no
> longer a need to check if unbind succeeded before calling put_pages().
> Make put_pages only BUG() after checking pin count.
>
> v4: Rebased on top of the new hangcheck work by Mika
> plumbed eb_destroy also
> Many checkpatch related fixes
>
> v5: Very large rebase
>
> v6:
> Change BUG_ON to WARN_ON (Daniel)
> Rename vm to ggtt in preallocate stolen, since it is always ggtt when
> dealing with stolen memory. (Daniel)
> list_for_each will short-circuit already (Daniel)
> remove superflous space (Daniel)
> Use per object list of vmas (Daniel)
> Make obj_bound_any() use obj_bound for each vm (Ben)
> s/bind_to_gtt/bind_to_vm/ (Ben)
>
> Fixed up the inactive shrinker. As Daniel noticed the code could
> potentially count the same object multiple times. While it's not
> possible in the current case, since 1 object can only ever be bound into
> 1 address space thus far - we may as well try to get something more
> future proof in place now. With a prep patch before this to switch over
> to using the bound list + inactive check, we're now able to carry that
> forward for every address space an object is bound into.
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
[danvet: Rebase on top of the loss of "drm/i915: Cleanup more of VMA
in destroy".]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-08-01 08:00:10 +08:00
|
|
|
dev_priv->mm.interruptible = was_interruptible;
|
|
|
|
}
|
2012-04-24 22:47:31 +08:00
|
|
|
}
|
|
|
|
|
2013-06-01 05:46:20 +08:00
|
|
|
/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
|
|
|
|
* before progressing. */
|
|
|
|
if (obj->stolen)
|
|
|
|
i915_gem_object_unpin_pages(obj);
|
|
|
|
|
2014-06-19 05:28:09 +08:00
|
|
|
WARN_ON(obj->frontbuffer_bits);
|
|
|
|
|
2014-11-20 16:26:30 +08:00
|
|
|
if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
|
|
|
|
dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
|
|
|
|
obj->tiling_mode != I915_TILING_NONE)
|
|
|
|
i915_gem_object_unpin_pages(obj);
|
|
|
|
|
2013-06-01 02:28:47 +08:00
|
|
|
if (WARN_ON(obj->pages_pin_count))
|
|
|
|
obj->pages_pin_count = 0;
|
2014-05-22 16:16:52 +08:00
|
|
|
if (discard_backing_storage(obj))
|
2014-03-25 21:23:06 +08:00
|
|
|
obj->madv = I915_MADV_DONTNEED;
|
2012-06-07 22:38:42 +08:00
|
|
|
i915_gem_object_put_pages(obj);
|
2012-08-11 22:41:03 +08:00
|
|
|
i915_gem_object_free_mmap_offset(obj);
|
2008-11-13 02:03:55 +08:00
|
|
|
|
2012-06-01 22:20:22 +08:00
|
|
|
BUG_ON(obj->pages);
|
|
|
|
|
2012-09-05 04:02:58 +08:00
|
|
|
if (obj->base.import_attach)
|
|
|
|
drm_prime_gem_destroy(&obj->base, NULL);
|
2008-11-13 02:03:55 +08:00
|
|
|
|
drm/i915: Introduce mapping of user pages into video memory (userptr) ioctl
By exporting the ability to map user address and inserting PTEs
representing their backing pages into the GTT, we can exploit UMA in order
to utilize normal application data as a texture source or even as a
render target (depending upon the capabilities of the chipset). This has
a number of uses, with zero-copy downloads to the GPU and efficient
readback making the intermixed streaming of CPU and GPU operations
fairly efficient. This ability has many widespread implications from
faster rendering of client-side software rasterisers (chromium),
mitigation of stalls due to read back (firefox) and to faster pipelining
of texture data (such as pixel buffer objects in GL or data blobs in CL).
v2: Compile with CONFIG_MMU_NOTIFIER
v3: We can sleep while performing invalidate-range, which we can utilise
to drop our page references prior to the kernel manipulating the vma
(for either discard or cloning) and so protect normal users.
v4: Only run the invalidate notifier if the range intercepts the bo.
v5: Prevent userspace from attempting to GTT mmap non-page aligned buffers
v6: Recheck after reacquire mutex for lost mmu.
v7: Fix implicit padding of ioctl struct by rounding to next 64bit boundary.
v8: Fix rebasing error after forwarding porting the back port.
v9: Limit the userptr to page aligned entries. We now expect userspace
to handle all the offset-in-page adjustments itself.
v10: Prevent vma from being copied across fork to avoid issues with cow.
v11: Drop vma behaviour changes -- locking is nigh on impossible.
Use a worker to load user pages to avoid lock inversions.
v12: Use get_task_mm()/mmput() for correct refcounting of mm.
v13: Use a worker to release the mmu_notifier to avoid lock inversion
v14: Decouple mmu_notifier from struct_mutex using a custom mmu_notifer
with its own locking and tree of objects for each mm/mmu_notifier.
v15: Prevent overlapping userptr objects, and invalidate all objects
within the mmu_notifier range
v16: Fix a typo for iterating over multiple objects in the range and
rearrange error path to destroy the mmu_notifier locklessly.
Also close a race between invalidate_range and the get_pages_worker.
v17: Close a race between get_pages_worker/invalidate_range and fresh
allocations of the same userptr range - and notice that
struct_mutex was presumed to be held when during creation it wasn't.
v18: Sigh. Fix the refactor of st_set_pages() to allocate enough memory
for the struct sg_table and to clear it before reporting an error.
v19: Always error out on read-only userptr requests as we don't have the
hardware infrastructure to support them at the moment.
v20: Refuse to implement read-only support until we have the required
infrastructure - but reserve the bit in flags for future use.
v21: use_mm() is not required for get_user_pages(). It is only meant to
be used to fix up the kernel thread's current->mm for use with
copy_user().
v22: Use sg_alloc_table_from_pages for that chunky feeling
v23: Export a function for sanity checking dma-buf rather than encode
userptr details elsewhere, and clean up comments based on
suggestions by Bradley.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Cc: "Gong, Zhipeng" <zhipeng.gong@intel.com>
Cc: Akash Goel <akash.goel@intel.com>
Cc: "Volkin, Bradley D" <bradley.d.volkin@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Reviewed-by: Brad Volkin <bradley.d.volkin@intel.com>
[danvet: Frob ioctl allocation to pick the next one - will cause a bit
of fuss with create2 apparently, but such are the rules.]
[danvet2: oops, forgot to git add after manual patch application]
[danvet3: Appease sparse.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2014-05-16 21:22:37 +08:00
|
|
|
if (obj->ops->release)
|
|
|
|
obj->ops->release(obj);
|
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
drm_gem_object_release(&obj->base);
|
|
|
|
i915_gem_info_remove_obj(dev_priv, obj->base.size);
|
2010-04-10 03:05:07 +08:00
|
|
|
|
2010-11-09 03:18:58 +08:00
|
|
|
kfree(obj->bit_17);
|
2012-11-15 19:32:30 +08:00
|
|
|
i915_gem_object_free(obj);
|
2013-11-28 04:20:34 +08:00
|
|
|
|
|
|
|
intel_runtime_pm_put(dev_priv);
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
2015-03-16 20:11:13 +08:00
|
|
|
struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
|
|
|
|
struct i915_address_space *vm)
|
2013-08-14 20:14:04 +08:00
|
|
|
{
|
|
|
|
struct i915_vma *vma;
|
2016-02-26 19:03:19 +08:00
|
|
|
list_for_each_entry(vma, &obj->vma_list, obj_link) {
|
2015-11-12 19:59:55 +08:00
|
|
|
if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL &&
|
|
|
|
vma->vm == vm)
|
2013-08-14 20:14:04 +08:00
|
|
|
return vma;
|
2015-03-16 20:11:13 +08:00
|
|
|
}
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
|
|
|
|
const struct i915_ggtt_view *view)
|
|
|
|
{
|
2016-03-30 21:57:10 +08:00
|
|
|
struct drm_device *dev = obj->base.dev;
|
|
|
|
struct drm_i915_private *dev_priv = to_i915(dev);
|
|
|
|
struct i915_ggtt *ggtt = &dev_priv->ggtt;
|
2015-03-16 20:11:13 +08:00
|
|
|
struct i915_vma *vma;
|
2013-08-14 20:14:04 +08:00
|
|
|
|
2016-03-24 23:54:20 +08:00
|
|
|
BUG_ON(!view);
|
2015-03-16 20:11:13 +08:00
|
|
|
|
2016-02-26 19:03:19 +08:00
|
|
|
list_for_each_entry(vma, &obj->vma_list, obj_link)
|
2016-03-30 21:57:10 +08:00
|
|
|
if (vma->vm == &ggtt->base &&
|
2015-03-27 19:09:22 +08:00
|
|
|
i915_ggtt_view_equal(&vma->ggtt_view, view))
|
2015-03-16 20:11:13 +08:00
|
|
|
return vma;
|
2013-08-14 20:14:04 +08:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2013-07-18 03:19:03 +08:00
|
|
|
void i915_gem_vma_destroy(struct i915_vma *vma)
|
|
|
|
{
|
|
|
|
WARN_ON(vma->node.allocated);
|
2013-08-20 19:56:40 +08:00
|
|
|
|
|
|
|
/* Keep the vma as a placeholder in the execbuffer reservation lists */
|
|
|
|
if (!list_empty(&vma->exec_list))
|
|
|
|
return;
|
|
|
|
|
2016-02-26 19:03:20 +08:00
|
|
|
if (!vma->is_ggtt)
|
|
|
|
i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
|
2014-08-06 21:04:44 +08:00
|
|
|
|
2016-02-26 19:03:19 +08:00
|
|
|
list_del(&vma->obj_link);
|
2013-08-26 17:23:47 +08:00
|
|
|
|
2015-04-07 23:20:58 +08:00
|
|
|
kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
|
2013-07-18 03:19:03 +08:00
|
|
|
}
|
|
|
|
|
2014-04-09 16:19:41 +08:00
|
|
|
static void
|
2016-03-16 19:00:40 +08:00
|
|
|
i915_gem_stop_engines(struct drm_device *dev)
|
2014-04-09 16:19:41 +08:00
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2016-03-16 19:00:36 +08:00
|
|
|
struct intel_engine_cs *engine;
|
2014-04-09 16:19:41 +08:00
|
|
|
|
2016-03-24 19:20:38 +08:00
|
|
|
for_each_engine(engine, dev_priv)
|
2016-03-16 19:00:40 +08:00
|
|
|
dev_priv->gt.stop_engine(engine);
|
2014-04-09 16:19:41 +08:00
|
|
|
}
|
|
|
|
|
2010-01-07 18:39:13 +08:00
|
|
|
int
|
2013-10-16 18:50:01 +08:00
|
|
|
i915_gem_suspend(struct drm_device *dev)
|
2010-01-07 18:39:13 +08:00
|
|
|
{
|
2014-03-31 19:27:16 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2013-10-16 18:50:01 +08:00
|
|
|
int ret = 0;
|
2008-11-14 07:00:55 +08:00
|
|
|
|
2013-10-16 18:50:01 +08:00
|
|
|
mutex_lock(&dev->struct_mutex);
|
2012-04-27 07:02:58 +08:00
|
|
|
ret = i915_gpu_idle(dev);
|
2013-09-14 06:57:04 +08:00
|
|
|
if (ret)
|
2013-10-16 18:50:01 +08:00
|
|
|
goto err;
|
2013-09-14 06:57:04 +08:00
|
|
|
|
2012-04-27 07:02:58 +08:00
|
|
|
i915_gem_retire_requests(dev);
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2016-03-16 19:00:40 +08:00
|
|
|
i915_gem_stop_engines(dev);
|
2013-10-16 18:50:01 +08:00
|
|
|
mutex_unlock(&dev->struct_mutex);
|
|
|
|
|
2015-01-27 00:03:03 +08:00
|
|
|
cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
|
2010-01-07 18:39:13 +08:00
|
|
|
cancel_delayed_work_sync(&dev_priv->mm.retire_work);
|
2014-08-05 22:51:20 +08:00
|
|
|
flush_delayed_work(&dev_priv->mm.idle_work);
|
2010-01-07 18:39:13 +08:00
|
|
|
|
2014-11-25 19:56:33 +08:00
|
|
|
/* Assert that we sucessfully flushed all the work and
|
|
|
|
* reset the GPU back to its idle, low power state.
|
|
|
|
*/
|
|
|
|
WARN_ON(dev_priv->mm.busy);
|
|
|
|
|
2008-07-31 03:06:12 +08:00
|
|
|
return 0;
|
2013-10-16 18:50:01 +08:00
|
|
|
|
|
|
|
err:
|
|
|
|
mutex_unlock(&dev->struct_mutex);
|
|
|
|
return ret;
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
|
|
|
|
2015-05-30 00:43:51 +08:00
|
|
|
int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice)
|
2012-05-26 07:56:24 +08:00
|
|
|
{
|
2016-03-16 19:00:38 +08:00
|
|
|
struct intel_engine_cs *engine = req->engine;
|
2016-03-16 19:00:36 +08:00
|
|
|
struct drm_device *dev = engine->dev;
|
2014-03-31 19:27:16 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2013-09-20 02:13:41 +08:00
|
|
|
u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
|
2013-09-18 12:12:44 +08:00
|
|
|
int i, ret;
|
2012-05-26 07:56:24 +08:00
|
|
|
|
2013-09-20 02:01:40 +08:00
|
|
|
if (!HAS_L3_DPF(dev) || !remap_info)
|
2013-09-18 12:12:44 +08:00
|
|
|
return 0;
|
2012-05-26 07:56:24 +08:00
|
|
|
|
2015-05-30 00:44:07 +08:00
|
|
|
ret = intel_ring_begin(req, GEN7_L3LOG_SIZE / 4 * 3);
|
2013-09-18 12:12:44 +08:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
2012-05-26 07:56:24 +08:00
|
|
|
|
2013-09-18 12:12:44 +08:00
|
|
|
/*
|
|
|
|
* Note: We do not worry about the concurrent register cacheline hang
|
|
|
|
* here because no other code should access these registers other than
|
|
|
|
* at initialization time.
|
|
|
|
*/
|
2015-11-05 05:20:02 +08:00
|
|
|
for (i = 0; i < GEN7_L3LOG_SIZE / 4; i++) {
|
2016-03-16 19:00:36 +08:00
|
|
|
intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(1));
|
|
|
|
intel_ring_emit_reg(engine, GEN7_L3LOG(slice, i));
|
|
|
|
intel_ring_emit(engine, remap_info[i]);
|
2012-05-26 07:56:24 +08:00
|
|
|
}
|
|
|
|
|
2016-03-16 19:00:36 +08:00
|
|
|
intel_ring_advance(engine);
|
2012-05-26 07:56:24 +08:00
|
|
|
|
2013-09-18 12:12:44 +08:00
|
|
|
return ret;
|
2012-05-26 07:56:24 +08:00
|
|
|
}
|
|
|
|
|
2012-02-02 16:58:12 +08:00
|
|
|
void i915_gem_init_swizzling(struct drm_device *dev)
|
|
|
|
{
|
2014-03-31 19:27:16 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2012-02-02 16:58:12 +08:00
|
|
|
|
2012-01-31 23:47:55 +08:00
|
|
|
if (INTEL_INFO(dev)->gen < 5 ||
|
2012-02-02 16:58:12 +08:00
|
|
|
dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
|
|
|
|
return;
|
|
|
|
|
|
|
|
I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
|
|
|
|
DISP_TILE_SURFACE_SWIZZLING);
|
|
|
|
|
2012-01-31 23:47:55 +08:00
|
|
|
if (IS_GEN5(dev))
|
|
|
|
return;
|
|
|
|
|
2012-02-02 16:58:12 +08:00
|
|
|
I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
|
|
|
|
if (IS_GEN6(dev))
|
2012-04-24 20:04:12 +08:00
|
|
|
I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
|
2012-12-19 02:31:23 +08:00
|
|
|
else if (IS_GEN7(dev))
|
2012-04-24 20:04:12 +08:00
|
|
|
I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
|
2013-11-03 12:07:04 +08:00
|
|
|
else if (IS_GEN8(dev))
|
|
|
|
I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
|
2012-12-19 02:31:23 +08:00
|
|
|
else
|
|
|
|
BUG();
|
2012-02-02 16:58:12 +08:00
|
|
|
}
|
2012-02-10 03:53:27 +08:00
|
|
|
|
2014-08-15 06:21:55 +08:00
|
|
|
static void init_unused_ring(struct drm_device *dev, u32 base)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
|
|
|
|
|
|
|
I915_WRITE(RING_CTL(base), 0);
|
|
|
|
I915_WRITE(RING_HEAD(base), 0);
|
|
|
|
I915_WRITE(RING_TAIL(base), 0);
|
|
|
|
I915_WRITE(RING_START(base), 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void init_unused_rings(struct drm_device *dev)
|
|
|
|
{
|
|
|
|
if (IS_I830(dev)) {
|
|
|
|
init_unused_ring(dev, PRB1_BASE);
|
|
|
|
init_unused_ring(dev, SRB0_BASE);
|
|
|
|
init_unused_ring(dev, SRB1_BASE);
|
|
|
|
init_unused_ring(dev, SRB2_BASE);
|
|
|
|
init_unused_ring(dev, SRB3_BASE);
|
|
|
|
} else if (IS_GEN2(dev)) {
|
|
|
|
init_unused_ring(dev, SRB0_BASE);
|
|
|
|
init_unused_ring(dev, SRB1_BASE);
|
|
|
|
} else if (IS_GEN3(dev)) {
|
|
|
|
init_unused_ring(dev, PRB1_BASE);
|
|
|
|
init_unused_ring(dev, PRB2_BASE);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-03-16 19:00:40 +08:00
|
|
|
int i915_gem_init_engines(struct drm_device *dev)
|
2010-05-21 09:08:55 +08:00
|
|
|
{
|
2013-02-09 03:49:24 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2010-05-21 09:08:55 +08:00
|
|
|
int ret;
|
2010-05-27 20:18:22 +08:00
|
|
|
|
2010-09-16 10:43:11 +08:00
|
|
|
ret = intel_init_render_ring_buffer(dev);
|
2010-05-27 20:18:22 +08:00
|
|
|
if (ret)
|
2010-11-12 18:46:37 +08:00
|
|
|
return ret;
|
2010-05-27 20:18:22 +08:00
|
|
|
|
|
|
|
if (HAS_BSD(dev)) {
|
2010-09-16 10:43:11 +08:00
|
|
|
ret = intel_init_bsd_ring_buffer(dev);
|
2010-05-27 20:18:22 +08:00
|
|
|
if (ret)
|
|
|
|
goto cleanup_render_ring;
|
2010-05-21 09:08:57 +08:00
|
|
|
}
|
2010-05-27 20:18:22 +08:00
|
|
|
|
2015-10-07 16:17:44 +08:00
|
|
|
if (HAS_BLT(dev)) {
|
2010-10-19 18:19:32 +08:00
|
|
|
ret = intel_init_blt_ring_buffer(dev);
|
|
|
|
if (ret)
|
|
|
|
goto cleanup_bsd_ring;
|
|
|
|
}
|
|
|
|
|
2013-05-29 10:22:23 +08:00
|
|
|
if (HAS_VEBOX(dev)) {
|
|
|
|
ret = intel_init_vebox_ring_buffer(dev);
|
|
|
|
if (ret)
|
|
|
|
goto cleanup_blt_ring;
|
|
|
|
}
|
|
|
|
|
2014-04-17 10:37:37 +08:00
|
|
|
if (HAS_BSD2(dev)) {
|
|
|
|
ret = intel_init_bsd2_ring_buffer(dev);
|
|
|
|
if (ret)
|
|
|
|
goto cleanup_vebox_ring;
|
|
|
|
}
|
2013-05-29 10:22:23 +08:00
|
|
|
|
2013-02-09 03:49:24 +08:00
|
|
|
return 0;
|
|
|
|
|
2013-05-29 10:22:23 +08:00
|
|
|
cleanup_vebox_ring:
|
2016-03-16 19:00:40 +08:00
|
|
|
intel_cleanup_engine(&dev_priv->engine[VECS]);
|
2013-02-09 03:49:24 +08:00
|
|
|
cleanup_blt_ring:
|
2016-03-16 19:00:40 +08:00
|
|
|
intel_cleanup_engine(&dev_priv->engine[BCS]);
|
2013-02-09 03:49:24 +08:00
|
|
|
cleanup_bsd_ring:
|
2016-03-16 19:00:40 +08:00
|
|
|
intel_cleanup_engine(&dev_priv->engine[VCS]);
|
2013-02-09 03:49:24 +08:00
|
|
|
cleanup_render_ring:
|
2016-03-16 19:00:40 +08:00
|
|
|
intel_cleanup_engine(&dev_priv->engine[RCS]);
|
2013-02-09 03:49:24 +08:00
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
i915_gem_init_hw(struct drm_device *dev)
|
|
|
|
{
|
2014-03-31 19:27:16 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2016-03-16 19:00:36 +08:00
|
|
|
struct intel_engine_cs *engine;
|
2016-03-24 19:20:38 +08:00
|
|
|
int ret, j;
|
2013-02-09 03:49:24 +08:00
|
|
|
|
|
|
|
if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
|
|
|
|
return -EIO;
|
|
|
|
|
2015-02-13 22:35:59 +08:00
|
|
|
/* Double layer security blanket, see i915_gem_init() */
|
|
|
|
intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
|
|
|
|
|
2016-04-13 22:26:43 +08:00
|
|
|
if (HAS_EDRAM(dev) && INTEL_GEN(dev_priv) < 9)
|
2013-07-05 02:02:04 +08:00
|
|
|
I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
|
2013-02-09 03:49:24 +08:00
|
|
|
|
2013-11-29 20:56:12 +08:00
|
|
|
if (IS_HASWELL(dev))
|
|
|
|
I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
|
|
|
|
LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
|
2013-08-29 03:45:46 +08:00
|
|
|
|
2013-04-06 04:12:43 +08:00
|
|
|
if (HAS_PCH_NOP(dev)) {
|
2014-01-23 06:39:30 +08:00
|
|
|
if (IS_IVYBRIDGE(dev)) {
|
|
|
|
u32 temp = I915_READ(GEN7_MSG_CTL);
|
|
|
|
temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
|
|
|
|
I915_WRITE(GEN7_MSG_CTL, temp);
|
|
|
|
} else if (INTEL_INFO(dev)->gen >= 7) {
|
|
|
|
u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
|
|
|
|
temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
|
|
|
|
I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
|
|
|
|
}
|
2013-04-06 04:12:43 +08:00
|
|
|
}
|
|
|
|
|
2013-02-09 03:49:24 +08:00
|
|
|
i915_gem_init_swizzling(dev);
|
|
|
|
|
2014-11-20 16:45:19 +08:00
|
|
|
/*
|
|
|
|
* At least 830 can leave some of the unused rings
|
|
|
|
* "active" (ie. head != tail) after resume which
|
|
|
|
* will prevent c3 entry. Makes sure all unused rings
|
|
|
|
* are totally idle.
|
|
|
|
*/
|
|
|
|
init_unused_rings(dev);
|
|
|
|
|
2016-01-20 03:02:54 +08:00
|
|
|
BUG_ON(!dev_priv->kernel_context);
|
2015-05-30 00:43:37 +08:00
|
|
|
|
2015-06-18 20:11:20 +08:00
|
|
|
ret = i915_ppgtt_init_hw(dev);
|
|
|
|
if (ret) {
|
|
|
|
DRM_ERROR("PPGTT enable HW failed %d\n", ret);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Need to do basic initialisation of all rings first: */
|
2016-03-24 19:20:38 +08:00
|
|
|
for_each_engine(engine, dev_priv) {
|
2016-03-16 19:00:36 +08:00
|
|
|
ret = engine->init_hw(engine);
|
2014-11-20 07:33:07 +08:00
|
|
|
if (ret)
|
2015-02-13 22:35:59 +08:00
|
|
|
goto out;
|
2014-11-20 07:33:07 +08:00
|
|
|
}
|
2013-01-22 20:12:17 +08:00
|
|
|
|
2015-08-12 22:43:36 +08:00
|
|
|
/* We can't enable contexts until all firmware is loaded */
|
2015-09-11 05:55:00 +08:00
|
|
|
if (HAS_GUC_UCODE(dev)) {
|
|
|
|
ret = intel_guc_ucode_load(dev);
|
|
|
|
if (ret) {
|
2015-10-23 17:10:59 +08:00
|
|
|
DRM_ERROR("Failed to initialize GuC, error %d\n", ret);
|
|
|
|
ret = -EIO;
|
|
|
|
goto out;
|
2015-09-11 05:55:00 +08:00
|
|
|
}
|
2015-08-12 22:43:36 +08:00
|
|
|
}
|
|
|
|
|
2015-09-11 19:53:46 +08:00
|
|
|
/*
|
|
|
|
* Increment the next seqno by 0x100 so we have a visible break
|
|
|
|
* on re-initialisation
|
|
|
|
*/
|
|
|
|
ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100);
|
|
|
|
if (ret)
|
|
|
|
goto out;
|
|
|
|
|
2015-06-18 20:11:20 +08:00
|
|
|
/* Now it is safe to go back round and do everything else: */
|
2016-03-24 19:20:38 +08:00
|
|
|
for_each_engine(engine, dev_priv) {
|
2015-05-30 00:43:39 +08:00
|
|
|
struct drm_i915_gem_request *req;
|
|
|
|
|
2016-03-16 19:00:36 +08:00
|
|
|
req = i915_gem_request_alloc(engine, NULL);
|
drm/i915: simplify allocation of driver-internal requests
There are a number of places where the driver needs a request, but isn't
working on behalf of any specific user or in a specific context. At
present, we associate them with the per-engine default context. A future
patch will abolish those per-engine context pointers; but we can already
eliminate a lot of the references to them, just by making the allocator
allow NULL as a shorthand for "an appropriate context for this ring",
which will mean that the callers don't need to know anything about how
the "appropriate context" is found (e.g. per-ring vs per-device, etc).
So this patch renames the existing i915_gem_request_alloc(), and makes
it local (static inline), and replaces it with a wrapper that provides
a default if the context is NULL, and also has a nicer calling
convention (doesn't require a pointer to an output parameter). Then we
change all callers to use the new convention:
OLD:
err = i915_gem_request_alloc(ring, user_ctx, &req);
if (err) ...
NEW:
req = i915_gem_request_alloc(ring, user_ctx);
if (IS_ERR(req)) ...
OLD:
err = i915_gem_request_alloc(ring, ring->default_context, &req);
if (err) ...
NEW:
req = i915_gem_request_alloc(ring, NULL);
if (IS_ERR(req)) ...
v4: Rebased
Signed-off-by: Dave Gordon <david.s.gordon@intel.com>
Reviewed-by: Nick Hoath <nicholas.hoath@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1453230175-19330-2-git-send-email-david.s.gordon@intel.com
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2016-01-20 03:02:53 +08:00
|
|
|
if (IS_ERR(req)) {
|
|
|
|
ret = PTR_ERR(req);
|
2016-03-16 19:00:40 +08:00
|
|
|
i915_gem_cleanup_engines(dev);
|
2015-05-30 00:43:39 +08:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2016-03-16 19:00:36 +08:00
|
|
|
if (engine->id == RCS) {
|
2015-06-18 20:11:20 +08:00
|
|
|
for (j = 0; j < NUM_L3_SLICES(dev); j++)
|
2015-05-30 00:43:51 +08:00
|
|
|
i915_gem_l3_remap(req, j);
|
2015-06-18 20:11:20 +08:00
|
|
|
}
|
2013-09-18 12:12:44 +08:00
|
|
|
|
2015-05-30 00:43:40 +08:00
|
|
|
ret = i915_ppgtt_init_ring(req);
|
2015-06-18 20:11:20 +08:00
|
|
|
if (ret && ret != -EIO) {
|
2016-03-24 19:20:38 +08:00
|
|
|
DRM_ERROR("PPGTT enable %s failed %d\n",
|
|
|
|
engine->name, ret);
|
2015-05-30 00:43:39 +08:00
|
|
|
i915_gem_request_cancel(req);
|
2016-03-16 19:00:40 +08:00
|
|
|
i915_gem_cleanup_engines(dev);
|
2015-06-18 20:11:20 +08:00
|
|
|
goto out;
|
|
|
|
}
|
2014-08-07 02:19:53 +08:00
|
|
|
|
2015-05-30 00:43:40 +08:00
|
|
|
ret = i915_gem_context_enable(req);
|
2015-05-30 00:43:37 +08:00
|
|
|
if (ret && ret != -EIO) {
|
2016-03-24 19:20:38 +08:00
|
|
|
DRM_ERROR("Context enable %s failed %d\n",
|
|
|
|
engine->name, ret);
|
2015-05-30 00:43:39 +08:00
|
|
|
i915_gem_request_cancel(req);
|
2016-03-16 19:00:40 +08:00
|
|
|
i915_gem_cleanup_engines(dev);
|
2015-05-30 00:43:37 +08:00
|
|
|
goto out;
|
|
|
|
}
|
2015-05-30 00:43:39 +08:00
|
|
|
|
2015-05-30 00:43:49 +08:00
|
|
|
i915_add_request_no_flush(req);
|
2013-04-09 09:43:56 +08:00
|
|
|
}
|
2012-02-10 03:53:27 +08:00
|
|
|
|
2015-02-13 22:35:59 +08:00
|
|
|
out:
|
|
|
|
intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
|
drm/i915: Split context enabling from init
We **need** to do this for exactly 1 reason, because we want to embed a
PPGTT into the context, but we don't want to special case the default
context.
To achieve that, we must be able to initialize contexts after the GTT is
setup (so we can allocate and pin the default context's BO), but before
the PPGTT and rings are initialized. This is because, currently, context
initialization requires ring usage. We don't have rings until after the
GTT is setup. If we split the enabling part of context initialization,
the part requiring the ringbuffer, we can untangle this, and then later
embed the PPGTT
Incidentally this allows us to also adhere to the original design of
context init/fini in future patches: they were only ever meant to be
called at driver load and unload.
v2: Move hw_contexts_disabled test in i915_gem_context_enable() (Chris)
v3: BUG_ON after checking for disabled contexts. Or else it blows up pre
gen6 (Ben)
v4: Forward port
Modified enable for each ring, since that patch is earlier in the series
Dropped ring arg from create_default_context so it can be used by others
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-12-07 06:11:04 +08:00
|
|
|
return ret;
|
2010-05-21 09:08:55 +08:00
|
|
|
}
|
|
|
|
|
2012-04-24 22:47:41 +08:00
|
|
|
int i915_gem_init(struct drm_device *dev)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
|
|
|
int ret;
|
|
|
|
|
2014-07-25 00:04:11 +08:00
|
|
|
i915.enable_execlists = intel_sanitize_enable_execlists(dev,
|
|
|
|
i915.enable_execlists);
|
|
|
|
|
2012-04-24 22:47:41 +08:00
|
|
|
mutex_lock(&dev->struct_mutex);
|
2013-03-09 02:45:53 +08:00
|
|
|
|
2014-07-25 00:04:21 +08:00
|
|
|
if (!i915.enable_execlists) {
|
2015-03-19 20:30:06 +08:00
|
|
|
dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
|
2016-03-16 19:00:40 +08:00
|
|
|
dev_priv->gt.init_engines = i915_gem_init_engines;
|
|
|
|
dev_priv->gt.cleanup_engine = intel_cleanup_engine;
|
|
|
|
dev_priv->gt.stop_engine = intel_stop_engine;
|
2014-07-25 00:04:22 +08:00
|
|
|
} else {
|
2015-03-19 20:30:06 +08:00
|
|
|
dev_priv->gt.execbuf_submit = intel_execlists_submission;
|
2016-03-16 19:00:40 +08:00
|
|
|
dev_priv->gt.init_engines = intel_logical_rings_init;
|
|
|
|
dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
|
|
|
|
dev_priv->gt.stop_engine = intel_logical_ring_stop;
|
2014-07-25 00:04:21 +08:00
|
|
|
}
|
|
|
|
|
2015-02-13 22:35:59 +08:00
|
|
|
/* This is just a security blanket to placate dragons.
|
|
|
|
* On some systems, we very sporadically observe that the first TLBs
|
|
|
|
* used by the CS may be stale, despite us poking the TLB reset. If
|
|
|
|
* we hold the forcewake during initialisation these problems
|
|
|
|
* just magically go away.
|
|
|
|
*/
|
|
|
|
intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
|
|
|
|
|
2014-08-06 21:04:50 +08:00
|
|
|
ret = i915_gem_init_userptr(dev);
|
2014-12-05 20:17:42 +08:00
|
|
|
if (ret)
|
|
|
|
goto out_unlock;
|
2014-08-06 21:04:50 +08:00
|
|
|
|
2016-03-24 22:47:46 +08:00
|
|
|
i915_gem_init_ggtt(dev);
|
2013-03-09 02:45:53 +08:00
|
|
|
|
drm/i915: Split context enabling from init
We **need** to do this for exactly 1 reason, because we want to embed a
PPGTT into the context, but we don't want to special case the default
context.
To achieve that, we must be able to initialize contexts after the GTT is
setup (so we can allocate and pin the default context's BO), but before
the PPGTT and rings are initialized. This is because, currently, context
initialization requires ring usage. We don't have rings until after the
GTT is setup. If we split the enabling part of context initialization,
the part requiring the ringbuffer, we can untangle this, and then later
embed the PPGTT
Incidentally this allows us to also adhere to the original design of
context init/fini in future patches: they were only ever meant to be
called at driver load and unload.
v2: Move hw_contexts_disabled test in i915_gem_context_enable() (Chris)
v3: BUG_ON after checking for disabled contexts. Or else it blows up pre
gen6 (Ben)
v4: Forward port
Modified enable for each ring, since that patch is earlier in the series
Dropped ring arg from create_default_context so it can be used by others
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-12-07 06:11:04 +08:00
|
|
|
ret = i915_gem_context_init(dev);
|
2014-12-05 20:17:42 +08:00
|
|
|
if (ret)
|
|
|
|
goto out_unlock;
|
drm/i915: Split context enabling from init
We **need** to do this for exactly 1 reason, because we want to embed a
PPGTT into the context, but we don't want to special case the default
context.
To achieve that, we must be able to initialize contexts after the GTT is
setup (so we can allocate and pin the default context's BO), but before
the PPGTT and rings are initialized. This is because, currently, context
initialization requires ring usage. We don't have rings until after the
GTT is setup. If we split the enabling part of context initialization,
the part requiring the ringbuffer, we can untangle this, and then later
embed the PPGTT
Incidentally this allows us to also adhere to the original design of
context init/fini in future patches: they were only ever meant to be
called at driver load and unload.
v2: Move hw_contexts_disabled test in i915_gem_context_enable() (Chris)
v3: BUG_ON after checking for disabled contexts. Or else it blows up pre
gen6 (Ben)
v4: Forward port
Modified enable for each ring, since that patch is earlier in the series
Dropped ring arg from create_default_context so it can be used by others
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-12-07 06:11:04 +08:00
|
|
|
|
2016-03-16 19:00:40 +08:00
|
|
|
ret = dev_priv->gt.init_engines(dev);
|
2014-11-20 07:33:07 +08:00
|
|
|
if (ret)
|
2014-12-05 20:17:42 +08:00
|
|
|
goto out_unlock;
|
drm/i915: Split context enabling from init
We **need** to do this for exactly 1 reason, because we want to embed a
PPGTT into the context, but we don't want to special case the default
context.
To achieve that, we must be able to initialize contexts after the GTT is
setup (so we can allocate and pin the default context's BO), but before
the PPGTT and rings are initialized. This is because, currently, context
initialization requires ring usage. We don't have rings until after the
GTT is setup. If we split the enabling part of context initialization,
the part requiring the ringbuffer, we can untangle this, and then later
embed the PPGTT
Incidentally this allows us to also adhere to the original design of
context init/fini in future patches: they were only ever meant to be
called at driver load and unload.
v2: Move hw_contexts_disabled test in i915_gem_context_enable() (Chris)
v3: BUG_ON after checking for disabled contexts. Or else it blows up pre
gen6 (Ben)
v4: Forward port
Modified enable for each ring, since that patch is earlier in the series
Dropped ring arg from create_default_context so it can be used by others
Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-12-07 06:11:04 +08:00
|
|
|
|
2012-04-24 22:47:41 +08:00
|
|
|
ret = i915_gem_init_hw(dev);
|
2014-04-09 16:19:42 +08:00
|
|
|
if (ret == -EIO) {
|
|
|
|
/* Allow ring initialisation to fail by marking the GPU as
|
|
|
|
* wedged. But we only want to do this where the GPU is angry,
|
|
|
|
* for all other failure, such as an allocation failure, bail.
|
|
|
|
*/
|
|
|
|
DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
|
2015-04-24 07:12:32 +08:00
|
|
|
atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
|
2014-04-09 16:19:42 +08:00
|
|
|
ret = 0;
|
2012-04-24 22:47:41 +08:00
|
|
|
}
|
2014-12-05 20:17:42 +08:00
|
|
|
|
|
|
|
out_unlock:
|
2015-02-13 22:35:59 +08:00
|
|
|
intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
|
2014-04-09 16:19:42 +08:00
|
|
|
mutex_unlock(&dev->struct_mutex);
|
2012-04-24 22:47:41 +08:00
|
|
|
|
2014-04-09 16:19:42 +08:00
|
|
|
return ret;
|
2012-04-24 22:47:41 +08:00
|
|
|
}
|
|
|
|
|
2010-05-21 09:08:55 +08:00
|
|
|
void
|
2016-03-16 19:00:40 +08:00
|
|
|
i915_gem_cleanup_engines(struct drm_device *dev)
|
2010-05-21 09:08:55 +08:00
|
|
|
{
|
2014-03-31 19:27:16 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2016-03-16 19:00:36 +08:00
|
|
|
struct intel_engine_cs *engine;
|
2010-05-21 09:08:55 +08:00
|
|
|
|
2016-03-24 19:20:38 +08:00
|
|
|
for_each_engine(engine, dev_priv)
|
2016-03-16 19:00:40 +08:00
|
|
|
dev_priv->gt.cleanup_engine(engine);
|
2015-07-04 00:27:34 +08:00
|
|
|
|
2016-03-16 23:54:00 +08:00
|
|
|
if (i915.enable_execlists)
|
|
|
|
/*
|
|
|
|
* Neither the BIOS, ourselves or any other kernel
|
|
|
|
* expects the system to be in execlists mode on startup,
|
|
|
|
* so we need to reset the GPU back to legacy mode.
|
|
|
|
*/
|
|
|
|
intel_gpu_reset(dev, ALL_ENGINES);
|
2010-05-21 09:08:55 +08:00
|
|
|
}
|
|
|
|
|
2010-10-24 19:38:05 +08:00
|
|
|
static void
|
2016-03-16 19:00:39 +08:00
|
|
|
init_engine_lists(struct intel_engine_cs *engine)
|
2010-10-24 19:38:05 +08:00
|
|
|
{
|
2016-03-16 19:00:37 +08:00
|
|
|
INIT_LIST_HEAD(&engine->active_list);
|
|
|
|
INIT_LIST_HEAD(&engine->request_list);
|
2010-10-24 19:38:05 +08:00
|
|
|
}
|
|
|
|
|
2016-03-16 20:54:03 +08:00
|
|
|
void
|
|
|
|
i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
|
|
|
|
{
|
|
|
|
struct drm_device *dev = dev_priv->dev;
|
|
|
|
|
|
|
|
if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
|
|
|
|
!IS_CHERRYVIEW(dev_priv))
|
|
|
|
dev_priv->num_fence_regs = 32;
|
|
|
|
else if (INTEL_INFO(dev_priv)->gen >= 4 || IS_I945G(dev_priv) ||
|
|
|
|
IS_I945GM(dev_priv) || IS_G33(dev_priv))
|
|
|
|
dev_priv->num_fence_regs = 16;
|
|
|
|
else
|
|
|
|
dev_priv->num_fence_regs = 8;
|
|
|
|
|
|
|
|
if (intel_vgpu_active(dev))
|
|
|
|
dev_priv->num_fence_regs =
|
|
|
|
I915_READ(vgtif_reg(avail_rs.fence_num));
|
|
|
|
|
|
|
|
/* Initialize fence registers to zero */
|
|
|
|
i915_gem_restore_fences(dev);
|
|
|
|
|
|
|
|
i915_gem_detect_bit_6_swizzle(dev);
|
|
|
|
}
|
|
|
|
|
2008-07-31 03:06:12 +08:00
|
|
|
void
|
2016-01-19 21:26:29 +08:00
|
|
|
i915_gem_load_init(struct drm_device *dev)
|
2008-07-31 03:06:12 +08:00
|
|
|
{
|
2014-03-31 19:27:16 +08:00
|
|
|
struct drm_i915_private *dev_priv = dev->dev_private;
|
2012-11-15 19:32:30 +08:00
|
|
|
int i;
|
|
|
|
|
2015-04-07 23:20:57 +08:00
|
|
|
dev_priv->objects =
|
2012-11-15 19:32:30 +08:00
|
|
|
kmem_cache_create("i915_gem_object",
|
|
|
|
sizeof(struct drm_i915_gem_object), 0,
|
|
|
|
SLAB_HWCACHE_ALIGN,
|
|
|
|
NULL);
|
2015-04-07 23:20:58 +08:00
|
|
|
dev_priv->vmas =
|
|
|
|
kmem_cache_create("i915_gem_vma",
|
|
|
|
sizeof(struct i915_vma), 0,
|
|
|
|
SLAB_HWCACHE_ALIGN,
|
|
|
|
NULL);
|
2015-04-07 23:20:57 +08:00
|
|
|
dev_priv->requests =
|
|
|
|
kmem_cache_create("i915_gem_request",
|
|
|
|
sizeof(struct drm_i915_gem_request), 0,
|
|
|
|
SLAB_HWCACHE_ALIGN,
|
|
|
|
NULL);
|
2008-07-31 03:06:12 +08:00
|
|
|
|
2013-08-01 07:59:54 +08:00
|
|
|
INIT_LIST_HEAD(&dev_priv->vm_list);
|
2013-09-18 12:12:45 +08:00
|
|
|
INIT_LIST_HEAD(&dev_priv->context_list);
|
drm/i915: Track unbound pages
When dealing with a working set larger than the GATT, or even the
mappable aperture when touching through the GTT, we end up with evicting
objects only to rebind them at a new offset again later. Moving an
object into and out of the GTT requires clflushing the pages, thus
causing a double-clflush penalty for rebinding.
To avoid having to clflush on rebinding, we can track the pages as they
are evicted from the GTT and only relinquish those pages on memory
pressure.
As usual, if it were not for the handling of out-of-memory condition and
having to manually shrink our own bo caches, it would be a net reduction
of code. Alas.
Note: The patch also contains a few changes to the last-hope
evict_everything logic in i916_gem_execbuffer.c - we no longer try to
only evict the purgeable stuff in a first try (since that's superflous
and only helps in OOM corner-cases, not fragmented-gtt trashing
situations).
Also, the extraction of the get_pages retry loop from bind_to_gtt (and
other callsites) to get_pages should imo have been a separate patch.
v2: Ditch the newly added put_pages (for unbound objects only) in
i915_gem_reset. A quick irc discussion hasn't revealed any important
reason for this, so if we need this, I'd like to have a git blame'able
explanation for it.
v3: Undo the s/drm_malloc_ab/kmalloc/ in get_pages that Chris noticed.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
[danvet: Split out code movements and rant a bit in the commit message
with a few Notes. Done v2]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2012-08-20 17:40:46 +08:00
|
|
|
INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
|
|
|
|
INIT_LIST_HEAD(&dev_priv->mm.bound_list);
|
2009-08-30 03:49:51 +08:00
|
|
|
INIT_LIST_HEAD(&dev_priv->mm.fence_list);
|
2016-03-16 19:00:39 +08:00
|
|
|
for (i = 0; i < I915_NUM_ENGINES; i++)
|
|
|
|
init_engine_lists(&dev_priv->engine[i]);
|
2011-10-10 03:52:02 +08:00
|
|
|
for (i = 0; i < I915_MAX_NUM_FENCES; i++)
|
2010-04-28 17:02:31 +08:00
|
|
|
INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
|
2008-07-31 03:06:12 +08:00
|
|
|
INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
|
|
|
|
i915_gem_retire_work_handler);
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
|
|
|
|
i915_gem_idle_work_handler);
|
2012-11-16 00:17:22 +08:00
|
|
|
init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
|
2009-09-14 23:50:28 +08:00
|
|
|
|
2010-12-19 19:42:05 +08:00
|
|
|
dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
|
|
|
|
|
2015-09-11 19:53:46 +08:00
|
|
|
/*
|
|
|
|
* Set initial sequence number for requests.
|
|
|
|
* Using this number allows the wraparound to happen early,
|
|
|
|
* catching any obvious problems.
|
|
|
|
*/
|
|
|
|
dev_priv->next_seqno = ((u32)~0 - 0x1100);
|
|
|
|
dev_priv->last_seqno = ((u32)~0 - 0x1101);
|
|
|
|
|
2013-06-12 17:15:12 +08:00
|
|
|
INIT_LIST_HEAD(&dev_priv->mm.fence_list);
|
2011-05-07 04:53:49 +08:00
|
|
|
|
2009-11-19 00:25:18 +08:00
|
|
|
init_waitqueue_head(&dev_priv->pending_flip_queue);
|
2010-10-28 19:51:39 +08:00
|
|
|
|
2011-02-21 22:43:56 +08:00
|
|
|
dev_priv->mm.interruptible = true;
|
|
|
|
|
drm/i915: Track frontbuffer invalidation/flushing
So these are the guts of the new beast. This tracks when a frontbuffer
gets invalidated (due to frontbuffer rendering) and hence should be
constantly scaned out, and when it's flushed again and can be
compressed/one-shot-upload.
Rules for flushing are simple: The frontbuffer needs one more full
upload starting from the next vblank. Which means that the flushing
can _only_ be called once the frontbuffer update has been latched.
But this poses a problem for pageflips: We can't just delay the
flushing until the pageflip is latched, since that would pose the risk
that we override frontbuffer rendering that has been scheduled
in-between the pageflip ioctl and the actual latching.
To handle this track asynchronous invalidations (and also pageflip)
state per-ring and delay any in-between flushing until the rendering
has completed. And also cancel any delayed flushing if we get a new
invalidation request (whether delayed or not).
Also call intel_mark_fb_busy in both cases in all cases to make sure
that we keep the screen at the highest refresh rate both on flips,
synchronous plane updates and for frontbuffer rendering.
v2: Lots of improvements
Suggestions from Chris:
- Move invalidate/flush in flush_*_domain and set_to_*_domain.
- Drop the flush in busy_ioctl since it's redundant. Was a leftover
from an earlier concept to track flips/delayed flushes.
- Don't forget about the initial modeset enable/final disable.
Suggested by Chris.
Track flips accurately, too. Since flips complete independently of
rendering we need to track pending flips in a separate mask. Again if
an invalidate happens we need to cancel the evenutal flush to avoid
races.
v3:
Provide correct header declarations for flip functions. Currently not
needed outside of intel_display.c, but part of the proper interface.
v4: Add proper domain management to fbcon so that the fbcon buffer is
also tracked correctly.
v5: Fixup locking around the fbcon set_to_gtt_domain call.
v6: More comments from Chris:
- Split out fbcon changes.
- Drop superflous checks for potential scanout before calling intel_fb
functions - we can micro-optimize this later.
- s/intel_fb_/intel_fb_obj_/ to make it clear that this deals in gem
object. We already have precedence for fb_obj in the pin_and_fence
functions.
v7: Clarify the semantics of the flip flush handling by renaming
things a bit:
- Don't go through a gem object but take the relevant frontbuffer bits
directly. These functions center on the plane, the actual object is
irrelevant - even a flip to the same object as already active should
cause a flush.
- Add a new intel_frontbuffer_flip for synchronous plane updates. It
currently just calls intel_frontbuffer_flush since the implemenation
differs.
This way we achieve a clear split between one-shot update events on
one side and frontbuffer rendering with potentially a very long delay
between the invalidate and flush.
Chris and I also had some discussions about mark_busy and whether it
is appropriate to call from flush. But mark busy is a state which
should be derived from the 3 events (invalidate, flush, flip) we now
have by the users, like psr does by tracking relevant information in
psr.busy_frontbuffer_bits. DRRS (the only real use of mark_busy for
frontbuffer) needs to have similar logic. With that the overall
mark_busy in the core could be removed.
v8: Only when retiring gpu buffers only flush frontbuffer bits we
actually invalidated in a batch. Just for safety since before any
additional usage/invalidate we should always retire current rendering.
Suggested by Chris Wilson.
v9: Actually use intel_frontbuffer_flip in all appropriate places.
Spotted by Chris.
v10: Address more comments from Chris:
- Don't call _flip in set_base when the crtc is inactive, avoids redunancy
in the modeset case with the initial enabling of all planes.
- Add comments explaining that the initial/final plane enable/disable
still has work left to do before it's fully generic.
v11: Only invalidate for gtt/cpu access when writing. Spotted by Chris.
v12: s/_flush/_flip/ in intel_overlay.c per Chris' comment.
Cc: Rodrigo Vivi <rodrigo.vivi@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2014-06-19 22:01:59 +08:00
|
|
|
mutex_init(&dev_priv->fb_tracking.lock);
|
2008-07-31 03:06:12 +08:00
|
|
|
}
|
2008-12-30 18:31:46 +08:00
|
|
|
|
2016-01-19 21:26:29 +08:00
|
|
|
void i915_gem_load_cleanup(struct drm_device *dev)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = to_i915(dev);
|
|
|
|
|
|
|
|
kmem_cache_destroy(dev_priv->requests);
|
|
|
|
kmem_cache_destroy(dev_priv->vmas);
|
|
|
|
kmem_cache_destroy(dev_priv->objects);
|
|
|
|
}
|
|
|
|
|
2010-09-24 23:02:42 +08:00
|
|
|
void i915_gem_release(struct drm_device *dev, struct drm_file *file)
|
2009-06-03 15:27:35 +08:00
|
|
|
{
|
2010-09-24 23:02:42 +08:00
|
|
|
struct drm_i915_file_private *file_priv = file->driver_priv;
|
2009-06-03 15:27:35 +08:00
|
|
|
|
|
|
|
/* Clean up our request list when the client is going away, so that
|
|
|
|
* later retire_requests won't dereference our soon-to-be-gone
|
|
|
|
* file_priv.
|
|
|
|
*/
|
2010-09-26 18:03:27 +08:00
|
|
|
spin_lock(&file_priv->mm.lock);
|
2010-09-24 23:02:42 +08:00
|
|
|
while (!list_empty(&file_priv->mm.request_list)) {
|
|
|
|
struct drm_i915_gem_request *request;
|
|
|
|
|
|
|
|
request = list_first_entry(&file_priv->mm.request_list,
|
|
|
|
struct drm_i915_gem_request,
|
|
|
|
client_list);
|
|
|
|
list_del(&request->client_list);
|
|
|
|
request->file_priv = NULL;
|
|
|
|
}
|
2010-09-26 18:03:27 +08:00
|
|
|
spin_unlock(&file_priv->mm.lock);
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
|
2015-04-27 20:41:22 +08:00
|
|
|
if (!list_empty(&file_priv->rps.link)) {
|
2015-05-22 04:01:47 +08:00
|
|
|
spin_lock(&to_i915(dev)->rps.client_lock);
|
2015-04-27 20:41:22 +08:00
|
|
|
list_del(&file_priv->rps.link);
|
2015-05-22 04:01:47 +08:00
|
|
|
spin_unlock(&to_i915(dev)->rps.client_lock);
|
2015-04-07 23:20:32 +08:00
|
|
|
}
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
int i915_gem_open(struct drm_device *dev, struct drm_file *file)
|
|
|
|
{
|
|
|
|
struct drm_i915_file_private *file_priv;
|
2013-12-07 06:10:58 +08:00
|
|
|
int ret;
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
|
|
|
|
DRM_DEBUG_DRIVER("\n");
|
|
|
|
|
|
|
|
file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
|
|
|
|
if (!file_priv)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
file->driver_priv = file_priv;
|
|
|
|
file_priv->dev_priv = dev->dev_private;
|
2014-02-25 23:11:24 +08:00
|
|
|
file_priv->file = file;
|
2015-04-27 20:41:22 +08:00
|
|
|
INIT_LIST_HEAD(&file_priv->rps.link);
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
|
|
|
|
spin_lock_init(&file_priv->mm.lock);
|
|
|
|
INIT_LIST_HEAD(&file_priv->mm.request_list);
|
|
|
|
|
2016-01-15 23:12:50 +08:00
|
|
|
file_priv->bsd_ring = -1;
|
|
|
|
|
2013-12-07 06:10:58 +08:00
|
|
|
ret = i915_gem_context_open(dev, file);
|
|
|
|
if (ret)
|
|
|
|
kfree(file_priv);
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
|
2013-12-07 06:10:58 +08:00
|
|
|
return ret;
|
drm/i915: Boost RPS frequency for CPU stalls
If we encounter a situation where the CPU blocks waiting for results
from the GPU, give the GPU a kick to boost its the frequency.
This should work to reduce user interface stalls and to quickly promote
mesa to high frequencies - but the cost is that our requested frequency
stalls high (as we do not idle for long enough before rc6 to start
reducing frequencies, nor are we aggressive at down clocking an
underused GPU). However, this should be mitigated by rc6 itself powering
off the GPU when idle, and that energy use is dependent upon the workload
of the GPU in addition to its frequency (e.g. the math or sampler
functions only consume power when used). Still, this is likely to
adversely affect light workloads.
In particular, this nearly eliminates the highly noticeable wake-up lag
in animations from idle. For example, expose or workspace transitions.
(However, given the situation where we fail to downclock, our requested
frequency is almost always the maximum, except for Baytrail where we
manually downclock upon idling. This often masks the latency of
upclocking after being idle, so animations are typically smooth - at the
cost of increased power consumption.)
Stéphane raised the concern that this will punish good applications and
reward bad applications - but due to the nature of how mesa performs its
client throttling, I believe all mesa applications will be roughly
equally affected. To address this concern, and to prevent applications
like compositors from permanently boosting the RPS state, we ratelimit the
frequency of the wait-boosts each client recieves.
Unfortunately, this techinique is ineffective with Ironlake - which also
has dynamic render power states and suffers just as dramatically. For
Ironlake, the thermal/power headroom is shared with the CPU through
Intelligent Power Sharing and the intel-ips module. This leaves us with
no GPU boost frequencies available when coming out of idle, and due to
hardware limitations we cannot change the arbitration between the CPU and
GPU quickly enough to be effective.
v2: Limit each client to receiving a single boost for each active period.
Tested by QA to only marginally increase power, and to demonstrably
increase throughput in games. No latency measurements yet.
v3: Cater for front-buffer rendering with manual throttling.
v4: Tidy up.
v5: Sadly the compositor needs frequent boosts as it may never idle, but
due to its picking mechanism (using ReadPixels) may require frequent
waits. Those waits, along with the waits for the vrefresh swap, conspire
to keep the GPU at low frequencies despite the interactive latency. To
overcome this we ditch the one-boost-per-active-period and just ratelimit
the number of wait-boosts each client can receive.
Reported-and-tested-by: Paul Neumann <paul104x@yahoo.de>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=68716
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Kenneth Graunke <kenneth@whitecape.org>
Cc: Stéphane Marchesin <stephane.marchesin@gmail.com>
Cc: Owen Taylor <otaylor@redhat.com>
Cc: "Meng, Mengmeng" <mengmeng.meng@intel.com>
Cc: "Zhuang, Lena" <lena.zhuang@intel.com>
Reviewed-by: Jesse Barnes <jbarnes@virtuousgeek.org>
[danvet: No extern for function prototypes in headers.]
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2013-09-26 00:34:56 +08:00
|
|
|
}
|
|
|
|
|
2014-09-20 00:27:27 +08:00
|
|
|
/**
|
|
|
|
* i915_gem_track_fb - update frontbuffer tracking
|
2015-09-15 20:58:44 +08:00
|
|
|
* @old: current GEM buffer for the frontbuffer slots
|
|
|
|
* @new: new GEM buffer for the frontbuffer slots
|
|
|
|
* @frontbuffer_bits: bitmask of frontbuffer slots
|
2014-09-20 00:27:27 +08:00
|
|
|
*
|
|
|
|
* This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
|
|
|
|
* from @old and setting them in @new. Both @old and @new can be NULL.
|
|
|
|
*/
|
2014-06-19 05:28:09 +08:00
|
|
|
void i915_gem_track_fb(struct drm_i915_gem_object *old,
|
|
|
|
struct drm_i915_gem_object *new,
|
|
|
|
unsigned frontbuffer_bits)
|
|
|
|
{
|
|
|
|
if (old) {
|
|
|
|
WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
|
|
|
|
WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
|
|
|
|
old->frontbuffer_bits &= ~frontbuffer_bits;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (new) {
|
|
|
|
WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
|
|
|
|
WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
|
|
|
|
new->frontbuffer_bits |= frontbuffer_bits;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-08-01 07:59:56 +08:00
|
|
|
/* All the new VM stuff */
|
2015-08-08 00:40:17 +08:00
|
|
|
u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
|
|
|
|
struct i915_address_space *vm)
|
2013-08-01 07:59:56 +08:00
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = o->base.dev->dev_private;
|
|
|
|
struct i915_vma *vma;
|
|
|
|
|
2014-08-06 21:04:51 +08:00
|
|
|
WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
|
2013-08-01 07:59:56 +08:00
|
|
|
|
2016-02-26 19:03:19 +08:00
|
|
|
list_for_each_entry(vma, &o->vma_list, obj_link) {
|
2016-02-26 19:03:20 +08:00
|
|
|
if (vma->is_ggtt &&
|
2015-03-16 20:11:13 +08:00
|
|
|
vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
|
|
|
|
continue;
|
|
|
|
if (vma->vm == vm)
|
2013-08-01 07:59:56 +08:00
|
|
|
return vma->node.start;
|
|
|
|
}
|
2015-03-16 20:11:13 +08:00
|
|
|
|
2014-06-18 04:34:38 +08:00
|
|
|
WARN(1, "%s vma for this object not found.\n",
|
|
|
|
i915_is_ggtt(vm) ? "global" : "ppgtt");
|
2013-08-01 07:59:56 +08:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2015-08-08 00:40:17 +08:00
|
|
|
u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
|
|
|
|
const struct i915_ggtt_view *view)
|
2013-08-01 07:59:56 +08:00
|
|
|
{
|
2016-03-30 21:57:10 +08:00
|
|
|
struct drm_i915_private *dev_priv = to_i915(o->base.dev);
|
|
|
|
struct i915_ggtt *ggtt = &dev_priv->ggtt;
|
2013-08-01 07:59:56 +08:00
|
|
|
struct i915_vma *vma;
|
|
|
|
|
2016-02-26 19:03:19 +08:00
|
|
|
list_for_each_entry(vma, &o->vma_list, obj_link)
|
2016-03-30 21:57:10 +08:00
|
|
|
if (vma->vm == &ggtt->base &&
|
2015-03-27 19:09:22 +08:00
|
|
|
i915_ggtt_view_equal(&vma->ggtt_view, view))
|
2015-03-16 20:11:13 +08:00
|
|
|
return vma->node.start;
|
|
|
|
|
2015-03-17 22:45:29 +08:00
|
|
|
WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
|
2015-03-16 20:11:13 +08:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
|
|
|
|
struct i915_address_space *vm)
|
|
|
|
{
|
|
|
|
struct i915_vma *vma;
|
|
|
|
|
2016-02-26 19:03:19 +08:00
|
|
|
list_for_each_entry(vma, &o->vma_list, obj_link) {
|
2016-02-26 19:03:20 +08:00
|
|
|
if (vma->is_ggtt &&
|
2015-03-16 20:11:13 +08:00
|
|
|
vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
|
|
|
|
continue;
|
|
|
|
if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
|
2015-03-27 19:09:22 +08:00
|
|
|
const struct i915_ggtt_view *view)
|
2015-03-16 20:11:13 +08:00
|
|
|
{
|
2016-03-30 21:57:10 +08:00
|
|
|
struct drm_i915_private *dev_priv = to_i915(o->base.dev);
|
|
|
|
struct i915_ggtt *ggtt = &dev_priv->ggtt;
|
2015-03-16 20:11:13 +08:00
|
|
|
struct i915_vma *vma;
|
|
|
|
|
2016-02-26 19:03:19 +08:00
|
|
|
list_for_each_entry(vma, &o->vma_list, obj_link)
|
2016-03-30 21:57:10 +08:00
|
|
|
if (vma->vm == &ggtt->base &&
|
2015-03-27 19:09:22 +08:00
|
|
|
i915_ggtt_view_equal(&vma->ggtt_view, view) &&
|
2014-12-11 01:27:58 +08:00
|
|
|
drm_mm_node_allocated(&vma->node))
|
2013-08-01 07:59:56 +08:00
|
|
|
return true;
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
|
|
|
|
{
|
2013-09-10 18:27:37 +08:00
|
|
|
struct i915_vma *vma;
|
2013-08-01 07:59:56 +08:00
|
|
|
|
2016-02-26 19:03:19 +08:00
|
|
|
list_for_each_entry(vma, &o->vma_list, obj_link)
|
2013-09-10 18:27:37 +08:00
|
|
|
if (drm_mm_node_allocated(&vma->node))
|
2013-08-01 07:59:56 +08:00
|
|
|
return true;
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
|
|
|
|
struct i915_address_space *vm)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = o->base.dev->dev_private;
|
|
|
|
struct i915_vma *vma;
|
|
|
|
|
2014-08-06 21:04:51 +08:00
|
|
|
WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
|
2013-08-01 07:59:56 +08:00
|
|
|
|
|
|
|
BUG_ON(list_empty(&o->vma_list));
|
|
|
|
|
2016-02-26 19:03:19 +08:00
|
|
|
list_for_each_entry(vma, &o->vma_list, obj_link) {
|
2016-02-26 19:03:20 +08:00
|
|
|
if (vma->is_ggtt &&
|
2015-03-16 20:11:13 +08:00
|
|
|
vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
|
|
|
|
continue;
|
2013-08-01 07:59:56 +08:00
|
|
|
if (vma->vm == vm)
|
|
|
|
return vma->node.size;
|
2015-03-16 20:11:13 +08:00
|
|
|
}
|
2013-08-01 07:59:56 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-03-16 20:11:13 +08:00
|
|
|
bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
|
2013-09-25 00:57:57 +08:00
|
|
|
{
|
|
|
|
struct i915_vma *vma;
|
2016-02-26 19:03:19 +08:00
|
|
|
list_for_each_entry(vma, &obj->vma_list, obj_link)
|
2015-03-16 20:11:13 +08:00
|
|
|
if (vma->pin_count > 0)
|
|
|
|
return true;
|
2015-05-06 19:34:58 +08:00
|
|
|
|
2015-03-16 20:11:13 +08:00
|
|
|
return false;
|
2013-09-25 00:57:57 +08:00
|
|
|
}
|
2015-07-10 02:29:02 +08:00
|
|
|
|
2015-12-11 02:51:23 +08:00
|
|
|
/* Like i915_gem_object_get_page(), but mark the returned page dirty */
|
|
|
|
struct page *
|
|
|
|
i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
|
|
|
|
{
|
|
|
|
struct page *page;
|
|
|
|
|
|
|
|
/* Only default objects have per-page dirty tracking */
|
2016-01-23 02:32:31 +08:00
|
|
|
if (WARN_ON((obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE) == 0))
|
2015-12-11 02:51:23 +08:00
|
|
|
return NULL;
|
|
|
|
|
|
|
|
page = i915_gem_object_get_page(obj, n);
|
|
|
|
set_page_dirty(page);
|
|
|
|
return page;
|
|
|
|
}
|
|
|
|
|
2015-07-10 02:29:02 +08:00
|
|
|
/* Allocate a new GEM object and fill it with the supplied data */
|
|
|
|
struct drm_i915_gem_object *
|
|
|
|
i915_gem_object_create_from_data(struct drm_device *dev,
|
|
|
|
const void *data, size_t size)
|
|
|
|
{
|
|
|
|
struct drm_i915_gem_object *obj;
|
|
|
|
struct sg_table *sg;
|
|
|
|
size_t bytes;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
obj = i915_gem_alloc_object(dev, round_up(size, PAGE_SIZE));
|
|
|
|
if (IS_ERR_OR_NULL(obj))
|
|
|
|
return obj;
|
|
|
|
|
|
|
|
ret = i915_gem_object_set_to_cpu_domain(obj, true);
|
|
|
|
if (ret)
|
|
|
|
goto fail;
|
|
|
|
|
|
|
|
ret = i915_gem_object_get_pages(obj);
|
|
|
|
if (ret)
|
|
|
|
goto fail;
|
|
|
|
|
|
|
|
i915_gem_object_pin_pages(obj);
|
|
|
|
sg = obj->pages;
|
|
|
|
bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
|
2015-12-11 02:51:24 +08:00
|
|
|
obj->dirty = 1; /* Backing store is now out of date */
|
2015-07-10 02:29:02 +08:00
|
|
|
i915_gem_object_unpin_pages(obj);
|
|
|
|
|
|
|
|
if (WARN_ON(bytes != size)) {
|
|
|
|
DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
|
|
|
|
ret = -EFAULT;
|
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
|
|
|
return obj;
|
|
|
|
|
|
|
|
fail:
|
|
|
|
drm_gem_object_unreference(&obj->base);
|
|
|
|
return ERR_PTR(ret);
|
|
|
|
}
|