304 lines
6.7 KiB
C
304 lines
6.7 KiB
C
// SPDX-License-Identifier: MIT
|
|
/*
|
|
* Copyright © 2019 Intel Corporation
|
|
*/
|
|
#include "i915_gem_client_blt.h"
|
|
|
|
#include "i915_gem_object_blt.h"
|
|
#include "intel_drv.h"
|
|
|
|
struct i915_sleeve {
|
|
struct i915_vma *vma;
|
|
struct drm_i915_gem_object *obj;
|
|
struct sg_table *pages;
|
|
struct i915_page_sizes page_sizes;
|
|
};
|
|
|
|
static int vma_set_pages(struct i915_vma *vma)
|
|
{
|
|
struct i915_sleeve *sleeve = vma->private;
|
|
|
|
vma->pages = sleeve->pages;
|
|
vma->page_sizes = sleeve->page_sizes;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void vma_clear_pages(struct i915_vma *vma)
|
|
{
|
|
GEM_BUG_ON(!vma->pages);
|
|
vma->pages = NULL;
|
|
}
|
|
|
|
static int vma_bind(struct i915_vma *vma,
|
|
enum i915_cache_level cache_level,
|
|
u32 flags)
|
|
{
|
|
return vma->vm->vma_ops.bind_vma(vma, cache_level, flags);
|
|
}
|
|
|
|
static void vma_unbind(struct i915_vma *vma)
|
|
{
|
|
vma->vm->vma_ops.unbind_vma(vma);
|
|
}
|
|
|
|
static const struct i915_vma_ops proxy_vma_ops = {
|
|
.set_pages = vma_set_pages,
|
|
.clear_pages = vma_clear_pages,
|
|
.bind_vma = vma_bind,
|
|
.unbind_vma = vma_unbind,
|
|
};
|
|
|
|
static struct i915_sleeve *create_sleeve(struct i915_address_space *vm,
|
|
struct drm_i915_gem_object *obj,
|
|
struct sg_table *pages,
|
|
struct i915_page_sizes *page_sizes)
|
|
{
|
|
struct i915_sleeve *sleeve;
|
|
struct i915_vma *vma;
|
|
int err;
|
|
|
|
sleeve = kzalloc(sizeof(*sleeve), GFP_KERNEL);
|
|
if (!sleeve)
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
vma = i915_vma_instance(obj, vm, NULL);
|
|
if (IS_ERR(vma)) {
|
|
err = PTR_ERR(vma);
|
|
goto err_free;
|
|
}
|
|
|
|
vma->private = sleeve;
|
|
vma->ops = &proxy_vma_ops;
|
|
|
|
sleeve->vma = vma;
|
|
sleeve->pages = pages;
|
|
sleeve->page_sizes = *page_sizes;
|
|
|
|
return sleeve;
|
|
|
|
err_free:
|
|
kfree(sleeve);
|
|
return ERR_PTR(err);
|
|
}
|
|
|
|
static void destroy_sleeve(struct i915_sleeve *sleeve)
|
|
{
|
|
kfree(sleeve);
|
|
}
|
|
|
|
struct clear_pages_work {
|
|
struct dma_fence dma;
|
|
struct dma_fence_cb cb;
|
|
struct i915_sw_fence wait;
|
|
struct work_struct work;
|
|
struct irq_work irq_work;
|
|
struct i915_sleeve *sleeve;
|
|
struct intel_context *ce;
|
|
u32 value;
|
|
};
|
|
|
|
static const char *clear_pages_work_driver_name(struct dma_fence *fence)
|
|
{
|
|
return DRIVER_NAME;
|
|
}
|
|
|
|
static const char *clear_pages_work_timeline_name(struct dma_fence *fence)
|
|
{
|
|
return "clear";
|
|
}
|
|
|
|
static void clear_pages_work_release(struct dma_fence *fence)
|
|
{
|
|
struct clear_pages_work *w = container_of(fence, typeof(*w), dma);
|
|
|
|
destroy_sleeve(w->sleeve);
|
|
|
|
i915_sw_fence_fini(&w->wait);
|
|
|
|
BUILD_BUG_ON(offsetof(typeof(*w), dma));
|
|
dma_fence_free(&w->dma);
|
|
}
|
|
|
|
static const struct dma_fence_ops clear_pages_work_ops = {
|
|
.get_driver_name = clear_pages_work_driver_name,
|
|
.get_timeline_name = clear_pages_work_timeline_name,
|
|
.release = clear_pages_work_release,
|
|
};
|
|
|
|
static void clear_pages_signal_irq_worker(struct irq_work *work)
|
|
{
|
|
struct clear_pages_work *w = container_of(work, typeof(*w), irq_work);
|
|
|
|
dma_fence_signal(&w->dma);
|
|
dma_fence_put(&w->dma);
|
|
}
|
|
|
|
static void clear_pages_dma_fence_cb(struct dma_fence *fence,
|
|
struct dma_fence_cb *cb)
|
|
{
|
|
struct clear_pages_work *w = container_of(cb, typeof(*w), cb);
|
|
|
|
if (fence->error)
|
|
dma_fence_set_error(&w->dma, fence->error);
|
|
|
|
/*
|
|
* Push the signalling of the fence into yet another worker to avoid
|
|
* the nightmare locking around the fence spinlock.
|
|
*/
|
|
irq_work_queue(&w->irq_work);
|
|
}
|
|
|
|
static void clear_pages_worker(struct work_struct *work)
|
|
{
|
|
struct clear_pages_work *w = container_of(work, typeof(*w), work);
|
|
struct drm_i915_private *i915 = w->ce->engine->i915;
|
|
struct drm_i915_gem_object *obj = w->sleeve->vma->obj;
|
|
struct i915_vma *vma = w->sleeve->vma;
|
|
struct i915_request *rq;
|
|
int err = w->dma.error;
|
|
|
|
if (unlikely(err))
|
|
goto out_signal;
|
|
|
|
if (obj->cache_dirty) {
|
|
if (i915_gem_object_has_struct_page(obj))
|
|
drm_clflush_sg(w->sleeve->pages);
|
|
obj->cache_dirty = false;
|
|
}
|
|
obj->read_domains = I915_GEM_GPU_DOMAINS;
|
|
obj->write_domain = 0;
|
|
|
|
/* XXX: we need to kill this */
|
|
mutex_lock(&i915->drm.struct_mutex);
|
|
err = i915_vma_pin(vma, 0, 0, PIN_USER);
|
|
if (unlikely(err))
|
|
goto out_unlock;
|
|
|
|
rq = i915_request_create(w->ce);
|
|
if (IS_ERR(rq)) {
|
|
err = PTR_ERR(rq);
|
|
goto out_unpin;
|
|
}
|
|
|
|
/* There's no way the fence has signalled */
|
|
if (dma_fence_add_callback(&rq->fence, &w->cb,
|
|
clear_pages_dma_fence_cb))
|
|
GEM_BUG_ON(1);
|
|
|
|
if (w->ce->engine->emit_init_breadcrumb) {
|
|
err = w->ce->engine->emit_init_breadcrumb(rq);
|
|
if (unlikely(err))
|
|
goto out_request;
|
|
}
|
|
|
|
/*
|
|
* w->dma is already exported via (vma|obj)->resv we need only
|
|
* keep track of the GPU activity within this vma/request, and
|
|
* propagate the signal from the request to w->dma.
|
|
*/
|
|
err = i915_active_ref(&vma->active, rq->fence.context, rq);
|
|
if (err)
|
|
goto out_request;
|
|
|
|
err = intel_emit_vma_fill_blt(rq, vma, w->value);
|
|
out_request:
|
|
if (unlikely(err)) {
|
|
i915_request_skip(rq, err);
|
|
err = 0;
|
|
}
|
|
|
|
i915_request_add(rq);
|
|
out_unpin:
|
|
i915_vma_unpin(vma);
|
|
out_unlock:
|
|
mutex_unlock(&i915->drm.struct_mutex);
|
|
out_signal:
|
|
if (unlikely(err)) {
|
|
dma_fence_set_error(&w->dma, err);
|
|
dma_fence_signal(&w->dma);
|
|
dma_fence_put(&w->dma);
|
|
}
|
|
}
|
|
|
|
static int __i915_sw_fence_call
|
|
clear_pages_work_notify(struct i915_sw_fence *fence,
|
|
enum i915_sw_fence_notify state)
|
|
{
|
|
struct clear_pages_work *w = container_of(fence, typeof(*w), wait);
|
|
|
|
switch (state) {
|
|
case FENCE_COMPLETE:
|
|
schedule_work(&w->work);
|
|
break;
|
|
|
|
case FENCE_FREE:
|
|
dma_fence_put(&w->dma);
|
|
break;
|
|
}
|
|
|
|
return NOTIFY_DONE;
|
|
}
|
|
|
|
static DEFINE_SPINLOCK(fence_lock);
|
|
|
|
/* XXX: better name please */
|
|
int i915_gem_schedule_fill_pages_blt(struct drm_i915_gem_object *obj,
|
|
struct intel_context *ce,
|
|
struct sg_table *pages,
|
|
struct i915_page_sizes *page_sizes,
|
|
u32 value)
|
|
{
|
|
struct drm_i915_private *i915 = to_i915(obj->base.dev);
|
|
struct clear_pages_work *work;
|
|
struct i915_sleeve *sleeve;
|
|
int err;
|
|
|
|
sleeve = create_sleeve(ce->vm, obj, pages, page_sizes);
|
|
if (IS_ERR(sleeve))
|
|
return PTR_ERR(sleeve);
|
|
|
|
work = kmalloc(sizeof(*work), GFP_KERNEL);
|
|
if (!work) {
|
|
destroy_sleeve(sleeve);
|
|
return -ENOMEM;
|
|
}
|
|
|
|
work->value = value;
|
|
work->sleeve = sleeve;
|
|
work->ce = ce;
|
|
|
|
INIT_WORK(&work->work, clear_pages_worker);
|
|
|
|
init_irq_work(&work->irq_work, clear_pages_signal_irq_worker);
|
|
|
|
dma_fence_init(&work->dma,
|
|
&clear_pages_work_ops,
|
|
&fence_lock,
|
|
i915->mm.unordered_timeline,
|
|
0);
|
|
i915_sw_fence_init(&work->wait, clear_pages_work_notify);
|
|
|
|
i915_gem_object_lock(obj);
|
|
err = i915_sw_fence_await_reservation(&work->wait,
|
|
obj->base.resv, NULL,
|
|
true, I915_FENCE_TIMEOUT,
|
|
I915_FENCE_GFP);
|
|
if (err < 0) {
|
|
dma_fence_set_error(&work->dma, err);
|
|
} else {
|
|
reservation_object_add_excl_fence(obj->base.resv, &work->dma);
|
|
err = 0;
|
|
}
|
|
i915_gem_object_unlock(obj);
|
|
|
|
dma_fence_get(&work->dma);
|
|
i915_sw_fence_commit(&work->wait);
|
|
|
|
return err;
|
|
}
|
|
|
|
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
|
|
#include "selftests/i915_gem_client_blt.c"
|
|
#endif
|