#include <linux/swap.h>
#include <linux/pci.h>
-static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
+static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
struct drm_i915_gem_pwrite *args,
struct drm_file *file_priv);
+static void i915_gem_free_object_tail(struct drm_gem_object *obj);
static LIST_HEAD(shrink_list);
static DEFINE_SPINLOCK(shrink_list_lock);
return -ENOMEM;
ret = drm_gem_handle_create(file_priv, obj, &handle);
- drm_gem_object_handle_unreference_unlocked(obj);
-
+ drm_gem_object_unreference_unlocked(obj);
if (ret)
return ret;
char *vaddr_atomic;
unsigned long unwritten;
- vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
+ vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
user_data, length);
- io_mapping_unmap_atomic(vaddr_atomic);
+ io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
if (unwritten)
return -EFAULT;
return 0;
/**
* This function clears the request list as sequence numbers are passed.
*/
-void
-i915_gem_retire_requests(struct drm_device *dev,
- struct intel_ring_buffer *ring)
+static void
+i915_gem_retire_requests_ring(struct drm_device *dev,
+ struct intel_ring_buffer *ring)
{
drm_i915_private_t *dev_priv = dev->dev_private;
uint32_t seqno;
}
}
+void
+i915_gem_retire_requests(struct drm_device *dev)
+{
+ drm_i915_private_t *dev_priv = dev->dev_private;
+
+ if (!list_empty(&dev_priv->mm.deferred_free_list)) {
+ struct drm_i915_gem_object *obj_priv, *tmp;
+
+ /* We must be careful that during unbind() we do not
+ * accidentally infinitely recurse into retire requests.
+ * Currently:
+ * retire -> free -> unbind -> wait -> retire_ring
+ */
+ list_for_each_entry_safe(obj_priv, tmp,
+ &dev_priv->mm.deferred_free_list,
+ list)
+ i915_gem_free_object_tail(&obj_priv->base);
+ }
+
+ i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
+ if (HAS_BSD(dev))
+ i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
+}
+
void
i915_gem_retire_work_handler(struct work_struct *work)
{
dev = dev_priv->dev;
mutex_lock(&dev->struct_mutex);
- i915_gem_retire_requests(dev, &dev_priv->render_ring);
-
- if (HAS_BSD(dev))
- i915_gem_retire_requests(dev, &dev_priv->bsd_ring);
+ i915_gem_retire_requests(dev);
if (!dev_priv->mm.suspended &&
(!list_empty(&dev_priv->render_ring.request_list) ||
* a separate wait queue to handle that.
*/
if (ret == 0)
- i915_gem_retire_requests(dev, ring);
+ i915_gem_retire_requests_ring(dev, ring);
return ret;
}
* before we unbind.
*/
ret = i915_gem_object_set_to_cpu_domain(obj, 1);
- if (ret) {
- if (ret != -ERESTARTSYS)
- DRM_ERROR("set_domain failed: %d\n", ret);
+ if (ret == -ERESTARTSYS)
return ret;
- }
+ /* Continue on if we fail due to EIO, the GPU is hung so we
+ * should be safe and we need to cleanup or else we might
+ * cause memory corruption through use-after-free.
+ */
BUG_ON(obj_priv->active);
trace_i915_gem_object_unbind(obj);
- return 0;
+ return ret;
}
static struct drm_gem_object *
struct intel_ring_buffer *render_ring = &dev_priv->render_ring;
struct intel_ring_buffer *bsd_ring = &dev_priv->bsd_ring;
for (;;) {
- i915_gem_retire_requests(dev, render_ring);
-
- if (HAS_BSD(dev))
- i915_gem_retire_requests(dev, bsd_ring);
+ i915_gem_retire_requests(dev);
/* If there's an inactive buffer available now, grab it
* and be done.
if (!IS_I965G(dev)) {
int ret;
- i915_gem_object_flush_gpu_write_domain(obj);
+ ret = i915_gem_object_flush_gpu_write_domain(obj);
+ if (ret != 0)
+ return ret;
+
ret = i915_gem_object_wait_rendering(obj);
if (ret != 0)
return ret;
if (free_space != NULL) {
obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
alignment);
- if (obj_priv->gtt_space != NULL) {
- obj_priv->gtt_space->private = obj;
+ if (obj_priv->gtt_space != NULL)
obj_priv->gtt_offset = obj_priv->gtt_space->start;
- }
}
if (obj_priv->gtt_space == NULL) {
/* If the gtt is empty and we're still having trouble
}
/** Flushes any GPU write domain for the object if it's dirty. */
-static void
+static int
i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
{
struct drm_device *dev = obj->dev;
struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
- return;
+ return 0;
/* Queue the GPU write cache flushing we need. */
old_write_domain = obj->write_domain;
i915_gem_flush(dev, 0, obj->write_domain);
- (void) i915_add_request(dev, NULL, obj->write_domain, obj_priv->ring);
- BUG_ON(obj->write_domain);
+ if (i915_add_request(dev, NULL, obj->write_domain, obj_priv->ring) == 0)
+ return -ENOMEM;
trace_i915_gem_object_change_domain(obj,
obj->read_domains,
old_write_domain);
+ return 0;
}
/** Flushes the GTT write domain for the object if it's dirty. */
old_write_domain);
}
-void
+int
i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
{
+ int ret = 0;
+
switch (obj->write_domain) {
case I915_GEM_DOMAIN_GTT:
i915_gem_object_flush_gtt_write_domain(obj);
i915_gem_object_flush_cpu_write_domain(obj);
break;
default:
- i915_gem_object_flush_gpu_write_domain(obj);
+ ret = i915_gem_object_flush_gpu_write_domain(obj);
break;
}
+
+ return ret;
}
/**
if (obj_priv->gtt_space == NULL)
return -EINVAL;
- i915_gem_object_flush_gpu_write_domain(obj);
+ ret = i915_gem_object_flush_gpu_write_domain(obj);
+ if (ret != 0)
+ return ret;
+
/* Wait on any GPU rendering and flushing to occur. */
ret = i915_gem_object_wait_rendering(obj);
if (ret != 0)
if (obj_priv->gtt_space == NULL)
return -EINVAL;
- i915_gem_object_flush_gpu_write_domain(obj);
+ ret = i915_gem_object_flush_gpu_write_domain(obj);
+ if (ret)
+ return ret;
/* Wait on any GPU rendering and flushing to occur. */
if (obj_priv->active) {
uint32_t old_write_domain, old_read_domains;
int ret;
- i915_gem_object_flush_gpu_write_domain(obj);
+ ret = i915_gem_object_flush_gpu_write_domain(obj);
+ if (ret)
+ return ret;
+
/* Wait on any GPU rendering and flushing to occur. */
ret = i915_gem_object_wait_rendering(obj);
if (ret != 0)
i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
{
struct drm_device *dev = obj->dev;
+ drm_i915_private_t *dev_priv = dev->dev_private;
struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
uint32_t invalidate_domains = 0;
uint32_t flush_domains = 0;
obj->pending_write_domain = obj->write_domain;
obj->read_domains = obj->pending_read_domains;
+ if (flush_domains & I915_GEM_GPU_DOMAINS) {
+ if (obj_priv->ring == &dev_priv->render_ring)
+ dev_priv->flush_rings |= FLUSH_RENDER_RING;
+ else if (obj_priv->ring == &dev_priv->bsd_ring)
+ dev_priv->flush_rings |= FLUSH_BSD_RING;
+ }
+
dev->invalidate_domains |= invalidate_domains;
dev->flush_domains |= flush_domains;
#if WATCH_BUF
if (offset == 0 && size == obj->size)
return i915_gem_object_set_to_cpu_domain(obj, 0);
- i915_gem_object_flush_gpu_write_domain(obj);
+ ret = i915_gem_object_flush_gpu_write_domain(obj);
+ if (ret)
+ return ret;
+
/* Wait on any GPU rendering and flushing to occur. */
ret = i915_gem_object_wait_rendering(obj);
if (ret != 0)
reloc_offset = obj_priv->gtt_offset + reloc->offset;
reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
(reloc_offset &
- ~(PAGE_SIZE - 1)));
+ ~(PAGE_SIZE - 1)),
+ KM_USER0);
reloc_entry = (uint32_t __iomem *)(reloc_page +
(reloc_offset & (PAGE_SIZE - 1)));
reloc_val = target_obj_priv->gtt_offset + reloc->delta;
readl(reloc_entry), reloc_val);
#endif
writel(reloc_val, reloc_entry);
- io_mapping_unmap_atomic(reloc_page);
+ io_mapping_unmap_atomic(reloc_page, KM_USER0);
/* The updated presumed offset for this entry will be
* copied back out to the user.
return ret;
}
+
int
i915_gem_do_execbuffer(struct drm_device *dev, void *data,
struct drm_file *file_priv,
ring = &dev_priv->render_ring;
}
-
if (args->buffer_count < 1) {
DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
return -EINVAL;
unsigned long long total_size = 0;
int num_fences = 0;
for (i = 0; i < args->buffer_count; i++) {
- obj_priv = object_list[i]->driver_private;
+ obj_priv = to_intel_bo(object_list[i]);
total_size += object_list[i]->size;
num_fences +=
*/
dev->invalidate_domains = 0;
dev->flush_domains = 0;
+ dev_priv->flush_rings = 0;
for (i = 0; i < args->buffer_count; i++) {
struct drm_gem_object *obj = object_list[i];
i915_gem_flush(dev,
dev->invalidate_domains,
dev->flush_domains);
- if (dev->flush_domains & I915_GEM_GPU_DOMAINS) {
+ if (dev_priv->flush_rings & FLUSH_RENDER_RING)
(void)i915_add_request(dev, file_priv,
- dev->flush_domains,
- &dev_priv->render_ring);
-
- if (HAS_BSD(dev))
- (void)i915_add_request(dev, file_priv,
- dev->flush_domains,
- &dev_priv->bsd_ring);
- }
+ dev->flush_domains,
+ &dev_priv->render_ring);
+ if (dev_priv->flush_rings & FLUSH_BSD_RING)
+ (void)i915_add_request(dev, file_priv,
+ dev->flush_domains,
+ &dev_priv->bsd_ring);
}
for (i = 0; i < args->buffer_count; i++) {
struct drm_i915_gem_busy *args = data;
struct drm_gem_object *obj;
struct drm_i915_gem_object *obj_priv;
- drm_i915_private_t *dev_priv = dev->dev_private;
obj = drm_gem_object_lookup(dev, file_priv, args->handle);
if (obj == NULL) {
}
mutex_lock(&dev->struct_mutex);
- /* Update the active list for the hardware's current position.
- * Otherwise this only updates on a delayed timer or when irqs are
- * actually unmasked, and our working set ends up being larger than
- * required.
+
+ /* Count all active objects as busy, even if they are currently not used
+ * by the gpu. Users of this interface expect objects to eventually
+ * become non-busy without any further actions, therefore emit any
+ * necessary flushes here.
*/
- i915_gem_retire_requests(dev, &dev_priv->render_ring);
+ obj_priv = to_intel_bo(obj);
+ args->busy = obj_priv->active;
+ if (args->busy) {
+ /* Unconditionally flush objects, even when the gpu still uses this
+ * object. Userspace calling this function indicates that it wants to
+ * use this buffer rather sooner than later, so issuing the required
+ * flush earlier is beneficial.
+ */
+ if (obj->write_domain) {
+ i915_gem_flush(dev, 0, obj->write_domain);
+ (void)i915_add_request(dev, file_priv, obj->write_domain, obj_priv->ring);
+ }
- if (HAS_BSD(dev))
- i915_gem_retire_requests(dev, &dev_priv->bsd_ring);
+ /* Update the active list for the hardware's current position.
+ * Otherwise this only updates on a delayed timer or when irqs
+ * are actually unmasked, and our working set ends up being
+ * larger than required.
+ */
+ i915_gem_retire_requests_ring(dev, obj_priv->ring);
- obj_priv = to_intel_bo(obj);
- /* Don't count being on the flushing list against the object being
- * done. Otherwise, a buffer left on the flushing list but not getting
- * flushed (because nobody's flushing that domain) won't ever return
- * unbusy and get reused by libdrm's bo cache. The other expected
- * consumer of this interface, OpenGL's occlusion queries, also specs
- * that the objects get unbusy "eventually" without any interference.
- */
- args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
+ args->busy = obj_priv->active;
+ }
drm_gem_object_unreference(obj);
mutex_unlock(&dev->struct_mutex);
return 0;
}
-void i915_gem_free_object(struct drm_gem_object *obj)
+static void i915_gem_free_object_tail(struct drm_gem_object *obj)
{
struct drm_device *dev = obj->dev;
+ drm_i915_private_t *dev_priv = dev->dev_private;
struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+ int ret;
- trace_i915_gem_object_destroy(obj);
-
- while (obj_priv->pin_count > 0)
- i915_gem_object_unpin(obj);
-
- if (obj_priv->phys_obj)
- i915_gem_detach_phys_object(dev, obj);
-
- i915_gem_object_unbind(obj);
+ ret = i915_gem_object_unbind(obj);
+ if (ret == -ERESTARTSYS) {
+ list_move(&obj_priv->list,
+ &dev_priv->mm.deferred_free_list);
+ return;
+ }
if (obj_priv->mmap_offset)
i915_gem_free_mmap_offset(obj);
kfree(obj_priv);
}
+void i915_gem_free_object(struct drm_gem_object *obj)
+{
+ struct drm_device *dev = obj->dev;
+ struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
+
+ trace_i915_gem_object_destroy(obj);
+
+ while (obj_priv->pin_count > 0)
+ i915_gem_object_unpin(obj);
+
+ if (obj_priv->phys_obj)
+ i915_gem_detach_phys_object(dev, obj);
+
+ i915_gem_free_object_tail(obj);
+}
+
/** Unbinds all inactive objects. */
static int
i915_gem_evict_from_inactive_list(struct drm_device *dev)
BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
mutex_unlock(&dev->struct_mutex);
- drm_irq_install(dev);
+ ret = drm_irq_install(dev);
+ if (ret)
+ goto cleanup_ringbuffer;
return 0;
+
+cleanup_ringbuffer:
+ mutex_lock(&dev->struct_mutex);
+ i915_gem_cleanup_ringbuffer(dev);
+ dev_priv->mm.suspended = 1;
+ mutex_unlock(&dev->struct_mutex);
+
+ return ret;
}
int
INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
INIT_LIST_HEAD(&dev_priv->mm.fence_list);
+ INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
if (HAS_BSD(dev)) {
continue;
spin_unlock(&shrink_list_lock);
- i915_gem_retire_requests(dev, &dev_priv->render_ring);
-
- if (HAS_BSD(dev))
- i915_gem_retire_requests(dev, &dev_priv->bsd_ring);
+ i915_gem_retire_requests(dev);
list_for_each_entry_safe(obj_priv, next_obj,
&dev_priv->mm.inactive_list,