#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/pci.h>
+#include <linux/intel-gtt.h>
+static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
unsigned alignment);
static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
-static int i915_gem_evict_something(struct drm_device *dev, int min_size);
-static int i915_gem_evict_from_inactive_list(struct drm_device *dev);
static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
struct drm_i915_gem_pwrite *args,
struct drm_file *file_priv);
static LIST_HEAD(shrink_list);
static DEFINE_SPINLOCK(shrink_list_lock);
+static inline bool
+i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
+{
+ return obj_priv->gtt_space &&
+ !obj_priv->active &&
+ obj_priv->pin_count == 0;
+}
+
int i915_gem_do_init(struct drm_device *dev, unsigned long start,
unsigned long end)
{
return -ENOMEM;
ret = drm_gem_handle_create(file_priv, obj, &handle);
- drm_gem_object_unreference_unlocked(obj);
- if (ret)
+ if (ret) {
+ drm_gem_object_unreference_unlocked(obj);
return ret;
+ }
- args->handle = handle;
+ /* Sink the floating reference from kref_init(handlecount) */
+ drm_gem_object_handle_unreference_unlocked(obj);
+ args->handle = handle;
return 0;
}
if (ret == -ENOMEM) {
struct drm_device *dev = obj->dev;
- ret = i915_gem_evict_something(dev, obj->size);
+ ret = i915_gem_evict_something(dev, obj->size,
+ i915_gem_get_gtt_alignment(obj));
if (ret)
return ret;
obj = drm_gem_object_lookup(dev, file_priv, args->handle);
if (obj == NULL)
- return -EBADF;
+ return -ENOENT;
obj_priv = to_intel_bo(obj);
/* Bounds check source.
obj = drm_gem_object_lookup(dev, file_priv, args->handle);
if (obj == NULL)
- return -EBADF;
+ return -ENOENT;
obj_priv = to_intel_bo(obj);
/* Bounds check destination.
obj = drm_gem_object_lookup(dev, file_priv, args->handle);
if (obj == NULL)
- return -EBADF;
+ return -ENOENT;
obj_priv = to_intel_bo(obj);
mutex_lock(&dev->struct_mutex);
ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
}
+
+ /* Maintain LRU order of "inactive" objects */
+ if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
+ list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
+
drm_gem_object_unreference(obj);
mutex_unlock(&dev->struct_mutex);
return ret;
obj = drm_gem_object_lookup(dev, file_priv, args->handle);
if (obj == NULL) {
mutex_unlock(&dev->struct_mutex);
- return -EBADF;
+ return -ENOENT;
}
#if WATCH_BUF
obj = drm_gem_object_lookup(dev, file_priv, args->handle);
if (obj == NULL)
- return -EBADF;
+ return -ENOENT;
offset = args->offset;
{
struct drm_gem_object *obj = vma->vm_private_data;
struct drm_device *dev = obj->dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
+ drm_i915_private_t *dev_priv = dev->dev_private;
struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
pgoff_t page_offset;
unsigned long pfn;
if (ret)
goto unlock;
- list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
-
ret = i915_gem_object_set_to_gtt_domain(obj, write);
if (ret)
goto unlock;
goto unlock;
}
+ if (i915_gem_object_is_inactive(obj_priv))
+ list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
+
pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
page_offset;
struct drm_file *file_priv)
{
struct drm_i915_gem_mmap_gtt *args = data;
- struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_gem_object *obj;
struct drm_i915_gem_object *obj_priv;
int ret;
obj = drm_gem_object_lookup(dev, file_priv, args->handle);
if (obj == NULL)
- return -EBADF;
+ return -ENOENT;
mutex_lock(&dev->struct_mutex);
mutex_unlock(&dev->struct_mutex);
return ret;
}
- list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
}
drm_gem_object_unreference(obj);
struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
struct inode *inode;
+ /* Our goal here is to return as much of the memory as
+ * is possible back to the system as we are called from OOM.
+ * To do this we must instruct the shmfs to drop all of its
+ * backing pages, *now*. Here we mirror the actions taken
+ * when by shmem_delete_inode() to release the backing store.
+ */
inode = obj->filp->f_path.dentry->d_inode;
- if (inode->i_op->truncate)
- inode->i_op->truncate (inode);
+ truncate_inode_pages(inode->i_mapping, 0);
+ if (inode->i_op->truncate_range)
+ inode->i_op->truncate_range(inode, 0, (loff_t)-1);
obj_priv->madv = __I915_MADV_PURGED;
}
flush_domains);
}
-static void
-i915_gem_flush_ring(struct drm_device *dev,
- uint32_t invalidate_domains,
- uint32_t flush_domains,
- struct intel_ring_buffer *ring)
-{
- if (flush_domains & I915_GEM_DOMAIN_CPU)
- drm_agp_chipset_flush(dev);
- ring->flush(dev, ring,
- invalidate_domains,
- flush_domains);
-}
-
/**
* Ensures that all rendering to the object has completed and the object is
* safe to unbind from the GTT or access from the CPU.
* cause memory corruption through use-after-free.
*/
- BUG_ON(obj_priv->active);
-
/* release the fence reg _after_ flushing */
if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
i915_gem_clear_fence_reg(obj);
return ret;
}
-static struct drm_gem_object *
-i915_gem_find_inactive_object(struct drm_device *dev, int min_size)
-{
- drm_i915_private_t *dev_priv = dev->dev_private;
- struct drm_i915_gem_object *obj_priv;
- struct drm_gem_object *best = NULL;
- struct drm_gem_object *first = NULL;
-
- /* Try to find the smallest clean object */
- list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
- struct drm_gem_object *obj = &obj_priv->base;
- if (obj->size >= min_size) {
- if ((!obj_priv->dirty ||
- i915_gem_object_is_purgeable(obj_priv)) &&
- (!best || obj->size < best->size)) {
- best = obj;
- if (best->size == min_size)
- return best;
- }
- if (!first)
- first = obj;
- }
- }
-
- return best ? best : first;
-}
-
-static int
+int
i915_gpu_idle(struct drm_device *dev)
{
drm_i915_private_t *dev_priv = dev->dev_private;
return ret;
}
-static int
-i915_gem_evict_everything(struct drm_device *dev)
-{
- drm_i915_private_t *dev_priv = dev->dev_private;
- int ret;
- bool lists_empty;
-
- spin_lock(&dev_priv->mm.active_list_lock);
- lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
- list_empty(&dev_priv->mm.flushing_list) &&
- list_empty(&dev_priv->render_ring.active_list) &&
- (!HAS_BSD(dev)
- || list_empty(&dev_priv->bsd_ring.active_list)));
- spin_unlock(&dev_priv->mm.active_list_lock);
-
- if (lists_empty)
- return -ENOSPC;
-
- /* Flush everything (on to the inactive lists) and evict */
- ret = i915_gpu_idle(dev);
- if (ret)
- return ret;
-
- BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
-
- ret = i915_gem_evict_from_inactive_list(dev);
- if (ret)
- return ret;
-
- spin_lock(&dev_priv->mm.active_list_lock);
- lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
- list_empty(&dev_priv->mm.flushing_list) &&
- list_empty(&dev_priv->render_ring.active_list) &&
- (!HAS_BSD(dev)
- || list_empty(&dev_priv->bsd_ring.active_list)));
- spin_unlock(&dev_priv->mm.active_list_lock);
- BUG_ON(!lists_empty);
-
- return 0;
-}
-
-static int
-i915_gem_evict_something(struct drm_device *dev, int min_size)
-{
- drm_i915_private_t *dev_priv = dev->dev_private;
- struct drm_gem_object *obj;
- int ret;
-
- struct intel_ring_buffer *render_ring = &dev_priv->render_ring;
- struct intel_ring_buffer *bsd_ring = &dev_priv->bsd_ring;
- for (;;) {
- i915_gem_retire_requests(dev);
-
- /* If there's an inactive buffer available now, grab it
- * and be done.
- */
- obj = i915_gem_find_inactive_object(dev, min_size);
- if (obj) {
- struct drm_i915_gem_object *obj_priv;
-
-#if WATCH_LRU
- DRM_INFO("%s: evicting %p\n", __func__, obj);
-#endif
- obj_priv = to_intel_bo(obj);
- BUG_ON(obj_priv->pin_count != 0);
- BUG_ON(obj_priv->active);
-
- /* Wait on the rendering and unbind the buffer. */
- return i915_gem_object_unbind(obj);
- }
-
- /* If we didn't get anything, but the ring is still processing
- * things, wait for the next to finish and hopefully leave us
- * a buffer to evict.
- */
- if (!list_empty(&render_ring->request_list)) {
- struct drm_i915_gem_request *request;
-
- request = list_first_entry(&render_ring->request_list,
- struct drm_i915_gem_request,
- list);
-
- ret = i915_wait_request(dev,
- request->seqno, request->ring);
- if (ret)
- return ret;
-
- continue;
- }
-
- if (HAS_BSD(dev) && !list_empty(&bsd_ring->request_list)) {
- struct drm_i915_gem_request *request;
-
- request = list_first_entry(&bsd_ring->request_list,
- struct drm_i915_gem_request,
- list);
-
- ret = i915_wait_request(dev,
- request->seqno, request->ring);
- if (ret)
- return ret;
-
- continue;
- }
-
- /* If we didn't have anything on the request list but there
- * are buffers awaiting a flush, emit one and try again.
- * When we wait on it, those buffers waiting for that flush
- * will get moved to inactive.
- */
- if (!list_empty(&dev_priv->mm.flushing_list)) {
- struct drm_i915_gem_object *obj_priv;
-
- /* Find an object that we can immediately reuse */
- list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
- obj = &obj_priv->base;
- if (obj->size >= min_size)
- break;
-
- obj = NULL;
- }
-
- if (obj != NULL) {
- uint32_t seqno;
-
- i915_gem_flush_ring(dev,
- obj->write_domain,
- obj->write_domain,
- obj_priv->ring);
- seqno = i915_add_request(dev, NULL,
- obj->write_domain,
- obj_priv->ring);
- if (seqno == 0)
- return -ENOMEM;
- continue;
- }
- }
-
- /* If we didn't do any of the above, there's no single buffer
- * large enough to swap out for the new one, so just evict
- * everything and start again. (This should be rare.)
- */
- if (!list_empty (&dev_priv->mm.inactive_list))
- return i915_gem_evict_from_inactive_list(dev);
- else
- return i915_gem_evict_everything(dev);
- }
-}
-
int
i915_gem_object_get_pages(struct drm_gem_object *obj,
gfp_t gfpmask)
#if WATCH_LRU
DRM_INFO("%s: GTT full, evicting something\n", __func__);
#endif
- ret = i915_gem_evict_something(dev, obj->size);
+ ret = i915_gem_evict_something(dev, obj->size, alignment);
if (ret)
return ret;
if (ret == -ENOMEM) {
/* first try to clear up some space from the GTT */
- ret = i915_gem_evict_something(dev, obj->size);
+ ret = i915_gem_evict_something(dev, obj->size,
+ alignment);
if (ret) {
/* now try to shrink everyone else */
if (gfpmask) {
drm_mm_put_block(obj_priv->gtt_space);
obj_priv->gtt_space = NULL;
- ret = i915_gem_evict_something(dev, obj->size);
+ ret = i915_gem_evict_something(dev, obj->size, alignment);
if (ret)
return ret;
atomic_inc(&dev->gtt_count);
atomic_add(obj->size, &dev->gtt_memory);
+ /* keep track of bounds object by adding it to the inactive list */
+ list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
+
/* Assert that the object is not currently in any GPU domain. As it
* wasn't in the GTT, there shouldn't be any way it could have been in
* a GPU cache
i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
{
struct drm_device *dev = obj->dev;
+ drm_i915_private_t *dev_priv = dev->dev_private;
struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
uint32_t invalidate_domains = 0;
uint32_t flush_domains = 0;
obj->pending_write_domain = obj->write_domain;
obj->read_domains = obj->pending_read_domains;
+ if (flush_domains & I915_GEM_GPU_DOMAINS) {
+ if (obj_priv->ring == &dev_priv->render_ring)
+ dev_priv->flush_rings |= FLUSH_RENDER_RING;
+ else if (obj_priv->ring == &dev_priv->bsd_ring)
+ dev_priv->flush_rings |= FLUSH_BSD_RING;
+ }
+
dev->invalidate_domains |= invalidate_domains;
dev->flush_domains |= flush_domains;
#if WATCH_BUF
reloc->target_handle);
if (target_obj == NULL) {
i915_gem_object_unpin(obj);
- return -EBADF;
+ return -ENOENT;
}
target_obj_priv = to_intel_bo(target_obj);
ring = &dev_priv->render_ring;
}
-
if (args->buffer_count < 1) {
DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
return -EINVAL;
if (ret != 0) {
DRM_ERROR("copy %d cliprects failed: %d\n",
args->num_cliprects, ret);
+ ret = -EFAULT;
goto pre_mutex_err;
}
}
exec_list[i].handle, i);
/* prevent error path from reading uninitialized data */
args->buffer_count = i + 1;
- ret = -EBADF;
+ ret = -ENOENT;
goto err;
}
object_list[i]);
/* prevent error path from reading uninitialized data */
args->buffer_count = i + 1;
- ret = -EBADF;
+ ret = -EINVAL;
goto err;
}
obj_priv->in_execbuffer = true;
*/
dev->invalidate_domains = 0;
dev->flush_domains = 0;
+ dev_priv->flush_rings = 0;
for (i = 0; i < args->buffer_count; i++) {
struct drm_gem_object *obj = object_list[i];
i915_gem_flush(dev,
dev->invalidate_domains,
dev->flush_domains);
- if (dev->flush_domains & I915_GEM_GPU_DOMAINS) {
+ if (dev_priv->flush_rings & FLUSH_RENDER_RING)
(void)i915_add_request(dev, file_priv,
- dev->flush_domains,
- &dev_priv->render_ring);
-
- if (HAS_BSD(dev))
- (void)i915_add_request(dev, file_priv,
- dev->flush_domains,
- &dev_priv->bsd_ring);
- }
+ dev->flush_domains,
+ &dev_priv->render_ring);
+ if (dev_priv->flush_rings & FLUSH_BSD_RING)
+ (void)i915_add_request(dev, file_priv,
+ dev->flush_domains,
+ &dev_priv->bsd_ring);
}
for (i = 0; i < args->buffer_count; i++) {
if (alignment == 0)
alignment = i915_gem_get_gtt_alignment(obj);
if (obj_priv->gtt_offset & (alignment - 1)) {
+ WARN(obj_priv->pin_count,
+ "bo is already pinned with incorrect alignment:"
+ " offset=%x, req.alignment=%x\n",
+ obj_priv->gtt_offset, alignment);
ret = i915_gem_object_unbind(obj);
if (ret)
return ret;
atomic_inc(&dev->pin_count);
atomic_add(obj->size, &dev->pin_memory);
if (!obj_priv->active &&
- (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0 &&
- !list_empty(&obj_priv->list))
+ (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
list_del_init(&obj_priv->list);
}
i915_verify_inactive(dev, __FILE__, __LINE__);
DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
args->handle);
mutex_unlock(&dev->struct_mutex);
- return -EBADF;
+ return -ENOENT;
}
obj_priv = to_intel_bo(obj);
DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
args->handle);
mutex_unlock(&dev->struct_mutex);
- return -EBADF;
+ return -ENOENT;
}
obj_priv = to_intel_bo(obj);
if (obj == NULL) {
DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
args->handle);
- return -EBADF;
+ return -ENOENT;
}
mutex_lock(&dev->struct_mutex);
- /* Update the active list for the hardware's current position.
- * Otherwise this only updates on a delayed timer or when irqs are
- * actually unmasked, and our working set ends up being larger than
- * required.
- */
- i915_gem_retire_requests(dev);
- obj_priv = to_intel_bo(obj);
- /* Don't count being on the flushing list against the object being
- * done. Otherwise, a buffer left on the flushing list but not getting
- * flushed (because nobody's flushing that domain) won't ever return
- * unbusy and get reused by libdrm's bo cache. The other expected
- * consumer of this interface, OpenGL's occlusion queries, also specs
- * that the objects get unbusy "eventually" without any interference.
+ /* Count all active objects as busy, even if they are currently not used
+ * by the gpu. Users of this interface expect objects to eventually
+ * become non-busy without any further actions, therefore emit any
+ * necessary flushes here.
*/
- args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
+ obj_priv = to_intel_bo(obj);
+ args->busy = obj_priv->active;
+ if (args->busy) {
+ /* Unconditionally flush objects, even when the gpu still uses this
+ * object. Userspace calling this function indicates that it wants to
+ * use this buffer rather sooner than later, so issuing the required
+ * flush earlier is beneficial.
+ */
+ if (obj->write_domain) {
+ i915_gem_flush(dev, 0, obj->write_domain);
+ (void)i915_add_request(dev, file_priv, obj->write_domain, obj_priv->ring);
+ }
+
+ /* Update the active list for the hardware's current position.
+ * Otherwise this only updates on a delayed timer or when irqs
+ * are actually unmasked, and our working set ends up being
+ * larger than required.
+ */
+ i915_gem_retire_requests_ring(dev, obj_priv->ring);
+
+ args->busy = obj_priv->active;
+ }
drm_gem_object_unreference(obj);
mutex_unlock(&dev->struct_mutex);
if (obj == NULL) {
DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
args->handle);
- return -EBADF;
+ return -ENOENT;
}
mutex_lock(&dev->struct_mutex);
i915_gem_free_object_tail(obj);
}
-/** Unbinds all inactive objects. */
-static int
-i915_gem_evict_from_inactive_list(struct drm_device *dev)
-{
- drm_i915_private_t *dev_priv = dev->dev_private;
-
- while (!list_empty(&dev_priv->mm.inactive_list)) {
- struct drm_gem_object *obj;
- int ret;
-
- obj = &list_first_entry(&dev_priv->mm.inactive_list,
- struct drm_i915_gem_object,
- list)->base;
-
- ret = i915_gem_object_unbind(obj);
- if (ret != 0) {
- DRM_ERROR("Error unbinding object: %d\n", ret);
- return ret;
- }
- }
-
- return 0;
-}
-
int
i915_gem_idle(struct drm_device *dev)
{
/* Under UMS, be paranoid and evict. */
if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
- ret = i915_gem_evict_from_inactive_list(dev);
+ ret = i915_gem_evict_inactive(dev);
if (ret) {
mutex_unlock(&dev->struct_mutex);
return ret;
goto cleanup_render_ring;
}
+ dev_priv->next_seqno = 1;
+
return 0;
cleanup_render_ring:
* e.g. for cursor + overlay regs
*/
int i915_gem_init_phys_object(struct drm_device *dev,
- int id, int size)
+ int id, int size, int align)
{
drm_i915_private_t *dev_priv = dev->dev_private;
struct drm_i915_gem_phys_object *phys_obj;
phys_obj->id = id;
- phys_obj->handle = drm_pci_alloc(dev, size, 0);
+ phys_obj->handle = drm_pci_alloc(dev, size, align);
if (!phys_obj->handle) {
ret = -ENOMEM;
goto kfree_obj;
int
i915_gem_attach_phys_object(struct drm_device *dev,
- struct drm_gem_object *obj, int id)
+ struct drm_gem_object *obj,
+ int id,
+ int align)
{
drm_i915_private_t *dev_priv = dev->dev_private;
struct drm_i915_gem_object *obj_priv;
i915_gem_detach_phys_object(dev, obj);
}
-
/* create a new object */
if (!dev_priv->mm.phys_objs[id - 1]) {
ret = i915_gem_init_phys_object(dev, id,
- obj->size);
+ obj->size, align);
if (ret) {
DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
goto out;
projects / firefly-linux-kernel-4.4.55.git / blobdiff