drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37 #include <linux/intel-gtt.h>
  38
  39 static uint32_t i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj_priv);
  40 static uint32_t i915_gem_get_gtt_size(struct drm_i915_gem_object *obj_priv);
  41
  42 static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
  43                                                   bool pipelined);
  44 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  45 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  46 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  47                                              int write);
  48 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  49                                                      uint64_t offset,
  50                                                      uint64_t size);
  51 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  52 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
  53                                           bool interruptible);
  54 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  55                                        unsigned alignment,
  56                                        bool mappable,
  57                                        bool need_fence);
  58 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  59 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  60                                 struct drm_i915_gem_pwrite *args,
  61                                 struct drm_file *file_priv);
  62 static void i915_gem_free_object_tail(struct drm_gem_object *obj);
  63
  64 static int i915_gem_inactive_shrink(struct shrinker *shrinker,
  65                                     int nr_to_scan,
  66                                     gfp_t gfp_mask);
  67
  68
  69 /* some bookkeeping */
  70 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  71                                   size_t size)
  72 {
  73         dev_priv->mm.object_count++;
  74         dev_priv->mm.object_memory += size;
  75 }
  76
  77 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  78                                      size_t size)
  79 {
  80         dev_priv->mm.object_count--;
  81         dev_priv->mm.object_memory -= size;
  82 }
  83
  84 static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
  85                                   struct drm_i915_gem_object *obj)
  86 {
  87         dev_priv->mm.gtt_count++;
  88         dev_priv->mm.gtt_memory += obj->gtt_space->size;
  89         if (obj->gtt_offset < dev_priv->mm.gtt_mappable_end) {
  90                 dev_priv->mm.mappable_gtt_used +=
  91                         min_t(size_t, obj->gtt_space->size,
  92                               dev_priv->mm.gtt_mappable_end - obj->gtt_offset);
  93         }
  94 }
  95
  96 static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
  97                                      struct drm_i915_gem_object *obj)
  98 {
  99         dev_priv->mm.gtt_count--;
 100         dev_priv->mm.gtt_memory -= obj->gtt_space->size;
 101         if (obj->gtt_offset < dev_priv->mm.gtt_mappable_end) {
 102                 dev_priv->mm.mappable_gtt_used -=
 103                         min_t(size_t, obj->gtt_space->size,
 104                               dev_priv->mm.gtt_mappable_end - obj->gtt_offset);
 105         }
 106 }
 107
 108 /**
 109  * Update the mappable working set counters. Call _only_ when there is a change
 110  * in one of (pin|fault)_mappable and update *_mappable _before_ calling.
 111  * @mappable: new state the changed mappable flag (either pin_ or fault_).
 112  */
 113 static void
 114 i915_gem_info_update_mappable(struct drm_i915_private *dev_priv,
 115                               struct drm_i915_gem_object *obj,
 116                               bool mappable)
 117 {
 118         if (mappable) {
 119                 if (obj->pin_mappable && obj->fault_mappable)
 120                         /* Combined state was already mappable. */
 121                         return;
 122                 dev_priv->mm.gtt_mappable_count++;
 123                 dev_priv->mm.gtt_mappable_memory += obj->gtt_space->size;
 124         } else {
 125                 if (obj->pin_mappable || obj->fault_mappable)
 126                         /* Combined state still mappable. */
 127                         return;
 128                 dev_priv->mm.gtt_mappable_count--;
 129                 dev_priv->mm.gtt_mappable_memory -= obj->gtt_space->size;
 130         }
 131 }
 132
 133 static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
 134                                   struct drm_i915_gem_object *obj,
 135                                   bool mappable)
 136 {
 137         dev_priv->mm.pin_count++;
 138         dev_priv->mm.pin_memory += obj->gtt_space->size;
 139         if (mappable) {
 140                 obj->pin_mappable = true;
 141                 i915_gem_info_update_mappable(dev_priv, obj, true);
 142         }
 143 }
 144
 145 static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
 146                                      struct drm_i915_gem_object *obj)
 147 {
 148         dev_priv->mm.pin_count--;
 149         dev_priv->mm.pin_memory -= obj->gtt_space->size;
 150         if (obj->pin_mappable) {
 151                 obj->pin_mappable = false;
 152                 i915_gem_info_update_mappable(dev_priv, obj, false);
 153         }
 154 }
 155
 156 int
 157 i915_gem_check_is_wedged(struct drm_device *dev)
 158 {
 159         struct drm_i915_private *dev_priv = dev->dev_private;
 160         struct completion *x = &dev_priv->error_completion;
 161         unsigned long flags;
 162         int ret;
 163
 164         if (!atomic_read(&dev_priv->mm.wedged))
 165                 return 0;
 166
 167         ret = wait_for_completion_interruptible(x);
 168         if (ret)
 169                 return ret;
 170
 171         /* Success, we reset the GPU! */
 172         if (!atomic_read(&dev_priv->mm.wedged))
 173                 return 0;
 174
 175         /* GPU is hung, bump the completion count to account for
 176          * the token we just consumed so that we never hit zero and
 177          * end up waiting upon a subsequent completion event that
 178          * will never happen.
 179          */
 180         spin_lock_irqsave(&x->wait.lock, flags);
 181         x->done++;
 182         spin_unlock_irqrestore(&x->wait.lock, flags);
 183         return -EIO;
 184 }
 185
 186 static int i915_mutex_lock_interruptible(struct drm_device *dev)
 187 {
 188         struct drm_i915_private *dev_priv = dev->dev_private;
 189         int ret;
 190
 191         ret = i915_gem_check_is_wedged(dev);
 192         if (ret)
 193                 return ret;
 194
 195         ret = mutex_lock_interruptible(&dev->struct_mutex);
 196         if (ret)
 197                 return ret;
 198
 199         if (atomic_read(&dev_priv->mm.wedged)) {
 200                 mutex_unlock(&dev->struct_mutex);
 201                 return -EAGAIN;
 202         }
 203
 204         WARN_ON(i915_verify_lists(dev));
 205         return 0;
 206 }
 207
 208 static inline bool
 209 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
 210 {
 211         return obj_priv->gtt_space &&
 212                 !obj_priv->active &&
 213                 obj_priv->pin_count == 0;
 214 }
 215
 216 int i915_gem_do_init(struct drm_device *dev,
 217                      unsigned long start,
 218                      unsigned long mappable_end,
 219                      unsigned long end)
 220 {
 221         drm_i915_private_t *dev_priv = dev->dev_private;
 222
 223         if (start >= end ||
 224             (start & (PAGE_SIZE - 1)) != 0 ||
 225             (end & (PAGE_SIZE - 1)) != 0) {
 226                 return -EINVAL;
 227         }
 228
 229         drm_mm_init(&dev_priv->mm.gtt_space, start,
 230                     end - start);
 231
 232         dev_priv->mm.gtt_total = end - start;
 233         dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
 234         dev_priv->mm.gtt_mappable_end = mappable_end;
 235
 236         return 0;
 237 }
 238
 239 int
 240 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 241                     struct drm_file *file_priv)
 242 {
 243         struct drm_i915_gem_init *args = data;
 244         int ret;
 245
 246         mutex_lock(&dev->struct_mutex);
 247         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
 248         mutex_unlock(&dev->struct_mutex);
 249
 250         return ret;
 251 }
 252
 253 int
 254 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 255                             struct drm_file *file_priv)
 256 {
 257         struct drm_i915_private *dev_priv = dev->dev_private;
 258         struct drm_i915_gem_get_aperture *args = data;
 259
 260         if (!(dev->driver->driver_features & DRIVER_GEM))
 261                 return -ENODEV;
 262
 263         mutex_lock(&dev->struct_mutex);
 264         args->aper_size = dev_priv->mm.gtt_total;
 265         args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory;
 266         mutex_unlock(&dev->struct_mutex);
 267
 268         return 0;
 269 }
 270
 271
 272 /**
 273  * Creates a new mm object and returns a handle to it.
 274  */
 275 int
 276 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 277                       struct drm_file *file_priv)
 278 {
 279         struct drm_i915_gem_create *args = data;
 280         struct drm_gem_object *obj;
 281         int ret;
 282         u32 handle;
 283
 284         args->size = roundup(args->size, PAGE_SIZE);
 285
 286         /* Allocate the new object */
 287         obj = i915_gem_alloc_object(dev, args->size);
 288         if (obj == NULL)
 289                 return -ENOMEM;
 290
 291         ret = drm_gem_handle_create(file_priv, obj, &handle);
 292         if (ret) {
 293                 drm_gem_object_release(obj);
 294                 i915_gem_info_remove_obj(dev->dev_private, obj->size);
 295                 kfree(obj);
 296                 return ret;
 297         }
 298
 299         /* drop reference from allocate - handle holds it now */
 300         drm_gem_object_unreference(obj);
 301         trace_i915_gem_object_create(obj);
 302
 303         args->handle = handle;
 304         return 0;
 305 }
 306
 307 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 308 {
 309         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 310         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 311
 312         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 313                 obj_priv->tiling_mode != I915_TILING_NONE;
 314 }
 315
 316 static inline void
 317 slow_shmem_copy(struct page *dst_page,
 318                 int dst_offset,
 319                 struct page *src_page,
 320                 int src_offset,
 321                 int length)
 322 {
 323         char *dst_vaddr, *src_vaddr;
 324
 325         dst_vaddr = kmap(dst_page);
 326         src_vaddr = kmap(src_page);
 327
 328         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 329
 330         kunmap(src_page);
 331         kunmap(dst_page);
 332 }
 333
 334 static inline void
 335 slow_shmem_bit17_copy(struct page *gpu_page,
 336                       int gpu_offset,
 337                       struct page *cpu_page,
 338                       int cpu_offset,
 339                       int length,
 340                       int is_read)
 341 {
 342         char *gpu_vaddr, *cpu_vaddr;
 343
 344         /* Use the unswizzled path if this page isn't affected. */
 345         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 346                 if (is_read)
 347                         return slow_shmem_copy(cpu_page, cpu_offset,
 348                                                gpu_page, gpu_offset, length);
 349                 else
 350                         return slow_shmem_copy(gpu_page, gpu_offset,
 351                                                cpu_page, cpu_offset, length);
 352         }
 353
 354         gpu_vaddr = kmap(gpu_page);
 355         cpu_vaddr = kmap(cpu_page);
 356
 357         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 358          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 359          */
 360         while (length > 0) {
 361                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 362                 int this_length = min(cacheline_end - gpu_offset, length);
 363                 int swizzled_gpu_offset = gpu_offset ^ 64;
 364
 365                 if (is_read) {
 366                         memcpy(cpu_vaddr + cpu_offset,
 367                                gpu_vaddr + swizzled_gpu_offset,
 368                                this_length);
 369                 } else {
 370                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 371                                cpu_vaddr + cpu_offset,
 372                                this_length);
 373                 }
 374                 cpu_offset += this_length;
 375                 gpu_offset += this_length;
 376                 length -= this_length;
 377         }
 378
 379         kunmap(cpu_page);
 380         kunmap(gpu_page);
 381 }
 382
 383 /**
 384  * This is the fast shmem pread path, which attempts to copy_from_user directly
 385  * from the backing pages of the object to the user's address space.  On a
 386  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 387  */
 388 static int
 389 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 390                           struct drm_i915_gem_pread *args,
 391                           struct drm_file *file_priv)
 392 {
 393         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 394         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 395         ssize_t remain;
 396         loff_t offset;
 397         char __user *user_data;
 398         int page_offset, page_length;
 399
 400         user_data = (char __user *) (uintptr_t) args->data_ptr;
 401         remain = args->size;
 402
 403         obj_priv = to_intel_bo(obj);
 404         offset = args->offset;
 405
 406         while (remain > 0) {
 407                 struct page *page;
 408                 char *vaddr;
 409                 int ret;
 410
 411                 /* Operation in this page
 412                  *
 413                  * page_offset = offset within page
 414                  * page_length = bytes to copy for this page
 415                  */
 416                 page_offset = offset & (PAGE_SIZE-1);
 417                 page_length = remain;
 418                 if ((page_offset + remain) > PAGE_SIZE)
 419                         page_length = PAGE_SIZE - page_offset;
 420
 421                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 422                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 423                 if (IS_ERR(page))
 424                         return PTR_ERR(page);
 425
 426                 vaddr = kmap_atomic(page);
 427                 ret = __copy_to_user_inatomic(user_data,
 428                                               vaddr + page_offset,
 429                                               page_length);
 430                 kunmap_atomic(vaddr);
 431
 432                 mark_page_accessed(page);
 433                 page_cache_release(page);
 434                 if (ret)
 435                         return -EFAULT;
 436
 437                 remain -= page_length;
 438                 user_data += page_length;
 439                 offset += page_length;
 440         }
 441
 442         return 0;
 443 }
 444
 445 /**
 446  * This is the fallback shmem pread path, which allocates temporary storage
 447  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 448  * can copy out of the object's backing pages while holding the struct mutex
 449  * and not take page faults.
 450  */
 451 static int
 452 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 453                           struct drm_i915_gem_pread *args,
 454                           struct drm_file *file_priv)
 455 {
 456         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 457         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 458         struct mm_struct *mm = current->mm;
 459         struct page **user_pages;
 460         ssize_t remain;
 461         loff_t offset, pinned_pages, i;
 462         loff_t first_data_page, last_data_page, num_pages;
 463         int shmem_page_offset;
 464         int data_page_index, data_page_offset;
 465         int page_length;
 466         int ret;
 467         uint64_t data_ptr = args->data_ptr;
 468         int do_bit17_swizzling;
 469
 470         remain = args->size;
 471
 472         /* Pin the user pages containing the data.  We can't fault while
 473          * holding the struct mutex, yet we want to hold it while
 474          * dereferencing the user data.
 475          */
 476         first_data_page = data_ptr / PAGE_SIZE;
 477         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 478         num_pages = last_data_page - first_data_page + 1;
 479
 480         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 481         if (user_pages == NULL)
 482                 return -ENOMEM;
 483
 484         mutex_unlock(&dev->struct_mutex);
 485         down_read(&mm->mmap_sem);
 486         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 487                                       num_pages, 1, 0, user_pages, NULL);
 488         up_read(&mm->mmap_sem);
 489         mutex_lock(&dev->struct_mutex);
 490         if (pinned_pages < num_pages) {
 491                 ret = -EFAULT;
 492                 goto out;
 493         }
 494
 495         ret = i915_gem_object_set_cpu_read_domain_range(obj,
 496                                                         args->offset,
 497                                                         args->size);
 498         if (ret)
 499                 goto out;
 500
 501         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 502
 503         obj_priv = to_intel_bo(obj);
 504         offset = args->offset;
 505
 506         while (remain > 0) {
 507                 struct page *page;
 508
 509                 /* Operation in this page
 510                  *
 511                  * shmem_page_offset = offset within page in shmem file
 512                  * data_page_index = page number in get_user_pages return
 513                  * data_page_offset = offset with data_page_index page.
 514                  * page_length = bytes to copy for this page
 515                  */
 516                 shmem_page_offset = offset & ~PAGE_MASK;
 517                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 518                 data_page_offset = data_ptr & ~PAGE_MASK;
 519
 520                 page_length = remain;
 521                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 522                         page_length = PAGE_SIZE - shmem_page_offset;
 523                 if ((data_page_offset + page_length) > PAGE_SIZE)
 524                         page_length = PAGE_SIZE - data_page_offset;
 525
 526                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 527                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 528                 if (IS_ERR(page))
 529                         return PTR_ERR(page);
 530
 531                 if (do_bit17_swizzling) {
 532                         slow_shmem_bit17_copy(page,
 533                                               shmem_page_offset,
 534                                               user_pages[data_page_index],
 535                                               data_page_offset,
 536                                               page_length,
 537                                               1);
 538                 } else {
 539                         slow_shmem_copy(user_pages[data_page_index],
 540                                         data_page_offset,
 541                                         page,
 542                                         shmem_page_offset,
 543                                         page_length);
 544                 }
 545
 546                 mark_page_accessed(page);
 547                 page_cache_release(page);
 548
 549                 remain -= page_length;
 550                 data_ptr += page_length;
 551                 offset += page_length;
 552         }
 553
 554 out:
 555         for (i = 0; i < pinned_pages; i++) {
 556                 SetPageDirty(user_pages[i]);
 557                 mark_page_accessed(user_pages[i]);
 558                 page_cache_release(user_pages[i]);
 559         }
 560         drm_free_large(user_pages);
 561
 562         return ret;
 563 }
 564
 565 /**
 566  * Reads data from the object referenced by handle.
 567  *
 568  * On error, the contents of *data are undefined.
 569  */
 570 int
 571 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 572                      struct drm_file *file_priv)
 573 {
 574         struct drm_i915_gem_pread *args = data;
 575         struct drm_gem_object *obj;
 576         struct drm_i915_gem_object *obj_priv;
 577         int ret = 0;
 578
 579         ret = i915_mutex_lock_interruptible(dev);
 580         if (ret)
 581                 return ret;
 582
 583         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 584         if (obj == NULL) {
 585                 ret = -ENOENT;
 586                 goto unlock;
 587         }
 588         obj_priv = to_intel_bo(obj);
 589
 590         /* Bounds check source.  */
 591         if (args->offset > obj->size || args->size > obj->size - args->offset) {
 592                 ret = -EINVAL;
 593                 goto out;
 594         }
 595
 596         if (args->size == 0)
 597                 goto out;
 598
 599         if (!access_ok(VERIFY_WRITE,
 600                        (char __user *)(uintptr_t)args->data_ptr,
 601                        args->size)) {
 602                 ret = -EFAULT;
 603                 goto out;
 604         }
 605
 606         ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
 607                                        args->size);
 608         if (ret) {
 609                 ret = -EFAULT;
 610                 goto out;
 611         }
 612
 613         ret = i915_gem_object_set_cpu_read_domain_range(obj,
 614                                                         args->offset,
 615                                                         args->size);
 616         if (ret)
 617                 goto out;
 618
 619         ret = -EFAULT;
 620         if (!i915_gem_object_needs_bit17_swizzle(obj))
 621                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 622         if (ret == -EFAULT)
 623                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 624
 625 out:
 626         drm_gem_object_unreference(obj);
 627 unlock:
 628         mutex_unlock(&dev->struct_mutex);
 629         return ret;
 630 }
 631
 632 /* This is the fast write path which cannot handle
 633  * page faults in the source data
 634  */
 635
 636 static inline int
 637 fast_user_write(struct io_mapping *mapping,
 638                 loff_t page_base, int page_offset,
 639                 char __user *user_data,
 640                 int length)
 641 {
 642         char *vaddr_atomic;
 643         unsigned long unwritten;
 644
 645         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 646         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 647                                                       user_data, length);
 648         io_mapping_unmap_atomic(vaddr_atomic);
 649         return unwritten;
 650 }
 651
 652 /* Here's the write path which can sleep for
 653  * page faults
 654  */
 655
 656 static inline void
 657 slow_kernel_write(struct io_mapping *mapping,
 658                   loff_t gtt_base, int gtt_offset,
 659                   struct page *user_page, int user_offset,
 660                   int length)
 661 {
 662         char __iomem *dst_vaddr;
 663         char *src_vaddr;
 664
 665         dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
 666         src_vaddr = kmap(user_page);
 667
 668         memcpy_toio(dst_vaddr + gtt_offset,
 669                     src_vaddr + user_offset,
 670                     length);
 671
 672         kunmap(user_page);
 673         io_mapping_unmap(dst_vaddr);
 674 }
 675
 676 /**
 677  * This is the fast pwrite path, where we copy the data directly from the
 678  * user into the GTT, uncached.
 679  */
 680 static int
 681 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 682                          struct drm_i915_gem_pwrite *args,
 683                          struct drm_file *file_priv)
 684 {
 685         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 686         drm_i915_private_t *dev_priv = dev->dev_private;
 687         ssize_t remain;
 688         loff_t offset, page_base;
 689         char __user *user_data;
 690         int page_offset, page_length;
 691
 692         user_data = (char __user *) (uintptr_t) args->data_ptr;
 693         remain = args->size;
 694
 695         obj_priv = to_intel_bo(obj);
 696         offset = obj_priv->gtt_offset + args->offset;
 697
 698         while (remain > 0) {
 699                 /* Operation in this page
 700                  *
 701                  * page_base = page offset within aperture
 702                  * page_offset = offset within page
 703                  * page_length = bytes to copy for this page
 704                  */
 705                 page_base = (offset & ~(PAGE_SIZE-1));
 706                 page_offset = offset & (PAGE_SIZE-1);
 707                 page_length = remain;
 708                 if ((page_offset + remain) > PAGE_SIZE)
 709                         page_length = PAGE_SIZE - page_offset;
 710
 711                 /* If we get a fault while copying data, then (presumably) our
 712                  * source page isn't available.  Return the error and we'll
 713                  * retry in the slow path.
 714                  */
 715                 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
 716                                     page_offset, user_data, page_length))
 717
 718                         return -EFAULT;
 719
 720                 remain -= page_length;
 721                 user_data += page_length;
 722                 offset += page_length;
 723         }
 724
 725         return 0;
 726 }
 727
 728 /**
 729  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 730  * the memory and maps it using kmap_atomic for copying.
 731  *
 732  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 733  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 734  */
 735 static int
 736 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 737                          struct drm_i915_gem_pwrite *args,
 738                          struct drm_file *file_priv)
 739 {
 740         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 741         drm_i915_private_t *dev_priv = dev->dev_private;
 742         ssize_t remain;
 743         loff_t gtt_page_base, offset;
 744         loff_t first_data_page, last_data_page, num_pages;
 745         loff_t pinned_pages, i;
 746         struct page **user_pages;
 747         struct mm_struct *mm = current->mm;
 748         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 749         int ret;
 750         uint64_t data_ptr = args->data_ptr;
 751
 752         remain = args->size;
 753
 754         /* Pin the user pages containing the data.  We can't fault while
 755          * holding the struct mutex, and all of the pwrite implementations
 756          * want to hold it while dereferencing the user data.
 757          */
 758         first_data_page = data_ptr / PAGE_SIZE;
 759         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 760         num_pages = last_data_page - first_data_page + 1;
 761
 762         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 763         if (user_pages == NULL)
 764                 return -ENOMEM;
 765
 766         mutex_unlock(&dev->struct_mutex);
 767         down_read(&mm->mmap_sem);
 768         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 769                                       num_pages, 0, 0, user_pages, NULL);
 770         up_read(&mm->mmap_sem);
 771         mutex_lock(&dev->struct_mutex);
 772         if (pinned_pages < num_pages) {
 773                 ret = -EFAULT;
 774                 goto out_unpin_pages;
 775         }
 776
 777         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 778         if (ret)
 779                 goto out_unpin_pages;
 780
 781         obj_priv = to_intel_bo(obj);
 782         offset = obj_priv->gtt_offset + args->offset;
 783
 784         while (remain > 0) {
 785                 /* Operation in this page
 786                  *
 787                  * gtt_page_base = page offset within aperture
 788                  * gtt_page_offset = offset within page in aperture
 789                  * data_page_index = page number in get_user_pages return
 790                  * data_page_offset = offset with data_page_index page.
 791                  * page_length = bytes to copy for this page
 792                  */
 793                 gtt_page_base = offset & PAGE_MASK;
 794                 gtt_page_offset = offset & ~PAGE_MASK;
 795                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 796                 data_page_offset = data_ptr & ~PAGE_MASK;
 797
 798                 page_length = remain;
 799                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 800                         page_length = PAGE_SIZE - gtt_page_offset;
 801                 if ((data_page_offset + page_length) > PAGE_SIZE)
 802                         page_length = PAGE_SIZE - data_page_offset;
 803
 804                 slow_kernel_write(dev_priv->mm.gtt_mapping,
 805                                   gtt_page_base, gtt_page_offset,
 806                                   user_pages[data_page_index],
 807                                   data_page_offset,
 808                                   page_length);
 809
 810                 remain -= page_length;
 811                 offset += page_length;
 812                 data_ptr += page_length;
 813         }
 814
 815 out_unpin_pages:
 816         for (i = 0; i < pinned_pages; i++)
 817                 page_cache_release(user_pages[i]);
 818         drm_free_large(user_pages);
 819
 820         return ret;
 821 }
 822
 823 /**
 824  * This is the fast shmem pwrite path, which attempts to directly
 825  * copy_from_user into the kmapped pages backing the object.
 826  */
 827 static int
 828 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 829                            struct drm_i915_gem_pwrite *args,
 830                            struct drm_file *file_priv)
 831 {
 832         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 833         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 834         ssize_t remain;
 835         loff_t offset;
 836         char __user *user_data;
 837         int page_offset, page_length;
 838
 839         user_data = (char __user *) (uintptr_t) args->data_ptr;
 840         remain = args->size;
 841
 842         obj_priv = to_intel_bo(obj);
 843         offset = args->offset;
 844         obj_priv->dirty = 1;
 845
 846         while (remain > 0) {
 847                 struct page *page;
 848                 char *vaddr;
 849                 int ret;
 850
 851                 /* Operation in this page
 852                  *
 853                  * page_offset = offset within page
 854                  * page_length = bytes to copy for this page
 855                  */
 856                 page_offset = offset & (PAGE_SIZE-1);
 857                 page_length = remain;
 858                 if ((page_offset + remain) > PAGE_SIZE)
 859                         page_length = PAGE_SIZE - page_offset;
 860
 861                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 862                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 863                 if (IS_ERR(page))
 864                         return PTR_ERR(page);
 865
 866                 vaddr = kmap_atomic(page, KM_USER0);
 867                 ret = __copy_from_user_inatomic(vaddr + page_offset,
 868                                                 user_data,
 869                                                 page_length);
 870                 kunmap_atomic(vaddr, KM_USER0);
 871
 872                 set_page_dirty(page);
 873                 mark_page_accessed(page);
 874                 page_cache_release(page);
 875
 876                 /* If we get a fault while copying data, then (presumably) our
 877                  * source page isn't available.  Return the error and we'll
 878                  * retry in the slow path.
 879                  */
 880                 if (ret)
 881                         return -EFAULT;
 882
 883                 remain -= page_length;
 884                 user_data += page_length;
 885                 offset += page_length;
 886         }
 887
 888         return 0;
 889 }
 890
 891 /**
 892  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 893  * the memory and maps it using kmap_atomic for copying.
 894  *
 895  * This avoids taking mmap_sem for faulting on the user's address while the
 896  * struct_mutex is held.
 897  */
 898 static int
 899 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 900                            struct drm_i915_gem_pwrite *args,
 901                            struct drm_file *file_priv)
 902 {
 903         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 904         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 905         struct mm_struct *mm = current->mm;
 906         struct page **user_pages;
 907         ssize_t remain;
 908         loff_t offset, pinned_pages, i;
 909         loff_t first_data_page, last_data_page, num_pages;
 910         int shmem_page_offset;
 911         int data_page_index,  data_page_offset;
 912         int page_length;
 913         int ret;
 914         uint64_t data_ptr = args->data_ptr;
 915         int do_bit17_swizzling;
 916
 917         remain = args->size;
 918
 919         /* Pin the user pages containing the data.  We can't fault while
 920          * holding the struct mutex, and all of the pwrite implementations
 921          * want to hold it while dereferencing the user data.
 922          */
 923         first_data_page = data_ptr / PAGE_SIZE;
 924         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 925         num_pages = last_data_page - first_data_page + 1;
 926
 927         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 928         if (user_pages == NULL)
 929                 return -ENOMEM;
 930
 931         mutex_unlock(&dev->struct_mutex);
 932         down_read(&mm->mmap_sem);
 933         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 934                                       num_pages, 0, 0, user_pages, NULL);
 935         up_read(&mm->mmap_sem);
 936         mutex_lock(&dev->struct_mutex);
 937         if (pinned_pages < num_pages) {
 938                 ret = -EFAULT;
 939                 goto out;
 940         }
 941
 942         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 943         if (ret)
 944                 goto out;
 945
 946         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 947
 948         obj_priv = to_intel_bo(obj);
 949         offset = args->offset;
 950         obj_priv->dirty = 1;
 951
 952         while (remain > 0) {
 953                 struct page *page;
 954
 955                 /* Operation in this page
 956                  *
 957                  * shmem_page_offset = offset within page in shmem file
 958                  * data_page_index = page number in get_user_pages return
 959                  * data_page_offset = offset with data_page_index page.
 960                  * page_length = bytes to copy for this page
 961                  */
 962                 shmem_page_offset = offset & ~PAGE_MASK;
 963                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 964                 data_page_offset = data_ptr & ~PAGE_MASK;
 965
 966                 page_length = remain;
 967                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 968                         page_length = PAGE_SIZE - shmem_page_offset;
 969                 if ((data_page_offset + page_length) > PAGE_SIZE)
 970                         page_length = PAGE_SIZE - data_page_offset;
 971
 972                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 973                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 974                 if (IS_ERR(page)) {
 975                         ret = PTR_ERR(page);
 976                         goto out;
 977                 }
 978
 979                 if (do_bit17_swizzling) {
 980                         slow_shmem_bit17_copy(page,
 981                                               shmem_page_offset,
 982                                               user_pages[data_page_index],
 983                                               data_page_offset,
 984                                               page_length,
 985                                               0);
 986                 } else {
 987                         slow_shmem_copy(page,
 988                                         shmem_page_offset,
 989                                         user_pages[data_page_index],
 990                                         data_page_offset,
 991                                         page_length);
 992                 }
 993
 994                 set_page_dirty(page);
 995                 mark_page_accessed(page);
 996                 page_cache_release(page);
 997
 998                 remain -= page_length;
 999                 data_ptr += page_length;
1000                 offset += page_length;
1001         }
1002
1003 out:
1004         for (i = 0; i < pinned_pages; i++)
1005                 page_cache_release(user_pages[i]);
1006         drm_free_large(user_pages);
1007
1008         return ret;
1009 }
1010
1011 /**
1012  * Writes data to the object referenced by handle.
1013  *
1014  * On error, the contents of the buffer that were to be modified are undefined.
1015  */
1016 int
1017 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1018                       struct drm_file *file)
1019 {
1020         struct drm_i915_gem_pwrite *args = data;
1021         struct drm_gem_object *obj;
1022         struct drm_i915_gem_object *obj_priv;
1023         int ret = 0;
1024
1025         ret = i915_mutex_lock_interruptible(dev);
1026         if (ret)
1027                 return ret;
1028
1029         obj = drm_gem_object_lookup(dev, file, args->handle);
1030         if (obj == NULL) {
1031                 ret = -ENOENT;
1032                 goto unlock;
1033         }
1034         obj_priv = to_intel_bo(obj);
1035
1036
1037         /* Bounds check destination. */
1038         if (args->offset > obj->size || args->size > obj->size - args->offset) {
1039                 ret = -EINVAL;
1040                 goto out;
1041         }
1042
1043         if (args->size == 0)
1044                 goto out;
1045
1046         if (!access_ok(VERIFY_READ,
1047                        (char __user *)(uintptr_t)args->data_ptr,
1048                        args->size)) {
1049                 ret = -EFAULT;
1050                 goto out;
1051         }
1052
1053         ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
1054                                       args->size);
1055         if (ret) {
1056                 ret = -EFAULT;
1057                 goto out;
1058         }
1059
1060         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1061          * it would end up going through the fenced access, and we'll get
1062          * different detiling behavior between reading and writing.
1063          * pread/pwrite currently are reading and writing from the CPU
1064          * perspective, requiring manual detiling by the client.
1065          */
1066         if (obj_priv->phys_obj)
1067                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
1068         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
1069                  obj_priv->gtt_space &&
1070                  obj->write_domain != I915_GEM_DOMAIN_CPU) {
1071                 ret = i915_gem_object_pin(obj, 0, true, false);
1072                 if (ret)
1073                         goto out;
1074
1075                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
1076                 if (ret)
1077                         goto out_unpin;
1078
1079                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1080                 if (ret == -EFAULT)
1081                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1082
1083 out_unpin:
1084                 i915_gem_object_unpin(obj);
1085         } else {
1086                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1087                 if (ret)
1088                         goto out;
1089
1090                 ret = -EFAULT;
1091                 if (!i915_gem_object_needs_bit17_swizzle(obj))
1092                         ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1093                 if (ret == -EFAULT)
1094                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1095         }
1096
1097 out:
1098         drm_gem_object_unreference(obj);
1099 unlock:
1100         mutex_unlock(&dev->struct_mutex);
1101         return ret;
1102 }
1103
1104 /**
1105  * Called when user space prepares to use an object with the CPU, either
1106  * through the mmap ioctl's mapping or a GTT mapping.
1107  */
1108 int
1109 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1110                           struct drm_file *file_priv)
1111 {
1112         struct drm_i915_private *dev_priv = dev->dev_private;
1113         struct drm_i915_gem_set_domain *args = data;
1114         struct drm_gem_object *obj;
1115         struct drm_i915_gem_object *obj_priv;
1116         uint32_t read_domains = args->read_domains;
1117         uint32_t write_domain = args->write_domain;
1118         int ret;
1119
1120         if (!(dev->driver->driver_features & DRIVER_GEM))
1121                 return -ENODEV;
1122
1123         /* Only handle setting domains to types used by the CPU. */
1124         if (write_domain & I915_GEM_GPU_DOMAINS)
1125                 return -EINVAL;
1126
1127         if (read_domains & I915_GEM_GPU_DOMAINS)
1128                 return -EINVAL;
1129
1130         /* Having something in the write domain implies it's in the read
1131          * domain, and only that read domain.  Enforce that in the request.
1132          */
1133         if (write_domain != 0 && read_domains != write_domain)
1134                 return -EINVAL;
1135
1136         ret = i915_mutex_lock_interruptible(dev);
1137         if (ret)
1138                 return ret;
1139
1140         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1141         if (obj == NULL) {
1142                 ret = -ENOENT;
1143                 goto unlock;
1144         }
1145         obj_priv = to_intel_bo(obj);
1146
1147         intel_mark_busy(dev, obj);
1148
1149         if (read_domains & I915_GEM_DOMAIN_GTT) {
1150                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1151
1152                 /* Update the LRU on the fence for the CPU access that's
1153                  * about to occur.
1154                  */
1155                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1156                         struct drm_i915_fence_reg *reg =
1157                                 &dev_priv->fence_regs[obj_priv->fence_reg];
1158                         list_move_tail(&reg->lru_list,
1159                                        &dev_priv->mm.fence_list);
1160                 }
1161
1162                 /* Silently promote "you're not bound, there was nothing to do"
1163                  * to success, since the client was just asking us to
1164                  * make sure everything was done.
1165                  */
1166                 if (ret == -EINVAL)
1167                         ret = 0;
1168         } else {
1169                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1170         }
1171
1172         /* Maintain LRU order of "inactive" objects */
1173         if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1174                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1175
1176         drm_gem_object_unreference(obj);
1177 unlock:
1178         mutex_unlock(&dev->struct_mutex);
1179         return ret;
1180 }
1181
1182 /**
1183  * Called when user space has done writes to this buffer
1184  */
1185 int
1186 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1187                       struct drm_file *file_priv)
1188 {
1189         struct drm_i915_gem_sw_finish *args = data;
1190         struct drm_gem_object *obj;
1191         int ret = 0;
1192
1193         if (!(dev->driver->driver_features & DRIVER_GEM))
1194                 return -ENODEV;
1195
1196         ret = i915_mutex_lock_interruptible(dev);
1197         if (ret)
1198                 return ret;
1199
1200         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1201         if (obj == NULL) {
1202                 ret = -ENOENT;
1203                 goto unlock;
1204         }
1205
1206         /* Pinned buffers may be scanout, so flush the cache */
1207         if (to_intel_bo(obj)->pin_count)
1208                 i915_gem_object_flush_cpu_write_domain(obj);
1209
1210         drm_gem_object_unreference(obj);
1211 unlock:
1212         mutex_unlock(&dev->struct_mutex);
1213         return ret;
1214 }
1215
1216 /**
1217  * Maps the contents of an object, returning the address it is mapped
1218  * into.
1219  *
1220  * While the mapping holds a reference on the contents of the object, it doesn't
1221  * imply a ref on the object itself.
1222  */
1223 int
1224 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1225                    struct drm_file *file_priv)
1226 {
1227         struct drm_i915_private *dev_priv = dev->dev_private;
1228         struct drm_i915_gem_mmap *args = data;
1229         struct drm_gem_object *obj;
1230         loff_t offset;
1231         unsigned long addr;
1232
1233         if (!(dev->driver->driver_features & DRIVER_GEM))
1234                 return -ENODEV;
1235
1236         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1237         if (obj == NULL)
1238                 return -ENOENT;
1239
1240         if (obj->size > dev_priv->mm.gtt_mappable_end) {
1241                 drm_gem_object_unreference_unlocked(obj);
1242                 return -E2BIG;
1243         }
1244
1245         offset = args->offset;
1246
1247         down_write(&current->mm->mmap_sem);
1248         addr = do_mmap(obj->filp, 0, args->size,
1249                        PROT_READ | PROT_WRITE, MAP_SHARED,
1250                        args->offset);
1251         up_write(&current->mm->mmap_sem);
1252         drm_gem_object_unreference_unlocked(obj);
1253         if (IS_ERR((void *)addr))
1254                 return addr;
1255
1256         args->addr_ptr = (uint64_t) addr;
1257
1258         return 0;
1259 }
1260
1261 /**
1262  * i915_gem_fault - fault a page into the GTT
1263  * vma: VMA in question
1264  * vmf: fault info
1265  *
1266  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1267  * from userspace.  The fault handler takes care of binding the object to
1268  * the GTT (if needed), allocating and programming a fence register (again,
1269  * only if needed based on whether the old reg is still valid or the object
1270  * is tiled) and inserting a new PTE into the faulting process.
1271  *
1272  * Note that the faulting process may involve evicting existing objects
1273  * from the GTT and/or fence registers to make room.  So performance may
1274  * suffer if the GTT working set is large or there are few fence registers
1275  * left.
1276  */
1277 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1278 {
1279         struct drm_gem_object *obj = vma->vm_private_data;
1280         struct drm_device *dev = obj->dev;
1281         drm_i915_private_t *dev_priv = dev->dev_private;
1282         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1283         pgoff_t page_offset;
1284         unsigned long pfn;
1285         int ret = 0;
1286         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1287
1288         /* We don't use vmf->pgoff since that has the fake offset */
1289         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1290                 PAGE_SHIFT;
1291
1292         /* Now bind it into the GTT if needed */
1293         mutex_lock(&dev->struct_mutex);
1294         BUG_ON(obj_priv->pin_count && !obj_priv->pin_mappable);
1295
1296         if (obj_priv->gtt_space) {
1297                 if (!obj_priv->mappable ||
1298                     (obj_priv->tiling_mode && !obj_priv->fenceable)) {
1299                         ret = i915_gem_object_unbind(obj);
1300                         if (ret)
1301                                 goto unlock;
1302                 }
1303         }
1304
1305         if (!obj_priv->gtt_space) {
1306                 ret = i915_gem_object_bind_to_gtt(obj, 0,
1307                                                   true, obj_priv->tiling_mode);
1308                 if (ret)
1309                         goto unlock;
1310         }
1311
1312         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1313         if (ret)
1314                 goto unlock;
1315
1316         if (!obj_priv->fault_mappable) {
1317                 obj_priv->fault_mappable = true;
1318                 i915_gem_info_update_mappable(dev_priv, obj_priv, true);
1319         }
1320
1321         /* Need a new fence register? */
1322         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1323                 ret = i915_gem_object_get_fence_reg(obj, true);
1324                 if (ret)
1325                         goto unlock;
1326         }
1327
1328         if (i915_gem_object_is_inactive(obj_priv))
1329                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1330
1331         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1332                 page_offset;
1333
1334         /* Finally, remap it using the new GTT offset */
1335         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1336 unlock:
1337         mutex_unlock(&dev->struct_mutex);
1338
1339         switch (ret) {
1340         case 0:
1341         case -ERESTARTSYS:
1342                 return VM_FAULT_NOPAGE;
1343         case -ENOMEM:
1344         case -EAGAIN:
1345                 return VM_FAULT_OOM;
1346         default:
1347                 return VM_FAULT_SIGBUS;
1348         }
1349 }
1350
1351 /**
1352  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1353  * @obj: obj in question
1354  *
1355  * GEM memory mapping works by handing back to userspace a fake mmap offset
1356  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1357  * up the object based on the offset and sets up the various memory mapping
1358  * structures.
1359  *
1360  * This routine allocates and attaches a fake offset for @obj.
1361  */
1362 static int
1363 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1364 {
1365         struct drm_device *dev = obj->dev;
1366         struct drm_gem_mm *mm = dev->mm_private;
1367         struct drm_map_list *list;
1368         struct drm_local_map *map;
1369         int ret = 0;
1370
1371         /* Set the object up for mmap'ing */
1372         list = &obj->map_list;
1373         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1374         if (!list->map)
1375                 return -ENOMEM;
1376
1377         map = list->map;
1378         map->type = _DRM_GEM;
1379         map->size = obj->size;
1380         map->handle = obj;
1381
1382         /* Get a DRM GEM mmap offset allocated... */
1383         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1384                                                     obj->size / PAGE_SIZE, 0, 0);
1385         if (!list->file_offset_node) {
1386                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1387                 ret = -ENOSPC;
1388                 goto out_free_list;
1389         }
1390
1391         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1392                                                   obj->size / PAGE_SIZE, 0);
1393         if (!list->file_offset_node) {
1394                 ret = -ENOMEM;
1395                 goto out_free_list;
1396         }
1397
1398         list->hash.key = list->file_offset_node->start;
1399         ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1400         if (ret) {
1401                 DRM_ERROR("failed to add to map hash\n");
1402                 goto out_free_mm;
1403         }
1404
1405         return 0;
1406
1407 out_free_mm:
1408         drm_mm_put_block(list->file_offset_node);
1409 out_free_list:
1410         kfree(list->map);
1411         list->map = NULL;
1412
1413         return ret;
1414 }
1415
1416 /**
1417  * i915_gem_release_mmap - remove physical page mappings
1418  * @obj: obj in question
1419  *
1420  * Preserve the reservation of the mmapping with the DRM core code, but
1421  * relinquish ownership of the pages back to the system.
1422  *
1423  * It is vital that we remove the page mapping if we have mapped a tiled
1424  * object through the GTT and then lose the fence register due to
1425  * resource pressure. Similarly if the object has been moved out of the
1426  * aperture, than pages mapped into userspace must be revoked. Removing the
1427  * mapping will then trigger a page fault on the next user access, allowing
1428  * fixup by i915_gem_fault().
1429  */
1430 void
1431 i915_gem_release_mmap(struct drm_gem_object *obj)
1432 {
1433         struct drm_device *dev = obj->dev;
1434         struct drm_i915_private *dev_priv = dev->dev_private;
1435         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1436
1437         if (unlikely(obj->map_list.map && dev->dev_mapping))
1438                 unmap_mapping_range(dev->dev_mapping,
1439                                     (loff_t)obj->map_list.hash.key<<PAGE_SHIFT,
1440                                     obj->size, 1);
1441
1442         if (obj_priv->fault_mappable) {
1443                 obj_priv->fault_mappable = false;
1444                 i915_gem_info_update_mappable(dev_priv, obj_priv, false);
1445         }
1446 }
1447
1448 static void
1449 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1450 {
1451         struct drm_device *dev = obj->dev;
1452         struct drm_gem_mm *mm = dev->mm_private;
1453         struct drm_map_list *list = &obj->map_list;
1454
1455         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1456         drm_mm_put_block(list->file_offset_node);
1457         kfree(list->map);
1458         list->map = NULL;
1459 }
1460
1461 /**
1462  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1463  * @obj: object to check
1464  *
1465  * Return the required GTT alignment for an object, taking into account
1466  * potential fence register mapping if needed.
1467  */
1468 static uint32_t
1469 i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj_priv)
1470 {
1471         struct drm_device *dev = obj_priv->base.dev;
1472
1473         /*
1474          * Minimum alignment is 4k (GTT page size), but might be greater
1475          * if a fence register is needed for the object.
1476          */
1477         if (INTEL_INFO(dev)->gen >= 4 ||
1478             obj_priv->tiling_mode == I915_TILING_NONE)
1479                 return 4096;
1480
1481         /*
1482          * Previous chips need to be aligned to the size of the smallest
1483          * fence register that can contain the object.
1484          */
1485         return i915_gem_get_gtt_size(obj_priv);
1486 }
1487
1488 static uint32_t
1489 i915_gem_get_gtt_size(struct drm_i915_gem_object *obj_priv)
1490 {
1491         struct drm_device *dev = obj_priv->base.dev;
1492         uint32_t size;
1493
1494         /*
1495          * Minimum alignment is 4k (GTT page size), but might be greater
1496          * if a fence register is needed for the object.
1497          */
1498         if (INTEL_INFO(dev)->gen >= 4)
1499                 return obj_priv->base.size;
1500
1501         /*
1502          * Previous chips need to be aligned to the size of the smallest
1503          * fence register that can contain the object.
1504          */
1505         if (INTEL_INFO(dev)->gen == 3)
1506                 size = 1024*1024;
1507         else
1508                 size = 512*1024;
1509
1510         while (size < obj_priv->base.size)
1511                 size <<= 1;
1512
1513         return size;
1514 }
1515
1516 /**
1517  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1518  * @dev: DRM device
1519  * @data: GTT mapping ioctl data
1520  * @file_priv: GEM object info
1521  *
1522  * Simply returns the fake offset to userspace so it can mmap it.
1523  * The mmap call will end up in drm_gem_mmap(), which will set things
1524  * up so we can get faults in the handler above.
1525  *
1526  * The fault handler will take care of binding the object into the GTT
1527  * (since it may have been evicted to make room for something), allocating
1528  * a fence register, and mapping the appropriate aperture address into
1529  * userspace.
1530  */
1531 int
1532 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1533                         struct drm_file *file_priv)
1534 {
1535         struct drm_i915_private *dev_priv = dev->dev_private;
1536         struct drm_i915_gem_mmap_gtt *args = data;
1537         struct drm_gem_object *obj;
1538         struct drm_i915_gem_object *obj_priv;
1539         int ret;
1540
1541         if (!(dev->driver->driver_features & DRIVER_GEM))
1542                 return -ENODEV;
1543
1544         ret = i915_mutex_lock_interruptible(dev);
1545         if (ret)
1546                 return ret;
1547
1548         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1549         if (obj == NULL) {
1550                 ret = -ENOENT;
1551                 goto unlock;
1552         }
1553         obj_priv = to_intel_bo(obj);
1554
1555         if (obj->size > dev_priv->mm.gtt_mappable_end) {
1556                 ret = -E2BIG;
1557                 goto unlock;
1558         }
1559
1560         if (obj_priv->madv != I915_MADV_WILLNEED) {
1561                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1562                 ret = -EINVAL;
1563                 goto out;
1564         }
1565
1566         if (!obj->map_list.map) {
1567                 ret = i915_gem_create_mmap_offset(obj);
1568                 if (ret)
1569                         goto out;
1570         }
1571
1572         args->offset = (u64)obj->map_list.hash.key << PAGE_SHIFT;
1573
1574 out:
1575         drm_gem_object_unreference(obj);
1576 unlock:
1577         mutex_unlock(&dev->struct_mutex);
1578         return ret;
1579 }
1580
1581 static int
1582 i915_gem_object_get_pages_gtt(struct drm_gem_object *obj,
1583                               gfp_t gfpmask)
1584 {
1585         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1586         int page_count, i;
1587         struct address_space *mapping;
1588         struct inode *inode;
1589         struct page *page;
1590
1591         /* Get the list of pages out of our struct file.  They'll be pinned
1592          * at this point until we release them.
1593          */
1594         page_count = obj->size / PAGE_SIZE;
1595         BUG_ON(obj_priv->pages != NULL);
1596         obj_priv->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1597         if (obj_priv->pages == NULL)
1598                 return -ENOMEM;
1599
1600         inode = obj->filp->f_path.dentry->d_inode;
1601         mapping = inode->i_mapping;
1602         for (i = 0; i < page_count; i++) {
1603                 page = read_cache_page_gfp(mapping, i,
1604                                            GFP_HIGHUSER |
1605                                            __GFP_COLD |
1606                                            __GFP_RECLAIMABLE |
1607                                            gfpmask);
1608                 if (IS_ERR(page))
1609                         goto err_pages;
1610
1611                 obj_priv->pages[i] = page;
1612         }
1613
1614         if (obj_priv->tiling_mode != I915_TILING_NONE)
1615                 i915_gem_object_do_bit_17_swizzle(obj);
1616
1617         return 0;
1618
1619 err_pages:
1620         while (i--)
1621                 page_cache_release(obj_priv->pages[i]);
1622
1623         drm_free_large(obj_priv->pages);
1624         obj_priv->pages = NULL;
1625         return PTR_ERR(page);
1626 }
1627
1628 static void
1629 i915_gem_object_put_pages_gtt(struct drm_gem_object *obj)
1630 {
1631         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1632         int page_count = obj->size / PAGE_SIZE;
1633         int i;
1634
1635         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1636
1637         if (obj_priv->tiling_mode != I915_TILING_NONE)
1638                 i915_gem_object_save_bit_17_swizzle(obj);
1639
1640         if (obj_priv->madv == I915_MADV_DONTNEED)
1641                 obj_priv->dirty = 0;
1642
1643         for (i = 0; i < page_count; i++) {
1644                 if (obj_priv->dirty)
1645                         set_page_dirty(obj_priv->pages[i]);
1646
1647                 if (obj_priv->madv == I915_MADV_WILLNEED)
1648                         mark_page_accessed(obj_priv->pages[i]);
1649
1650                 page_cache_release(obj_priv->pages[i]);
1651         }
1652         obj_priv->dirty = 0;
1653
1654         drm_free_large(obj_priv->pages);
1655         obj_priv->pages = NULL;
1656 }
1657
1658 static uint32_t
1659 i915_gem_next_request_seqno(struct drm_device *dev,
1660                             struct intel_ring_buffer *ring)
1661 {
1662         drm_i915_private_t *dev_priv = dev->dev_private;
1663
1664         ring->outstanding_lazy_request = true;
1665         return dev_priv->next_seqno;
1666 }
1667
1668 static void
1669 i915_gem_object_move_to_active(struct drm_gem_object *obj,
1670                                struct intel_ring_buffer *ring)
1671 {
1672         struct drm_device *dev = obj->dev;
1673         struct drm_i915_private *dev_priv = dev->dev_private;
1674         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1675         uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
1676
1677         BUG_ON(ring == NULL);
1678         obj_priv->ring = ring;
1679
1680         /* Add a reference if we're newly entering the active list. */
1681         if (!obj_priv->active) {
1682                 drm_gem_object_reference(obj);
1683                 obj_priv->active = 1;
1684         }
1685
1686         /* Move from whatever list we were on to the tail of execution. */
1687         list_move_tail(&obj_priv->mm_list, &dev_priv->mm.active_list);
1688         list_move_tail(&obj_priv->ring_list, &ring->active_list);
1689         obj_priv->last_rendering_seqno = seqno;
1690 }
1691
1692 static void
1693 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1694 {
1695         struct drm_device *dev = obj->dev;
1696         drm_i915_private_t *dev_priv = dev->dev_private;
1697         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1698
1699         BUG_ON(!obj_priv->active);
1700         list_move_tail(&obj_priv->mm_list, &dev_priv->mm.flushing_list);
1701         list_del_init(&obj_priv->ring_list);
1702         obj_priv->last_rendering_seqno = 0;
1703 }
1704
1705 /* Immediately discard the backing storage */
1706 static void
1707 i915_gem_object_truncate(struct drm_gem_object *obj)
1708 {
1709         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1710         struct inode *inode;
1711
1712         /* Our goal here is to return as much of the memory as
1713          * is possible back to the system as we are called from OOM.
1714          * To do this we must instruct the shmfs to drop all of its
1715          * backing pages, *now*. Here we mirror the actions taken
1716          * when by shmem_delete_inode() to release the backing store.
1717          */
1718         inode = obj->filp->f_path.dentry->d_inode;
1719         truncate_inode_pages(inode->i_mapping, 0);
1720         if (inode->i_op->truncate_range)
1721                 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1722
1723         obj_priv->madv = __I915_MADV_PURGED;
1724 }
1725
1726 static inline int
1727 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1728 {
1729         return obj_priv->madv == I915_MADV_DONTNEED;
1730 }
1731
1732 static void
1733 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1734 {
1735         struct drm_device *dev = obj->dev;
1736         drm_i915_private_t *dev_priv = dev->dev_private;
1737         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1738
1739         if (obj_priv->pin_count != 0)
1740                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.pinned_list);
1741         else
1742                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1743         list_del_init(&obj_priv->ring_list);
1744
1745         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1746
1747         obj_priv->last_rendering_seqno = 0;
1748         obj_priv->ring = NULL;
1749         if (obj_priv->active) {
1750                 obj_priv->active = 0;
1751                 drm_gem_object_unreference(obj);
1752         }
1753         WARN_ON(i915_verify_lists(dev));
1754 }
1755
1756 static void
1757 i915_gem_process_flushing_list(struct drm_device *dev,
1758                                uint32_t flush_domains,
1759                                struct intel_ring_buffer *ring)
1760 {
1761         drm_i915_private_t *dev_priv = dev->dev_private;
1762         struct drm_i915_gem_object *obj_priv, *next;
1763
1764         list_for_each_entry_safe(obj_priv, next,
1765                                  &ring->gpu_write_list,
1766                                  gpu_write_list) {
1767                 struct drm_gem_object *obj = &obj_priv->base;
1768
1769                 if (obj->write_domain & flush_domains) {
1770                         uint32_t old_write_domain = obj->write_domain;
1771
1772                         obj->write_domain = 0;
1773                         list_del_init(&obj_priv->gpu_write_list);
1774                         i915_gem_object_move_to_active(obj, ring);
1775
1776                         /* update the fence lru list */
1777                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1778                                 struct drm_i915_fence_reg *reg =
1779                                         &dev_priv->fence_regs[obj_priv->fence_reg];
1780                                 list_move_tail(&reg->lru_list,
1781                                                 &dev_priv->mm.fence_list);
1782                         }
1783
1784                         trace_i915_gem_object_change_domain(obj,
1785                                                             obj->read_domains,
1786                                                             old_write_domain);
1787                 }
1788         }
1789 }
1790
1791 int
1792 i915_add_request(struct drm_device *dev,
1793                  struct drm_file *file,
1794                  struct drm_i915_gem_request *request,
1795                  struct intel_ring_buffer *ring)
1796 {
1797         drm_i915_private_t *dev_priv = dev->dev_private;
1798         struct drm_i915_file_private *file_priv = NULL;
1799         uint32_t seqno;
1800         int was_empty;
1801         int ret;
1802
1803         BUG_ON(request == NULL);
1804
1805         if (file != NULL)
1806                 file_priv = file->driver_priv;
1807
1808         ret = ring->add_request(ring, &seqno);
1809         if (ret)
1810             return ret;
1811
1812         ring->outstanding_lazy_request = false;
1813
1814         request->seqno = seqno;
1815         request->ring = ring;
1816         request->emitted_jiffies = jiffies;
1817         was_empty = list_empty(&ring->request_list);
1818         list_add_tail(&request->list, &ring->request_list);
1819
1820         if (file_priv) {
1821                 spin_lock(&file_priv->mm.lock);
1822                 request->file_priv = file_priv;
1823                 list_add_tail(&request->client_list,
1824                               &file_priv->mm.request_list);
1825                 spin_unlock(&file_priv->mm.lock);
1826         }
1827
1828         if (!dev_priv->mm.suspended) {
1829                 mod_timer(&dev_priv->hangcheck_timer,
1830                           jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1831                 if (was_empty)
1832                         queue_delayed_work(dev_priv->wq,
1833                                            &dev_priv->mm.retire_work, HZ);
1834         }
1835         return 0;
1836 }
1837
1838 /**
1839  * Command execution barrier
1840  *
1841  * Ensures that all commands in the ring are finished
1842  * before signalling the CPU
1843  */
1844 static void
1845 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1846 {
1847         uint32_t flush_domains = 0;
1848
1849         /* The sampler always gets flushed on i965 (sigh) */
1850         if (INTEL_INFO(dev)->gen >= 4)
1851                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1852
1853         ring->flush(ring, I915_GEM_DOMAIN_COMMAND, flush_domains);
1854 }
1855
1856 static inline void
1857 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1858 {
1859         struct drm_i915_file_private *file_priv = request->file_priv;
1860
1861         if (!file_priv)
1862                 return;
1863
1864         spin_lock(&file_priv->mm.lock);
1865         list_del(&request->client_list);
1866         request->file_priv = NULL;
1867         spin_unlock(&file_priv->mm.lock);
1868 }
1869
1870 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1871                                       struct intel_ring_buffer *ring)
1872 {
1873         while (!list_empty(&ring->request_list)) {
1874                 struct drm_i915_gem_request *request;
1875
1876                 request = list_first_entry(&ring->request_list,
1877                                            struct drm_i915_gem_request,
1878                                            list);
1879
1880                 list_del(&request->list);
1881                 i915_gem_request_remove_from_client(request);
1882                 kfree(request);
1883         }
1884
1885         while (!list_empty(&ring->active_list)) {
1886                 struct drm_i915_gem_object *obj_priv;
1887
1888                 obj_priv = list_first_entry(&ring->active_list,
1889                                             struct drm_i915_gem_object,
1890                                             ring_list);
1891
1892                 obj_priv->base.write_domain = 0;
1893                 list_del_init(&obj_priv->gpu_write_list);
1894                 i915_gem_object_move_to_inactive(&obj_priv->base);
1895         }
1896 }
1897
1898 void i915_gem_reset(struct drm_device *dev)
1899 {
1900         struct drm_i915_private *dev_priv = dev->dev_private;
1901         struct drm_i915_gem_object *obj_priv;
1902         int i;
1903
1904         i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1905         i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1906         i915_gem_reset_ring_lists(dev_priv, &dev_priv->blt_ring);
1907
1908         /* Remove anything from the flushing lists. The GPU cache is likely
1909          * to be lost on reset along with the data, so simply move the
1910          * lost bo to the inactive list.
1911          */
1912         while (!list_empty(&dev_priv->mm.flushing_list)) {
1913                 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1914                                             struct drm_i915_gem_object,
1915                                             mm_list);
1916
1917                 obj_priv->base.write_domain = 0;
1918                 list_del_init(&obj_priv->gpu_write_list);
1919                 i915_gem_object_move_to_inactive(&obj_priv->base);
1920         }
1921
1922         /* Move everything out of the GPU domains to ensure we do any
1923          * necessary invalidation upon reuse.
1924          */
1925         list_for_each_entry(obj_priv,
1926                             &dev_priv->mm.inactive_list,
1927                             mm_list)
1928         {
1929                 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1930         }
1931
1932         /* The fence registers are invalidated so clear them out */
1933         for (i = 0; i < 16; i++) {
1934                 struct drm_i915_fence_reg *reg;
1935
1936                 reg = &dev_priv->fence_regs[i];
1937                 if (!reg->obj)
1938                         continue;
1939
1940                 i915_gem_clear_fence_reg(reg->obj);
1941         }
1942 }
1943
1944 /**
1945  * This function clears the request list as sequence numbers are passed.
1946  */
1947 static void
1948 i915_gem_retire_requests_ring(struct drm_device *dev,
1949                               struct intel_ring_buffer *ring)
1950 {
1951         drm_i915_private_t *dev_priv = dev->dev_private;
1952         uint32_t seqno;
1953
1954         if (!ring->status_page.page_addr ||
1955             list_empty(&ring->request_list))
1956                 return;
1957
1958         WARN_ON(i915_verify_lists(dev));
1959
1960         seqno = ring->get_seqno(ring);
1961         while (!list_empty(&ring->request_list)) {
1962                 struct drm_i915_gem_request *request;
1963
1964                 request = list_first_entry(&ring->request_list,
1965                                            struct drm_i915_gem_request,
1966                                            list);
1967
1968                 if (!i915_seqno_passed(seqno, request->seqno))
1969                         break;
1970
1971                 trace_i915_gem_request_retire(dev, request->seqno);
1972
1973                 list_del(&request->list);
1974                 i915_gem_request_remove_from_client(request);
1975                 kfree(request);
1976         }
1977
1978         /* Move any buffers on the active list that are no longer referenced
1979          * by the ringbuffer to the flushing/inactive lists as appropriate.
1980          */
1981         while (!list_empty(&ring->active_list)) {
1982                 struct drm_gem_object *obj;
1983                 struct drm_i915_gem_object *obj_priv;
1984
1985                 obj_priv = list_first_entry(&ring->active_list,
1986                                             struct drm_i915_gem_object,
1987                                             ring_list);
1988
1989                 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
1990                         break;
1991
1992                 obj = &obj_priv->base;
1993                 if (obj->write_domain != 0)
1994                         i915_gem_object_move_to_flushing(obj);
1995                 else
1996                         i915_gem_object_move_to_inactive(obj);
1997         }
1998
1999         if (unlikely (dev_priv->trace_irq_seqno &&
2000                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
2001                 ring->user_irq_put(ring);
2002                 dev_priv->trace_irq_seqno = 0;
2003         }
2004
2005         WARN_ON(i915_verify_lists(dev));
2006 }
2007
2008 void
2009 i915_gem_retire_requests(struct drm_device *dev)
2010 {
2011         drm_i915_private_t *dev_priv = dev->dev_private;
2012
2013         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
2014             struct drm_i915_gem_object *obj_priv, *tmp;
2015
2016             /* We must be careful that during unbind() we do not
2017              * accidentally infinitely recurse into retire requests.
2018              * Currently:
2019              *   retire -> free -> unbind -> wait -> retire_ring
2020              */
2021             list_for_each_entry_safe(obj_priv, tmp,
2022                                      &dev_priv->mm.deferred_free_list,
2023                                      mm_list)
2024                     i915_gem_free_object_tail(&obj_priv->base);
2025         }
2026
2027         i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
2028         i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
2029         i915_gem_retire_requests_ring(dev, &dev_priv->blt_ring);
2030 }
2031
2032 static void
2033 i915_gem_retire_work_handler(struct work_struct *work)
2034 {
2035         drm_i915_private_t *dev_priv;
2036         struct drm_device *dev;
2037
2038         dev_priv = container_of(work, drm_i915_private_t,
2039                                 mm.retire_work.work);
2040         dev = dev_priv->dev;
2041
2042         /* Come back later if the device is busy... */
2043         if (!mutex_trylock(&dev->struct_mutex)) {
2044                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2045                 return;
2046         }
2047
2048         i915_gem_retire_requests(dev);
2049
2050         if (!dev_priv->mm.suspended &&
2051                 (!list_empty(&dev_priv->render_ring.request_list) ||
2052                  !list_empty(&dev_priv->bsd_ring.request_list) ||
2053                  !list_empty(&dev_priv->blt_ring.request_list)))
2054                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2055         mutex_unlock(&dev->struct_mutex);
2056 }
2057
2058 int
2059 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
2060                      bool interruptible, struct intel_ring_buffer *ring)
2061 {
2062         drm_i915_private_t *dev_priv = dev->dev_private;
2063         u32 ier;
2064         int ret = 0;
2065
2066         BUG_ON(seqno == 0);
2067
2068         if (atomic_read(&dev_priv->mm.wedged))
2069                 return -EAGAIN;
2070
2071         if (ring->outstanding_lazy_request) {
2072                 struct drm_i915_gem_request *request;
2073
2074                 request = kzalloc(sizeof(*request), GFP_KERNEL);
2075                 if (request == NULL)
2076                         return -ENOMEM;
2077
2078                 ret = i915_add_request(dev, NULL, request, ring);
2079                 if (ret) {
2080                         kfree(request);
2081                         return ret;
2082                 }
2083
2084                 seqno = request->seqno;
2085         }
2086         BUG_ON(seqno == dev_priv->next_seqno);
2087
2088         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
2089                 if (HAS_PCH_SPLIT(dev))
2090                         ier = I915_READ(DEIER) | I915_READ(GTIER);
2091                 else
2092                         ier = I915_READ(IER);
2093                 if (!ier) {
2094                         DRM_ERROR("something (likely vbetool) disabled "
2095                                   "interrupts, re-enabling\n");
2096                         i915_driver_irq_preinstall(dev);
2097                         i915_driver_irq_postinstall(dev);
2098                 }
2099
2100                 trace_i915_gem_request_wait_begin(dev, seqno);
2101
2102                 ring->waiting_seqno = seqno;
2103                 ring->user_irq_get(ring);
2104                 if (interruptible)
2105                         ret = wait_event_interruptible(ring->irq_queue,
2106                                 i915_seqno_passed(ring->get_seqno(ring), seqno)
2107                                 || atomic_read(&dev_priv->mm.wedged));
2108                 else
2109                         wait_event(ring->irq_queue,
2110                                 i915_seqno_passed(ring->get_seqno(ring), seqno)
2111                                 || atomic_read(&dev_priv->mm.wedged));
2112
2113                 ring->user_irq_put(ring);
2114                 ring->waiting_seqno = 0;
2115
2116                 trace_i915_gem_request_wait_end(dev, seqno);
2117         }
2118         if (atomic_read(&dev_priv->mm.wedged))
2119                 ret = -EAGAIN;
2120
2121         if (ret && ret != -ERESTARTSYS)
2122                 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
2123                           __func__, ret, seqno, ring->get_seqno(ring),
2124                           dev_priv->next_seqno);
2125
2126         /* Directly dispatch request retiring.  While we have the work queue
2127          * to handle this, the waiter on a request often wants an associated
2128          * buffer to have made it to the inactive list, and we would need
2129          * a separate wait queue to handle that.
2130          */
2131         if (ret == 0)
2132                 i915_gem_retire_requests_ring(dev, ring);
2133
2134         return ret;
2135 }
2136
2137 /**
2138  * Waits for a sequence number to be signaled, and cleans up the
2139  * request and object lists appropriately for that event.
2140  */
2141 static int
2142 i915_wait_request(struct drm_device *dev, uint32_t seqno,
2143                   struct intel_ring_buffer *ring)
2144 {
2145         return i915_do_wait_request(dev, seqno, 1, ring);
2146 }
2147
2148 static void
2149 i915_gem_flush_ring(struct drm_device *dev,
2150                     struct drm_file *file_priv,
2151                     struct intel_ring_buffer *ring,
2152                     uint32_t invalidate_domains,
2153                     uint32_t flush_domains)
2154 {
2155         ring->flush(ring, invalidate_domains, flush_domains);
2156         i915_gem_process_flushing_list(dev, flush_domains, ring);
2157 }
2158
2159 static void
2160 i915_gem_flush(struct drm_device *dev,
2161                struct drm_file *file_priv,
2162                uint32_t invalidate_domains,
2163                uint32_t flush_domains,
2164                uint32_t flush_rings)
2165 {
2166         drm_i915_private_t *dev_priv = dev->dev_private;
2167
2168         if (flush_domains & I915_GEM_DOMAIN_CPU)
2169                 drm_agp_chipset_flush(dev);
2170
2171         if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2172                 if (flush_rings & RING_RENDER)
2173                         i915_gem_flush_ring(dev, file_priv,
2174                                             &dev_priv->render_ring,
2175                                             invalidate_domains, flush_domains);
2176                 if (flush_rings & RING_BSD)
2177                         i915_gem_flush_ring(dev, file_priv,
2178                                             &dev_priv->bsd_ring,
2179                                             invalidate_domains, flush_domains);
2180                 if (flush_rings & RING_BLT)
2181                         i915_gem_flush_ring(dev, file_priv,
2182                                             &dev_priv->blt_ring,
2183                                             invalidate_domains, flush_domains);
2184         }
2185 }
2186
2187 /**
2188  * Ensures that all rendering to the object has completed and the object is
2189  * safe to unbind from the GTT or access from the CPU.
2190  */
2191 static int
2192 i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2193                                bool interruptible)
2194 {
2195         struct drm_device *dev = obj->dev;
2196         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2197         int ret;
2198
2199         /* This function only exists to support waiting for existing rendering,
2200          * not for emitting required flushes.
2201          */
2202         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
2203
2204         /* If there is rendering queued on the buffer being evicted, wait for
2205          * it.
2206          */
2207         if (obj_priv->active) {
2208                 ret = i915_do_wait_request(dev,
2209                                            obj_priv->last_rendering_seqno,
2210                                            interruptible,
2211                                            obj_priv->ring);
2212                 if (ret)
2213                         return ret;
2214         }
2215
2216         return 0;
2217 }
2218
2219 /**
2220  * Unbinds an object from the GTT aperture.
2221  */
2222 int
2223 i915_gem_object_unbind(struct drm_gem_object *obj)
2224 {
2225         struct drm_device *dev = obj->dev;
2226         struct drm_i915_private *dev_priv = dev->dev_private;
2227         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2228         int ret = 0;
2229
2230         if (obj_priv->gtt_space == NULL)
2231                 return 0;
2232
2233         if (obj_priv->pin_count != 0) {
2234                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2235                 return -EINVAL;
2236         }
2237
2238         /* blow away mappings if mapped through GTT */
2239         i915_gem_release_mmap(obj);
2240
2241         /* Move the object to the CPU domain to ensure that
2242          * any possible CPU writes while it's not in the GTT
2243          * are flushed when we go to remap it. This will
2244          * also ensure that all pending GPU writes are finished
2245          * before we unbind.
2246          */
2247         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2248         if (ret == -ERESTARTSYS)
2249                 return ret;
2250         /* Continue on if we fail due to EIO, the GPU is hung so we
2251          * should be safe and we need to cleanup or else we might
2252          * cause memory corruption through use-after-free.
2253          */
2254         if (ret) {
2255                 i915_gem_clflush_object(obj);
2256                 obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU;
2257         }
2258
2259         /* release the fence reg _after_ flushing */
2260         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2261                 i915_gem_clear_fence_reg(obj);
2262
2263         drm_unbind_agp(obj_priv->agp_mem);
2264         drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2265
2266         i915_gem_object_put_pages_gtt(obj);
2267
2268         i915_gem_info_remove_gtt(dev_priv, obj_priv);
2269         list_del_init(&obj_priv->mm_list);
2270         obj_priv->fenceable = true;
2271         obj_priv->mappable = true;
2272
2273         drm_mm_put_block(obj_priv->gtt_space);
2274         obj_priv->gtt_space = NULL;
2275         obj_priv->gtt_offset = 0;
2276
2277         if (i915_gem_object_is_purgeable(obj_priv))
2278                 i915_gem_object_truncate(obj);
2279
2280         trace_i915_gem_object_unbind(obj);
2281
2282         return ret;
2283 }
2284
2285 static int i915_ring_idle(struct drm_device *dev,
2286                           struct intel_ring_buffer *ring)
2287 {
2288         if (list_empty(&ring->gpu_write_list))
2289                 return 0;
2290
2291         i915_gem_flush_ring(dev, NULL, ring,
2292                             I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2293         return i915_wait_request(dev,
2294                                  i915_gem_next_request_seqno(dev, ring),
2295                                  ring);
2296 }
2297
2298 int
2299 i915_gpu_idle(struct drm_device *dev)
2300 {
2301         drm_i915_private_t *dev_priv = dev->dev_private;
2302         bool lists_empty;
2303         int ret;
2304
2305         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2306                        list_empty(&dev_priv->render_ring.active_list) &&
2307                        list_empty(&dev_priv->bsd_ring.active_list) &&
2308                        list_empty(&dev_priv->blt_ring.active_list));
2309         if (lists_empty)
2310                 return 0;
2311
2312         /* Flush everything onto the inactive list. */
2313         ret = i915_ring_idle(dev, &dev_priv->render_ring);
2314         if (ret)
2315                 return ret;
2316
2317         ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
2318         if (ret)
2319                 return ret;
2320
2321         ret = i915_ring_idle(dev, &dev_priv->blt_ring);
2322         if (ret)
2323                 return ret;
2324
2325         return 0;
2326 }
2327
2328 static void sandybridge_write_fence_reg(struct drm_gem_object *obj)
2329 {
2330         struct drm_device *dev = obj->dev;
2331         drm_i915_private_t *dev_priv = dev->dev_private;
2332         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2333         u32 size = i915_gem_get_gtt_size(obj_priv);
2334         int regnum = obj_priv->fence_reg;
2335         uint64_t val;
2336
2337         val = (uint64_t)((obj_priv->gtt_offset + size - 4096) &
2338                     0xfffff000) << 32;
2339         val |= obj_priv->gtt_offset & 0xfffff000;
2340         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2341                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2342
2343         if (obj_priv->tiling_mode == I915_TILING_Y)
2344                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2345         val |= I965_FENCE_REG_VALID;
2346
2347         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2348 }
2349
2350 static void i965_write_fence_reg(struct drm_gem_object *obj)
2351 {
2352         struct drm_device *dev = obj->dev;
2353         drm_i915_private_t *dev_priv = dev->dev_private;
2354         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2355         u32 size = i915_gem_get_gtt_size(obj_priv);
2356         int regnum = obj_priv->fence_reg;
2357         uint64_t val;
2358
2359         val = (uint64_t)((obj_priv->gtt_offset + size - 4096) &
2360                     0xfffff000) << 32;
2361         val |= obj_priv->gtt_offset & 0xfffff000;
2362         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2363         if (obj_priv->tiling_mode == I915_TILING_Y)
2364                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2365         val |= I965_FENCE_REG_VALID;
2366
2367         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2368 }
2369
2370 static void i915_write_fence_reg(struct drm_gem_object *obj)
2371 {
2372         struct drm_device *dev = obj->dev;
2373         drm_i915_private_t *dev_priv = dev->dev_private;
2374         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2375         u32 size = i915_gem_get_gtt_size(obj_priv);
2376         uint32_t fence_reg, val, pitch_val;
2377         int tile_width;
2378
2379         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2380             (obj_priv->gtt_offset & (size - 1))) {
2381                 WARN(1, "%s: object 0x%08x [fenceable? %d] not 1M or size (0x%08x) aligned [gtt_space offset=%lx, size=%lx]\n",
2382                      __func__, obj_priv->gtt_offset, obj_priv->fenceable, size,
2383                      obj_priv->gtt_space->start, obj_priv->gtt_space->size);
2384                 return;
2385         }
2386
2387         if (obj_priv->tiling_mode == I915_TILING_Y &&
2388             HAS_128_BYTE_Y_TILING(dev))
2389                 tile_width = 128;
2390         else
2391                 tile_width = 512;
2392
2393         /* Note: pitch better be a power of two tile widths */
2394         pitch_val = obj_priv->stride / tile_width;
2395         pitch_val = ffs(pitch_val) - 1;
2396
2397         if (obj_priv->tiling_mode == I915_TILING_Y &&
2398             HAS_128_BYTE_Y_TILING(dev))
2399                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2400         else
2401                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2402
2403         val = obj_priv->gtt_offset;
2404         if (obj_priv->tiling_mode == I915_TILING_Y)
2405                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2406         val |= I915_FENCE_SIZE_BITS(size);
2407         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2408         val |= I830_FENCE_REG_VALID;
2409
2410         fence_reg = obj_priv->fence_reg;
2411         if (fence_reg < 8)
2412                 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2413         else
2414                 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2415         I915_WRITE(fence_reg, val);
2416 }
2417
2418 static void i830_write_fence_reg(struct drm_gem_object *obj)
2419 {
2420         struct drm_device *dev = obj->dev;
2421         drm_i915_private_t *dev_priv = dev->dev_private;
2422         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2423         u32 size = i915_gem_get_gtt_size(obj_priv);
2424         int regnum = obj_priv->fence_reg;
2425         uint32_t val;
2426         uint32_t pitch_val;
2427         uint32_t fence_size_bits;
2428
2429         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2430             (obj_priv->gtt_offset & (obj->size - 1))) {
2431                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2432                      __func__, obj_priv->gtt_offset);
2433                 return;
2434         }
2435
2436         pitch_val = obj_priv->stride / 128;
2437         pitch_val = ffs(pitch_val) - 1;
2438         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2439
2440         val = obj_priv->gtt_offset;
2441         if (obj_priv->tiling_mode == I915_TILING_Y)
2442                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2443         fence_size_bits = I830_FENCE_SIZE_BITS(size);
2444         WARN_ON(fence_size_bits & ~0x00000f00);
2445         val |= fence_size_bits;
2446         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2447         val |= I830_FENCE_REG_VALID;
2448
2449         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2450 }
2451
2452 static int i915_find_fence_reg(struct drm_device *dev,
2453                                bool interruptible)
2454 {
2455         struct drm_i915_private *dev_priv = dev->dev_private;
2456         struct drm_i915_fence_reg *reg;
2457         struct drm_i915_gem_object *obj_priv = NULL;
2458         int i, avail, ret;
2459
2460         /* First try to find a free reg */
2461         avail = 0;
2462         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2463                 reg = &dev_priv->fence_regs[i];
2464                 if (!reg->obj)
2465                         return i;
2466
2467                 obj_priv = to_intel_bo(reg->obj);
2468                 if (!obj_priv->pin_count)
2469                     avail++;
2470         }
2471
2472         if (avail == 0)
2473                 return -ENOSPC;
2474
2475         /* None available, try to steal one or wait for a user to finish */
2476         avail = I915_FENCE_REG_NONE;
2477         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2478                             lru_list) {
2479                 obj_priv = to_intel_bo(reg->obj);
2480                 if (obj_priv->pin_count)
2481                         continue;
2482
2483                 /* found one! */
2484                 avail = obj_priv->fence_reg;
2485                 break;
2486         }
2487
2488         BUG_ON(avail == I915_FENCE_REG_NONE);
2489
2490         /* We only have a reference on obj from the active list. put_fence_reg
2491          * might drop that one, causing a use-after-free in it. So hold a
2492          * private reference to obj like the other callers of put_fence_reg
2493          * (set_tiling ioctl) do. */
2494         drm_gem_object_reference(&obj_priv->base);
2495         ret = i915_gem_object_put_fence_reg(&obj_priv->base, interruptible);
2496         drm_gem_object_unreference(&obj_priv->base);
2497         if (ret != 0)
2498                 return ret;
2499
2500         return avail;
2501 }
2502
2503 /**
2504  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2505  * @obj: object to map through a fence reg
2506  *
2507  * When mapping objects through the GTT, userspace wants to be able to write
2508  * to them without having to worry about swizzling if the object is tiled.
2509  *
2510  * This function walks the fence regs looking for a free one for @obj,
2511  * stealing one if it can't find any.
2512  *
2513  * It then sets up the reg based on the object's properties: address, pitch
2514  * and tiling format.
2515  */
2516 int
2517 i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2518                               bool interruptible)
2519 {
2520         struct drm_device *dev = obj->dev;
2521         struct drm_i915_private *dev_priv = dev->dev_private;
2522         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2523         struct drm_i915_fence_reg *reg = NULL;
2524         int ret;
2525
2526         /* Just update our place in the LRU if our fence is getting used. */
2527         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2528                 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2529                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2530                 return 0;
2531         }
2532
2533         switch (obj_priv->tiling_mode) {
2534         case I915_TILING_NONE:
2535                 WARN(1, "allocating a fence for non-tiled object?\n");
2536                 break;
2537         case I915_TILING_X:
2538                 if (!obj_priv->stride)
2539                         return -EINVAL;
2540                 WARN((obj_priv->stride & (512 - 1)),
2541                      "object 0x%08x is X tiled but has non-512B pitch\n",
2542                      obj_priv->gtt_offset);
2543                 break;
2544         case I915_TILING_Y:
2545                 if (!obj_priv->stride)
2546                         return -EINVAL;
2547                 WARN((obj_priv->stride & (128 - 1)),
2548                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2549                      obj_priv->gtt_offset);
2550                 break;
2551         }
2552
2553         ret = i915_find_fence_reg(dev, interruptible);
2554         if (ret < 0)
2555                 return ret;
2556
2557         obj_priv->fence_reg = ret;
2558         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2559         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2560
2561         reg->obj = obj;
2562
2563         switch (INTEL_INFO(dev)->gen) {
2564         case 6:
2565                 sandybridge_write_fence_reg(obj);
2566                 break;
2567         case 5:
2568         case 4:
2569                 i965_write_fence_reg(obj);
2570                 break;
2571         case 3:
2572                 i915_write_fence_reg(obj);
2573                 break;
2574         case 2:
2575                 i830_write_fence_reg(obj);
2576                 break;
2577         }
2578
2579         trace_i915_gem_object_get_fence(obj,
2580                                         obj_priv->fence_reg,
2581                                         obj_priv->tiling_mode);
2582
2583         return 0;
2584 }
2585
2586 /**
2587  * i915_gem_clear_fence_reg - clear out fence register info
2588  * @obj: object to clear
2589  *
2590  * Zeroes out the fence register itself and clears out the associated
2591  * data structures in dev_priv and obj_priv.
2592  */
2593 static void
2594 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2595 {
2596         struct drm_device *dev = obj->dev;
2597         drm_i915_private_t *dev_priv = dev->dev_private;
2598         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2599         struct drm_i915_fence_reg *reg =
2600                 &dev_priv->fence_regs[obj_priv->fence_reg];
2601         uint32_t fence_reg;
2602
2603         switch (INTEL_INFO(dev)->gen) {
2604         case 6:
2605                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2606                              (obj_priv->fence_reg * 8), 0);
2607                 break;
2608         case 5:
2609         case 4:
2610                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2611                 break;
2612         case 3:
2613                 if (obj_priv->fence_reg >= 8)
2614                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
2615                 else
2616         case 2:
2617                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2618
2619                 I915_WRITE(fence_reg, 0);
2620                 break;
2621         }
2622
2623         reg->obj = NULL;
2624         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2625         list_del_init(&reg->lru_list);
2626 }
2627
2628 /**
2629  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2630  * to the buffer to finish, and then resets the fence register.
2631  * @obj: tiled object holding a fence register.
2632  * @bool: whether the wait upon the fence is interruptible
2633  *
2634  * Zeroes out the fence register itself and clears out the associated
2635  * data structures in dev_priv and obj_priv.
2636  */
2637 int
2638 i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2639                               bool interruptible)
2640 {
2641         struct drm_device *dev = obj->dev;
2642         struct drm_i915_private *dev_priv = dev->dev_private;
2643         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2644         struct drm_i915_fence_reg *reg;
2645
2646         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2647                 return 0;
2648
2649         /* If we've changed tiling, GTT-mappings of the object
2650          * need to re-fault to ensure that the correct fence register
2651          * setup is in place.
2652          */
2653         i915_gem_release_mmap(obj);
2654
2655         /* On the i915, GPU access to tiled buffers is via a fence,
2656          * therefore we must wait for any outstanding access to complete
2657          * before clearing the fence.
2658          */
2659         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2660         if (reg->gpu) {
2661                 int ret;
2662
2663                 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2664                 if (ret)
2665                         return ret;
2666
2667                 ret = i915_gem_object_wait_rendering(obj, interruptible);
2668                 if (ret)
2669                         return ret;
2670
2671                 reg->gpu = false;
2672         }
2673
2674         i915_gem_object_flush_gtt_write_domain(obj);
2675         i915_gem_clear_fence_reg(obj);
2676
2677         return 0;
2678 }
2679
2680 /**
2681  * Finds free space in the GTT aperture and binds the object there.
2682  */
2683 static int
2684 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
2685                             unsigned alignment,
2686                             bool mappable,
2687                             bool need_fence)
2688 {
2689         struct drm_device *dev = obj->dev;
2690         drm_i915_private_t *dev_priv = dev->dev_private;
2691         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2692         struct drm_mm_node *free_space;
2693         gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
2694         u32 size, fence_size, fence_alignment;
2695         int ret;
2696
2697         if (obj_priv->madv != I915_MADV_WILLNEED) {
2698                 DRM_ERROR("Attempting to bind a purgeable object\n");
2699                 return -EINVAL;
2700         }
2701
2702         fence_size = i915_gem_get_gtt_size(obj_priv);
2703         fence_alignment = i915_gem_get_gtt_alignment(obj_priv);
2704
2705         if (alignment == 0)
2706                 alignment = need_fence ? fence_alignment : 4096;
2707         if (need_fence && alignment & (fence_alignment - 1)) {
2708                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2709                 return -EINVAL;
2710         }
2711
2712         size = need_fence ? fence_size : obj->size;
2713
2714         /* If the object is bigger than the entire aperture, reject it early
2715          * before evicting everything in a vain attempt to find space.
2716          */
2717         if (obj->size >
2718             (mappable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2719                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2720                 return -E2BIG;
2721         }
2722
2723  search_free:
2724         if (mappable)
2725                 free_space =
2726                         drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2727                                                     size, alignment, 0,
2728                                                     dev_priv->mm.gtt_mappable_end,
2729                                                     0);
2730         else
2731                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2732                                                 size, alignment, 0);
2733
2734         if (free_space != NULL) {
2735                 if (mappable)
2736                         obj_priv->gtt_space =
2737                                 drm_mm_get_block_range_generic(free_space,
2738                                                                size, alignment, 0,
2739                                                                dev_priv->mm.gtt_mappable_end,
2740                                                                0);
2741                 else
2742                         obj_priv->gtt_space =
2743                                 drm_mm_get_block(free_space, size, alignment);
2744         }
2745         if (obj_priv->gtt_space == NULL) {
2746                 /* If the gtt is empty and we're still having trouble
2747                  * fitting our object in, we're out of memory.
2748                  */
2749                 ret = i915_gem_evict_something(dev, size, alignment, mappable);
2750                 if (ret)
2751                         return ret;
2752
2753                 goto search_free;
2754         }
2755
2756         ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2757         if (ret) {
2758                 drm_mm_put_block(obj_priv->gtt_space);
2759                 obj_priv->gtt_space = NULL;
2760
2761                 if (ret == -ENOMEM) {
2762                         /* first try to clear up some space from the GTT */
2763                         ret = i915_gem_evict_something(dev, size,
2764                                                        alignment, mappable);
2765                         if (ret) {
2766                                 /* now try to shrink everyone else */
2767                                 if (gfpmask) {
2768                                         gfpmask = 0;
2769                                         goto search_free;
2770                                 }
2771
2772                                 return ret;
2773                         }
2774
2775                         goto search_free;
2776                 }
2777
2778                 return ret;
2779         }
2780
2781         /* Create an AGP memory structure pointing at our pages, and bind it
2782          * into the GTT.
2783          */
2784         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2785                                                obj_priv->pages,
2786                                                obj->size >> PAGE_SHIFT,
2787                                                obj_priv->gtt_space->start,
2788                                                obj_priv->agp_type);
2789         if (obj_priv->agp_mem == NULL) {
2790                 i915_gem_object_put_pages_gtt(obj);
2791                 drm_mm_put_block(obj_priv->gtt_space);
2792                 obj_priv->gtt_space = NULL;
2793
2794                 ret = i915_gem_evict_something(dev, size,
2795                                                alignment, mappable);
2796                 if (ret)
2797                         return ret;
2798
2799                 goto search_free;
2800         }
2801
2802         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2803
2804         /* keep track of bounds object by adding it to the inactive list */
2805         list_add_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
2806         i915_gem_info_add_gtt(dev_priv, obj_priv);
2807
2808         /* Assert that the object is not currently in any GPU domain. As it
2809          * wasn't in the GTT, there shouldn't be any way it could have been in
2810          * a GPU cache
2811          */
2812         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2813         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2814
2815         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset, mappable);
2816
2817         obj_priv->fenceable =
2818                 obj_priv->gtt_space->size == fence_size &&
2819                 (obj_priv->gtt_space->start & (fence_alignment -1)) == 0;
2820
2821         obj_priv->mappable =
2822                 obj_priv->gtt_offset + obj->size <= dev_priv->mm.gtt_mappable_end;
2823
2824         return 0;
2825 }
2826
2827 void
2828 i915_gem_clflush_object(struct drm_gem_object *obj)
2829 {
2830         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2831
2832         /* If we don't have a page list set up, then we're not pinned
2833          * to GPU, and we can ignore the cache flush because it'll happen
2834          * again at bind time.
2835          */
2836         if (obj_priv->pages == NULL)
2837                 return;
2838
2839         trace_i915_gem_object_clflush(obj);
2840
2841         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2842 }
2843
2844 /** Flushes any GPU write domain for the object if it's dirty. */
2845 static int
2846 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
2847                                        bool pipelined)
2848 {
2849         struct drm_device *dev = obj->dev;
2850         uint32_t old_write_domain;
2851
2852         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2853                 return 0;
2854
2855         /* Queue the GPU write cache flushing we need. */
2856         old_write_domain = obj->write_domain;
2857         i915_gem_flush_ring(dev, NULL,
2858                             to_intel_bo(obj)->ring,
2859                             0, obj->write_domain);
2860         BUG_ON(obj->write_domain);
2861
2862         trace_i915_gem_object_change_domain(obj,
2863                                             obj->read_domains,
2864                                             old_write_domain);
2865
2866         if (pipelined)
2867                 return 0;
2868
2869         return i915_gem_object_wait_rendering(obj, true);
2870 }
2871
2872 /** Flushes the GTT write domain for the object if it's dirty. */
2873 static void
2874 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2875 {
2876         uint32_t old_write_domain;
2877
2878         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2879                 return;
2880
2881         /* No actual flushing is required for the GTT write domain.   Writes
2882          * to it immediately go to main memory as far as we know, so there's
2883          * no chipset flush.  It also doesn't land in render cache.
2884          */
2885         i915_gem_release_mmap(obj);
2886
2887         old_write_domain = obj->write_domain;
2888         obj->write_domain = 0;
2889
2890         trace_i915_gem_object_change_domain(obj,
2891                                             obj->read_domains,
2892                                             old_write_domain);
2893 }
2894
2895 /** Flushes the CPU write domain for the object if it's dirty. */
2896 static void
2897 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2898 {
2899         struct drm_device *dev = obj->dev;
2900         uint32_t old_write_domain;
2901
2902         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2903                 return;
2904
2905         i915_gem_clflush_object(obj);
2906         drm_agp_chipset_flush(dev);
2907         old_write_domain = obj->write_domain;
2908         obj->write_domain = 0;
2909
2910         trace_i915_gem_object_change_domain(obj,
2911                                             obj->read_domains,
2912                                             old_write_domain);
2913 }
2914
2915 /**
2916  * Moves a single object to the GTT read, and possibly write domain.
2917  *
2918  * This function returns when the move is complete, including waiting on
2919  * flushes to occur.
2920  */
2921 int
2922 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2923 {
2924         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2925         uint32_t old_write_domain, old_read_domains;
2926         int ret;
2927
2928         /* Not valid to be called on unbound objects. */
2929         if (obj_priv->gtt_space == NULL)
2930                 return -EINVAL;
2931
2932         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
2933         if (ret != 0)
2934                 return ret;
2935
2936         i915_gem_object_flush_cpu_write_domain(obj);
2937
2938         if (write) {
2939                 ret = i915_gem_object_wait_rendering(obj, true);
2940                 if (ret)
2941                         return ret;
2942         }
2943
2944         old_write_domain = obj->write_domain;
2945         old_read_domains = obj->read_domains;
2946
2947         /* It should now be out of any other write domains, and we can update
2948          * the domain values for our changes.
2949          */
2950         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2951         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2952         if (write) {
2953                 obj->read_domains = I915_GEM_DOMAIN_GTT;
2954                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2955                 obj_priv->dirty = 1;
2956         }
2957
2958         trace_i915_gem_object_change_domain(obj,
2959                                             old_read_domains,
2960                                             old_write_domain);
2961
2962         return 0;
2963 }
2964
2965 /*
2966  * Prepare buffer for display plane. Use uninterruptible for possible flush
2967  * wait, as in modesetting process we're not supposed to be interrupted.
2968  */
2969 int
2970 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2971                                      bool pipelined)
2972 {
2973         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2974         uint32_t old_read_domains;
2975         int ret;
2976
2977         /* Not valid to be called on unbound objects. */
2978         if (obj_priv->gtt_space == NULL)
2979                 return -EINVAL;
2980
2981         ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2982         if (ret)
2983                 return ret;
2984
2985         /* Currently, we are always called from an non-interruptible context. */
2986         if (!pipelined) {
2987                 ret = i915_gem_object_wait_rendering(obj, false);
2988                 if (ret)
2989                         return ret;
2990         }
2991
2992         i915_gem_object_flush_cpu_write_domain(obj);
2993
2994         old_read_domains = obj->read_domains;
2995         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2996
2997         trace_i915_gem_object_change_domain(obj,
2998                                             old_read_domains,
2999                                             obj->write_domain);
3000
3001         return 0;
3002 }
3003
3004 /**
3005  * Moves a single object to the CPU read, and possibly write domain.
3006  *
3007  * This function returns when the move is complete, including waiting on
3008  * flushes to occur.
3009  */
3010 static int
3011 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
3012 {
3013         uint32_t old_write_domain, old_read_domains;
3014         int ret;
3015
3016         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
3017         if (ret != 0)
3018                 return ret;
3019
3020         i915_gem_object_flush_gtt_write_domain(obj);
3021
3022         /* If we have a partially-valid cache of the object in the CPU,
3023          * finish invalidating it and free the per-page flags.
3024          */
3025         i915_gem_object_set_to_full_cpu_read_domain(obj);
3026
3027         if (write) {
3028                 ret = i915_gem_object_wait_rendering(obj, true);
3029                 if (ret)
3030                         return ret;
3031         }
3032
3033         old_write_domain = obj->write_domain;
3034         old_read_domains = obj->read_domains;
3035
3036         /* Flush the CPU cache if it's still invalid. */
3037         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3038                 i915_gem_clflush_object(obj);
3039
3040                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3041         }
3042
3043         /* It should now be out of any other write domains, and we can update
3044          * the domain values for our changes.
3045          */
3046         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3047
3048         /* If we're writing through the CPU, then the GPU read domains will
3049          * need to be invalidated at next use.
3050          */
3051         if (write) {
3052                 obj->read_domains = I915_GEM_DOMAIN_CPU;
3053                 obj->write_domain = I915_GEM_DOMAIN_CPU;
3054         }
3055
3056         trace_i915_gem_object_change_domain(obj,
3057                                             old_read_domains,
3058                                             old_write_domain);
3059
3060         return 0;
3061 }
3062
3063 /*
3064  * Set the next domain for the specified object. This
3065  * may not actually perform the necessary flushing/invaliding though,
3066  * as that may want to be batched with other set_domain operations
3067  *
3068  * This is (we hope) the only really tricky part of gem. The goal
3069  * is fairly simple -- track which caches hold bits of the object
3070  * and make sure they remain coherent. A few concrete examples may
3071  * help to explain how it works. For shorthand, we use the notation
3072  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
3073  * a pair of read and write domain masks.
3074  *
3075  * Case 1: the batch buffer
3076  *
3077  *      1. Allocated
3078  *      2. Written by CPU
3079  *      3. Mapped to GTT
3080  *      4. Read by GPU
3081  *      5. Unmapped from GTT
3082  *      6. Freed
3083  *
3084  *      Let's take these a step at a time
3085  *
3086  *      1. Allocated
3087  *              Pages allocated from the kernel may still have
3088  *              cache contents, so we set them to (CPU, CPU) always.
3089  *      2. Written by CPU (using pwrite)
3090  *              The pwrite function calls set_domain (CPU, CPU) and
3091  *              this function does nothing (as nothing changes)
3092  *      3. Mapped by GTT
3093  *              This function asserts that the object is not
3094  *              currently in any GPU-based read or write domains
3095  *      4. Read by GPU
3096  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
3097  *              As write_domain is zero, this function adds in the
3098  *              current read domains (CPU+COMMAND, 0).
3099  *              flush_domains is set to CPU.
3100  *              invalidate_domains is set to COMMAND
3101  *              clflush is run to get data out of the CPU caches
3102  *              then i915_dev_set_domain calls i915_gem_flush to
3103  *              emit an MI_FLUSH and drm_agp_chipset_flush
3104  *      5. Unmapped from GTT
3105  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
3106  *              flush_domains and invalidate_domains end up both zero
3107  *              so no flushing/invalidating happens
3108  *      6. Freed
3109  *              yay, done
3110  *
3111  * Case 2: The shared render buffer
3112  *
3113  *      1. Allocated
3114  *      2. Mapped to GTT
3115  *      3. Read/written by GPU
3116  *      4. set_domain to (CPU,CPU)
3117  *      5. Read/written by CPU
3118  *      6. Read/written by GPU
3119  *
3120  *      1. Allocated
3121  *              Same as last example, (CPU, CPU)
3122  *      2. Mapped to GTT
3123  *              Nothing changes (assertions find that it is not in the GPU)
3124  *      3. Read/written by GPU
3125  *              execbuffer calls set_domain (RENDER, RENDER)
3126  *              flush_domains gets CPU
3127  *              invalidate_domains gets GPU
3128  *              clflush (obj)
3129  *              MI_FLUSH and drm_agp_chipset_flush
3130  *      4. set_domain (CPU, CPU)
3131  *              flush_domains gets GPU
3132  *              invalidate_domains gets CPU
3133  *              wait_rendering (obj) to make sure all drawing is complete.
3134  *              This will include an MI_FLUSH to get the data from GPU
3135  *              to memory
3136  *              clflush (obj) to invalidate the CPU cache
3137  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3138  *      5. Read/written by CPU
3139  *              cache lines are loaded and dirtied
3140  *      6. Read written by GPU
3141  *              Same as last GPU access
3142  *
3143  * Case 3: The constant buffer
3144  *
3145  *      1. Allocated
3146  *      2. Written by CPU
3147  *      3. Read by GPU
3148  *      4. Updated (written) by CPU again
3149  *      5. Read by GPU
3150  *
3151  *      1. Allocated
3152  *              (CPU, CPU)
3153  *      2. Written by CPU
3154  *              (CPU, CPU)
3155  *      3. Read by GPU
3156  *              (CPU+RENDER, 0)
3157  *              flush_domains = CPU
3158  *              invalidate_domains = RENDER
3159  *              clflush (obj)
3160  *              MI_FLUSH
3161  *              drm_agp_chipset_flush
3162  *      4. Updated (written) by CPU again
3163  *              (CPU, CPU)
3164  *              flush_domains = 0 (no previous write domain)
3165  *              invalidate_domains = 0 (no new read domains)
3166  *      5. Read by GPU
3167  *              (CPU+RENDER, 0)
3168  *              flush_domains = CPU
3169  *              invalidate_domains = RENDER
3170  *              clflush (obj)
3171  *              MI_FLUSH
3172  *              drm_agp_chipset_flush
3173  */
3174 static void
3175 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
3176                                   struct intel_ring_buffer *ring)
3177 {
3178         struct drm_device               *dev = obj->dev;
3179         struct drm_i915_private         *dev_priv = dev->dev_private;
3180         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
3181         uint32_t                        invalidate_domains = 0;
3182         uint32_t                        flush_domains = 0;
3183
3184         /*
3185          * If the object isn't moving to a new write domain,
3186          * let the object stay in multiple read domains
3187          */
3188         if (obj->pending_write_domain == 0)
3189                 obj->pending_read_domains |= obj->read_domains;
3190
3191         /*
3192          * Flush the current write domain if
3193          * the new read domains don't match. Invalidate
3194          * any read domains which differ from the old
3195          * write domain
3196          */
3197         if (obj->write_domain &&
3198             obj->write_domain != obj->pending_read_domains) {
3199                 flush_domains |= obj->write_domain;
3200                 invalidate_domains |=
3201                         obj->pending_read_domains & ~obj->write_domain;
3202         }
3203         /*
3204          * Invalidate any read caches which may have
3205          * stale data. That is, any new read domains.
3206          */
3207         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3208         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
3209                 i915_gem_clflush_object(obj);
3210
3211         /* blow away mappings if mapped through GTT */
3212         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_GTT)
3213                 i915_gem_release_mmap(obj);
3214
3215         /* The actual obj->write_domain will be updated with
3216          * pending_write_domain after we emit the accumulated flush for all
3217          * of our domain changes in execbuffers (which clears objects'
3218          * write_domains).  So if we have a current write domain that we
3219          * aren't changing, set pending_write_domain to that.
3220          */
3221         if (flush_domains == 0 && obj->pending_write_domain == 0)
3222                 obj->pending_write_domain = obj->write_domain;
3223
3224         dev->invalidate_domains |= invalidate_domains;
3225         dev->flush_domains |= flush_domains;
3226         if (flush_domains & I915_GEM_GPU_DOMAINS)
3227                 dev_priv->mm.flush_rings |= obj_priv->ring->id;
3228         if (invalidate_domains & I915_GEM_GPU_DOMAINS)
3229                 dev_priv->mm.flush_rings |= ring->id;
3230 }
3231
3232 /**
3233  * Moves the object from a partially CPU read to a full one.
3234  *
3235  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3236  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3237  */
3238 static void
3239 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3240 {
3241         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3242
3243         if (!obj_priv->page_cpu_valid)
3244                 return;
3245
3246         /* If we're partially in the CPU read domain, finish moving it in.
3247          */
3248         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3249                 int i;
3250
3251                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3252                         if (obj_priv->page_cpu_valid[i])
3253                                 continue;
3254                         drm_clflush_pages(obj_priv->pages + i, 1);
3255                 }
3256         }
3257
3258         /* Free the page_cpu_valid mappings which are now stale, whether
3259          * or not we've got I915_GEM_DOMAIN_CPU.
3260          */
3261         kfree(obj_priv->page_cpu_valid);
3262         obj_priv->page_cpu_valid = NULL;
3263 }
3264
3265 /**
3266  * Set the CPU read domain on a range of the object.
3267  *
3268  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3269  * not entirely valid.  The page_cpu_valid member of the object flags which
3270  * pages have been flushed, and will be respected by
3271  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3272  * of the whole object.
3273  *
3274  * This function returns when the move is complete, including waiting on
3275  * flushes to occur.
3276  */
3277 static int
3278 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3279                                           uint64_t offset, uint64_t size)
3280 {
3281         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3282         uint32_t old_read_domains;
3283         int i, ret;
3284
3285         if (offset == 0 && size == obj->size)
3286                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3287
3288         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
3289         if (ret != 0)
3290                 return ret;
3291         i915_gem_object_flush_gtt_write_domain(obj);
3292
3293         /* If we're already fully in the CPU read domain, we're done. */
3294         if (obj_priv->page_cpu_valid == NULL &&
3295             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3296                 return 0;
3297
3298         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3299          * newly adding I915_GEM_DOMAIN_CPU
3300          */
3301         if (obj_priv->page_cpu_valid == NULL) {
3302                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3303                                                    GFP_KERNEL);
3304                 if (obj_priv->page_cpu_valid == NULL)
3305                         return -ENOMEM;
3306         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3307                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3308
3309         /* Flush the cache on any pages that are still invalid from the CPU's
3310          * perspective.
3311          */
3312         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3313              i++) {
3314                 if (obj_priv->page_cpu_valid[i])
3315                         continue;
3316
3317                 drm_clflush_pages(obj_priv->pages + i, 1);
3318
3319                 obj_priv->page_cpu_valid[i] = 1;
3320         }
3321
3322         /* It should now be out of any other write domains, and we can update
3323          * the domain values for our changes.
3324          */
3325         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3326
3327         old_read_domains = obj->read_domains;
3328         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3329
3330         trace_i915_gem_object_change_domain(obj,
3331                                             old_read_domains,
3332                                             obj->write_domain);
3333
3334         return 0;
3335 }
3336
3337 /**
3338  * Pin an object to the GTT and evaluate the relocations landing in it.
3339  */
3340 static int
3341 i915_gem_execbuffer_relocate(struct drm_i915_gem_object *obj,
3342                              struct drm_file *file_priv,
3343                              struct drm_i915_gem_exec_object2 *entry)
3344 {
3345         struct drm_device *dev = obj->base.dev;
3346         drm_i915_private_t *dev_priv = dev->dev_private;
3347         struct drm_i915_gem_relocation_entry __user *user_relocs;
3348         struct drm_gem_object *target_obj = NULL;
3349         uint32_t target_handle = 0;
3350         int i, ret = 0;
3351
3352         user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
3353         for (i = 0; i < entry->relocation_count; i++) {
3354                 struct drm_i915_gem_relocation_entry reloc;
3355                 uint32_t target_offset;
3356
3357                 if (__copy_from_user_inatomic(&reloc,
3358                                               user_relocs+i,
3359                                               sizeof(reloc))) {
3360                         ret = -EFAULT;
3361                         break;
3362                 }
3363
3364                 if (reloc.target_handle != target_handle) {
3365                         drm_gem_object_unreference(target_obj);
3366
3367                         target_obj = drm_gem_object_lookup(dev, file_priv,
3368                                                            reloc.target_handle);
3369                         if (target_obj == NULL) {
3370                                 ret = -ENOENT;
3371                                 break;
3372                         }
3373
3374                         target_handle = reloc.target_handle;
3375                 }
3376                 target_offset = to_intel_bo(target_obj)->gtt_offset;
3377
3378 #if WATCH_RELOC
3379                 DRM_INFO("%s: obj %p offset %08x target %d "
3380                          "read %08x write %08x gtt %08x "
3381                          "presumed %08x delta %08x\n",
3382                          __func__,
3383                          obj,
3384                          (int) reloc.offset,
3385                          (int) reloc.target_handle,
3386                          (int) reloc.read_domains,
3387                          (int) reloc.write_domain,
3388                          (int) target_offset,
3389                          (int) reloc.presumed_offset,
3390                          reloc.delta);
3391 #endif
3392
3393                 /* The target buffer should have appeared before us in the
3394                  * exec_object list, so it should have a GTT space bound by now.
3395                  */
3396                 if (target_offset == 0) {
3397                         DRM_ERROR("No GTT space found for object %d\n",
3398                                   reloc.target_handle);
3399                         ret = -EINVAL;
3400                         break;
3401                 }
3402
3403                 /* Validate that the target is in a valid r/w GPU domain */
3404                 if (reloc.write_domain & (reloc.write_domain - 1)) {
3405                         DRM_ERROR("reloc with multiple write domains: "
3406                                   "obj %p target %d offset %d "
3407                                   "read %08x write %08x",
3408                                   obj, reloc.target_handle,
3409                                   (int) reloc.offset,
3410                                   reloc.read_domains,
3411                                   reloc.write_domain);
3412                         ret = -EINVAL;
3413                         break;
3414                 }
3415                 if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
3416                     reloc.read_domains & I915_GEM_DOMAIN_CPU) {
3417                         DRM_ERROR("reloc with read/write CPU domains: "
3418                                   "obj %p target %d offset %d "
3419                                   "read %08x write %08x",
3420                                   obj, reloc.target_handle,
3421                                   (int) reloc.offset,
3422                                   reloc.read_domains,
3423                                   reloc.write_domain);
3424                         ret = -EINVAL;
3425                         break;
3426                 }
3427                 if (reloc.write_domain && target_obj->pending_write_domain &&
3428                     reloc.write_domain != target_obj->pending_write_domain) {
3429                         DRM_ERROR("Write domain conflict: "
3430                                   "obj %p target %d offset %d "
3431                                   "new %08x old %08x\n",
3432                                   obj, reloc.target_handle,
3433                                   (int) reloc.offset,
3434                                   reloc.write_domain,
3435                                   target_obj->pending_write_domain);
3436                         ret = -EINVAL;
3437                         break;
3438                 }
3439
3440                 target_obj->pending_read_domains |= reloc.read_domains;
3441                 target_obj->pending_write_domain |= reloc.write_domain;
3442
3443                 /* If the relocation already has the right value in it, no
3444                  * more work needs to be done.
3445                  */
3446                 if (target_offset == reloc.presumed_offset)
3447                         continue;
3448
3449                 /* Check that the relocation address is valid... */
3450                 if (reloc.offset > obj->base.size - 4) {
3451                         DRM_ERROR("Relocation beyond object bounds: "
3452                                   "obj %p target %d offset %d size %d.\n",
3453                                   obj, reloc.target_handle,
3454                                   (int) reloc.offset, (int) obj->base.size);
3455                         ret = -EINVAL;
3456                         break;
3457                 }
3458                 if (reloc.offset & 3) {
3459                         DRM_ERROR("Relocation not 4-byte aligned: "
3460                                   "obj %p target %d offset %d.\n",
3461                                   obj, reloc.target_handle,
3462                                   (int) reloc.offset);
3463                         ret = -EINVAL;
3464                         break;
3465                 }
3466
3467                 /* and points to somewhere within the target object. */
3468                 if (reloc.delta >= target_obj->size) {
3469                         DRM_ERROR("Relocation beyond target object bounds: "
3470                                   "obj %p target %d delta %d size %d.\n",
3471                                   obj, reloc.target_handle,
3472                                   (int) reloc.delta, (int) target_obj->size);
3473                         ret = -EINVAL;
3474                         break;
3475                 }
3476
3477                 reloc.delta += target_offset;
3478                 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
3479                         uint32_t page_offset = reloc.offset & ~PAGE_MASK;
3480                         char *vaddr;
3481
3482                         vaddr = kmap_atomic(obj->pages[reloc.offset >> PAGE_SHIFT]);
3483                         *(uint32_t *)(vaddr + page_offset) = reloc.delta;
3484                         kunmap_atomic(vaddr);
3485                 } else {
3486                         uint32_t __iomem *reloc_entry;
3487                         void __iomem *reloc_page;
3488
3489                         ret = i915_gem_object_set_to_gtt_domain(&obj->base, 1);
3490                         if (ret)
3491                                 break;
3492
3493                         /* Map the page containing the relocation we're going to perform.  */
3494                         reloc.offset += obj->gtt_offset;
3495                         reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3496                                                               reloc.offset & PAGE_MASK);
3497                         reloc_entry = (uint32_t __iomem *)
3498                                 (reloc_page + (reloc.offset & ~PAGE_MASK));
3499                         iowrite32(reloc.delta, reloc_entry);
3500                         io_mapping_unmap_atomic(reloc_page);
3501                 }
3502
3503                 /* and update the user's relocation entry */
3504                 reloc.presumed_offset = target_offset;
3505                 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
3506                                               &reloc.presumed_offset,
3507                                               sizeof(reloc.presumed_offset))) {
3508                     ret = -EFAULT;
3509                     break;
3510                 }
3511         }
3512
3513         drm_gem_object_unreference(target_obj);
3514         return ret;
3515 }
3516
3517 static int
3518 i915_gem_execbuffer_pin(struct drm_device *dev,
3519                         struct drm_file *file,
3520                         struct drm_gem_object **object_list,
3521                         struct drm_i915_gem_exec_object2 *exec_list,
3522                         int count)
3523 {
3524         struct drm_i915_private *dev_priv = dev->dev_private;
3525         int ret, i, retry;
3526
3527         /* attempt to pin all of the buffers into the GTT */
3528         for (retry = 0; retry < 2; retry++) {
3529                 ret = 0;
3530                 for (i = 0; i < count; i++) {
3531                         struct drm_i915_gem_exec_object2 *entry = &exec_list[i];
3532                         struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3533                         bool need_fence =
3534                                 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3535                                 obj->tiling_mode != I915_TILING_NONE;
3536
3537                         /* g33/pnv can't fence buffers in the unmappable part */
3538                         bool need_mappable =
3539                                 entry->relocation_count ? true : need_fence;
3540
3541                         /* Check fence reg constraints and rebind if necessary */
3542                         if ((need_fence && !obj->fenceable) ||
3543                             (need_mappable && !obj->mappable)) {
3544                                 ret = i915_gem_object_unbind(&obj->base);
3545                                 if (ret)
3546                                         break;
3547                         }
3548
3549                         ret = i915_gem_object_pin(&obj->base,
3550                                                   entry->alignment,
3551                                                   need_mappable,
3552                                                   need_fence);
3553                         if (ret)
3554                                 break;
3555
3556                         /*
3557                          * Pre-965 chips need a fence register set up in order
3558                          * to properly handle blits to/from tiled surfaces.
3559                          */
3560                         if (need_fence) {
3561                                 ret = i915_gem_object_get_fence_reg(&obj->base, true);
3562                                 if (ret) {
3563                                         i915_gem_object_unpin(&obj->base);
3564                                         break;
3565                                 }
3566
3567                                 dev_priv->fence_regs[obj->fence_reg].gpu = true;
3568                         }
3569
3570                         entry->offset = obj->gtt_offset;
3571                 }
3572
3573                 while (i--)
3574                         i915_gem_object_unpin(object_list[i]);
3575
3576                 if (ret == 0)
3577                         break;
3578
3579                 if (ret != -ENOSPC || retry)
3580                         return ret;
3581
3582                 ret = i915_gem_evict_everything(dev);
3583                 if (ret)
3584                         return ret;
3585         }
3586
3587         return 0;
3588 }
3589
3590 /* Throttle our rendering by waiting until the ring has completed our requests
3591  * emitted over 20 msec ago.
3592  *
3593  * Note that if we were to use the current jiffies each time around the loop,
3594  * we wouldn't escape the function with any frames outstanding if the time to
3595  * render a frame was over 20ms.
3596  *
3597  * This should get us reasonable parallelism between CPU and GPU but also
3598  * relatively low latency when blocking on a particular request to finish.
3599  */
3600 static int
3601 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3602 {
3603         struct drm_i915_private *dev_priv = dev->dev_private;
3604         struct drm_i915_file_private *file_priv = file->driver_priv;
3605         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3606         struct drm_i915_gem_request *request;
3607         struct intel_ring_buffer *ring = NULL;
3608         u32 seqno = 0;
3609         int ret;
3610
3611         spin_lock(&file_priv->mm.lock);
3612         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3613                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3614                         break;
3615
3616                 ring = request->ring;
3617                 seqno = request->seqno;
3618         }
3619         spin_unlock(&file_priv->mm.lock);
3620
3621         if (seqno == 0)
3622                 return 0;
3623
3624         ret = 0;
3625         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
3626                 /* And wait for the seqno passing without holding any locks and
3627                  * causing extra latency for others. This is safe as the irq
3628                  * generation is designed to be run atomically and so is
3629                  * lockless.
3630                  */
3631                 ring->user_irq_get(ring);
3632                 ret = wait_event_interruptible(ring->irq_queue,
3633                                                i915_seqno_passed(ring->get_seqno(ring), seqno)
3634                                                || atomic_read(&dev_priv->mm.wedged));
3635                 ring->user_irq_put(ring);
3636
3637                 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3638                         ret = -EIO;
3639         }
3640
3641         if (ret == 0)
3642                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3643
3644         return ret;
3645 }
3646
3647 static int
3648 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
3649                           uint64_t exec_offset)
3650 {
3651         uint32_t exec_start, exec_len;
3652
3653         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3654         exec_len = (uint32_t) exec->batch_len;
3655
3656         if ((exec_start | exec_len) & 0x7)
3657                 return -EINVAL;
3658
3659         if (!exec_start)
3660                 return -EINVAL;
3661
3662         return 0;
3663 }
3664
3665 static int
3666 validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3667                    int count)
3668 {
3669         int i;
3670
3671         for (i = 0; i < count; i++) {
3672                 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
3673                 size_t length = exec[i].relocation_count * sizeof(struct drm_i915_gem_relocation_entry);
3674
3675                 if (!access_ok(VERIFY_READ, ptr, length))
3676                         return -EFAULT;
3677
3678                 /* we may also need to update the presumed offsets */
3679                 if (!access_ok(VERIFY_WRITE, ptr, length))
3680                         return -EFAULT;
3681
3682                 if (fault_in_pages_readable(ptr, length))
3683                         return -EFAULT;
3684         }
3685
3686         return 0;
3687 }
3688
3689 static int
3690 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3691                        struct drm_file *file,
3692                        struct drm_i915_gem_execbuffer2 *args,
3693                        struct drm_i915_gem_exec_object2 *exec_list)
3694 {
3695         drm_i915_private_t *dev_priv = dev->dev_private;
3696         struct drm_gem_object **object_list = NULL;
3697         struct drm_gem_object *batch_obj;
3698         struct drm_clip_rect *cliprects = NULL;
3699         struct drm_i915_gem_request *request = NULL;
3700         int ret, i, flips;
3701         uint64_t exec_offset;
3702
3703         struct intel_ring_buffer *ring = NULL;
3704
3705         ret = i915_gem_check_is_wedged(dev);
3706         if (ret)
3707                 return ret;
3708
3709         ret = validate_exec_list(exec_list, args->buffer_count);
3710         if (ret)
3711                 return ret;
3712
3713 #if WATCH_EXEC
3714         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3715                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3716 #endif
3717         switch (args->flags & I915_EXEC_RING_MASK) {
3718         case I915_EXEC_DEFAULT:
3719         case I915_EXEC_RENDER:
3720                 ring = &dev_priv->render_ring;
3721                 break;
3722         case I915_EXEC_BSD:
3723                 if (!HAS_BSD(dev)) {
3724                         DRM_ERROR("execbuf with invalid ring (BSD)\n");
3725                         return -EINVAL;
3726                 }
3727                 ring = &dev_priv->bsd_ring;
3728                 break;
3729         case I915_EXEC_BLT:
3730                 if (!HAS_BLT(dev)) {
3731                         DRM_ERROR("execbuf with invalid ring (BLT)\n");
3732                         return -EINVAL;
3733                 }
3734                 ring = &dev_priv->blt_ring;
3735                 break;
3736         default:
3737                 DRM_ERROR("execbuf with unknown ring: %d\n",
3738                           (int)(args->flags & I915_EXEC_RING_MASK));
3739                 return -EINVAL;
3740         }
3741
3742         if (args->buffer_count < 1) {
3743                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3744                 return -EINVAL;
3745         }
3746         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3747         if (object_list == NULL) {
3748                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3749                           args->buffer_count);
3750                 ret = -ENOMEM;
3751                 goto pre_mutex_err;
3752         }
3753
3754         if (args->num_cliprects != 0) {
3755                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3756                                     GFP_KERNEL);
3757                 if (cliprects == NULL) {
3758                         ret = -ENOMEM;
3759                         goto pre_mutex_err;
3760                 }
3761
3762                 ret = copy_from_user(cliprects,
3763                                      (struct drm_clip_rect __user *)
3764                                      (uintptr_t) args->cliprects_ptr,
3765                                      sizeof(*cliprects) * args->num_cliprects);
3766                 if (ret != 0) {
3767                         DRM_ERROR("copy %d cliprects failed: %d\n",
3768                                   args->num_cliprects, ret);
3769                         ret = -EFAULT;
3770                         goto pre_mutex_err;
3771                 }
3772         }
3773
3774         request = kzalloc(sizeof(*request), GFP_KERNEL);
3775         if (request == NULL) {
3776                 ret = -ENOMEM;
3777                 goto pre_mutex_err;
3778         }
3779
3780         ret = i915_mutex_lock_interruptible(dev);
3781         if (ret)
3782                 goto pre_mutex_err;
3783
3784         if (dev_priv->mm.suspended) {
3785                 mutex_unlock(&dev->struct_mutex);
3786                 ret = -EBUSY;
3787                 goto pre_mutex_err;
3788         }
3789
3790         /* Look up object handles */
3791         for (i = 0; i < args->buffer_count; i++) {
3792                 struct drm_i915_gem_object *obj_priv;
3793
3794                 object_list[i] = drm_gem_object_lookup(dev, file,
3795                                                        exec_list[i].handle);
3796                 if (object_list[i] == NULL) {
3797                         DRM_ERROR("Invalid object handle %d at index %d\n",
3798                                    exec_list[i].handle, i);
3799                         /* prevent error path from reading uninitialized data */
3800                         args->buffer_count = i + 1;
3801                         ret = -ENOENT;
3802                         goto err;
3803                 }
3804
3805                 obj_priv = to_intel_bo(object_list[i]);
3806                 if (obj_priv->in_execbuffer) {
3807                         DRM_ERROR("Object %p appears more than once in object list\n",
3808                                    object_list[i]);
3809                         /* prevent error path from reading uninitialized data */
3810                         args->buffer_count = i + 1;
3811                         ret = -EINVAL;
3812                         goto err;
3813                 }
3814                 obj_priv->in_execbuffer = true;
3815         }
3816
3817         /* Move the objects en-masse into the GTT, evicting if necessary. */
3818         ret = i915_gem_execbuffer_pin(dev, file,
3819                                       object_list, exec_list,
3820                                       args->buffer_count);
3821         if (ret)
3822                 goto err;
3823
3824         /* The objects are in their final locations, apply the relocations. */
3825         for (i = 0; i < args->buffer_count; i++) {
3826                 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3827                 obj->base.pending_read_domains = 0;
3828                 obj->base.pending_write_domain = 0;
3829                 ret = i915_gem_execbuffer_relocate(obj, file, &exec_list[i]);
3830                 if (ret)
3831                         goto err;
3832         }
3833
3834         /* Set the pending read domains for the batch buffer to COMMAND */
3835         batch_obj = object_list[args->buffer_count-1];
3836         if (batch_obj->pending_write_domain) {
3837                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3838                 ret = -EINVAL;
3839                 goto err;
3840         }
3841         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3842
3843         /* Sanity check the batch buffer */
3844         exec_offset = to_intel_bo(batch_obj)->gtt_offset;
3845         ret = i915_gem_check_execbuffer(args, exec_offset);
3846         if (ret != 0) {
3847                 DRM_ERROR("execbuf with invalid offset/length\n");
3848                 goto err;
3849         }
3850
3851         /* Zero the global flush/invalidate flags. These
3852          * will be modified as new domains are computed
3853          * for each object
3854          */
3855         dev->invalidate_domains = 0;
3856         dev->flush_domains = 0;
3857         dev_priv->mm.flush_rings = 0;
3858         for (i = 0; i < args->buffer_count; i++)
3859                 i915_gem_object_set_to_gpu_domain(object_list[i], ring);
3860
3861         if (dev->invalidate_domains | dev->flush_domains) {
3862 #if WATCH_EXEC
3863                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3864                           __func__,
3865                          dev->invalidate_domains,
3866                          dev->flush_domains);
3867 #endif
3868                 i915_gem_flush(dev, file,
3869                                dev->invalidate_domains,
3870                                dev->flush_domains,
3871                                dev_priv->mm.flush_rings);
3872         }
3873
3874 #if WATCH_COHERENCY
3875         for (i = 0; i < args->buffer_count; i++) {
3876                 i915_gem_object_check_coherency(object_list[i],
3877                                                 exec_list[i].handle);
3878         }
3879 #endif
3880
3881 #if WATCH_EXEC
3882         i915_gem_dump_object(batch_obj,
3883                               args->batch_len,
3884                               __func__,
3885                               ~0);
3886 #endif
3887
3888         /* Check for any pending flips. As we only maintain a flip queue depth
3889          * of 1, we can simply insert a WAIT for the next display flip prior
3890          * to executing the batch and avoid stalling the CPU.
3891          */
3892         flips = 0;
3893         for (i = 0; i < args->buffer_count; i++) {
3894                 if (object_list[i]->write_domain)
3895                         flips |= atomic_read(&to_intel_bo(object_list[i])->pending_flip);
3896         }
3897         if (flips) {
3898                 int plane, flip_mask;
3899
3900                 for (plane = 0; flips >> plane; plane++) {
3901                         if (((flips >> plane) & 1) == 0)
3902                                 continue;
3903
3904                         if (plane)
3905                                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
3906                         else
3907                                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
3908
3909                         ret = intel_ring_begin(ring, 2);
3910                         if (ret)
3911                                 goto err;
3912
3913                         intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
3914                         intel_ring_emit(ring, MI_NOOP);
3915                         intel_ring_advance(ring);
3916                 }
3917         }
3918
3919         /* Exec the batchbuffer */
3920         ret = ring->dispatch_execbuffer(ring, args, cliprects, exec_offset);
3921         if (ret) {
3922                 DRM_ERROR("dispatch failed %d\n", ret);
3923                 goto err;
3924         }
3925
3926         for (i = 0; i < args->buffer_count; i++) {
3927                 struct drm_gem_object *obj = object_list[i];
3928
3929                 obj->read_domains = obj->pending_read_domains;
3930                 obj->write_domain = obj->pending_write_domain;
3931
3932                 i915_gem_object_move_to_active(obj, ring);
3933                 if (obj->write_domain) {
3934                         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3935                         obj_priv->dirty = 1;
3936                         list_move_tail(&obj_priv->gpu_write_list,
3937                                        &ring->gpu_write_list);
3938                         intel_mark_busy(dev, obj);
3939                 }
3940
3941                 trace_i915_gem_object_change_domain(obj,
3942                                                     obj->read_domains,
3943                                                     obj->write_domain);
3944         }
3945
3946         /*
3947          * Ensure that the commands in the batch buffer are
3948          * finished before the interrupt fires
3949          */
3950         i915_retire_commands(dev, ring);
3951
3952         if (i915_add_request(dev, file, request, ring))
3953                 ring->outstanding_lazy_request = true;
3954         else
3955                 request = NULL;
3956
3957 err:
3958         for (i = 0; i < args->buffer_count; i++) {
3959                 if (object_list[i] == NULL)
3960                     break;
3961
3962                 to_intel_bo(object_list[i])->in_execbuffer = false;
3963                 drm_gem_object_unreference(object_list[i]);
3964         }
3965
3966         mutex_unlock(&dev->struct_mutex);
3967
3968 pre_mutex_err:
3969         drm_free_large(object_list);
3970         kfree(cliprects);
3971         kfree(request);
3972
3973         return ret;
3974 }
3975
3976 /*
3977  * Legacy execbuffer just creates an exec2 list from the original exec object
3978  * list array and passes it to the real function.
3979  */
3980 int
3981 i915_gem_execbuffer(struct drm_device *dev, void *data,
3982                     struct drm_file *file_priv)
3983 {
3984         struct drm_i915_gem_execbuffer *args = data;
3985         struct drm_i915_gem_execbuffer2 exec2;
3986         struct drm_i915_gem_exec_object *exec_list = NULL;
3987         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3988         int ret, i;
3989
3990 #if WATCH_EXEC
3991         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3992                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3993 #endif
3994
3995         if (args->buffer_count < 1) {
3996                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3997                 return -EINVAL;
3998         }
3999
4000         /* Copy in the exec list from userland */
4001         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4002         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4003         if (exec_list == NULL || exec2_list == NULL) {
4004                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4005                           args->buffer_count);
4006                 drm_free_large(exec_list);
4007                 drm_free_large(exec2_list);
4008                 return -ENOMEM;
4009         }
4010         ret = copy_from_user(exec_list,
4011                              (struct drm_i915_relocation_entry __user *)
4012                              (uintptr_t) args->buffers_ptr,
4013                              sizeof(*exec_list) * args->buffer_count);
4014         if (ret != 0) {
4015                 DRM_ERROR("copy %d exec entries failed %d\n",
4016                           args->buffer_count, ret);
4017                 drm_free_large(exec_list);
4018                 drm_free_large(exec2_list);
4019                 return -EFAULT;
4020         }
4021
4022         for (i = 0; i < args->buffer_count; i++) {
4023                 exec2_list[i].handle = exec_list[i].handle;
4024                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4025                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4026                 exec2_list[i].alignment = exec_list[i].alignment;
4027                 exec2_list[i].offset = exec_list[i].offset;
4028                 if (INTEL_INFO(dev)->gen < 4)
4029                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4030                 else
4031                         exec2_list[i].flags = 0;
4032         }
4033
4034         exec2.buffers_ptr = args->buffers_ptr;
4035         exec2.buffer_count = args->buffer_count;
4036         exec2.batch_start_offset = args->batch_start_offset;
4037         exec2.batch_len = args->batch_len;
4038         exec2.DR1 = args->DR1;
4039         exec2.DR4 = args->DR4;
4040         exec2.num_cliprects = args->num_cliprects;
4041         exec2.cliprects_ptr = args->cliprects_ptr;
4042         exec2.flags = I915_EXEC_RENDER;
4043
4044         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4045         if (!ret) {
4046                 /* Copy the new buffer offsets back to the user's exec list. */
4047                 for (i = 0; i < args->buffer_count; i++)
4048                         exec_list[i].offset = exec2_list[i].offset;
4049                 /* ... and back out to userspace */
4050                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4051                                    (uintptr_t) args->buffers_ptr,
4052                                    exec_list,
4053                                    sizeof(*exec_list) * args->buffer_count);
4054                 if (ret) {
4055                         ret = -EFAULT;
4056                         DRM_ERROR("failed to copy %d exec entries "
4057                                   "back to user (%d)\n",
4058                                   args->buffer_count, ret);
4059                 }
4060         }
4061
4062         drm_free_large(exec_list);
4063         drm_free_large(exec2_list);
4064         return ret;
4065 }
4066
4067 int
4068 i915_gem_execbuffer2(struct drm_device *dev, void *data,
4069                      struct drm_file *file_priv)
4070 {
4071         struct drm_i915_gem_execbuffer2 *args = data;
4072         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4073         int ret;
4074
4075 #if WATCH_EXEC
4076         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4077                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4078 #endif
4079
4080         if (args->buffer_count < 1) {
4081                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4082                 return -EINVAL;
4083         }
4084
4085         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4086         if (exec2_list == NULL) {
4087                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4088                           args->buffer_count);
4089                 return -ENOMEM;
4090         }
4091         ret = copy_from_user(exec2_list,
4092                              (struct drm_i915_relocation_entry __user *)
4093                              (uintptr_t) args->buffers_ptr,
4094                              sizeof(*exec2_list) * args->buffer_count);
4095         if (ret != 0) {
4096                 DRM_ERROR("copy %d exec entries failed %d\n",
4097                           args->buffer_count, ret);
4098                 drm_free_large(exec2_list);
4099                 return -EFAULT;
4100         }
4101
4102         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4103         if (!ret) {
4104                 /* Copy the new buffer offsets back to the user's exec list. */
4105                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4106                                    (uintptr_t) args->buffers_ptr,
4107                                    exec2_list,
4108                                    sizeof(*exec2_list) * args->buffer_count);
4109                 if (ret) {
4110                         ret = -EFAULT;
4111                         DRM_ERROR("failed to copy %d exec entries "
4112                                   "back to user (%d)\n",
4113                                   args->buffer_count, ret);
4114                 }
4115         }
4116
4117         drm_free_large(exec2_list);
4118         return ret;
4119 }
4120
4121 int
4122 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment,
4123                     bool mappable, bool need_fence)
4124 {
4125         struct drm_device *dev = obj->dev;
4126         struct drm_i915_private *dev_priv = dev->dev_private;
4127         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4128         int ret;
4129
4130         BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4131         WARN_ON(i915_verify_lists(dev));
4132
4133         if (obj_priv->gtt_space != NULL) {
4134                 if ((alignment && obj_priv->gtt_offset & (alignment - 1)) ||
4135                     (need_fence && !obj_priv->fenceable) ||
4136                     (mappable && !obj_priv->mappable)) {
4137                         WARN(obj_priv->pin_count,
4138                              "bo is already pinned with incorrect alignment:"
4139                              " offset=%x, req.alignment=%x, need_fence=%d, fenceable=%d, mappable=%d, cpu_accessible=%d\n",
4140                              obj_priv->gtt_offset, alignment,
4141                              need_fence, obj_priv->fenceable,
4142                              mappable, obj_priv->mappable);
4143                         ret = i915_gem_object_unbind(obj);
4144                         if (ret)
4145                                 return ret;
4146                 }
4147         }
4148
4149         if (obj_priv->gtt_space == NULL) {
4150                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
4151                                                   mappable, need_fence);
4152                 if (ret)
4153                         return ret;
4154         }
4155
4156         if (obj_priv->pin_count++ == 0) {
4157                 i915_gem_info_add_pin(dev_priv, obj_priv, mappable);
4158                 if (!obj_priv->active)
4159                         list_move_tail(&obj_priv->mm_list,
4160                                        &dev_priv->mm.pinned_list);
4161         }
4162         BUG_ON(!obj_priv->pin_mappable && mappable);
4163
4164         WARN_ON(i915_verify_lists(dev));
4165         return 0;
4166 }
4167
4168 void
4169 i915_gem_object_unpin(struct drm_gem_object *obj)
4170 {
4171         struct drm_device *dev = obj->dev;
4172         drm_i915_private_t *dev_priv = dev->dev_private;
4173         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4174
4175         WARN_ON(i915_verify_lists(dev));
4176         BUG_ON(obj_priv->pin_count == 0);
4177         BUG_ON(obj_priv->gtt_space == NULL);
4178
4179         if (--obj_priv->pin_count == 0) {
4180                 if (!obj_priv->active)
4181                         list_move_tail(&obj_priv->mm_list,
4182                                        &dev_priv->mm.inactive_list);
4183                 i915_gem_info_remove_pin(dev_priv, obj_priv);
4184         }
4185         WARN_ON(i915_verify_lists(dev));
4186 }
4187
4188 int
4189 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4190                    struct drm_file *file_priv)
4191 {
4192         struct drm_i915_gem_pin *args = data;
4193         struct drm_gem_object *obj;
4194         struct drm_i915_gem_object *obj_priv;
4195         int ret;
4196
4197         ret = i915_mutex_lock_interruptible(dev);
4198         if (ret)
4199                 return ret;
4200
4201         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4202         if (obj == NULL) {
4203                 ret = -ENOENT;
4204                 goto unlock;
4205         }
4206         obj_priv = to_intel_bo(obj);
4207
4208         if (obj_priv->madv != I915_MADV_WILLNEED) {
4209                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4210                 ret = -EINVAL;
4211                 goto out;
4212         }
4213
4214         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4215                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4216                           args->handle);
4217                 ret = -EINVAL;
4218                 goto out;
4219         }
4220
4221         obj_priv->user_pin_count++;
4222         obj_priv->pin_filp = file_priv;
4223         if (obj_priv->user_pin_count == 1) {
4224                 ret = i915_gem_object_pin(obj, args->alignment,
4225                                           true, obj_priv->tiling_mode);
4226                 if (ret)
4227                         goto out;
4228         }
4229
4230         /* XXX - flush the CPU caches for pinned objects
4231          * as the X server doesn't manage domains yet
4232          */
4233         i915_gem_object_flush_cpu_write_domain(obj);
4234         args->offset = obj_priv->gtt_offset;
4235 out:
4236         drm_gem_object_unreference(obj);
4237 unlock:
4238         mutex_unlock(&dev->struct_mutex);
4239         return ret;
4240 }
4241
4242 int
4243 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4244                      struct drm_file *file_priv)
4245 {
4246         struct drm_i915_gem_pin *args = data;
4247         struct drm_gem_object *obj;
4248         struct drm_i915_gem_object *obj_priv;
4249         int ret;
4250
4251         ret = i915_mutex_lock_interruptible(dev);
4252         if (ret)
4253                 return ret;
4254
4255         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4256         if (obj == NULL) {
4257                 ret = -ENOENT;
4258                 goto unlock;
4259         }
4260         obj_priv = to_intel_bo(obj);
4261
4262         if (obj_priv->pin_filp != file_priv) {
4263                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4264                           args->handle);
4265                 ret = -EINVAL;
4266                 goto out;
4267         }
4268         obj_priv->user_pin_count--;
4269         if (obj_priv->user_pin_count == 0) {
4270                 obj_priv->pin_filp = NULL;
4271                 i915_gem_object_unpin(obj);
4272         }
4273
4274 out:
4275         drm_gem_object_unreference(obj);
4276 unlock:
4277         mutex_unlock(&dev->struct_mutex);
4278         return ret;
4279 }
4280
4281 int
4282 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4283                     struct drm_file *file_priv)
4284 {
4285         struct drm_i915_gem_busy *args = data;
4286         struct drm_gem_object *obj;
4287         struct drm_i915_gem_object *obj_priv;
4288         int ret;
4289
4290         ret = i915_mutex_lock_interruptible(dev);
4291         if (ret)
4292                 return ret;
4293
4294         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4295         if (obj == NULL) {
4296                 ret = -ENOENT;
4297                 goto unlock;
4298         }
4299         obj_priv = to_intel_bo(obj);
4300
4301         /* Count all active objects as busy, even if they are currently not used
4302          * by the gpu. Users of this interface expect objects to eventually
4303          * become non-busy without any further actions, therefore emit any
4304          * necessary flushes here.
4305          */
4306         args->busy = obj_priv->active;
4307         if (args->busy) {
4308                 /* Unconditionally flush objects, even when the gpu still uses this
4309                  * object. Userspace calling this function indicates that it wants to
4310                  * use this buffer rather sooner than later, so issuing the required
4311                  * flush earlier is beneficial.
4312                  */
4313                 if (obj->write_domain & I915_GEM_GPU_DOMAINS)
4314                         i915_gem_flush_ring(dev, file_priv,
4315                                             obj_priv->ring,
4316                                             0, obj->write_domain);
4317
4318                 /* Update the active list for the hardware's current position.
4319                  * Otherwise this only updates on a delayed timer or when irqs
4320                  * are actually unmasked, and our working set ends up being
4321                  * larger than required.
4322                  */
4323                 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4324
4325                 args->busy = obj_priv->active;
4326         }
4327
4328         drm_gem_object_unreference(obj);
4329 unlock:
4330         mutex_unlock(&dev->struct_mutex);
4331         return ret;
4332 }
4333
4334 int
4335 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4336                         struct drm_file *file_priv)
4337 {
4338     return i915_gem_ring_throttle(dev, file_priv);
4339 }
4340
4341 int
4342 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4343                        struct drm_file *file_priv)
4344 {
4345         struct drm_i915_gem_madvise *args = data;
4346         struct drm_gem_object *obj;
4347         struct drm_i915_gem_object *obj_priv;
4348         int ret;
4349
4350         switch (args->madv) {
4351         case I915_MADV_DONTNEED:
4352         case I915_MADV_WILLNEED:
4353             break;
4354         default:
4355             return -EINVAL;
4356         }
4357
4358         ret = i915_mutex_lock_interruptible(dev);
4359         if (ret)
4360                 return ret;
4361
4362         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4363         if (obj == NULL) {
4364                 ret = -ENOENT;
4365                 goto unlock;
4366         }
4367         obj_priv = to_intel_bo(obj);
4368
4369         if (obj_priv->pin_count) {
4370                 ret = -EINVAL;
4371                 goto out;
4372         }
4373
4374         if (obj_priv->madv != __I915_MADV_PURGED)
4375                 obj_priv->madv = args->madv;
4376
4377         /* if the object is no longer bound, discard its backing storage */
4378         if (i915_gem_object_is_purgeable(obj_priv) &&
4379             obj_priv->gtt_space == NULL)
4380                 i915_gem_object_truncate(obj);
4381
4382         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4383
4384 out:
4385         drm_gem_object_unreference(obj);
4386 unlock:
4387         mutex_unlock(&dev->struct_mutex);
4388         return ret;
4389 }
4390
4391 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4392                                               size_t size)
4393 {
4394         struct drm_i915_private *dev_priv = dev->dev_private;
4395         struct drm_i915_gem_object *obj;
4396
4397         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4398         if (obj == NULL)
4399                 return NULL;
4400
4401         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4402                 kfree(obj);
4403                 return NULL;
4404         }
4405
4406         i915_gem_info_add_obj(dev_priv, size);
4407
4408         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4409         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4410
4411         obj->agp_type = AGP_USER_MEMORY;
4412         obj->base.driver_private = NULL;
4413         obj->fence_reg = I915_FENCE_REG_NONE;
4414         INIT_LIST_HEAD(&obj->mm_list);
4415         INIT_LIST_HEAD(&obj->ring_list);
4416         INIT_LIST_HEAD(&obj->gpu_write_list);
4417         obj->madv = I915_MADV_WILLNEED;
4418         obj->fenceable = true;
4419         obj->mappable = true;
4420
4421         return &obj->base;
4422 }
4423
4424 int i915_gem_init_object(struct drm_gem_object *obj)
4425 {
4426         BUG();
4427
4428         return 0;
4429 }
4430
4431 static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4432 {
4433         struct drm_device *dev = obj->dev;
4434         drm_i915_private_t *dev_priv = dev->dev_private;
4435         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4436         int ret;
4437
4438         ret = i915_gem_object_unbind(obj);
4439         if (ret == -ERESTARTSYS) {
4440                 list_move(&obj_priv->mm_list,
4441                           &dev_priv->mm.deferred_free_list);
4442                 return;
4443         }
4444
4445         if (obj->map_list.map)
4446                 i915_gem_free_mmap_offset(obj);
4447
4448         drm_gem_object_release(obj);
4449         i915_gem_info_remove_obj(dev_priv, obj->size);
4450
4451         kfree(obj_priv->page_cpu_valid);
4452         kfree(obj_priv->bit_17);
4453         kfree(obj_priv);
4454 }
4455
4456 void i915_gem_free_object(struct drm_gem_object *obj)
4457 {
4458         struct drm_device *dev = obj->dev;
4459         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4460
4461         trace_i915_gem_object_destroy(obj);
4462
4463         while (obj_priv->pin_count > 0)
4464                 i915_gem_object_unpin(obj);
4465
4466         if (obj_priv->phys_obj)
4467                 i915_gem_detach_phys_object(dev, obj);
4468
4469         i915_gem_free_object_tail(obj);
4470 }
4471
4472 int
4473 i915_gem_idle(struct drm_device *dev)
4474 {
4475         drm_i915_private_t *dev_priv = dev->dev_private;
4476         int ret;
4477
4478         mutex_lock(&dev->struct_mutex);
4479
4480         if (dev_priv->mm.suspended) {
4481                 mutex_unlock(&dev->struct_mutex);
4482                 return 0;
4483         }
4484
4485         ret = i915_gpu_idle(dev);
4486         if (ret) {
4487                 mutex_unlock(&dev->struct_mutex);
4488                 return ret;
4489         }
4490
4491         /* Under UMS, be paranoid and evict. */
4492         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4493                 ret = i915_gem_evict_inactive(dev);
4494                 if (ret) {
4495                         mutex_unlock(&dev->struct_mutex);
4496                         return ret;
4497                 }
4498         }
4499
4500         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4501          * We need to replace this with a semaphore, or something.
4502          * And not confound mm.suspended!
4503          */
4504         dev_priv->mm.suspended = 1;
4505         del_timer_sync(&dev_priv->hangcheck_timer);
4506
4507         i915_kernel_lost_context(dev);
4508         i915_gem_cleanup_ringbuffer(dev);
4509
4510         mutex_unlock(&dev->struct_mutex);
4511
4512         /* Cancel the retire work handler, which should be idle now. */
4513         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4514
4515         return 0;
4516 }
4517
4518 /*
4519  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4520  * over cache flushing.
4521  */
4522 static int
4523 i915_gem_init_pipe_control(struct drm_device *dev)
4524 {
4525         drm_i915_private_t *dev_priv = dev->dev_private;
4526         struct drm_gem_object *obj;
4527         struct drm_i915_gem_object *obj_priv;
4528         int ret;
4529
4530         obj = i915_gem_alloc_object(dev, 4096);
4531         if (obj == NULL) {
4532                 DRM_ERROR("Failed to allocate seqno page\n");
4533                 ret = -ENOMEM;
4534                 goto err;
4535         }
4536         obj_priv = to_intel_bo(obj);
4537         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4538
4539         ret = i915_gem_object_pin(obj, 4096, true, false);
4540         if (ret)
4541                 goto err_unref;
4542
4543         dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4544         dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
4545         if (dev_priv->seqno_page == NULL)
4546                 goto err_unpin;
4547
4548         dev_priv->seqno_obj = obj;
4549         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4550
4551         return 0;
4552
4553 err_unpin:
4554         i915_gem_object_unpin(obj);
4555 err_unref:
4556         drm_gem_object_unreference(obj);
4557 err:
4558         return ret;
4559 }
4560
4561
4562 static void
4563 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4564 {
4565         drm_i915_private_t *dev_priv = dev->dev_private;
4566         struct drm_gem_object *obj;
4567         struct drm_i915_gem_object *obj_priv;
4568
4569         obj = dev_priv->seqno_obj;
4570         obj_priv = to_intel_bo(obj);
4571         kunmap(obj_priv->pages[0]);
4572         i915_gem_object_unpin(obj);
4573         drm_gem_object_unreference(obj);
4574         dev_priv->seqno_obj = NULL;
4575
4576         dev_priv->seqno_page = NULL;
4577 }
4578
4579 int
4580 i915_gem_init_ringbuffer(struct drm_device *dev)
4581 {
4582         drm_i915_private_t *dev_priv = dev->dev_private;
4583         int ret;
4584
4585         if (HAS_PIPE_CONTROL(dev)) {
4586                 ret = i915_gem_init_pipe_control(dev);
4587                 if (ret)
4588                         return ret;
4589         }
4590
4591         ret = intel_init_render_ring_buffer(dev);
4592         if (ret)
4593                 goto cleanup_pipe_control;
4594
4595         if (HAS_BSD(dev)) {
4596                 ret = intel_init_bsd_ring_buffer(dev);
4597                 if (ret)
4598                         goto cleanup_render_ring;
4599         }
4600
4601         if (HAS_BLT(dev)) {
4602                 ret = intel_init_blt_ring_buffer(dev);
4603                 if (ret)
4604                         goto cleanup_bsd_ring;
4605         }
4606
4607         dev_priv->next_seqno = 1;
4608
4609         return 0;
4610
4611 cleanup_bsd_ring:
4612         intel_cleanup_ring_buffer(&dev_priv->bsd_ring);
4613 cleanup_render_ring:
4614         intel_cleanup_ring_buffer(&dev_priv->render_ring);
4615 cleanup_pipe_control:
4616         if (HAS_PIPE_CONTROL(dev))
4617                 i915_gem_cleanup_pipe_control(dev);
4618         return ret;
4619 }
4620
4621 void
4622 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4623 {
4624         drm_i915_private_t *dev_priv = dev->dev_private;
4625
4626         intel_cleanup_ring_buffer(&dev_priv->render_ring);
4627         intel_cleanup_ring_buffer(&dev_priv->bsd_ring);
4628         intel_cleanup_ring_buffer(&dev_priv->blt_ring);
4629         if (HAS_PIPE_CONTROL(dev))
4630                 i915_gem_cleanup_pipe_control(dev);
4631 }
4632
4633 int
4634 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4635                        struct drm_file *file_priv)
4636 {
4637         drm_i915_private_t *dev_priv = dev->dev_private;
4638         int ret;
4639
4640         if (drm_core_check_feature(dev, DRIVER_MODESET))
4641                 return 0;
4642
4643         if (atomic_read(&dev_priv->mm.wedged)) {
4644                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4645                 atomic_set(&dev_priv->mm.wedged, 0);
4646         }
4647
4648         mutex_lock(&dev->struct_mutex);
4649         dev_priv->mm.suspended = 0;
4650
4651         ret = i915_gem_init_ringbuffer(dev);
4652         if (ret != 0) {
4653                 mutex_unlock(&dev->struct_mutex);
4654                 return ret;
4655         }
4656
4657         BUG_ON(!list_empty(&dev_priv->mm.active_list));
4658         BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4659         BUG_ON(!list_empty(&dev_priv->bsd_ring.active_list));
4660         BUG_ON(!list_empty(&dev_priv->blt_ring.active_list));
4661         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4662         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4663         BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4664         BUG_ON(!list_empty(&dev_priv->bsd_ring.request_list));
4665         BUG_ON(!list_empty(&dev_priv->blt_ring.request_list));
4666         mutex_unlock(&dev->struct_mutex);
4667
4668         ret = drm_irq_install(dev);
4669         if (ret)
4670                 goto cleanup_ringbuffer;
4671
4672         return 0;
4673
4674 cleanup_ringbuffer:
4675         mutex_lock(&dev->struct_mutex);
4676         i915_gem_cleanup_ringbuffer(dev);
4677         dev_priv->mm.suspended = 1;
4678         mutex_unlock(&dev->struct_mutex);
4679
4680         return ret;
4681 }
4682
4683 int
4684 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4685                        struct drm_file *file_priv)
4686 {
4687         if (drm_core_check_feature(dev, DRIVER_MODESET))
4688                 return 0;
4689
4690         drm_irq_uninstall(dev);
4691         return i915_gem_idle(dev);
4692 }
4693
4694 void
4695 i915_gem_lastclose(struct drm_device *dev)
4696 {
4697         int ret;
4698
4699         if (drm_core_check_feature(dev, DRIVER_MODESET))
4700                 return;
4701
4702         ret = i915_gem_idle(dev);
4703         if (ret)
4704                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4705 }
4706
4707 static void
4708 init_ring_lists(struct intel_ring_buffer *ring)
4709 {
4710         INIT_LIST_HEAD(&ring->active_list);
4711         INIT_LIST_HEAD(&ring->request_list);
4712         INIT_LIST_HEAD(&ring->gpu_write_list);
4713 }
4714
4715 void
4716 i915_gem_load(struct drm_device *dev)
4717 {
4718         int i;
4719         drm_i915_private_t *dev_priv = dev->dev_private;
4720
4721         INIT_LIST_HEAD(&dev_priv->mm.active_list);
4722         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4723         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4724         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
4725         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4726         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4727         init_ring_lists(&dev_priv->render_ring);
4728         init_ring_lists(&dev_priv->bsd_ring);
4729         init_ring_lists(&dev_priv->blt_ring);
4730         for (i = 0; i < 16; i++)
4731                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4732         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4733                           i915_gem_retire_work_handler);
4734         init_completion(&dev_priv->error_completion);
4735
4736         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4737         if (IS_GEN3(dev)) {
4738                 u32 tmp = I915_READ(MI_ARB_STATE);
4739                 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4740                         /* arb state is a masked write, so set bit + bit in mask */
4741                         tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4742                         I915_WRITE(MI_ARB_STATE, tmp);
4743                 }
4744         }
4745
4746         /* Old X drivers will take 0-2 for front, back, depth buffers */
4747         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4748                 dev_priv->fence_reg_start = 3;
4749
4750         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4751                 dev_priv->num_fence_regs = 16;
4752         else
4753                 dev_priv->num_fence_regs = 8;
4754
4755         /* Initialize fence registers to zero */
4756         switch (INTEL_INFO(dev)->gen) {
4757         case 6:
4758                 for (i = 0; i < 16; i++)
4759                         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4760                 break;
4761         case 5:
4762         case 4:
4763                 for (i = 0; i < 16; i++)
4764                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4765                 break;
4766         case 3:
4767                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4768                         for (i = 0; i < 8; i++)
4769                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4770         case 2:
4771                 for (i = 0; i < 8; i++)
4772                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4773                 break;
4774         }
4775         i915_gem_detect_bit_6_swizzle(dev);
4776         init_waitqueue_head(&dev_priv->pending_flip_queue);
4777
4778         dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
4779         dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
4780         register_shrinker(&dev_priv->mm.inactive_shrinker);
4781 }
4782
4783 /*
4784  * Create a physically contiguous memory object for this object
4785  * e.g. for cursor + overlay regs
4786  */
4787 static int i915_gem_init_phys_object(struct drm_device *dev,
4788                                      int id, int size, int align)
4789 {
4790         drm_i915_private_t *dev_priv = dev->dev_private;
4791         struct drm_i915_gem_phys_object *phys_obj;
4792         int ret;
4793
4794         if (dev_priv->mm.phys_objs[id - 1] || !size)
4795                 return 0;
4796
4797         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4798         if (!phys_obj)
4799                 return -ENOMEM;
4800
4801         phys_obj->id = id;
4802
4803         phys_obj->handle = drm_pci_alloc(dev, size, align);
4804         if (!phys_obj->handle) {
4805                 ret = -ENOMEM;
4806                 goto kfree_obj;
4807         }
4808 #ifdef CONFIG_X86
4809         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4810 #endif
4811
4812         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4813
4814         return 0;
4815 kfree_obj:
4816         kfree(phys_obj);
4817         return ret;
4818 }
4819
4820 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4821 {
4822         drm_i915_private_t *dev_priv = dev->dev_private;
4823         struct drm_i915_gem_phys_object *phys_obj;
4824
4825         if (!dev_priv->mm.phys_objs[id - 1])
4826                 return;
4827
4828         phys_obj = dev_priv->mm.phys_objs[id - 1];
4829         if (phys_obj->cur_obj) {
4830                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4831         }
4832
4833 #ifdef CONFIG_X86
4834         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4835 #endif
4836         drm_pci_free(dev, phys_obj->handle);
4837         kfree(phys_obj);
4838         dev_priv->mm.phys_objs[id - 1] = NULL;
4839 }
4840
4841 void i915_gem_free_all_phys_object(struct drm_device *dev)
4842 {
4843         int i;
4844
4845         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4846                 i915_gem_free_phys_object(dev, i);
4847 }
4848
4849 void i915_gem_detach_phys_object(struct drm_device *dev,
4850                                  struct drm_gem_object *obj)
4851 {
4852         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
4853         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4854         char *vaddr;
4855         int i;
4856         int page_count;
4857
4858         if (!obj_priv->phys_obj)
4859                 return;
4860         vaddr = obj_priv->phys_obj->handle->vaddr;
4861
4862         page_count = obj->size / PAGE_SIZE;
4863
4864         for (i = 0; i < page_count; i++) {
4865                 struct page *page = read_cache_page_gfp(mapping, i,
4866                                                         GFP_HIGHUSER | __GFP_RECLAIMABLE);
4867                 if (!IS_ERR(page)) {
4868                         char *dst = kmap_atomic(page);
4869                         memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4870                         kunmap_atomic(dst);
4871
4872                         drm_clflush_pages(&page, 1);
4873
4874                         set_page_dirty(page);
4875                         mark_page_accessed(page);
4876                         page_cache_release(page);
4877                 }
4878         }
4879         drm_agp_chipset_flush(dev);
4880
4881         obj_priv->phys_obj->cur_obj = NULL;
4882         obj_priv->phys_obj = NULL;
4883 }
4884
4885 int
4886 i915_gem_attach_phys_object(struct drm_device *dev,
4887                             struct drm_gem_object *obj,
4888                             int id,
4889                             int align)
4890 {
4891         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
4892         drm_i915_private_t *dev_priv = dev->dev_private;
4893         struct drm_i915_gem_object *obj_priv;
4894         int ret = 0;
4895         int page_count;
4896         int i;
4897
4898         if (id > I915_MAX_PHYS_OBJECT)
4899                 return -EINVAL;
4900
4901         obj_priv = to_intel_bo(obj);
4902
4903         if (obj_priv->phys_obj) {
4904                 if (obj_priv->phys_obj->id == id)
4905                         return 0;
4906                 i915_gem_detach_phys_object(dev, obj);
4907         }
4908
4909         /* create a new object */
4910         if (!dev_priv->mm.phys_objs[id - 1]) {
4911                 ret = i915_gem_init_phys_object(dev, id,
4912                                                 obj->size, align);
4913                 if (ret) {
4914                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4915                         return ret;
4916                 }
4917         }
4918
4919         /* bind to the object */
4920         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4921         obj_priv->phys_obj->cur_obj = obj;
4922
4923         page_count = obj->size / PAGE_SIZE;
4924
4925         for (i = 0; i < page_count; i++) {
4926                 struct page *page;
4927                 char *dst, *src;
4928
4929                 page = read_cache_page_gfp(mapping, i,
4930                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
4931                 if (IS_ERR(page))
4932                         return PTR_ERR(page);
4933
4934                 src = kmap_atomic(obj_priv->pages[i]);
4935                 dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4936                 memcpy(dst, src, PAGE_SIZE);
4937                 kunmap_atomic(src);
4938
4939                 mark_page_accessed(page);
4940                 page_cache_release(page);
4941         }
4942
4943         return 0;
4944 }
4945
4946 static int
4947 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4948                      struct drm_i915_gem_pwrite *args,
4949                      struct drm_file *file_priv)
4950 {
4951         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4952         void *obj_addr;
4953         int ret;
4954         char __user *user_data;
4955
4956         user_data = (char __user *) (uintptr_t) args->data_ptr;
4957         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4958
4959         DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
4960         ret = copy_from_user(obj_addr, user_data, args->size);
4961         if (ret)
4962                 return -EFAULT;
4963
4964         drm_agp_chipset_flush(dev);
4965         return 0;
4966 }
4967
4968 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4969 {
4970         struct drm_i915_file_private *file_priv = file->driver_priv;
4971
4972         /* Clean up our request list when the client is going away, so that
4973          * later retire_requests won't dereference our soon-to-be-gone
4974          * file_priv.
4975          */
4976         spin_lock(&file_priv->mm.lock);
4977         while (!list_empty(&file_priv->mm.request_list)) {
4978                 struct drm_i915_gem_request *request;
4979
4980                 request = list_first_entry(&file_priv->mm.request_list,
4981                                            struct drm_i915_gem_request,
4982                                            client_list);
4983                 list_del(&request->client_list);
4984                 request->file_priv = NULL;
4985         }
4986         spin_unlock(&file_priv->mm.lock);
4987 }
4988
4989 static int
4990 i915_gpu_is_active(struct drm_device *dev)
4991 {
4992         drm_i915_private_t *dev_priv = dev->dev_private;
4993         int lists_empty;
4994
4995         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
4996                       list_empty(&dev_priv->mm.active_list);
4997
4998         return !lists_empty;
4999 }
5000
5001 static int
5002 i915_gem_inactive_shrink(struct shrinker *shrinker,
5003                          int nr_to_scan,
5004                          gfp_t gfp_mask)
5005 {
5006         struct drm_i915_private *dev_priv =
5007                 container_of(shrinker,
5008                              struct drm_i915_private,
5009                              mm.inactive_shrinker);
5010         struct drm_device *dev = dev_priv->dev;
5011         struct drm_i915_gem_object *obj, *next;
5012         int cnt;
5013
5014         if (!mutex_trylock(&dev->struct_mutex))
5015                 return 0;
5016
5017         /* "fast-path" to count number of available objects */
5018         if (nr_to_scan == 0) {
5019                 cnt = 0;
5020                 list_for_each_entry(obj,
5021                                     &dev_priv->mm.inactive_list,
5022                                     mm_list)
5023                         cnt++;
5024                 mutex_unlock(&dev->struct_mutex);
5025                 return cnt / 100 * sysctl_vfs_cache_pressure;
5026         }
5027
5028 rescan:
5029         /* first scan for clean buffers */
5030         i915_gem_retire_requests(dev);
5031
5032         list_for_each_entry_safe(obj, next,
5033                                  &dev_priv->mm.inactive_list,
5034                                  mm_list) {
5035                 if (i915_gem_object_is_purgeable(obj)) {
5036                         i915_gem_object_unbind(&obj->base);
5037                         if (--nr_to_scan == 0)
5038                                 break;
5039                 }
5040         }
5041
5042         /* second pass, evict/count anything still on the inactive list */
5043         cnt = 0;
5044         list_for_each_entry_safe(obj, next,
5045                                  &dev_priv->mm.inactive_list,
5046                                  mm_list) {
5047                 if (nr_to_scan) {
5048                         i915_gem_object_unbind(&obj->base);
5049                         nr_to_scan--;
5050                 } else
5051                         cnt++;
5052         }
5053
5054         if (nr_to_scan && i915_gpu_is_active(dev)) {
5055                 /*
5056                  * We are desperate for pages, so as a last resort, wait
5057                  * for the GPU to finish and discard whatever we can.
5058                  * This has a dramatic impact to reduce the number of
5059                  * OOM-killer events whilst running the GPU aggressively.
5060                  */
5061                 if (i915_gpu_idle(dev) == 0)
5062                         goto rescan;
5063         }
5064         mutex_unlock(&dev->struct_mutex);
5065         return cnt / 100 * sysctl_vfs_cache_pressure;
5066 }