drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include "drmP.h"
  31 #include "drm.h"
  32 #include "i915_drv.h"
  33 #include "i915_drm.h"
  34 #include "i915_trace.h"
  35 #include "intel_drv.h"
  36
  37 static inline int ring_space(struct intel_ring_buffer *ring)
  38 {
  39         int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
  40         if (space < 0)
  41                 space += ring->size;
  42         return space;
  43 }
  44
  45 static u32 i915_gem_get_seqno(struct drm_device *dev)
  46 {
  47         drm_i915_private_t *dev_priv = dev->dev_private;
  48         u32 seqno;
  49
  50         seqno = dev_priv->next_seqno;
  51
  52         /* reserve 0 for non-seqno */
  53         if (++dev_priv->next_seqno == 0)
  54                 dev_priv->next_seqno = 1;
  55
  56         return seqno;
  57 }
  58
  59 static int
  60 render_ring_flush(struct intel_ring_buffer *ring,
  61                   u32   invalidate_domains,
  62                   u32   flush_domains)
  63 {
  64         struct drm_device *dev = ring->dev;
  65         u32 cmd;
  66         int ret;
  67
  68         /*
  69          * read/write caches:
  70          *
  71          * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  72          * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  73          * also flushed at 2d versus 3d pipeline switches.
  74          *
  75          * read-only caches:
  76          *
  77          * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
  78          * MI_READ_FLUSH is set, and is always flushed on 965.
  79          *
  80          * I915_GEM_DOMAIN_COMMAND may not exist?
  81          *
  82          * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
  83          * invalidated when MI_EXE_FLUSH is set.
  84          *
  85          * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
  86          * invalidated with every MI_FLUSH.
  87          *
  88          * TLBs:
  89          *
  90          * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
  91          * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
  92          * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
  93          * are flushed at any MI_FLUSH.
  94          */
  95
  96         cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
  97         if ((invalidate_domains|flush_domains) &
  98             I915_GEM_DOMAIN_RENDER)
  99                 cmd &= ~MI_NO_WRITE_FLUSH;
 100         if (INTEL_INFO(dev)->gen < 4) {
 101                 /*
 102                  * On the 965, the sampler cache always gets flushed
 103                  * and this bit is reserved.
 104                  */
 105                 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
 106                         cmd |= MI_READ_FLUSH;
 107         }
 108         if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
 109                 cmd |= MI_EXE_FLUSH;
 110
 111         if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
 112             (IS_G4X(dev) || IS_GEN5(dev)))
 113                 cmd |= MI_INVALIDATE_ISP;
 114
 115         ret = intel_ring_begin(ring, 2);
 116         if (ret)
 117                 return ret;
 118
 119         intel_ring_emit(ring, cmd);
 120         intel_ring_emit(ring, MI_NOOP);
 121         intel_ring_advance(ring);
 122
 123         return 0;
 124 }
 125
 126 static void ring_write_tail(struct intel_ring_buffer *ring,
 127                             u32 value)
 128 {
 129         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 130         I915_WRITE_TAIL(ring, value);
 131 }
 132
 133 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
 134 {
 135         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 136         u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
 137                         RING_ACTHD(ring->mmio_base) : ACTHD;
 138
 139         return I915_READ(acthd_reg);
 140 }
 141
 142 static int init_ring_common(struct intel_ring_buffer *ring)
 143 {
 144         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 145         struct drm_i915_gem_object *obj = ring->obj;
 146         u32 head;
 147
 148         /* Stop the ring if it's running. */
 149         I915_WRITE_CTL(ring, 0);
 150         I915_WRITE_HEAD(ring, 0);
 151         ring->write_tail(ring, 0);
 152
 153         /* Initialize the ring. */
 154         I915_WRITE_START(ring, obj->gtt_offset);
 155         head = I915_READ_HEAD(ring) & HEAD_ADDR;
 156
 157         /* G45 ring initialization fails to reset head to zero */
 158         if (head != 0) {
 159                 DRM_DEBUG_KMS("%s head not reset to zero "
 160                               "ctl %08x head %08x tail %08x start %08x\n",
 161                               ring->name,
 162                               I915_READ_CTL(ring),
 163                               I915_READ_HEAD(ring),
 164                               I915_READ_TAIL(ring),
 165                               I915_READ_START(ring));
 166
 167                 I915_WRITE_HEAD(ring, 0);
 168
 169                 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
 170                         DRM_ERROR("failed to set %s head to zero "
 171                                   "ctl %08x head %08x tail %08x start %08x\n",
 172                                   ring->name,
 173                                   I915_READ_CTL(ring),
 174                                   I915_READ_HEAD(ring),
 175                                   I915_READ_TAIL(ring),
 176                                   I915_READ_START(ring));
 177                 }
 178         }
 179
 180         I915_WRITE_CTL(ring,
 181                         ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
 182                         | RING_REPORT_64K | RING_VALID);
 183
 184         /* If the head is still not zero, the ring is dead */
 185         if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
 186             I915_READ_START(ring) != obj->gtt_offset ||
 187             (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
 188                 DRM_ERROR("%s initialization failed "
 189                                 "ctl %08x head %08x tail %08x start %08x\n",
 190                                 ring->name,
 191                                 I915_READ_CTL(ring),
 192                                 I915_READ_HEAD(ring),
 193                                 I915_READ_TAIL(ring),
 194                                 I915_READ_START(ring));
 195                 return -EIO;
 196         }
 197
 198         if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
 199                 i915_kernel_lost_context(ring->dev);
 200         else {
 201                 ring->head = I915_READ_HEAD(ring);
 202                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
 203                 ring->space = ring_space(ring);
 204         }
 205
 206         return 0;
 207 }
 208
 209 /*
 210  * 965+ support PIPE_CONTROL commands, which provide finer grained control
 211  * over cache flushing.
 212  */
 213 struct pipe_control {
 214         struct drm_i915_gem_object *obj;
 215         volatile u32 *cpu_page;
 216         u32 gtt_offset;
 217 };
 218
 219 static int
 220 init_pipe_control(struct intel_ring_buffer *ring)
 221 {
 222         struct pipe_control *pc;
 223         struct drm_i915_gem_object *obj;
 224         int ret;
 225
 226         if (ring->private)
 227                 return 0;
 228
 229         pc = kmalloc(sizeof(*pc), GFP_KERNEL);
 230         if (!pc)
 231                 return -ENOMEM;
 232
 233         obj = i915_gem_alloc_object(ring->dev, 4096);
 234         if (obj == NULL) {
 235                 DRM_ERROR("Failed to allocate seqno page\n");
 236                 ret = -ENOMEM;
 237                 goto err;
 238         }
 239
 240         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 241
 242         ret = i915_gem_object_pin(obj, 4096, true);
 243         if (ret)
 244                 goto err_unref;
 245
 246         pc->gtt_offset = obj->gtt_offset;
 247         pc->cpu_page =  kmap(obj->pages[0]);
 248         if (pc->cpu_page == NULL)
 249                 goto err_unpin;
 250
 251         pc->obj = obj;
 252         ring->private = pc;
 253         return 0;
 254
 255 err_unpin:
 256         i915_gem_object_unpin(obj);
 257 err_unref:
 258         drm_gem_object_unreference(&obj->base);
 259 err:
 260         kfree(pc);
 261         return ret;
 262 }
 263
 264 static void
 265 cleanup_pipe_control(struct intel_ring_buffer *ring)
 266 {
 267         struct pipe_control *pc = ring->private;
 268         struct drm_i915_gem_object *obj;
 269
 270         if (!ring->private)
 271                 return;
 272
 273         obj = pc->obj;
 274         kunmap(obj->pages[0]);
 275         i915_gem_object_unpin(obj);
 276         drm_gem_object_unreference(&obj->base);
 277
 278         kfree(pc);
 279         ring->private = NULL;
 280 }
 281
 282 static int init_render_ring(struct intel_ring_buffer *ring)
 283 {
 284         struct drm_device *dev = ring->dev;
 285         struct drm_i915_private *dev_priv = dev->dev_private;
 286         int ret = init_ring_common(ring);
 287
 288         if (INTEL_INFO(dev)->gen > 3) {
 289                 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
 290                 if (IS_GEN6(dev) || IS_GEN7(dev))
 291                         mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
 292                 I915_WRITE(MI_MODE, mode);
 293                 if (IS_GEN7(dev))
 294                         I915_WRITE(GFX_MODE_GEN7,
 295                                    GFX_MODE_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
 296                                    GFX_MODE_ENABLE(GFX_REPLAY_MODE));
 297         }
 298
 299         if (INTEL_INFO(dev)->gen >= 6) {
 300         } else if (IS_GEN5(dev)) {
 301                 ret = init_pipe_control(ring);
 302                 if (ret)
 303                         return ret;
 304         }
 305
 306         return ret;
 307 }
 308
 309 static void render_ring_cleanup(struct intel_ring_buffer *ring)
 310 {
 311         if (!ring->private)
 312                 return;
 313
 314         cleanup_pipe_control(ring);
 315 }
 316
 317 static void
 318 update_mboxes(struct intel_ring_buffer *ring,
 319             u32 seqno,
 320             u32 mmio_offset)
 321 {
 322         intel_ring_emit(ring, MI_SEMAPHORE_MBOX |
 323                               MI_SEMAPHORE_GLOBAL_GTT |
 324                               MI_SEMAPHORE_REGISTER |
 325                               MI_SEMAPHORE_UPDATE);
 326         intel_ring_emit(ring, seqno);
 327         intel_ring_emit(ring, mmio_offset);
 328 }
 329
 330 /**
 331  * gen6_add_request - Update the semaphore mailbox registers
 332  *
 333  * @ring - ring that is adding a request
 334  * @seqno - return seqno stuck into the ring
 335  *
 336  * Update the mailbox registers in the *other* rings with the current seqno.
 337  * This acts like a signal in the canonical semaphore.
 338  */
 339 static int
 340 gen6_add_request(struct intel_ring_buffer *ring,
 341                  u32 *seqno)
 342 {
 343         u32 mbox1_reg;
 344         u32 mbox2_reg;
 345         int ret;
 346
 347         ret = intel_ring_begin(ring, 10);
 348         if (ret)
 349                 return ret;
 350
 351         mbox1_reg = ring->signal_mbox[0];
 352         mbox2_reg = ring->signal_mbox[1];
 353
 354         *seqno = i915_gem_get_seqno(ring->dev);
 355
 356         update_mboxes(ring, *seqno, mbox1_reg);
 357         update_mboxes(ring, *seqno, mbox2_reg);
 358         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 359         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 360         intel_ring_emit(ring, *seqno);
 361         intel_ring_emit(ring, MI_USER_INTERRUPT);
 362         intel_ring_advance(ring);
 363
 364         return 0;
 365 }
 366
 367 /**
 368  * intel_ring_sync - sync the waiter to the signaller on seqno
 369  *
 370  * @waiter - ring that is waiting
 371  * @signaller - ring which has, or will signal
 372  * @seqno - seqno which the waiter will block on
 373  */
 374 static int
 375 intel_ring_sync(struct intel_ring_buffer *waiter,
 376                 struct intel_ring_buffer *signaller,
 377                 int ring,
 378                 u32 seqno)
 379 {
 380         int ret;
 381         u32 dw1 = MI_SEMAPHORE_MBOX |
 382                   MI_SEMAPHORE_COMPARE |
 383                   MI_SEMAPHORE_REGISTER;
 384
 385         ret = intel_ring_begin(waiter, 4);
 386         if (ret)
 387                 return ret;
 388
 389         intel_ring_emit(waiter, dw1 | signaller->semaphore_register[ring]);
 390         intel_ring_emit(waiter, seqno);
 391         intel_ring_emit(waiter, 0);
 392         intel_ring_emit(waiter, MI_NOOP);
 393         intel_ring_advance(waiter);
 394
 395         return 0;
 396 }
 397
 398 /* VCS->RCS (RVSYNC) or BCS->RCS (RBSYNC) */
 399 int
 400 render_ring_sync_to(struct intel_ring_buffer *waiter,
 401                     struct intel_ring_buffer *signaller,
 402                     u32 seqno)
 403 {
 404         WARN_ON(signaller->semaphore_register[RCS] == MI_SEMAPHORE_SYNC_INVALID);
 405         return intel_ring_sync(waiter,
 406                                signaller,
 407                                RCS,
 408                                seqno);
 409 }
 410
 411 /* RCS->VCS (VRSYNC) or BCS->VCS (VBSYNC) */
 412 int
 413 gen6_bsd_ring_sync_to(struct intel_ring_buffer *waiter,
 414                       struct intel_ring_buffer *signaller,
 415                       u32 seqno)
 416 {
 417         WARN_ON(signaller->semaphore_register[VCS] == MI_SEMAPHORE_SYNC_INVALID);
 418         return intel_ring_sync(waiter,
 419                                signaller,
 420                                VCS,
 421                                seqno);
 422 }
 423
 424 /* RCS->BCS (BRSYNC) or VCS->BCS (BVSYNC) */
 425 int
 426 gen6_blt_ring_sync_to(struct intel_ring_buffer *waiter,
 427                       struct intel_ring_buffer *signaller,
 428                       u32 seqno)
 429 {
 430         WARN_ON(signaller->semaphore_register[BCS] == MI_SEMAPHORE_SYNC_INVALID);
 431         return intel_ring_sync(waiter,
 432                                signaller,
 433                                BCS,
 434                                seqno);
 435 }
 436
 437
 438
 439 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
 440 do {                                                                    \
 441         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |                \
 442                  PIPE_CONTROL_DEPTH_STALL);                             \
 443         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
 444         intel_ring_emit(ring__, 0);                                                     \
 445         intel_ring_emit(ring__, 0);                                                     \
 446 } while (0)
 447
 448 static int
 449 pc_render_add_request(struct intel_ring_buffer *ring,
 450                       u32 *result)
 451 {
 452         struct drm_device *dev = ring->dev;
 453         u32 seqno = i915_gem_get_seqno(dev);
 454         struct pipe_control *pc = ring->private;
 455         u32 scratch_addr = pc->gtt_offset + 128;
 456         int ret;
 457
 458         /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
 459          * incoherent with writes to memory, i.e. completely fubar,
 460          * so we need to use PIPE_NOTIFY instead.
 461          *
 462          * However, we also need to workaround the qword write
 463          * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
 464          * memory before requesting an interrupt.
 465          */
 466         ret = intel_ring_begin(ring, 32);
 467         if (ret)
 468                 return ret;
 469
 470         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
 471                         PIPE_CONTROL_WRITE_FLUSH |
 472                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
 473         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
 474         intel_ring_emit(ring, seqno);
 475         intel_ring_emit(ring, 0);
 476         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 477         scratch_addr += 128; /* write to separate cachelines */
 478         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 479         scratch_addr += 128;
 480         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 481         scratch_addr += 128;
 482         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 483         scratch_addr += 128;
 484         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 485         scratch_addr += 128;
 486         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 487         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
 488                         PIPE_CONTROL_WRITE_FLUSH |
 489                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
 490                         PIPE_CONTROL_NOTIFY);
 491         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
 492         intel_ring_emit(ring, seqno);
 493         intel_ring_emit(ring, 0);
 494         intel_ring_advance(ring);
 495
 496         *result = seqno;
 497         return 0;
 498 }
 499
 500 static int
 501 render_ring_add_request(struct intel_ring_buffer *ring,
 502                         u32 *result)
 503 {
 504         struct drm_device *dev = ring->dev;
 505         u32 seqno = i915_gem_get_seqno(dev);
 506         int ret;
 507
 508         ret = intel_ring_begin(ring, 4);
 509         if (ret)
 510                 return ret;
 511
 512         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 513         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 514         intel_ring_emit(ring, seqno);
 515         intel_ring_emit(ring, MI_USER_INTERRUPT);
 516         intel_ring_advance(ring);
 517
 518         *result = seqno;
 519         return 0;
 520 }
 521
 522 static u32
 523 ring_get_seqno(struct intel_ring_buffer *ring)
 524 {
 525         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 526 }
 527
 528 static u32
 529 pc_render_get_seqno(struct intel_ring_buffer *ring)
 530 {
 531         struct pipe_control *pc = ring->private;
 532         return pc->cpu_page[0];
 533 }
 534
 535 static void
 536 ironlake_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
 537 {
 538         dev_priv->gt_irq_mask &= ~mask;
 539         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 540         POSTING_READ(GTIMR);
 541 }
 542
 543 static void
 544 ironlake_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
 545 {
 546         dev_priv->gt_irq_mask |= mask;
 547         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 548         POSTING_READ(GTIMR);
 549 }
 550
 551 static void
 552 i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
 553 {
 554         dev_priv->irq_mask &= ~mask;
 555         I915_WRITE(IMR, dev_priv->irq_mask);
 556         POSTING_READ(IMR);
 557 }
 558
 559 static void
 560 i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
 561 {
 562         dev_priv->irq_mask |= mask;
 563         I915_WRITE(IMR, dev_priv->irq_mask);
 564         POSTING_READ(IMR);
 565 }
 566
 567 static bool
 568 render_ring_get_irq(struct intel_ring_buffer *ring)
 569 {
 570         struct drm_device *dev = ring->dev;
 571         drm_i915_private_t *dev_priv = dev->dev_private;
 572
 573         if (!dev->irq_enabled)
 574                 return false;
 575
 576         spin_lock(&ring->irq_lock);
 577         if (ring->irq_refcount++ == 0) {
 578                 if (HAS_PCH_SPLIT(dev))
 579                         ironlake_enable_irq(dev_priv,
 580                                             GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
 581                 else
 582                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
 583         }
 584         spin_unlock(&ring->irq_lock);
 585
 586         return true;
 587 }
 588
 589 static void
 590 render_ring_put_irq(struct intel_ring_buffer *ring)
 591 {
 592         struct drm_device *dev = ring->dev;
 593         drm_i915_private_t *dev_priv = dev->dev_private;
 594
 595         spin_lock(&ring->irq_lock);
 596         if (--ring->irq_refcount == 0) {
 597                 if (HAS_PCH_SPLIT(dev))
 598                         ironlake_disable_irq(dev_priv,
 599                                              GT_USER_INTERRUPT |
 600                                              GT_PIPE_NOTIFY);
 601                 else
 602                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
 603         }
 604         spin_unlock(&ring->irq_lock);
 605 }
 606
 607 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
 608 {
 609         struct drm_device *dev = ring->dev;
 610         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 611         u32 mmio = 0;
 612
 613         /* The ring status page addresses are no longer next to the rest of
 614          * the ring registers as of gen7.
 615          */
 616         if (IS_GEN7(dev)) {
 617                 switch (ring->id) {
 618                 case RING_RENDER:
 619                         mmio = RENDER_HWS_PGA_GEN7;
 620                         break;
 621                 case RING_BLT:
 622                         mmio = BLT_HWS_PGA_GEN7;
 623                         break;
 624                 case RING_BSD:
 625                         mmio = BSD_HWS_PGA_GEN7;
 626                         break;
 627                 }
 628         } else if (IS_GEN6(ring->dev)) {
 629                 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
 630         } else {
 631                 mmio = RING_HWS_PGA(ring->mmio_base);
 632         }
 633
 634         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
 635         POSTING_READ(mmio);
 636 }
 637
 638 static int
 639 bsd_ring_flush(struct intel_ring_buffer *ring,
 640                u32     invalidate_domains,
 641                u32     flush_domains)
 642 {
 643         int ret;
 644
 645         ret = intel_ring_begin(ring, 2);
 646         if (ret)
 647                 return ret;
 648
 649         intel_ring_emit(ring, MI_FLUSH);
 650         intel_ring_emit(ring, MI_NOOP);
 651         intel_ring_advance(ring);
 652         return 0;
 653 }
 654
 655 static int
 656 ring_add_request(struct intel_ring_buffer *ring,
 657                  u32 *result)
 658 {
 659         u32 seqno;
 660         int ret;
 661
 662         ret = intel_ring_begin(ring, 4);
 663         if (ret)
 664                 return ret;
 665
 666         seqno = i915_gem_get_seqno(ring->dev);
 667
 668         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 669         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 670         intel_ring_emit(ring, seqno);
 671         intel_ring_emit(ring, MI_USER_INTERRUPT);
 672         intel_ring_advance(ring);
 673
 674         *result = seqno;
 675         return 0;
 676 }
 677
 678 static bool
 679 gen6_ring_get_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
 680 {
 681         struct drm_device *dev = ring->dev;
 682         drm_i915_private_t *dev_priv = dev->dev_private;
 683
 684         if (!dev->irq_enabled)
 685                return false;
 686
 687         spin_lock(&ring->irq_lock);
 688         if (ring->irq_refcount++ == 0) {
 689                 ring->irq_mask &= ~rflag;
 690                 I915_WRITE_IMR(ring, ring->irq_mask);
 691                 ironlake_enable_irq(dev_priv, gflag);
 692         }
 693         spin_unlock(&ring->irq_lock);
 694
 695         return true;
 696 }
 697
 698 static void
 699 gen6_ring_put_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
 700 {
 701         struct drm_device *dev = ring->dev;
 702         drm_i915_private_t *dev_priv = dev->dev_private;
 703
 704         spin_lock(&ring->irq_lock);
 705         if (--ring->irq_refcount == 0) {
 706                 ring->irq_mask |= rflag;
 707                 I915_WRITE_IMR(ring, ring->irq_mask);
 708                 ironlake_disable_irq(dev_priv, gflag);
 709         }
 710         spin_unlock(&ring->irq_lock);
 711 }
 712
 713 static bool
 714 bsd_ring_get_irq(struct intel_ring_buffer *ring)
 715 {
 716         struct drm_device *dev = ring->dev;
 717         drm_i915_private_t *dev_priv = dev->dev_private;
 718
 719         if (!dev->irq_enabled)
 720                 return false;
 721
 722         spin_lock(&ring->irq_lock);
 723         if (ring->irq_refcount++ == 0) {
 724                 if (IS_G4X(dev))
 725                         i915_enable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
 726                 else
 727                         ironlake_enable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
 728         }
 729         spin_unlock(&ring->irq_lock);
 730
 731         return true;
 732 }
 733 static void
 734 bsd_ring_put_irq(struct intel_ring_buffer *ring)
 735 {
 736         struct drm_device *dev = ring->dev;
 737         drm_i915_private_t *dev_priv = dev->dev_private;
 738
 739         spin_lock(&ring->irq_lock);
 740         if (--ring->irq_refcount == 0) {
 741                 if (IS_G4X(dev))
 742                         i915_disable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
 743                 else
 744                         ironlake_disable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
 745         }
 746         spin_unlock(&ring->irq_lock);
 747 }
 748
 749 static int
 750 ring_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
 751 {
 752         int ret;
 753
 754         ret = intel_ring_begin(ring, 2);
 755         if (ret)
 756                 return ret;
 757
 758         intel_ring_emit(ring,
 759                         MI_BATCH_BUFFER_START | (2 << 6) |
 760                         MI_BATCH_NON_SECURE_I965);
 761         intel_ring_emit(ring, offset);
 762         intel_ring_advance(ring);
 763
 764         return 0;
 765 }
 766
 767 static int
 768 render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
 769                                 u32 offset, u32 len)
 770 {
 771         struct drm_device *dev = ring->dev;
 772         int ret;
 773
 774         if (IS_I830(dev) || IS_845G(dev)) {
 775                 ret = intel_ring_begin(ring, 4);
 776                 if (ret)
 777                         return ret;
 778
 779                 intel_ring_emit(ring, MI_BATCH_BUFFER);
 780                 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
 781                 intel_ring_emit(ring, offset + len - 8);
 782                 intel_ring_emit(ring, 0);
 783         } else {
 784                 ret = intel_ring_begin(ring, 2);
 785                 if (ret)
 786                         return ret;
 787
 788                 if (INTEL_INFO(dev)->gen >= 4) {
 789                         intel_ring_emit(ring,
 790                                         MI_BATCH_BUFFER_START | (2 << 6) |
 791                                         MI_BATCH_NON_SECURE_I965);
 792                         intel_ring_emit(ring, offset);
 793                 } else {
 794                         intel_ring_emit(ring,
 795                                         MI_BATCH_BUFFER_START | (2 << 6));
 796                         intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
 797                 }
 798         }
 799         intel_ring_advance(ring);
 800
 801         return 0;
 802 }
 803
 804 static void cleanup_status_page(struct intel_ring_buffer *ring)
 805 {
 806         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 807         struct drm_i915_gem_object *obj;
 808
 809         obj = ring->status_page.obj;
 810         if (obj == NULL)
 811                 return;
 812
 813         kunmap(obj->pages[0]);
 814         i915_gem_object_unpin(obj);
 815         drm_gem_object_unreference(&obj->base);
 816         ring->status_page.obj = NULL;
 817
 818         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 819 }
 820
 821 static int init_status_page(struct intel_ring_buffer *ring)
 822 {
 823         struct drm_device *dev = ring->dev;
 824         drm_i915_private_t *dev_priv = dev->dev_private;
 825         struct drm_i915_gem_object *obj;
 826         int ret;
 827
 828         obj = i915_gem_alloc_object(dev, 4096);
 829         if (obj == NULL) {
 830                 DRM_ERROR("Failed to allocate status page\n");
 831                 ret = -ENOMEM;
 832                 goto err;
 833         }
 834
 835         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 836
 837         ret = i915_gem_object_pin(obj, 4096, true);
 838         if (ret != 0) {
 839                 goto err_unref;
 840         }
 841
 842         ring->status_page.gfx_addr = obj->gtt_offset;
 843         ring->status_page.page_addr = kmap(obj->pages[0]);
 844         if (ring->status_page.page_addr == NULL) {
 845                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 846                 goto err_unpin;
 847         }
 848         ring->status_page.obj = obj;
 849         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
 850
 851         intel_ring_setup_status_page(ring);
 852         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
 853                         ring->name, ring->status_page.gfx_addr);
 854
 855         return 0;
 856
 857 err_unpin:
 858         i915_gem_object_unpin(obj);
 859 err_unref:
 860         drm_gem_object_unreference(&obj->base);
 861 err:
 862         return ret;
 863 }
 864
 865 int intel_init_ring_buffer(struct drm_device *dev,
 866                            struct intel_ring_buffer *ring)
 867 {
 868         struct drm_i915_gem_object *obj;
 869         int ret;
 870
 871         ring->dev = dev;
 872         INIT_LIST_HEAD(&ring->active_list);
 873         INIT_LIST_HEAD(&ring->request_list);
 874         INIT_LIST_HEAD(&ring->gpu_write_list);
 875
 876         init_waitqueue_head(&ring->irq_queue);
 877         spin_lock_init(&ring->irq_lock);
 878         ring->irq_mask = ~0;
 879
 880         if (I915_NEED_GFX_HWS(dev)) {
 881                 ret = init_status_page(ring);
 882                 if (ret)
 883                         return ret;
 884         }
 885
 886         obj = i915_gem_alloc_object(dev, ring->size);
 887         if (obj == NULL) {
 888                 DRM_ERROR("Failed to allocate ringbuffer\n");
 889                 ret = -ENOMEM;
 890                 goto err_hws;
 891         }
 892
 893         ring->obj = obj;
 894
 895         ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
 896         if (ret)
 897                 goto err_unref;
 898
 899         ring->map.size = ring->size;
 900         ring->map.offset = dev->agp->base + obj->gtt_offset;
 901         ring->map.type = 0;
 902         ring->map.flags = 0;
 903         ring->map.mtrr = 0;
 904
 905         drm_core_ioremap_wc(&ring->map, dev);
 906         if (ring->map.handle == NULL) {
 907                 DRM_ERROR("Failed to map ringbuffer.\n");
 908                 ret = -EINVAL;
 909                 goto err_unpin;
 910         }
 911
 912         ring->virtual_start = ring->map.handle;
 913         ret = ring->init(ring);
 914         if (ret)
 915                 goto err_unmap;
 916
 917         /* Workaround an erratum on the i830 which causes a hang if
 918          * the TAIL pointer points to within the last 2 cachelines
 919          * of the buffer.
 920          */
 921         ring->effective_size = ring->size;
 922         if (IS_I830(ring->dev))
 923                 ring->effective_size -= 128;
 924
 925         return 0;
 926
 927 err_unmap:
 928         drm_core_ioremapfree(&ring->map, dev);
 929 err_unpin:
 930         i915_gem_object_unpin(obj);
 931 err_unref:
 932         drm_gem_object_unreference(&obj->base);
 933         ring->obj = NULL;
 934 err_hws:
 935         cleanup_status_page(ring);
 936         return ret;
 937 }
 938
 939 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
 940 {
 941         struct drm_i915_private *dev_priv;
 942         int ret;
 943
 944         if (ring->obj == NULL)
 945                 return;
 946
 947         /* Disable the ring buffer. The ring must be idle at this point */
 948         dev_priv = ring->dev->dev_private;
 949         ret = intel_wait_ring_idle(ring);
 950         if (ret)
 951                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
 952                           ring->name, ret);
 953
 954         I915_WRITE_CTL(ring, 0);
 955
 956         drm_core_ioremapfree(&ring->map, ring->dev);
 957
 958         i915_gem_object_unpin(ring->obj);
 959         drm_gem_object_unreference(&ring->obj->base);
 960         ring->obj = NULL;
 961
 962         if (ring->cleanup)
 963                 ring->cleanup(ring);
 964
 965         cleanup_status_page(ring);
 966 }
 967
 968 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
 969 {
 970         unsigned int *virt;
 971         int rem = ring->size - ring->tail;
 972
 973         if (ring->space < rem) {
 974                 int ret = intel_wait_ring_buffer(ring, rem);
 975                 if (ret)
 976                         return ret;
 977         }
 978
 979         virt = (unsigned int *)(ring->virtual_start + ring->tail);
 980         rem /= 8;
 981         while (rem--) {
 982                 *virt++ = MI_NOOP;
 983                 *virt++ = MI_NOOP;
 984         }
 985
 986         ring->tail = 0;
 987         ring->space = ring_space(ring);
 988
 989         return 0;
 990 }
 991
 992 int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
 993 {
 994         struct drm_device *dev = ring->dev;
 995         struct drm_i915_private *dev_priv = dev->dev_private;
 996         unsigned long end;
 997         u32 head;
 998
 999         /* If the reported head position has wrapped or hasn't advanced,
1000          * fallback to the slow and accurate path.
1001          */
1002         head = intel_read_status_page(ring, 4);
1003         if (head > ring->head) {
1004                 ring->head = head;
1005                 ring->space = ring_space(ring);
1006                 if (ring->space >= n)
1007                         return 0;
1008         }
1009
1010         trace_i915_ring_wait_begin(ring);
1011         end = jiffies + 3 * HZ;
1012         do {
1013                 ring->head = I915_READ_HEAD(ring);
1014                 ring->space = ring_space(ring);
1015                 if (ring->space >= n) {
1016                         trace_i915_ring_wait_end(ring);
1017                         return 0;
1018                 }
1019
1020                 if (dev->primary->master) {
1021                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
1022                         if (master_priv->sarea_priv)
1023                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
1024                 }
1025
1026                 msleep(1);
1027                 if (atomic_read(&dev_priv->mm.wedged))
1028                         return -EAGAIN;
1029         } while (!time_after(jiffies, end));
1030         trace_i915_ring_wait_end(ring);
1031         return -EBUSY;
1032 }
1033
1034 int intel_ring_begin(struct intel_ring_buffer *ring,
1035                      int num_dwords)
1036 {
1037         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1038         int n = 4*num_dwords;
1039         int ret;
1040
1041         if (unlikely(atomic_read(&dev_priv->mm.wedged)))
1042                 return -EIO;
1043
1044         if (unlikely(ring->tail + n > ring->effective_size)) {
1045                 ret = intel_wrap_ring_buffer(ring);
1046                 if (unlikely(ret))
1047                         return ret;
1048         }
1049
1050         if (unlikely(ring->space < n)) {
1051                 ret = intel_wait_ring_buffer(ring, n);
1052                 if (unlikely(ret))
1053                         return ret;
1054         }
1055
1056         ring->space -= n;
1057         return 0;
1058 }
1059
1060 void intel_ring_advance(struct intel_ring_buffer *ring)
1061 {
1062         ring->tail &= ring->size - 1;
1063         ring->write_tail(ring, ring->tail);
1064 }
1065
1066 static const struct intel_ring_buffer render_ring = {
1067         .name                   = "render ring",
1068         .id                     = RING_RENDER,
1069         .mmio_base              = RENDER_RING_BASE,
1070         .size                   = 32 * PAGE_SIZE,
1071         .init                   = init_render_ring,
1072         .write_tail             = ring_write_tail,
1073         .flush                  = render_ring_flush,
1074         .add_request            = render_ring_add_request,
1075         .get_seqno              = ring_get_seqno,
1076         .irq_get                = render_ring_get_irq,
1077         .irq_put                = render_ring_put_irq,
1078         .dispatch_execbuffer    = render_ring_dispatch_execbuffer,
1079         .cleanup                = render_ring_cleanup,
1080         .sync_to                = render_ring_sync_to,
1081         .semaphore_register     = {MI_SEMAPHORE_SYNC_INVALID,
1082                                    MI_SEMAPHORE_SYNC_RV,
1083                                    MI_SEMAPHORE_SYNC_RB},
1084         .signal_mbox            = {GEN6_VRSYNC, GEN6_BRSYNC},
1085 };
1086
1087 /* ring buffer for bit-stream decoder */
1088
1089 static const struct intel_ring_buffer bsd_ring = {
1090         .name                   = "bsd ring",
1091         .id                     = RING_BSD,
1092         .mmio_base              = BSD_RING_BASE,
1093         .size                   = 32 * PAGE_SIZE,
1094         .init                   = init_ring_common,
1095         .write_tail             = ring_write_tail,
1096         .flush                  = bsd_ring_flush,
1097         .add_request            = ring_add_request,
1098         .get_seqno              = ring_get_seqno,
1099         .irq_get                = bsd_ring_get_irq,
1100         .irq_put                = bsd_ring_put_irq,
1101         .dispatch_execbuffer    = ring_dispatch_execbuffer,
1102 };
1103
1104
1105 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
1106                                      u32 value)
1107 {
1108         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1109
1110        /* Every tail move must follow the sequence below */
1111         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1112                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1113                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
1114         I915_WRITE(GEN6_BSD_RNCID, 0x0);
1115
1116         if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1117                 GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
1118                 50))
1119         DRM_ERROR("timed out waiting for IDLE Indicator\n");
1120
1121         I915_WRITE_TAIL(ring, value);
1122         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1123                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1124                 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
1125 }
1126
1127 static int gen6_ring_flush(struct intel_ring_buffer *ring,
1128                            u32 invalidate, u32 flush)
1129 {
1130         uint32_t cmd;
1131         int ret;
1132
1133         ret = intel_ring_begin(ring, 4);
1134         if (ret)
1135                 return ret;
1136
1137         cmd = MI_FLUSH_DW;
1138         if (invalidate & I915_GEM_GPU_DOMAINS)
1139                 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
1140         intel_ring_emit(ring, cmd);
1141         intel_ring_emit(ring, 0);
1142         intel_ring_emit(ring, 0);
1143         intel_ring_emit(ring, MI_NOOP);
1144         intel_ring_advance(ring);
1145         return 0;
1146 }
1147
1148 static int
1149 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1150                               u32 offset, u32 len)
1151 {
1152         int ret;
1153
1154         ret = intel_ring_begin(ring, 2);
1155         if (ret)
1156                 return ret;
1157
1158         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
1159         /* bit0-7 is the length on GEN6+ */
1160         intel_ring_emit(ring, offset);
1161         intel_ring_advance(ring);
1162
1163         return 0;
1164 }
1165
1166 static bool
1167 gen6_render_ring_get_irq(struct intel_ring_buffer *ring)
1168 {
1169         return gen6_ring_get_irq(ring,
1170                                  GT_USER_INTERRUPT,
1171                                  GEN6_RENDER_USER_INTERRUPT);
1172 }
1173
1174 static void
1175 gen6_render_ring_put_irq(struct intel_ring_buffer *ring)
1176 {
1177         return gen6_ring_put_irq(ring,
1178                                  GT_USER_INTERRUPT,
1179                                  GEN6_RENDER_USER_INTERRUPT);
1180 }
1181
1182 static bool
1183 gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
1184 {
1185         return gen6_ring_get_irq(ring,
1186                                  GT_GEN6_BSD_USER_INTERRUPT,
1187                                  GEN6_BSD_USER_INTERRUPT);
1188 }
1189
1190 static void
1191 gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
1192 {
1193         return gen6_ring_put_irq(ring,
1194                                  GT_GEN6_BSD_USER_INTERRUPT,
1195                                  GEN6_BSD_USER_INTERRUPT);
1196 }
1197
1198 /* ring buffer for Video Codec for Gen6+ */
1199 static const struct intel_ring_buffer gen6_bsd_ring = {
1200         .name                   = "gen6 bsd ring",
1201         .id                     = RING_BSD,
1202         .mmio_base              = GEN6_BSD_RING_BASE,
1203         .size                   = 32 * PAGE_SIZE,
1204         .init                   = init_ring_common,
1205         .write_tail             = gen6_bsd_ring_write_tail,
1206         .flush                  = gen6_ring_flush,
1207         .add_request            = gen6_add_request,
1208         .get_seqno              = ring_get_seqno,
1209         .irq_get                = gen6_bsd_ring_get_irq,
1210         .irq_put                = gen6_bsd_ring_put_irq,
1211         .dispatch_execbuffer    = gen6_ring_dispatch_execbuffer,
1212         .sync_to                = gen6_bsd_ring_sync_to,
1213         .semaphore_register     = {MI_SEMAPHORE_SYNC_VR,
1214                                    MI_SEMAPHORE_SYNC_INVALID,
1215                                    MI_SEMAPHORE_SYNC_VB},
1216         .signal_mbox            = {GEN6_RVSYNC, GEN6_BVSYNC},
1217 };
1218
1219 /* Blitter support (SandyBridge+) */
1220
1221 static bool
1222 blt_ring_get_irq(struct intel_ring_buffer *ring)
1223 {
1224         return gen6_ring_get_irq(ring,
1225                                  GT_BLT_USER_INTERRUPT,
1226                                  GEN6_BLITTER_USER_INTERRUPT);
1227 }
1228
1229 static void
1230 blt_ring_put_irq(struct intel_ring_buffer *ring)
1231 {
1232         gen6_ring_put_irq(ring,
1233                           GT_BLT_USER_INTERRUPT,
1234                           GEN6_BLITTER_USER_INTERRUPT);
1235 }
1236
1237
1238 /* Workaround for some stepping of SNB,
1239  * each time when BLT engine ring tail moved,
1240  * the first command in the ring to be parsed
1241  * should be MI_BATCH_BUFFER_START
1242  */
1243 #define NEED_BLT_WORKAROUND(dev) \
1244         (IS_GEN6(dev) && (dev->pdev->revision < 8))
1245
1246 static inline struct drm_i915_gem_object *
1247 to_blt_workaround(struct intel_ring_buffer *ring)
1248 {
1249         return ring->private;
1250 }
1251
1252 static int blt_ring_init(struct intel_ring_buffer *ring)
1253 {
1254         if (NEED_BLT_WORKAROUND(ring->dev)) {
1255                 struct drm_i915_gem_object *obj;
1256                 u32 *ptr;
1257                 int ret;
1258
1259                 obj = i915_gem_alloc_object(ring->dev, 4096);
1260                 if (obj == NULL)
1261                         return -ENOMEM;
1262
1263                 ret = i915_gem_object_pin(obj, 4096, true);
1264                 if (ret) {
1265                         drm_gem_object_unreference(&obj->base);
1266                         return ret;
1267                 }
1268
1269                 ptr = kmap(obj->pages[0]);
1270                 *ptr++ = MI_BATCH_BUFFER_END;
1271                 *ptr++ = MI_NOOP;
1272                 kunmap(obj->pages[0]);
1273
1274                 ret = i915_gem_object_set_to_gtt_domain(obj, false);
1275                 if (ret) {
1276                         i915_gem_object_unpin(obj);
1277                         drm_gem_object_unreference(&obj->base);
1278                         return ret;
1279                 }
1280
1281                 ring->private = obj;
1282         }
1283
1284         return init_ring_common(ring);
1285 }
1286
1287 static int blt_ring_begin(struct intel_ring_buffer *ring,
1288                           int num_dwords)
1289 {
1290         if (ring->private) {
1291                 int ret = intel_ring_begin(ring, num_dwords+2);
1292                 if (ret)
1293                         return ret;
1294
1295                 intel_ring_emit(ring, MI_BATCH_BUFFER_START);
1296                 intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
1297
1298                 return 0;
1299         } else
1300                 return intel_ring_begin(ring, 4);
1301 }
1302
1303 static int blt_ring_flush(struct intel_ring_buffer *ring,
1304                           u32 invalidate, u32 flush)
1305 {
1306         uint32_t cmd;
1307         int ret;
1308
1309         ret = blt_ring_begin(ring, 4);
1310         if (ret)
1311                 return ret;
1312
1313         cmd = MI_FLUSH_DW;
1314         if (invalidate & I915_GEM_DOMAIN_RENDER)
1315                 cmd |= MI_INVALIDATE_TLB;
1316         intel_ring_emit(ring, cmd);
1317         intel_ring_emit(ring, 0);
1318         intel_ring_emit(ring, 0);
1319         intel_ring_emit(ring, MI_NOOP);
1320         intel_ring_advance(ring);
1321         return 0;
1322 }
1323
1324 static void blt_ring_cleanup(struct intel_ring_buffer *ring)
1325 {
1326         if (!ring->private)
1327                 return;
1328
1329         i915_gem_object_unpin(ring->private);
1330         drm_gem_object_unreference(ring->private);
1331         ring->private = NULL;
1332 }
1333
1334 static const struct intel_ring_buffer gen6_blt_ring = {
1335         .name                   = "blt ring",
1336         .id                     = RING_BLT,
1337         .mmio_base              = BLT_RING_BASE,
1338         .size                   = 32 * PAGE_SIZE,
1339         .init                   = blt_ring_init,
1340         .write_tail             = ring_write_tail,
1341         .flush                  = blt_ring_flush,
1342         .add_request            = gen6_add_request,
1343         .get_seqno              = ring_get_seqno,
1344         .irq_get                = blt_ring_get_irq,
1345         .irq_put                = blt_ring_put_irq,
1346         .dispatch_execbuffer    = gen6_ring_dispatch_execbuffer,
1347         .cleanup                = blt_ring_cleanup,
1348         .sync_to                = gen6_blt_ring_sync_to,
1349         .semaphore_register     = {MI_SEMAPHORE_SYNC_BR,
1350                                    MI_SEMAPHORE_SYNC_BV,
1351                                    MI_SEMAPHORE_SYNC_INVALID},
1352         .signal_mbox            = {GEN6_RBSYNC, GEN6_VBSYNC},
1353 };
1354
1355 int intel_init_render_ring_buffer(struct drm_device *dev)
1356 {
1357         drm_i915_private_t *dev_priv = dev->dev_private;
1358         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1359
1360         *ring = render_ring;
1361         if (INTEL_INFO(dev)->gen >= 6) {
1362                 ring->add_request = gen6_add_request;
1363                 ring->irq_get = gen6_render_ring_get_irq;
1364                 ring->irq_put = gen6_render_ring_put_irq;
1365         } else if (IS_GEN5(dev)) {
1366                 ring->add_request = pc_render_add_request;
1367                 ring->get_seqno = pc_render_get_seqno;
1368         }
1369
1370         if (!I915_NEED_GFX_HWS(dev)) {
1371                 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1372                 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1373         }
1374
1375         return intel_init_ring_buffer(dev, ring);
1376 }
1377
1378 int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
1379 {
1380         drm_i915_private_t *dev_priv = dev->dev_private;
1381         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1382
1383         *ring = render_ring;
1384         if (INTEL_INFO(dev)->gen >= 6) {
1385                 ring->add_request = gen6_add_request;
1386                 ring->irq_get = gen6_render_ring_get_irq;
1387                 ring->irq_put = gen6_render_ring_put_irq;
1388         } else if (IS_GEN5(dev)) {
1389                 ring->add_request = pc_render_add_request;
1390                 ring->get_seqno = pc_render_get_seqno;
1391         }
1392
1393         if (!I915_NEED_GFX_HWS(dev))
1394                 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1395
1396         ring->dev = dev;
1397         INIT_LIST_HEAD(&ring->active_list);
1398         INIT_LIST_HEAD(&ring->request_list);
1399         INIT_LIST_HEAD(&ring->gpu_write_list);
1400
1401         ring->size = size;
1402         ring->effective_size = ring->size;
1403         if (IS_I830(ring->dev))
1404                 ring->effective_size -= 128;
1405
1406         ring->map.offset = start;
1407         ring->map.size = size;
1408         ring->map.type = 0;
1409         ring->map.flags = 0;
1410         ring->map.mtrr = 0;
1411
1412         drm_core_ioremap_wc(&ring->map, dev);
1413         if (ring->map.handle == NULL) {
1414                 DRM_ERROR("can not ioremap virtual address for"
1415                           " ring buffer\n");
1416                 return -ENOMEM;
1417         }
1418
1419         ring->virtual_start = (void __force __iomem *)ring->map.handle;
1420         return 0;
1421 }
1422
1423 int intel_init_bsd_ring_buffer(struct drm_device *dev)
1424 {
1425         drm_i915_private_t *dev_priv = dev->dev_private;
1426         struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
1427
1428         if (IS_GEN6(dev) || IS_GEN7(dev))
1429                 *ring = gen6_bsd_ring;
1430         else
1431                 *ring = bsd_ring;
1432
1433         return intel_init_ring_buffer(dev, ring);
1434 }
1435
1436 int intel_init_blt_ring_buffer(struct drm_device *dev)
1437 {
1438         drm_i915_private_t *dev_priv = dev->dev_private;
1439         struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
1440
1441         *ring = gen6_blt_ring;
1442
1443         return intel_init_ring_buffer(dev, ring);
1444 }