drivers/gpu/drm/i915/i915_irq.c

   1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
   2  */
   3 /*
   4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  30
  31 #include <linux/sysrq.h>
  32 #include <linux/slab.h>
  33 #include <linux/circ_buf.h>
  34 #include <drm/drmP.h>
  35 #include <drm/i915_drm.h>
  36 #include "i915_drv.h"
  37 #include "i915_trace.h"
  38 #include "intel_drv.h"
  39
  40 /**
  41  * DOC: interrupt handling
  42  *
  43  * These functions provide the basic support for enabling and disabling the
  44  * interrupt handling support. There's a lot more functionality in i915_irq.c
  45  * and related files, but that will be described in separate chapters.
  46  */
  47
  48 static const u32 hpd_ibx[HPD_NUM_PINS] = {
  49         [HPD_CRT] = SDE_CRT_HOTPLUG,
  50         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
  51         [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
  52         [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
  53         [HPD_PORT_D] = SDE_PORTD_HOTPLUG
  54 };
  55
  56 static const u32 hpd_cpt[HPD_NUM_PINS] = {
  57         [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
  58         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
  59         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
  60         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
  61         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
  62 };
  63
  64 static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
  65         [HPD_CRT] = CRT_HOTPLUG_INT_EN,
  66         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
  67         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
  68         [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
  69         [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
  70         [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
  71 };
  72
  73 static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
  74         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  75         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
  76         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
  77         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  78         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  79         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
  80 };
  81
  82 static const u32 hpd_status_i915[HPD_NUM_PINS] = {
  83         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  84         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
  85         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
  86         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  87         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  88         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
  89 };
  90
  91 /* BXT hpd list */
  92 static const u32 hpd_bxt[HPD_NUM_PINS] = {
  93         [HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
  94         [HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
  95 };
  96
  97 /* IIR can theoretically queue up two events. Be paranoid. */
  98 #define GEN8_IRQ_RESET_NDX(type, which) do { \
  99         I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
 100         POSTING_READ(GEN8_##type##_IMR(which)); \
 101         I915_WRITE(GEN8_##type##_IER(which), 0); \
 102         I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
 103         POSTING_READ(GEN8_##type##_IIR(which)); \
 104         I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
 105         POSTING_READ(GEN8_##type##_IIR(which)); \
 106 } while (0)
 107
 108 #define GEN5_IRQ_RESET(type) do { \
 109         I915_WRITE(type##IMR, 0xffffffff); \
 110         POSTING_READ(type##IMR); \
 111         I915_WRITE(type##IER, 0); \
 112         I915_WRITE(type##IIR, 0xffffffff); \
 113         POSTING_READ(type##IIR); \
 114         I915_WRITE(type##IIR, 0xffffffff); \
 115         POSTING_READ(type##IIR); \
 116 } while (0)
 117
 118 /*
 119  * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 120  */
 121 #define GEN5_ASSERT_IIR_IS_ZERO(reg) do { \
 122         u32 val = I915_READ(reg); \
 123         if (val) { \
 124                 WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", \
 125                      (reg), val); \
 126                 I915_WRITE((reg), 0xffffffff); \
 127                 POSTING_READ(reg); \
 128                 I915_WRITE((reg), 0xffffffff); \
 129                 POSTING_READ(reg); \
 130         } \
 131 } while (0)
 132
 133 #define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
 134         GEN5_ASSERT_IIR_IS_ZERO(GEN8_##type##_IIR(which)); \
 135         I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
 136         I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
 137         POSTING_READ(GEN8_##type##_IMR(which)); \
 138 } while (0)
 139
 140 #define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
 141         GEN5_ASSERT_IIR_IS_ZERO(type##IIR); \
 142         I915_WRITE(type##IER, (ier_val)); \
 143         I915_WRITE(type##IMR, (imr_val)); \
 144         POSTING_READ(type##IMR); \
 145 } while (0)
 146
 147 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
 148
 149 /* For display hotplug interrupt */
 150 void
 151 ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
 152 {
 153         assert_spin_locked(&dev_priv->irq_lock);
 154
 155         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 156                 return;
 157
 158         if ((dev_priv->irq_mask & mask) != 0) {
 159                 dev_priv->irq_mask &= ~mask;
 160                 I915_WRITE(DEIMR, dev_priv->irq_mask);
 161                 POSTING_READ(DEIMR);
 162         }
 163 }
 164
 165 void
 166 ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
 167 {
 168         assert_spin_locked(&dev_priv->irq_lock);
 169
 170         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 171                 return;
 172
 173         if ((dev_priv->irq_mask & mask) != mask) {
 174                 dev_priv->irq_mask |= mask;
 175                 I915_WRITE(DEIMR, dev_priv->irq_mask);
 176                 POSTING_READ(DEIMR);
 177         }
 178 }
 179
 180 /**
 181  * ilk_update_gt_irq - update GTIMR
 182  * @dev_priv: driver private
 183  * @interrupt_mask: mask of interrupt bits to update
 184  * @enabled_irq_mask: mask of interrupt bits to enable
 185  */
 186 static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
 187                               uint32_t interrupt_mask,
 188                               uint32_t enabled_irq_mask)
 189 {
 190         assert_spin_locked(&dev_priv->irq_lock);
 191
 192         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 193
 194         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 195                 return;
 196
 197         dev_priv->gt_irq_mask &= ~interrupt_mask;
 198         dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
 199         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 200         POSTING_READ(GTIMR);
 201 }
 202
 203 void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 204 {
 205         ilk_update_gt_irq(dev_priv, mask, mask);
 206 }
 207
 208 void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 209 {
 210         ilk_update_gt_irq(dev_priv, mask, 0);
 211 }
 212
 213 static u32 gen6_pm_iir(struct drm_i915_private *dev_priv)
 214 {
 215         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
 216 }
 217
 218 static u32 gen6_pm_imr(struct drm_i915_private *dev_priv)
 219 {
 220         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
 221 }
 222
 223 static u32 gen6_pm_ier(struct drm_i915_private *dev_priv)
 224 {
 225         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
 226 }
 227
 228 /**
 229   * snb_update_pm_irq - update GEN6_PMIMR
 230   * @dev_priv: driver private
 231   * @interrupt_mask: mask of interrupt bits to update
 232   * @enabled_irq_mask: mask of interrupt bits to enable
 233   */
 234 static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
 235                               uint32_t interrupt_mask,
 236                               uint32_t enabled_irq_mask)
 237 {
 238         uint32_t new_val;
 239
 240         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 241
 242         assert_spin_locked(&dev_priv->irq_lock);
 243
 244         new_val = dev_priv->pm_irq_mask;
 245         new_val &= ~interrupt_mask;
 246         new_val |= (~enabled_irq_mask & interrupt_mask);
 247
 248         if (new_val != dev_priv->pm_irq_mask) {
 249                 dev_priv->pm_irq_mask = new_val;
 250                 I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
 251                 POSTING_READ(gen6_pm_imr(dev_priv));
 252         }
 253 }
 254
 255 void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 256 {
 257         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 258                 return;
 259
 260         snb_update_pm_irq(dev_priv, mask, mask);
 261 }
 262
 263 static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
 264                                   uint32_t mask)
 265 {
 266         snb_update_pm_irq(dev_priv, mask, 0);
 267 }
 268
 269 void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 270 {
 271         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 272                 return;
 273
 274         __gen6_disable_pm_irq(dev_priv, mask);
 275 }
 276
 277 void gen6_reset_rps_interrupts(struct drm_device *dev)
 278 {
 279         struct drm_i915_private *dev_priv = dev->dev_private;
 280         uint32_t reg = gen6_pm_iir(dev_priv);
 281
 282         spin_lock_irq(&dev_priv->irq_lock);
 283         I915_WRITE(reg, dev_priv->pm_rps_events);
 284         I915_WRITE(reg, dev_priv->pm_rps_events);
 285         POSTING_READ(reg);
 286         dev_priv->rps.pm_iir = 0;
 287         spin_unlock_irq(&dev_priv->irq_lock);
 288 }
 289
 290 void gen6_enable_rps_interrupts(struct drm_device *dev)
 291 {
 292         struct drm_i915_private *dev_priv = dev->dev_private;
 293
 294         spin_lock_irq(&dev_priv->irq_lock);
 295
 296         WARN_ON(dev_priv->rps.pm_iir);
 297         WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
 298         dev_priv->rps.interrupts_enabled = true;
 299         I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) |
 300                                 dev_priv->pm_rps_events);
 301         gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 302
 303         spin_unlock_irq(&dev_priv->irq_lock);
 304 }
 305
 306 u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
 307 {
 308         /*
 309          * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
 310          * if GEN6_PM_UP_EI_EXPIRED is masked.
 311          *
 312          * TODO: verify if this can be reproduced on VLV,CHV.
 313          */
 314         if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
 315                 mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;
 316
 317         if (INTEL_INFO(dev_priv)->gen >= 8)
 318                 mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;
 319
 320         return mask;
 321 }
 322
 323 void gen6_disable_rps_interrupts(struct drm_device *dev)
 324 {
 325         struct drm_i915_private *dev_priv = dev->dev_private;
 326
 327         spin_lock_irq(&dev_priv->irq_lock);
 328         dev_priv->rps.interrupts_enabled = false;
 329         spin_unlock_irq(&dev_priv->irq_lock);
 330
 331         cancel_work_sync(&dev_priv->rps.work);
 332
 333         spin_lock_irq(&dev_priv->irq_lock);
 334
 335         I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
 336
 337         __gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 338         I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
 339                                 ~dev_priv->pm_rps_events);
 340
 341         spin_unlock_irq(&dev_priv->irq_lock);
 342
 343         synchronize_irq(dev->irq);
 344 }
 345
 346 /**
 347  * ibx_display_interrupt_update - update SDEIMR
 348  * @dev_priv: driver private
 349  * @interrupt_mask: mask of interrupt bits to update
 350  * @enabled_irq_mask: mask of interrupt bits to enable
 351  */
 352 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
 353                                   uint32_t interrupt_mask,
 354                                   uint32_t enabled_irq_mask)
 355 {
 356         uint32_t sdeimr = I915_READ(SDEIMR);
 357         sdeimr &= ~interrupt_mask;
 358         sdeimr |= (~enabled_irq_mask & interrupt_mask);
 359
 360         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 361
 362         assert_spin_locked(&dev_priv->irq_lock);
 363
 364         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 365                 return;
 366
 367         I915_WRITE(SDEIMR, sdeimr);
 368         POSTING_READ(SDEIMR);
 369 }
 370
 371 static void
 372 __i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 373                        u32 enable_mask, u32 status_mask)
 374 {
 375         u32 reg = PIPESTAT(pipe);
 376         u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
 377
 378         assert_spin_locked(&dev_priv->irq_lock);
 379         WARN_ON(!intel_irqs_enabled(dev_priv));
 380
 381         if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
 382                       status_mask & ~PIPESTAT_INT_STATUS_MASK,
 383                       "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
 384                       pipe_name(pipe), enable_mask, status_mask))
 385                 return;
 386
 387         if ((pipestat & enable_mask) == enable_mask)
 388                 return;
 389
 390         dev_priv->pipestat_irq_mask[pipe] |= status_mask;
 391
 392         /* Enable the interrupt, clear any pending status */
 393         pipestat |= enable_mask | status_mask;
 394         I915_WRITE(reg, pipestat);
 395         POSTING_READ(reg);
 396 }
 397
 398 static void
 399 __i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 400                         u32 enable_mask, u32 status_mask)
 401 {
 402         u32 reg = PIPESTAT(pipe);
 403         u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
 404
 405         assert_spin_locked(&dev_priv->irq_lock);
 406         WARN_ON(!intel_irqs_enabled(dev_priv));
 407
 408         if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
 409                       status_mask & ~PIPESTAT_INT_STATUS_MASK,
 410                       "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
 411                       pipe_name(pipe), enable_mask, status_mask))
 412                 return;
 413
 414         if ((pipestat & enable_mask) == 0)
 415                 return;
 416
 417         dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
 418
 419         pipestat &= ~enable_mask;
 420         I915_WRITE(reg, pipestat);
 421         POSTING_READ(reg);
 422 }
 423
 424 static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
 425 {
 426         u32 enable_mask = status_mask << 16;
 427
 428         /*
 429          * On pipe A we don't support the PSR interrupt yet,
 430          * on pipe B and C the same bit MBZ.
 431          */
 432         if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
 433                 return 0;
 434         /*
 435          * On pipe B and C we don't support the PSR interrupt yet, on pipe
 436          * A the same bit is for perf counters which we don't use either.
 437          */
 438         if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
 439                 return 0;
 440
 441         enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
 442                          SPRITE0_FLIP_DONE_INT_EN_VLV |
 443                          SPRITE1_FLIP_DONE_INT_EN_VLV);
 444         if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
 445                 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
 446         if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
 447                 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
 448
 449         return enable_mask;
 450 }
 451
 452 void
 453 i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 454                      u32 status_mask)
 455 {
 456         u32 enable_mask;
 457
 458         if (IS_VALLEYVIEW(dev_priv->dev))
 459                 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
 460                                                            status_mask);
 461         else
 462                 enable_mask = status_mask << 16;
 463         __i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
 464 }
 465
 466 void
 467 i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 468                       u32 status_mask)
 469 {
 470         u32 enable_mask;
 471
 472         if (IS_VALLEYVIEW(dev_priv->dev))
 473                 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
 474                                                            status_mask);
 475         else
 476                 enable_mask = status_mask << 16;
 477         __i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
 478 }
 479
 480 /**
 481  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
 482  */
 483 static void i915_enable_asle_pipestat(struct drm_device *dev)
 484 {
 485         struct drm_i915_private *dev_priv = dev->dev_private;
 486
 487         if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
 488                 return;
 489
 490         spin_lock_irq(&dev_priv->irq_lock);
 491
 492         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
 493         if (INTEL_INFO(dev)->gen >= 4)
 494                 i915_enable_pipestat(dev_priv, PIPE_A,
 495                                      PIPE_LEGACY_BLC_EVENT_STATUS);
 496
 497         spin_unlock_irq(&dev_priv->irq_lock);
 498 }
 499
 500 /*
 501  * This timing diagram depicts the video signal in and
 502  * around the vertical blanking period.
 503  *
 504  * Assumptions about the fictitious mode used in this example:
 505  *  vblank_start >= 3
 506  *  vsync_start = vblank_start + 1
 507  *  vsync_end = vblank_start + 2
 508  *  vtotal = vblank_start + 3
 509  *
 510  *           start of vblank:
 511  *           latch double buffered registers
 512  *           increment frame counter (ctg+)
 513  *           generate start of vblank interrupt (gen4+)
 514  *           |
 515  *           |          frame start:
 516  *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 517  *           |          may be shifted forward 1-3 extra lines via PIPECONF
 518  *           |          |
 519  *           |          |  start of vsync:
 520  *           |          |  generate vsync interrupt
 521  *           |          |  |
 522  * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 523  *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 524  * ----va---> <-----------------vb--------------------> <--------va-------------
 525  *       |          |       <----vs----->                     |
 526  * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 527  * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 528  * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 529  *       |          |                                         |
 530  *       last visible pixel                                   first visible pixel
 531  *                  |                                         increment frame counter (gen3/4)
 532  *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 533  *
 534  * x  = horizontal active
 535  * _  = horizontal blanking
 536  * hs = horizontal sync
 537  * va = vertical active
 538  * vb = vertical blanking
 539  * vs = vertical sync
 540  * vbs = vblank_start (number)
 541  *
 542  * Summary:
 543  * - most events happen at the start of horizontal sync
 544  * - frame start happens at the start of horizontal blank, 1-4 lines
 545  *   (depending on PIPECONF settings) after the start of vblank
 546  * - gen3/4 pixel and frame counter are synchronized with the start
 547  *   of horizontal active on the first line of vertical active
 548  */
 549
 550 static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe)
 551 {
 552         /* Gen2 doesn't have a hardware frame counter */
 553         return 0;
 554 }
 555
 556 /* Called from drm generic code, passed a 'crtc', which
 557  * we use as a pipe index
 558  */
 559 static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
 560 {
 561         struct drm_i915_private *dev_priv = dev->dev_private;
 562         unsigned long high_frame;
 563         unsigned long low_frame;
 564         u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
 565         struct intel_crtc *intel_crtc =
 566                 to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
 567         const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
 568
 569         htotal = mode->crtc_htotal;
 570         hsync_start = mode->crtc_hsync_start;
 571         vbl_start = mode->crtc_vblank_start;
 572         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 573                 vbl_start = DIV_ROUND_UP(vbl_start, 2);
 574
 575         /* Convert to pixel count */
 576         vbl_start *= htotal;
 577
 578         /* Start of vblank event occurs at start of hsync */
 579         vbl_start -= htotal - hsync_start;
 580
 581         high_frame = PIPEFRAME(pipe);
 582         low_frame = PIPEFRAMEPIXEL(pipe);
 583
 584         /*
 585          * High & low register fields aren't synchronized, so make sure
 586          * we get a low value that's stable across two reads of the high
 587          * register.
 588          */
 589         do {
 590                 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 591                 low   = I915_READ(low_frame);
 592                 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 593         } while (high1 != high2);
 594
 595         high1 >>= PIPE_FRAME_HIGH_SHIFT;
 596         pixel = low & PIPE_PIXEL_MASK;
 597         low >>= PIPE_FRAME_LOW_SHIFT;
 598
 599         /*
 600          * The frame counter increments at beginning of active.
 601          * Cook up a vblank counter by also checking the pixel
 602          * counter against vblank start.
 603          */
 604         return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
 605 }
 606
 607 static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
 608 {
 609         struct drm_i915_private *dev_priv = dev->dev_private;
 610         int reg = PIPE_FRMCOUNT_GM45(pipe);
 611
 612         return I915_READ(reg);
 613 }
 614
 615 /* raw reads, only for fast reads of display block, no need for forcewake etc. */
 616 #define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
 617
 618 static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
 619 {
 620         struct drm_device *dev = crtc->base.dev;
 621         struct drm_i915_private *dev_priv = dev->dev_private;
 622         const struct drm_display_mode *mode = &crtc->base.hwmode;
 623         enum pipe pipe = crtc->pipe;
 624         int position, vtotal;
 625
 626         vtotal = mode->crtc_vtotal;
 627         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 628                 vtotal /= 2;
 629
 630         if (IS_GEN2(dev))
 631                 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
 632         else
 633                 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
 634
 635         /*
 636          * See update_scanline_offset() for the details on the
 637          * scanline_offset adjustment.
 638          */
 639         return (position + crtc->scanline_offset) % vtotal;
 640 }
 641
 642 static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
 643                                     unsigned int flags, int *vpos, int *hpos,
 644                                     ktime_t *stime, ktime_t *etime)
 645 {
 646         struct drm_i915_private *dev_priv = dev->dev_private;
 647         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
 648         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 649         const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
 650         int position;
 651         int vbl_start, vbl_end, hsync_start, htotal, vtotal;
 652         bool in_vbl = true;
 653         int ret = 0;
 654         unsigned long irqflags;
 655
 656         if (WARN_ON(!mode->crtc_clock)) {
 657                 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
 658                                  "pipe %c\n", pipe_name(pipe));
 659                 return 0;
 660         }
 661
 662         htotal = mode->crtc_htotal;
 663         hsync_start = mode->crtc_hsync_start;
 664         vtotal = mode->crtc_vtotal;
 665         vbl_start = mode->crtc_vblank_start;
 666         vbl_end = mode->crtc_vblank_end;
 667
 668         if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
 669                 vbl_start = DIV_ROUND_UP(vbl_start, 2);
 670                 vbl_end /= 2;
 671                 vtotal /= 2;
 672         }
 673
 674         ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
 675
 676         /*
 677          * Lock uncore.lock, as we will do multiple timing critical raw
 678          * register reads, potentially with preemption disabled, so the
 679          * following code must not block on uncore.lock.
 680          */
 681         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 682
 683         /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
 684
 685         /* Get optional system timestamp before query. */
 686         if (stime)
 687                 *stime = ktime_get();
 688
 689         if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
 690                 /* No obvious pixelcount register. Only query vertical
 691                  * scanout position from Display scan line register.
 692                  */
 693                 position = __intel_get_crtc_scanline(intel_crtc);
 694         } else {
 695                 /* Have access to pixelcount since start of frame.
 696                  * We can split this into vertical and horizontal
 697                  * scanout position.
 698                  */
 699                 position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
 700
 701                 /* convert to pixel counts */
 702                 vbl_start *= htotal;
 703                 vbl_end *= htotal;
 704                 vtotal *= htotal;
 705
 706                 /*
 707                  * In interlaced modes, the pixel counter counts all pixels,
 708                  * so one field will have htotal more pixels. In order to avoid
 709                  * the reported position from jumping backwards when the pixel
 710                  * counter is beyond the length of the shorter field, just
 711                  * clamp the position the length of the shorter field. This
 712                  * matches how the scanline counter based position works since
 713                  * the scanline counter doesn't count the two half lines.
 714                  */
 715                 if (position >= vtotal)
 716                         position = vtotal - 1;
 717
 718                 /*
 719                  * Start of vblank interrupt is triggered at start of hsync,
 720                  * just prior to the first active line of vblank. However we
 721                  * consider lines to start at the leading edge of horizontal
 722                  * active. So, should we get here before we've crossed into
 723                  * the horizontal active of the first line in vblank, we would
 724                  * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
 725                  * always add htotal-hsync_start to the current pixel position.
 726                  */
 727                 position = (position + htotal - hsync_start) % vtotal;
 728         }
 729
 730         /* Get optional system timestamp after query. */
 731         if (etime)
 732                 *etime = ktime_get();
 733
 734         /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
 735
 736         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 737
 738         in_vbl = position >= vbl_start && position < vbl_end;
 739
 740         /*
 741          * While in vblank, position will be negative
 742          * counting up towards 0 at vbl_end. And outside
 743          * vblank, position will be positive counting
 744          * up since vbl_end.
 745          */
 746         if (position >= vbl_start)
 747                 position -= vbl_end;
 748         else
 749                 position += vtotal - vbl_end;
 750
 751         if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
 752                 *vpos = position;
 753                 *hpos = 0;
 754         } else {
 755                 *vpos = position / htotal;
 756                 *hpos = position - (*vpos * htotal);
 757         }
 758
 759         /* In vblank? */
 760         if (in_vbl)
 761                 ret |= DRM_SCANOUTPOS_IN_VBLANK;
 762
 763         return ret;
 764 }
 765
 766 int intel_get_crtc_scanline(struct intel_crtc *crtc)
 767 {
 768         struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
 769         unsigned long irqflags;
 770         int position;
 771
 772         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 773         position = __intel_get_crtc_scanline(crtc);
 774         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 775
 776         return position;
 777 }
 778
 779 static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
 780                               int *max_error,
 781                               struct timeval *vblank_time,
 782                               unsigned flags)
 783 {
 784         struct drm_crtc *crtc;
 785
 786         if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
 787                 DRM_ERROR("Invalid crtc %d\n", pipe);
 788                 return -EINVAL;
 789         }
 790
 791         /* Get drm_crtc to timestamp: */
 792         crtc = intel_get_crtc_for_pipe(dev, pipe);
 793         if (crtc == NULL) {
 794                 DRM_ERROR("Invalid crtc %d\n", pipe);
 795                 return -EINVAL;
 796         }
 797
 798         if (!crtc->hwmode.crtc_clock) {
 799                 DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
 800                 return -EBUSY;
 801         }
 802
 803         /* Helper routine in DRM core does all the work: */
 804         return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
 805                                                      vblank_time, flags,
 806                                                      crtc,
 807                                                      &crtc->hwmode);
 808 }
 809
 810 static void ironlake_rps_change_irq_handler(struct drm_device *dev)
 811 {
 812         struct drm_i915_private *dev_priv = dev->dev_private;
 813         u32 busy_up, busy_down, max_avg, min_avg;
 814         u8 new_delay;
 815
 816         spin_lock(&mchdev_lock);
 817
 818         I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
 819
 820         new_delay = dev_priv->ips.cur_delay;
 821
 822         I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
 823         busy_up = I915_READ(RCPREVBSYTUPAVG);
 824         busy_down = I915_READ(RCPREVBSYTDNAVG);
 825         max_avg = I915_READ(RCBMAXAVG);
 826         min_avg = I915_READ(RCBMINAVG);
 827
 828         /* Handle RCS change request from hw */
 829         if (busy_up > max_avg) {
 830                 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
 831                         new_delay = dev_priv->ips.cur_delay - 1;
 832                 if (new_delay < dev_priv->ips.max_delay)
 833                         new_delay = dev_priv->ips.max_delay;
 834         } else if (busy_down < min_avg) {
 835                 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
 836                         new_delay = dev_priv->ips.cur_delay + 1;
 837                 if (new_delay > dev_priv->ips.min_delay)
 838                         new_delay = dev_priv->ips.min_delay;
 839         }
 840
 841         if (ironlake_set_drps(dev, new_delay))
 842                 dev_priv->ips.cur_delay = new_delay;
 843
 844         spin_unlock(&mchdev_lock);
 845
 846         return;
 847 }
 848
 849 static void notify_ring(struct intel_engine_cs *ring)
 850 {
 851         if (!intel_ring_initialized(ring))
 852                 return;
 853
 854         trace_i915_gem_request_notify(ring);
 855
 856         wake_up_all(&ring->irq_queue);
 857 }
 858
 859 static void vlv_c0_read(struct drm_i915_private *dev_priv,
 860                         struct intel_rps_ei *ei)
 861 {
 862         ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
 863         ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
 864         ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
 865 }
 866
 867 static bool vlv_c0_above(struct drm_i915_private *dev_priv,
 868                          const struct intel_rps_ei *old,
 869                          const struct intel_rps_ei *now,
 870                          int threshold)
 871 {
 872         u64 time, c0;
 873
 874         if (old->cz_clock == 0)
 875                 return false;
 876
 877         time = now->cz_clock - old->cz_clock;
 878         time *= threshold * dev_priv->mem_freq;
 879
 880         /* Workload can be split between render + media, e.g. SwapBuffers
 881          * being blitted in X after being rendered in mesa. To account for
 882          * this we need to combine both engines into our activity counter.
 883          */
 884         c0 = now->render_c0 - old->render_c0;
 885         c0 += now->media_c0 - old->media_c0;
 886         c0 *= 100 * VLV_CZ_CLOCK_TO_MILLI_SEC * 4 / 1000;
 887
 888         return c0 >= time;
 889 }
 890
 891 void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
 892 {
 893         vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
 894         dev_priv->rps.up_ei = dev_priv->rps.down_ei;
 895 }
 896
 897 static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
 898 {
 899         struct intel_rps_ei now;
 900         u32 events = 0;
 901
 902         if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
 903                 return 0;
 904
 905         vlv_c0_read(dev_priv, &now);
 906         if (now.cz_clock == 0)
 907                 return 0;
 908
 909         if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
 910                 if (!vlv_c0_above(dev_priv,
 911                                   &dev_priv->rps.down_ei, &now,
 912                                   dev_priv->rps.down_threshold))
 913                         events |= GEN6_PM_RP_DOWN_THRESHOLD;
 914                 dev_priv->rps.down_ei = now;
 915         }
 916
 917         if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
 918                 if (vlv_c0_above(dev_priv,
 919                                  &dev_priv->rps.up_ei, &now,
 920                                  dev_priv->rps.up_threshold))
 921                         events |= GEN6_PM_RP_UP_THRESHOLD;
 922                 dev_priv->rps.up_ei = now;
 923         }
 924
 925         return events;
 926 }
 927
 928 static bool any_waiters(struct drm_i915_private *dev_priv)
 929 {
 930         struct intel_engine_cs *ring;
 931         int i;
 932
 933         for_each_ring(ring, dev_priv, i)
 934                 if (ring->irq_refcount)
 935                         return true;
 936
 937         return false;
 938 }
 939
 940 static void gen6_pm_rps_work(struct work_struct *work)
 941 {
 942         struct drm_i915_private *dev_priv =
 943                 container_of(work, struct drm_i915_private, rps.work);
 944         bool client_boost;
 945         int new_delay, adj, min, max;
 946         u32 pm_iir;
 947
 948         spin_lock_irq(&dev_priv->irq_lock);
 949         /* Speed up work cancelation during disabling rps interrupts. */
 950         if (!dev_priv->rps.interrupts_enabled) {
 951                 spin_unlock_irq(&dev_priv->irq_lock);
 952                 return;
 953         }
 954         pm_iir = dev_priv->rps.pm_iir;
 955         dev_priv->rps.pm_iir = 0;
 956         /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
 957         gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 958         client_boost = dev_priv->rps.client_boost;
 959         dev_priv->rps.client_boost = false;
 960         spin_unlock_irq(&dev_priv->irq_lock);
 961
 962         /* Make sure we didn't queue anything we're not going to process. */
 963         WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
 964
 965         if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
 966                 return;
 967
 968         mutex_lock(&dev_priv->rps.hw_lock);
 969
 970         pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);
 971
 972         adj = dev_priv->rps.last_adj;
 973         new_delay = dev_priv->rps.cur_freq;
 974         min = dev_priv->rps.min_freq_softlimit;
 975         max = dev_priv->rps.max_freq_softlimit;
 976
 977         if (client_boost) {
 978                 new_delay = dev_priv->rps.max_freq_softlimit;
 979                 adj = 0;
 980         } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
 981                 if (adj > 0)
 982                         adj *= 2;
 983                 else /* CHV needs even encode values */
 984                         adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
 985                 /*
 986                  * For better performance, jump directly
 987                  * to RPe if we're below it.
 988                  */
 989                 if (new_delay < dev_priv->rps.efficient_freq - adj) {
 990                         new_delay = dev_priv->rps.efficient_freq;
 991                         adj = 0;
 992                 }
 993         } else if (any_waiters(dev_priv)) {
 994                 adj = 0;
 995         } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
 996                 if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
 997                         new_delay = dev_priv->rps.efficient_freq;
 998                 else
 999                         new_delay = dev_priv->rps.min_freq_softlimit;
1000                 adj = 0;
1001         } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1002                 if (adj < 0)
1003                         adj *= 2;
1004                 else /* CHV needs even encode values */
1005                         adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1006         } else { /* unknown event */
1007                 adj = 0;
1008         }
1009
1010         dev_priv->rps.last_adj = adj;
1011
1012         /* sysfs frequency interfaces may have snuck in while servicing the
1013          * interrupt
1014          */
1015         new_delay += adj;
1016         new_delay = clamp_t(int, new_delay, min, max);
1017
1018         intel_set_rps(dev_priv->dev, new_delay);
1019
1020         mutex_unlock(&dev_priv->rps.hw_lock);
1021 }
1022
1023
1024 /**
1025  * ivybridge_parity_work - Workqueue called when a parity error interrupt
1026  * occurred.
1027  * @work: workqueue struct
1028  *
1029  * Doesn't actually do anything except notify userspace. As a consequence of
1030  * this event, userspace should try to remap the bad rows since statistically
1031  * it is likely the same row is more likely to go bad again.
1032  */
1033 static void ivybridge_parity_work(struct work_struct *work)
1034 {
1035         struct drm_i915_private *dev_priv =
1036                 container_of(work, struct drm_i915_private, l3_parity.error_work);
1037         u32 error_status, row, bank, subbank;
1038         char *parity_event[6];
1039         uint32_t misccpctl;
1040         uint8_t slice = 0;
1041
1042         /* We must turn off DOP level clock gating to access the L3 registers.
1043          * In order to prevent a get/put style interface, acquire struct mutex
1044          * any time we access those registers.
1045          */
1046         mutex_lock(&dev_priv->dev->struct_mutex);
1047
1048         /* If we've screwed up tracking, just let the interrupt fire again */
1049         if (WARN_ON(!dev_priv->l3_parity.which_slice))
1050                 goto out;
1051
1052         misccpctl = I915_READ(GEN7_MISCCPCTL);
1053         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
1054         POSTING_READ(GEN7_MISCCPCTL);
1055
1056         while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1057                 u32 reg;
1058
1059                 slice--;
1060                 if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
1061                         break;
1062
1063                 dev_priv->l3_parity.which_slice &= ~(1<<slice);
1064
1065                 reg = GEN7_L3CDERRST1 + (slice * 0x200);
1066
1067                 error_status = I915_READ(reg);
1068                 row = GEN7_PARITY_ERROR_ROW(error_status);
1069                 bank = GEN7_PARITY_ERROR_BANK(error_status);
1070                 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
1071
1072                 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
1073                 POSTING_READ(reg);
1074
1075                 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
1076                 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
1077                 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
1078                 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
1079                 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
1080                 parity_event[5] = NULL;
1081
1082                 kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
1083                                    KOBJ_CHANGE, parity_event);
1084
1085                 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
1086                           slice, row, bank, subbank);
1087
1088                 kfree(parity_event[4]);
1089                 kfree(parity_event[3]);
1090                 kfree(parity_event[2]);
1091                 kfree(parity_event[1]);
1092         }
1093
1094         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1095
1096 out:
1097         WARN_ON(dev_priv->l3_parity.which_slice);
1098         spin_lock_irq(&dev_priv->irq_lock);
1099         gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
1100         spin_unlock_irq(&dev_priv->irq_lock);
1101
1102         mutex_unlock(&dev_priv->dev->struct_mutex);
1103 }
1104
1105 static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
1106 {
1107         struct drm_i915_private *dev_priv = dev->dev_private;
1108
1109         if (!HAS_L3_DPF(dev))
1110                 return;
1111
1112         spin_lock(&dev_priv->irq_lock);
1113         gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
1114         spin_unlock(&dev_priv->irq_lock);
1115
1116         iir &= GT_PARITY_ERROR(dev);
1117         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
1118                 dev_priv->l3_parity.which_slice |= 1 << 1;
1119
1120         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
1121                 dev_priv->l3_parity.which_slice |= 1 << 0;
1122
1123         queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1124 }
1125
1126 static void ilk_gt_irq_handler(struct drm_device *dev,
1127                                struct drm_i915_private *dev_priv,
1128                                u32 gt_iir)
1129 {
1130         if (gt_iir &
1131             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1132                 notify_ring(&dev_priv->ring[RCS]);
1133         if (gt_iir & ILK_BSD_USER_INTERRUPT)
1134                 notify_ring(&dev_priv->ring[VCS]);
1135 }
1136
1137 static void snb_gt_irq_handler(struct drm_device *dev,
1138                                struct drm_i915_private *dev_priv,
1139                                u32 gt_iir)
1140 {
1141
1142         if (gt_iir &
1143             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1144                 notify_ring(&dev_priv->ring[RCS]);
1145         if (gt_iir & GT_BSD_USER_INTERRUPT)
1146                 notify_ring(&dev_priv->ring[VCS]);
1147         if (gt_iir & GT_BLT_USER_INTERRUPT)
1148                 notify_ring(&dev_priv->ring[BCS]);
1149
1150         if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
1151                       GT_BSD_CS_ERROR_INTERRUPT |
1152                       GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
1153                 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1154
1155         if (gt_iir & GT_PARITY_ERROR(dev))
1156                 ivybridge_parity_error_irq_handler(dev, gt_iir);
1157 }
1158
1159 static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
1160                                        u32 master_ctl)
1161 {
1162         irqreturn_t ret = IRQ_NONE;
1163
1164         if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1165                 u32 tmp = I915_READ_FW(GEN8_GT_IIR(0));
1166                 if (tmp) {
1167                         I915_WRITE_FW(GEN8_GT_IIR(0), tmp);
1168                         ret = IRQ_HANDLED;
1169
1170                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
1171                                 intel_lrc_irq_handler(&dev_priv->ring[RCS]);
1172                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
1173                                 notify_ring(&dev_priv->ring[RCS]);
1174
1175                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
1176                                 intel_lrc_irq_handler(&dev_priv->ring[BCS]);
1177                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
1178                                 notify_ring(&dev_priv->ring[BCS]);
1179                 } else
1180                         DRM_ERROR("The master control interrupt lied (GT0)!\n");
1181         }
1182
1183         if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1184                 u32 tmp = I915_READ_FW(GEN8_GT_IIR(1));
1185                 if (tmp) {
1186                         I915_WRITE_FW(GEN8_GT_IIR(1), tmp);
1187                         ret = IRQ_HANDLED;
1188
1189                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
1190                                 intel_lrc_irq_handler(&dev_priv->ring[VCS]);
1191                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
1192                                 notify_ring(&dev_priv->ring[VCS]);
1193
1194                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
1195                                 intel_lrc_irq_handler(&dev_priv->ring[VCS2]);
1196                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
1197                                 notify_ring(&dev_priv->ring[VCS2]);
1198                 } else
1199                         DRM_ERROR("The master control interrupt lied (GT1)!\n");
1200         }
1201
1202         if (master_ctl & GEN8_GT_VECS_IRQ) {
1203                 u32 tmp = I915_READ_FW(GEN8_GT_IIR(3));
1204                 if (tmp) {
1205                         I915_WRITE_FW(GEN8_GT_IIR(3), tmp);
1206                         ret = IRQ_HANDLED;
1207
1208                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
1209                                 intel_lrc_irq_handler(&dev_priv->ring[VECS]);
1210                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
1211                                 notify_ring(&dev_priv->ring[VECS]);
1212                 } else
1213                         DRM_ERROR("The master control interrupt lied (GT3)!\n");
1214         }
1215
1216         if (master_ctl & GEN8_GT_PM_IRQ) {
1217                 u32 tmp = I915_READ_FW(GEN8_GT_IIR(2));
1218                 if (tmp & dev_priv->pm_rps_events) {
1219                         I915_WRITE_FW(GEN8_GT_IIR(2),
1220                                       tmp & dev_priv->pm_rps_events);
1221                         ret = IRQ_HANDLED;
1222                         gen6_rps_irq_handler(dev_priv, tmp);
1223                 } else
1224                         DRM_ERROR("The master control interrupt lied (PM)!\n");
1225         }
1226
1227         return ret;
1228 }
1229
1230 static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1231 {
1232         switch (port) {
1233         case PORT_B:
1234                 return val & PORTB_HOTPLUG_LONG_DETECT;
1235         case PORT_C:
1236                 return val & PORTC_HOTPLUG_LONG_DETECT;
1237         case PORT_D:
1238                 return val & PORTD_HOTPLUG_LONG_DETECT;
1239         default:
1240                 return false;
1241         }
1242 }
1243
1244 static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1245 {
1246         switch (port) {
1247         case PORT_B:
1248                 return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1249         case PORT_C:
1250                 return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1251         case PORT_D:
1252                 return val & PORTD_HOTPLUG_INT_LONG_PULSE;
1253         default:
1254                 return false;
1255         }
1256 }
1257
1258 /* Get a bit mask of pins that have triggered, and which ones may be long. */
1259 static void pch_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1260                              u32 hotplug_trigger, u32 dig_hotplug_reg,
1261                              const u32 hpd[HPD_NUM_PINS])
1262 {
1263         enum port port;
1264         int i;
1265
1266         *pin_mask = 0;
1267         *long_mask = 0;
1268
1269         for_each_hpd_pin(i) {
1270                 if ((hpd[i] & hotplug_trigger) == 0)
1271                         continue;
1272
1273                 *pin_mask |= BIT(i);
1274
1275                 port = intel_hpd_pin_to_port(i);
1276                 if (pch_port_hotplug_long_detect(port, dig_hotplug_reg))
1277                         *long_mask |= BIT(i);
1278         }
1279
1280         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
1281                          hotplug_trigger, dig_hotplug_reg, *pin_mask);
1282
1283 }
1284
1285 /* Get a bit mask of pins that have triggered, and which ones may be long. */
1286 static void i9xx_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1287                               u32 hotplug_trigger, const u32 hpd[HPD_NUM_PINS])
1288 {
1289         enum port port;
1290         int i;
1291
1292         *pin_mask = 0;
1293         *long_mask = 0;
1294
1295         if (!hotplug_trigger)
1296                 return;
1297
1298         for_each_hpd_pin(i) {
1299                 if ((hpd[i] & hotplug_trigger) == 0)
1300                         continue;
1301
1302                 *pin_mask |= BIT(i);
1303
1304                 port = intel_hpd_pin_to_port(i);
1305                 if (i9xx_port_hotplug_long_detect(port, hotplug_trigger))
1306                         *long_mask |= BIT(i);
1307         }
1308
1309         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, pins 0x%08x\n",
1310                          hotplug_trigger, *pin_mask);
1311 }
1312
1313 static void gmbus_irq_handler(struct drm_device *dev)
1314 {
1315         struct drm_i915_private *dev_priv = dev->dev_private;
1316
1317         wake_up_all(&dev_priv->gmbus_wait_queue);
1318 }
1319
1320 static void dp_aux_irq_handler(struct drm_device *dev)
1321 {
1322         struct drm_i915_private *dev_priv = dev->dev_private;
1323
1324         wake_up_all(&dev_priv->gmbus_wait_queue);
1325 }
1326
1327 #if defined(CONFIG_DEBUG_FS)
1328 static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1329                                          uint32_t crc0, uint32_t crc1,
1330                                          uint32_t crc2, uint32_t crc3,
1331                                          uint32_t crc4)
1332 {
1333         struct drm_i915_private *dev_priv = dev->dev_private;
1334         struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
1335         struct intel_pipe_crc_entry *entry;
1336         int head, tail;
1337
1338         spin_lock(&pipe_crc->lock);
1339
1340         if (!pipe_crc->entries) {
1341                 spin_unlock(&pipe_crc->lock);
1342                 DRM_DEBUG_KMS("spurious interrupt\n");
1343                 return;
1344         }
1345
1346         head = pipe_crc->head;
1347         tail = pipe_crc->tail;
1348
1349         if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1350                 spin_unlock(&pipe_crc->lock);
1351                 DRM_ERROR("CRC buffer overflowing\n");
1352                 return;
1353         }
1354
1355         entry = &pipe_crc->entries[head];
1356
1357         entry->frame = dev->driver->get_vblank_counter(dev, pipe);
1358         entry->crc[0] = crc0;
1359         entry->crc[1] = crc1;
1360         entry->crc[2] = crc2;
1361         entry->crc[3] = crc3;
1362         entry->crc[4] = crc4;
1363
1364         head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1365         pipe_crc->head = head;
1366
1367         spin_unlock(&pipe_crc->lock);
1368
1369         wake_up_interruptible(&pipe_crc->wq);
1370 }
1371 #else
1372 static inline void
1373 display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1374                              uint32_t crc0, uint32_t crc1,
1375                              uint32_t crc2, uint32_t crc3,
1376                              uint32_t crc4) {}
1377 #endif
1378
1379
1380 static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1381 {
1382         struct drm_i915_private *dev_priv = dev->dev_private;
1383
1384         display_pipe_crc_irq_handler(dev, pipe,
1385                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1386                                      0, 0, 0, 0);
1387 }
1388
1389 static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1390 {
1391         struct drm_i915_private *dev_priv = dev->dev_private;
1392
1393         display_pipe_crc_irq_handler(dev, pipe,
1394                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1395                                      I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1396                                      I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1397                                      I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
1398                                      I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1399 }
1400
1401 static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1402 {
1403         struct drm_i915_private *dev_priv = dev->dev_private;
1404         uint32_t res1, res2;
1405
1406         if (INTEL_INFO(dev)->gen >= 3)
1407                 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
1408         else
1409                 res1 = 0;
1410
1411         if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
1412                 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
1413         else
1414                 res2 = 0;
1415
1416         display_pipe_crc_irq_handler(dev, pipe,
1417                                      I915_READ(PIPE_CRC_RES_RED(pipe)),
1418                                      I915_READ(PIPE_CRC_RES_GREEN(pipe)),
1419                                      I915_READ(PIPE_CRC_RES_BLUE(pipe)),
1420                                      res1, res2);
1421 }
1422
1423 /* The RPS events need forcewake, so we add them to a work queue and mask their
1424  * IMR bits until the work is done. Other interrupts can be processed without
1425  * the work queue. */
1426 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1427 {
1428         if (pm_iir & dev_priv->pm_rps_events) {
1429                 spin_lock(&dev_priv->irq_lock);
1430                 gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
1431                 if (dev_priv->rps.interrupts_enabled) {
1432                         dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
1433                         queue_work(dev_priv->wq, &dev_priv->rps.work);
1434                 }
1435                 spin_unlock(&dev_priv->irq_lock);
1436         }
1437
1438         if (INTEL_INFO(dev_priv)->gen >= 8)
1439                 return;
1440
1441         if (HAS_VEBOX(dev_priv->dev)) {
1442                 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1443                         notify_ring(&dev_priv->ring[VECS]);
1444
1445                 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1446                         DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1447         }
1448 }
1449
1450 static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
1451 {
1452         if (!drm_handle_vblank(dev, pipe))
1453                 return false;
1454
1455         return true;
1456 }
1457
1458 static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
1459 {
1460         struct drm_i915_private *dev_priv = dev->dev_private;
1461         u32 pipe_stats[I915_MAX_PIPES] = { };
1462         int pipe;
1463
1464         spin_lock(&dev_priv->irq_lock);
1465         for_each_pipe(dev_priv, pipe) {
1466                 int reg;
1467                 u32 mask, iir_bit = 0;
1468
1469                 /*
1470                  * PIPESTAT bits get signalled even when the interrupt is
1471                  * disabled with the mask bits, and some of the status bits do
1472                  * not generate interrupts at all (like the underrun bit). Hence
1473                  * we need to be careful that we only handle what we want to
1474                  * handle.
1475                  */
1476
1477                 /* fifo underruns are filterered in the underrun handler. */
1478                 mask = PIPE_FIFO_UNDERRUN_STATUS;
1479
1480                 switch (pipe) {
1481                 case PIPE_A:
1482                         iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1483                         break;
1484                 case PIPE_B:
1485                         iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1486                         break;
1487                 case PIPE_C:
1488                         iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
1489                         break;
1490                 }
1491                 if (iir & iir_bit)
1492                         mask |= dev_priv->pipestat_irq_mask[pipe];
1493
1494                 if (!mask)
1495                         continue;
1496
1497                 reg = PIPESTAT(pipe);
1498                 mask |= PIPESTAT_INT_ENABLE_MASK;
1499                 pipe_stats[pipe] = I915_READ(reg) & mask;
1500
1501                 /*
1502                  * Clear the PIPE*STAT regs before the IIR
1503                  */
1504                 if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
1505                                         PIPESTAT_INT_STATUS_MASK))
1506                         I915_WRITE(reg, pipe_stats[pipe]);
1507         }
1508         spin_unlock(&dev_priv->irq_lock);
1509
1510         for_each_pipe(dev_priv, pipe) {
1511                 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
1512                     intel_pipe_handle_vblank(dev, pipe))
1513                         intel_check_page_flip(dev, pipe);
1514
1515                 if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
1516                         intel_prepare_page_flip(dev, pipe);
1517                         intel_finish_page_flip(dev, pipe);
1518                 }
1519
1520                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1521                         i9xx_pipe_crc_irq_handler(dev, pipe);
1522
1523                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1524                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1525         }
1526
1527         if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1528                 gmbus_irq_handler(dev);
1529 }
1530
1531 static void i9xx_hpd_irq_handler(struct drm_device *dev)
1532 {
1533         struct drm_i915_private *dev_priv = dev->dev_private;
1534         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
1535         u32 pin_mask, long_mask;
1536
1537         if (!hotplug_status)
1538                 return;
1539
1540         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1541         /*
1542          * Make sure hotplug status is cleared before we clear IIR, or else we
1543          * may miss hotplug events.
1544          */
1545         POSTING_READ(PORT_HOTPLUG_STAT);
1546
1547         if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
1548                 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1549
1550                 i9xx_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, hpd_status_g4x);
1551                 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1552
1553                 if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1554                         dp_aux_irq_handler(dev);
1555         } else {
1556                 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1557
1558                 i9xx_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, hpd_status_i915);
1559                 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1560         }
1561 }
1562
1563 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
1564 {
1565         struct drm_device *dev = arg;
1566         struct drm_i915_private *dev_priv = dev->dev_private;
1567         u32 iir, gt_iir, pm_iir;
1568         irqreturn_t ret = IRQ_NONE;
1569
1570         if (!intel_irqs_enabled(dev_priv))
1571                 return IRQ_NONE;
1572
1573         while (true) {
1574                 /* Find, clear, then process each source of interrupt */
1575
1576                 gt_iir = I915_READ(GTIIR);
1577                 if (gt_iir)
1578                         I915_WRITE(GTIIR, gt_iir);
1579
1580                 pm_iir = I915_READ(GEN6_PMIIR);
1581                 if (pm_iir)
1582                         I915_WRITE(GEN6_PMIIR, pm_iir);
1583
1584                 iir = I915_READ(VLV_IIR);
1585                 if (iir) {
1586                         /* Consume port before clearing IIR or we'll miss events */
1587                         if (iir & I915_DISPLAY_PORT_INTERRUPT)
1588                                 i9xx_hpd_irq_handler(dev);
1589                         I915_WRITE(VLV_IIR, iir);
1590                 }
1591
1592                 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1593                         goto out;
1594
1595                 ret = IRQ_HANDLED;
1596
1597                 if (gt_iir)
1598                         snb_gt_irq_handler(dev, dev_priv, gt_iir);
1599                 if (pm_iir)
1600                         gen6_rps_irq_handler(dev_priv, pm_iir);
1601                 /* Call regardless, as some status bits might not be
1602                  * signalled in iir */
1603                 valleyview_pipestat_irq_handler(dev, iir);
1604         }
1605
1606 out:
1607         return ret;
1608 }
1609
1610 static irqreturn_t cherryview_irq_handler(int irq, void *arg)
1611 {
1612         struct drm_device *dev = arg;
1613         struct drm_i915_private *dev_priv = dev->dev_private;
1614         u32 master_ctl, iir;
1615         irqreturn_t ret = IRQ_NONE;
1616
1617         if (!intel_irqs_enabled(dev_priv))
1618                 return IRQ_NONE;
1619
1620         for (;;) {
1621                 master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
1622                 iir = I915_READ(VLV_IIR);
1623
1624                 if (master_ctl == 0 && iir == 0)
1625                         break;
1626
1627                 ret = IRQ_HANDLED;
1628
1629                 I915_WRITE(GEN8_MASTER_IRQ, 0);
1630
1631                 /* Find, clear, then process each source of interrupt */
1632
1633                 if (iir) {
1634                         /* Consume port before clearing IIR or we'll miss events */
1635                         if (iir & I915_DISPLAY_PORT_INTERRUPT)
1636                                 i9xx_hpd_irq_handler(dev);
1637                         I915_WRITE(VLV_IIR, iir);
1638                 }
1639
1640                 gen8_gt_irq_handler(dev_priv, master_ctl);
1641
1642                 /* Call regardless, as some status bits might not be
1643                  * signalled in iir */
1644                 valleyview_pipestat_irq_handler(dev, iir);
1645
1646                 I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
1647                 POSTING_READ(GEN8_MASTER_IRQ);
1648         }
1649
1650         return ret;
1651 }
1652
1653 static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1654 {
1655         struct drm_i915_private *dev_priv = dev->dev_private;
1656         int pipe;
1657         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1658
1659         if (hotplug_trigger) {
1660                 u32 dig_hotplug_reg, pin_mask, long_mask;
1661
1662                 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1663                 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1664
1665                 pch_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1666                                  dig_hotplug_reg, hpd_ibx);
1667                 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1668         }
1669
1670         if (pch_iir & SDE_AUDIO_POWER_MASK) {
1671                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1672                                SDE_AUDIO_POWER_SHIFT);
1673                 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1674                                  port_name(port));
1675         }
1676
1677         if (pch_iir & SDE_AUX_MASK)
1678                 dp_aux_irq_handler(dev);
1679
1680         if (pch_iir & SDE_GMBUS)
1681                 gmbus_irq_handler(dev);
1682
1683         if (pch_iir & SDE_AUDIO_HDCP_MASK)
1684                 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
1685
1686         if (pch_iir & SDE_AUDIO_TRANS_MASK)
1687                 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
1688
1689         if (pch_iir & SDE_POISON)
1690                 DRM_ERROR("PCH poison interrupt\n");
1691
1692         if (pch_iir & SDE_FDI_MASK)
1693                 for_each_pipe(dev_priv, pipe)
1694                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1695                                          pipe_name(pipe),
1696                                          I915_READ(FDI_RX_IIR(pipe)));
1697
1698         if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1699                 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
1700
1701         if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1702                 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
1703
1704         if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1705                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1706
1707         if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1708                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1709 }
1710
1711 static void ivb_err_int_handler(struct drm_device *dev)
1712 {
1713         struct drm_i915_private *dev_priv = dev->dev_private;
1714         u32 err_int = I915_READ(GEN7_ERR_INT);
1715         enum pipe pipe;
1716
1717         if (err_int & ERR_INT_POISON)
1718                 DRM_ERROR("Poison interrupt\n");
1719
1720         for_each_pipe(dev_priv, pipe) {
1721                 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
1722                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1723
1724                 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
1725                         if (IS_IVYBRIDGE(dev))
1726                                 ivb_pipe_crc_irq_handler(dev, pipe);
1727                         else
1728                                 hsw_pipe_crc_irq_handler(dev, pipe);
1729                 }
1730         }
1731
1732         I915_WRITE(GEN7_ERR_INT, err_int);
1733 }
1734
1735 static void cpt_serr_int_handler(struct drm_device *dev)
1736 {
1737         struct drm_i915_private *dev_priv = dev->dev_private;
1738         u32 serr_int = I915_READ(SERR_INT);
1739
1740         if (serr_int & SERR_INT_POISON)
1741                 DRM_ERROR("PCH poison interrupt\n");
1742
1743         if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1744                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1745
1746         if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1747                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1748
1749         if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1750                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
1751
1752         I915_WRITE(SERR_INT, serr_int);
1753 }
1754
1755 static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
1756 {
1757         struct drm_i915_private *dev_priv = dev->dev_private;
1758         int pipe;
1759         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1760
1761         if (hotplug_trigger) {
1762                 u32 dig_hotplug_reg, pin_mask, long_mask;
1763
1764                 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1765                 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1766                 pch_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1767                                  dig_hotplug_reg, hpd_cpt);
1768                 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1769         }
1770
1771         if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1772                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1773                                SDE_AUDIO_POWER_SHIFT_CPT);
1774                 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
1775                                  port_name(port));
1776         }
1777
1778         if (pch_iir & SDE_AUX_MASK_CPT)
1779                 dp_aux_irq_handler(dev);
1780
1781         if (pch_iir & SDE_GMBUS_CPT)
1782                 gmbus_irq_handler(dev);
1783
1784         if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1785                 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
1786
1787         if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1788                 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
1789
1790         if (pch_iir & SDE_FDI_MASK_CPT)
1791                 for_each_pipe(dev_priv, pipe)
1792                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1793                                          pipe_name(pipe),
1794                                          I915_READ(FDI_RX_IIR(pipe)));
1795
1796         if (pch_iir & SDE_ERROR_CPT)
1797                 cpt_serr_int_handler(dev);
1798 }
1799
1800 static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
1801 {
1802         struct drm_i915_private *dev_priv = dev->dev_private;
1803         enum pipe pipe;
1804
1805         if (de_iir & DE_AUX_CHANNEL_A)
1806                 dp_aux_irq_handler(dev);
1807
1808         if (de_iir & DE_GSE)
1809                 intel_opregion_asle_intr(dev);
1810
1811         if (de_iir & DE_POISON)
1812                 DRM_ERROR("Poison interrupt\n");
1813
1814         for_each_pipe(dev_priv, pipe) {
1815                 if (de_iir & DE_PIPE_VBLANK(pipe) &&
1816                     intel_pipe_handle_vblank(dev, pipe))
1817                         intel_check_page_flip(dev, pipe);
1818
1819                 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
1820                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1821
1822                 if (de_iir & DE_PIPE_CRC_DONE(pipe))
1823                         i9xx_pipe_crc_irq_handler(dev, pipe);
1824
1825                 /* plane/pipes map 1:1 on ilk+ */
1826                 if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
1827                         intel_prepare_page_flip(dev, pipe);
1828                         intel_finish_page_flip_plane(dev, pipe);
1829                 }
1830         }
1831
1832         /* check event from PCH */
1833         if (de_iir & DE_PCH_EVENT) {
1834                 u32 pch_iir = I915_READ(SDEIIR);
1835
1836                 if (HAS_PCH_CPT(dev))
1837                         cpt_irq_handler(dev, pch_iir);
1838                 else
1839                         ibx_irq_handler(dev, pch_iir);
1840
1841                 /* should clear PCH hotplug event before clear CPU irq */
1842                 I915_WRITE(SDEIIR, pch_iir);
1843         }
1844
1845         if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
1846                 ironlake_rps_change_irq_handler(dev);
1847 }
1848
1849 static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
1850 {
1851         struct drm_i915_private *dev_priv = dev->dev_private;
1852         enum pipe pipe;
1853
1854         if (de_iir & DE_ERR_INT_IVB)
1855                 ivb_err_int_handler(dev);
1856
1857         if (de_iir & DE_AUX_CHANNEL_A_IVB)
1858                 dp_aux_irq_handler(dev);
1859
1860         if (de_iir & DE_GSE_IVB)
1861                 intel_opregion_asle_intr(dev);
1862
1863         for_each_pipe(dev_priv, pipe) {
1864                 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
1865                     intel_pipe_handle_vblank(dev, pipe))
1866                         intel_check_page_flip(dev, pipe);
1867
1868                 /* plane/pipes map 1:1 on ilk+ */
1869                 if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
1870                         intel_prepare_page_flip(dev, pipe);
1871                         intel_finish_page_flip_plane(dev, pipe);
1872                 }
1873         }
1874
1875         /* check event from PCH */
1876         if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
1877                 u32 pch_iir = I915_READ(SDEIIR);
1878
1879                 cpt_irq_handler(dev, pch_iir);
1880
1881                 /* clear PCH hotplug event before clear CPU irq */
1882                 I915_WRITE(SDEIIR, pch_iir);
1883         }
1884 }
1885
1886 /*
1887  * To handle irqs with the minimum potential races with fresh interrupts, we:
1888  * 1 - Disable Master Interrupt Control.
1889  * 2 - Find the source(s) of the interrupt.
1890  * 3 - Clear the Interrupt Identity bits (IIR).
1891  * 4 - Process the interrupt(s) that had bits set in the IIRs.
1892  * 5 - Re-enable Master Interrupt Control.
1893  */
1894 static irqreturn_t ironlake_irq_handler(int irq, void *arg)
1895 {
1896         struct drm_device *dev = arg;
1897         struct drm_i915_private *dev_priv = dev->dev_private;
1898         u32 de_iir, gt_iir, de_ier, sde_ier = 0;
1899         irqreturn_t ret = IRQ_NONE;
1900
1901         if (!intel_irqs_enabled(dev_priv))
1902                 return IRQ_NONE;
1903
1904         /* We get interrupts on unclaimed registers, so check for this before we
1905          * do any I915_{READ,WRITE}. */
1906         intel_uncore_check_errors(dev);
1907
1908         /* disable master interrupt before clearing iir  */
1909         de_ier = I915_READ(DEIER);
1910         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
1911         POSTING_READ(DEIER);
1912
1913         /* Disable south interrupts. We'll only write to SDEIIR once, so further
1914          * interrupts will will be stored on its back queue, and then we'll be
1915          * able to process them after we restore SDEIER (as soon as we restore
1916          * it, we'll get an interrupt if SDEIIR still has something to process
1917          * due to its back queue). */
1918         if (!HAS_PCH_NOP(dev)) {
1919                 sde_ier = I915_READ(SDEIER);
1920                 I915_WRITE(SDEIER, 0);
1921                 POSTING_READ(SDEIER);
1922         }
1923
1924         /* Find, clear, then process each source of interrupt */
1925
1926         gt_iir = I915_READ(GTIIR);
1927         if (gt_iir) {
1928                 I915_WRITE(GTIIR, gt_iir);
1929                 ret = IRQ_HANDLED;
1930                 if (INTEL_INFO(dev)->gen >= 6)
1931                         snb_gt_irq_handler(dev, dev_priv, gt_iir);
1932                 else
1933                         ilk_gt_irq_handler(dev, dev_priv, gt_iir);
1934         }
1935
1936         de_iir = I915_READ(DEIIR);
1937         if (de_iir) {
1938                 I915_WRITE(DEIIR, de_iir);
1939                 ret = IRQ_HANDLED;
1940                 if (INTEL_INFO(dev)->gen >= 7)
1941                         ivb_display_irq_handler(dev, de_iir);
1942                 else
1943                         ilk_display_irq_handler(dev, de_iir);
1944         }
1945
1946         if (INTEL_INFO(dev)->gen >= 6) {
1947                 u32 pm_iir = I915_READ(GEN6_PMIIR);
1948                 if (pm_iir) {
1949                         I915_WRITE(GEN6_PMIIR, pm_iir);
1950                         ret = IRQ_HANDLED;
1951                         gen6_rps_irq_handler(dev_priv, pm_iir);
1952                 }
1953         }
1954
1955         I915_WRITE(DEIER, de_ier);
1956         POSTING_READ(DEIER);
1957         if (!HAS_PCH_NOP(dev)) {
1958                 I915_WRITE(SDEIER, sde_ier);
1959                 POSTING_READ(SDEIER);
1960         }
1961
1962         return ret;
1963 }
1964
1965 static void bxt_hpd_handler(struct drm_device *dev, uint32_t iir_status)
1966 {
1967         struct drm_i915_private *dev_priv = dev->dev_private;
1968         u32 hp_control, hp_trigger;
1969         u32 pin_mask, long_mask;
1970
1971         /* Get the status */
1972         hp_trigger = iir_status & BXT_DE_PORT_HOTPLUG_MASK;
1973         hp_control = I915_READ(BXT_HOTPLUG_CTL);
1974
1975         /* Hotplug not enabled ? */
1976         if (!(hp_control & BXT_HOTPLUG_CTL_MASK)) {
1977                 DRM_ERROR("Interrupt when HPD disabled\n");
1978                 return;
1979         }
1980
1981         /* Clear sticky bits in hpd status */
1982         I915_WRITE(BXT_HOTPLUG_CTL, hp_control);
1983
1984         pch_get_hpd_pins(&pin_mask, &long_mask, hp_trigger, hp_control, hpd_bxt);
1985         intel_hpd_irq_handler(dev, pin_mask, long_mask);
1986 }
1987
1988 static irqreturn_t gen8_irq_handler(int irq, void *arg)
1989 {
1990         struct drm_device *dev = arg;
1991         struct drm_i915_private *dev_priv = dev->dev_private;
1992         u32 master_ctl;
1993         irqreturn_t ret = IRQ_NONE;
1994         uint32_t tmp = 0;
1995         enum pipe pipe;
1996         u32 aux_mask = GEN8_AUX_CHANNEL_A;
1997
1998         if (!intel_irqs_enabled(dev_priv))
1999                 return IRQ_NONE;
2000
2001         if (IS_GEN9(dev))
2002                 aux_mask |=  GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
2003                         GEN9_AUX_CHANNEL_D;
2004
2005         master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
2006         master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
2007         if (!master_ctl)
2008                 return IRQ_NONE;
2009
2010         I915_WRITE_FW(GEN8_MASTER_IRQ, 0);
2011
2012         /* Find, clear, then process each source of interrupt */
2013
2014         ret = gen8_gt_irq_handler(dev_priv, master_ctl);
2015
2016         if (master_ctl & GEN8_DE_MISC_IRQ) {
2017                 tmp = I915_READ(GEN8_DE_MISC_IIR);
2018                 if (tmp) {
2019                         I915_WRITE(GEN8_DE_MISC_IIR, tmp);
2020                         ret = IRQ_HANDLED;
2021                         if (tmp & GEN8_DE_MISC_GSE)
2022                                 intel_opregion_asle_intr(dev);
2023                         else
2024                                 DRM_ERROR("Unexpected DE Misc interrupt\n");
2025                 }
2026                 else
2027                         DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2028         }
2029
2030         if (master_ctl & GEN8_DE_PORT_IRQ) {
2031                 tmp = I915_READ(GEN8_DE_PORT_IIR);
2032                 if (tmp) {
2033                         bool found = false;
2034
2035                         I915_WRITE(GEN8_DE_PORT_IIR, tmp);
2036                         ret = IRQ_HANDLED;
2037
2038                         if (tmp & aux_mask) {
2039                                 dp_aux_irq_handler(dev);
2040                                 found = true;
2041                         }
2042
2043                         if (IS_BROXTON(dev) && tmp & BXT_DE_PORT_HOTPLUG_MASK) {
2044                                 bxt_hpd_handler(dev, tmp);
2045                                 found = true;
2046                         }
2047
2048                         if (IS_BROXTON(dev) && (tmp & BXT_DE_PORT_GMBUS)) {
2049                                 gmbus_irq_handler(dev);
2050                                 found = true;
2051                         }
2052
2053                         if (!found)
2054                                 DRM_ERROR("Unexpected DE Port interrupt\n");
2055                 }
2056                 else
2057                         DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2058         }
2059
2060         for_each_pipe(dev_priv, pipe) {
2061                 uint32_t pipe_iir, flip_done = 0, fault_errors = 0;
2062
2063                 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2064                         continue;
2065
2066                 pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
2067                 if (pipe_iir) {
2068                         ret = IRQ_HANDLED;
2069                         I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
2070
2071                         if (pipe_iir & GEN8_PIPE_VBLANK &&
2072                             intel_pipe_handle_vblank(dev, pipe))
2073                                 intel_check_page_flip(dev, pipe);
2074
2075                         if (IS_GEN9(dev))
2076                                 flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
2077                         else
2078                                 flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;
2079
2080                         if (flip_done) {
2081                                 intel_prepare_page_flip(dev, pipe);
2082                                 intel_finish_page_flip_plane(dev, pipe);
2083                         }
2084
2085                         if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
2086                                 hsw_pipe_crc_irq_handler(dev, pipe);
2087
2088                         if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN)
2089                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
2090                                                                     pipe);
2091
2092
2093                         if (IS_GEN9(dev))
2094                                 fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
2095                         else
2096                                 fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2097
2098                         if (fault_errors)
2099                                 DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
2100                                           pipe_name(pipe),
2101                                           pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
2102                 } else
2103                         DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
2104         }
2105
2106         if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) &&
2107             master_ctl & GEN8_DE_PCH_IRQ) {
2108                 /*
2109                  * FIXME(BDW): Assume for now that the new interrupt handling
2110                  * scheme also closed the SDE interrupt handling race we've seen
2111                  * on older pch-split platforms. But this needs testing.
2112                  */
2113                 u32 pch_iir = I915_READ(SDEIIR);
2114                 if (pch_iir) {
2115                         I915_WRITE(SDEIIR, pch_iir);
2116                         ret = IRQ_HANDLED;
2117                         cpt_irq_handler(dev, pch_iir);
2118                 } else
2119                         DRM_ERROR("The master control interrupt lied (SDE)!\n");
2120
2121         }
2122
2123         I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2124         POSTING_READ_FW(GEN8_MASTER_IRQ);
2125
2126         return ret;
2127 }
2128
2129 static void i915_error_wake_up(struct drm_i915_private *dev_priv,
2130                                bool reset_completed)
2131 {
2132         struct intel_engine_cs *ring;
2133         int i;
2134
2135         /*
2136          * Notify all waiters for GPU completion events that reset state has
2137          * been changed, and that they need to restart their wait after
2138          * checking for potential errors (and bail out to drop locks if there is
2139          * a gpu reset pending so that i915_error_work_func can acquire them).
2140          */
2141
2142         /* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
2143         for_each_ring(ring, dev_priv, i)
2144                 wake_up_all(&ring->irq_queue);
2145
2146         /* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
2147         wake_up_all(&dev_priv->pending_flip_queue);
2148
2149         /*
2150          * Signal tasks blocked in i915_gem_wait_for_error that the pending
2151          * reset state is cleared.
2152          */
2153         if (reset_completed)
2154                 wake_up_all(&dev_priv->gpu_error.reset_queue);
2155 }
2156
2157 /**
2158  * i915_reset_and_wakeup - do process context error handling work
2159  *
2160  * Fire an error uevent so userspace can see that a hang or error
2161  * was detected.
2162  */
2163 static void i915_reset_and_wakeup(struct drm_device *dev)
2164 {
2165         struct drm_i915_private *dev_priv = to_i915(dev);
2166         struct i915_gpu_error *error = &dev_priv->gpu_error;
2167         char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
2168         char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
2169         char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2170         int ret;
2171
2172         kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
2173
2174         /*
2175          * Note that there's only one work item which does gpu resets, so we
2176          * need not worry about concurrent gpu resets potentially incrementing
2177          * error->reset_counter twice. We only need to take care of another
2178          * racing irq/hangcheck declaring the gpu dead for a second time. A
2179          * quick check for that is good enough: schedule_work ensures the
2180          * correct ordering between hang detection and this work item, and since
2181          * the reset in-progress bit is only ever set by code outside of this
2182          * work we don't need to worry about any other races.
2183          */
2184         if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
2185                 DRM_DEBUG_DRIVER("resetting chip\n");
2186                 kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
2187                                    reset_event);
2188
2189                 /*
2190                  * In most cases it's guaranteed that we get here with an RPM
2191                  * reference held, for example because there is a pending GPU
2192                  * request that won't finish until the reset is done. This
2193                  * isn't the case at least when we get here by doing a
2194                  * simulated reset via debugs, so get an RPM reference.
2195                  */
2196                 intel_runtime_pm_get(dev_priv);
2197
2198                 intel_prepare_reset(dev);
2199
2200                 /*
2201                  * All state reset _must_ be completed before we update the
2202                  * reset counter, for otherwise waiters might miss the reset
2203                  * pending state and not properly drop locks, resulting in
2204                  * deadlocks with the reset work.
2205                  */
2206                 ret = i915_reset(dev);
2207
2208                 intel_finish_reset(dev);
2209
2210                 intel_runtime_pm_put(dev_priv);
2211
2212                 if (ret == 0) {
2213                         /*
2214                          * After all the gem state is reset, increment the reset
2215                          * counter and wake up everyone waiting for the reset to
2216                          * complete.
2217                          *
2218                          * Since unlock operations are a one-sided barrier only,
2219                          * we need to insert a barrier here to order any seqno
2220                          * updates before
2221                          * the counter increment.
2222                          */
2223                         smp_mb__before_atomic();
2224                         atomic_inc(&dev_priv->gpu_error.reset_counter);
2225
2226                         kobject_uevent_env(&dev->primary->kdev->kobj,
2227                                            KOBJ_CHANGE, reset_done_event);
2228                 } else {
2229                         atomic_set_mask(I915_WEDGED, &error->reset_counter);
2230                 }
2231
2232                 /*
2233                  * Note: The wake_up also serves as a memory barrier so that
2234                  * waiters see the update value of the reset counter atomic_t.
2235                  */
2236                 i915_error_wake_up(dev_priv, true);
2237         }
2238 }
2239
2240 static void i915_report_and_clear_eir(struct drm_device *dev)
2241 {
2242         struct drm_i915_private *dev_priv = dev->dev_private;
2243         uint32_t instdone[I915_NUM_INSTDONE_REG];
2244         u32 eir = I915_READ(EIR);
2245         int pipe, i;
2246
2247         if (!eir)
2248                 return;
2249
2250         pr_err("render error detected, EIR: 0x%08x\n", eir);
2251
2252         i915_get_extra_instdone(dev, instdone);
2253
2254         if (IS_G4X(dev)) {
2255                 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
2256                         u32 ipeir = I915_READ(IPEIR_I965);
2257
2258                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2259                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2260                         for (i = 0; i < ARRAY_SIZE(instdone); i++)
2261                                 pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2262                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
2263                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2264                         I915_WRITE(IPEIR_I965, ipeir);
2265                         POSTING_READ(IPEIR_I965);
2266                 }
2267                 if (eir & GM45_ERROR_PAGE_TABLE) {
2268                         u32 pgtbl_err = I915_READ(PGTBL_ER);
2269                         pr_err("page table error\n");
2270                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2271                         I915_WRITE(PGTBL_ER, pgtbl_err);
2272                         POSTING_READ(PGTBL_ER);
2273                 }
2274         }
2275
2276         if (!IS_GEN2(dev)) {
2277                 if (eir & I915_ERROR_PAGE_TABLE) {
2278                         u32 pgtbl_err = I915_READ(PGTBL_ER);
2279                         pr_err("page table error\n");
2280                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2281                         I915_WRITE(PGTBL_ER, pgtbl_err);
2282                         POSTING_READ(PGTBL_ER);
2283                 }
2284         }
2285
2286         if (eir & I915_ERROR_MEMORY_REFRESH) {
2287                 pr_err("memory refresh error:\n");
2288                 for_each_pipe(dev_priv, pipe)
2289                         pr_err("pipe %c stat: 0x%08x\n",
2290                                pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
2291                 /* pipestat has already been acked */
2292         }
2293         if (eir & I915_ERROR_INSTRUCTION) {
2294                 pr_err("instruction error\n");
2295                 pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
2296                 for (i = 0; i < ARRAY_SIZE(instdone); i++)
2297                         pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2298                 if (INTEL_INFO(dev)->gen < 4) {
2299                         u32 ipeir = I915_READ(IPEIR);
2300
2301                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
2302                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
2303                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
2304                         I915_WRITE(IPEIR, ipeir);
2305                         POSTING_READ(IPEIR);
2306                 } else {
2307                         u32 ipeir = I915_READ(IPEIR_I965);
2308
2309                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2310                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2311                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
2312                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2313                         I915_WRITE(IPEIR_I965, ipeir);
2314                         POSTING_READ(IPEIR_I965);
2315                 }
2316         }
2317
2318         I915_WRITE(EIR, eir);
2319         POSTING_READ(EIR);
2320         eir = I915_READ(EIR);
2321         if (eir) {
2322                 /*
2323                  * some errors might have become stuck,
2324                  * mask them.
2325                  */
2326                 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
2327                 I915_WRITE(EMR, I915_READ(EMR) | eir);
2328                 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2329         }
2330 }
2331
2332 /**
2333  * i915_handle_error - handle a gpu error
2334  * @dev: drm device
2335  *
2336  * Do some basic checking of regsiter state at error time and
2337  * dump it to the syslog.  Also call i915_capture_error_state() to make
2338  * sure we get a record and make it available in debugfs.  Fire a uevent
2339  * so userspace knows something bad happened (should trigger collection
2340  * of a ring dump etc.).
2341  */
2342 void i915_handle_error(struct drm_device *dev, bool wedged,
2343                        const char *fmt, ...)
2344 {
2345         struct drm_i915_private *dev_priv = dev->dev_private;
2346         va_list args;
2347         char error_msg[80];
2348
2349         va_start(args, fmt);
2350         vscnprintf(error_msg, sizeof(error_msg), fmt, args);
2351         va_end(args);
2352
2353         i915_capture_error_state(dev, wedged, error_msg);
2354         i915_report_and_clear_eir(dev);
2355
2356         if (wedged) {
2357                 atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
2358                                 &dev_priv->gpu_error.reset_counter);
2359
2360                 /*
2361                  * Wakeup waiting processes so that the reset function
2362                  * i915_reset_and_wakeup doesn't deadlock trying to grab
2363                  * various locks. By bumping the reset counter first, the woken
2364                  * processes will see a reset in progress and back off,
2365                  * releasing their locks and then wait for the reset completion.
2366                  * We must do this for _all_ gpu waiters that might hold locks
2367                  * that the reset work needs to acquire.
2368                  *
2369                  * Note: The wake_up serves as the required memory barrier to
2370                  * ensure that the waiters see the updated value of the reset
2371                  * counter atomic_t.
2372                  */
2373                 i915_error_wake_up(dev_priv, false);
2374         }
2375
2376         i915_reset_and_wakeup(dev);
2377 }
2378
2379 /* Called from drm generic code, passed 'crtc' which
2380  * we use as a pipe index
2381  */
2382 static int i915_enable_vblank(struct drm_device *dev, int pipe)
2383 {
2384         struct drm_i915_private *dev_priv = dev->dev_private;
2385         unsigned long irqflags;
2386
2387         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2388         if (INTEL_INFO(dev)->gen >= 4)
2389                 i915_enable_pipestat(dev_priv, pipe,
2390                                      PIPE_START_VBLANK_INTERRUPT_STATUS);
2391         else
2392                 i915_enable_pipestat(dev_priv, pipe,
2393                                      PIPE_VBLANK_INTERRUPT_STATUS);
2394         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2395
2396         return 0;
2397 }
2398
2399 static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
2400 {
2401         struct drm_i915_private *dev_priv = dev->dev_private;
2402         unsigned long irqflags;
2403         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2404                                                      DE_PIPE_VBLANK(pipe);
2405
2406         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2407         ironlake_enable_display_irq(dev_priv, bit);
2408         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2409
2410         return 0;
2411 }
2412
2413 static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
2414 {
2415         struct drm_i915_private *dev_priv = dev->dev_private;
2416         unsigned long irqflags;
2417
2418         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2419         i915_enable_pipestat(dev_priv, pipe,
2420                              PIPE_START_VBLANK_INTERRUPT_STATUS);
2421         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2422
2423         return 0;
2424 }
2425
2426 static int gen8_enable_vblank(struct drm_device *dev, int pipe)
2427 {
2428         struct drm_i915_private *dev_priv = dev->dev_private;
2429         unsigned long irqflags;
2430
2431         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2432         dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
2433         I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2434         POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2435         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2436         return 0;
2437 }
2438
2439 /* Called from drm generic code, passed 'crtc' which
2440  * we use as a pipe index
2441  */
2442 static void i915_disable_vblank(struct drm_device *dev, int pipe)
2443 {
2444         struct drm_i915_private *dev_priv = dev->dev_private;
2445         unsigned long irqflags;
2446
2447         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2448         i915_disable_pipestat(dev_priv, pipe,
2449                               PIPE_VBLANK_INTERRUPT_STATUS |
2450                               PIPE_START_VBLANK_INTERRUPT_STATUS);
2451         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2452 }
2453
2454 static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
2455 {
2456         struct drm_i915_private *dev_priv = dev->dev_private;
2457         unsigned long irqflags;
2458         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2459                                                      DE_PIPE_VBLANK(pipe);
2460
2461         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2462         ironlake_disable_display_irq(dev_priv, bit);
2463         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2464 }
2465
2466 static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
2467 {
2468         struct drm_i915_private *dev_priv = dev->dev_private;
2469         unsigned long irqflags;
2470
2471         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2472         i915_disable_pipestat(dev_priv, pipe,
2473                               PIPE_START_VBLANK_INTERRUPT_STATUS);
2474         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2475 }
2476
2477 static void gen8_disable_vblank(struct drm_device *dev, int pipe)
2478 {
2479         struct drm_i915_private *dev_priv = dev->dev_private;
2480         unsigned long irqflags;
2481
2482         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2483         dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
2484         I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2485         POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2486         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2487 }
2488
2489 static struct drm_i915_gem_request *
2490 ring_last_request(struct intel_engine_cs *ring)
2491 {
2492         return list_entry(ring->request_list.prev,
2493                           struct drm_i915_gem_request, list);
2494 }
2495
2496 static bool
2497 ring_idle(struct intel_engine_cs *ring)
2498 {
2499         return (list_empty(&ring->request_list) ||
2500                 i915_gem_request_completed(ring_last_request(ring), false));
2501 }
2502
2503 static bool
2504 ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
2505 {
2506         if (INTEL_INFO(dev)->gen >= 8) {
2507                 return (ipehr >> 23) == 0x1c;
2508         } else {
2509                 ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
2510                 return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
2511                                  MI_SEMAPHORE_REGISTER);
2512         }
2513 }
2514
2515 static struct intel_engine_cs *
2516 semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
2517 {
2518         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2519         struct intel_engine_cs *signaller;
2520         int i;
2521
2522         if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
2523                 for_each_ring(signaller, dev_priv, i) {
2524                         if (ring == signaller)
2525                                 continue;
2526
2527                         if (offset == signaller->semaphore.signal_ggtt[ring->id])
2528                                 return signaller;
2529                 }
2530         } else {
2531                 u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
2532
2533                 for_each_ring(signaller, dev_priv, i) {
2534                         if(ring == signaller)
2535                                 continue;
2536
2537                         if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
2538                                 return signaller;
2539                 }
2540         }
2541
2542         DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
2543                   ring->id, ipehr, offset);
2544
2545         return NULL;
2546 }
2547
2548 static struct intel_engine_cs *
2549 semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
2550 {
2551         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2552         u32 cmd, ipehr, head;
2553         u64 offset = 0;
2554         int i, backwards;
2555
2556         ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
2557         if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
2558                 return NULL;
2559
2560         /*
2561          * HEAD is likely pointing to the dword after the actual command,
2562          * so scan backwards until we find the MBOX. But limit it to just 3
2563          * or 4 dwords depending on the semaphore wait command size.
2564          * Note that we don't care about ACTHD here since that might
2565          * point at at batch, and semaphores are always emitted into the
2566          * ringbuffer itself.
2567          */
2568         head = I915_READ_HEAD(ring) & HEAD_ADDR;
2569         backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
2570
2571         for (i = backwards; i; --i) {
2572                 /*
2573                  * Be paranoid and presume the hw has gone off into the wild -
2574                  * our ring is smaller than what the hardware (and hence
2575                  * HEAD_ADDR) allows. Also handles wrap-around.
2576                  */
2577                 head &= ring->buffer->size - 1;
2578
2579                 /* This here seems to blow up */
2580                 cmd = ioread32(ring->buffer->virtual_start + head);
2581                 if (cmd == ipehr)
2582                         break;
2583
2584                 head -= 4;
2585         }
2586
2587         if (!i)
2588                 return NULL;
2589
2590         *seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
2591         if (INTEL_INFO(ring->dev)->gen >= 8) {
2592                 offset = ioread32(ring->buffer->virtual_start + head + 12);
2593                 offset <<= 32;
2594                 offset = ioread32(ring->buffer->virtual_start + head + 8);
2595         }
2596         return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
2597 }
2598
2599 static int semaphore_passed(struct intel_engine_cs *ring)
2600 {
2601         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2602         struct intel_engine_cs *signaller;
2603         u32 seqno;
2604
2605         ring->hangcheck.deadlock++;
2606
2607         signaller = semaphore_waits_for(ring, &seqno);
2608         if (signaller == NULL)
2609                 return -1;
2610
2611         /* Prevent pathological recursion due to driver bugs */
2612         if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
2613                 return -1;
2614
2615         if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
2616                 return 1;
2617
2618         /* cursory check for an unkickable deadlock */
2619         if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
2620             semaphore_passed(signaller) < 0)
2621                 return -1;
2622
2623         return 0;
2624 }
2625
2626 static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
2627 {
2628         struct intel_engine_cs *ring;
2629         int i;
2630
2631         for_each_ring(ring, dev_priv, i)
2632                 ring->hangcheck.deadlock = 0;
2633 }
2634
2635 static enum intel_ring_hangcheck_action
2636 ring_stuck(struct intel_engine_cs *ring, u64 acthd)
2637 {
2638         struct drm_device *dev = ring->dev;
2639         struct drm_i915_private *dev_priv = dev->dev_private;
2640         u32 tmp;
2641
2642         if (acthd != ring->hangcheck.acthd) {
2643                 if (acthd > ring->hangcheck.max_acthd) {
2644                         ring->hangcheck.max_acthd = acthd;
2645                         return HANGCHECK_ACTIVE;
2646                 }
2647
2648                 return HANGCHECK_ACTIVE_LOOP;
2649         }
2650
2651         if (IS_GEN2(dev))
2652                 return HANGCHECK_HUNG;
2653
2654         /* Is the chip hanging on a WAIT_FOR_EVENT?
2655          * If so we can simply poke the RB_WAIT bit
2656          * and break the hang. This should work on
2657          * all but the second generation chipsets.
2658          */
2659         tmp = I915_READ_CTL(ring);
2660         if (tmp & RING_WAIT) {
2661                 i915_handle_error(dev, false,
2662                                   "Kicking stuck wait on %s",
2663                                   ring->name);
2664                 I915_WRITE_CTL(ring, tmp);
2665                 return HANGCHECK_KICK;
2666         }
2667
2668         if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
2669                 switch (semaphore_passed(ring)) {
2670                 default:
2671                         return HANGCHECK_HUNG;
2672                 case 1:
2673                         i915_handle_error(dev, false,
2674                                           "Kicking stuck semaphore on %s",
2675                                           ring->name);
2676                         I915_WRITE_CTL(ring, tmp);
2677                         return HANGCHECK_KICK;
2678                 case 0:
2679                         return HANGCHECK_WAIT;
2680                 }
2681         }
2682
2683         return HANGCHECK_HUNG;
2684 }
2685
2686 /*
2687  * This is called when the chip hasn't reported back with completed
2688  * batchbuffers in a long time. We keep track per ring seqno progress and
2689  * if there are no progress, hangcheck score for that ring is increased.
2690  * Further, acthd is inspected to see if the ring is stuck. On stuck case
2691  * we kick the ring. If we see no progress on three subsequent calls
2692  * we assume chip is wedged and try to fix it by resetting the chip.
2693  */
2694 static void i915_hangcheck_elapsed(struct work_struct *work)
2695 {
2696         struct drm_i915_private *dev_priv =
2697                 container_of(work, typeof(*dev_priv),
2698                              gpu_error.hangcheck_work.work);
2699         struct drm_device *dev = dev_priv->dev;
2700         struct intel_engine_cs *ring;
2701         int i;
2702         int busy_count = 0, rings_hung = 0;
2703         bool stuck[I915_NUM_RINGS] = { 0 };
2704 #define BUSY 1
2705 #define KICK 5
2706 #define HUNG 20
2707
2708         if (!i915.enable_hangcheck)
2709                 return;
2710
2711         for_each_ring(ring, dev_priv, i) {
2712                 u64 acthd;
2713                 u32 seqno;
2714                 bool busy = true;
2715
2716                 semaphore_clear_deadlocks(dev_priv);
2717
2718                 seqno = ring->get_seqno(ring, false);
2719                 acthd = intel_ring_get_active_head(ring);
2720
2721                 if (ring->hangcheck.seqno == seqno) {
2722                         if (ring_idle(ring)) {
2723                                 ring->hangcheck.action = HANGCHECK_IDLE;
2724
2725                                 if (waitqueue_active(&ring->irq_queue)) {
2726                                         /* Issue a wake-up to catch stuck h/w. */
2727                                         if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
2728                                                 if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
2729                                                         DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
2730                                                                   ring->name);
2731                                                 else
2732                                                         DRM_INFO("Fake missed irq on %s\n",
2733                                                                  ring->name);
2734                                                 wake_up_all(&ring->irq_queue);
2735                                         }
2736                                         /* Safeguard against driver failure */
2737                                         ring->hangcheck.score += BUSY;
2738                                 } else
2739                                         busy = false;
2740                         } else {
2741                                 /* We always increment the hangcheck score
2742                                  * if the ring is busy and still processing
2743                                  * the same request, so that no single request
2744                                  * can run indefinitely (such as a chain of
2745                                  * batches). The only time we do not increment
2746                                  * the hangcheck score on this ring, if this
2747                                  * ring is in a legitimate wait for another
2748                                  * ring. In that case the waiting ring is a
2749                                  * victim and we want to be sure we catch the
2750                                  * right culprit. Then every time we do kick
2751                                  * the ring, add a small increment to the
2752                                  * score so that we can catch a batch that is
2753                                  * being repeatedly kicked and so responsible
2754                                  * for stalling the machine.
2755                                  */
2756                                 ring->hangcheck.action = ring_stuck(ring,
2757                                                                     acthd);
2758
2759                                 switch (ring->hangcheck.action) {
2760                                 case HANGCHECK_IDLE:
2761                                 case HANGCHECK_WAIT:
2762                                 case HANGCHECK_ACTIVE:
2763                                         break;
2764                                 case HANGCHECK_ACTIVE_LOOP:
2765                                         ring->hangcheck.score += BUSY;
2766                                         break;
2767                                 case HANGCHECK_KICK:
2768                                         ring->hangcheck.score += KICK;
2769                                         break;
2770                                 case HANGCHECK_HUNG:
2771                                         ring->hangcheck.score += HUNG;
2772                                         stuck[i] = true;
2773                                         break;
2774                                 }
2775                         }
2776                 } else {
2777                         ring->hangcheck.action = HANGCHECK_ACTIVE;
2778
2779                         /* Gradually reduce the count so that we catch DoS
2780                          * attempts across multiple batches.
2781                          */
2782                         if (ring->hangcheck.score > 0)
2783                                 ring->hangcheck.score--;
2784
2785                         ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
2786                 }
2787
2788                 ring->hangcheck.seqno = seqno;
2789                 ring->hangcheck.acthd = acthd;
2790                 busy_count += busy;
2791         }
2792
2793         for_each_ring(ring, dev_priv, i) {
2794                 if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
2795                         DRM_INFO("%s on %s\n",
2796                                  stuck[i] ? "stuck" : "no progress",
2797                                  ring->name);
2798                         rings_hung++;
2799                 }
2800         }
2801
2802         if (rings_hung)
2803                 return i915_handle_error(dev, true, "Ring hung");
2804
2805         if (busy_count)
2806                 /* Reset timer case chip hangs without another request
2807                  * being added */
2808                 i915_queue_hangcheck(dev);
2809 }
2810
2811 void i915_queue_hangcheck(struct drm_device *dev)
2812 {
2813         struct i915_gpu_error *e = &to_i915(dev)->gpu_error;
2814
2815         if (!i915.enable_hangcheck)
2816                 return;
2817
2818         /* Don't continually defer the hangcheck so that it is always run at
2819          * least once after work has been scheduled on any ring. Otherwise,
2820          * we will ignore a hung ring if a second ring is kept busy.
2821          */
2822
2823         queue_delayed_work(e->hangcheck_wq, &e->hangcheck_work,
2824                            round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES));
2825 }
2826
2827 static void ibx_irq_reset(struct drm_device *dev)
2828 {
2829         struct drm_i915_private *dev_priv = dev->dev_private;
2830
2831         if (HAS_PCH_NOP(dev))
2832                 return;
2833
2834         GEN5_IRQ_RESET(SDE);
2835
2836         if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
2837                 I915_WRITE(SERR_INT, 0xffffffff);
2838 }
2839
2840 /*
2841  * SDEIER is also touched by the interrupt handler to work around missed PCH
2842  * interrupts. Hence we can't update it after the interrupt handler is enabled -
2843  * instead we unconditionally enable all PCH interrupt sources here, but then
2844  * only unmask them as needed with SDEIMR.
2845  *
2846  * This function needs to be called before interrupts are enabled.
2847  */
2848 static void ibx_irq_pre_postinstall(struct drm_device *dev)
2849 {
2850         struct drm_i915_private *dev_priv = dev->dev_private;
2851
2852         if (HAS_PCH_NOP(dev))
2853                 return;
2854
2855         WARN_ON(I915_READ(SDEIER) != 0);
2856         I915_WRITE(SDEIER, 0xffffffff);
2857         POSTING_READ(SDEIER);
2858 }
2859
2860 static void gen5_gt_irq_reset(struct drm_device *dev)
2861 {
2862         struct drm_i915_private *dev_priv = dev->dev_private;
2863
2864         GEN5_IRQ_RESET(GT);
2865         if (INTEL_INFO(dev)->gen >= 6)
2866                 GEN5_IRQ_RESET(GEN6_PM);
2867 }
2868
2869 /* drm_dma.h hooks
2870 */
2871 static void ironlake_irq_reset(struct drm_device *dev)
2872 {
2873         struct drm_i915_private *dev_priv = dev->dev_private;
2874
2875         I915_WRITE(HWSTAM, 0xffffffff);
2876
2877         GEN5_IRQ_RESET(DE);
2878         if (IS_GEN7(dev))
2879                 I915_WRITE(GEN7_ERR_INT, 0xffffffff);
2880
2881         gen5_gt_irq_reset(dev);
2882
2883         ibx_irq_reset(dev);
2884 }
2885
2886 static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
2887 {
2888         enum pipe pipe;
2889
2890         I915_WRITE(PORT_HOTPLUG_EN, 0);
2891         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2892
2893         for_each_pipe(dev_priv, pipe)
2894                 I915_WRITE(PIPESTAT(pipe), 0xffff);
2895
2896         GEN5_IRQ_RESET(VLV_);
2897 }
2898
2899 static void valleyview_irq_preinstall(struct drm_device *dev)
2900 {
2901         struct drm_i915_private *dev_priv = dev->dev_private;
2902
2903         /* VLV magic */
2904         I915_WRITE(VLV_IMR, 0);
2905         I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
2906         I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
2907         I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
2908
2909         gen5_gt_irq_reset(dev);
2910
2911         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
2912
2913         vlv_display_irq_reset(dev_priv);
2914 }
2915
2916 static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
2917 {
2918         GEN8_IRQ_RESET_NDX(GT, 0);
2919         GEN8_IRQ_RESET_NDX(GT, 1);
2920         GEN8_IRQ_RESET_NDX(GT, 2);
2921         GEN8_IRQ_RESET_NDX(GT, 3);
2922 }
2923
2924 static void gen8_irq_reset(struct drm_device *dev)
2925 {
2926         struct drm_i915_private *dev_priv = dev->dev_private;
2927         int pipe;
2928
2929         I915_WRITE(GEN8_MASTER_IRQ, 0);
2930         POSTING_READ(GEN8_MASTER_IRQ);
2931
2932         gen8_gt_irq_reset(dev_priv);
2933
2934         for_each_pipe(dev_priv, pipe)
2935                 if (intel_display_power_is_enabled(dev_priv,
2936                                                    POWER_DOMAIN_PIPE(pipe)))
2937                         GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
2938
2939         GEN5_IRQ_RESET(GEN8_DE_PORT_);
2940         GEN5_IRQ_RESET(GEN8_DE_MISC_);
2941         GEN5_IRQ_RESET(GEN8_PCU_);
2942
2943         if (HAS_PCH_SPLIT(dev))
2944                 ibx_irq_reset(dev);
2945 }
2946
2947 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
2948                                      unsigned int pipe_mask)
2949 {
2950         uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
2951
2952         spin_lock_irq(&dev_priv->irq_lock);
2953         if (pipe_mask & 1 << PIPE_A)
2954                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_A,
2955                                   dev_priv->de_irq_mask[PIPE_A],
2956                                   ~dev_priv->de_irq_mask[PIPE_A] | extra_ier);
2957         if (pipe_mask & 1 << PIPE_B)
2958                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B,
2959                                   dev_priv->de_irq_mask[PIPE_B],
2960                                   ~dev_priv->de_irq_mask[PIPE_B] | extra_ier);
2961         if (pipe_mask & 1 << PIPE_C)
2962                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C,
2963                                   dev_priv->de_irq_mask[PIPE_C],
2964                                   ~dev_priv->de_irq_mask[PIPE_C] | extra_ier);
2965         spin_unlock_irq(&dev_priv->irq_lock);
2966 }
2967
2968 static void cherryview_irq_preinstall(struct drm_device *dev)
2969 {
2970         struct drm_i915_private *dev_priv = dev->dev_private;
2971
2972         I915_WRITE(GEN8_MASTER_IRQ, 0);
2973         POSTING_READ(GEN8_MASTER_IRQ);
2974
2975         gen8_gt_irq_reset(dev_priv);
2976
2977         GEN5_IRQ_RESET(GEN8_PCU_);
2978
2979         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
2980
2981         vlv_display_irq_reset(dev_priv);
2982 }
2983
2984 static void ibx_hpd_irq_setup(struct drm_device *dev)
2985 {
2986         struct drm_i915_private *dev_priv = dev->dev_private;
2987         struct intel_encoder *intel_encoder;
2988         u32 hotplug_irqs, hotplug, enabled_irqs = 0;
2989
2990         if (HAS_PCH_IBX(dev)) {
2991                 hotplug_irqs = SDE_HOTPLUG_MASK;
2992                 for_each_intel_encoder(dev, intel_encoder)
2993                         if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED)
2994                                 enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
2995         } else {
2996                 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
2997                 for_each_intel_encoder(dev, intel_encoder)
2998                         if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED)
2999                                 enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
3000         }
3001
3002         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3003
3004         /*
3005          * Enable digital hotplug on the PCH, and configure the DP short pulse
3006          * duration to 2ms (which is the minimum in the Display Port spec)
3007          *
3008          * This register is the same on all known PCH chips.
3009          */
3010         hotplug = I915_READ(PCH_PORT_HOTPLUG);
3011         hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
3012         hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
3013         hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
3014         hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3015         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3016 }
3017
3018 static void bxt_hpd_irq_setup(struct drm_device *dev)
3019 {
3020         struct drm_i915_private *dev_priv = dev->dev_private;
3021         struct intel_encoder *intel_encoder;
3022         u32 hotplug_port = 0;
3023         u32 hotplug_ctrl;
3024
3025         /* Now, enable HPD */
3026         for_each_intel_encoder(dev, intel_encoder) {
3027                 if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state
3028                                 == HPD_ENABLED)
3029                         hotplug_port |= hpd_bxt[intel_encoder->hpd_pin];
3030         }
3031
3032         /* Mask all HPD control bits */
3033         hotplug_ctrl = I915_READ(BXT_HOTPLUG_CTL) & ~BXT_HOTPLUG_CTL_MASK;
3034
3035         /* Enable requested port in hotplug control */
3036         /* TODO: implement (short) HPD support on port A */
3037         WARN_ON_ONCE(hotplug_port & BXT_DE_PORT_HP_DDIA);
3038         if (hotplug_port & BXT_DE_PORT_HP_DDIB)
3039                 hotplug_ctrl |= BXT_DDIB_HPD_ENABLE;
3040         if (hotplug_port & BXT_DE_PORT_HP_DDIC)
3041                 hotplug_ctrl |= BXT_DDIC_HPD_ENABLE;
3042         I915_WRITE(BXT_HOTPLUG_CTL, hotplug_ctrl);
3043
3044         /* Unmask DDI hotplug in IMR */
3045         hotplug_ctrl = I915_READ(GEN8_DE_PORT_IMR) & ~hotplug_port;
3046         I915_WRITE(GEN8_DE_PORT_IMR, hotplug_ctrl);
3047
3048         /* Enable DDI hotplug in IER */
3049         hotplug_ctrl = I915_READ(GEN8_DE_PORT_IER) | hotplug_port;
3050         I915_WRITE(GEN8_DE_PORT_IER, hotplug_ctrl);
3051         POSTING_READ(GEN8_DE_PORT_IER);
3052 }
3053
3054 static void ibx_irq_postinstall(struct drm_device *dev)
3055 {
3056         struct drm_i915_private *dev_priv = dev->dev_private;
3057         u32 mask;
3058
3059         if (HAS_PCH_NOP(dev))
3060                 return;
3061
3062         if (HAS_PCH_IBX(dev))
3063                 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3064         else
3065                 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3066
3067         GEN5_ASSERT_IIR_IS_ZERO(SDEIIR);
3068         I915_WRITE(SDEIMR, ~mask);
3069 }
3070
3071 static void gen5_gt_irq_postinstall(struct drm_device *dev)
3072 {
3073         struct drm_i915_private *dev_priv = dev->dev_private;
3074         u32 pm_irqs, gt_irqs;
3075
3076         pm_irqs = gt_irqs = 0;
3077
3078         dev_priv->gt_irq_mask = ~0;
3079         if (HAS_L3_DPF(dev)) {
3080                 /* L3 parity interrupt is always unmasked. */
3081                 dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
3082                 gt_irqs |= GT_PARITY_ERROR(dev);
3083         }
3084
3085         gt_irqs |= GT_RENDER_USER_INTERRUPT;
3086         if (IS_GEN5(dev)) {
3087                 gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
3088                            ILK_BSD_USER_INTERRUPT;
3089         } else {
3090                 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
3091         }
3092
3093         GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3094
3095         if (INTEL_INFO(dev)->gen >= 6) {
3096                 /*
3097                  * RPS interrupts will get enabled/disabled on demand when RPS
3098                  * itself is enabled/disabled.
3099                  */
3100                 if (HAS_VEBOX(dev))
3101                         pm_irqs |= PM_VEBOX_USER_INTERRUPT;
3102
3103                 dev_priv->pm_irq_mask = 0xffffffff;
3104                 GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
3105         }
3106 }
3107
3108 static int ironlake_irq_postinstall(struct drm_device *dev)
3109 {
3110         struct drm_i915_private *dev_priv = dev->dev_private;
3111         u32 display_mask, extra_mask;
3112
3113         if (INTEL_INFO(dev)->gen >= 7) {
3114                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3115                                 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
3116                                 DE_PLANEB_FLIP_DONE_IVB |
3117                                 DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3118                 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3119                               DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB);
3120         } else {
3121                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3122                                 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3123                                 DE_AUX_CHANNEL_A |
3124                                 DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
3125                                 DE_POISON);
3126                 extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
3127                                 DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN;
3128         }
3129
3130         dev_priv->irq_mask = ~display_mask;
3131
3132         I915_WRITE(HWSTAM, 0xeffe);
3133
3134         ibx_irq_pre_postinstall(dev);
3135
3136         GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3137
3138         gen5_gt_irq_postinstall(dev);
3139
3140         ibx_irq_postinstall(dev);
3141
3142         if (IS_IRONLAKE_M(dev)) {
3143                 /* Enable PCU event interrupts
3144                  *
3145                  * spinlocking not required here for correctness since interrupt
3146                  * setup is guaranteed to run in single-threaded context. But we
3147                  * need it to make the assert_spin_locked happy. */
3148                 spin_lock_irq(&dev_priv->irq_lock);
3149                 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
3150                 spin_unlock_irq(&dev_priv->irq_lock);
3151         }
3152
3153         return 0;
3154 }
3155
3156 static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
3157 {
3158         u32 pipestat_mask;
3159         u32 iir_mask;
3160         enum pipe pipe;
3161
3162         pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3163                         PIPE_FIFO_UNDERRUN_STATUS;
3164
3165         for_each_pipe(dev_priv, pipe)
3166                 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3167         POSTING_READ(PIPESTAT(PIPE_A));
3168
3169         pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3170                         PIPE_CRC_DONE_INTERRUPT_STATUS;
3171
3172         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3173         for_each_pipe(dev_priv, pipe)
3174                       i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
3175
3176         iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3177                    I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3178                    I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3179         if (IS_CHERRYVIEW(dev_priv))
3180                 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3181         dev_priv->irq_mask &= ~iir_mask;
3182
3183         I915_WRITE(VLV_IIR, iir_mask);
3184         I915_WRITE(VLV_IIR, iir_mask);
3185         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3186         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3187         POSTING_READ(VLV_IMR);
3188 }
3189
3190 static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
3191 {
3192         u32 pipestat_mask;
3193         u32 iir_mask;
3194         enum pipe pipe;
3195
3196         iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3197                    I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3198                    I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3199         if (IS_CHERRYVIEW(dev_priv))
3200                 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3201
3202         dev_priv->irq_mask |= iir_mask;
3203         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3204         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3205         I915_WRITE(VLV_IIR, iir_mask);
3206         I915_WRITE(VLV_IIR, iir_mask);
3207         POSTING_READ(VLV_IIR);
3208
3209         pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3210                         PIPE_CRC_DONE_INTERRUPT_STATUS;
3211
3212         i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3213         for_each_pipe(dev_priv, pipe)
3214                 i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
3215
3216         pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3217                         PIPE_FIFO_UNDERRUN_STATUS;
3218
3219         for_each_pipe(dev_priv, pipe)
3220                 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3221         POSTING_READ(PIPESTAT(PIPE_A));
3222 }
3223
3224 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3225 {
3226         assert_spin_locked(&dev_priv->irq_lock);
3227
3228         if (dev_priv->display_irqs_enabled)
3229                 return;
3230
3231         dev_priv->display_irqs_enabled = true;
3232
3233         if (intel_irqs_enabled(dev_priv))
3234                 valleyview_display_irqs_install(dev_priv);
3235 }
3236
3237 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3238 {
3239         assert_spin_locked(&dev_priv->irq_lock);
3240
3241         if (!dev_priv->display_irqs_enabled)
3242                 return;
3243
3244         dev_priv->display_irqs_enabled = false;
3245
3246         if (intel_irqs_enabled(dev_priv))
3247                 valleyview_display_irqs_uninstall(dev_priv);
3248 }
3249
3250 static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
3251 {
3252         dev_priv->irq_mask = ~0;
3253
3254         I915_WRITE(PORT_HOTPLUG_EN, 0);
3255         POSTING_READ(PORT_HOTPLUG_EN);
3256
3257         I915_WRITE(VLV_IIR, 0xffffffff);
3258         I915_WRITE(VLV_IIR, 0xffffffff);
3259         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3260         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3261         POSTING_READ(VLV_IMR);
3262
3263         /* Interrupt setup is already guaranteed to be single-threaded, this is
3264          * just to make the assert_spin_locked check happy. */
3265         spin_lock_irq(&dev_priv->irq_lock);
3266         if (dev_priv->display_irqs_enabled)
3267                 valleyview_display_irqs_install(dev_priv);
3268         spin_unlock_irq(&dev_priv->irq_lock);
3269 }
3270
3271 static int valleyview_irq_postinstall(struct drm_device *dev)
3272 {
3273         struct drm_i915_private *dev_priv = dev->dev_private;
3274
3275         vlv_display_irq_postinstall(dev_priv);
3276
3277         gen5_gt_irq_postinstall(dev);
3278
3279         /* ack & enable invalid PTE error interrupts */
3280 #if 0 /* FIXME: add support to irq handler for checking these bits */
3281         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3282         I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
3283 #endif
3284
3285         I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3286
3287         return 0;
3288 }
3289
3290 static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
3291 {
3292         /* These are interrupts we'll toggle with the ring mask register */
3293         uint32_t gt_interrupts[] = {
3294                 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3295                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3296                         GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
3297                         GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
3298                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3299                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3300                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3301                         GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
3302                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3303                 0,
3304                 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
3305                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3306                 };
3307
3308         dev_priv->pm_irq_mask = 0xffffffff;
3309         GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
3310         GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3311         /*
3312          * RPS interrupts will get enabled/disabled on demand when RPS itself
3313          * is enabled/disabled.
3314          */
3315         GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
3316         GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3317 }
3318
3319 static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3320 {
3321         uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
3322         uint32_t de_pipe_enables;
3323         int pipe;
3324         u32 de_port_en = GEN8_AUX_CHANNEL_A;
3325
3326         if (IS_GEN9(dev_priv)) {
3327                 de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
3328                                   GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3329                 de_port_en |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
3330                         GEN9_AUX_CHANNEL_D;
3331
3332                 if (IS_BROXTON(dev_priv))
3333                         de_port_en |= BXT_DE_PORT_GMBUS;
3334         } else
3335                 de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
3336                                   GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3337
3338         de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
3339                                            GEN8_PIPE_FIFO_UNDERRUN;
3340
3341         dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
3342         dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
3343         dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3344
3345         for_each_pipe(dev_priv, pipe)
3346                 if (intel_display_power_is_enabled(dev_priv,
3347                                 POWER_DOMAIN_PIPE(pipe)))
3348                         GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
3349                                           dev_priv->de_irq_mask[pipe],
3350                                           de_pipe_enables);
3351
3352         GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_en, de_port_en);
3353 }
3354
3355 static int gen8_irq_postinstall(struct drm_device *dev)
3356 {
3357         struct drm_i915_private *dev_priv = dev->dev_private;
3358
3359         if (HAS_PCH_SPLIT(dev))
3360                 ibx_irq_pre_postinstall(dev);
3361
3362         gen8_gt_irq_postinstall(dev_priv);
3363         gen8_de_irq_postinstall(dev_priv);
3364
3365         if (HAS_PCH_SPLIT(dev))
3366                 ibx_irq_postinstall(dev);
3367
3368         I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
3369         POSTING_READ(GEN8_MASTER_IRQ);
3370
3371         return 0;
3372 }
3373
3374 static int cherryview_irq_postinstall(struct drm_device *dev)
3375 {
3376         struct drm_i915_private *dev_priv = dev->dev_private;
3377
3378         vlv_display_irq_postinstall(dev_priv);
3379
3380         gen8_gt_irq_postinstall(dev_priv);
3381
3382         I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
3383         POSTING_READ(GEN8_MASTER_IRQ);
3384
3385         return 0;
3386 }
3387
3388 static void gen8_irq_uninstall(struct drm_device *dev)
3389 {
3390         struct drm_i915_private *dev_priv = dev->dev_private;
3391
3392         if (!dev_priv)
3393                 return;
3394
3395         gen8_irq_reset(dev);
3396 }
3397
3398 static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
3399 {
3400         /* Interrupt setup is already guaranteed to be single-threaded, this is
3401          * just to make the assert_spin_locked check happy. */
3402         spin_lock_irq(&dev_priv->irq_lock);
3403         if (dev_priv->display_irqs_enabled)
3404                 valleyview_display_irqs_uninstall(dev_priv);
3405         spin_unlock_irq(&dev_priv->irq_lock);
3406
3407         vlv_display_irq_reset(dev_priv);
3408
3409         dev_priv->irq_mask = ~0;
3410 }
3411
3412 static void valleyview_irq_uninstall(struct drm_device *dev)
3413 {
3414         struct drm_i915_private *dev_priv = dev->dev_private;
3415
3416         if (!dev_priv)
3417                 return;
3418
3419         I915_WRITE(VLV_MASTER_IER, 0);
3420
3421         gen5_gt_irq_reset(dev);
3422
3423         I915_WRITE(HWSTAM, 0xffffffff);
3424
3425         vlv_display_irq_uninstall(dev_priv);
3426 }
3427
3428 static void cherryview_irq_uninstall(struct drm_device *dev)
3429 {
3430         struct drm_i915_private *dev_priv = dev->dev_private;
3431
3432         if (!dev_priv)
3433                 return;
3434
3435         I915_WRITE(GEN8_MASTER_IRQ, 0);
3436         POSTING_READ(GEN8_MASTER_IRQ);
3437
3438         gen8_gt_irq_reset(dev_priv);
3439
3440         GEN5_IRQ_RESET(GEN8_PCU_);
3441
3442         vlv_display_irq_uninstall(dev_priv);
3443 }
3444
3445 static void ironlake_irq_uninstall(struct drm_device *dev)
3446 {
3447         struct drm_i915_private *dev_priv = dev->dev_private;
3448
3449         if (!dev_priv)
3450                 return;
3451
3452         ironlake_irq_reset(dev);
3453 }
3454
3455 static void i8xx_irq_preinstall(struct drm_device * dev)
3456 {
3457         struct drm_i915_private *dev_priv = dev->dev_private;
3458         int pipe;
3459
3460         for_each_pipe(dev_priv, pipe)
3461                 I915_WRITE(PIPESTAT(pipe), 0);
3462         I915_WRITE16(IMR, 0xffff);
3463         I915_WRITE16(IER, 0x0);
3464         POSTING_READ16(IER);
3465 }
3466
3467 static int i8xx_irq_postinstall(struct drm_device *dev)
3468 {
3469         struct drm_i915_private *dev_priv = dev->dev_private;
3470
3471         I915_WRITE16(EMR,
3472                      ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3473
3474         /* Unmask the interrupts that we always want on. */
3475         dev_priv->irq_mask =
3476                 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3477                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3478                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3479                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3480         I915_WRITE16(IMR, dev_priv->irq_mask);
3481
3482         I915_WRITE16(IER,
3483                      I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3484                      I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3485                      I915_USER_INTERRUPT);
3486         POSTING_READ16(IER);
3487
3488         /* Interrupt setup is already guaranteed to be single-threaded, this is
3489          * just to make the assert_spin_locked check happy. */
3490         spin_lock_irq(&dev_priv->irq_lock);
3491         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3492         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3493         spin_unlock_irq(&dev_priv->irq_lock);
3494
3495         return 0;
3496 }
3497
3498 /*
3499  * Returns true when a page flip has completed.
3500  */
3501 static bool i8xx_handle_vblank(struct drm_device *dev,
3502                                int plane, int pipe, u32 iir)
3503 {
3504         struct drm_i915_private *dev_priv = dev->dev_private;
3505         u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3506
3507         if (!intel_pipe_handle_vblank(dev, pipe))
3508                 return false;
3509
3510         if ((iir & flip_pending) == 0)
3511                 goto check_page_flip;
3512
3513         /* We detect FlipDone by looking for the change in PendingFlip from '1'
3514          * to '0' on the following vblank, i.e. IIR has the Pendingflip
3515          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3516          * the flip is completed (no longer pending). Since this doesn't raise
3517          * an interrupt per se, we watch for the change at vblank.
3518          */
3519         if (I915_READ16(ISR) & flip_pending)
3520                 goto check_page_flip;
3521
3522         intel_prepare_page_flip(dev, plane);
3523         intel_finish_page_flip(dev, pipe);
3524         return true;
3525
3526 check_page_flip:
3527         intel_check_page_flip(dev, pipe);
3528         return false;
3529 }
3530
3531 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
3532 {
3533         struct drm_device *dev = arg;
3534         struct drm_i915_private *dev_priv = dev->dev_private;
3535         u16 iir, new_iir;
3536         u32 pipe_stats[2];
3537         int pipe;
3538         u16 flip_mask =
3539                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3540                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3541
3542         if (!intel_irqs_enabled(dev_priv))
3543                 return IRQ_NONE;
3544
3545         iir = I915_READ16(IIR);
3546         if (iir == 0)
3547                 return IRQ_NONE;
3548
3549         while (iir & ~flip_mask) {
3550                 /* Can't rely on pipestat interrupt bit in iir as it might
3551                  * have been cleared after the pipestat interrupt was received.
3552                  * It doesn't set the bit in iir again, but it still produces
3553                  * interrupts (for non-MSI).
3554                  */
3555                 spin_lock(&dev_priv->irq_lock);
3556                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3557                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3558
3559                 for_each_pipe(dev_priv, pipe) {
3560                         int reg = PIPESTAT(pipe);
3561                         pipe_stats[pipe] = I915_READ(reg);
3562
3563                         /*
3564                          * Clear the PIPE*STAT regs before the IIR
3565                          */
3566                         if (pipe_stats[pipe] & 0x8000ffff)
3567                                 I915_WRITE(reg, pipe_stats[pipe]);
3568                 }
3569                 spin_unlock(&dev_priv->irq_lock);
3570
3571                 I915_WRITE16(IIR, iir & ~flip_mask);
3572                 new_iir = I915_READ16(IIR); /* Flush posted writes */
3573
3574                 if (iir & I915_USER_INTERRUPT)
3575                         notify_ring(&dev_priv->ring[RCS]);
3576
3577                 for_each_pipe(dev_priv, pipe) {
3578                         int plane = pipe;
3579                         if (HAS_FBC(dev))
3580                                 plane = !plane;
3581
3582                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3583                             i8xx_handle_vblank(dev, plane, pipe, iir))
3584                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3585
3586                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3587                                 i9xx_pipe_crc_irq_handler(dev, pipe);
3588
3589                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3590                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
3591                                                                     pipe);
3592                 }
3593
3594                 iir = new_iir;
3595         }
3596
3597         return IRQ_HANDLED;
3598 }
3599
3600 static void i8xx_irq_uninstall(struct drm_device * dev)
3601 {
3602         struct drm_i915_private *dev_priv = dev->dev_private;
3603         int pipe;
3604
3605         for_each_pipe(dev_priv, pipe) {
3606                 /* Clear enable bits; then clear status bits */
3607                 I915_WRITE(PIPESTAT(pipe), 0);
3608                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3609         }
3610         I915_WRITE16(IMR, 0xffff);
3611         I915_WRITE16(IER, 0x0);
3612         I915_WRITE16(IIR, I915_READ16(IIR));
3613 }
3614
3615 static void i915_irq_preinstall(struct drm_device * dev)
3616 {
3617         struct drm_i915_private *dev_priv = dev->dev_private;
3618         int pipe;
3619
3620         if (I915_HAS_HOTPLUG(dev)) {
3621                 I915_WRITE(PORT_HOTPLUG_EN, 0);
3622                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3623         }
3624
3625         I915_WRITE16(HWSTAM, 0xeffe);
3626         for_each_pipe(dev_priv, pipe)
3627                 I915_WRITE(PIPESTAT(pipe), 0);
3628         I915_WRITE(IMR, 0xffffffff);
3629         I915_WRITE(IER, 0x0);
3630         POSTING_READ(IER);
3631 }
3632
3633 static int i915_irq_postinstall(struct drm_device *dev)
3634 {
3635         struct drm_i915_private *dev_priv = dev->dev_private;
3636         u32 enable_mask;
3637
3638         I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3639
3640         /* Unmask the interrupts that we always want on. */
3641         dev_priv->irq_mask =
3642                 ~(I915_ASLE_INTERRUPT |
3643                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3644                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3645                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3646                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3647
3648         enable_mask =
3649                 I915_ASLE_INTERRUPT |
3650                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3651                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3652                 I915_USER_INTERRUPT;
3653
3654         if (I915_HAS_HOTPLUG(dev)) {
3655                 I915_WRITE(PORT_HOTPLUG_EN, 0);
3656                 POSTING_READ(PORT_HOTPLUG_EN);
3657
3658                 /* Enable in IER... */
3659                 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
3660                 /* and unmask in IMR */
3661                 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
3662         }
3663
3664         I915_WRITE(IMR, dev_priv->irq_mask);
3665         I915_WRITE(IER, enable_mask);
3666         POSTING_READ(IER);
3667
3668         i915_enable_asle_pipestat(dev);
3669
3670         /* Interrupt setup is already guaranteed to be single-threaded, this is
3671          * just to make the assert_spin_locked check happy. */
3672         spin_lock_irq(&dev_priv->irq_lock);
3673         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3674         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3675         spin_unlock_irq(&dev_priv->irq_lock);
3676
3677         return 0;
3678 }
3679
3680 /*
3681  * Returns true when a page flip has completed.
3682  */
3683 static bool i915_handle_vblank(struct drm_device *dev,
3684                                int plane, int pipe, u32 iir)
3685 {
3686         struct drm_i915_private *dev_priv = dev->dev_private;
3687         u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3688
3689         if (!intel_pipe_handle_vblank(dev, pipe))
3690                 return false;
3691
3692         if ((iir & flip_pending) == 0)
3693                 goto check_page_flip;
3694
3695         /* We detect FlipDone by looking for the change in PendingFlip from '1'
3696          * to '0' on the following vblank, i.e. IIR has the Pendingflip
3697          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3698          * the flip is completed (no longer pending). Since this doesn't raise
3699          * an interrupt per se, we watch for the change at vblank.
3700          */
3701         if (I915_READ(ISR) & flip_pending)
3702                 goto check_page_flip;
3703
3704         intel_prepare_page_flip(dev, plane);
3705         intel_finish_page_flip(dev, pipe);
3706         return true;
3707
3708 check_page_flip:
3709         intel_check_page_flip(dev, pipe);
3710         return false;
3711 }
3712
3713 static irqreturn_t i915_irq_handler(int irq, void *arg)
3714 {
3715         struct drm_device *dev = arg;
3716         struct drm_i915_private *dev_priv = dev->dev_private;
3717         u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
3718         u32 flip_mask =
3719                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3720                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3721         int pipe, ret = IRQ_NONE;
3722
3723         if (!intel_irqs_enabled(dev_priv))
3724                 return IRQ_NONE;
3725
3726         iir = I915_READ(IIR);
3727         do {
3728                 bool irq_received = (iir & ~flip_mask) != 0;
3729                 bool blc_event = false;
3730
3731                 /* Can't rely on pipestat interrupt bit in iir as it might
3732                  * have been cleared after the pipestat interrupt was received.
3733                  * It doesn't set the bit in iir again, but it still produces
3734                  * interrupts (for non-MSI).
3735                  */
3736                 spin_lock(&dev_priv->irq_lock);
3737                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3738                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3739
3740                 for_each_pipe(dev_priv, pipe) {
3741                         int reg = PIPESTAT(pipe);
3742                         pipe_stats[pipe] = I915_READ(reg);
3743
3744                         /* Clear the PIPE*STAT regs before the IIR */
3745                         if (pipe_stats[pipe] & 0x8000ffff) {
3746                                 I915_WRITE(reg, pipe_stats[pipe]);
3747                                 irq_received = true;
3748                         }
3749                 }
3750                 spin_unlock(&dev_priv->irq_lock);
3751
3752                 if (!irq_received)
3753                         break;
3754
3755                 /* Consume port.  Then clear IIR or we'll miss events */
3756                 if (I915_HAS_HOTPLUG(dev) &&
3757                     iir & I915_DISPLAY_PORT_INTERRUPT)
3758                         i9xx_hpd_irq_handler(dev);
3759
3760                 I915_WRITE(IIR, iir & ~flip_mask);
3761                 new_iir = I915_READ(IIR); /* Flush posted writes */
3762
3763                 if (iir & I915_USER_INTERRUPT)
3764                         notify_ring(&dev_priv->ring[RCS]);
3765
3766                 for_each_pipe(dev_priv, pipe) {
3767                         int plane = pipe;
3768                         if (HAS_FBC(dev))
3769                                 plane = !plane;
3770
3771                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3772                             i915_handle_vblank(dev, plane, pipe, iir))
3773                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3774
3775                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
3776                                 blc_event = true;
3777
3778                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3779                                 i9xx_pipe_crc_irq_handler(dev, pipe);
3780
3781                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3782                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
3783                                                                     pipe);
3784                 }
3785
3786                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
3787                         intel_opregion_asle_intr(dev);
3788
3789                 /* With MSI, interrupts are only generated when iir
3790                  * transitions from zero to nonzero.  If another bit got
3791                  * set while we were handling the existing iir bits, then
3792                  * we would never get another interrupt.
3793                  *
3794                  * This is fine on non-MSI as well, as if we hit this path
3795                  * we avoid exiting the interrupt handler only to generate
3796                  * another one.
3797                  *
3798                  * Note that for MSI this could cause a stray interrupt report
3799                  * if an interrupt landed in the time between writing IIR and
3800                  * the posting read.  This should be rare enough to never
3801                  * trigger the 99% of 100,000 interrupts test for disabling
3802                  * stray interrupts.
3803                  */
3804                 ret = IRQ_HANDLED;
3805                 iir = new_iir;
3806         } while (iir & ~flip_mask);
3807
3808         return ret;
3809 }
3810
3811 static void i915_irq_uninstall(struct drm_device * dev)
3812 {
3813         struct drm_i915_private *dev_priv = dev->dev_private;
3814         int pipe;
3815
3816         if (I915_HAS_HOTPLUG(dev)) {
3817                 I915_WRITE(PORT_HOTPLUG_EN, 0);
3818                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3819         }
3820
3821         I915_WRITE16(HWSTAM, 0xffff);
3822         for_each_pipe(dev_priv, pipe) {
3823                 /* Clear enable bits; then clear status bits */
3824                 I915_WRITE(PIPESTAT(pipe), 0);
3825                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3826         }
3827         I915_WRITE(IMR, 0xffffffff);
3828         I915_WRITE(IER, 0x0);
3829
3830         I915_WRITE(IIR, I915_READ(IIR));
3831 }
3832
3833 static void i965_irq_preinstall(struct drm_device * dev)
3834 {
3835         struct drm_i915_private *dev_priv = dev->dev_private;
3836         int pipe;
3837
3838         I915_WRITE(PORT_HOTPLUG_EN, 0);
3839         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3840
3841         I915_WRITE(HWSTAM, 0xeffe);
3842         for_each_pipe(dev_priv, pipe)
3843                 I915_WRITE(PIPESTAT(pipe), 0);
3844         I915_WRITE(IMR, 0xffffffff);
3845         I915_WRITE(IER, 0x0);
3846         POSTING_READ(IER);
3847 }
3848
3849 static int i965_irq_postinstall(struct drm_device *dev)
3850 {
3851         struct drm_i915_private *dev_priv = dev->dev_private;
3852         u32 enable_mask;
3853         u32 error_mask;
3854
3855         /* Unmask the interrupts that we always want on. */
3856         dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
3857                                I915_DISPLAY_PORT_INTERRUPT |
3858                                I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3859                                I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3860                                I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3861                                I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
3862                                I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
3863
3864         enable_mask = ~dev_priv->irq_mask;
3865         enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3866                          I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3867         enable_mask |= I915_USER_INTERRUPT;
3868
3869         if (IS_G4X(dev))
3870                 enable_mask |= I915_BSD_USER_INTERRUPT;
3871
3872         /* Interrupt setup is already guaranteed to be single-threaded, this is
3873          * just to make the assert_spin_locked check happy. */
3874         spin_lock_irq(&dev_priv->irq_lock);
3875         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3876         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3877         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3878         spin_unlock_irq(&dev_priv->irq_lock);
3879
3880         /*
3881          * Enable some error detection, note the instruction error mask
3882          * bit is reserved, so we leave it masked.
3883          */
3884         if (IS_G4X(dev)) {
3885                 error_mask = ~(GM45_ERROR_PAGE_TABLE |
3886                                GM45_ERROR_MEM_PRIV |
3887                                GM45_ERROR_CP_PRIV |
3888                                I915_ERROR_MEMORY_REFRESH);
3889         } else {
3890                 error_mask = ~(I915_ERROR_PAGE_TABLE |
3891                                I915_ERROR_MEMORY_REFRESH);
3892         }
3893         I915_WRITE(EMR, error_mask);
3894
3895         I915_WRITE(IMR, dev_priv->irq_mask);
3896         I915_WRITE(IER, enable_mask);
3897         POSTING_READ(IER);
3898
3899         I915_WRITE(PORT_HOTPLUG_EN, 0);
3900         POSTING_READ(PORT_HOTPLUG_EN);
3901
3902         i915_enable_asle_pipestat(dev);
3903
3904         return 0;
3905 }
3906
3907 static void i915_hpd_irq_setup(struct drm_device *dev)
3908 {
3909         struct drm_i915_private *dev_priv = dev->dev_private;
3910         struct intel_encoder *intel_encoder;
3911         u32 hotplug_en;
3912
3913         assert_spin_locked(&dev_priv->irq_lock);
3914
3915         hotplug_en = I915_READ(PORT_HOTPLUG_EN);
3916         hotplug_en &= ~HOTPLUG_INT_EN_MASK;
3917         /* Note HDMI and DP share hotplug bits */
3918         /* enable bits are the same for all generations */
3919         for_each_intel_encoder(dev, intel_encoder)
3920                 if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED)
3921                         hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
3922         /* Programming the CRT detection parameters tends
3923            to generate a spurious hotplug event about three
3924            seconds later.  So just do it once.
3925         */
3926         if (IS_G4X(dev))
3927                 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
3928         hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
3929         hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
3930
3931         /* Ignore TV since it's buggy */
3932         I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
3933 }
3934
3935 static irqreturn_t i965_irq_handler(int irq, void *arg)
3936 {
3937         struct drm_device *dev = arg;
3938         struct drm_i915_private *dev_priv = dev->dev_private;
3939         u32 iir, new_iir;
3940         u32 pipe_stats[I915_MAX_PIPES];
3941         int ret = IRQ_NONE, pipe;
3942         u32 flip_mask =
3943                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3944                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3945
3946         if (!intel_irqs_enabled(dev_priv))
3947                 return IRQ_NONE;
3948
3949         iir = I915_READ(IIR);
3950
3951         for (;;) {
3952                 bool irq_received = (iir & ~flip_mask) != 0;
3953                 bool blc_event = false;
3954
3955                 /* Can't rely on pipestat interrupt bit in iir as it might
3956                  * have been cleared after the pipestat interrupt was received.
3957                  * It doesn't set the bit in iir again, but it still produces
3958                  * interrupts (for non-MSI).
3959                  */
3960                 spin_lock(&dev_priv->irq_lock);
3961                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3962                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3963
3964                 for_each_pipe(dev_priv, pipe) {
3965                         int reg = PIPESTAT(pipe);
3966                         pipe_stats[pipe] = I915_READ(reg);
3967
3968                         /*
3969                          * Clear the PIPE*STAT regs before the IIR
3970                          */
3971                         if (pipe_stats[pipe] & 0x8000ffff) {
3972                                 I915_WRITE(reg, pipe_stats[pipe]);
3973                                 irq_received = true;
3974                         }
3975                 }
3976                 spin_unlock(&dev_priv->irq_lock);
3977
3978                 if (!irq_received)
3979                         break;
3980
3981                 ret = IRQ_HANDLED;
3982
3983                 /* Consume port.  Then clear IIR or we'll miss events */
3984                 if (iir & I915_DISPLAY_PORT_INTERRUPT)
3985                         i9xx_hpd_irq_handler(dev);
3986
3987                 I915_WRITE(IIR, iir & ~flip_mask);
3988                 new_iir = I915_READ(IIR); /* Flush posted writes */
3989
3990                 if (iir & I915_USER_INTERRUPT)
3991                         notify_ring(&dev_priv->ring[RCS]);
3992                 if (iir & I915_BSD_USER_INTERRUPT)
3993                         notify_ring(&dev_priv->ring[VCS]);
3994
3995                 for_each_pipe(dev_priv, pipe) {
3996                         if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
3997                             i915_handle_vblank(dev, pipe, pipe, iir))
3998                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
3999
4000                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
4001                                 blc_event = true;
4002
4003                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4004                                 i9xx_pipe_crc_irq_handler(dev, pipe);
4005
4006                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
4007                                 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4008                 }
4009
4010                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
4011                         intel_opregion_asle_intr(dev);
4012
4013                 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
4014                         gmbus_irq_handler(dev);
4015
4016                 /* With MSI, interrupts are only generated when iir
4017                  * transitions from zero to nonzero.  If another bit got
4018                  * set while we were handling the existing iir bits, then
4019                  * we would never get another interrupt.
4020                  *
4021                  * This is fine on non-MSI as well, as if we hit this path
4022                  * we avoid exiting the interrupt handler only to generate
4023                  * another one.
4024                  *
4025                  * Note that for MSI this could cause a stray interrupt report
4026                  * if an interrupt landed in the time between writing IIR and
4027                  * the posting read.  This should be rare enough to never
4028                  * trigger the 99% of 100,000 interrupts test for disabling
4029                  * stray interrupts.
4030                  */
4031                 iir = new_iir;
4032         }
4033
4034         return ret;
4035 }
4036
4037 static void i965_irq_uninstall(struct drm_device * dev)
4038 {
4039         struct drm_i915_private *dev_priv = dev->dev_private;
4040         int pipe;
4041
4042         if (!dev_priv)
4043                 return;
4044
4045         I915_WRITE(PORT_HOTPLUG_EN, 0);
4046         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4047
4048         I915_WRITE(HWSTAM, 0xffffffff);
4049         for_each_pipe(dev_priv, pipe)
4050                 I915_WRITE(PIPESTAT(pipe), 0);
4051         I915_WRITE(IMR, 0xffffffff);
4052         I915_WRITE(IER, 0x0);
4053
4054         for_each_pipe(dev_priv, pipe)
4055                 I915_WRITE(PIPESTAT(pipe),
4056                            I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
4057         I915_WRITE(IIR, I915_READ(IIR));
4058 }
4059
4060 /**
4061  * intel_irq_init - initializes irq support
4062  * @dev_priv: i915 device instance
4063  *
4064  * This function initializes all the irq support including work items, timers
4065  * and all the vtables. It does not setup the interrupt itself though.
4066  */
4067 void intel_irq_init(struct drm_i915_private *dev_priv)
4068 {
4069         struct drm_device *dev = dev_priv->dev;
4070
4071         intel_hpd_init_work(dev_priv);
4072
4073         INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4074         INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4075
4076         /* Let's track the enabled rps events */
4077         if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
4078                 /* WaGsvRC0ResidencyMethod:vlv */
4079                 dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED;
4080         else
4081                 dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4082
4083         INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
4084                           i915_hangcheck_elapsed);
4085
4086         pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
4087
4088         if (IS_GEN2(dev_priv)) {
4089                 dev->max_vblank_count = 0;
4090                 dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
4091         } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4092                 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4093                 dev->driver->get_vblank_counter = gm45_get_vblank_counter;
4094         } else {
4095                 dev->driver->get_vblank_counter = i915_get_vblank_counter;
4096                 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4097         }
4098
4099         /*
4100          * Opt out of the vblank disable timer on everything except gen2.
4101          * Gen2 doesn't have a hardware frame counter and so depends on
4102          * vblank interrupts to produce sane vblank seuquence numbers.
4103          */
4104         if (!IS_GEN2(dev_priv))
4105                 dev->vblank_disable_immediate = true;
4106
4107         dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
4108         dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4109
4110         if (IS_CHERRYVIEW(dev_priv)) {
4111                 dev->driver->irq_handler = cherryview_irq_handler;
4112                 dev->driver->irq_preinstall = cherryview_irq_preinstall;
4113                 dev->driver->irq_postinstall = cherryview_irq_postinstall;
4114                 dev->driver->irq_uninstall = cherryview_irq_uninstall;
4115                 dev->driver->enable_vblank = valleyview_enable_vblank;
4116                 dev->driver->disable_vblank = valleyview_disable_vblank;
4117                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4118         } else if (IS_VALLEYVIEW(dev_priv)) {
4119                 dev->driver->irq_handler = valleyview_irq_handler;
4120                 dev->driver->irq_preinstall = valleyview_irq_preinstall;
4121                 dev->driver->irq_postinstall = valleyview_irq_postinstall;
4122                 dev->driver->irq_uninstall = valleyview_irq_uninstall;
4123                 dev->driver->enable_vblank = valleyview_enable_vblank;
4124                 dev->driver->disable_vblank = valleyview_disable_vblank;
4125                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4126         } else if (INTEL_INFO(dev_priv)->gen >= 8) {
4127                 dev->driver->irq_handler = gen8_irq_handler;
4128                 dev->driver->irq_preinstall = gen8_irq_reset;
4129                 dev->driver->irq_postinstall = gen8_irq_postinstall;
4130                 dev->driver->irq_uninstall = gen8_irq_uninstall;
4131                 dev->driver->enable_vblank = gen8_enable_vblank;
4132                 dev->driver->disable_vblank = gen8_disable_vblank;
4133                 if (HAS_PCH_SPLIT(dev))
4134                         dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
4135                 else
4136                         dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4137         } else if (HAS_PCH_SPLIT(dev)) {
4138                 dev->driver->irq_handler = ironlake_irq_handler;
4139                 dev->driver->irq_preinstall = ironlake_irq_reset;
4140                 dev->driver->irq_postinstall = ironlake_irq_postinstall;
4141                 dev->driver->irq_uninstall = ironlake_irq_uninstall;
4142                 dev->driver->enable_vblank = ironlake_enable_vblank;
4143                 dev->driver->disable_vblank = ironlake_disable_vblank;
4144                 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
4145         } else {
4146                 if (INTEL_INFO(dev_priv)->gen == 2) {
4147                         dev->driver->irq_preinstall = i8xx_irq_preinstall;
4148                         dev->driver->irq_postinstall = i8xx_irq_postinstall;
4149                         dev->driver->irq_handler = i8xx_irq_handler;
4150                         dev->driver->irq_uninstall = i8xx_irq_uninstall;
4151                 } else if (INTEL_INFO(dev_priv)->gen == 3) {
4152                         dev->driver->irq_preinstall = i915_irq_preinstall;
4153                         dev->driver->irq_postinstall = i915_irq_postinstall;
4154                         dev->driver->irq_uninstall = i915_irq_uninstall;
4155                         dev->driver->irq_handler = i915_irq_handler;
4156                 } else {
4157                         dev->driver->irq_preinstall = i965_irq_preinstall;
4158                         dev->driver->irq_postinstall = i965_irq_postinstall;
4159                         dev->driver->irq_uninstall = i965_irq_uninstall;
4160                         dev->driver->irq_handler = i965_irq_handler;
4161                 }
4162                 if (I915_HAS_HOTPLUG(dev_priv))
4163                         dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4164                 dev->driver->enable_vblank = i915_enable_vblank;
4165                 dev->driver->disable_vblank = i915_disable_vblank;
4166         }
4167 }
4168
4169 /**
4170  * intel_irq_install - enables the hardware interrupt
4171  * @dev_priv: i915 device instance
4172  *
4173  * This function enables the hardware interrupt handling, but leaves the hotplug
4174  * handling still disabled. It is called after intel_irq_init().
4175  *
4176  * In the driver load and resume code we need working interrupts in a few places
4177  * but don't want to deal with the hassle of concurrent probe and hotplug
4178  * workers. Hence the split into this two-stage approach.
4179  */
4180 int intel_irq_install(struct drm_i915_private *dev_priv)
4181 {
4182         /*
4183          * We enable some interrupt sources in our postinstall hooks, so mark
4184          * interrupts as enabled _before_ actually enabling them to avoid
4185          * special cases in our ordering checks.
4186          */
4187         dev_priv->pm.irqs_enabled = true;
4188
4189         return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
4190 }
4191
4192 /**
4193  * intel_irq_uninstall - finilizes all irq handling
4194  * @dev_priv: i915 device instance
4195  *
4196  * This stops interrupt and hotplug handling and unregisters and frees all
4197  * resources acquired in the init functions.
4198  */
4199 void intel_irq_uninstall(struct drm_i915_private *dev_priv)
4200 {
4201         drm_irq_uninstall(dev_priv->dev);
4202         intel_hpd_cancel_work(dev_priv);
4203         dev_priv->pm.irqs_enabled = false;
4204 }
4205
4206 /**
4207  * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
4208  * @dev_priv: i915 device instance
4209  *
4210  * This function is used to disable interrupts at runtime, both in the runtime
4211  * pm and the system suspend/resume code.
4212  */
4213 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4214 {
4215         dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
4216         dev_priv->pm.irqs_enabled = false;
4217         synchronize_irq(dev_priv->dev->irq);
4218 }
4219
4220 /**
4221  * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
4222  * @dev_priv: i915 device instance
4223  *
4224  * This function is used to enable interrupts at runtime, both in the runtime
4225  * pm and the system suspend/resume code.
4226  */
4227 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4228 {
4229         dev_priv->pm.irqs_enabled = true;
4230         dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
4231         dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
4232 }