drivers/gpu/drm/i915/i915_irq.c

   1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
   2  */
   3 /*
   4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  30
  31 #include <linux/sysrq.h>
  32 #include <linux/slab.h>
  33 #include <drm/drmP.h>
  34 #include <drm/i915_drm.h>
  35 #include "i915_drv.h"
  36 #include "i915_trace.h"
  37 #include "intel_drv.h"
  38
  39 static const u32 hpd_ibx[] = {
  40         [HPD_CRT] = SDE_CRT_HOTPLUG,
  41         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
  42         [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
  43         [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
  44         [HPD_PORT_D] = SDE_PORTD_HOTPLUG
  45 };
  46
  47 static const u32 hpd_cpt[] = {
  48         [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
  49         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
  50         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
  51         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
  52         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
  53 };
  54
  55 static const u32 hpd_mask_i915[] = {
  56         [HPD_CRT] = CRT_HOTPLUG_INT_EN,
  57         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
  58         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
  59         [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
  60         [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
  61         [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
  62 };
  63
  64 static const u32 hpd_status_gen4[] = {
  65         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  66         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
  67         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
  68         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  69         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  70         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
  71 };
  72
  73 static const u32 hpd_status_i915[] = { /* i915 and valleyview are the same */
  74         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  75         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
  76         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
  77         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  78         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  79         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
  80 };
  81
  82 /* For display hotplug interrupt */
  83 static void
  84 ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
  85 {
  86         assert_spin_locked(&dev_priv->irq_lock);
  87
  88         if ((dev_priv->irq_mask & mask) != 0) {
  89                 dev_priv->irq_mask &= ~mask;
  90                 I915_WRITE(DEIMR, dev_priv->irq_mask);
  91                 POSTING_READ(DEIMR);
  92         }
  93 }
  94
  95 static void
  96 ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
  97 {
  98         assert_spin_locked(&dev_priv->irq_lock);
  99
 100         if ((dev_priv->irq_mask & mask) != mask) {
 101                 dev_priv->irq_mask |= mask;
 102                 I915_WRITE(DEIMR, dev_priv->irq_mask);
 103                 POSTING_READ(DEIMR);
 104         }
 105 }
 106
 107 /**
 108  * ilk_update_gt_irq - update GTIMR
 109  * @dev_priv: driver private
 110  * @interrupt_mask: mask of interrupt bits to update
 111  * @enabled_irq_mask: mask of interrupt bits to enable
 112  */
 113 static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
 114                               uint32_t interrupt_mask,
 115                               uint32_t enabled_irq_mask)
 116 {
 117         assert_spin_locked(&dev_priv->irq_lock);
 118
 119         dev_priv->gt_irq_mask &= ~interrupt_mask;
 120         dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
 121         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 122         POSTING_READ(GTIMR);
 123 }
 124
 125 void ilk_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 126 {
 127         ilk_update_gt_irq(dev_priv, mask, mask);
 128 }
 129
 130 void ilk_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 131 {
 132         ilk_update_gt_irq(dev_priv, mask, 0);
 133 }
 134
 135 /**
 136   * snb_update_pm_irq - update GEN6_PMIMR
 137   * @dev_priv: driver private
 138   * @interrupt_mask: mask of interrupt bits to update
 139   * @enabled_irq_mask: mask of interrupt bits to enable
 140   */
 141 static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
 142                               uint32_t interrupt_mask,
 143                               uint32_t enabled_irq_mask)
 144 {
 145         uint32_t pmimr = I915_READ(GEN6_PMIMR);
 146         pmimr &= ~interrupt_mask;
 147         pmimr |= (~enabled_irq_mask & interrupt_mask);
 148
 149         assert_spin_locked(&dev_priv->irq_lock);
 150
 151         I915_WRITE(GEN6_PMIMR, pmimr);
 152         POSTING_READ(GEN6_PMIMR);
 153 }
 154
 155 void snb_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 156 {
 157         snb_update_pm_irq(dev_priv, mask, mask);
 158 }
 159
 160 void snb_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 161 {
 162         snb_update_pm_irq(dev_priv, mask, 0);
 163 }
 164
 165 static void snb_set_pm_irq(struct drm_i915_private *dev_priv, uint32_t val)
 166 {
 167         snb_update_pm_irq(dev_priv, 0xffffffff, ~val);
 168 }
 169
 170 static bool ivb_can_enable_err_int(struct drm_device *dev)
 171 {
 172         struct drm_i915_private *dev_priv = dev->dev_private;
 173         struct intel_crtc *crtc;
 174         enum pipe pipe;
 175
 176         assert_spin_locked(&dev_priv->irq_lock);
 177
 178         for_each_pipe(pipe) {
 179                 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
 180
 181                 if (crtc->cpu_fifo_underrun_disabled)
 182                         return false;
 183         }
 184
 185         return true;
 186 }
 187
 188 static bool cpt_can_enable_serr_int(struct drm_device *dev)
 189 {
 190         struct drm_i915_private *dev_priv = dev->dev_private;
 191         enum pipe pipe;
 192         struct intel_crtc *crtc;
 193
 194         assert_spin_locked(&dev_priv->irq_lock);
 195
 196         for_each_pipe(pipe) {
 197                 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
 198
 199                 if (crtc->pch_fifo_underrun_disabled)
 200                         return false;
 201         }
 202
 203         return true;
 204 }
 205
 206 static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev,
 207                                                  enum pipe pipe, bool enable)
 208 {
 209         struct drm_i915_private *dev_priv = dev->dev_private;
 210         uint32_t bit = (pipe == PIPE_A) ? DE_PIPEA_FIFO_UNDERRUN :
 211                                           DE_PIPEB_FIFO_UNDERRUN;
 212
 213         if (enable)
 214                 ironlake_enable_display_irq(dev_priv, bit);
 215         else
 216                 ironlake_disable_display_irq(dev_priv, bit);
 217 }
 218
 219 static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev,
 220                                                   enum pipe pipe, bool enable)
 221 {
 222         struct drm_i915_private *dev_priv = dev->dev_private;
 223         if (enable) {
 224                 I915_WRITE(GEN7_ERR_INT, ERR_INT_FIFO_UNDERRUN(pipe));
 225
 226                 if (!ivb_can_enable_err_int(dev))
 227                         return;
 228
 229                 ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
 230         } else {
 231                 bool was_enabled = !(I915_READ(DEIMR) & DE_ERR_INT_IVB);
 232
 233                 /* Change the state _after_ we've read out the current one. */
 234                 ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
 235
 236                 if (!was_enabled &&
 237                     (I915_READ(GEN7_ERR_INT) & ERR_INT_FIFO_UNDERRUN(pipe))) {
 238                         DRM_DEBUG_KMS("uncleared fifo underrun on pipe %c\n",
 239                                       pipe_name(pipe));
 240                 }
 241         }
 242 }
 243
 244 /**
 245  * ibx_display_interrupt_update - update SDEIMR
 246  * @dev_priv: driver private
 247  * @interrupt_mask: mask of interrupt bits to update
 248  * @enabled_irq_mask: mask of interrupt bits to enable
 249  */
 250 static void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
 251                                          uint32_t interrupt_mask,
 252                                          uint32_t enabled_irq_mask)
 253 {
 254         uint32_t sdeimr = I915_READ(SDEIMR);
 255         sdeimr &= ~interrupt_mask;
 256         sdeimr |= (~enabled_irq_mask & interrupt_mask);
 257
 258         assert_spin_locked(&dev_priv->irq_lock);
 259
 260         I915_WRITE(SDEIMR, sdeimr);
 261         POSTING_READ(SDEIMR);
 262 }
 263 #define ibx_enable_display_interrupt(dev_priv, bits) \
 264         ibx_display_interrupt_update((dev_priv), (bits), (bits))
 265 #define ibx_disable_display_interrupt(dev_priv, bits) \
 266         ibx_display_interrupt_update((dev_priv), (bits), 0)
 267
 268 static void ibx_set_fifo_underrun_reporting(struct drm_device *dev,
 269                                             enum transcoder pch_transcoder,
 270                                             bool enable)
 271 {
 272         struct drm_i915_private *dev_priv = dev->dev_private;
 273         uint32_t bit = (pch_transcoder == TRANSCODER_A) ?
 274                        SDE_TRANSA_FIFO_UNDER : SDE_TRANSB_FIFO_UNDER;
 275
 276         if (enable)
 277                 ibx_enable_display_interrupt(dev_priv, bit);
 278         else
 279                 ibx_disable_display_interrupt(dev_priv, bit);
 280 }
 281
 282 static void cpt_set_fifo_underrun_reporting(struct drm_device *dev,
 283                                             enum transcoder pch_transcoder,
 284                                             bool enable)
 285 {
 286         struct drm_i915_private *dev_priv = dev->dev_private;
 287
 288         if (enable) {
 289                 I915_WRITE(SERR_INT,
 290                            SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder));
 291
 292                 if (!cpt_can_enable_serr_int(dev))
 293                         return;
 294
 295                 ibx_enable_display_interrupt(dev_priv, SDE_ERROR_CPT);
 296         } else {
 297                 uint32_t tmp = I915_READ(SERR_INT);
 298                 bool was_enabled = !(I915_READ(SDEIMR) & SDE_ERROR_CPT);
 299
 300                 /* Change the state _after_ we've read out the current one. */
 301                 ibx_disable_display_interrupt(dev_priv, SDE_ERROR_CPT);
 302
 303                 if (!was_enabled &&
 304                     (tmp & SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder))) {
 305                         DRM_DEBUG_KMS("uncleared pch fifo underrun on pch transcoder %c\n",
 306                                       transcoder_name(pch_transcoder));
 307                 }
 308         }
 309 }
 310
 311 /**
 312  * intel_set_cpu_fifo_underrun_reporting - enable/disable FIFO underrun messages
 313  * @dev: drm device
 314  * @pipe: pipe
 315  * @enable: true if we want to report FIFO underrun errors, false otherwise
 316  *
 317  * This function makes us disable or enable CPU fifo underruns for a specific
 318  * pipe. Notice that on some Gens (e.g. IVB, HSW), disabling FIFO underrun
 319  * reporting for one pipe may also disable all the other CPU error interruts for
 320  * the other pipes, due to the fact that there's just one interrupt mask/enable
 321  * bit for all the pipes.
 322  *
 323  * Returns the previous state of underrun reporting.
 324  */
 325 bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
 326                                            enum pipe pipe, bool enable)
 327 {
 328         struct drm_i915_private *dev_priv = dev->dev_private;
 329         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
 330         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 331         unsigned long flags;
 332         bool ret;
 333
 334         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 335
 336         ret = !intel_crtc->cpu_fifo_underrun_disabled;
 337
 338         if (enable == ret)
 339                 goto done;
 340
 341         intel_crtc->cpu_fifo_underrun_disabled = !enable;
 342
 343         if (IS_GEN5(dev) || IS_GEN6(dev))
 344                 ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
 345         else if (IS_GEN7(dev))
 346                 ivybridge_set_fifo_underrun_reporting(dev, pipe, enable);
 347
 348 done:
 349         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 350         return ret;
 351 }
 352
 353 /**
 354  * intel_set_pch_fifo_underrun_reporting - enable/disable FIFO underrun messages
 355  * @dev: drm device
 356  * @pch_transcoder: the PCH transcoder (same as pipe on IVB and older)
 357  * @enable: true if we want to report FIFO underrun errors, false otherwise
 358  *
 359  * This function makes us disable or enable PCH fifo underruns for a specific
 360  * PCH transcoder. Notice that on some PCHs (e.g. CPT/PPT), disabling FIFO
 361  * underrun reporting for one transcoder may also disable all the other PCH
 362  * error interruts for the other transcoders, due to the fact that there's just
 363  * one interrupt mask/enable bit for all the transcoders.
 364  *
 365  * Returns the previous state of underrun reporting.
 366  */
 367 bool intel_set_pch_fifo_underrun_reporting(struct drm_device *dev,
 368                                            enum transcoder pch_transcoder,
 369                                            bool enable)
 370 {
 371         struct drm_i915_private *dev_priv = dev->dev_private;
 372         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder];
 373         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 374         unsigned long flags;
 375         bool ret;
 376
 377         /*
 378          * NOTE: Pre-LPT has a fixed cpu pipe -> pch transcoder mapping, but LPT
 379          * has only one pch transcoder A that all pipes can use. To avoid racy
 380          * pch transcoder -> pipe lookups from interrupt code simply store the
 381          * underrun statistics in crtc A. Since we never expose this anywhere
 382          * nor use it outside of the fifo underrun code here using the "wrong"
 383          * crtc on LPT won't cause issues.
 384          */
 385
 386         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 387
 388         ret = !intel_crtc->pch_fifo_underrun_disabled;
 389
 390         if (enable == ret)
 391                 goto done;
 392
 393         intel_crtc->pch_fifo_underrun_disabled = !enable;
 394
 395         if (HAS_PCH_IBX(dev))
 396                 ibx_set_fifo_underrun_reporting(dev, pch_transcoder, enable);
 397         else
 398                 cpt_set_fifo_underrun_reporting(dev, pch_transcoder, enable);
 399
 400 done:
 401         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 402         return ret;
 403 }
 404
 405
 406 void
 407 i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
 408 {
 409         u32 reg = PIPESTAT(pipe);
 410         u32 pipestat = I915_READ(reg) & 0x7fff0000;
 411
 412         assert_spin_locked(&dev_priv->irq_lock);
 413
 414         if ((pipestat & mask) == mask)
 415                 return;
 416
 417         /* Enable the interrupt, clear any pending status */
 418         pipestat |= mask | (mask >> 16);
 419         I915_WRITE(reg, pipestat);
 420         POSTING_READ(reg);
 421 }
 422
 423 void
 424 i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
 425 {
 426         u32 reg = PIPESTAT(pipe);
 427         u32 pipestat = I915_READ(reg) & 0x7fff0000;
 428
 429         assert_spin_locked(&dev_priv->irq_lock);
 430
 431         if ((pipestat & mask) == 0)
 432                 return;
 433
 434         pipestat &= ~mask;
 435         I915_WRITE(reg, pipestat);
 436         POSTING_READ(reg);
 437 }
 438
 439 /**
 440  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
 441  */
 442 static void i915_enable_asle_pipestat(struct drm_device *dev)
 443 {
 444         drm_i915_private_t *dev_priv = dev->dev_private;
 445         unsigned long irqflags;
 446
 447         if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
 448                 return;
 449
 450         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
 451
 452         i915_enable_pipestat(dev_priv, 1, PIPE_LEGACY_BLC_EVENT_ENABLE);
 453         if (INTEL_INFO(dev)->gen >= 4)
 454                 i915_enable_pipestat(dev_priv, 0, PIPE_LEGACY_BLC_EVENT_ENABLE);
 455
 456         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 457 }
 458
 459 /**
 460  * i915_pipe_enabled - check if a pipe is enabled
 461  * @dev: DRM device
 462  * @pipe: pipe to check
 463  *
 464  * Reading certain registers when the pipe is disabled can hang the chip.
 465  * Use this routine to make sure the PLL is running and the pipe is active
 466  * before reading such registers if unsure.
 467  */
 468 static int
 469 i915_pipe_enabled(struct drm_device *dev, int pipe)
 470 {
 471         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 472
 473         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 474                 /* Locking is horribly broken here, but whatever. */
 475                 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
 476                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 477
 478                 return intel_crtc->active;
 479         } else {
 480                 return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE;
 481         }
 482 }
 483
 484 /* Called from drm generic code, passed a 'crtc', which
 485  * we use as a pipe index
 486  */
 487 static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
 488 {
 489         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 490         unsigned long high_frame;
 491         unsigned long low_frame;
 492         u32 high1, high2, low;
 493
 494         if (!i915_pipe_enabled(dev, pipe)) {
 495                 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
 496                                 "pipe %c\n", pipe_name(pipe));
 497                 return 0;
 498         }
 499
 500         high_frame = PIPEFRAME(pipe);
 501         low_frame = PIPEFRAMEPIXEL(pipe);
 502
 503         /*
 504          * High & low register fields aren't synchronized, so make sure
 505          * we get a low value that's stable across two reads of the high
 506          * register.
 507          */
 508         do {
 509                 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 510                 low   = I915_READ(low_frame)  & PIPE_FRAME_LOW_MASK;
 511                 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 512         } while (high1 != high2);
 513
 514         high1 >>= PIPE_FRAME_HIGH_SHIFT;
 515         low >>= PIPE_FRAME_LOW_SHIFT;
 516         return (high1 << 8) | low;
 517 }
 518
 519 static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
 520 {
 521         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 522         int reg = PIPE_FRMCOUNT_GM45(pipe);
 523
 524         if (!i915_pipe_enabled(dev, pipe)) {
 525                 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
 526                                  "pipe %c\n", pipe_name(pipe));
 527                 return 0;
 528         }
 529
 530         return I915_READ(reg);
 531 }
 532
 533 static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
 534                              int *vpos, int *hpos)
 535 {
 536         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 537         u32 vbl = 0, position = 0;
 538         int vbl_start, vbl_end, htotal, vtotal;
 539         bool in_vbl = true;
 540         int ret = 0;
 541         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
 542                                                                       pipe);
 543
 544         if (!i915_pipe_enabled(dev, pipe)) {
 545                 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
 546                                  "pipe %c\n", pipe_name(pipe));
 547                 return 0;
 548         }
 549
 550         /* Get vtotal. */
 551         vtotal = 1 + ((I915_READ(VTOTAL(cpu_transcoder)) >> 16) & 0x1fff);
 552
 553         if (INTEL_INFO(dev)->gen >= 4) {
 554                 /* No obvious pixelcount register. Only query vertical
 555                  * scanout position from Display scan line register.
 556                  */
 557                 position = I915_READ(PIPEDSL(pipe));
 558
 559                 /* Decode into vertical scanout position. Don't have
 560                  * horizontal scanout position.
 561                  */
 562                 *vpos = position & 0x1fff;
 563                 *hpos = 0;
 564         } else {
 565                 /* Have access to pixelcount since start of frame.
 566                  * We can split this into vertical and horizontal
 567                  * scanout position.
 568                  */
 569                 position = (I915_READ(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
 570
 571                 htotal = 1 + ((I915_READ(HTOTAL(cpu_transcoder)) >> 16) & 0x1fff);
 572                 *vpos = position / htotal;
 573                 *hpos = position - (*vpos * htotal);
 574         }
 575
 576         /* Query vblank area. */
 577         vbl = I915_READ(VBLANK(cpu_transcoder));
 578
 579         /* Test position against vblank region. */
 580         vbl_start = vbl & 0x1fff;
 581         vbl_end = (vbl >> 16) & 0x1fff;
 582
 583         if ((*vpos < vbl_start) || (*vpos > vbl_end))
 584                 in_vbl = false;
 585
 586         /* Inside "upper part" of vblank area? Apply corrective offset: */
 587         if (in_vbl && (*vpos >= vbl_start))
 588                 *vpos = *vpos - vtotal;
 589
 590         /* Readouts valid? */
 591         if (vbl > 0)
 592                 ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
 593
 594         /* In vblank? */
 595         if (in_vbl)
 596                 ret |= DRM_SCANOUTPOS_INVBL;
 597
 598         return ret;
 599 }
 600
 601 static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
 602                               int *max_error,
 603                               struct timeval *vblank_time,
 604                               unsigned flags)
 605 {
 606         struct drm_crtc *crtc;
 607
 608         if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
 609                 DRM_ERROR("Invalid crtc %d\n", pipe);
 610                 return -EINVAL;
 611         }
 612
 613         /* Get drm_crtc to timestamp: */
 614         crtc = intel_get_crtc_for_pipe(dev, pipe);
 615         if (crtc == NULL) {
 616                 DRM_ERROR("Invalid crtc %d\n", pipe);
 617                 return -EINVAL;
 618         }
 619
 620         if (!crtc->enabled) {
 621                 DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
 622                 return -EBUSY;
 623         }
 624
 625         /* Helper routine in DRM core does all the work: */
 626         return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
 627                                                      vblank_time, flags,
 628                                                      crtc);
 629 }
 630
 631 static int intel_hpd_irq_event(struct drm_device *dev, struct drm_connector *connector)
 632 {
 633         enum drm_connector_status old_status;
 634
 635         WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
 636         old_status = connector->status;
 637
 638         connector->status = connector->funcs->detect(connector, false);
 639         DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %d to %d\n",
 640                       connector->base.id,
 641                       drm_get_connector_name(connector),
 642                       old_status, connector->status);
 643         return (old_status != connector->status);
 644 }
 645
 646 /*
 647  * Handle hotplug events outside the interrupt handler proper.
 648  */
 649 #define I915_REENABLE_HOTPLUG_DELAY (2*60*1000)
 650
 651 static void i915_hotplug_work_func(struct work_struct *work)
 652 {
 653         drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 654                                                     hotplug_work);
 655         struct drm_device *dev = dev_priv->dev;
 656         struct drm_mode_config *mode_config = &dev->mode_config;
 657         struct intel_connector *intel_connector;
 658         struct intel_encoder *intel_encoder;
 659         struct drm_connector *connector;
 660         unsigned long irqflags;
 661         bool hpd_disabled = false;
 662         bool changed = false;
 663         u32 hpd_event_bits;
 664
 665         /* HPD irq before everything is fully set up. */
 666         if (!dev_priv->enable_hotplug_processing)
 667                 return;
 668
 669         mutex_lock(&mode_config->mutex);
 670         DRM_DEBUG_KMS("running encoder hotplug functions\n");
 671
 672         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
 673
 674         hpd_event_bits = dev_priv->hpd_event_bits;
 675         dev_priv->hpd_event_bits = 0;
 676         list_for_each_entry(connector, &mode_config->connector_list, head) {
 677                 intel_connector = to_intel_connector(connector);
 678                 intel_encoder = intel_connector->encoder;
 679                 if (intel_encoder->hpd_pin > HPD_NONE &&
 680                     dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_MARK_DISABLED &&
 681                     connector->polled == DRM_CONNECTOR_POLL_HPD) {
 682                         DRM_INFO("HPD interrupt storm detected on connector %s: "
 683                                  "switching from hotplug detection to polling\n",
 684                                 drm_get_connector_name(connector));
 685                         dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark = HPD_DISABLED;
 686                         connector->polled = DRM_CONNECTOR_POLL_CONNECT
 687                                 | DRM_CONNECTOR_POLL_DISCONNECT;
 688                         hpd_disabled = true;
 689                 }
 690                 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
 691                         DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n",
 692                                       drm_get_connector_name(connector), intel_encoder->hpd_pin);
 693                 }
 694         }
 695          /* if there were no outputs to poll, poll was disabled,
 696           * therefore make sure it's enabled when disabling HPD on
 697           * some connectors */
 698         if (hpd_disabled) {
 699                 drm_kms_helper_poll_enable(dev);
 700                 mod_timer(&dev_priv->hotplug_reenable_timer,
 701                           jiffies + msecs_to_jiffies(I915_REENABLE_HOTPLUG_DELAY));
 702         }
 703
 704         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 705
 706         list_for_each_entry(connector, &mode_config->connector_list, head) {
 707                 intel_connector = to_intel_connector(connector);
 708                 intel_encoder = intel_connector->encoder;
 709                 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
 710                         if (intel_encoder->hot_plug)
 711                                 intel_encoder->hot_plug(intel_encoder);
 712                         if (intel_hpd_irq_event(dev, connector))
 713                                 changed = true;
 714                 }
 715         }
 716         mutex_unlock(&mode_config->mutex);
 717
 718         if (changed)
 719                 drm_kms_helper_hotplug_event(dev);
 720 }
 721
 722 static void ironlake_rps_change_irq_handler(struct drm_device *dev)
 723 {
 724         drm_i915_private_t *dev_priv = dev->dev_private;
 725         u32 busy_up, busy_down, max_avg, min_avg;
 726         u8 new_delay;
 727
 728         spin_lock(&mchdev_lock);
 729
 730         I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
 731
 732         new_delay = dev_priv->ips.cur_delay;
 733
 734         I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
 735         busy_up = I915_READ(RCPREVBSYTUPAVG);
 736         busy_down = I915_READ(RCPREVBSYTDNAVG);
 737         max_avg = I915_READ(RCBMAXAVG);
 738         min_avg = I915_READ(RCBMINAVG);
 739
 740         /* Handle RCS change request from hw */
 741         if (busy_up > max_avg) {
 742                 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
 743                         new_delay = dev_priv->ips.cur_delay - 1;
 744                 if (new_delay < dev_priv->ips.max_delay)
 745                         new_delay = dev_priv->ips.max_delay;
 746         } else if (busy_down < min_avg) {
 747                 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
 748                         new_delay = dev_priv->ips.cur_delay + 1;
 749                 if (new_delay > dev_priv->ips.min_delay)
 750                         new_delay = dev_priv->ips.min_delay;
 751         }
 752
 753         if (ironlake_set_drps(dev, new_delay))
 754                 dev_priv->ips.cur_delay = new_delay;
 755
 756         spin_unlock(&mchdev_lock);
 757
 758         return;
 759 }
 760
 761 static void notify_ring(struct drm_device *dev,
 762                         struct intel_ring_buffer *ring)
 763 {
 764         if (ring->obj == NULL)
 765                 return;
 766
 767         trace_i915_gem_request_complete(ring, ring->get_seqno(ring, false));
 768
 769         wake_up_all(&ring->irq_queue);
 770         i915_queue_hangcheck(dev);
 771 }
 772
 773 static void gen6_pm_rps_work(struct work_struct *work)
 774 {
 775         drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 776                                                     rps.work);
 777         u32 pm_iir;
 778         u8 new_delay;
 779
 780         spin_lock_irq(&dev_priv->irq_lock);
 781         pm_iir = dev_priv->rps.pm_iir;
 782         dev_priv->rps.pm_iir = 0;
 783         /* Make sure not to corrupt PMIMR state used by ringbuffer code */
 784         snb_enable_pm_irq(dev_priv, GEN6_PM_RPS_EVENTS);
 785         spin_unlock_irq(&dev_priv->irq_lock);
 786
 787         if ((pm_iir & GEN6_PM_RPS_EVENTS) == 0)
 788                 return;
 789
 790         mutex_lock(&dev_priv->rps.hw_lock);
 791
 792         if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
 793                 new_delay = dev_priv->rps.cur_delay + 1;
 794
 795                 /*
 796                  * For better performance, jump directly
 797                  * to RPe if we're below it.
 798                  */
 799                 if (IS_VALLEYVIEW(dev_priv->dev) &&
 800                     dev_priv->rps.cur_delay < dev_priv->rps.rpe_delay)
 801                         new_delay = dev_priv->rps.rpe_delay;
 802         } else
 803                 new_delay = dev_priv->rps.cur_delay - 1;
 804
 805         /* sysfs frequency interfaces may have snuck in while servicing the
 806          * interrupt
 807          */
 808         if (new_delay >= dev_priv->rps.min_delay &&
 809             new_delay <= dev_priv->rps.max_delay) {
 810                 if (IS_VALLEYVIEW(dev_priv->dev))
 811                         valleyview_set_rps(dev_priv->dev, new_delay);
 812                 else
 813                         gen6_set_rps(dev_priv->dev, new_delay);
 814         }
 815
 816         if (IS_VALLEYVIEW(dev_priv->dev)) {
 817                 /*
 818                  * On VLV, when we enter RC6 we may not be at the minimum
 819                  * voltage level, so arm a timer to check.  It should only
 820                  * fire when there's activity or once after we've entered
 821                  * RC6, and then won't be re-armed until the next RPS interrupt.
 822                  */
 823                 mod_delayed_work(dev_priv->wq, &dev_priv->rps.vlv_work,
 824                                  msecs_to_jiffies(100));
 825         }
 826
 827         mutex_unlock(&dev_priv->rps.hw_lock);
 828 }
 829
 830
 831 /**
 832  * ivybridge_parity_work - Workqueue called when a parity error interrupt
 833  * occurred.
 834  * @work: workqueue struct
 835  *
 836  * Doesn't actually do anything except notify userspace. As a consequence of
 837  * this event, userspace should try to remap the bad rows since statistically
 838  * it is likely the same row is more likely to go bad again.
 839  */
 840 static void ivybridge_parity_work(struct work_struct *work)
 841 {
 842         drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 843                                                     l3_parity.error_work);
 844         u32 error_status, row, bank, subbank;
 845         char *parity_event[5];
 846         uint32_t misccpctl;
 847         unsigned long flags;
 848
 849         /* We must turn off DOP level clock gating to access the L3 registers.
 850          * In order to prevent a get/put style interface, acquire struct mutex
 851          * any time we access those registers.
 852          */
 853         mutex_lock(&dev_priv->dev->struct_mutex);
 854
 855         misccpctl = I915_READ(GEN7_MISCCPCTL);
 856         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
 857         POSTING_READ(GEN7_MISCCPCTL);
 858
 859         error_status = I915_READ(GEN7_L3CDERRST1);
 860         row = GEN7_PARITY_ERROR_ROW(error_status);
 861         bank = GEN7_PARITY_ERROR_BANK(error_status);
 862         subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
 863
 864         I915_WRITE(GEN7_L3CDERRST1, GEN7_PARITY_ERROR_VALID |
 865                                     GEN7_L3CDERRST1_ENABLE);
 866         POSTING_READ(GEN7_L3CDERRST1);
 867
 868         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
 869
 870         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 871         ilk_enable_gt_irq(dev_priv, GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
 872         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 873
 874         mutex_unlock(&dev_priv->dev->struct_mutex);
 875
 876         parity_event[0] = I915_L3_PARITY_UEVENT "=1";
 877         parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
 878         parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
 879         parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
 880         parity_event[4] = NULL;
 881
 882         kobject_uevent_env(&dev_priv->dev->primary->kdev.kobj,
 883                            KOBJ_CHANGE, parity_event);
 884
 885         DRM_DEBUG("Parity error: Row = %d, Bank = %d, Sub bank = %d.\n",
 886                   row, bank, subbank);
 887
 888         kfree(parity_event[3]);
 889         kfree(parity_event[2]);
 890         kfree(parity_event[1]);
 891 }
 892
 893 static void ivybridge_parity_error_irq_handler(struct drm_device *dev)
 894 {
 895         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 896
 897         if (!HAS_L3_GPU_CACHE(dev))
 898                 return;
 899
 900         spin_lock(&dev_priv->irq_lock);
 901         ilk_disable_gt_irq(dev_priv, GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
 902         spin_unlock(&dev_priv->irq_lock);
 903
 904         queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
 905 }
 906
 907 static void ilk_gt_irq_handler(struct drm_device *dev,
 908                                struct drm_i915_private *dev_priv,
 909                                u32 gt_iir)
 910 {
 911         if (gt_iir &
 912             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
 913                 notify_ring(dev, &dev_priv->ring[RCS]);
 914         if (gt_iir & ILK_BSD_USER_INTERRUPT)
 915                 notify_ring(dev, &dev_priv->ring[VCS]);
 916 }
 917
 918 static void snb_gt_irq_handler(struct drm_device *dev,
 919                                struct drm_i915_private *dev_priv,
 920                                u32 gt_iir)
 921 {
 922
 923         if (gt_iir &
 924             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
 925                 notify_ring(dev, &dev_priv->ring[RCS]);
 926         if (gt_iir & GT_BSD_USER_INTERRUPT)
 927                 notify_ring(dev, &dev_priv->ring[VCS]);
 928         if (gt_iir & GT_BLT_USER_INTERRUPT)
 929                 notify_ring(dev, &dev_priv->ring[BCS]);
 930
 931         if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
 932                       GT_BSD_CS_ERROR_INTERRUPT |
 933                       GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) {
 934                 DRM_ERROR("GT error interrupt 0x%08x\n", gt_iir);
 935                 i915_handle_error(dev, false);
 936         }
 937
 938         if (gt_iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
 939                 ivybridge_parity_error_irq_handler(dev);
 940 }
 941
 942 /* Legacy way of handling PM interrupts */
 943 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv,
 944                                  u32 pm_iir)
 945 {
 946         /*
 947          * IIR bits should never already be set because IMR should
 948          * prevent an interrupt from being shown in IIR. The warning
 949          * displays a case where we've unsafely cleared
 950          * dev_priv->rps.pm_iir. Although missing an interrupt of the same
 951          * type is not a problem, it displays a problem in the logic.
 952          *
 953          * The mask bit in IMR is cleared by dev_priv->rps.work.
 954          */
 955
 956         spin_lock(&dev_priv->irq_lock);
 957         dev_priv->rps.pm_iir |= pm_iir;
 958         snb_set_pm_irq(dev_priv, dev_priv->rps.pm_iir);
 959         spin_unlock(&dev_priv->irq_lock);
 960
 961         queue_work(dev_priv->wq, &dev_priv->rps.work);
 962 }
 963
 964 #define HPD_STORM_DETECT_PERIOD 1000
 965 #define HPD_STORM_THRESHOLD 5
 966
 967 static inline void intel_hpd_irq_handler(struct drm_device *dev,
 968                                          u32 hotplug_trigger,
 969                                          const u32 *hpd)
 970 {
 971         drm_i915_private_t *dev_priv = dev->dev_private;
 972         int i;
 973         bool storm_detected = false;
 974
 975         if (!hotplug_trigger)
 976                 return;
 977
 978         spin_lock(&dev_priv->irq_lock);
 979         for (i = 1; i < HPD_NUM_PINS; i++) {
 980
 981                 WARN(((hpd[i] & hotplug_trigger) &&
 982                       dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED),
 983                      "Received HPD interrupt although disabled\n");
 984
 985                 if (!(hpd[i] & hotplug_trigger) ||
 986                     dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED)
 987                         continue;
 988
 989                 dev_priv->hpd_event_bits |= (1 << i);
 990                 if (!time_in_range(jiffies, dev_priv->hpd_stats[i].hpd_last_jiffies,
 991                                    dev_priv->hpd_stats[i].hpd_last_jiffies
 992                                    + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD))) {
 993                         dev_priv->hpd_stats[i].hpd_last_jiffies = jiffies;
 994                         dev_priv->hpd_stats[i].hpd_cnt = 0;
 995                         DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: 0\n", i);
 996                 } else if (dev_priv->hpd_stats[i].hpd_cnt > HPD_STORM_THRESHOLD) {
 997                         dev_priv->hpd_stats[i].hpd_mark = HPD_MARK_DISABLED;
 998                         dev_priv->hpd_event_bits &= ~(1 << i);
 999                         DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", i);
1000                         storm_detected = true;
1001                 } else {
1002                         dev_priv->hpd_stats[i].hpd_cnt++;
1003                         DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: %d\n", i,
1004                                       dev_priv->hpd_stats[i].hpd_cnt);
1005                 }
1006         }
1007
1008         if (storm_detected)
1009                 dev_priv->display.hpd_irq_setup(dev);
1010         spin_unlock(&dev_priv->irq_lock);
1011
1012         queue_work(dev_priv->wq,
1013                    &dev_priv->hotplug_work);
1014 }
1015
1016 static void gmbus_irq_handler(struct drm_device *dev)
1017 {
1018         struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
1019
1020         wake_up_all(&dev_priv->gmbus_wait_queue);
1021 }
1022
1023 static void dp_aux_irq_handler(struct drm_device *dev)
1024 {
1025         struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
1026
1027         wake_up_all(&dev_priv->gmbus_wait_queue);
1028 }
1029
1030 /* Unlike gen6_rps_irq_handler() from which this function is originally derived,
1031  * we must be able to deal with other PM interrupts. This is complicated because
1032  * of the way in which we use the masks to defer the RPS work (which for
1033  * posterity is necessary because of forcewake).
1034  */
1035 static void hsw_pm_irq_handler(struct drm_i915_private *dev_priv,
1036                                u32 pm_iir)
1037 {
1038         if (pm_iir & GEN6_PM_RPS_EVENTS) {
1039                 spin_lock(&dev_priv->irq_lock);
1040                 dev_priv->rps.pm_iir |= pm_iir & GEN6_PM_RPS_EVENTS;
1041                 snb_set_pm_irq(dev_priv, dev_priv->rps.pm_iir);
1042                 /* never want to mask useful interrupts. */
1043                 WARN_ON(I915_READ_NOTRACE(GEN6_PMIMR) & ~GEN6_PM_RPS_EVENTS);
1044                 spin_unlock(&dev_priv->irq_lock);
1045
1046                 queue_work(dev_priv->wq, &dev_priv->rps.work);
1047         }
1048
1049         if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1050                 notify_ring(dev_priv->dev, &dev_priv->ring[VECS]);
1051
1052         if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) {
1053                 DRM_ERROR("VEBOX CS error interrupt 0x%08x\n", pm_iir);
1054                 i915_handle_error(dev_priv->dev, false);
1055         }
1056 }
1057
1058 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
1059 {
1060         struct drm_device *dev = (struct drm_device *) arg;
1061         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1062         u32 iir, gt_iir, pm_iir;
1063         irqreturn_t ret = IRQ_NONE;
1064         unsigned long irqflags;
1065         int pipe;
1066         u32 pipe_stats[I915_MAX_PIPES];
1067
1068         atomic_inc(&dev_priv->irq_received);
1069
1070         while (true) {
1071                 iir = I915_READ(VLV_IIR);
1072                 gt_iir = I915_READ(GTIIR);
1073                 pm_iir = I915_READ(GEN6_PMIIR);
1074
1075                 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1076                         goto out;
1077
1078                 ret = IRQ_HANDLED;
1079
1080                 snb_gt_irq_handler(dev, dev_priv, gt_iir);
1081
1082                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1083                 for_each_pipe(pipe) {
1084                         int reg = PIPESTAT(pipe);
1085                         pipe_stats[pipe] = I915_READ(reg);
1086
1087                         /*
1088                          * Clear the PIPE*STAT regs before the IIR
1089                          */
1090                         if (pipe_stats[pipe] & 0x8000ffff) {
1091                                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1092                                         DRM_DEBUG_DRIVER("pipe %c underrun\n",
1093                                                          pipe_name(pipe));
1094                                 I915_WRITE(reg, pipe_stats[pipe]);
1095                         }
1096                 }
1097                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1098
1099                 for_each_pipe(pipe) {
1100                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1101                                 drm_handle_vblank(dev, pipe);
1102
1103                         if (pipe_stats[pipe] & PLANE_FLIPDONE_INT_STATUS_VLV) {
1104                                 intel_prepare_page_flip(dev, pipe);
1105                                 intel_finish_page_flip(dev, pipe);
1106                         }
1107                 }
1108
1109                 /* Consume port.  Then clear IIR or we'll miss events */
1110                 if (iir & I915_DISPLAY_PORT_INTERRUPT) {
1111                         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
1112                         u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1113
1114                         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
1115                                          hotplug_status);
1116
1117                         intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
1118
1119                         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1120                         I915_READ(PORT_HOTPLUG_STAT);
1121                 }
1122
1123                 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1124                         gmbus_irq_handler(dev);
1125
1126                 if (pm_iir & GEN6_PM_RPS_EVENTS)
1127                         gen6_rps_irq_handler(dev_priv, pm_iir);
1128
1129                 I915_WRITE(GTIIR, gt_iir);
1130                 I915_WRITE(GEN6_PMIIR, pm_iir);
1131                 I915_WRITE(VLV_IIR, iir);
1132         }
1133
1134 out:
1135         return ret;
1136 }
1137
1138 static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1139 {
1140         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1141         int pipe;
1142         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1143
1144         intel_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
1145
1146         if (pch_iir & SDE_AUDIO_POWER_MASK) {
1147                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1148                                SDE_AUDIO_POWER_SHIFT);
1149                 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1150                                  port_name(port));
1151         }
1152
1153         if (pch_iir & SDE_AUX_MASK)
1154                 dp_aux_irq_handler(dev);
1155
1156         if (pch_iir & SDE_GMBUS)
1157                 gmbus_irq_handler(dev);
1158
1159         if (pch_iir & SDE_AUDIO_HDCP_MASK)
1160                 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
1161
1162         if (pch_iir & SDE_AUDIO_TRANS_MASK)
1163                 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
1164
1165         if (pch_iir & SDE_POISON)
1166                 DRM_ERROR("PCH poison interrupt\n");
1167
1168         if (pch_iir & SDE_FDI_MASK)
1169                 for_each_pipe(pipe)
1170                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1171                                          pipe_name(pipe),
1172                                          I915_READ(FDI_RX_IIR(pipe)));
1173
1174         if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1175                 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
1176
1177         if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1178                 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
1179
1180         if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1181                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
1182                                                           false))
1183                         DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
1184
1185         if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1186                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
1187                                                           false))
1188                         DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
1189 }
1190
1191 static void ivb_err_int_handler(struct drm_device *dev)
1192 {
1193         struct drm_i915_private *dev_priv = dev->dev_private;
1194         u32 err_int = I915_READ(GEN7_ERR_INT);
1195
1196         if (err_int & ERR_INT_POISON)
1197                 DRM_ERROR("Poison interrupt\n");
1198
1199         if (err_int & ERR_INT_FIFO_UNDERRUN_A)
1200                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_A, false))
1201                         DRM_DEBUG_DRIVER("Pipe A FIFO underrun\n");
1202
1203         if (err_int & ERR_INT_FIFO_UNDERRUN_B)
1204                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_B, false))
1205                         DRM_DEBUG_DRIVER("Pipe B FIFO underrun\n");
1206
1207         if (err_int & ERR_INT_FIFO_UNDERRUN_C)
1208                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_C, false))
1209                         DRM_DEBUG_DRIVER("Pipe C FIFO underrun\n");
1210
1211         I915_WRITE(GEN7_ERR_INT, err_int);
1212 }
1213
1214 static void cpt_serr_int_handler(struct drm_device *dev)
1215 {
1216         struct drm_i915_private *dev_priv = dev->dev_private;
1217         u32 serr_int = I915_READ(SERR_INT);
1218
1219         if (serr_int & SERR_INT_POISON)
1220                 DRM_ERROR("PCH poison interrupt\n");
1221
1222         if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1223                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
1224                                                           false))
1225                         DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
1226
1227         if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1228                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
1229                                                           false))
1230                         DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
1231
1232         if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1233                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_C,
1234                                                           false))
1235                         DRM_DEBUG_DRIVER("PCH transcoder C FIFO underrun\n");
1236
1237         I915_WRITE(SERR_INT, serr_int);
1238 }
1239
1240 static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
1241 {
1242         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1243         int pipe;
1244         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1245
1246         intel_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
1247
1248         if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1249                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1250                                SDE_AUDIO_POWER_SHIFT_CPT);
1251                 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
1252                                  port_name(port));
1253         }
1254
1255         if (pch_iir & SDE_AUX_MASK_CPT)
1256                 dp_aux_irq_handler(dev);
1257
1258         if (pch_iir & SDE_GMBUS_CPT)
1259                 gmbus_irq_handler(dev);
1260
1261         if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1262                 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
1263
1264         if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1265                 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
1266
1267         if (pch_iir & SDE_FDI_MASK_CPT)
1268                 for_each_pipe(pipe)
1269                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1270                                          pipe_name(pipe),
1271                                          I915_READ(FDI_RX_IIR(pipe)));
1272
1273         if (pch_iir & SDE_ERROR_CPT)
1274                 cpt_serr_int_handler(dev);
1275 }
1276
1277 static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
1278 {
1279         struct drm_i915_private *dev_priv = dev->dev_private;
1280
1281         if (de_iir & DE_AUX_CHANNEL_A)
1282                 dp_aux_irq_handler(dev);
1283
1284         if (de_iir & DE_GSE)
1285                 intel_opregion_asle_intr(dev);
1286
1287         if (de_iir & DE_PIPEA_VBLANK)
1288                 drm_handle_vblank(dev, 0);
1289
1290         if (de_iir & DE_PIPEB_VBLANK)
1291                 drm_handle_vblank(dev, 1);
1292
1293         if (de_iir & DE_POISON)
1294                 DRM_ERROR("Poison interrupt\n");
1295
1296         if (de_iir & DE_PIPEA_FIFO_UNDERRUN)
1297                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_A, false))
1298                         DRM_DEBUG_DRIVER("Pipe A FIFO underrun\n");
1299
1300         if (de_iir & DE_PIPEB_FIFO_UNDERRUN)
1301                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_B, false))
1302                         DRM_DEBUG_DRIVER("Pipe B FIFO underrun\n");
1303
1304         if (de_iir & DE_PLANEA_FLIP_DONE) {
1305                 intel_prepare_page_flip(dev, 0);
1306                 intel_finish_page_flip_plane(dev, 0);
1307         }
1308
1309         if (de_iir & DE_PLANEB_FLIP_DONE) {
1310                 intel_prepare_page_flip(dev, 1);
1311                 intel_finish_page_flip_plane(dev, 1);
1312         }
1313
1314         /* check event from PCH */
1315         if (de_iir & DE_PCH_EVENT) {
1316                 u32 pch_iir = I915_READ(SDEIIR);
1317
1318                 if (HAS_PCH_CPT(dev))
1319                         cpt_irq_handler(dev, pch_iir);
1320                 else
1321                         ibx_irq_handler(dev, pch_iir);
1322
1323                 /* should clear PCH hotplug event before clear CPU irq */
1324                 I915_WRITE(SDEIIR, pch_iir);
1325         }
1326
1327         if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
1328                 ironlake_rps_change_irq_handler(dev);
1329 }
1330
1331 static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
1332 {
1333         struct drm_i915_private *dev_priv = dev->dev_private;
1334         int i;
1335
1336         if (de_iir & DE_ERR_INT_IVB)
1337                 ivb_err_int_handler(dev);
1338
1339         if (de_iir & DE_AUX_CHANNEL_A_IVB)
1340                 dp_aux_irq_handler(dev);
1341
1342         if (de_iir & DE_GSE_IVB)
1343                 intel_opregion_asle_intr(dev);
1344
1345         for (i = 0; i < 3; i++) {
1346                 if (de_iir & (DE_PIPEA_VBLANK_IVB << (5 * i)))
1347                         drm_handle_vblank(dev, i);
1348                 if (de_iir & (DE_PLANEA_FLIP_DONE_IVB << (5 * i))) {
1349                         intel_prepare_page_flip(dev, i);
1350                         intel_finish_page_flip_plane(dev, i);
1351                 }
1352         }
1353
1354         /* check event from PCH */
1355         if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
1356                 u32 pch_iir = I915_READ(SDEIIR);
1357
1358                 cpt_irq_handler(dev, pch_iir);
1359
1360                 /* clear PCH hotplug event before clear CPU irq */
1361                 I915_WRITE(SDEIIR, pch_iir);
1362         }
1363 }
1364
1365 static irqreturn_t ironlake_irq_handler(int irq, void *arg)
1366 {
1367         struct drm_device *dev = (struct drm_device *) arg;
1368         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1369         u32 de_iir, gt_iir, de_ier, sde_ier = 0;
1370         irqreturn_t ret = IRQ_NONE;
1371
1372         atomic_inc(&dev_priv->irq_received);
1373
1374         /* We get interrupts on unclaimed registers, so check for this before we
1375          * do any I915_{READ,WRITE}. */
1376         intel_uncore_check_errors(dev);
1377
1378         /* disable master interrupt before clearing iir  */
1379         de_ier = I915_READ(DEIER);
1380         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
1381         POSTING_READ(DEIER);
1382
1383         /* Disable south interrupts. We'll only write to SDEIIR once, so further
1384          * interrupts will will be stored on its back queue, and then we'll be
1385          * able to process them after we restore SDEIER (as soon as we restore
1386          * it, we'll get an interrupt if SDEIIR still has something to process
1387          * due to its back queue). */
1388         if (!HAS_PCH_NOP(dev)) {
1389                 sde_ier = I915_READ(SDEIER);
1390                 I915_WRITE(SDEIER, 0);
1391                 POSTING_READ(SDEIER);
1392         }
1393
1394         /* On Haswell, also mask ERR_INT because we don't want to risk
1395          * generating "unclaimed register" interrupts from inside the interrupt
1396          * handler. */
1397         if (IS_HASWELL(dev)) {
1398                 spin_lock(&dev_priv->irq_lock);
1399                 ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
1400                 spin_unlock(&dev_priv->irq_lock);
1401         }
1402
1403         gt_iir = I915_READ(GTIIR);
1404         if (gt_iir) {
1405                 if (INTEL_INFO(dev)->gen >= 6)
1406                         snb_gt_irq_handler(dev, dev_priv, gt_iir);
1407                 else
1408                         ilk_gt_irq_handler(dev, dev_priv, gt_iir);
1409                 I915_WRITE(GTIIR, gt_iir);
1410                 ret = IRQ_HANDLED;
1411         }
1412
1413         de_iir = I915_READ(DEIIR);
1414         if (de_iir) {
1415                 if (INTEL_INFO(dev)->gen >= 7)
1416                         ivb_display_irq_handler(dev, de_iir);
1417                 else
1418                         ilk_display_irq_handler(dev, de_iir);
1419                 I915_WRITE(DEIIR, de_iir);
1420                 ret = IRQ_HANDLED;
1421         }
1422
1423         if (INTEL_INFO(dev)->gen >= 6) {
1424                 u32 pm_iir = I915_READ(GEN6_PMIIR);
1425                 if (pm_iir) {
1426                         if (IS_HASWELL(dev))
1427                                 hsw_pm_irq_handler(dev_priv, pm_iir);
1428                         else if (pm_iir & GEN6_PM_RPS_EVENTS)
1429                                 gen6_rps_irq_handler(dev_priv, pm_iir);
1430                         I915_WRITE(GEN6_PMIIR, pm_iir);
1431                         ret = IRQ_HANDLED;
1432                 }
1433         }
1434
1435         if (IS_HASWELL(dev)) {
1436                 spin_lock(&dev_priv->irq_lock);
1437                 if (ivb_can_enable_err_int(dev))
1438                         ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
1439                 spin_unlock(&dev_priv->irq_lock);
1440         }
1441
1442         I915_WRITE(DEIER, de_ier);
1443         POSTING_READ(DEIER);
1444         if (!HAS_PCH_NOP(dev)) {
1445                 I915_WRITE(SDEIER, sde_ier);
1446                 POSTING_READ(SDEIER);
1447         }
1448
1449         return ret;
1450 }
1451
1452 /**
1453  * i915_error_work_func - do process context error handling work
1454  * @work: work struct
1455  *
1456  * Fire an error uevent so userspace can see that a hang or error
1457  * was detected.
1458  */
1459 static void i915_error_work_func(struct work_struct *work)
1460 {
1461         struct i915_gpu_error *error = container_of(work, struct i915_gpu_error,
1462                                                     work);
1463         drm_i915_private_t *dev_priv = container_of(error, drm_i915_private_t,
1464                                                     gpu_error);
1465         struct drm_device *dev = dev_priv->dev;
1466         struct intel_ring_buffer *ring;
1467         char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
1468         char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
1469         char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
1470         int i, ret;
1471
1472         kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, error_event);
1473
1474         /*
1475          * Note that there's only one work item which does gpu resets, so we
1476          * need not worry about concurrent gpu resets potentially incrementing
1477          * error->reset_counter twice. We only need to take care of another
1478          * racing irq/hangcheck declaring the gpu dead for a second time. A
1479          * quick check for that is good enough: schedule_work ensures the
1480          * correct ordering between hang detection and this work item, and since
1481          * the reset in-progress bit is only ever set by code outside of this
1482          * work we don't need to worry about any other races.
1483          */
1484         if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
1485                 DRM_DEBUG_DRIVER("resetting chip\n");
1486                 kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE,
1487                                    reset_event);
1488
1489                 ret = i915_reset(dev);
1490
1491                 if (ret == 0) {
1492                         /*
1493                          * After all the gem state is reset, increment the reset
1494                          * counter and wake up everyone waiting for the reset to
1495                          * complete.
1496                          *
1497                          * Since unlock operations are a one-sided barrier only,
1498                          * we need to insert a barrier here to order any seqno
1499                          * updates before
1500                          * the counter increment.
1501                          */
1502                         smp_mb__before_atomic_inc();
1503                         atomic_inc(&dev_priv->gpu_error.reset_counter);
1504
1505                         kobject_uevent_env(&dev->primary->kdev.kobj,
1506                                            KOBJ_CHANGE, reset_done_event);
1507                 } else {
1508                         atomic_set(&error->reset_counter, I915_WEDGED);
1509                 }
1510
1511                 for_each_ring(ring, dev_priv, i)
1512                         wake_up_all(&ring->irq_queue);
1513
1514                 intel_display_handle_reset(dev);
1515
1516                 wake_up_all(&dev_priv->gpu_error.reset_queue);
1517         }
1518 }
1519
1520 static void i915_report_and_clear_eir(struct drm_device *dev)
1521 {
1522         struct drm_i915_private *dev_priv = dev->dev_private;
1523         uint32_t instdone[I915_NUM_INSTDONE_REG];
1524         u32 eir = I915_READ(EIR);
1525         int pipe, i;
1526
1527         if (!eir)
1528                 return;
1529
1530         pr_err("render error detected, EIR: 0x%08x\n", eir);
1531
1532         i915_get_extra_instdone(dev, instdone);
1533
1534         if (IS_G4X(dev)) {
1535                 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
1536                         u32 ipeir = I915_READ(IPEIR_I965);
1537
1538                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
1539                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
1540                         for (i = 0; i < ARRAY_SIZE(instdone); i++)
1541                                 pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
1542                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
1543                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
1544                         I915_WRITE(IPEIR_I965, ipeir);
1545                         POSTING_READ(IPEIR_I965);
1546                 }
1547                 if (eir & GM45_ERROR_PAGE_TABLE) {
1548                         u32 pgtbl_err = I915_READ(PGTBL_ER);
1549                         pr_err("page table error\n");
1550                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
1551                         I915_WRITE(PGTBL_ER, pgtbl_err);
1552                         POSTING_READ(PGTBL_ER);
1553                 }
1554         }
1555
1556         if (!IS_GEN2(dev)) {
1557                 if (eir & I915_ERROR_PAGE_TABLE) {
1558                         u32 pgtbl_err = I915_READ(PGTBL_ER);
1559                         pr_err("page table error\n");
1560                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
1561                         I915_WRITE(PGTBL_ER, pgtbl_err);
1562                         POSTING_READ(PGTBL_ER);
1563                 }
1564         }
1565
1566         if (eir & I915_ERROR_MEMORY_REFRESH) {
1567                 pr_err("memory refresh error:\n");
1568                 for_each_pipe(pipe)
1569                         pr_err("pipe %c stat: 0x%08x\n",
1570                                pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
1571                 /* pipestat has already been acked */
1572         }
1573         if (eir & I915_ERROR_INSTRUCTION) {
1574                 pr_err("instruction error\n");
1575                 pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
1576                 for (i = 0; i < ARRAY_SIZE(instdone); i++)
1577                         pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
1578                 if (INTEL_INFO(dev)->gen < 4) {
1579                         u32 ipeir = I915_READ(IPEIR);
1580
1581                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
1582                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
1583                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
1584                         I915_WRITE(IPEIR, ipeir);
1585                         POSTING_READ(IPEIR);
1586                 } else {
1587                         u32 ipeir = I915_READ(IPEIR_I965);
1588
1589                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
1590                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
1591                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
1592                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
1593                         I915_WRITE(IPEIR_I965, ipeir);
1594                         POSTING_READ(IPEIR_I965);
1595                 }
1596         }
1597
1598         I915_WRITE(EIR, eir);
1599         POSTING_READ(EIR);
1600         eir = I915_READ(EIR);
1601         if (eir) {
1602                 /*
1603                  * some errors might have become stuck,
1604                  * mask them.
1605                  */
1606                 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
1607                 I915_WRITE(EMR, I915_READ(EMR) | eir);
1608                 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
1609         }
1610 }
1611
1612 /**
1613  * i915_handle_error - handle an error interrupt
1614  * @dev: drm device
1615  *
1616  * Do some basic checking of regsiter state at error interrupt time and
1617  * dump it to the syslog.  Also call i915_capture_error_state() to make
1618  * sure we get a record and make it available in debugfs.  Fire a uevent
1619  * so userspace knows something bad happened (should trigger collection
1620  * of a ring dump etc.).
1621  */
1622 void i915_handle_error(struct drm_device *dev, bool wedged)
1623 {
1624         struct drm_i915_private *dev_priv = dev->dev_private;
1625         struct intel_ring_buffer *ring;
1626         int i;
1627
1628         i915_capture_error_state(dev);
1629         i915_report_and_clear_eir(dev);
1630
1631         if (wedged) {
1632                 atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
1633                                 &dev_priv->gpu_error.reset_counter);
1634
1635                 /*
1636                  * Wakeup waiting processes so that the reset work item
1637                  * doesn't deadlock trying to grab various locks.
1638                  */
1639                 for_each_ring(ring, dev_priv, i)
1640                         wake_up_all(&ring->irq_queue);
1641         }
1642
1643         queue_work(dev_priv->wq, &dev_priv->gpu_error.work);
1644 }
1645
1646 static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, int pipe)
1647 {
1648         drm_i915_private_t *dev_priv = dev->dev_private;
1649         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
1650         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1651         struct drm_i915_gem_object *obj;
1652         struct intel_unpin_work *work;
1653         unsigned long flags;
1654         bool stall_detected;
1655
1656         /* Ignore early vblank irqs */
1657         if (intel_crtc == NULL)
1658                 return;
1659
1660         spin_lock_irqsave(&dev->event_lock, flags);
1661         work = intel_crtc->unpin_work;
1662
1663         if (work == NULL ||
1664             atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE ||
1665             !work->enable_stall_check) {
1666                 /* Either the pending flip IRQ arrived, or we're too early. Don't check */
1667                 spin_unlock_irqrestore(&dev->event_lock, flags);
1668                 return;
1669         }
1670
1671         /* Potential stall - if we see that the flip has happened, assume a missed interrupt */
1672         obj = work->pending_flip_obj;
1673         if (INTEL_INFO(dev)->gen >= 4) {
1674                 int dspsurf = DSPSURF(intel_crtc->plane);
1675                 stall_detected = I915_HI_DISPBASE(I915_READ(dspsurf)) ==
1676                                         i915_gem_obj_ggtt_offset(obj);
1677         } else {
1678                 int dspaddr = DSPADDR(intel_crtc->plane);
1679                 stall_detected = I915_READ(dspaddr) == (i915_gem_obj_ggtt_offset(obj) +
1680                                                         crtc->y * crtc->fb->pitches[0] +
1681                                                         crtc->x * crtc->fb->bits_per_pixel/8);
1682         }
1683
1684         spin_unlock_irqrestore(&dev->event_lock, flags);
1685
1686         if (stall_detected) {
1687                 DRM_DEBUG_DRIVER("Pageflip stall detected\n");
1688                 intel_prepare_page_flip(dev, intel_crtc->plane);
1689         }
1690 }
1691
1692 /* Called from drm generic code, passed 'crtc' which
1693  * we use as a pipe index
1694  */
1695 static int i915_enable_vblank(struct drm_device *dev, int pipe)
1696 {
1697         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1698         unsigned long irqflags;
1699
1700         if (!i915_pipe_enabled(dev, pipe))
1701                 return -EINVAL;
1702
1703         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1704         if (INTEL_INFO(dev)->gen >= 4)
1705                 i915_enable_pipestat(dev_priv, pipe,
1706                                      PIPE_START_VBLANK_INTERRUPT_ENABLE);
1707         else
1708                 i915_enable_pipestat(dev_priv, pipe,
1709                                      PIPE_VBLANK_INTERRUPT_ENABLE);
1710
1711         /* maintain vblank delivery even in deep C-states */
1712         if (dev_priv->info->gen == 3)
1713                 I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_AGPBUSY_DIS));
1714         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1715
1716         return 0;
1717 }
1718
1719 static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
1720 {
1721         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1722         unsigned long irqflags;
1723         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
1724                                                      DE_PIPE_VBLANK_ILK(pipe);
1725
1726         if (!i915_pipe_enabled(dev, pipe))
1727                 return -EINVAL;
1728
1729         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1730         ironlake_enable_display_irq(dev_priv, bit);
1731         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1732
1733         return 0;
1734 }
1735
1736 static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
1737 {
1738         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1739         unsigned long irqflags;
1740         u32 imr;
1741
1742         if (!i915_pipe_enabled(dev, pipe))
1743                 return -EINVAL;
1744
1745         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1746         imr = I915_READ(VLV_IMR);
1747         if (pipe == 0)
1748                 imr &= ~I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
1749         else
1750                 imr &= ~I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
1751         I915_WRITE(VLV_IMR, imr);
1752         i915_enable_pipestat(dev_priv, pipe,
1753                              PIPE_START_VBLANK_INTERRUPT_ENABLE);
1754         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1755
1756         return 0;
1757 }
1758
1759 /* Called from drm generic code, passed 'crtc' which
1760  * we use as a pipe index
1761  */
1762 static void i915_disable_vblank(struct drm_device *dev, int pipe)
1763 {
1764         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1765         unsigned long irqflags;
1766
1767         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1768         if (dev_priv->info->gen == 3)
1769                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_DIS));
1770
1771         i915_disable_pipestat(dev_priv, pipe,
1772                               PIPE_VBLANK_INTERRUPT_ENABLE |
1773                               PIPE_START_VBLANK_INTERRUPT_ENABLE);
1774         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1775 }
1776
1777 static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
1778 {
1779         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1780         unsigned long irqflags;
1781         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
1782                                                      DE_PIPE_VBLANK_ILK(pipe);
1783
1784         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1785         ironlake_disable_display_irq(dev_priv, bit);
1786         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1787 }
1788
1789 static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
1790 {
1791         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1792         unsigned long irqflags;
1793         u32 imr;
1794
1795         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1796         i915_disable_pipestat(dev_priv, pipe,
1797                               PIPE_START_VBLANK_INTERRUPT_ENABLE);
1798         imr = I915_READ(VLV_IMR);
1799         if (pipe == 0)
1800                 imr |= I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
1801         else
1802                 imr |= I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
1803         I915_WRITE(VLV_IMR, imr);
1804         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1805 }
1806
1807 static u32
1808 ring_last_seqno(struct intel_ring_buffer *ring)
1809 {
1810         return list_entry(ring->request_list.prev,
1811                           struct drm_i915_gem_request, list)->seqno;
1812 }
1813
1814 static bool
1815 ring_idle(struct intel_ring_buffer *ring, u32 seqno)
1816 {
1817         return (list_empty(&ring->request_list) ||
1818                 i915_seqno_passed(seqno, ring_last_seqno(ring)));
1819 }
1820
1821 static struct intel_ring_buffer *
1822 semaphore_waits_for(struct intel_ring_buffer *ring, u32 *seqno)
1823 {
1824         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1825         u32 cmd, ipehr, acthd, acthd_min;
1826
1827         ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
1828         if ((ipehr & ~(0x3 << 16)) !=
1829             (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | MI_SEMAPHORE_REGISTER))
1830                 return NULL;
1831
1832         /* ACTHD is likely pointing to the dword after the actual command,
1833          * so scan backwards until we find the MBOX.
1834          */
1835         acthd = intel_ring_get_active_head(ring) & HEAD_ADDR;
1836         acthd_min = max((int)acthd - 3 * 4, 0);
1837         do {
1838                 cmd = ioread32(ring->virtual_start + acthd);
1839                 if (cmd == ipehr)
1840                         break;
1841
1842                 acthd -= 4;
1843                 if (acthd < acthd_min)
1844                         return NULL;
1845         } while (1);
1846
1847         *seqno = ioread32(ring->virtual_start+acthd+4)+1;
1848         return &dev_priv->ring[(ring->id + (((ipehr >> 17) & 1) + 1)) % 3];
1849 }
1850
1851 static int semaphore_passed(struct intel_ring_buffer *ring)
1852 {
1853         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1854         struct intel_ring_buffer *signaller;
1855         u32 seqno, ctl;
1856
1857         ring->hangcheck.deadlock = true;
1858
1859         signaller = semaphore_waits_for(ring, &seqno);
1860         if (signaller == NULL || signaller->hangcheck.deadlock)
1861                 return -1;
1862
1863         /* cursory check for an unkickable deadlock */
1864         ctl = I915_READ_CTL(signaller);
1865         if (ctl & RING_WAIT_SEMAPHORE && semaphore_passed(signaller) < 0)
1866                 return -1;
1867
1868         return i915_seqno_passed(signaller->get_seqno(signaller, false), seqno);
1869 }
1870
1871 static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
1872 {
1873         struct intel_ring_buffer *ring;
1874         int i;
1875
1876         for_each_ring(ring, dev_priv, i)
1877                 ring->hangcheck.deadlock = false;
1878 }
1879
1880 static enum intel_ring_hangcheck_action
1881 ring_stuck(struct intel_ring_buffer *ring, u32 acthd)
1882 {
1883         struct drm_device *dev = ring->dev;
1884         struct drm_i915_private *dev_priv = dev->dev_private;
1885         u32 tmp;
1886
1887         if (ring->hangcheck.acthd != acthd)
1888                 return HANGCHECK_ACTIVE;
1889
1890         if (IS_GEN2(dev))
1891                 return HANGCHECK_HUNG;
1892
1893         /* Is the chip hanging on a WAIT_FOR_EVENT?
1894          * If so we can simply poke the RB_WAIT bit
1895          * and break the hang. This should work on
1896          * all but the second generation chipsets.
1897          */
1898         tmp = I915_READ_CTL(ring);
1899         if (tmp & RING_WAIT) {
1900                 DRM_ERROR("Kicking stuck wait on %s\n",
1901                           ring->name);
1902                 I915_WRITE_CTL(ring, tmp);
1903                 return HANGCHECK_KICK;
1904         }
1905
1906         if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
1907                 switch (semaphore_passed(ring)) {
1908                 default:
1909                         return HANGCHECK_HUNG;
1910                 case 1:
1911                         DRM_ERROR("Kicking stuck semaphore on %s\n",
1912                                   ring->name);
1913                         I915_WRITE_CTL(ring, tmp);
1914                         return HANGCHECK_KICK;
1915                 case 0:
1916                         return HANGCHECK_WAIT;
1917                 }
1918         }
1919
1920         return HANGCHECK_HUNG;
1921 }
1922
1923 /**
1924  * This is called when the chip hasn't reported back with completed
1925  * batchbuffers in a long time. We keep track per ring seqno progress and
1926  * if there are no progress, hangcheck score for that ring is increased.
1927  * Further, acthd is inspected to see if the ring is stuck. On stuck case
1928  * we kick the ring. If we see no progress on three subsequent calls
1929  * we assume chip is wedged and try to fix it by resetting the chip.
1930  */
1931 static void i915_hangcheck_elapsed(unsigned long data)
1932 {
1933         struct drm_device *dev = (struct drm_device *)data;
1934         drm_i915_private_t *dev_priv = dev->dev_private;
1935         struct intel_ring_buffer *ring;
1936         int i;
1937         int busy_count = 0, rings_hung = 0;
1938         bool stuck[I915_NUM_RINGS] = { 0 };
1939 #define BUSY 1
1940 #define KICK 5
1941 #define HUNG 20
1942 #define FIRE 30
1943
1944         if (!i915_enable_hangcheck)
1945                 return;
1946
1947         for_each_ring(ring, dev_priv, i) {
1948                 u32 seqno, acthd;
1949                 bool busy = true;
1950
1951                 semaphore_clear_deadlocks(dev_priv);
1952
1953                 seqno = ring->get_seqno(ring, false);
1954                 acthd = intel_ring_get_active_head(ring);
1955
1956                 if (ring->hangcheck.seqno == seqno) {
1957                         if (ring_idle(ring, seqno)) {
1958                                 if (waitqueue_active(&ring->irq_queue)) {
1959                                         /* Issue a wake-up to catch stuck h/w. */
1960                                         DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
1961                                                   ring->name);
1962                                         wake_up_all(&ring->irq_queue);
1963                                         ring->hangcheck.score += HUNG;
1964                                 } else
1965                                         busy = false;
1966                         } else {
1967                                 /* We always increment the hangcheck score
1968                                  * if the ring is busy and still processing
1969                                  * the same request, so that no single request
1970                                  * can run indefinitely (such as a chain of
1971                                  * batches). The only time we do not increment
1972                                  * the hangcheck score on this ring, if this
1973                                  * ring is in a legitimate wait for another
1974                                  * ring. In that case the waiting ring is a
1975                                  * victim and we want to be sure we catch the
1976                                  * right culprit. Then every time we do kick
1977                                  * the ring, add a small increment to the
1978                                  * score so that we can catch a batch that is
1979                                  * being repeatedly kicked and so responsible
1980                                  * for stalling the machine.
1981                                  */
1982                                 ring->hangcheck.action = ring_stuck(ring,
1983                                                                     acthd);
1984
1985                                 switch (ring->hangcheck.action) {
1986                                 case HANGCHECK_WAIT:
1987                                         break;
1988                                 case HANGCHECK_ACTIVE:
1989                                         ring->hangcheck.score += BUSY;
1990                                         break;
1991                                 case HANGCHECK_KICK:
1992                                         ring->hangcheck.score += KICK;
1993                                         break;
1994                                 case HANGCHECK_HUNG:
1995                                         ring->hangcheck.score += HUNG;
1996                                         stuck[i] = true;
1997                                         break;
1998                                 }
1999                         }
2000                 } else {
2001                         /* Gradually reduce the count so that we catch DoS
2002                          * attempts across multiple batches.
2003                          */
2004                         if (ring->hangcheck.score > 0)
2005                                 ring->hangcheck.score--;
2006                 }
2007
2008                 ring->hangcheck.seqno = seqno;
2009                 ring->hangcheck.acthd = acthd;
2010                 busy_count += busy;
2011         }
2012
2013         for_each_ring(ring, dev_priv, i) {
2014                 if (ring->hangcheck.score > FIRE) {
2015                         DRM_ERROR("%s on %s\n",
2016                                   stuck[i] ? "stuck" : "no progress",
2017                                   ring->name);
2018                         rings_hung++;
2019                 }
2020         }
2021
2022         if (rings_hung)
2023                 return i915_handle_error(dev, true);
2024
2025         if (busy_count)
2026                 /* Reset timer case chip hangs without another request
2027                  * being added */
2028                 i915_queue_hangcheck(dev);
2029 }
2030
2031 void i915_queue_hangcheck(struct drm_device *dev)
2032 {
2033         struct drm_i915_private *dev_priv = dev->dev_private;
2034         if (!i915_enable_hangcheck)
2035                 return;
2036
2037         mod_timer(&dev_priv->gpu_error.hangcheck_timer,
2038                   round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
2039 }
2040
2041 static void ibx_irq_preinstall(struct drm_device *dev)
2042 {
2043         struct drm_i915_private *dev_priv = dev->dev_private;
2044
2045         if (HAS_PCH_NOP(dev))
2046                 return;
2047
2048         /* south display irq */
2049         I915_WRITE(SDEIMR, 0xffffffff);
2050         /*
2051          * SDEIER is also touched by the interrupt handler to work around missed
2052          * PCH interrupts. Hence we can't update it after the interrupt handler
2053          * is enabled - instead we unconditionally enable all PCH interrupt
2054          * sources here, but then only unmask them as needed with SDEIMR.
2055          */
2056         I915_WRITE(SDEIER, 0xffffffff);
2057         POSTING_READ(SDEIER);
2058 }
2059
2060 static void gen5_gt_irq_preinstall(struct drm_device *dev)
2061 {
2062         struct drm_i915_private *dev_priv = dev->dev_private;
2063
2064         /* and GT */
2065         I915_WRITE(GTIMR, 0xffffffff);
2066         I915_WRITE(GTIER, 0x0);
2067         POSTING_READ(GTIER);
2068
2069         if (INTEL_INFO(dev)->gen >= 6) {
2070                 /* and PM */
2071                 I915_WRITE(GEN6_PMIMR, 0xffffffff);
2072                 I915_WRITE(GEN6_PMIER, 0x0);
2073                 POSTING_READ(GEN6_PMIER);
2074         }
2075 }
2076
2077 /* drm_dma.h hooks
2078 */
2079 static void ironlake_irq_preinstall(struct drm_device *dev)
2080 {
2081         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2082
2083         atomic_set(&dev_priv->irq_received, 0);
2084
2085         I915_WRITE(HWSTAM, 0xeffe);
2086
2087         I915_WRITE(DEIMR, 0xffffffff);
2088         I915_WRITE(DEIER, 0x0);
2089         POSTING_READ(DEIER);
2090
2091         gen5_gt_irq_preinstall(dev);
2092
2093         ibx_irq_preinstall(dev);
2094 }
2095
2096 static void valleyview_irq_preinstall(struct drm_device *dev)
2097 {
2098         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2099         int pipe;
2100
2101         atomic_set(&dev_priv->irq_received, 0);
2102
2103         /* VLV magic */
2104         I915_WRITE(VLV_IMR, 0);
2105         I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
2106         I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
2107         I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
2108
2109         /* and GT */
2110         I915_WRITE(GTIIR, I915_READ(GTIIR));
2111         I915_WRITE(GTIIR, I915_READ(GTIIR));
2112
2113         gen5_gt_irq_preinstall(dev);
2114
2115         I915_WRITE(DPINVGTT, 0xff);
2116
2117         I915_WRITE(PORT_HOTPLUG_EN, 0);
2118         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2119         for_each_pipe(pipe)
2120                 I915_WRITE(PIPESTAT(pipe), 0xffff);
2121         I915_WRITE(VLV_IIR, 0xffffffff);
2122         I915_WRITE(VLV_IMR, 0xffffffff);
2123         I915_WRITE(VLV_IER, 0x0);
2124         POSTING_READ(VLV_IER);
2125 }
2126
2127 static void ibx_hpd_irq_setup(struct drm_device *dev)
2128 {
2129         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2130         struct drm_mode_config *mode_config = &dev->mode_config;
2131         struct intel_encoder *intel_encoder;
2132         u32 hotplug_irqs, hotplug, enabled_irqs = 0;
2133
2134         if (HAS_PCH_IBX(dev)) {
2135                 hotplug_irqs = SDE_HOTPLUG_MASK;
2136                 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
2137                         if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
2138                                 enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
2139         } else {
2140                 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
2141                 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
2142                         if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
2143                                 enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
2144         }
2145
2146         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
2147
2148         /*
2149          * Enable digital hotplug on the PCH, and configure the DP short pulse
2150          * duration to 2ms (which is the minimum in the Display Port spec)
2151          *
2152          * This register is the same on all known PCH chips.
2153          */
2154         hotplug = I915_READ(PCH_PORT_HOTPLUG);
2155         hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
2156         hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
2157         hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
2158         hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
2159         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
2160 }
2161
2162 static void ibx_irq_postinstall(struct drm_device *dev)
2163 {
2164         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2165         u32 mask;
2166
2167         if (HAS_PCH_NOP(dev))
2168                 return;
2169
2170         if (HAS_PCH_IBX(dev)) {
2171                 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_TRANSB_FIFO_UNDER |
2172                        SDE_TRANSA_FIFO_UNDER | SDE_POISON;
2173         } else {
2174                 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT | SDE_ERROR_CPT;
2175
2176                 I915_WRITE(SERR_INT, I915_READ(SERR_INT));
2177         }
2178
2179         I915_WRITE(SDEIIR, I915_READ(SDEIIR));
2180         I915_WRITE(SDEIMR, ~mask);
2181 }
2182
2183 static void gen5_gt_irq_postinstall(struct drm_device *dev)
2184 {
2185         struct drm_i915_private *dev_priv = dev->dev_private;
2186         u32 pm_irqs, gt_irqs;
2187
2188         pm_irqs = gt_irqs = 0;
2189
2190         dev_priv->gt_irq_mask = ~0;
2191         if (HAS_L3_GPU_CACHE(dev)) {
2192                 /* L3 parity interrupt is always unmasked. */
2193                 dev_priv->gt_irq_mask = ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
2194                 gt_irqs |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
2195         }
2196
2197         gt_irqs |= GT_RENDER_USER_INTERRUPT;
2198         if (IS_GEN5(dev)) {
2199                 gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
2200                            ILK_BSD_USER_INTERRUPT;
2201         } else {
2202                 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
2203         }
2204
2205         I915_WRITE(GTIIR, I915_READ(GTIIR));
2206         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
2207         I915_WRITE(GTIER, gt_irqs);
2208         POSTING_READ(GTIER);
2209
2210         if (INTEL_INFO(dev)->gen >= 6) {
2211                 pm_irqs |= GEN6_PM_RPS_EVENTS;
2212
2213                 if (HAS_VEBOX(dev))
2214                         pm_irqs |= PM_VEBOX_USER_INTERRUPT;
2215
2216                 I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
2217                 I915_WRITE(GEN6_PMIMR, 0xffffffff);
2218                 I915_WRITE(GEN6_PMIER, pm_irqs);
2219                 POSTING_READ(GEN6_PMIER);
2220         }
2221 }
2222
2223 static int ironlake_irq_postinstall(struct drm_device *dev)
2224 {
2225         unsigned long irqflags;
2226         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2227         u32 display_mask, extra_mask;
2228
2229         if (INTEL_INFO(dev)->gen >= 7) {
2230                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
2231                                 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
2232                                 DE_PLANEB_FLIP_DONE_IVB |
2233                                 DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB |
2234                                 DE_ERR_INT_IVB);
2235                 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
2236                               DE_PIPEA_VBLANK_IVB);
2237
2238                 I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
2239         } else {
2240                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
2241                                 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
2242                                 DE_AUX_CHANNEL_A | DE_PIPEB_FIFO_UNDERRUN |
2243                                 DE_PIPEA_FIFO_UNDERRUN | DE_POISON);
2244                 extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT;
2245         }
2246
2247         dev_priv->irq_mask = ~display_mask;
2248
2249         /* should always can generate irq */
2250         I915_WRITE(DEIIR, I915_READ(DEIIR));
2251         I915_WRITE(DEIMR, dev_priv->irq_mask);
2252         I915_WRITE(DEIER, display_mask | extra_mask);
2253         POSTING_READ(DEIER);
2254
2255         gen5_gt_irq_postinstall(dev);
2256
2257         ibx_irq_postinstall(dev);
2258
2259         if (IS_IRONLAKE_M(dev)) {
2260                 /* Enable PCU event interrupts
2261                  *
2262                  * spinlocking not required here for correctness since interrupt
2263                  * setup is guaranteed to run in single-threaded context. But we
2264                  * need it to make the assert_spin_locked happy. */
2265                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2266                 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
2267                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2268         }
2269
2270         return 0;
2271 }
2272
2273 static int valleyview_irq_postinstall(struct drm_device *dev)
2274 {
2275         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2276         u32 enable_mask;
2277         u32 pipestat_enable = PLANE_FLIP_DONE_INT_EN_VLV;
2278         unsigned long irqflags;
2279
2280         enable_mask = I915_DISPLAY_PORT_INTERRUPT;
2281         enable_mask |= I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2282                 I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
2283                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2284                 I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
2285
2286         /*
2287          *Leave vblank interrupts masked initially.  enable/disable will
2288          * toggle them based on usage.
2289          */
2290         dev_priv->irq_mask = (~enable_mask) |
2291                 I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
2292                 I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
2293
2294         I915_WRITE(PORT_HOTPLUG_EN, 0);
2295         POSTING_READ(PORT_HOTPLUG_EN);
2296
2297         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
2298         I915_WRITE(VLV_IER, enable_mask);
2299         I915_WRITE(VLV_IIR, 0xffffffff);
2300         I915_WRITE(PIPESTAT(0), 0xffff);
2301         I915_WRITE(PIPESTAT(1), 0xffff);
2302         POSTING_READ(VLV_IER);
2303
2304         /* Interrupt setup is already guaranteed to be single-threaded, this is
2305          * just to make the assert_spin_locked check happy. */
2306         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2307         i915_enable_pipestat(dev_priv, 0, pipestat_enable);
2308         i915_enable_pipestat(dev_priv, 0, PIPE_GMBUS_EVENT_ENABLE);
2309         i915_enable_pipestat(dev_priv, 1, pipestat_enable);
2310         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2311
2312         I915_WRITE(VLV_IIR, 0xffffffff);
2313         I915_WRITE(VLV_IIR, 0xffffffff);
2314
2315         gen5_gt_irq_postinstall(dev);
2316
2317         /* ack & enable invalid PTE error interrupts */
2318 #if 0 /* FIXME: add support to irq handler for checking these bits */
2319         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
2320         I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
2321 #endif
2322
2323         I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
2324
2325         return 0;
2326 }
2327
2328 static void valleyview_irq_uninstall(struct drm_device *dev)
2329 {
2330         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2331         int pipe;
2332
2333         if (!dev_priv)
2334                 return;
2335
2336         del_timer_sync(&dev_priv->hotplug_reenable_timer);
2337
2338         for_each_pipe(pipe)
2339                 I915_WRITE(PIPESTAT(pipe), 0xffff);
2340
2341         I915_WRITE(HWSTAM, 0xffffffff);
2342         I915_WRITE(PORT_HOTPLUG_EN, 0);
2343         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2344         for_each_pipe(pipe)
2345                 I915_WRITE(PIPESTAT(pipe), 0xffff);
2346         I915_WRITE(VLV_IIR, 0xffffffff);
2347         I915_WRITE(VLV_IMR, 0xffffffff);
2348         I915_WRITE(VLV_IER, 0x0);
2349         POSTING_READ(VLV_IER);
2350 }
2351
2352 static void ironlake_irq_uninstall(struct drm_device *dev)
2353 {
2354         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2355
2356         if (!dev_priv)
2357                 return;
2358
2359         del_timer_sync(&dev_priv->hotplug_reenable_timer);
2360
2361         I915_WRITE(HWSTAM, 0xffffffff);
2362
2363         I915_WRITE(DEIMR, 0xffffffff);
2364         I915_WRITE(DEIER, 0x0);
2365         I915_WRITE(DEIIR, I915_READ(DEIIR));
2366         if (IS_GEN7(dev))
2367                 I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
2368
2369         I915_WRITE(GTIMR, 0xffffffff);
2370         I915_WRITE(GTIER, 0x0);
2371         I915_WRITE(GTIIR, I915_READ(GTIIR));
2372
2373         if (HAS_PCH_NOP(dev))
2374                 return;
2375
2376         I915_WRITE(SDEIMR, 0xffffffff);
2377         I915_WRITE(SDEIER, 0x0);
2378         I915_WRITE(SDEIIR, I915_READ(SDEIIR));
2379         if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
2380                 I915_WRITE(SERR_INT, I915_READ(SERR_INT));
2381 }
2382
2383 static void i8xx_irq_preinstall(struct drm_device * dev)
2384 {
2385         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2386         int pipe;
2387
2388         atomic_set(&dev_priv->irq_received, 0);
2389
2390         for_each_pipe(pipe)
2391                 I915_WRITE(PIPESTAT(pipe), 0);
2392         I915_WRITE16(IMR, 0xffff);
2393         I915_WRITE16(IER, 0x0);
2394         POSTING_READ16(IER);
2395 }
2396
2397 static int i8xx_irq_postinstall(struct drm_device *dev)
2398 {
2399         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2400
2401         I915_WRITE16(EMR,
2402                      ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
2403
2404         /* Unmask the interrupts that we always want on. */
2405         dev_priv->irq_mask =
2406                 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2407                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2408                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2409                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2410                   I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2411         I915_WRITE16(IMR, dev_priv->irq_mask);
2412
2413         I915_WRITE16(IER,
2414                      I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2415                      I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2416                      I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
2417                      I915_USER_INTERRUPT);
2418         POSTING_READ16(IER);
2419
2420         return 0;
2421 }
2422
2423 /*
2424  * Returns true when a page flip has completed.
2425  */
2426 static bool i8xx_handle_vblank(struct drm_device *dev,
2427                                int pipe, u16 iir)
2428 {
2429         drm_i915_private_t *dev_priv = dev->dev_private;
2430         u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(pipe);
2431
2432         if (!drm_handle_vblank(dev, pipe))
2433                 return false;
2434
2435         if ((iir & flip_pending) == 0)
2436                 return false;
2437
2438         intel_prepare_page_flip(dev, pipe);
2439
2440         /* We detect FlipDone by looking for the change in PendingFlip from '1'
2441          * to '0' on the following vblank, i.e. IIR has the Pendingflip
2442          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
2443          * the flip is completed (no longer pending). Since this doesn't raise
2444          * an interrupt per se, we watch for the change at vblank.
2445          */
2446         if (I915_READ16(ISR) & flip_pending)
2447                 return false;
2448
2449         intel_finish_page_flip(dev, pipe);
2450
2451         return true;
2452 }
2453
2454 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
2455 {
2456         struct drm_device *dev = (struct drm_device *) arg;
2457         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2458         u16 iir, new_iir;
2459         u32 pipe_stats[2];
2460         unsigned long irqflags;
2461         int pipe;
2462         u16 flip_mask =
2463                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2464                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
2465
2466         atomic_inc(&dev_priv->irq_received);
2467
2468         iir = I915_READ16(IIR);
2469         if (iir == 0)
2470                 return IRQ_NONE;
2471
2472         while (iir & ~flip_mask) {
2473                 /* Can't rely on pipestat interrupt bit in iir as it might
2474                  * have been cleared after the pipestat interrupt was received.
2475                  * It doesn't set the bit in iir again, but it still produces
2476                  * interrupts (for non-MSI).
2477                  */
2478                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2479                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2480                         i915_handle_error(dev, false);
2481
2482                 for_each_pipe(pipe) {
2483                         int reg = PIPESTAT(pipe);
2484                         pipe_stats[pipe] = I915_READ(reg);
2485
2486                         /*
2487                          * Clear the PIPE*STAT regs before the IIR
2488                          */
2489                         if (pipe_stats[pipe] & 0x8000ffff) {
2490                                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2491                                         DRM_DEBUG_DRIVER("pipe %c underrun\n",
2492                                                          pipe_name(pipe));
2493                                 I915_WRITE(reg, pipe_stats[pipe]);
2494                         }
2495                 }
2496                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2497
2498                 I915_WRITE16(IIR, iir & ~flip_mask);
2499                 new_iir = I915_READ16(IIR); /* Flush posted writes */
2500
2501                 i915_update_dri1_breadcrumb(dev);
2502
2503                 if (iir & I915_USER_INTERRUPT)
2504                         notify_ring(dev, &dev_priv->ring[RCS]);
2505
2506                 if (pipe_stats[0] & PIPE_VBLANK_INTERRUPT_STATUS &&
2507                     i8xx_handle_vblank(dev, 0, iir))
2508                         flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(0);
2509
2510                 if (pipe_stats[1] & PIPE_VBLANK_INTERRUPT_STATUS &&
2511                     i8xx_handle_vblank(dev, 1, iir))
2512                         flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(1);
2513
2514                 iir = new_iir;
2515         }
2516
2517         return IRQ_HANDLED;
2518 }
2519
2520 static void i8xx_irq_uninstall(struct drm_device * dev)
2521 {
2522         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2523         int pipe;
2524
2525         for_each_pipe(pipe) {
2526                 /* Clear enable bits; then clear status bits */
2527                 I915_WRITE(PIPESTAT(pipe), 0);
2528                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
2529         }
2530         I915_WRITE16(IMR, 0xffff);
2531         I915_WRITE16(IER, 0x0);
2532         I915_WRITE16(IIR, I915_READ16(IIR));
2533 }
2534
2535 static void i915_irq_preinstall(struct drm_device * dev)
2536 {
2537         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2538         int pipe;
2539
2540         atomic_set(&dev_priv->irq_received, 0);
2541
2542         if (I915_HAS_HOTPLUG(dev)) {
2543                 I915_WRITE(PORT_HOTPLUG_EN, 0);
2544                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2545         }
2546
2547         I915_WRITE16(HWSTAM, 0xeffe);
2548         for_each_pipe(pipe)
2549                 I915_WRITE(PIPESTAT(pipe), 0);
2550         I915_WRITE(IMR, 0xffffffff);
2551         I915_WRITE(IER, 0x0);
2552         POSTING_READ(IER);
2553 }
2554
2555 static int i915_irq_postinstall(struct drm_device *dev)
2556 {
2557         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2558         u32 enable_mask;
2559
2560         I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
2561
2562         /* Unmask the interrupts that we always want on. */
2563         dev_priv->irq_mask =
2564                 ~(I915_ASLE_INTERRUPT |
2565                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2566                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2567                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2568                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2569                   I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2570
2571         enable_mask =
2572                 I915_ASLE_INTERRUPT |
2573                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2574                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2575                 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
2576                 I915_USER_INTERRUPT;
2577
2578         if (I915_HAS_HOTPLUG(dev)) {
2579                 I915_WRITE(PORT_HOTPLUG_EN, 0);
2580                 POSTING_READ(PORT_HOTPLUG_EN);
2581
2582                 /* Enable in IER... */
2583                 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
2584                 /* and unmask in IMR */
2585                 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
2586         }
2587
2588         I915_WRITE(IMR, dev_priv->irq_mask);
2589         I915_WRITE(IER, enable_mask);
2590         POSTING_READ(IER);
2591
2592         i915_enable_asle_pipestat(dev);
2593
2594         return 0;
2595 }
2596
2597 /*
2598  * Returns true when a page flip has completed.
2599  */
2600 static bool i915_handle_vblank(struct drm_device *dev,
2601                                int plane, int pipe, u32 iir)
2602 {
2603         drm_i915_private_t *dev_priv = dev->dev_private;
2604         u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
2605
2606         if (!drm_handle_vblank(dev, pipe))
2607                 return false;
2608
2609         if ((iir & flip_pending) == 0)
2610                 return false;
2611
2612         intel_prepare_page_flip(dev, plane);
2613
2614         /* We detect FlipDone by looking for the change in PendingFlip from '1'
2615          * to '0' on the following vblank, i.e. IIR has the Pendingflip
2616          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
2617          * the flip is completed (no longer pending). Since this doesn't raise
2618          * an interrupt per se, we watch for the change at vblank.
2619          */
2620         if (I915_READ(ISR) & flip_pending)
2621                 return false;
2622
2623         intel_finish_page_flip(dev, pipe);
2624
2625         return true;
2626 }
2627
2628 static irqreturn_t i915_irq_handler(int irq, void *arg)
2629 {
2630         struct drm_device *dev = (struct drm_device *) arg;
2631         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2632         u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
2633         unsigned long irqflags;
2634         u32 flip_mask =
2635                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2636                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
2637         int pipe, ret = IRQ_NONE;
2638
2639         atomic_inc(&dev_priv->irq_received);
2640
2641         iir = I915_READ(IIR);
2642         do {
2643                 bool irq_received = (iir & ~flip_mask) != 0;
2644                 bool blc_event = false;
2645
2646                 /* Can't rely on pipestat interrupt bit in iir as it might
2647                  * have been cleared after the pipestat interrupt was received.
2648                  * It doesn't set the bit in iir again, but it still produces
2649                  * interrupts (for non-MSI).
2650                  */
2651                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2652                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2653                         i915_handle_error(dev, false);
2654
2655                 for_each_pipe(pipe) {
2656                         int reg = PIPESTAT(pipe);
2657                         pipe_stats[pipe] = I915_READ(reg);
2658
2659                         /* Clear the PIPE*STAT regs before the IIR */
2660                         if (pipe_stats[pipe] & 0x8000ffff) {
2661                                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2662                                         DRM_DEBUG_DRIVER("pipe %c underrun\n",
2663                                                          pipe_name(pipe));
2664                                 I915_WRITE(reg, pipe_stats[pipe]);
2665                                 irq_received = true;
2666                         }
2667                 }
2668                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2669
2670                 if (!irq_received)
2671                         break;
2672
2673                 /* Consume port.  Then clear IIR or we'll miss events */
2674                 if ((I915_HAS_HOTPLUG(dev)) &&
2675                     (iir & I915_DISPLAY_PORT_INTERRUPT)) {
2676                         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
2677                         u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
2678
2679                         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
2680                                   hotplug_status);
2681
2682                         intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
2683
2684                         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
2685                         POSTING_READ(PORT_HOTPLUG_STAT);
2686                 }
2687
2688                 I915_WRITE(IIR, iir & ~flip_mask);
2689                 new_iir = I915_READ(IIR); /* Flush posted writes */
2690
2691                 if (iir & I915_USER_INTERRUPT)
2692                         notify_ring(dev, &dev_priv->ring[RCS]);
2693
2694                 for_each_pipe(pipe) {
2695                         int plane = pipe;
2696                         if (IS_MOBILE(dev))
2697                                 plane = !plane;
2698
2699                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
2700                             i915_handle_vblank(dev, plane, pipe, iir))
2701                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
2702
2703                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
2704                                 blc_event = true;
2705                 }
2706
2707                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
2708                         intel_opregion_asle_intr(dev);
2709
2710                 /* With MSI, interrupts are only generated when iir
2711                  * transitions from zero to nonzero.  If another bit got
2712                  * set while we were handling the existing iir bits, then
2713                  * we would never get another interrupt.
2714                  *
2715                  * This is fine on non-MSI as well, as if we hit this path
2716                  * we avoid exiting the interrupt handler only to generate
2717                  * another one.
2718                  *
2719                  * Note that for MSI this could cause a stray interrupt report
2720                  * if an interrupt landed in the time between writing IIR and
2721                  * the posting read.  This should be rare enough to never
2722                  * trigger the 99% of 100,000 interrupts test for disabling
2723                  * stray interrupts.
2724                  */
2725                 ret = IRQ_HANDLED;
2726                 iir = new_iir;
2727         } while (iir & ~flip_mask);
2728
2729         i915_update_dri1_breadcrumb(dev);
2730
2731         return ret;
2732 }
2733
2734 static void i915_irq_uninstall(struct drm_device * dev)
2735 {
2736         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2737         int pipe;
2738
2739         del_timer_sync(&dev_priv->hotplug_reenable_timer);
2740
2741         if (I915_HAS_HOTPLUG(dev)) {
2742                 I915_WRITE(PORT_HOTPLUG_EN, 0);
2743                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2744         }
2745
2746         I915_WRITE16(HWSTAM, 0xffff);
2747         for_each_pipe(pipe) {
2748                 /* Clear enable bits; then clear status bits */
2749                 I915_WRITE(PIPESTAT(pipe), 0);
2750                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
2751         }
2752         I915_WRITE(IMR, 0xffffffff);
2753         I915_WRITE(IER, 0x0);
2754
2755         I915_WRITE(IIR, I915_READ(IIR));
2756 }
2757
2758 static void i965_irq_preinstall(struct drm_device * dev)
2759 {
2760         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2761         int pipe;
2762
2763         atomic_set(&dev_priv->irq_received, 0);
2764
2765         I915_WRITE(PORT_HOTPLUG_EN, 0);
2766         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2767
2768         I915_WRITE(HWSTAM, 0xeffe);
2769         for_each_pipe(pipe)
2770                 I915_WRITE(PIPESTAT(pipe), 0);
2771         I915_WRITE(IMR, 0xffffffff);
2772         I915_WRITE(IER, 0x0);
2773         POSTING_READ(IER);
2774 }
2775
2776 static int i965_irq_postinstall(struct drm_device *dev)
2777 {
2778         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2779         u32 enable_mask;
2780         u32 error_mask;
2781         unsigned long irqflags;
2782
2783         /* Unmask the interrupts that we always want on. */
2784         dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
2785                                I915_DISPLAY_PORT_INTERRUPT |
2786                                I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2787                                I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2788                                I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2789                                I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2790                                I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2791
2792         enable_mask = ~dev_priv->irq_mask;
2793         enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2794                          I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
2795         enable_mask |= I915_USER_INTERRUPT;
2796
2797         if (IS_G4X(dev))
2798                 enable_mask |= I915_BSD_USER_INTERRUPT;
2799
2800         /* Interrupt setup is already guaranteed to be single-threaded, this is
2801          * just to make the assert_spin_locked check happy. */
2802         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2803         i915_enable_pipestat(dev_priv, 0, PIPE_GMBUS_EVENT_ENABLE);
2804         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2805
2806         /*
2807          * Enable some error detection, note the instruction error mask
2808          * bit is reserved, so we leave it masked.
2809          */
2810         if (IS_G4X(dev)) {
2811                 error_mask = ~(GM45_ERROR_PAGE_TABLE |
2812                                GM45_ERROR_MEM_PRIV |
2813                                GM45_ERROR_CP_PRIV |
2814                                I915_ERROR_MEMORY_REFRESH);
2815         } else {
2816                 error_mask = ~(I915_ERROR_PAGE_TABLE |
2817                                I915_ERROR_MEMORY_REFRESH);
2818         }
2819         I915_WRITE(EMR, error_mask);
2820
2821         I915_WRITE(IMR, dev_priv->irq_mask);
2822         I915_WRITE(IER, enable_mask);
2823         POSTING_READ(IER);
2824
2825         I915_WRITE(PORT_HOTPLUG_EN, 0);
2826         POSTING_READ(PORT_HOTPLUG_EN);
2827
2828         i915_enable_asle_pipestat(dev);
2829
2830         return 0;
2831 }
2832
2833 static void i915_hpd_irq_setup(struct drm_device *dev)
2834 {
2835         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2836         struct drm_mode_config *mode_config = &dev->mode_config;
2837         struct intel_encoder *intel_encoder;
2838         u32 hotplug_en;
2839
2840         assert_spin_locked(&dev_priv->irq_lock);
2841
2842         if (I915_HAS_HOTPLUG(dev)) {
2843                 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
2844                 hotplug_en &= ~HOTPLUG_INT_EN_MASK;
2845                 /* Note HDMI and DP share hotplug bits */
2846                 /* enable bits are the same for all generations */
2847                 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
2848                         if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
2849                                 hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
2850                 /* Programming the CRT detection parameters tends
2851                    to generate a spurious hotplug event about three
2852                    seconds later.  So just do it once.
2853                 */
2854                 if (IS_G4X(dev))
2855                         hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
2856                 hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
2857                 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
2858
2859                 /* Ignore TV since it's buggy */
2860                 I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
2861         }
2862 }
2863
2864 static irqreturn_t i965_irq_handler(int irq, void *arg)
2865 {
2866         struct drm_device *dev = (struct drm_device *) arg;
2867         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2868         u32 iir, new_iir;
2869         u32 pipe_stats[I915_MAX_PIPES];
2870         unsigned long irqflags;
2871         int irq_received;
2872         int ret = IRQ_NONE, pipe;
2873         u32 flip_mask =
2874                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2875                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
2876
2877         atomic_inc(&dev_priv->irq_received);
2878
2879         iir = I915_READ(IIR);
2880
2881         for (;;) {
2882                 bool blc_event = false;
2883
2884                 irq_received = (iir & ~flip_mask) != 0;
2885
2886                 /* Can't rely on pipestat interrupt bit in iir as it might
2887                  * have been cleared after the pipestat interrupt was received.
2888                  * It doesn't set the bit in iir again, but it still produces
2889                  * interrupts (for non-MSI).
2890                  */
2891                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2892                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2893                         i915_handle_error(dev, false);
2894
2895                 for_each_pipe(pipe) {
2896                         int reg = PIPESTAT(pipe);
2897                         pipe_stats[pipe] = I915_READ(reg);
2898
2899                         /*
2900                          * Clear the PIPE*STAT regs before the IIR
2901                          */
2902                         if (pipe_stats[pipe] & 0x8000ffff) {
2903                                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2904                                         DRM_DEBUG_DRIVER("pipe %c underrun\n",
2905                                                          pipe_name(pipe));
2906                                 I915_WRITE(reg, pipe_stats[pipe]);
2907                                 irq_received = 1;
2908                         }
2909                 }
2910                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2911
2912                 if (!irq_received)
2913                         break;
2914
2915                 ret = IRQ_HANDLED;
2916
2917                 /* Consume port.  Then clear IIR or we'll miss events */
2918                 if (iir & I915_DISPLAY_PORT_INTERRUPT) {
2919                         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
2920                         u32 hotplug_trigger = hotplug_status & (IS_G4X(dev) ?
2921                                                                   HOTPLUG_INT_STATUS_G4X :
2922                                                                   HOTPLUG_INT_STATUS_I915);
2923
2924                         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
2925                                   hotplug_status);
2926
2927                         intel_hpd_irq_handler(dev, hotplug_trigger,
2928                                               IS_G4X(dev) ? hpd_status_gen4 : hpd_status_i915);
2929
2930                         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
2931                         I915_READ(PORT_HOTPLUG_STAT);
2932                 }
2933
2934                 I915_WRITE(IIR, iir & ~flip_mask);
2935                 new_iir = I915_READ(IIR); /* Flush posted writes */
2936
2937                 if (iir & I915_USER_INTERRUPT)
2938                         notify_ring(dev, &dev_priv->ring[RCS]);
2939                 if (iir & I915_BSD_USER_INTERRUPT)
2940                         notify_ring(dev, &dev_priv->ring[VCS]);
2941
2942                 for_each_pipe(pipe) {
2943                         if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
2944                             i915_handle_vblank(dev, pipe, pipe, iir))
2945                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
2946
2947                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
2948                                 blc_event = true;
2949                 }
2950
2951
2952                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
2953                         intel_opregion_asle_intr(dev);
2954
2955                 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
2956                         gmbus_irq_handler(dev);
2957
2958                 /* With MSI, interrupts are only generated when iir
2959                  * transitions from zero to nonzero.  If another bit got
2960                  * set while we were handling the existing iir bits, then
2961                  * we would never get another interrupt.
2962                  *
2963                  * This is fine on non-MSI as well, as if we hit this path
2964                  * we avoid exiting the interrupt handler only to generate
2965                  * another one.
2966                  *
2967                  * Note that for MSI this could cause a stray interrupt report
2968                  * if an interrupt landed in the time between writing IIR and
2969                  * the posting read.  This should be rare enough to never
2970                  * trigger the 99% of 100,000 interrupts test for disabling
2971                  * stray interrupts.
2972                  */
2973                 iir = new_iir;
2974         }
2975
2976         i915_update_dri1_breadcrumb(dev);
2977
2978         return ret;
2979 }
2980
2981 static void i965_irq_uninstall(struct drm_device * dev)
2982 {
2983         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2984         int pipe;
2985
2986         if (!dev_priv)
2987                 return;
2988
2989         del_timer_sync(&dev_priv->hotplug_reenable_timer);
2990
2991         I915_WRITE(PORT_HOTPLUG_EN, 0);
2992         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2993
2994         I915_WRITE(HWSTAM, 0xffffffff);
2995         for_each_pipe(pipe)
2996                 I915_WRITE(PIPESTAT(pipe), 0);
2997         I915_WRITE(IMR, 0xffffffff);
2998         I915_WRITE(IER, 0x0);
2999
3000         for_each_pipe(pipe)
3001                 I915_WRITE(PIPESTAT(pipe),
3002                            I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
3003         I915_WRITE(IIR, I915_READ(IIR));
3004 }
3005
3006 static void i915_reenable_hotplug_timer_func(unsigned long data)
3007 {
3008         drm_i915_private_t *dev_priv = (drm_i915_private_t *)data;
3009         struct drm_device *dev = dev_priv->dev;
3010         struct drm_mode_config *mode_config = &dev->mode_config;
3011         unsigned long irqflags;
3012         int i;
3013
3014         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3015         for (i = (HPD_NONE + 1); i < HPD_NUM_PINS; i++) {
3016                 struct drm_connector *connector;
3017
3018                 if (dev_priv->hpd_stats[i].hpd_mark != HPD_DISABLED)
3019                         continue;
3020
3021                 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
3022
3023                 list_for_each_entry(connector, &mode_config->connector_list, head) {
3024                         struct intel_connector *intel_connector = to_intel_connector(connector);
3025
3026                         if (intel_connector->encoder->hpd_pin == i) {
3027                                 if (connector->polled != intel_connector->polled)
3028                                         DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n",
3029                                                          drm_get_connector_name(connector));
3030                                 connector->polled = intel_connector->polled;
3031                                 if (!connector->polled)
3032                                         connector->polled = DRM_CONNECTOR_POLL_HPD;
3033                         }
3034                 }
3035         }
3036         if (dev_priv->display.hpd_irq_setup)
3037                 dev_priv->display.hpd_irq_setup(dev);
3038         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3039 }
3040
3041 void intel_irq_init(struct drm_device *dev)
3042 {
3043         struct drm_i915_private *dev_priv = dev->dev_private;
3044
3045         INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
3046         INIT_WORK(&dev_priv->gpu_error.work, i915_error_work_func);
3047         INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
3048         INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
3049
3050         setup_timer(&dev_priv->gpu_error.hangcheck_timer,
3051                     i915_hangcheck_elapsed,
3052                     (unsigned long) dev);
3053         setup_timer(&dev_priv->hotplug_reenable_timer, i915_reenable_hotplug_timer_func,
3054                     (unsigned long) dev_priv);
3055
3056         pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
3057
3058         dev->driver->get_vblank_counter = i915_get_vblank_counter;
3059         dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
3060         if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
3061                 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
3062                 dev->driver->get_vblank_counter = gm45_get_vblank_counter;
3063         }
3064
3065         if (drm_core_check_feature(dev, DRIVER_MODESET))
3066                 dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
3067         else
3068                 dev->driver->get_vblank_timestamp = NULL;
3069         dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
3070
3071         if (IS_VALLEYVIEW(dev)) {
3072                 dev->driver->irq_handler = valleyview_irq_handler;
3073                 dev->driver->irq_preinstall = valleyview_irq_preinstall;
3074                 dev->driver->irq_postinstall = valleyview_irq_postinstall;
3075                 dev->driver->irq_uninstall = valleyview_irq_uninstall;
3076                 dev->driver->enable_vblank = valleyview_enable_vblank;
3077                 dev->driver->disable_vblank = valleyview_disable_vblank;
3078                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
3079         } else if (HAS_PCH_SPLIT(dev)) {
3080                 dev->driver->irq_handler = ironlake_irq_handler;
3081                 dev->driver->irq_preinstall = ironlake_irq_preinstall;
3082                 dev->driver->irq_postinstall = ironlake_irq_postinstall;
3083                 dev->driver->irq_uninstall = ironlake_irq_uninstall;
3084                 dev->driver->enable_vblank = ironlake_enable_vblank;
3085                 dev->driver->disable_vblank = ironlake_disable_vblank;
3086                 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
3087         } else {
3088                 if (INTEL_INFO(dev)->gen == 2) {
3089                         dev->driver->irq_preinstall = i8xx_irq_preinstall;
3090                         dev->driver->irq_postinstall = i8xx_irq_postinstall;
3091                         dev->driver->irq_handler = i8xx_irq_handler;
3092                         dev->driver->irq_uninstall = i8xx_irq_uninstall;
3093                 } else if (INTEL_INFO(dev)->gen == 3) {
3094                         dev->driver->irq_preinstall = i915_irq_preinstall;
3095                         dev->driver->irq_postinstall = i915_irq_postinstall;
3096                         dev->driver->irq_uninstall = i915_irq_uninstall;
3097                         dev->driver->irq_handler = i915_irq_handler;
3098                         dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
3099                 } else {
3100                         dev->driver->irq_preinstall = i965_irq_preinstall;
3101                         dev->driver->irq_postinstall = i965_irq_postinstall;
3102                         dev->driver->irq_uninstall = i965_irq_uninstall;
3103                         dev->driver->irq_handler = i965_irq_handler;
3104                         dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
3105                 }
3106                 dev->driver->enable_vblank = i915_enable_vblank;
3107                 dev->driver->disable_vblank = i915_disable_vblank;
3108         }
3109 }
3110
3111 void intel_hpd_init(struct drm_device *dev)
3112 {
3113         struct drm_i915_private *dev_priv = dev->dev_private;
3114         struct drm_mode_config *mode_config = &dev->mode_config;
3115         struct drm_connector *connector;
3116         unsigned long irqflags;
3117         int i;
3118
3119         for (i = 1; i < HPD_NUM_PINS; i++) {
3120                 dev_priv->hpd_stats[i].hpd_cnt = 0;
3121                 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
3122         }
3123         list_for_each_entry(connector, &mode_config->connector_list, head) {
3124                 struct intel_connector *intel_connector = to_intel_connector(connector);
3125                 connector->polled = intel_connector->polled;
3126                 if (!connector->polled && I915_HAS_HOTPLUG(dev) && intel_connector->encoder->hpd_pin > HPD_NONE)
3127                         connector->polled = DRM_CONNECTOR_POLL_HPD;
3128         }
3129
3130         /* Interrupt setup is already guaranteed to be single-threaded, this is
3131          * just to make the assert_spin_locked checks happy. */
3132         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3133         if (dev_priv->display.hpd_irq_setup)
3134                 dev_priv->display.hpd_irq_setup(dev);
3135         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3136 }