drivers/gpu/drm/nouveau/nouveau_irq.c

   1 /*
   2  * Copyright (C) 2006 Ben Skeggs.
   3  *
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining
   7  * a copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sublicense, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial
  16  * portions of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  */
  27
  28 /*
  29  * Authors:
  30  *   Ben Skeggs <darktama@iinet.net.au>
  31  */
  32
  33 #include "drmP.h"
  34 #include "drm.h"
  35 #include "nouveau_drm.h"
  36 #include "nouveau_drv.h"
  37 #include "nouveau_reg.h"
  38 #include "nouveau_ramht.h"
  39 #include <linux/ratelimit.h>
  40
  41 /* needed for hotplug irq */
  42 #include "nouveau_connector.h"
  43 #include "nv50_display.h"
  44
  45 static DEFINE_RATELIMIT_STATE(nouveau_ratelimit_state, 3 * HZ, 20);
  46
  47 static int nouveau_ratelimit(void)
  48 {
  49         return __ratelimit(&nouveau_ratelimit_state);
  50 }
  51
  52 void
  53 nouveau_irq_preinstall(struct drm_device *dev)
  54 {
  55         struct drm_nouveau_private *dev_priv = dev->dev_private;
  56
  57         /* Master disable */
  58         nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
  59
  60         if (dev_priv->card_type >= NV_50) {
  61                 INIT_WORK(&dev_priv->irq_work, nv50_display_irq_handler_bh);
  62                 INIT_WORK(&dev_priv->hpd_work, nv50_display_irq_hotplug_bh);
  63                 spin_lock_init(&dev_priv->hpd_state.lock);
  64                 INIT_LIST_HEAD(&dev_priv->vbl_waiting);
  65         }
  66 }
  67
  68 int
  69 nouveau_irq_postinstall(struct drm_device *dev)
  70 {
  71         /* Master enable */
  72         nv_wr32(dev, NV03_PMC_INTR_EN_0, NV_PMC_INTR_EN_0_MASTER_ENABLE);
  73         return 0;
  74 }
  75
  76 void
  77 nouveau_irq_uninstall(struct drm_device *dev)
  78 {
  79         /* Master disable */
  80         nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
  81 }
  82
  83 static int
  84 nouveau_call_method(struct nouveau_channel *chan, int class, int mthd, int data)
  85 {
  86         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
  87         struct nouveau_pgraph_object_method *grm;
  88         struct nouveau_pgraph_object_class *grc;
  89
  90         grc = dev_priv->engine.graph.grclass;
  91         while (grc->id) {
  92                 if (grc->id == class)
  93                         break;
  94                 grc++;
  95         }
  96
  97         if (grc->id != class || !grc->methods)
  98                 return -ENOENT;
  99
 100         grm = grc->methods;
 101         while (grm->id) {
 102                 if (grm->id == mthd)
 103                         return grm->exec(chan, class, mthd, data);
 104                 grm++;
 105         }
 106
 107         return -ENOENT;
 108 }
 109
 110 static bool
 111 nouveau_fifo_swmthd(struct drm_device *dev, u32 chid, u32 addr, u32 data)
 112 {
 113         struct drm_nouveau_private *dev_priv = dev->dev_private;
 114         struct nouveau_channel *chan = NULL;
 115         struct nouveau_gpuobj *obj;
 116         unsigned long flags;
 117         const int subc = (addr >> 13) & 0x7;
 118         const int mthd = addr & 0x1ffc;
 119         bool handled = false;
 120         u32 engine;
 121
 122         spin_lock_irqsave(&dev_priv->channels.lock, flags);
 123         if (likely(chid >= 0 && chid < dev_priv->engine.fifo.channels))
 124                 chan = dev_priv->channels.ptr[chid];
 125         if (unlikely(!chan))
 126                 goto out;
 127
 128         switch (mthd) {
 129         case 0x0000: /* bind object to subchannel */
 130                 obj = nouveau_ramht_find(chan, data);
 131                 if (unlikely(!obj || obj->engine != NVOBJ_ENGINE_SW))
 132                         break;
 133
 134                 chan->sw_subchannel[subc] = obj->class;
 135                 engine = 0x0000000f << (subc * 4);
 136
 137                 nv_mask(dev, NV04_PFIFO_CACHE1_ENGINE, engine, 0x00000000);
 138                 handled = true;
 139                 break;
 140         default:
 141                 engine = nv_rd32(dev, NV04_PFIFO_CACHE1_ENGINE);
 142                 if (unlikely(((engine >> (subc * 4)) & 0xf) != 0))
 143                         break;
 144
 145                 if (!nouveau_call_method(chan, chan->sw_subchannel[subc],
 146                                          mthd, data))
 147                         handled = true;
 148                 break;
 149         }
 150
 151 out:
 152         spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 153         return handled;
 154 }
 155
 156 static void
 157 nouveau_fifo_irq_handler(struct drm_device *dev)
 158 {
 159         struct drm_nouveau_private *dev_priv = dev->dev_private;
 160         struct nouveau_engine *engine = &dev_priv->engine;
 161         uint32_t status, reassign;
 162         int cnt = 0;
 163
 164         reassign = nv_rd32(dev, NV03_PFIFO_CACHES) & 1;
 165         while ((status = nv_rd32(dev, NV03_PFIFO_INTR_0)) && (cnt++ < 100)) {
 166                 uint32_t chid, get;
 167
 168                 nv_wr32(dev, NV03_PFIFO_CACHES, 0);
 169
 170                 chid = engine->fifo.channel_id(dev);
 171                 get  = nv_rd32(dev, NV03_PFIFO_CACHE1_GET);
 172
 173                 if (status & NV_PFIFO_INTR_CACHE_ERROR) {
 174                         uint32_t mthd, data;
 175                         int ptr;
 176
 177                         /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before
 178                          * wrapping on my G80 chips, but CACHE1 isn't big
 179                          * enough for this much data.. Tests show that it
 180                          * wraps around to the start at GET=0x800.. No clue
 181                          * as to why..
 182                          */
 183                         ptr = (get & 0x7ff) >> 2;
 184
 185                         if (dev_priv->card_type < NV_40) {
 186                                 mthd = nv_rd32(dev,
 187                                         NV04_PFIFO_CACHE1_METHOD(ptr));
 188                                 data = nv_rd32(dev,
 189                                         NV04_PFIFO_CACHE1_DATA(ptr));
 190                         } else {
 191                                 mthd = nv_rd32(dev,
 192                                         NV40_PFIFO_CACHE1_METHOD(ptr));
 193                                 data = nv_rd32(dev,
 194                                         NV40_PFIFO_CACHE1_DATA(ptr));
 195                         }
 196
 197                         if (!nouveau_fifo_swmthd(dev, chid, mthd, data)) {
 198                                 NV_INFO(dev, "PFIFO_CACHE_ERROR - Ch %d/%d "
 199                                              "Mthd 0x%04x Data 0x%08x\n",
 200                                         chid, (mthd >> 13) & 7, mthd & 0x1ffc,
 201                                         data);
 202                         }
 203
 204                         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH, 0);
 205                         nv_wr32(dev, NV03_PFIFO_INTR_0,
 206                                                 NV_PFIFO_INTR_CACHE_ERROR);
 207
 208                         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
 209                                 nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) & ~1);
 210                         nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
 211                         nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
 212                                 nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) | 1);
 213                         nv_wr32(dev, NV04_PFIFO_CACHE1_HASH, 0);
 214
 215                         nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH,
 216                                 nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_PUSH) | 1);
 217                         nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
 218
 219                         status &= ~NV_PFIFO_INTR_CACHE_ERROR;
 220                 }
 221
 222                 if (status & NV_PFIFO_INTR_DMA_PUSHER) {
 223                         u32 dma_get = nv_rd32(dev, 0x003244);
 224                         u32 dma_put = nv_rd32(dev, 0x003240);
 225                         u32 push = nv_rd32(dev, 0x003220);
 226                         u32 state = nv_rd32(dev, 0x003228);
 227
 228                         if (dev_priv->card_type == NV_50) {
 229                                 u32 ho_get = nv_rd32(dev, 0x003328);
 230                                 u32 ho_put = nv_rd32(dev, 0x003320);
 231                                 u32 ib_get = nv_rd32(dev, 0x003334);
 232                                 u32 ib_put = nv_rd32(dev, 0x003330);
 233
 234                                 if (nouveau_ratelimit())
 235                                         NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%02x%08x "
 236                                              "Put 0x%02x%08x IbGet 0x%08x IbPut 0x%08x "
 237                                              "State 0x%08x Push 0x%08x\n",
 238                                                 chid, ho_get, dma_get, ho_put,
 239                                                 dma_put, ib_get, ib_put, state,
 240                                                 push);
 241
 242                                 /* METHOD_COUNT, in DMA_STATE on earlier chipsets */
 243                                 nv_wr32(dev, 0x003364, 0x00000000);
 244                                 if (dma_get != dma_put || ho_get != ho_put) {
 245                                         nv_wr32(dev, 0x003244, dma_put);
 246                                         nv_wr32(dev, 0x003328, ho_put);
 247                                 } else
 248                                 if (ib_get != ib_put) {
 249                                         nv_wr32(dev, 0x003334, ib_put);
 250                                 }
 251                         } else {
 252                                 NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%08x "
 253                                              "Put 0x%08x State 0x%08x Push 0x%08x\n",
 254                                         chid, dma_get, dma_put, state, push);
 255
 256                                 if (dma_get != dma_put)
 257                                         nv_wr32(dev, 0x003244, dma_put);
 258                         }
 259
 260                         nv_wr32(dev, 0x003228, 0x00000000);
 261                         nv_wr32(dev, 0x003220, 0x00000001);
 262                         nv_wr32(dev, 0x002100, NV_PFIFO_INTR_DMA_PUSHER);
 263                         status &= ~NV_PFIFO_INTR_DMA_PUSHER;
 264                 }
 265
 266                 if (status & NV_PFIFO_INTR_SEMAPHORE) {
 267                         uint32_t sem;
 268
 269                         status &= ~NV_PFIFO_INTR_SEMAPHORE;
 270                         nv_wr32(dev, NV03_PFIFO_INTR_0,
 271                                 NV_PFIFO_INTR_SEMAPHORE);
 272
 273                         sem = nv_rd32(dev, NV10_PFIFO_CACHE1_SEMAPHORE);
 274                         nv_wr32(dev, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1);
 275
 276                         nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
 277                         nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
 278                 }
 279
 280                 if (dev_priv->card_type == NV_50) {
 281                         if (status & 0x00000010) {
 282                                 nv50_fb_vm_trap(dev, 1, "PFIFO_BAR_FAULT");
 283                                 status &= ~0x00000010;
 284                                 nv_wr32(dev, 0x002100, 0x00000010);
 285                         }
 286                 }
 287
 288                 if (status) {
 289                         if (nouveau_ratelimit())
 290                                 NV_INFO(dev, "PFIFO_INTR 0x%08x - Ch %d\n",
 291                                         status, chid);
 292                         nv_wr32(dev, NV03_PFIFO_INTR_0, status);
 293                         status = 0;
 294                 }
 295
 296                 nv_wr32(dev, NV03_PFIFO_CACHES, reassign);
 297         }
 298
 299         if (status) {
 300                 NV_INFO(dev, "PFIFO still angry after %d spins, halt\n", cnt);
 301                 nv_wr32(dev, 0x2140, 0);
 302                 nv_wr32(dev, 0x140, 0);
 303         }
 304
 305         nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PFIFO_PENDING);
 306 }
 307
 308 struct nouveau_bitfield_names {
 309         uint32_t mask;
 310         const char *name;
 311 };
 312
 313 static struct nouveau_bitfield_names nstatus_names[] =
 314 {
 315         { NV04_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
 316         { NV04_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
 317         { NV04_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
 318         { NV04_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" }
 319 };
 320
 321 static struct nouveau_bitfield_names nstatus_names_nv10[] =
 322 {
 323         { NV10_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
 324         { NV10_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
 325         { NV10_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
 326         { NV10_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" }
 327 };
 328
 329 static struct nouveau_bitfield_names nsource_names[] =
 330 {
 331         { NV03_PGRAPH_NSOURCE_NOTIFICATION,       "NOTIFICATION" },
 332         { NV03_PGRAPH_NSOURCE_DATA_ERROR,         "DATA_ERROR" },
 333         { NV03_PGRAPH_NSOURCE_PROTECTION_ERROR,   "PROTECTION_ERROR" },
 334         { NV03_PGRAPH_NSOURCE_RANGE_EXCEPTION,    "RANGE_EXCEPTION" },
 335         { NV03_PGRAPH_NSOURCE_LIMIT_COLOR,        "LIMIT_COLOR" },
 336         { NV03_PGRAPH_NSOURCE_LIMIT_ZETA,         "LIMIT_ZETA" },
 337         { NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD,       "ILLEGAL_MTHD" },
 338         { NV03_PGRAPH_NSOURCE_DMA_R_PROTECTION,   "DMA_R_PROTECTION" },
 339         { NV03_PGRAPH_NSOURCE_DMA_W_PROTECTION,   "DMA_W_PROTECTION" },
 340         { NV03_PGRAPH_NSOURCE_FORMAT_EXCEPTION,   "FORMAT_EXCEPTION" },
 341         { NV03_PGRAPH_NSOURCE_PATCH_EXCEPTION,    "PATCH_EXCEPTION" },
 342         { NV03_PGRAPH_NSOURCE_STATE_INVALID,      "STATE_INVALID" },
 343         { NV03_PGRAPH_NSOURCE_DOUBLE_NOTIFY,      "DOUBLE_NOTIFY" },
 344         { NV03_PGRAPH_NSOURCE_NOTIFY_IN_USE,      "NOTIFY_IN_USE" },
 345         { NV03_PGRAPH_NSOURCE_METHOD_CNT,         "METHOD_CNT" },
 346         { NV03_PGRAPH_NSOURCE_BFR_NOTIFICATION,   "BFR_NOTIFICATION" },
 347         { NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION, "DMA_VTX_PROTECTION" },
 348         { NV03_PGRAPH_NSOURCE_DMA_WIDTH_A,        "DMA_WIDTH_A" },
 349         { NV03_PGRAPH_NSOURCE_DMA_WIDTH_B,        "DMA_WIDTH_B" },
 350 };
 351
 352 static void
 353 nouveau_print_bitfield_names_(uint32_t value,
 354                                 const struct nouveau_bitfield_names *namelist,
 355                                 const int namelist_len)
 356 {
 357         /*
 358          * Caller must have already printed the KERN_* log level for us.
 359          * Also the caller is responsible for adding the newline.
 360          */
 361         int i;
 362         for (i = 0; i < namelist_len; ++i) {
 363                 uint32_t mask = namelist[i].mask;
 364                 if (value & mask) {
 365                         printk(" %s", namelist[i].name);
 366                         value &= ~mask;
 367                 }
 368         }
 369         if (value)
 370                 printk(" (unknown bits 0x%08x)", value);
 371 }
 372 #define nouveau_print_bitfield_names(val, namelist) \
 373         nouveau_print_bitfield_names_((val), (namelist), ARRAY_SIZE(namelist))
 374
 375 struct nouveau_enum_names {
 376         uint32_t value;
 377         const char *name;
 378 };
 379
 380 static void
 381 nouveau_print_enum_names_(uint32_t value,
 382                                 const struct nouveau_enum_names *namelist,
 383                                 const int namelist_len)
 384 {
 385         /*
 386          * Caller must have already printed the KERN_* log level for us.
 387          * Also the caller is responsible for adding the newline.
 388          */
 389         int i;
 390         for (i = 0; i < namelist_len; ++i) {
 391                 if (value == namelist[i].value) {
 392                         printk("%s", namelist[i].name);
 393                         return;
 394                 }
 395         }
 396         printk("unknown value 0x%08x", value);
 397 }
 398 #define nouveau_print_enum_names(val, namelist) \
 399         nouveau_print_enum_names_((val), (namelist), ARRAY_SIZE(namelist))
 400
 401 static int
 402 nouveau_graph_chid_from_grctx(struct drm_device *dev)
 403 {
 404         struct drm_nouveau_private *dev_priv = dev->dev_private;
 405         struct nouveau_channel *chan;
 406         unsigned long flags;
 407         uint32_t inst;
 408         int i;
 409
 410         if (dev_priv->card_type < NV_40)
 411                 return dev_priv->engine.fifo.channels;
 412         else
 413         if (dev_priv->card_type < NV_50) {
 414                 inst = (nv_rd32(dev, 0x40032c) & 0xfffff) << 4;
 415
 416                 spin_lock_irqsave(&dev_priv->channels.lock, flags);
 417                 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
 418                         chan = dev_priv->channels.ptr[i];
 419                         if (!chan || !chan->ramin_grctx)
 420                                 continue;
 421
 422                         if (inst == chan->ramin_grctx->pinst)
 423                                 break;
 424                 }
 425                 spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 426         } else {
 427                 inst = (nv_rd32(dev, 0x40032c) & 0xfffff) << 12;
 428
 429                 spin_lock_irqsave(&dev_priv->channels.lock, flags);
 430                 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
 431                         chan = dev_priv->channels.ptr[i];
 432                         if (!chan || !chan->ramin)
 433                                 continue;
 434
 435                         if (inst == chan->ramin->vinst)
 436                                 break;
 437                 }
 438                 spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 439         }
 440
 441
 442         return i;
 443 }
 444
 445 static int
 446 nouveau_graph_trapped_channel(struct drm_device *dev, int *channel_ret)
 447 {
 448         struct drm_nouveau_private *dev_priv = dev->dev_private;
 449         struct nouveau_engine *engine = &dev_priv->engine;
 450         int channel;
 451
 452         if (dev_priv->card_type < NV_10)
 453                 channel = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 24) & 0xf;
 454         else
 455         if (dev_priv->card_type < NV_40)
 456                 channel = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 20) & 0x1f;
 457         else
 458                 channel = nouveau_graph_chid_from_grctx(dev);
 459
 460         if (channel >= engine->fifo.channels ||
 461             !dev_priv->channels.ptr[channel]) {
 462                 NV_ERROR(dev, "AIII, invalid/inactive channel id %d\n", channel);
 463                 return -EINVAL;
 464         }
 465
 466         *channel_ret = channel;
 467         return 0;
 468 }
 469
 470 struct nouveau_pgraph_trap {
 471         int channel;
 472         int class;
 473         int subc, mthd, size;
 474         uint32_t data, data2;
 475         uint32_t nsource, nstatus;
 476 };
 477
 478 static void
 479 nouveau_graph_trap_info(struct drm_device *dev,
 480                         struct nouveau_pgraph_trap *trap)
 481 {
 482         struct drm_nouveau_private *dev_priv = dev->dev_private;
 483         uint32_t address;
 484
 485         trap->nsource = trap->nstatus = 0;
 486         if (dev_priv->card_type < NV_50) {
 487                 trap->nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
 488                 trap->nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
 489         }
 490
 491         if (nouveau_graph_trapped_channel(dev, &trap->channel))
 492                 trap->channel = -1;
 493         address = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
 494
 495         trap->mthd = address & 0x1FFC;
 496         trap->data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
 497         if (dev_priv->card_type < NV_10) {
 498                 trap->subc  = (address >> 13) & 0x7;
 499         } else {
 500                 trap->subc  = (address >> 16) & 0x7;
 501                 trap->data2 = nv_rd32(dev, NV10_PGRAPH_TRAPPED_DATA_HIGH);
 502         }
 503
 504         if (dev_priv->card_type < NV_10)
 505                 trap->class = nv_rd32(dev, 0x400180 + trap->subc*4) & 0xFF;
 506         else if (dev_priv->card_type < NV_40)
 507                 trap->class = nv_rd32(dev, 0x400160 + trap->subc*4) & 0xFFF;
 508         else if (dev_priv->card_type < NV_50)
 509                 trap->class = nv_rd32(dev, 0x400160 + trap->subc*4) & 0xFFFF;
 510         else
 511                 trap->class = nv_rd32(dev, 0x400814);
 512 }
 513
 514 static void
 515 nouveau_graph_dump_trap_info(struct drm_device *dev, const char *id,
 516                              struct nouveau_pgraph_trap *trap)
 517 {
 518         struct drm_nouveau_private *dev_priv = dev->dev_private;
 519         uint32_t nsource = trap->nsource, nstatus = trap->nstatus;
 520
 521         if (dev_priv->card_type < NV_50) {
 522                 NV_INFO(dev, "%s - nSource:", id);
 523                 nouveau_print_bitfield_names(nsource, nsource_names);
 524                 printk(", nStatus:");
 525                 if (dev_priv->card_type < NV_10)
 526                         nouveau_print_bitfield_names(nstatus, nstatus_names);
 527                 else
 528                         nouveau_print_bitfield_names(nstatus, nstatus_names_nv10);
 529                 printk("\n");
 530         }
 531
 532         NV_INFO(dev, "%s - Ch %d/%d Class 0x%04x Mthd 0x%04x "
 533                                         "Data 0x%08x:0x%08x\n",
 534                                         id, trap->channel, trap->subc,
 535                                         trap->class, trap->mthd,
 536                                         trap->data2, trap->data);
 537 }
 538
 539 static int
 540 nouveau_pgraph_intr_swmthd(struct drm_device *dev,
 541                            struct nouveau_pgraph_trap *trap)
 542 {
 543         struct drm_nouveau_private *dev_priv = dev->dev_private;
 544         unsigned long flags;
 545         int ret = -EINVAL;
 546
 547         spin_lock_irqsave(&dev_priv->channels.lock, flags);
 548         if (trap->channel > 0 &&
 549             trap->channel < dev_priv->engine.fifo.channels &&
 550             dev_priv->channels.ptr[trap->channel]) {
 551                 ret = nouveau_call_method(dev_priv->channels.ptr[trap->channel],
 552                                           trap->class, trap->mthd, trap->data);
 553         }
 554         spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
 555
 556         return ret;
 557 }
 558
 559 static inline void
 560 nouveau_pgraph_intr_notify(struct drm_device *dev, uint32_t nsource)
 561 {
 562         struct nouveau_pgraph_trap trap;
 563         int unhandled = 0;
 564
 565         nouveau_graph_trap_info(dev, &trap);
 566
 567         if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
 568                 if (nouveau_pgraph_intr_swmthd(dev, &trap))
 569                         unhandled = 1;
 570         } else {
 571                 unhandled = 1;
 572         }
 573
 574         if (unhandled)
 575                 nouveau_graph_dump_trap_info(dev, "PGRAPH_NOTIFY", &trap);
 576 }
 577
 578
 579 static inline void
 580 nouveau_pgraph_intr_error(struct drm_device *dev, uint32_t nsource)
 581 {
 582         struct nouveau_pgraph_trap trap;
 583         int unhandled = 0;
 584
 585         nouveau_graph_trap_info(dev, &trap);
 586         trap.nsource = nsource;
 587
 588         if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
 589                 if (nouveau_pgraph_intr_swmthd(dev, &trap))
 590                         unhandled = 1;
 591         } else if (nsource & NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION) {
 592                 uint32_t v = nv_rd32(dev, 0x402000);
 593                 nv_wr32(dev, 0x402000, v);
 594
 595                 /* dump the error anyway for now: it's useful for
 596                    Gallium development */
 597                 unhandled = 1;
 598         } else {
 599                 unhandled = 1;
 600         }
 601
 602         if (unhandled && nouveau_ratelimit())
 603                 nouveau_graph_dump_trap_info(dev, "PGRAPH_ERROR", &trap);
 604 }
 605
 606 static inline void
 607 nouveau_pgraph_intr_context_switch(struct drm_device *dev)
 608 {
 609         struct drm_nouveau_private *dev_priv = dev->dev_private;
 610         struct nouveau_engine *engine = &dev_priv->engine;
 611         uint32_t chid;
 612
 613         chid = engine->fifo.channel_id(dev);
 614         NV_DEBUG(dev, "PGRAPH context switch interrupt channel %x\n", chid);
 615
 616         switch (dev_priv->card_type) {
 617         case NV_04:
 618                 nv04_graph_context_switch(dev);
 619                 break;
 620         case NV_10:
 621                 nv10_graph_context_switch(dev);
 622                 break;
 623         default:
 624                 NV_ERROR(dev, "Context switch not implemented\n");
 625                 break;
 626         }
 627 }
 628
 629 static void
 630 nouveau_pgraph_irq_handler(struct drm_device *dev)
 631 {
 632         uint32_t status;
 633
 634         while ((status = nv_rd32(dev, NV03_PGRAPH_INTR))) {
 635                 uint32_t nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
 636
 637                 if (status & NV_PGRAPH_INTR_NOTIFY) {
 638                         nouveau_pgraph_intr_notify(dev, nsource);
 639
 640                         status &= ~NV_PGRAPH_INTR_NOTIFY;
 641                         nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_NOTIFY);
 642                 }
 643
 644                 if (status & NV_PGRAPH_INTR_ERROR) {
 645                         nouveau_pgraph_intr_error(dev, nsource);
 646
 647                         status &= ~NV_PGRAPH_INTR_ERROR;
 648                         nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_ERROR);
 649                 }
 650
 651                 if (status & NV_PGRAPH_INTR_CONTEXT_SWITCH) {
 652                         status &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
 653                         nv_wr32(dev, NV03_PGRAPH_INTR,
 654                                  NV_PGRAPH_INTR_CONTEXT_SWITCH);
 655
 656                         nouveau_pgraph_intr_context_switch(dev);
 657                 }
 658
 659                 if (status) {
 660                         NV_INFO(dev, "Unhandled PGRAPH_INTR - 0x%08x\n", status);
 661                         nv_wr32(dev, NV03_PGRAPH_INTR, status);
 662                 }
 663
 664                 if ((nv_rd32(dev, NV04_PGRAPH_FIFO) & (1 << 0)) == 0)
 665                         nv_wr32(dev, NV04_PGRAPH_FIFO, 1);
 666         }
 667
 668         nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PGRAPH_PENDING);
 669 }
 670
 671 static struct nouveau_enum_names nv50_mp_exec_error_names[] =
 672 {
 673         { 3, "STACK_UNDERFLOW" },
 674         { 4, "QUADON_ACTIVE" },
 675         { 8, "TIMEOUT" },
 676         { 0x10, "INVALID_OPCODE" },
 677         { 0x40, "BREAKPOINT" },
 678 };
 679
 680 static void
 681 nv50_pgraph_mp_trap(struct drm_device *dev, int tpid, int display)
 682 {
 683         struct drm_nouveau_private *dev_priv = dev->dev_private;
 684         uint32_t units = nv_rd32(dev, 0x1540);
 685         uint32_t addr, mp10, status, pc, oplow, ophigh;
 686         int i;
 687         int mps = 0;
 688         for (i = 0; i < 4; i++) {
 689                 if (!(units & 1 << (i+24)))
 690                         continue;
 691                 if (dev_priv->chipset < 0xa0)
 692                         addr = 0x408200 + (tpid << 12) + (i << 7);
 693                 else
 694                         addr = 0x408100 + (tpid << 11) + (i << 7);
 695                 mp10 = nv_rd32(dev, addr + 0x10);
 696                 status = nv_rd32(dev, addr + 0x14);
 697                 if (!status)
 698                         continue;
 699                 if (display) {
 700                         nv_rd32(dev, addr + 0x20);
 701                         pc = nv_rd32(dev, addr + 0x24);
 702                         oplow = nv_rd32(dev, addr + 0x70);
 703                         ophigh= nv_rd32(dev, addr + 0x74);
 704                         NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - "
 705                                         "TP %d MP %d: ", tpid, i);
 706                         nouveau_print_enum_names(status,
 707                                         nv50_mp_exec_error_names);
 708                         printk(" at %06x warp %d, opcode %08x %08x\n",
 709                                         pc&0xffffff, pc >> 24,
 710                                         oplow, ophigh);
 711                 }
 712                 nv_wr32(dev, addr + 0x10, mp10);
 713                 nv_wr32(dev, addr + 0x14, 0);
 714                 mps++;
 715         }
 716         if (!mps && display)
 717                 NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - TP %d: "
 718                                 "No MPs claiming errors?\n", tpid);
 719 }
 720
 721 static void
 722 nv50_pgraph_tp_trap(struct drm_device *dev, int type, uint32_t ustatus_old,
 723                 uint32_t ustatus_new, int display, const char *name)
 724 {
 725         struct drm_nouveau_private *dev_priv = dev->dev_private;
 726         int tps = 0;
 727         uint32_t units = nv_rd32(dev, 0x1540);
 728         int i, r;
 729         uint32_t ustatus_addr, ustatus;
 730         for (i = 0; i < 16; i++) {
 731                 if (!(units & (1 << i)))
 732                         continue;
 733                 if (dev_priv->chipset < 0xa0)
 734                         ustatus_addr = ustatus_old + (i << 12);
 735                 else
 736                         ustatus_addr = ustatus_new + (i << 11);
 737                 ustatus = nv_rd32(dev, ustatus_addr) & 0x7fffffff;
 738                 if (!ustatus)
 739                         continue;
 740                 tps++;
 741                 switch (type) {
 742                 case 6: /* texture error... unknown for now */
 743                         nv50_fb_vm_trap(dev, display, name);
 744                         if (display) {
 745                                 NV_ERROR(dev, "magic set %d:\n", i);
 746                                 for (r = ustatus_addr + 4; r <= ustatus_addr + 0x10; r += 4)
 747                                         NV_ERROR(dev, "\t0x%08x: 0x%08x\n", r,
 748                                                 nv_rd32(dev, r));
 749                         }
 750                         break;
 751                 case 7: /* MP error */
 752                         if (ustatus & 0x00010000) {
 753                                 nv50_pgraph_mp_trap(dev, i, display);
 754                                 ustatus &= ~0x00010000;
 755                         }
 756                         break;
 757                 case 8: /* TPDMA error */
 758                         {
 759                         uint32_t e0c = nv_rd32(dev, ustatus_addr + 4);
 760                         uint32_t e10 = nv_rd32(dev, ustatus_addr + 8);
 761                         uint32_t e14 = nv_rd32(dev, ustatus_addr + 0xc);
 762                         uint32_t e18 = nv_rd32(dev, ustatus_addr + 0x10);
 763                         uint32_t e1c = nv_rd32(dev, ustatus_addr + 0x14);
 764                         uint32_t e20 = nv_rd32(dev, ustatus_addr + 0x18);
 765                         uint32_t e24 = nv_rd32(dev, ustatus_addr + 0x1c);
 766                         nv50_fb_vm_trap(dev, display, name);
 767                         /* 2d engine destination */
 768                         if (ustatus & 0x00000010) {
 769                                 if (display) {
 770                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - Unknown fault at address %02x%08x\n",
 771                                                         i, e14, e10);
 772                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
 773                                                         i, e0c, e18, e1c, e20, e24);
 774                                 }
 775                                 ustatus &= ~0x00000010;
 776                         }
 777                         /* Render target */
 778                         if (ustatus & 0x00000040) {
 779                                 if (display) {
 780                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - Unknown fault at address %02x%08x\n",
 781                                                         i, e14, e10);
 782                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
 783                                                         i, e0c, e18, e1c, e20, e24);
 784                                 }
 785                                 ustatus &= ~0x00000040;
 786                         }
 787                         /* CUDA memory: l[], g[] or stack. */
 788                         if (ustatus & 0x00000080) {
 789                                 if (display) {
 790                                         if (e18 & 0x80000000) {
 791                                                 /* g[] read fault? */
 792                                                 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global read fault at address %02x%08x\n",
 793                                                                 i, e14, e10 | ((e18 >> 24) & 0x1f));
 794                                                 e18 &= ~0x1f000000;
 795                                         } else if (e18 & 0xc) {
 796                                                 /* g[] write fault? */
 797                                                 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global write fault at address %02x%08x\n",
 798                                                                 i, e14, e10 | ((e18 >> 7) & 0x1f));
 799                                                 e18 &= ~0x00000f80;
 800                                         } else {
 801                                                 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Unknown CUDA fault at address %02x%08x\n",
 802                                                                 i, e14, e10);
 803                                         }
 804                                         NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
 805                                                         i, e0c, e18, e1c, e20, e24);
 806                                 }
 807                                 ustatus &= ~0x00000080;
 808                         }
 809                         }
 810                         break;
 811                 }
 812                 if (ustatus) {
 813                         if (display)
 814                                 NV_INFO(dev, "%s - TP%d: Unhandled ustatus 0x%08x\n", name, i, ustatus);
 815                 }
 816                 nv_wr32(dev, ustatus_addr, 0xc0000000);
 817         }
 818
 819         if (!tps && display)
 820                 NV_INFO(dev, "%s - No TPs claiming errors?\n", name);
 821 }
 822
 823 static void
 824 nv50_pgraph_trap_handler(struct drm_device *dev)
 825 {
 826         struct nouveau_pgraph_trap trap;
 827         uint32_t status = nv_rd32(dev, 0x400108);
 828         uint32_t ustatus;
 829         int display = nouveau_ratelimit();
 830
 831
 832         if (!status && display) {
 833                 nouveau_graph_trap_info(dev, &trap);
 834                 nouveau_graph_dump_trap_info(dev, "PGRAPH_TRAP", &trap);
 835                 NV_INFO(dev, "PGRAPH_TRAP - no units reporting traps?\n");
 836         }
 837
 838         /* DISPATCH: Relays commands to other units and handles NOTIFY,
 839          * COND, QUERY. If you get a trap from it, the command is still stuck
 840          * in DISPATCH and you need to do something about it. */
 841         if (status & 0x001) {
 842                 ustatus = nv_rd32(dev, 0x400804) & 0x7fffffff;
 843                 if (!ustatus && display) {
 844                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - no ustatus?\n");
 845                 }
 846
 847                 /* Known to be triggered by screwed up NOTIFY and COND... */
 848                 if (ustatus & 0x00000001) {
 849                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_DISPATCH_FAULT");
 850                         nv_wr32(dev, 0x400500, 0);
 851                         if (nv_rd32(dev, 0x400808) & 0x80000000) {
 852                                 if (display) {
 853                                         if (nouveau_graph_trapped_channel(dev, &trap.channel))
 854                                                 trap.channel = -1;
 855                                         trap.class = nv_rd32(dev, 0x400814);
 856                                         trap.mthd = nv_rd32(dev, 0x400808) & 0x1ffc;
 857                                         trap.subc = (nv_rd32(dev, 0x400808) >> 16) & 0x7;
 858                                         trap.data = nv_rd32(dev, 0x40080c);
 859                                         trap.data2 = nv_rd32(dev, 0x400810);
 860                                         nouveau_graph_dump_trap_info(dev,
 861                                                         "PGRAPH_TRAP_DISPATCH_FAULT", &trap);
 862                                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - 400808: %08x\n", nv_rd32(dev, 0x400808));
 863                                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - 400848: %08x\n", nv_rd32(dev, 0x400848));
 864                                 }
 865                                 nv_wr32(dev, 0x400808, 0);
 866                         } else if (display) {
 867                                 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - No stuck command?\n");
 868                         }
 869                         nv_wr32(dev, 0x4008e8, nv_rd32(dev, 0x4008e8) & 3);
 870                         nv_wr32(dev, 0x400848, 0);
 871                         ustatus &= ~0x00000001;
 872                 }
 873                 if (ustatus & 0x00000002) {
 874                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_DISPATCH_QUERY");
 875                         nv_wr32(dev, 0x400500, 0);
 876                         if (nv_rd32(dev, 0x40084c) & 0x80000000) {
 877                                 if (display) {
 878                                         if (nouveau_graph_trapped_channel(dev, &trap.channel))
 879                                                 trap.channel = -1;
 880                                         trap.class = nv_rd32(dev, 0x400814);
 881                                         trap.mthd = nv_rd32(dev, 0x40084c) & 0x1ffc;
 882                                         trap.subc = (nv_rd32(dev, 0x40084c) >> 16) & 0x7;
 883                                         trap.data = nv_rd32(dev, 0x40085c);
 884                                         trap.data2 = 0;
 885                                         nouveau_graph_dump_trap_info(dev,
 886                                                         "PGRAPH_TRAP_DISPATCH_QUERY", &trap);
 887                                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_QUERY - 40084c: %08x\n", nv_rd32(dev, 0x40084c));
 888                                 }
 889                                 nv_wr32(dev, 0x40084c, 0);
 890                         } else if (display) {
 891                                 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_QUERY - No stuck command?\n");
 892                         }
 893                         ustatus &= ~0x00000002;
 894                 }
 895                 if (ustatus && display)
 896                         NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - Unhandled ustatus 0x%08x\n", ustatus);
 897                 nv_wr32(dev, 0x400804, 0xc0000000);
 898                 nv_wr32(dev, 0x400108, 0x001);
 899                 status &= ~0x001;
 900         }
 901
 902         /* TRAPs other than dispatch use the "normal" trap regs. */
 903         if (status && display) {
 904                 nouveau_graph_trap_info(dev, &trap);
 905                 nouveau_graph_dump_trap_info(dev,
 906                                 "PGRAPH_TRAP", &trap);
 907         }
 908
 909         /* M2MF: Memory to memory copy engine. */
 910         if (status & 0x002) {
 911                 ustatus = nv_rd32(dev, 0x406800) & 0x7fffffff;
 912                 if (!ustatus && display) {
 913                         NV_INFO(dev, "PGRAPH_TRAP_M2MF - no ustatus?\n");
 914                 }
 915                 if (ustatus & 0x00000001) {
 916                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_NOTIFY");
 917                         ustatus &= ~0x00000001;
 918                 }
 919                 if (ustatus & 0x00000002) {
 920                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_IN");
 921                         ustatus &= ~0x00000002;
 922                 }
 923                 if (ustatus & 0x00000004) {
 924                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_OUT");
 925                         ustatus &= ~0x00000004;
 926                 }
 927                 NV_INFO (dev, "PGRAPH_TRAP_M2MF - %08x %08x %08x %08x\n",
 928                                 nv_rd32(dev, 0x406804),
 929                                 nv_rd32(dev, 0x406808),
 930                                 nv_rd32(dev, 0x40680c),
 931                                 nv_rd32(dev, 0x406810));
 932                 if (ustatus && display)
 933                         NV_INFO(dev, "PGRAPH_TRAP_M2MF - Unhandled ustatus 0x%08x\n", ustatus);
 934                 /* No sane way found yet -- just reset the bugger. */
 935                 nv_wr32(dev, 0x400040, 2);
 936                 nv_wr32(dev, 0x400040, 0);
 937                 nv_wr32(dev, 0x406800, 0xc0000000);
 938                 nv_wr32(dev, 0x400108, 0x002);
 939                 status &= ~0x002;
 940         }
 941
 942         /* VFETCH: Fetches data from vertex buffers. */
 943         if (status & 0x004) {
 944                 ustatus = nv_rd32(dev, 0x400c04) & 0x7fffffff;
 945                 if (!ustatus && display) {
 946                         NV_INFO(dev, "PGRAPH_TRAP_VFETCH - no ustatus?\n");
 947                 }
 948                 if (ustatus & 0x00000001) {
 949                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_VFETCH_FAULT");
 950                         NV_INFO (dev, "PGRAPH_TRAP_VFETCH_FAULT - %08x %08x %08x %08x\n",
 951                                         nv_rd32(dev, 0x400c00),
 952                                         nv_rd32(dev, 0x400c08),
 953                                         nv_rd32(dev, 0x400c0c),
 954                                         nv_rd32(dev, 0x400c10));
 955                         ustatus &= ~0x00000001;
 956                 }
 957                 if (ustatus && display)
 958                         NV_INFO(dev, "PGRAPH_TRAP_VFETCH - Unhandled ustatus 0x%08x\n", ustatus);
 959                 nv_wr32(dev, 0x400c04, 0xc0000000);
 960                 nv_wr32(dev, 0x400108, 0x004);
 961                 status &= ~0x004;
 962         }
 963
 964         /* STRMOUT: DirectX streamout / OpenGL transform feedback. */
 965         if (status & 0x008) {
 966                 ustatus = nv_rd32(dev, 0x401800) & 0x7fffffff;
 967                 if (!ustatus && display) {
 968                         NV_INFO(dev, "PGRAPH_TRAP_STRMOUT - no ustatus?\n");
 969                 }
 970                 if (ustatus & 0x00000001) {
 971                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_STRMOUT_FAULT");
 972                         NV_INFO (dev, "PGRAPH_TRAP_STRMOUT_FAULT - %08x %08x %08x %08x\n",
 973                                         nv_rd32(dev, 0x401804),
 974                                         nv_rd32(dev, 0x401808),
 975                                         nv_rd32(dev, 0x40180c),
 976                                         nv_rd32(dev, 0x401810));
 977                         ustatus &= ~0x00000001;
 978                 }
 979                 if (ustatus && display)
 980                         NV_INFO(dev, "PGRAPH_TRAP_STRMOUT - Unhandled ustatus 0x%08x\n", ustatus);
 981                 /* No sane way found yet -- just reset the bugger. */
 982                 nv_wr32(dev, 0x400040, 0x80);
 983                 nv_wr32(dev, 0x400040, 0);
 984                 nv_wr32(dev, 0x401800, 0xc0000000);
 985                 nv_wr32(dev, 0x400108, 0x008);
 986                 status &= ~0x008;
 987         }
 988
 989         /* CCACHE: Handles code and c[] caches and fills them. */
 990         if (status & 0x010) {
 991                 ustatus = nv_rd32(dev, 0x405018) & 0x7fffffff;
 992                 if (!ustatus && display) {
 993                         NV_INFO(dev, "PGRAPH_TRAP_CCACHE - no ustatus?\n");
 994                 }
 995                 if (ustatus & 0x00000001) {
 996                         nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_CCACHE_FAULT");
 997                         NV_INFO (dev, "PGRAPH_TRAP_CCACHE_FAULT - %08x %08x %08x %08x %08x %08x %08x\n",
 998                                         nv_rd32(dev, 0x405800),
 999                                         nv_rd32(dev, 0x405804),
1000                                         nv_rd32(dev, 0x405808),
1001                                         nv_rd32(dev, 0x40580c),
1002                                         nv_rd32(dev, 0x405810),
1003                                         nv_rd32(dev, 0x405814),
1004                                         nv_rd32(dev, 0x40581c));
1005                         ustatus &= ~0x00000001;
1006                 }
1007                 if (ustatus && display)
1008                         NV_INFO(dev, "PGRAPH_TRAP_CCACHE - Unhandled ustatus 0x%08x\n", ustatus);
1009                 nv_wr32(dev, 0x405018, 0xc0000000);
1010                 nv_wr32(dev, 0x400108, 0x010);
1011                 status &= ~0x010;
1012         }
1013
1014         /* Unknown, not seen yet... 0x402000 is the only trap status reg
1015          * remaining, so try to handle it anyway. Perhaps related to that
1016          * unknown DMA slot on tesla? */
1017         if (status & 0x20) {
1018                 nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_UNKC04");
1019                 ustatus = nv_rd32(dev, 0x402000) & 0x7fffffff;
1020                 if (display)
1021                         NV_INFO(dev, "PGRAPH_TRAP_UNKC04 - Unhandled ustatus 0x%08x\n", ustatus);
1022                 nv_wr32(dev, 0x402000, 0xc0000000);
1023                 /* no status modifiction on purpose */
1024         }
1025
1026         /* TEXTURE: CUDA texturing units */
1027         if (status & 0x040) {
1028                 nv50_pgraph_tp_trap (dev, 6, 0x408900, 0x408600, display,
1029                                 "PGRAPH_TRAP_TEXTURE");
1030                 nv_wr32(dev, 0x400108, 0x040);
1031                 status &= ~0x040;
1032         }
1033
1034         /* MP: CUDA execution engines. */
1035         if (status & 0x080) {
1036                 nv50_pgraph_tp_trap (dev, 7, 0x408314, 0x40831c, display,
1037                                 "PGRAPH_TRAP_MP");
1038                 nv_wr32(dev, 0x400108, 0x080);
1039                 status &= ~0x080;
1040         }
1041
1042         /* TPDMA:  Handles TP-initiated uncached memory accesses:
1043          * l[], g[], stack, 2d surfaces, render targets. */
1044         if (status & 0x100) {
1045                 nv50_pgraph_tp_trap (dev, 8, 0x408e08, 0x408708, display,
1046                                 "PGRAPH_TRAP_TPDMA");
1047                 nv_wr32(dev, 0x400108, 0x100);
1048                 status &= ~0x100;
1049         }
1050
1051         if (status) {
1052                 if (display)
1053                         NV_INFO(dev, "PGRAPH_TRAP - Unknown trap 0x%08x\n",
1054                                 status);
1055                 nv_wr32(dev, 0x400108, status);
1056         }
1057 }
1058
1059 /* There must be a *lot* of these. Will take some time to gather them up. */
1060 static struct nouveau_enum_names nv50_data_error_names[] =
1061 {
1062         { 4,    "INVALID_VALUE" },
1063         { 5,    "INVALID_ENUM" },
1064         { 8,    "INVALID_OBJECT" },
1065         { 0xc,  "INVALID_BITFIELD" },
1066         { 0x28, "MP_NO_REG_SPACE" },
1067         { 0x2b, "MP_BLOCK_SIZE_MISMATCH" },
1068 };
1069
1070 static void
1071 nv50_pgraph_irq_handler(struct drm_device *dev)
1072 {
1073         struct nouveau_pgraph_trap trap;
1074         int unhandled = 0;
1075         uint32_t status;
1076
1077         while ((status = nv_rd32(dev, NV03_PGRAPH_INTR))) {
1078                 /* NOTIFY: You've set a NOTIFY an a command and it's done. */
1079                 if (status & 0x00000001) {
1080                         nouveau_graph_trap_info(dev, &trap);
1081                         if (nouveau_ratelimit())
1082                                 nouveau_graph_dump_trap_info(dev,
1083                                                 "PGRAPH_NOTIFY", &trap);
1084                         status &= ~0x00000001;
1085                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000001);
1086                 }
1087
1088                 /* COMPUTE_QUERY: Purpose and exact cause unknown, happens
1089                  * when you write 0x200 to 0x50c0 method 0x31c. */
1090                 if (status & 0x00000002) {
1091                         nouveau_graph_trap_info(dev, &trap);
1092                         if (nouveau_ratelimit())
1093                                 nouveau_graph_dump_trap_info(dev,
1094                                                 "PGRAPH_COMPUTE_QUERY", &trap);
1095                         status &= ~0x00000002;
1096                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000002);
1097                 }
1098
1099                 /* Unknown, never seen: 0x4 */
1100
1101                 /* ILLEGAL_MTHD: You used a wrong method for this class. */
1102                 if (status & 0x00000010) {
1103                         nouveau_graph_trap_info(dev, &trap);
1104                         if (nouveau_pgraph_intr_swmthd(dev, &trap))
1105                                 unhandled = 1;
1106                         if (unhandled && nouveau_ratelimit())
1107                                 nouveau_graph_dump_trap_info(dev,
1108                                                 "PGRAPH_ILLEGAL_MTHD", &trap);
1109                         status &= ~0x00000010;
1110                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000010);
1111                 }
1112
1113                 /* ILLEGAL_CLASS: You used a wrong class. */
1114                 if (status & 0x00000020) {
1115                         nouveau_graph_trap_info(dev, &trap);
1116                         if (nouveau_ratelimit())
1117                                 nouveau_graph_dump_trap_info(dev,
1118                                                 "PGRAPH_ILLEGAL_CLASS", &trap);
1119                         status &= ~0x00000020;
1120                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000020);
1121                 }
1122
1123                 /* DOUBLE_NOTIFY: You tried to set a NOTIFY on another NOTIFY. */
1124                 if (status & 0x00000040) {
1125                         nouveau_graph_trap_info(dev, &trap);
1126                         if (nouveau_ratelimit())
1127                                 nouveau_graph_dump_trap_info(dev,
1128                                                 "PGRAPH_DOUBLE_NOTIFY", &trap);
1129                         status &= ~0x00000040;
1130                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000040);
1131                 }
1132
1133                 /* CONTEXT_SWITCH: PGRAPH needs us to load a new context */
1134                 if (status & 0x00001000) {
1135                         nv_wr32(dev, 0x400500, 0x00000000);
1136                         nv_wr32(dev, NV03_PGRAPH_INTR,
1137                                 NV_PGRAPH_INTR_CONTEXT_SWITCH);
1138                         nv_wr32(dev, NV40_PGRAPH_INTR_EN, nv_rd32(dev,
1139                                 NV40_PGRAPH_INTR_EN) &
1140                                 ~NV_PGRAPH_INTR_CONTEXT_SWITCH);
1141                         nv_wr32(dev, 0x400500, 0x00010001);
1142
1143                         nv50_graph_context_switch(dev);
1144
1145                         status &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
1146                 }
1147
1148                 /* BUFFER_NOTIFY: Your m2mf transfer finished */
1149                 if (status & 0x00010000) {
1150                         nouveau_graph_trap_info(dev, &trap);
1151                         if (nouveau_ratelimit())
1152                                 nouveau_graph_dump_trap_info(dev,
1153                                                 "PGRAPH_BUFFER_NOTIFY", &trap);
1154                         status &= ~0x00010000;
1155                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00010000);
1156                 }
1157
1158                 /* DATA_ERROR: Invalid value for this method, or invalid
1159                  * state in current PGRAPH context for this operation */
1160                 if (status & 0x00100000) {
1161                         nouveau_graph_trap_info(dev, &trap);
1162                         if (nouveau_ratelimit()) {
1163                                 nouveau_graph_dump_trap_info(dev,
1164                                                 "PGRAPH_DATA_ERROR", &trap);
1165                                 NV_INFO (dev, "PGRAPH_DATA_ERROR - ");
1166                                 nouveau_print_enum_names(nv_rd32(dev, 0x400110),
1167                                                 nv50_data_error_names);
1168                                 printk("\n");
1169                         }
1170                         status &= ~0x00100000;
1171                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00100000);
1172                 }
1173
1174                 /* TRAP: Something bad happened in the middle of command
1175                  * execution.  Has a billion types, subtypes, and even
1176                  * subsubtypes. */
1177                 if (status & 0x00200000) {
1178                         nv50_pgraph_trap_handler(dev);
1179                         status &= ~0x00200000;
1180                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x00200000);
1181                 }
1182
1183                 /* Unknown, never seen: 0x00400000 */
1184
1185                 /* SINGLE_STEP: Happens on every method if you turned on
1186                  * single stepping in 40008c */
1187                 if (status & 0x01000000) {
1188                         nouveau_graph_trap_info(dev, &trap);
1189                         if (nouveau_ratelimit())
1190                                 nouveau_graph_dump_trap_info(dev,
1191                                                 "PGRAPH_SINGLE_STEP", &trap);
1192                         status &= ~0x01000000;
1193                         nv_wr32(dev, NV03_PGRAPH_INTR, 0x01000000);
1194                 }
1195
1196                 /* 0x02000000 happens when you pause a ctxprog...
1197                  * but the only way this can happen that I know is by
1198                  * poking the relevant MMIO register, and we don't
1199                  * do that. */
1200
1201                 if (status) {
1202                         NV_INFO(dev, "Unhandled PGRAPH_INTR - 0x%08x\n",
1203                                 status);
1204                         nv_wr32(dev, NV03_PGRAPH_INTR, status);
1205                 }
1206
1207                 {
1208                         const int isb = (1 << 16) | (1 << 0);
1209
1210                         if ((nv_rd32(dev, 0x400500) & isb) != isb)
1211                                 nv_wr32(dev, 0x400500,
1212                                         nv_rd32(dev, 0x400500) | isb);
1213                 }
1214         }
1215
1216         nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PGRAPH_PENDING);
1217         if (nv_rd32(dev, 0x400824) & (1 << 31))
1218                 nv_wr32(dev, 0x400824, nv_rd32(dev, 0x400824) & ~(1 << 31));
1219 }
1220
1221 static void
1222 nouveau_crtc_irq_handler(struct drm_device *dev, int crtc)
1223 {
1224         if (crtc & 1)
1225                 nv_wr32(dev, NV_CRTC0_INTSTAT, NV_CRTC_INTR_VBLANK);
1226
1227         if (crtc & 2)
1228                 nv_wr32(dev, NV_CRTC1_INTSTAT, NV_CRTC_INTR_VBLANK);
1229 }
1230
1231 irqreturn_t
1232 nouveau_irq_handler(DRM_IRQ_ARGS)
1233 {
1234         struct drm_device *dev = (struct drm_device *)arg;
1235         struct drm_nouveau_private *dev_priv = dev->dev_private;
1236         uint32_t status;
1237         unsigned long flags;
1238
1239         status = nv_rd32(dev, NV03_PMC_INTR_0);
1240         if (!status)
1241                 return IRQ_NONE;
1242
1243         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
1244
1245         if (status & NV_PMC_INTR_0_PFIFO_PENDING) {
1246                 nouveau_fifo_irq_handler(dev);
1247                 status &= ~NV_PMC_INTR_0_PFIFO_PENDING;
1248         }
1249
1250         if (status & NV_PMC_INTR_0_PGRAPH_PENDING) {
1251                 if (dev_priv->card_type >= NV_50)
1252                         nv50_pgraph_irq_handler(dev);
1253                 else
1254                         nouveau_pgraph_irq_handler(dev);
1255
1256                 status &= ~NV_PMC_INTR_0_PGRAPH_PENDING;
1257         }
1258
1259         if (status & NV_PMC_INTR_0_CRTCn_PENDING) {
1260                 nouveau_crtc_irq_handler(dev, (status>>24)&3);
1261                 status &= ~NV_PMC_INTR_0_CRTCn_PENDING;
1262         }
1263
1264         if (status & (NV_PMC_INTR_0_NV50_DISPLAY_PENDING |
1265                       NV_PMC_INTR_0_NV50_I2C_PENDING)) {
1266                 nv50_display_irq_handler(dev);
1267                 status &= ~(NV_PMC_INTR_0_NV50_DISPLAY_PENDING |
1268                             NV_PMC_INTR_0_NV50_I2C_PENDING);
1269         }
1270
1271         if (status)
1272                 NV_ERROR(dev, "Unhandled PMC INTR status bits 0x%08x\n", status);
1273
1274         spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
1275
1276         return IRQ_HANDLED;
1277 }