drivers/gpu/drm/nouveau/nouveau_object.c

   1 /*
   2  * Copyright (C) 2006 Ben Skeggs.
   3  *
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining
   7  * a copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sublicense, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial
  16  * portions of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  */
  27
  28 /*
  29  * Authors:
  30  *   Ben Skeggs <darktama@iinet.net.au>
  31  */
  32
  33 #include "drmP.h"
  34 #include "drm.h"
  35 #include "nouveau_drv.h"
  36 #include "nouveau_drm.h"
  37 #include "nouveau_ramht.h"
  38
  39 /* NVidia uses context objects to drive drawing operations.
  40
  41    Context objects can be selected into 8 subchannels in the FIFO,
  42    and then used via DMA command buffers.
  43
  44    A context object is referenced by a user defined handle (CARD32). The HW
  45    looks up graphics objects in a hash table in the instance RAM.
  46
  47    An entry in the hash table consists of 2 CARD32. The first CARD32 contains
  48    the handle, the second one a bitfield, that contains the address of the
  49    object in instance RAM.
  50
  51    The format of the second CARD32 seems to be:
  52
  53    NV4 to NV30:
  54
  55    15: 0  instance_addr >> 4
  56    17:16  engine (here uses 1 = graphics)
  57    28:24  channel id (here uses 0)
  58    31     valid (use 1)
  59
  60    NV40:
  61
  62    15: 0  instance_addr >> 4   (maybe 19-0)
  63    21:20  engine (here uses 1 = graphics)
  64    I'm unsure about the other bits, but using 0 seems to work.
  65
  66    The key into the hash table depends on the object handle and channel id and
  67    is given as:
  68 */
  69
  70 int
  71 nouveau_gpuobj_new(struct drm_device *dev, struct nouveau_channel *chan,
  72                    uint32_t size, int align, uint32_t flags,
  73                    struct nouveau_gpuobj **gpuobj_ret)
  74 {
  75         struct drm_nouveau_private *dev_priv = dev->dev_private;
  76         struct nouveau_engine *engine = &dev_priv->engine;
  77         struct nouveau_gpuobj *gpuobj;
  78         struct drm_mm_node *ramin = NULL;
  79         int ret;
  80
  81         NV_DEBUG(dev, "ch%d size=%u align=%d flags=0x%08x\n",
  82                  chan ? chan->id : -1, size, align, flags);
  83
  84         if (!dev_priv || !gpuobj_ret || *gpuobj_ret != NULL)
  85                 return -EINVAL;
  86
  87         gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
  88         if (!gpuobj)
  89                 return -ENOMEM;
  90         NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
  91         gpuobj->dev = dev;
  92         gpuobj->flags = flags;
  93         kref_init(&gpuobj->refcount);
  94         gpuobj->size = size;
  95
  96         spin_lock(&dev_priv->ramin_lock);
  97         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
  98         spin_unlock(&dev_priv->ramin_lock);
  99
 100         if (chan) {
 101                 NV_DEBUG(dev, "channel heap\n");
 102
 103                 ramin = drm_mm_search_free(&chan->ramin_heap, size, align, 0);
 104                 if (ramin)
 105                         ramin = drm_mm_get_block(ramin, size, align);
 106
 107                 if (!ramin) {
 108                         nouveau_gpuobj_ref(NULL, &gpuobj);
 109                         return -ENOMEM;
 110                 }
 111         } else {
 112                 NV_DEBUG(dev, "global heap\n");
 113
 114                 /* allocate backing pages, sets vinst */
 115                 ret = engine->instmem.populate(dev, gpuobj, &size, align);
 116                 if (ret) {
 117                         nouveau_gpuobj_ref(NULL, &gpuobj);
 118                         return ret;
 119                 }
 120
 121                 /* try and get aperture space */
 122                 do {
 123                         if (drm_mm_pre_get(&dev_priv->ramin_heap))
 124                                 return -ENOMEM;
 125
 126                         spin_lock(&dev_priv->ramin_lock);
 127                         ramin = drm_mm_search_free(&dev_priv->ramin_heap, size,
 128                                                    align, 0);
 129                         if (ramin == NULL) {
 130                                 spin_unlock(&dev_priv->ramin_lock);
 131                                 nouveau_gpuobj_ref(NULL, &gpuobj);
 132                                 return -ENOMEM;
 133                         }
 134
 135                         ramin = drm_mm_get_block_atomic(ramin, size, align);
 136                         spin_unlock(&dev_priv->ramin_lock);
 137                 } while (ramin == NULL);
 138
 139                 /* on nv50 it's ok to fail, we have a fallback path */
 140                 if (!ramin && dev_priv->card_type < NV_50) {
 141                         nouveau_gpuobj_ref(NULL, &gpuobj);
 142                         return -ENOMEM;
 143                 }
 144         }
 145
 146         /* if we got a chunk of the aperture, map pages into it */
 147         gpuobj->im_pramin = ramin;
 148         if (!chan && gpuobj->im_pramin && dev_priv->ramin_available) {
 149                 ret = engine->instmem.bind(dev, gpuobj);
 150                 if (ret) {
 151                         nouveau_gpuobj_ref(NULL, &gpuobj);
 152                         return ret;
 153                 }
 154         }
 155
 156         /* calculate the various different addresses for the object */
 157         if (chan) {
 158                 gpuobj->pinst = chan->ramin->pinst;
 159                 if (gpuobj->pinst != ~0)
 160                         gpuobj->pinst += gpuobj->im_pramin->start;
 161
 162                 if (dev_priv->card_type < NV_50) {
 163                         gpuobj->cinst = gpuobj->pinst;
 164                 } else {
 165                         gpuobj->cinst = gpuobj->im_pramin->start;
 166                         gpuobj->vinst = gpuobj->im_pramin->start +
 167                                         chan->ramin->vinst;
 168                 }
 169         } else {
 170                 if (gpuobj->im_pramin)
 171                         gpuobj->pinst = gpuobj->im_pramin->start;
 172                 else
 173                         gpuobj->pinst = ~0;
 174                 gpuobj->cinst = 0xdeadbeef;
 175         }
 176
 177         if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
 178                 int i;
 179
 180                 for (i = 0; i < gpuobj->size; i += 4)
 181                         nv_wo32(gpuobj, i, 0);
 182                 engine->instmem.flush(dev);
 183         }
 184
 185
 186         *gpuobj_ret = gpuobj;
 187         return 0;
 188 }
 189
 190 int
 191 nouveau_gpuobj_init(struct drm_device *dev)
 192 {
 193         struct drm_nouveau_private *dev_priv = dev->dev_private;
 194
 195         NV_DEBUG(dev, "\n");
 196
 197         INIT_LIST_HEAD(&dev_priv->gpuobj_list);
 198         spin_lock_init(&dev_priv->ramin_lock);
 199         dev_priv->ramin_base = ~0;
 200
 201         return 0;
 202 }
 203
 204 void
 205 nouveau_gpuobj_takedown(struct drm_device *dev)
 206 {
 207         struct drm_nouveau_private *dev_priv = dev->dev_private;
 208
 209         NV_DEBUG(dev, "\n");
 210
 211         BUG_ON(!list_empty(&dev_priv->gpuobj_list));
 212 }
 213
 214
 215 static void
 216 nouveau_gpuobj_del(struct kref *ref)
 217 {
 218         struct nouveau_gpuobj *gpuobj =
 219                 container_of(ref, struct nouveau_gpuobj, refcount);
 220         struct drm_device *dev = gpuobj->dev;
 221         struct drm_nouveau_private *dev_priv = dev->dev_private;
 222         struct nouveau_engine *engine = &dev_priv->engine;
 223         int i;
 224
 225         NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
 226
 227         if (gpuobj->im_pramin && (gpuobj->flags & NVOBJ_FLAG_ZERO_FREE)) {
 228                 for (i = 0; i < gpuobj->size; i += 4)
 229                         nv_wo32(gpuobj, i, 0);
 230                 engine->instmem.flush(dev);
 231         }
 232
 233         if (gpuobj->dtor)
 234                 gpuobj->dtor(dev, gpuobj);
 235
 236         if (gpuobj->im_backing)
 237                 engine->instmem.clear(dev, gpuobj);
 238
 239         spin_lock(&dev_priv->ramin_lock);
 240         if (gpuobj->im_pramin)
 241                 drm_mm_put_block(gpuobj->im_pramin);
 242         list_del(&gpuobj->list);
 243         spin_unlock(&dev_priv->ramin_lock);
 244
 245         kfree(gpuobj);
 246 }
 247
 248 void
 249 nouveau_gpuobj_ref(struct nouveau_gpuobj *ref, struct nouveau_gpuobj **ptr)
 250 {
 251         if (ref)
 252                 kref_get(&ref->refcount);
 253
 254         if (*ptr)
 255                 kref_put(&(*ptr)->refcount, nouveau_gpuobj_del);
 256
 257         *ptr = ref;
 258 }
 259
 260 int
 261 nouveau_gpuobj_new_fake(struct drm_device *dev, u32 pinst, u64 vinst,
 262                         u32 size, u32 flags, struct nouveau_gpuobj **pgpuobj)
 263 {
 264         struct drm_nouveau_private *dev_priv = dev->dev_private;
 265         struct nouveau_gpuobj *gpuobj = NULL;
 266         int i;
 267
 268         NV_DEBUG(dev,
 269                  "pinst=0x%08x vinst=0x%010llx size=0x%08x flags=0x%08x\n",
 270                  pinst, vinst, size, flags);
 271
 272         gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
 273         if (!gpuobj)
 274                 return -ENOMEM;
 275         NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
 276         gpuobj->dev = dev;
 277         gpuobj->flags = flags;
 278         kref_init(&gpuobj->refcount);
 279         gpuobj->size  = size;
 280         gpuobj->pinst = pinst;
 281         gpuobj->cinst = 0xdeadbeef;
 282         gpuobj->vinst = vinst;
 283
 284         if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
 285                 for (i = 0; i < gpuobj->size; i += 4)
 286                         nv_wo32(gpuobj, i, 0);
 287                 dev_priv->engine.instmem.flush(dev);
 288         }
 289
 290         spin_lock(&dev_priv->ramin_lock);
 291         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
 292         spin_unlock(&dev_priv->ramin_lock);
 293         *pgpuobj = gpuobj;
 294         return 0;
 295 }
 296
 297
 298 static uint32_t
 299 nouveau_gpuobj_class_instmem_size(struct drm_device *dev, int class)
 300 {
 301         struct drm_nouveau_private *dev_priv = dev->dev_private;
 302
 303         /*XXX: dodgy hack for now */
 304         if (dev_priv->card_type >= NV_50)
 305                 return 24;
 306         if (dev_priv->card_type >= NV_40)
 307                 return 32;
 308         return 16;
 309 }
 310
 311 /*
 312    DMA objects are used to reference a piece of memory in the
 313    framebuffer, PCI or AGP address space. Each object is 16 bytes big
 314    and looks as follows:
 315
 316    entry[0]
 317    11:0  class (seems like I can always use 0 here)
 318    12    page table present?
 319    13    page entry linear?
 320    15:14 access: 0 rw, 1 ro, 2 wo
 321    17:16 target: 0 NV memory, 1 NV memory tiled, 2 PCI, 3 AGP
 322    31:20 dma adjust (bits 0-11 of the address)
 323    entry[1]
 324    dma limit (size of transfer)
 325    entry[X]
 326    1     0 readonly, 1 readwrite
 327    31:12 dma frame address of the page (bits 12-31 of the address)
 328    entry[N]
 329    page table terminator, same value as the first pte, as does nvidia
 330    rivatv uses 0xffffffff
 331
 332    Non linear page tables need a list of frame addresses afterwards,
 333    the rivatv project has some info on this.
 334
 335    The method below creates a DMA object in instance RAM and returns a handle
 336    to it that can be used to set up context objects.
 337 */
 338 int
 339 nouveau_gpuobj_dma_new(struct nouveau_channel *chan, int class,
 340                        uint64_t offset, uint64_t size, int access,
 341                        int target, struct nouveau_gpuobj **gpuobj)
 342 {
 343         struct drm_device *dev = chan->dev;
 344         struct drm_nouveau_private *dev_priv = dev->dev_private;
 345         struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
 346         int ret;
 347
 348         NV_DEBUG(dev, "ch%d class=0x%04x offset=0x%llx size=0x%llx\n",
 349                  chan->id, class, offset, size);
 350         NV_DEBUG(dev, "access=%d target=%d\n", access, target);
 351
 352         switch (target) {
 353         case NV_DMA_TARGET_AGP:
 354                 offset += dev_priv->gart_info.aper_base;
 355                 break;
 356         default:
 357                 break;
 358         }
 359
 360         ret = nouveau_gpuobj_new(dev, chan,
 361                                  nouveau_gpuobj_class_instmem_size(dev, class),
 362                                  16, NVOBJ_FLAG_ZERO_ALLOC |
 363                                  NVOBJ_FLAG_ZERO_FREE, gpuobj);
 364         if (ret) {
 365                 NV_ERROR(dev, "Error creating gpuobj: %d\n", ret);
 366                 return ret;
 367         }
 368
 369         if (dev_priv->card_type < NV_50) {
 370                 uint32_t frame, adjust, pte_flags = 0;
 371
 372                 if (access != NV_DMA_ACCESS_RO)
 373                         pte_flags |= (1<<1);
 374                 adjust = offset &  0x00000fff;
 375                 frame  = offset & ~0x00000fff;
 376
 377                 nv_wo32(*gpuobj,  0, ((1<<12) | (1<<13) | (adjust << 20) |
 378                                       (access << 14) | (target << 16) |
 379                                       class));
 380                 nv_wo32(*gpuobj,  4, size - 1);
 381                 nv_wo32(*gpuobj,  8, frame | pte_flags);
 382                 nv_wo32(*gpuobj, 12, frame | pte_flags);
 383         } else {
 384                 uint64_t limit = offset + size - 1;
 385                 uint32_t flags0, flags5;
 386
 387                 if (target == NV_DMA_TARGET_VIDMEM) {
 388                         flags0 = 0x00190000;
 389                         flags5 = 0x00010000;
 390                 } else {
 391                         flags0 = 0x7fc00000;
 392                         flags5 = 0x00080000;
 393                 }
 394
 395                 nv_wo32(*gpuobj,  0, flags0 | class);
 396                 nv_wo32(*gpuobj,  4, lower_32_bits(limit));
 397                 nv_wo32(*gpuobj,  8, lower_32_bits(offset));
 398                 nv_wo32(*gpuobj, 12, ((upper_32_bits(limit) & 0xff) << 24) |
 399                                       (upper_32_bits(offset) & 0xff));
 400                 nv_wo32(*gpuobj, 20, flags5);
 401         }
 402
 403         instmem->flush(dev);
 404
 405         (*gpuobj)->engine = NVOBJ_ENGINE_SW;
 406         (*gpuobj)->class  = class;
 407         return 0;
 408 }
 409
 410 int
 411 nouveau_gpuobj_gart_dma_new(struct nouveau_channel *chan,
 412                             uint64_t offset, uint64_t size, int access,
 413                             struct nouveau_gpuobj **gpuobj,
 414                             uint32_t *o_ret)
 415 {
 416         struct drm_device *dev = chan->dev;
 417         struct drm_nouveau_private *dev_priv = dev->dev_private;
 418         int ret;
 419
 420         if (dev_priv->gart_info.type == NOUVEAU_GART_AGP ||
 421             (dev_priv->card_type >= NV_50 &&
 422              dev_priv->gart_info.type == NOUVEAU_GART_SGDMA)) {
 423                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 424                                              offset + dev_priv->vm_gart_base,
 425                                              size, access, NV_DMA_TARGET_AGP,
 426                                              gpuobj);
 427                 if (o_ret)
 428                         *o_ret = 0;
 429         } else
 430         if (dev_priv->gart_info.type == NOUVEAU_GART_SGDMA) {
 431                 nouveau_gpuobj_ref(dev_priv->gart_info.sg_ctxdma, gpuobj);
 432                 if (offset & ~0xffffffffULL) {
 433                         NV_ERROR(dev, "obj offset exceeds 32-bits\n");
 434                         return -EINVAL;
 435                 }
 436                 if (o_ret)
 437                         *o_ret = (uint32_t)offset;
 438                 ret = (*gpuobj != NULL) ? 0 : -EINVAL;
 439         } else {
 440                 NV_ERROR(dev, "Invalid GART type %d\n", dev_priv->gart_info.type);
 441                 return -EINVAL;
 442         }
 443
 444         return ret;
 445 }
 446
 447 /* Context objects in the instance RAM have the following structure.
 448  * On NV40 they are 32 byte long, on NV30 and smaller 16 bytes.
 449
 450    NV4 - NV30:
 451
 452    entry[0]
 453    11:0 class
 454    12   chroma key enable
 455    13   user clip enable
 456    14   swizzle enable
 457    17:15 patch config:
 458        scrcopy_and, rop_and, blend_and, scrcopy, srccopy_pre, blend_pre
 459    18   synchronize enable
 460    19   endian: 1 big, 0 little
 461    21:20 dither mode
 462    23    single step enable
 463    24    patch status: 0 invalid, 1 valid
 464    25    context_surface 0: 1 valid
 465    26    context surface 1: 1 valid
 466    27    context pattern: 1 valid
 467    28    context rop: 1 valid
 468    29,30 context beta, beta4
 469    entry[1]
 470    7:0   mono format
 471    15:8  color format
 472    31:16 notify instance address
 473    entry[2]
 474    15:0  dma 0 instance address
 475    31:16 dma 1 instance address
 476    entry[3]
 477    dma method traps
 478
 479    NV40:
 480    No idea what the exact format is. Here's what can be deducted:
 481
 482    entry[0]:
 483    11:0  class  (maybe uses more bits here?)
 484    17    user clip enable
 485    21:19 patch config
 486    25    patch status valid ?
 487    entry[1]:
 488    15:0  DMA notifier  (maybe 20:0)
 489    entry[2]:
 490    15:0  DMA 0 instance (maybe 20:0)
 491    24    big endian
 492    entry[3]:
 493    15:0  DMA 1 instance (maybe 20:0)
 494    entry[4]:
 495    entry[5]:
 496    set to 0?
 497 */
 498 static int
 499 nouveau_gpuobj_sw_new(struct nouveau_channel *chan, int class,
 500                       struct nouveau_gpuobj **gpuobj_ret)
 501 {
 502         struct drm_nouveau_private *dev_priv;
 503         struct nouveau_gpuobj *gpuobj;
 504
 505         if (!chan || !gpuobj_ret || *gpuobj_ret != NULL)
 506                 return -EINVAL;
 507         dev_priv = chan->dev->dev_private;
 508
 509         gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
 510         if (!gpuobj)
 511                 return -ENOMEM;
 512         gpuobj->dev = chan->dev;
 513         gpuobj->engine = NVOBJ_ENGINE_SW;
 514         gpuobj->class = class;
 515         kref_init(&gpuobj->refcount);
 516         gpuobj->cinst = 0x40;
 517
 518         spin_lock(&dev_priv->ramin_lock);
 519         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
 520         spin_unlock(&dev_priv->ramin_lock);
 521         *gpuobj_ret = gpuobj;
 522         return 0;
 523 }
 524
 525 int
 526 nouveau_gpuobj_gr_new(struct nouveau_channel *chan, int class,
 527                       struct nouveau_gpuobj **gpuobj)
 528 {
 529         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 530         struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
 531         struct nouveau_pgraph_object_class *grc;
 532         struct drm_device *dev = chan->dev;
 533         int ret;
 534
 535         NV_DEBUG(dev, "ch%d class=0x%04x\n", chan->id, class);
 536
 537         grc = pgraph->grclass;
 538         while (grc->id) {
 539                 if (grc->id == class)
 540                         break;
 541                 grc++;
 542         }
 543
 544         if (!grc->id) {
 545                 NV_ERROR(dev, "illegal object class: 0x%x\n", class);
 546                 return -EINVAL;
 547         }
 548
 549         if (grc->engine == NVOBJ_ENGINE_SW)
 550                 return nouveau_gpuobj_sw_new(chan, class, gpuobj);
 551
 552         ret = nouveau_gpuobj_new(dev, chan,
 553                                  nouveau_gpuobj_class_instmem_size(dev, class),
 554                                  16,
 555                                  NVOBJ_FLAG_ZERO_ALLOC | NVOBJ_FLAG_ZERO_FREE,
 556                                  gpuobj);
 557         if (ret) {
 558                 NV_ERROR(dev, "error creating gpuobj: %d\n", ret);
 559                 return ret;
 560         }
 561
 562         if (dev_priv->card_type >= NV_50) {
 563                 nv_wo32(*gpuobj,  0, class);
 564                 nv_wo32(*gpuobj, 20, 0x00010000);
 565         } else {
 566                 switch (class) {
 567                 case NV_CLASS_NULL:
 568                         nv_wo32(*gpuobj, 0, 0x00001030);
 569                         nv_wo32(*gpuobj, 4, 0xFFFFFFFF);
 570                         break;
 571                 default:
 572                         if (dev_priv->card_type >= NV_40) {
 573                                 nv_wo32(*gpuobj, 0, class);
 574 #ifdef __BIG_ENDIAN
 575                                 nv_wo32(*gpuobj, 8, 0x01000000);
 576 #endif
 577                         } else {
 578 #ifdef __BIG_ENDIAN
 579                                 nv_wo32(*gpuobj, 0, class | 0x00080000);
 580 #else
 581                                 nv_wo32(*gpuobj, 0, class);
 582 #endif
 583                         }
 584                 }
 585         }
 586         dev_priv->engine.instmem.flush(dev);
 587
 588         (*gpuobj)->engine = grc->engine;
 589         (*gpuobj)->class  = class;
 590         return 0;
 591 }
 592
 593 static int
 594 nouveau_gpuobj_channel_init_pramin(struct nouveau_channel *chan)
 595 {
 596         struct drm_device *dev = chan->dev;
 597         struct drm_nouveau_private *dev_priv = dev->dev_private;
 598         uint32_t size;
 599         uint32_t base;
 600         int ret;
 601
 602         NV_DEBUG(dev, "ch%d\n", chan->id);
 603
 604         /* Base amount for object storage (4KiB enough?) */
 605         size = 0x1000;
 606         base = 0;
 607
 608         /* PGRAPH context */
 609         size += dev_priv->engine.graph.grctx_size;
 610
 611         if (dev_priv->card_type == NV_50) {
 612                 /* Various fixed table thingos */
 613                 size += 0x1400; /* mostly unknown stuff */
 614                 size += 0x4000; /* vm pd */
 615                 base  = 0x6000;
 616                 /* RAMHT, not sure about setting size yet, 32KiB to be safe */
 617                 size += 0x8000;
 618                 /* RAMFC */
 619                 size += 0x1000;
 620         }
 621
 622         ret = nouveau_gpuobj_new(dev, NULL, size, 0x1000, 0, &chan->ramin);
 623         if (ret) {
 624                 NV_ERROR(dev, "Error allocating channel PRAMIN: %d\n", ret);
 625                 return ret;
 626         }
 627
 628         ret = drm_mm_init(&chan->ramin_heap, base, size);
 629         if (ret) {
 630                 NV_ERROR(dev, "Error creating PRAMIN heap: %d\n", ret);
 631                 nouveau_gpuobj_ref(NULL, &chan->ramin);
 632                 return ret;
 633         }
 634
 635         return 0;
 636 }
 637
 638 int
 639 nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
 640                             uint32_t vram_h, uint32_t tt_h)
 641 {
 642         struct drm_device *dev = chan->dev;
 643         struct drm_nouveau_private *dev_priv = dev->dev_private;
 644         struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
 645         struct nouveau_gpuobj *vram = NULL, *tt = NULL;
 646         int ret, i;
 647
 648         NV_DEBUG(dev, "ch%d vram=0x%08x tt=0x%08x\n", chan->id, vram_h, tt_h);
 649
 650         /* Allocate a chunk of memory for per-channel object storage */
 651         ret = nouveau_gpuobj_channel_init_pramin(chan);
 652         if (ret) {
 653                 NV_ERROR(dev, "init pramin\n");
 654                 return ret;
 655         }
 656
 657         /* NV50 VM
 658          *  - Allocate per-channel page-directory
 659          *  - Map GART and VRAM into the channel's address space at the
 660          *    locations determined during init.
 661          */
 662         if (dev_priv->card_type >= NV_50) {
 663                 u32 pgd_offs = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200;
 664                 u64 vm_vinst = chan->ramin->vinst + pgd_offs;
 665                 u32 vm_pinst = chan->ramin->pinst;
 666                 u32 pde;
 667
 668                 if (vm_pinst != ~0)
 669                         vm_pinst += pgd_offs;
 670
 671                 ret = nouveau_gpuobj_new_fake(dev, vm_pinst, vm_vinst, 0x4000,
 672                                               0, &chan->vm_pd);
 673                 if (ret)
 674                         return ret;
 675                 for (i = 0; i < 0x4000; i += 8) {
 676                         nv_wo32(chan->vm_pd, i + 0, 0x00000000);
 677                         nv_wo32(chan->vm_pd, i + 4, 0xdeadcafe);
 678                 }
 679
 680                 nouveau_gpuobj_ref(dev_priv->gart_info.sg_ctxdma,
 681                                    &chan->vm_gart_pt);
 682                 pde = (dev_priv->vm_gart_base / (512*1024*1024)) * 8;
 683                 nv_wo32(chan->vm_pd, pde + 0, chan->vm_gart_pt->vinst | 3);
 684                 nv_wo32(chan->vm_pd, pde + 4, 0x00000000);
 685
 686                 pde = (dev_priv->vm_vram_base / (512*1024*1024)) * 8;
 687                 for (i = 0; i < dev_priv->vm_vram_pt_nr; i++) {
 688                         nouveau_gpuobj_ref(dev_priv->vm_vram_pt[i],
 689                                            &chan->vm_vram_pt[i]);
 690
 691                         nv_wo32(chan->vm_pd, pde + 0,
 692                                 chan->vm_vram_pt[i]->vinst | 0x61);
 693                         nv_wo32(chan->vm_pd, pde + 4, 0x00000000);
 694                         pde += 8;
 695                 }
 696
 697                 instmem->flush(dev);
 698         }
 699
 700         /* RAMHT */
 701         if (dev_priv->card_type < NV_50) {
 702                 nouveau_ramht_ref(dev_priv->ramht, &chan->ramht, NULL);
 703         } else {
 704                 struct nouveau_gpuobj *ramht = NULL;
 705
 706                 ret = nouveau_gpuobj_new(dev, chan, 0x8000, 16,
 707                                          NVOBJ_FLAG_ZERO_ALLOC, &ramht);
 708                 if (ret)
 709                         return ret;
 710
 711                 ret = nouveau_ramht_new(dev, ramht, &chan->ramht);
 712                 nouveau_gpuobj_ref(NULL, &ramht);
 713                 if (ret)
 714                         return ret;
 715         }
 716
 717         /* VRAM ctxdma */
 718         if (dev_priv->card_type >= NV_50) {
 719                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 720                                              0, dev_priv->vm_end,
 721                                              NV_DMA_ACCESS_RW,
 722                                              NV_DMA_TARGET_AGP, &vram);
 723                 if (ret) {
 724                         NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
 725                         return ret;
 726                 }
 727         } else {
 728                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 729                                              0, dev_priv->fb_available_size,
 730                                              NV_DMA_ACCESS_RW,
 731                                              NV_DMA_TARGET_VIDMEM, &vram);
 732                 if (ret) {
 733                         NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
 734                         return ret;
 735                 }
 736         }
 737
 738         ret = nouveau_ramht_insert(chan, vram_h, vram);
 739         nouveau_gpuobj_ref(NULL, &vram);
 740         if (ret) {
 741                 NV_ERROR(dev, "Error adding VRAM ctxdma to RAMHT: %d\n", ret);
 742                 return ret;
 743         }
 744
 745         /* TT memory ctxdma */
 746         if (dev_priv->card_type >= NV_50) {
 747                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 748                                              0, dev_priv->vm_end,
 749                                              NV_DMA_ACCESS_RW,
 750                                              NV_DMA_TARGET_AGP, &tt);
 751                 if (ret) {
 752                         NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
 753                         return ret;
 754                 }
 755         } else
 756         if (dev_priv->gart_info.type != NOUVEAU_GART_NONE) {
 757                 ret = nouveau_gpuobj_gart_dma_new(chan, 0,
 758                                                   dev_priv->gart_info.aper_size,
 759                                                   NV_DMA_ACCESS_RW, &tt, NULL);
 760         } else {
 761                 NV_ERROR(dev, "Invalid GART type %d\n", dev_priv->gart_info.type);
 762                 ret = -EINVAL;
 763         }
 764
 765         if (ret) {
 766                 NV_ERROR(dev, "Error creating TT ctxdma: %d\n", ret);
 767                 return ret;
 768         }
 769
 770         ret = nouveau_ramht_insert(chan, tt_h, tt);
 771         nouveau_gpuobj_ref(NULL, &tt);
 772         if (ret) {
 773                 NV_ERROR(dev, "Error adding TT ctxdma to RAMHT: %d\n", ret);
 774                 return ret;
 775         }
 776
 777         return 0;
 778 }
 779
 780 void
 781 nouveau_gpuobj_channel_takedown(struct nouveau_channel *chan)
 782 {
 783         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 784         struct drm_device *dev = chan->dev;
 785         int i;
 786
 787         NV_DEBUG(dev, "ch%d\n", chan->id);
 788
 789         if (!chan->ramht)
 790                 return;
 791
 792         nouveau_ramht_ref(NULL, &chan->ramht, chan);
 793
 794         nouveau_gpuobj_ref(NULL, &chan->vm_pd);
 795         nouveau_gpuobj_ref(NULL, &chan->vm_gart_pt);
 796         for (i = 0; i < dev_priv->vm_vram_pt_nr; i++)
 797                 nouveau_gpuobj_ref(NULL, &chan->vm_vram_pt[i]);
 798
 799         if (chan->ramin_heap.free_stack.next)
 800                 drm_mm_takedown(&chan->ramin_heap);
 801         nouveau_gpuobj_ref(NULL, &chan->ramin);
 802 }
 803
 804 int
 805 nouveau_gpuobj_suspend(struct drm_device *dev)
 806 {
 807         struct drm_nouveau_private *dev_priv = dev->dev_private;
 808         struct nouveau_gpuobj *gpuobj;
 809         int i;
 810
 811         if (dev_priv->card_type < NV_50) {
 812                 dev_priv->susres.ramin_copy = vmalloc(dev_priv->ramin_rsvd_vram);
 813                 if (!dev_priv->susres.ramin_copy)
 814                         return -ENOMEM;
 815
 816                 for (i = 0; i < dev_priv->ramin_rsvd_vram; i += 4)
 817                         dev_priv->susres.ramin_copy[i/4] = nv_ri32(dev, i);
 818                 return 0;
 819         }
 820
 821         list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
 822                 if (!gpuobj->im_backing)
 823                         continue;
 824
 825                 gpuobj->im_backing_suspend = vmalloc(gpuobj->size);
 826                 if (!gpuobj->im_backing_suspend) {
 827                         nouveau_gpuobj_resume(dev);
 828                         return -ENOMEM;
 829                 }
 830
 831                 for (i = 0; i < gpuobj->size; i += 4)
 832                         gpuobj->im_backing_suspend[i/4] = nv_ro32(gpuobj, i);
 833         }
 834
 835         return 0;
 836 }
 837
 838 void
 839 nouveau_gpuobj_suspend_cleanup(struct drm_device *dev)
 840 {
 841         struct drm_nouveau_private *dev_priv = dev->dev_private;
 842         struct nouveau_gpuobj *gpuobj;
 843
 844         if (dev_priv->card_type < NV_50) {
 845                 vfree(dev_priv->susres.ramin_copy);
 846                 dev_priv->susres.ramin_copy = NULL;
 847                 return;
 848         }
 849
 850         list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
 851                 if (!gpuobj->im_backing_suspend)
 852                         continue;
 853
 854                 vfree(gpuobj->im_backing_suspend);
 855                 gpuobj->im_backing_suspend = NULL;
 856         }
 857 }
 858
 859 void
 860 nouveau_gpuobj_resume(struct drm_device *dev)
 861 {
 862         struct drm_nouveau_private *dev_priv = dev->dev_private;
 863         struct nouveau_gpuobj *gpuobj;
 864         int i;
 865
 866         if (dev_priv->card_type < NV_50) {
 867                 for (i = 0; i < dev_priv->ramin_rsvd_vram; i += 4)
 868                         nv_wi32(dev, i, dev_priv->susres.ramin_copy[i/4]);
 869                 nouveau_gpuobj_suspend_cleanup(dev);
 870                 return;
 871         }
 872
 873         list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
 874                 if (!gpuobj->im_backing_suspend)
 875                         continue;
 876
 877                 for (i = 0; i < gpuobj->size; i += 4)
 878                         nv_wo32(gpuobj, i, gpuobj->im_backing_suspend[i/4]);
 879                 dev_priv->engine.instmem.flush(dev);
 880         }
 881
 882         nouveau_gpuobj_suspend_cleanup(dev);
 883 }
 884
 885 int nouveau_ioctl_grobj_alloc(struct drm_device *dev, void *data,
 886                               struct drm_file *file_priv)
 887 {
 888         struct drm_nouveau_grobj_alloc *init = data;
 889         struct nouveau_gpuobj *gr = NULL;
 890         struct nouveau_channel *chan;
 891         int ret;
 892
 893         if (init->handle == ~0)
 894                 return -EINVAL;
 895
 896         chan = nouveau_channel_get(dev, file_priv, init->channel);
 897         if (IS_ERR(chan))
 898                 return PTR_ERR(chan);
 899
 900         if (nouveau_ramht_find(chan, init->handle)) {
 901                 ret = -EEXIST;
 902                 goto out;
 903         }
 904
 905         ret = nouveau_gpuobj_gr_new(chan, init->class, &gr);
 906         if (ret) {
 907                 NV_ERROR(dev, "Error creating object: %d (%d/0x%08x)\n",
 908                          ret, init->channel, init->handle);
 909                 goto out;
 910         }
 911
 912         ret = nouveau_ramht_insert(chan, init->handle, gr);
 913         nouveau_gpuobj_ref(NULL, &gr);
 914         if (ret) {
 915                 NV_ERROR(dev, "Error referencing object: %d (%d/0x%08x)\n",
 916                          ret, init->channel, init->handle);
 917         }
 918
 919 out:
 920         nouveau_channel_put(&chan);
 921         return ret;
 922 }
 923
 924 int nouveau_ioctl_gpuobj_free(struct drm_device *dev, void *data,
 925                               struct drm_file *file_priv)
 926 {
 927         struct drm_nouveau_gpuobj_free *objfree = data;
 928         struct nouveau_channel *chan;
 929         int ret;
 930
 931         chan = nouveau_channel_get(dev, file_priv, objfree->channel);
 932         if (IS_ERR(chan))
 933                 return PTR_ERR(chan);
 934
 935         ret = nouveau_ramht_remove(chan, objfree->handle);
 936         nouveau_channel_put(&chan);
 937         return ret;
 938 }
 939
 940 u32
 941 nv_ro32(struct nouveau_gpuobj *gpuobj, u32 offset)
 942 {
 943         struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
 944         struct drm_device *dev = gpuobj->dev;
 945
 946         if (gpuobj->pinst == ~0 || !dev_priv->ramin_available) {
 947                 u64  ptr = gpuobj->vinst + offset;
 948                 u32 base = ptr >> 16;
 949                 u32  val;
 950
 951                 spin_lock(&dev_priv->ramin_lock);
 952                 if (dev_priv->ramin_base != base) {
 953                         dev_priv->ramin_base = base;
 954                         nv_wr32(dev, 0x001700, dev_priv->ramin_base);
 955                 }
 956                 val = nv_rd32(dev, 0x700000 + (ptr & 0xffff));
 957                 spin_unlock(&dev_priv->ramin_lock);
 958                 return val;
 959         }
 960
 961         return nv_ri32(dev, gpuobj->pinst + offset);
 962 }
 963
 964 void
 965 nv_wo32(struct nouveau_gpuobj *gpuobj, u32 offset, u32 val)
 966 {
 967         struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
 968         struct drm_device *dev = gpuobj->dev;
 969
 970         if (gpuobj->pinst == ~0 || !dev_priv->ramin_available) {
 971                 u64  ptr = gpuobj->vinst + offset;
 972                 u32 base = ptr >> 16;
 973
 974                 spin_lock(&dev_priv->ramin_lock);
 975                 if (dev_priv->ramin_base != base) {
 976                         dev_priv->ramin_base = base;
 977                         nv_wr32(dev, 0x001700, dev_priv->ramin_base);
 978                 }
 979                 nv_wr32(dev, 0x700000 + (ptr & 0xffff), val);
 980                 spin_unlock(&dev_priv->ramin_lock);
 981                 return;
 982         }
 983
 984         nv_wi32(dev, gpuobj->pinst + offset, val);
 985 }