drivers/gpu/drm/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <drm/drmP.h>
  26 #include <drm/i915_drm.h>
  27 #include "i915_drv.h"
  28 #include "i915_trace.h"
  29 #include "intel_drv.h"
  30
  31 typedef uint32_t gtt_pte_t;
  32
  33 static inline gtt_pte_t pte_encode(struct drm_device *dev,
  34                                    dma_addr_t addr,
  35                                    enum i915_cache_level level)
  36 {
  37         gtt_pte_t pte = GEN6_PTE_VALID;
  38         pte |= GEN6_PTE_ADDR_ENCODE(addr);
  39
  40         switch (level) {
  41         case I915_CACHE_LLC_MLC:
  42                 /* Haswell doesn't set L3 this way */
  43                 if (IS_HASWELL(dev))
  44                         pte |= GEN6_PTE_CACHE_LLC;
  45                 else
  46                         pte |= GEN6_PTE_CACHE_LLC_MLC;
  47                 break;
  48         case I915_CACHE_LLC:
  49                 pte |= GEN6_PTE_CACHE_LLC;
  50                 break;
  51         case I915_CACHE_NONE:
  52                 if (IS_HASWELL(dev))
  53                         pte |= HSW_PTE_UNCACHED;
  54                 else
  55                         pte |= GEN6_PTE_UNCACHED;
  56                 break;
  57         default:
  58                 BUG();
  59         }
  60
  61
  62         return pte;
  63 }
  64
  65 /* PPGTT support for Sandybdrige/Gen6 and later */
  66 static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
  67                                    unsigned first_entry,
  68                                    unsigned num_entries)
  69 {
  70         gtt_pte_t *pt_vaddr;
  71         gtt_pte_t scratch_pte;
  72         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
  73         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
  74         unsigned last_pte, i;
  75
  76         scratch_pte = pte_encode(ppgtt->dev, ppgtt->scratch_page_dma_addr,
  77                                  I915_CACHE_LLC);
  78
  79         while (num_entries) {
  80                 last_pte = first_pte + num_entries;
  81                 if (last_pte > I915_PPGTT_PT_ENTRIES)
  82                         last_pte = I915_PPGTT_PT_ENTRIES;
  83
  84                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
  85
  86                 for (i = first_pte; i < last_pte; i++)
  87                         pt_vaddr[i] = scratch_pte;
  88
  89                 kunmap_atomic(pt_vaddr);
  90
  91                 num_entries -= last_pte - first_pte;
  92                 first_pte = 0;
  93                 act_pd++;
  94         }
  95 }
  96
  97 int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
  98 {
  99         struct drm_i915_private *dev_priv = dev->dev_private;
 100         struct i915_hw_ppgtt *ppgtt;
 101         unsigned first_pd_entry_in_global_pt;
 102         int i;
 103         int ret = -ENOMEM;
 104
 105         /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
 106          * entries. For aliasing ppgtt support we just steal them at the end for
 107          * now. */
 108         first_pd_entry_in_global_pt = dev_priv->mm.gtt->gtt_total_entries - I915_PPGTT_PD_ENTRIES;
 109
 110         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
 111         if (!ppgtt)
 112                 return ret;
 113
 114         ppgtt->dev = dev;
 115         ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
 116         ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
 117                                   GFP_KERNEL);
 118         if (!ppgtt->pt_pages)
 119                 goto err_ppgtt;
 120
 121         for (i = 0; i < ppgtt->num_pd_entries; i++) {
 122                 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
 123                 if (!ppgtt->pt_pages[i])
 124                         goto err_pt_alloc;
 125         }
 126
 127         if (dev_priv->mm.gtt->needs_dmar) {
 128                 ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t)
 129                                                 *ppgtt->num_pd_entries,
 130                                              GFP_KERNEL);
 131                 if (!ppgtt->pt_dma_addr)
 132                         goto err_pt_alloc;
 133
 134                 for (i = 0; i < ppgtt->num_pd_entries; i++) {
 135                         dma_addr_t pt_addr;
 136
 137                         pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i],
 138                                                0, 4096,
 139                                                PCI_DMA_BIDIRECTIONAL);
 140
 141                         if (pci_dma_mapping_error(dev->pdev,
 142                                                   pt_addr)) {
 143                                 ret = -EIO;
 144                                 goto err_pd_pin;
 145
 146                         }
 147                         ppgtt->pt_dma_addr[i] = pt_addr;
 148                 }
 149         }
 150
 151         ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt->scratch_page_dma;
 152
 153         i915_ppgtt_clear_range(ppgtt, 0,
 154                                ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
 155
 156         ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(gtt_pte_t);
 157
 158         dev_priv->mm.aliasing_ppgtt = ppgtt;
 159
 160         return 0;
 161
 162 err_pd_pin:
 163         if (ppgtt->pt_dma_addr) {
 164                 for (i--; i >= 0; i--)
 165                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
 166                                        4096, PCI_DMA_BIDIRECTIONAL);
 167         }
 168 err_pt_alloc:
 169         kfree(ppgtt->pt_dma_addr);
 170         for (i = 0; i < ppgtt->num_pd_entries; i++) {
 171                 if (ppgtt->pt_pages[i])
 172                         __free_page(ppgtt->pt_pages[i]);
 173         }
 174         kfree(ppgtt->pt_pages);
 175 err_ppgtt:
 176         kfree(ppgtt);
 177
 178         return ret;
 179 }
 180
 181 void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
 182 {
 183         struct drm_i915_private *dev_priv = dev->dev_private;
 184         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
 185         int i;
 186
 187         if (!ppgtt)
 188                 return;
 189
 190         if (ppgtt->pt_dma_addr) {
 191                 for (i = 0; i < ppgtt->num_pd_entries; i++)
 192                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
 193                                        4096, PCI_DMA_BIDIRECTIONAL);
 194         }
 195
 196         kfree(ppgtt->pt_dma_addr);
 197         for (i = 0; i < ppgtt->num_pd_entries; i++)
 198                 __free_page(ppgtt->pt_pages[i]);
 199         kfree(ppgtt->pt_pages);
 200         kfree(ppgtt);
 201 }
 202
 203 static void i915_ppgtt_insert_sg_entries(struct i915_hw_ppgtt *ppgtt,
 204                                          const struct sg_table *pages,
 205                                          unsigned first_entry,
 206                                          enum i915_cache_level cache_level)
 207 {
 208         gtt_pte_t *pt_vaddr;
 209         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
 210         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 211         unsigned i, j, m, segment_len;
 212         dma_addr_t page_addr;
 213         struct scatterlist *sg;
 214
 215         /* init sg walking */
 216         sg = pages->sgl;
 217         i = 0;
 218         segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
 219         m = 0;
 220
 221         while (i < pages->nents) {
 222                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
 223
 224                 for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) {
 225                         page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
 226                         pt_vaddr[j] = pte_encode(ppgtt->dev, page_addr,
 227                                                  cache_level);
 228
 229                         /* grab the next page */
 230                         if (++m == segment_len) {
 231                                 if (++i == pages->nents)
 232                                         break;
 233
 234                                 sg = sg_next(sg);
 235                                 segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
 236                                 m = 0;
 237                         }
 238                 }
 239
 240                 kunmap_atomic(pt_vaddr);
 241
 242                 first_pte = 0;
 243                 act_pd++;
 244         }
 245 }
 246
 247 void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
 248                             struct drm_i915_gem_object *obj,
 249                             enum i915_cache_level cache_level)
 250 {
 251         i915_ppgtt_insert_sg_entries(ppgtt,
 252                                      obj->pages,
 253                                      obj->gtt_space->start >> PAGE_SHIFT,
 254                                      cache_level);
 255 }
 256
 257 void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
 258                               struct drm_i915_gem_object *obj)
 259 {
 260         i915_ppgtt_clear_range(ppgtt,
 261                                obj->gtt_space->start >> PAGE_SHIFT,
 262                                obj->base.size >> PAGE_SHIFT);
 263 }
 264
 265 static bool do_idling(struct drm_i915_private *dev_priv)
 266 {
 267         bool ret = dev_priv->mm.interruptible;
 268
 269         if (unlikely(dev_priv->mm.gtt->do_idle_maps)) {
 270                 dev_priv->mm.interruptible = false;
 271                 if (i915_gpu_idle(dev_priv->dev)) {
 272                         DRM_ERROR("Couldn't idle GPU\n");
 273                         /* Wait a bit, in hopes it avoids the hang */
 274                         udelay(10);
 275                 }
 276         }
 277
 278         return ret;
 279 }
 280
 281 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
 282 {
 283         if (unlikely(dev_priv->mm.gtt->do_idle_maps))
 284                 dev_priv->mm.interruptible = interruptible;
 285 }
 286
 287
 288 static void i915_ggtt_clear_range(struct drm_device *dev,
 289                                  unsigned first_entry,
 290                                  unsigned num_entries)
 291 {
 292         struct drm_i915_private *dev_priv = dev->dev_private;
 293         gtt_pte_t scratch_pte;
 294         volatile void __iomem *gtt_base = dev_priv->mm.gtt->gtt + first_entry;
 295         const int max_entries = dev_priv->mm.gtt->gtt_total_entries - first_entry;
 296
 297         if (INTEL_INFO(dev)->gen < 6) {
 298                 intel_gtt_clear_range(first_entry, num_entries);
 299                 return;
 300         }
 301
 302         if (WARN(num_entries > max_entries,
 303                  "First entry = %d; Num entries = %d (max=%d)\n",
 304                  first_entry, num_entries, max_entries))
 305                 num_entries = max_entries;
 306
 307         scratch_pte = pte_encode(dev, dev_priv->mm.gtt->scratch_page_dma, I915_CACHE_LLC);
 308         memset_io(gtt_base, scratch_pte, num_entries * sizeof(scratch_pte));
 309         readl(gtt_base);
 310 }
 311
 312 void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 313 {
 314         struct drm_i915_private *dev_priv = dev->dev_private;
 315         struct drm_i915_gem_object *obj;
 316
 317         /* First fill our portion of the GTT with scratch pages */
 318         i915_ggtt_clear_range(dev, dev_priv->mm.gtt_start / PAGE_SIZE,
 319                               (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
 320
 321         list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
 322                 i915_gem_clflush_object(obj);
 323                 i915_gem_gtt_bind_object(obj, obj->cache_level);
 324         }
 325
 326         i915_gem_chipset_flush(dev);
 327 }
 328
 329 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
 330 {
 331         if (obj->has_dma_mapping)
 332                 return 0;
 333
 334         if (!dma_map_sg(&obj->base.dev->pdev->dev,
 335                         obj->pages->sgl, obj->pages->nents,
 336                         PCI_DMA_BIDIRECTIONAL))
 337                 return -ENOSPC;
 338
 339         return 0;
 340 }
 341
 342 /*
 343  * Binds an object into the global gtt with the specified cache level. The object
 344  * will be accessible to the GPU via commands whose operands reference offsets
 345  * within the global GTT as well as accessible by the GPU through the GMADR
 346  * mapped BAR (dev_priv->mm.gtt->gtt).
 347  */
 348 static void gen6_ggtt_bind_object(struct drm_i915_gem_object *obj,
 349                                   enum i915_cache_level level)
 350 {
 351         struct drm_device *dev = obj->base.dev;
 352         struct drm_i915_private *dev_priv = dev->dev_private;
 353         struct sg_table *st = obj->pages;
 354         struct scatterlist *sg = st->sgl;
 355         const int first_entry = obj->gtt_space->start >> PAGE_SHIFT;
 356         const int max_entries = dev_priv->mm.gtt->gtt_total_entries - first_entry;
 357         gtt_pte_t __iomem *gtt_entries = dev_priv->mm.gtt->gtt + first_entry;
 358         int unused, i = 0;
 359         unsigned int len, m = 0;
 360         dma_addr_t addr;
 361
 362         for_each_sg(st->sgl, sg, st->nents, unused) {
 363                 len = sg_dma_len(sg) >> PAGE_SHIFT;
 364                 for (m = 0; m < len; m++) {
 365                         addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
 366                         gtt_entries[i] = pte_encode(dev, addr, level);
 367                         i++;
 368                 }
 369         }
 370
 371         BUG_ON(i > max_entries);
 372         BUG_ON(i != obj->base.size / PAGE_SIZE);
 373
 374         /* XXX: This serves as a posting read to make sure that the PTE has
 375          * actually been updated. There is some concern that even though
 376          * registers and PTEs are within the same BAR that they are potentially
 377          * of NUMA access patterns. Therefore, even with the way we assume
 378          * hardware should work, we must keep this posting read for paranoia.
 379          */
 380         if (i != 0)
 381                 WARN_ON(readl(&gtt_entries[i-1]) != pte_encode(dev, addr, level));
 382 }
 383
 384 void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
 385                               enum i915_cache_level cache_level)
 386 {
 387         struct drm_device *dev = obj->base.dev;
 388         if (INTEL_INFO(dev)->gen < 6) {
 389                 unsigned int flags = (cache_level == I915_CACHE_NONE) ?
 390                         AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
 391                 intel_gtt_insert_sg_entries(obj->pages,
 392                                             obj->gtt_space->start >> PAGE_SHIFT,
 393                                             flags);
 394         } else {
 395                 gen6_ggtt_bind_object(obj, cache_level);
 396         }
 397
 398         obj->has_global_gtt_mapping = 1;
 399 }
 400
 401 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
 402 {
 403         i915_ggtt_clear_range(obj->base.dev,
 404                               obj->gtt_space->start >> PAGE_SHIFT,
 405                               obj->base.size >> PAGE_SHIFT);
 406
 407         obj->has_global_gtt_mapping = 0;
 408 }
 409
 410 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
 411 {
 412         struct drm_device *dev = obj->base.dev;
 413         struct drm_i915_private *dev_priv = dev->dev_private;
 414         bool interruptible;
 415
 416         interruptible = do_idling(dev_priv);
 417
 418         if (!obj->has_dma_mapping)
 419                 dma_unmap_sg(&dev->pdev->dev,
 420                              obj->pages->sgl, obj->pages->nents,
 421                              PCI_DMA_BIDIRECTIONAL);
 422
 423         undo_idling(dev_priv, interruptible);
 424 }
 425
 426 static void i915_gtt_color_adjust(struct drm_mm_node *node,
 427                                   unsigned long color,
 428                                   unsigned long *start,
 429                                   unsigned long *end)
 430 {
 431         if (node->color != color)
 432                 *start += 4096;
 433
 434         if (!list_empty(&node->node_list)) {
 435                 node = list_entry(node->node_list.next,
 436                                   struct drm_mm_node,
 437                                   node_list);
 438                 if (node->allocated && node->color != color)
 439                         *end -= 4096;
 440         }
 441 }
 442
 443 void i915_gem_init_global_gtt(struct drm_device *dev,
 444                               unsigned long start,
 445                               unsigned long mappable_end,
 446                               unsigned long end)
 447 {
 448         drm_i915_private_t *dev_priv = dev->dev_private;
 449
 450         /* Substract the guard page ... */
 451         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE);
 452         if (!HAS_LLC(dev))
 453                 dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust;
 454
 455         dev_priv->mm.gtt_start = start;
 456         dev_priv->mm.gtt_mappable_end = mappable_end;
 457         dev_priv->mm.gtt_end = end;
 458         dev_priv->mm.gtt_total = end - start;
 459         dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
 460
 461         /* ... but ensure that we clear the entire range. */
 462         i915_ggtt_clear_range(dev, start / PAGE_SIZE, (end-start) / PAGE_SIZE);
 463 }
 464
 465 static int setup_scratch_page(struct drm_device *dev)
 466 {
 467         struct drm_i915_private *dev_priv = dev->dev_private;
 468         struct page *page;
 469         dma_addr_t dma_addr;
 470
 471         page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
 472         if (page == NULL)
 473                 return -ENOMEM;
 474         get_page(page);
 475         set_pages_uc(page, 1);
 476
 477 #ifdef CONFIG_INTEL_IOMMU
 478         dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
 479                                 PCI_DMA_BIDIRECTIONAL);
 480         if (pci_dma_mapping_error(dev->pdev, dma_addr))
 481                 return -EINVAL;
 482 #else
 483         dma_addr = page_to_phys(page);
 484 #endif
 485         dev_priv->mm.gtt->scratch_page = page;
 486         dev_priv->mm.gtt->scratch_page_dma = dma_addr;
 487
 488         return 0;
 489 }
 490
 491 static void teardown_scratch_page(struct drm_device *dev)
 492 {
 493         struct drm_i915_private *dev_priv = dev->dev_private;
 494         set_pages_wb(dev_priv->mm.gtt->scratch_page, 1);
 495         pci_unmap_page(dev->pdev, dev_priv->mm.gtt->scratch_page_dma,
 496                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 497         put_page(dev_priv->mm.gtt->scratch_page);
 498         __free_page(dev_priv->mm.gtt->scratch_page);
 499 }
 500
 501 static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
 502 {
 503         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
 504         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
 505         return snb_gmch_ctl << 20;
 506 }
 507
 508 static inline unsigned int gen6_get_stolen_size(u16 snb_gmch_ctl)
 509 {
 510         snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
 511         snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
 512         return snb_gmch_ctl << 25; /* 32 MB units */
 513 }
 514
 515 static inline unsigned int gen7_get_stolen_size(u16 snb_gmch_ctl)
 516 {
 517         static const int stolen_decoder[] = {
 518                 0, 0, 0, 0, 0, 32, 48, 64, 128, 256, 96, 160, 224, 352};
 519         snb_gmch_ctl >>= IVB_GMCH_GMS_SHIFT;
 520         snb_gmch_ctl &= IVB_GMCH_GMS_MASK;
 521         return stolen_decoder[snb_gmch_ctl] << 20;
 522 }
 523
 524 int i915_gem_gtt_init(struct drm_device *dev)
 525 {
 526         struct drm_i915_private *dev_priv = dev->dev_private;
 527         phys_addr_t gtt_bus_addr;
 528         u16 snb_gmch_ctl;
 529         u32 tmp;
 530         int ret;
 531
 532         /* On modern platforms we need not worry ourself with the legacy
 533          * hostbridge query stuff. Skip it entirely
 534          */
 535         if (INTEL_INFO(dev)->gen < 6) {
 536                 ret = intel_gmch_probe(dev_priv->bridge_dev, dev->pdev, NULL);
 537                 if (!ret) {
 538                         DRM_ERROR("failed to set up gmch\n");
 539                         return -EIO;
 540                 }
 541
 542                 dev_priv->mm.gtt = intel_gtt_get();
 543                 if (!dev_priv->mm.gtt) {
 544                         DRM_ERROR("Failed to initialize GTT\n");
 545                         intel_gmch_remove();
 546                         return -ENODEV;
 547                 }
 548                 return 0;
 549         }
 550
 551         dev_priv->mm.gtt = kzalloc(sizeof(*dev_priv->mm.gtt), GFP_KERNEL);
 552         if (!dev_priv->mm.gtt)
 553                 return -ENOMEM;
 554
 555         if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
 556                 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
 557
 558         pci_read_config_dword(dev->pdev, PCI_BASE_ADDRESS_0, &tmp);
 559         /* For GEN6+ the PTEs for the ggtt live at 2MB + BAR0 */
 560         gtt_bus_addr = (tmp & PCI_BASE_ADDRESS_MEM_MASK) + (2<<20);
 561
 562         pci_read_config_dword(dev->pdev, PCI_BASE_ADDRESS_2, &tmp);
 563         dev_priv->mm.gtt->gma_bus_addr = tmp & PCI_BASE_ADDRESS_MEM_MASK;
 564
 565         /* i9xx_setup */
 566         pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 567         dev_priv->mm.gtt->gtt_total_entries =
 568                 gen6_get_total_gtt_size(snb_gmch_ctl) / sizeof(gtt_pte_t);
 569         if (INTEL_INFO(dev)->gen < 7)
 570                 dev_priv->mm.gtt->stolen_size = gen6_get_stolen_size(snb_gmch_ctl);
 571         else
 572                 dev_priv->mm.gtt->stolen_size = gen7_get_stolen_size(snb_gmch_ctl);
 573
 574         dev_priv->mm.gtt->gtt_mappable_entries = pci_resource_len(dev->pdev, 2) >> PAGE_SHIFT;
 575         /* 64/512MB is the current min/max we actually know of, but this is just a
 576          * coarse sanity check.
 577          */
 578         if ((dev_priv->mm.gtt->gtt_mappable_entries >> 8) < 64 ||
 579             dev_priv->mm.gtt->gtt_mappable_entries > dev_priv->mm.gtt->gtt_total_entries) {
 580                 DRM_ERROR("Unknown GMADR entries (%d)\n",
 581                           dev_priv->mm.gtt->gtt_mappable_entries);
 582                 ret = -ENXIO;
 583                 goto err_out;
 584         }
 585
 586         ret = setup_scratch_page(dev);
 587         if (ret) {
 588                 DRM_ERROR("Scratch setup failed\n");
 589                 goto err_out;
 590         }
 591
 592         dev_priv->mm.gtt->gtt = ioremap(gtt_bus_addr,
 593                                         dev_priv->mm.gtt->gtt_total_entries * sizeof(gtt_pte_t));
 594         if (!dev_priv->mm.gtt->gtt) {
 595                 DRM_ERROR("Failed to map the gtt page table\n");
 596                 teardown_scratch_page(dev);
 597                 ret = -ENOMEM;
 598                 goto err_out;
 599         }
 600
 601         /* GMADR is the PCI aperture used by SW to access tiled GFX surfaces in a linear fashion. */
 602         DRM_INFO("Memory Usable by graphics device = %dK\n", dev_priv->mm.gtt->gtt_total_entries >> 10);
 603         DRM_DEBUG_DRIVER("GMADR size = %dM\n", dev_priv->mm.gtt->gtt_mappable_entries >> 8);
 604         DRM_DEBUG_DRIVER("GTT stolen size = %dM\n", dev_priv->mm.gtt->stolen_size >> 20);
 605
 606         return 0;
 607
 608 err_out:
 609         kfree(dev_priv->mm.gtt);
 610         if (INTEL_INFO(dev)->gen < 6)
 611                 intel_gmch_remove();
 612         return ret;
 613 }
 614
 615 void i915_gem_gtt_fini(struct drm_device *dev)
 616 {
 617         struct drm_i915_private *dev_priv = dev->dev_private;
 618         iounmap(dev_priv->mm.gtt->gtt);
 619         teardown_scratch_page(dev);
 620         if (INTEL_INFO(dev)->gen < 6)
 621                 intel_gmch_remove();
 622         kfree(dev_priv->mm.gtt);
 623 }