2 * Copyright 2007 Dave Airlied
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
25 * Authors: Dave Airlied <airlied@linux.ie>
26 * Ben Skeggs <darktama@iinet.net.au>
27 * Jeremy Kolb <jkolb@brandeis.edu>
30 #include <linux/dma-mapping.h>
31 #include <linux/swiotlb.h>
33 #include "nouveau_drm.h"
34 #include "nouveau_dma.h"
35 #include "nouveau_fence.h"
37 #include "nouveau_bo.h"
38 #include "nouveau_ttm.h"
39 #include "nouveau_gem.h"
42 * NV10-NV40 tiling helpers
46 nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg,
47 u32 addr, u32 size, u32 pitch, u32 flags)
49 struct nouveau_drm *drm = nouveau_drm(dev);
50 int i = reg - drm->tile.reg;
51 struct nvkm_device *device = nvxx_device(&drm->device);
52 struct nvkm_fb *fb = device->fb;
53 struct nvkm_fb_tile *tile = &fb->tile.region[i];
55 nouveau_fence_unref(®->fence);
58 nvkm_fb_tile_fini(fb, i, tile);
61 nvkm_fb_tile_init(fb, i, addr, size, pitch, flags, tile);
63 nvkm_fb_tile_prog(fb, i, tile);
66 static struct nouveau_drm_tile *
67 nv10_bo_get_tile_region(struct drm_device *dev, int i)
69 struct nouveau_drm *drm = nouveau_drm(dev);
70 struct nouveau_drm_tile *tile = &drm->tile.reg[i];
72 spin_lock(&drm->tile.lock);
75 (!tile->fence || nouveau_fence_done(tile->fence)))
80 spin_unlock(&drm->tile.lock);
85 nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile,
88 struct nouveau_drm *drm = nouveau_drm(dev);
91 spin_lock(&drm->tile.lock);
92 tile->fence = (struct nouveau_fence *)fence_get(fence);
94 spin_unlock(&drm->tile.lock);
98 static struct nouveau_drm_tile *
99 nv10_bo_set_tiling(struct drm_device *dev, u32 addr,
100 u32 size, u32 pitch, u32 flags)
102 struct nouveau_drm *drm = nouveau_drm(dev);
103 struct nvkm_fb *fb = nvxx_fb(&drm->device);
104 struct nouveau_drm_tile *tile, *found = NULL;
107 for (i = 0; i < fb->tile.regions; i++) {
108 tile = nv10_bo_get_tile_region(dev, i);
110 if (pitch && !found) {
114 } else if (tile && fb->tile.region[i].pitch) {
115 /* Kill an unused tile region. */
116 nv10_bo_update_tile_region(dev, tile, 0, 0, 0, 0);
119 nv10_bo_put_tile_region(dev, tile, NULL);
123 nv10_bo_update_tile_region(dev, found, addr, size,
129 nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
131 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
132 struct drm_device *dev = drm->dev;
133 struct nouveau_bo *nvbo = nouveau_bo(bo);
135 if (unlikely(nvbo->gem.filp))
136 DRM_ERROR("bo %p still attached to GEM object\n", bo);
137 WARN_ON(nvbo->pin_refcnt > 0);
138 nv10_bo_put_tile_region(dev, nvbo->tile, NULL);
143 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags,
144 int *align, int *size)
146 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
147 struct nvif_device *device = &drm->device;
149 if (device->info.family < NV_DEVICE_INFO_V0_TESLA) {
150 if (nvbo->tile_mode) {
151 if (device->info.chipset >= 0x40) {
153 *size = roundup(*size, 64 * nvbo->tile_mode);
155 } else if (device->info.chipset >= 0x30) {
157 *size = roundup(*size, 64 * nvbo->tile_mode);
159 } else if (device->info.chipset >= 0x20) {
161 *size = roundup(*size, 64 * nvbo->tile_mode);
163 } else if (device->info.chipset >= 0x10) {
165 *size = roundup(*size, 32 * nvbo->tile_mode);
169 *size = roundup(*size, (1 << nvbo->page_shift));
170 *align = max((1 << nvbo->page_shift), *align);
173 *size = roundup(*size, PAGE_SIZE);
177 nouveau_bo_new(struct drm_device *dev, int size, int align,
178 uint32_t flags, uint32_t tile_mode, uint32_t tile_flags,
179 struct sg_table *sg, struct reservation_object *robj,
180 struct nouveau_bo **pnvbo)
182 struct nouveau_drm *drm = nouveau_drm(dev);
183 struct nouveau_bo *nvbo;
186 int type = ttm_bo_type_device;
191 lpg_shift = drm->client.vm->mmu->lpg_shift;
192 max_size = INT_MAX & ~((1 << lpg_shift) - 1);
194 if (size <= 0 || size > max_size) {
195 NV_WARN(drm, "skipped size %x\n", (u32)size);
200 type = ttm_bo_type_sg;
202 nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
205 INIT_LIST_HEAD(&nvbo->head);
206 INIT_LIST_HEAD(&nvbo->entry);
207 INIT_LIST_HEAD(&nvbo->vma_list);
208 nvbo->tile_mode = tile_mode;
209 nvbo->tile_flags = tile_flags;
210 nvbo->bo.bdev = &drm->ttm.bdev;
212 if (!nvxx_device(&drm->device)->func->cpu_coherent)
213 nvbo->force_coherent = flags & TTM_PL_FLAG_UNCACHED;
215 nvbo->page_shift = 12;
216 if (drm->client.vm) {
217 if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024)
218 nvbo->page_shift = drm->client.vm->mmu->lpg_shift;
221 nouveau_bo_fixup_align(nvbo, flags, &align, &size);
222 nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;
223 nouveau_bo_placement_set(nvbo, flags, 0);
225 acc_size = ttm_bo_dma_acc_size(&drm->ttm.bdev, size,
226 sizeof(struct nouveau_bo));
228 ret = ttm_bo_init(&drm->ttm.bdev, &nvbo->bo, size,
229 type, &nvbo->placement,
230 align >> PAGE_SHIFT, false, NULL, acc_size, sg,
231 robj, nouveau_bo_del_ttm);
233 /* ttm will call nouveau_bo_del_ttm if it fails.. */
242 set_placement_list(struct ttm_place *pl, unsigned *n, uint32_t type, uint32_t flags)
246 if (type & TTM_PL_FLAG_VRAM)
247 pl[(*n)++].flags = TTM_PL_FLAG_VRAM | flags;
248 if (type & TTM_PL_FLAG_TT)
249 pl[(*n)++].flags = TTM_PL_FLAG_TT | flags;
250 if (type & TTM_PL_FLAG_SYSTEM)
251 pl[(*n)++].flags = TTM_PL_FLAG_SYSTEM | flags;
255 set_placement_range(struct nouveau_bo *nvbo, uint32_t type)
257 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
258 u32 vram_pages = drm->device.info.ram_size >> PAGE_SHIFT;
259 unsigned i, fpfn, lpfn;
261 if (drm->device.info.family == NV_DEVICE_INFO_V0_CELSIUS &&
262 nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) &&
263 nvbo->bo.mem.num_pages < vram_pages / 4) {
265 * Make sure that the color and depth buffers are handled
266 * by independent memory controller units. Up to a 9x
267 * speed up when alpha-blending and depth-test are enabled
270 if (nvbo->tile_flags & NOUVEAU_GEM_TILE_ZETA) {
271 fpfn = vram_pages / 2;
275 lpfn = vram_pages / 2;
277 for (i = 0; i < nvbo->placement.num_placement; ++i) {
278 nvbo->placements[i].fpfn = fpfn;
279 nvbo->placements[i].lpfn = lpfn;
281 for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {
282 nvbo->busy_placements[i].fpfn = fpfn;
283 nvbo->busy_placements[i].lpfn = lpfn;
289 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy)
291 struct ttm_placement *pl = &nvbo->placement;
292 uint32_t flags = (nvbo->force_coherent ? TTM_PL_FLAG_UNCACHED :
293 TTM_PL_MASK_CACHING) |
294 (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0);
296 pl->placement = nvbo->placements;
297 set_placement_list(nvbo->placements, &pl->num_placement,
300 pl->busy_placement = nvbo->busy_placements;
301 set_placement_list(nvbo->busy_placements, &pl->num_busy_placement,
304 set_placement_range(nvbo, type);
308 nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype, bool contig)
310 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
311 struct ttm_buffer_object *bo = &nvbo->bo;
312 bool force = false, evict = false;
315 ret = ttm_bo_reserve(bo, false, false, false, NULL);
319 if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA &&
320 memtype == TTM_PL_FLAG_VRAM && contig) {
321 if (nvbo->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG) {
322 if (bo->mem.mem_type == TTM_PL_VRAM) {
323 struct nvkm_mem *mem = bo->mem.mm_node;
324 if (!list_is_singular(&mem->regions))
327 nvbo->tile_flags &= ~NOUVEAU_GEM_TILE_NONCONTIG;
332 if (nvbo->pin_refcnt) {
333 if (!(memtype & (1 << bo->mem.mem_type)) || evict) {
334 NV_ERROR(drm, "bo %p pinned elsewhere: "
335 "0x%08x vs 0x%08x\n", bo,
336 1 << bo->mem.mem_type, memtype);
344 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT, 0);
345 ret = nouveau_bo_validate(nvbo, false, false);
351 nouveau_bo_placement_set(nvbo, memtype, 0);
353 /* drop pin_refcnt temporarily, so we don't trip the assertion
354 * in nouveau_bo_move() that makes sure we're not trying to
355 * move a pinned buffer
358 ret = nouveau_bo_validate(nvbo, false, false);
363 switch (bo->mem.mem_type) {
365 drm->gem.vram_available -= bo->mem.size;
368 drm->gem.gart_available -= bo->mem.size;
376 nvbo->tile_flags |= NOUVEAU_GEM_TILE_NONCONTIG;
377 ttm_bo_unreserve(bo);
382 nouveau_bo_unpin(struct nouveau_bo *nvbo)
384 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
385 struct ttm_buffer_object *bo = &nvbo->bo;
388 ret = ttm_bo_reserve(bo, false, false, false, NULL);
392 ref = --nvbo->pin_refcnt;
393 WARN_ON_ONCE(ref < 0);
397 nouveau_bo_placement_set(nvbo, bo->mem.placement, 0);
399 ret = nouveau_bo_validate(nvbo, false, false);
401 switch (bo->mem.mem_type) {
403 drm->gem.vram_available += bo->mem.size;
406 drm->gem.gart_available += bo->mem.size;
414 ttm_bo_unreserve(bo);
419 nouveau_bo_map(struct nouveau_bo *nvbo)
423 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, NULL);
428 * TTM buffers allocated using the DMA API already have a mapping, let's
431 if (!nvbo->force_coherent)
432 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,
435 ttm_bo_unreserve(&nvbo->bo);
440 nouveau_bo_unmap(struct nouveau_bo *nvbo)
446 * TTM buffers allocated using the DMA API already had a coherent
447 * mapping which we used, no need to unmap.
449 if (!nvbo->force_coherent)
450 ttm_bo_kunmap(&nvbo->kmap);
454 nouveau_bo_sync_for_device(struct nouveau_bo *nvbo)
456 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
457 struct nvkm_device *device = nvxx_device(&drm->device);
458 struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;
464 /* Don't waste time looping if the object is coherent */
465 if (nvbo->force_coherent)
468 for (i = 0; i < ttm_dma->ttm.num_pages; i++)
469 dma_sync_single_for_device(device->dev, ttm_dma->dma_address[i],
470 PAGE_SIZE, DMA_TO_DEVICE);
474 nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo)
476 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
477 struct nvkm_device *device = nvxx_device(&drm->device);
478 struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;
484 /* Don't waste time looping if the object is coherent */
485 if (nvbo->force_coherent)
488 for (i = 0; i < ttm_dma->ttm.num_pages; i++)
489 dma_sync_single_for_cpu(device->dev, ttm_dma->dma_address[i],
490 PAGE_SIZE, DMA_FROM_DEVICE);
494 nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
499 ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement,
500 interruptible, no_wait_gpu);
504 nouveau_bo_sync_for_device(nvbo);
510 _nouveau_bo_mem_index(struct nouveau_bo *nvbo, unsigned index, void *mem, u8 sz)
512 struct ttm_dma_tt *dma_tt;
518 /* kmap'd address, return the corresponding offset */
521 /* DMA-API mapping, lookup the right address */
522 dma_tt = (struct ttm_dma_tt *)nvbo->bo.ttm;
523 m = dma_tt->cpu_address[index / PAGE_SIZE];
524 m += index % PAGE_SIZE;
529 #define nouveau_bo_mem_index(o, i, m) _nouveau_bo_mem_index(o, i, m, sizeof(*m))
532 nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val)
535 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
537 mem = nouveau_bo_mem_index(nvbo, index, mem);
540 iowrite16_native(val, (void __force __iomem *)mem);
546 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index)
549 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
551 mem = nouveau_bo_mem_index(nvbo, index, mem);
554 return ioread32_native((void __force __iomem *)mem);
560 nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
563 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
565 mem = nouveau_bo_mem_index(nvbo, index, mem);
568 iowrite32_native(val, (void __force __iomem *)mem);
573 static struct ttm_tt *
574 nouveau_ttm_tt_create(struct ttm_bo_device *bdev, unsigned long size,
575 uint32_t page_flags, struct page *dummy_read)
578 struct nouveau_drm *drm = nouveau_bdev(bdev);
579 struct drm_device *dev = drm->dev;
581 if (drm->agp.stat == ENABLED) {
582 return ttm_agp_tt_create(bdev, dev->agp->bridge, size,
583 page_flags, dummy_read);
587 return nouveau_sgdma_create_ttm(bdev, size, page_flags, dummy_read);
591 nouveau_bo_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
593 /* We'll do this from user space. */
598 nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
599 struct ttm_mem_type_manager *man)
601 struct nouveau_drm *drm = nouveau_bdev(bdev);
605 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
606 man->available_caching = TTM_PL_MASK_CACHING;
607 man->default_caching = TTM_PL_FLAG_CACHED;
610 man->flags = TTM_MEMTYPE_FLAG_FIXED |
611 TTM_MEMTYPE_FLAG_MAPPABLE;
612 man->available_caching = TTM_PL_FLAG_UNCACHED |
614 man->default_caching = TTM_PL_FLAG_WC;
616 if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
617 /* Some BARs do not support being ioremapped WC */
618 if (nvxx_bar(&drm->device)->iomap_uncached) {
619 man->available_caching = TTM_PL_FLAG_UNCACHED;
620 man->default_caching = TTM_PL_FLAG_UNCACHED;
623 man->func = &nouveau_vram_manager;
624 man->io_reserve_fastpath = false;
625 man->use_io_reserve_lru = true;
627 man->func = &ttm_bo_manager_func;
631 if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA)
632 man->func = &nouveau_gart_manager;
634 if (drm->agp.stat != ENABLED)
635 man->func = &nv04_gart_manager;
637 man->func = &ttm_bo_manager_func;
639 if (drm->agp.stat == ENABLED) {
640 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
641 man->available_caching = TTM_PL_FLAG_UNCACHED |
643 man->default_caching = TTM_PL_FLAG_WC;
645 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE |
646 TTM_MEMTYPE_FLAG_CMA;
647 man->available_caching = TTM_PL_MASK_CACHING;
648 man->default_caching = TTM_PL_FLAG_CACHED;
659 nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl)
661 struct nouveau_bo *nvbo = nouveau_bo(bo);
663 switch (bo->mem.mem_type) {
665 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT,
669 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_SYSTEM, 0);
673 *pl = nvbo->placement;
678 nve0_bo_move_init(struct nouveau_channel *chan, u32 handle)
680 int ret = RING_SPACE(chan, 2);
682 BEGIN_NVC0(chan, NvSubCopy, 0x0000, 1);
683 OUT_RING (chan, handle & 0x0000ffff);
690 nve0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
691 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
693 struct nvkm_mem *node = old_mem->mm_node;
694 int ret = RING_SPACE(chan, 10);
696 BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
697 OUT_RING (chan, upper_32_bits(node->vma[0].offset));
698 OUT_RING (chan, lower_32_bits(node->vma[0].offset));
699 OUT_RING (chan, upper_32_bits(node->vma[1].offset));
700 OUT_RING (chan, lower_32_bits(node->vma[1].offset));
701 OUT_RING (chan, PAGE_SIZE);
702 OUT_RING (chan, PAGE_SIZE);
703 OUT_RING (chan, PAGE_SIZE);
704 OUT_RING (chan, new_mem->num_pages);
705 BEGIN_IMC0(chan, NvSubCopy, 0x0300, 0x0386);
711 nvc0_bo_move_init(struct nouveau_channel *chan, u32 handle)
713 int ret = RING_SPACE(chan, 2);
715 BEGIN_NVC0(chan, NvSubCopy, 0x0000, 1);
716 OUT_RING (chan, handle);
722 nvc0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
723 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
725 struct nvkm_mem *node = old_mem->mm_node;
726 u64 src_offset = node->vma[0].offset;
727 u64 dst_offset = node->vma[1].offset;
728 u32 page_count = new_mem->num_pages;
731 page_count = new_mem->num_pages;
733 int line_count = (page_count > 8191) ? 8191 : page_count;
735 ret = RING_SPACE(chan, 11);
739 BEGIN_NVC0(chan, NvSubCopy, 0x030c, 8);
740 OUT_RING (chan, upper_32_bits(src_offset));
741 OUT_RING (chan, lower_32_bits(src_offset));
742 OUT_RING (chan, upper_32_bits(dst_offset));
743 OUT_RING (chan, lower_32_bits(dst_offset));
744 OUT_RING (chan, PAGE_SIZE);
745 OUT_RING (chan, PAGE_SIZE);
746 OUT_RING (chan, PAGE_SIZE);
747 OUT_RING (chan, line_count);
748 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
749 OUT_RING (chan, 0x00000110);
751 page_count -= line_count;
752 src_offset += (PAGE_SIZE * line_count);
753 dst_offset += (PAGE_SIZE * line_count);
760 nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
761 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
763 struct nvkm_mem *node = old_mem->mm_node;
764 u64 src_offset = node->vma[0].offset;
765 u64 dst_offset = node->vma[1].offset;
766 u32 page_count = new_mem->num_pages;
769 page_count = new_mem->num_pages;
771 int line_count = (page_count > 2047) ? 2047 : page_count;
773 ret = RING_SPACE(chan, 12);
777 BEGIN_NVC0(chan, NvSubCopy, 0x0238, 2);
778 OUT_RING (chan, upper_32_bits(dst_offset));
779 OUT_RING (chan, lower_32_bits(dst_offset));
780 BEGIN_NVC0(chan, NvSubCopy, 0x030c, 6);
781 OUT_RING (chan, upper_32_bits(src_offset));
782 OUT_RING (chan, lower_32_bits(src_offset));
783 OUT_RING (chan, PAGE_SIZE); /* src_pitch */
784 OUT_RING (chan, PAGE_SIZE); /* dst_pitch */
785 OUT_RING (chan, PAGE_SIZE); /* line_length */
786 OUT_RING (chan, line_count);
787 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
788 OUT_RING (chan, 0x00100110);
790 page_count -= line_count;
791 src_offset += (PAGE_SIZE * line_count);
792 dst_offset += (PAGE_SIZE * line_count);
799 nva3_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
800 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
802 struct nvkm_mem *node = old_mem->mm_node;
803 u64 src_offset = node->vma[0].offset;
804 u64 dst_offset = node->vma[1].offset;
805 u32 page_count = new_mem->num_pages;
808 page_count = new_mem->num_pages;
810 int line_count = (page_count > 8191) ? 8191 : page_count;
812 ret = RING_SPACE(chan, 11);
816 BEGIN_NV04(chan, NvSubCopy, 0x030c, 8);
817 OUT_RING (chan, upper_32_bits(src_offset));
818 OUT_RING (chan, lower_32_bits(src_offset));
819 OUT_RING (chan, upper_32_bits(dst_offset));
820 OUT_RING (chan, lower_32_bits(dst_offset));
821 OUT_RING (chan, PAGE_SIZE);
822 OUT_RING (chan, PAGE_SIZE);
823 OUT_RING (chan, PAGE_SIZE);
824 OUT_RING (chan, line_count);
825 BEGIN_NV04(chan, NvSubCopy, 0x0300, 1);
826 OUT_RING (chan, 0x00000110);
828 page_count -= line_count;
829 src_offset += (PAGE_SIZE * line_count);
830 dst_offset += (PAGE_SIZE * line_count);
837 nv98_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
838 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
840 struct nvkm_mem *node = old_mem->mm_node;
841 int ret = RING_SPACE(chan, 7);
843 BEGIN_NV04(chan, NvSubCopy, 0x0320, 6);
844 OUT_RING (chan, upper_32_bits(node->vma[0].offset));
845 OUT_RING (chan, lower_32_bits(node->vma[0].offset));
846 OUT_RING (chan, upper_32_bits(node->vma[1].offset));
847 OUT_RING (chan, lower_32_bits(node->vma[1].offset));
848 OUT_RING (chan, 0x00000000 /* COPY */);
849 OUT_RING (chan, new_mem->num_pages << PAGE_SHIFT);
855 nv84_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
856 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
858 struct nvkm_mem *node = old_mem->mm_node;
859 int ret = RING_SPACE(chan, 7);
861 BEGIN_NV04(chan, NvSubCopy, 0x0304, 6);
862 OUT_RING (chan, new_mem->num_pages << PAGE_SHIFT);
863 OUT_RING (chan, upper_32_bits(node->vma[0].offset));
864 OUT_RING (chan, lower_32_bits(node->vma[0].offset));
865 OUT_RING (chan, upper_32_bits(node->vma[1].offset));
866 OUT_RING (chan, lower_32_bits(node->vma[1].offset));
867 OUT_RING (chan, 0x00000000 /* MODE_COPY, QUERY_NONE */);
873 nv50_bo_move_init(struct nouveau_channel *chan, u32 handle)
875 int ret = RING_SPACE(chan, 6);
877 BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
878 OUT_RING (chan, handle);
879 BEGIN_NV04(chan, NvSubCopy, 0x0180, 3);
880 OUT_RING (chan, chan->drm->ntfy.handle);
881 OUT_RING (chan, chan->vram.handle);
882 OUT_RING (chan, chan->vram.handle);
889 nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
890 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
892 struct nvkm_mem *node = old_mem->mm_node;
893 u64 length = (new_mem->num_pages << PAGE_SHIFT);
894 u64 src_offset = node->vma[0].offset;
895 u64 dst_offset = node->vma[1].offset;
896 int src_tiled = !!node->memtype;
897 int dst_tiled = !!((struct nvkm_mem *)new_mem->mm_node)->memtype;
901 u32 amount, stride, height;
903 ret = RING_SPACE(chan, 18 + 6 * (src_tiled + dst_tiled));
907 amount = min(length, (u64)(4 * 1024 * 1024));
909 height = amount / stride;
912 BEGIN_NV04(chan, NvSubCopy, 0x0200, 7);
915 OUT_RING (chan, stride);
916 OUT_RING (chan, height);
921 BEGIN_NV04(chan, NvSubCopy, 0x0200, 1);
925 BEGIN_NV04(chan, NvSubCopy, 0x021c, 7);
928 OUT_RING (chan, stride);
929 OUT_RING (chan, height);
934 BEGIN_NV04(chan, NvSubCopy, 0x021c, 1);
938 BEGIN_NV04(chan, NvSubCopy, 0x0238, 2);
939 OUT_RING (chan, upper_32_bits(src_offset));
940 OUT_RING (chan, upper_32_bits(dst_offset));
941 BEGIN_NV04(chan, NvSubCopy, 0x030c, 8);
942 OUT_RING (chan, lower_32_bits(src_offset));
943 OUT_RING (chan, lower_32_bits(dst_offset));
944 OUT_RING (chan, stride);
945 OUT_RING (chan, stride);
946 OUT_RING (chan, stride);
947 OUT_RING (chan, height);
948 OUT_RING (chan, 0x00000101);
949 OUT_RING (chan, 0x00000000);
950 BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
954 src_offset += amount;
955 dst_offset += amount;
962 nv04_bo_move_init(struct nouveau_channel *chan, u32 handle)
964 int ret = RING_SPACE(chan, 4);
966 BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
967 OUT_RING (chan, handle);
968 BEGIN_NV04(chan, NvSubCopy, 0x0180, 1);
969 OUT_RING (chan, chan->drm->ntfy.handle);
975 static inline uint32_t
976 nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo,
977 struct nouveau_channel *chan, struct ttm_mem_reg *mem)
979 if (mem->mem_type == TTM_PL_TT)
981 return chan->vram.handle;
985 nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
986 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
988 u32 src_offset = old_mem->start << PAGE_SHIFT;
989 u32 dst_offset = new_mem->start << PAGE_SHIFT;
990 u32 page_count = new_mem->num_pages;
993 ret = RING_SPACE(chan, 3);
997 BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2);
998 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem));
999 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem));
1001 page_count = new_mem->num_pages;
1002 while (page_count) {
1003 int line_count = (page_count > 2047) ? 2047 : page_count;
1005 ret = RING_SPACE(chan, 11);
1009 BEGIN_NV04(chan, NvSubCopy,
1010 NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8);
1011 OUT_RING (chan, src_offset);
1012 OUT_RING (chan, dst_offset);
1013 OUT_RING (chan, PAGE_SIZE); /* src_pitch */
1014 OUT_RING (chan, PAGE_SIZE); /* dst_pitch */
1015 OUT_RING (chan, PAGE_SIZE); /* line_length */
1016 OUT_RING (chan, line_count);
1017 OUT_RING (chan, 0x00000101);
1018 OUT_RING (chan, 0x00000000);
1019 BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
1022 page_count -= line_count;
1023 src_offset += (PAGE_SIZE * line_count);
1024 dst_offset += (PAGE_SIZE * line_count);
1031 nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo,
1032 struct ttm_mem_reg *mem)
1034 struct nvkm_mem *old_node = bo->mem.mm_node;
1035 struct nvkm_mem *new_node = mem->mm_node;
1036 u64 size = (u64)mem->num_pages << PAGE_SHIFT;
1039 ret = nvkm_vm_get(drm->client.vm, size, old_node->page_shift,
1040 NV_MEM_ACCESS_RW, &old_node->vma[0]);
1044 ret = nvkm_vm_get(drm->client.vm, size, new_node->page_shift,
1045 NV_MEM_ACCESS_RW, &old_node->vma[1]);
1047 nvkm_vm_put(&old_node->vma[0]);
1051 nvkm_vm_map(&old_node->vma[0], old_node);
1052 nvkm_vm_map(&old_node->vma[1], new_node);
1057 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
1058 bool no_wait_gpu, struct ttm_mem_reg *new_mem)
1060 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1061 struct nouveau_channel *chan = drm->ttm.chan;
1062 struct nouveau_cli *cli = (void *)chan->user.client;
1063 struct nouveau_fence *fence;
1066 /* create temporary vmas for the transfer and attach them to the
1067 * old nvkm_mem node, these will get cleaned up after ttm has
1068 * destroyed the ttm_mem_reg
1070 if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
1071 ret = nouveau_bo_move_prep(drm, bo, new_mem);
1076 mutex_lock_nested(&cli->mutex, SINGLE_DEPTH_NESTING);
1077 ret = nouveau_fence_sync(nouveau_bo(bo), chan, true, intr);
1079 ret = drm->ttm.move(chan, bo, &bo->mem, new_mem);
1081 ret = nouveau_fence_new(chan, false, &fence);
1083 ret = ttm_bo_move_accel_cleanup(bo,
1088 nouveau_fence_unref(&fence);
1092 mutex_unlock(&cli->mutex);
1097 nouveau_bo_move_init(struct nouveau_drm *drm)
1099 static const struct {
1103 int (*exec)(struct nouveau_channel *,
1104 struct ttm_buffer_object *,
1105 struct ttm_mem_reg *, struct ttm_mem_reg *);
1106 int (*init)(struct nouveau_channel *, u32 handle);
1108 { "COPY", 4, 0xb0b5, nve0_bo_move_copy, nve0_bo_move_init },
1109 { "GRCE", 0, 0xb0b5, nve0_bo_move_copy, nvc0_bo_move_init },
1110 { "COPY", 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init },
1111 { "GRCE", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init },
1112 { "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init },
1113 { "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init },
1114 { "COPY", 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init },
1115 { "CRYPT", 0, 0x74c1, nv84_bo_move_exec, nv50_bo_move_init },
1116 { "M2MF", 0, 0x9039, nvc0_bo_move_m2mf, nvc0_bo_move_init },
1117 { "M2MF", 0, 0x5039, nv50_bo_move_m2mf, nv50_bo_move_init },
1118 { "M2MF", 0, 0x0039, nv04_bo_move_m2mf, nv04_bo_move_init },
1120 { "CRYPT", 0, 0x88b4, nv98_bo_move_exec, nv50_bo_move_init },
1121 }, *mthd = _methods;
1122 const char *name = "CPU";
1126 struct nouveau_channel *chan;
1131 chan = drm->channel;
1135 ret = nvif_object_init(&chan->user,
1136 mthd->oclass | (mthd->engine << 16),
1137 mthd->oclass, NULL, 0,
1140 ret = mthd->init(chan, drm->ttm.copy.handle);
1142 nvif_object_fini(&drm->ttm.copy);
1146 drm->ttm.move = mthd->exec;
1147 drm->ttm.chan = chan;
1151 } while ((++mthd)->exec);
1153 NV_INFO(drm, "MM: using %s for buffer copies\n", name);
1157 nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
1158 bool no_wait_gpu, struct ttm_mem_reg *new_mem)
1160 struct ttm_place placement_memtype = {
1163 .flags = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING
1165 struct ttm_placement placement;
1166 struct ttm_mem_reg tmp_mem;
1169 placement.num_placement = placement.num_busy_placement = 1;
1170 placement.placement = placement.busy_placement = &placement_memtype;
1173 tmp_mem.mm_node = NULL;
1174 ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_gpu);
1178 ret = ttm_tt_bind(bo->ttm, &tmp_mem);
1182 ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, &tmp_mem);
1186 ret = ttm_bo_move_ttm(bo, true, no_wait_gpu, new_mem);
1188 ttm_bo_mem_put(bo, &tmp_mem);
1193 nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr,
1194 bool no_wait_gpu, struct ttm_mem_reg *new_mem)
1196 struct ttm_place placement_memtype = {
1199 .flags = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING
1201 struct ttm_placement placement;
1202 struct ttm_mem_reg tmp_mem;
1205 placement.num_placement = placement.num_busy_placement = 1;
1206 placement.placement = placement.busy_placement = &placement_memtype;
1209 tmp_mem.mm_node = NULL;
1210 ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_gpu);
1214 ret = ttm_bo_move_ttm(bo, true, no_wait_gpu, &tmp_mem);
1218 ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, new_mem);
1223 ttm_bo_mem_put(bo, &tmp_mem);
1228 nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem)
1230 struct nouveau_bo *nvbo = nouveau_bo(bo);
1231 struct nvkm_vma *vma;
1233 /* ttm can now (stupidly) pass the driver bos it didn't create... */
1234 if (bo->destroy != nouveau_bo_del_ttm)
1237 list_for_each_entry(vma, &nvbo->vma_list, head) {
1238 if (new_mem && new_mem->mem_type != TTM_PL_SYSTEM &&
1239 (new_mem->mem_type == TTM_PL_VRAM ||
1240 nvbo->page_shift != vma->vm->mmu->lpg_shift)) {
1241 nvkm_vm_map(vma, new_mem->mm_node);
1249 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem,
1250 struct nouveau_drm_tile **new_tile)
1252 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1253 struct drm_device *dev = drm->dev;
1254 struct nouveau_bo *nvbo = nouveau_bo(bo);
1255 u64 offset = new_mem->start << PAGE_SHIFT;
1258 if (new_mem->mem_type != TTM_PL_VRAM)
1261 if (drm->device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) {
1262 *new_tile = nv10_bo_set_tiling(dev, offset, new_mem->size,
1271 nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
1272 struct nouveau_drm_tile *new_tile,
1273 struct nouveau_drm_tile **old_tile)
1275 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1276 struct drm_device *dev = drm->dev;
1277 struct fence *fence = reservation_object_get_excl(bo->resv);
1279 nv10_bo_put_tile_region(dev, *old_tile, fence);
1280 *old_tile = new_tile;
1284 nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
1285 bool no_wait_gpu, struct ttm_mem_reg *new_mem)
1287 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1288 struct nouveau_bo *nvbo = nouveau_bo(bo);
1289 struct ttm_mem_reg *old_mem = &bo->mem;
1290 struct nouveau_drm_tile *new_tile = NULL;
1293 if (nvbo->pin_refcnt)
1294 NV_WARN(drm, "Moving pinned object %p!\n", nvbo);
1296 if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1297 ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile);
1303 if (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm) {
1304 BUG_ON(bo->mem.mm_node != NULL);
1306 new_mem->mm_node = NULL;
1310 /* Hardware assisted copy. */
1311 if (drm->ttm.move) {
1312 if (new_mem->mem_type == TTM_PL_SYSTEM)
1313 ret = nouveau_bo_move_flipd(bo, evict, intr,
1314 no_wait_gpu, new_mem);
1315 else if (old_mem->mem_type == TTM_PL_SYSTEM)
1316 ret = nouveau_bo_move_flips(bo, evict, intr,
1317 no_wait_gpu, new_mem);
1319 ret = nouveau_bo_move_m2mf(bo, evict, intr,
1320 no_wait_gpu, new_mem);
1325 /* Fallback to software copy. */
1326 ret = ttm_bo_wait(bo, true, intr, no_wait_gpu);
1328 ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem);
1331 if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1333 nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
1335 nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);
1342 nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp)
1344 struct nouveau_bo *nvbo = nouveau_bo(bo);
1346 return drm_vma_node_verify_access(&nvbo->gem.vma_node, filp);
1350 nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1352 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1353 struct nouveau_drm *drm = nouveau_bdev(bdev);
1354 struct nvkm_device *device = nvxx_device(&drm->device);
1355 struct nvkm_mem *node = mem->mm_node;
1358 mem->bus.addr = NULL;
1359 mem->bus.offset = 0;
1360 mem->bus.size = mem->num_pages << PAGE_SHIFT;
1362 mem->bus.is_iomem = false;
1363 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
1365 switch (mem->mem_type) {
1371 if (drm->agp.stat == ENABLED) {
1372 mem->bus.offset = mem->start << PAGE_SHIFT;
1373 mem->bus.base = drm->agp.base;
1374 mem->bus.is_iomem = !drm->dev->agp->cant_use_aperture;
1377 if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA || !node->memtype)
1380 /* fallthrough, tiled memory */
1382 mem->bus.offset = mem->start << PAGE_SHIFT;
1383 mem->bus.base = device->func->resource_addr(device, 1);
1384 mem->bus.is_iomem = true;
1385 if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
1386 struct nvkm_bar *bar = nvxx_bar(&drm->device);
1387 int page_shift = 12;
1388 if (drm->device.info.family >= NV_DEVICE_INFO_V0_FERMI)
1389 page_shift = node->page_shift;
1391 ret = nvkm_bar_umap(bar, node->size << 12, page_shift,
1396 nvkm_vm_map(&node->bar_vma, node);
1397 mem->bus.offset = node->bar_vma.offset;
1407 nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1409 struct nvkm_mem *node = mem->mm_node;
1411 if (!node->bar_vma.node)
1414 nvkm_vm_unmap(&node->bar_vma);
1415 nvkm_vm_put(&node->bar_vma);
1419 nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
1421 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1422 struct nouveau_bo *nvbo = nouveau_bo(bo);
1423 struct nvkm_device *device = nvxx_device(&drm->device);
1424 u32 mappable = device->func->resource_size(device, 1) >> PAGE_SHIFT;
1427 /* as long as the bo isn't in vram, and isn't tiled, we've got
1428 * nothing to do here.
1430 if (bo->mem.mem_type != TTM_PL_VRAM) {
1431 if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA ||
1432 !nouveau_bo_tile_layout(nvbo))
1435 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
1436 nouveau_bo_placement_set(nvbo, TTM_PL_TT, 0);
1438 ret = nouveau_bo_validate(nvbo, false, false);
1445 /* make sure bo is in mappable vram */
1446 if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA ||
1447 bo->mem.start + bo->mem.num_pages < mappable)
1450 for (i = 0; i < nvbo->placement.num_placement; ++i) {
1451 nvbo->placements[i].fpfn = 0;
1452 nvbo->placements[i].lpfn = mappable;
1455 for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {
1456 nvbo->busy_placements[i].fpfn = 0;
1457 nvbo->busy_placements[i].lpfn = mappable;
1460 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_VRAM, 0);
1461 return nouveau_bo_validate(nvbo, false, false);
1465 nouveau_ttm_tt_populate(struct ttm_tt *ttm)
1467 struct ttm_dma_tt *ttm_dma = (void *)ttm;
1468 struct nouveau_drm *drm;
1469 struct nvkm_device *device;
1470 struct drm_device *dev;
1471 struct device *pdev;
1474 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1476 if (ttm->state != tt_unpopulated)
1479 if (slave && ttm->sg) {
1480 /* make userspace faulting work */
1481 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
1482 ttm_dma->dma_address, ttm->num_pages);
1483 ttm->state = tt_unbound;
1487 drm = nouveau_bdev(ttm->bdev);
1488 device = nvxx_device(&drm->device);
1493 * Objects matching this condition have been marked as force_coherent,
1494 * so use the DMA API for them.
1496 if (!nvxx_device(&drm->device)->func->cpu_coherent &&
1497 ttm->caching_state == tt_uncached)
1498 return ttm_dma_populate(ttm_dma, dev->dev);
1501 if (drm->agp.stat == ENABLED) {
1502 return ttm_agp_tt_populate(ttm);
1506 #ifdef CONFIG_SWIOTLB
1507 if (swiotlb_nr_tbl()) {
1508 return ttm_dma_populate((void *)ttm, dev->dev);
1512 r = ttm_pool_populate(ttm);
1517 for (i = 0; i < ttm->num_pages; i++) {
1520 addr = dma_map_page(pdev, ttm->pages[i], 0, PAGE_SIZE,
1523 if (dma_mapping_error(pdev, addr)) {
1525 dma_unmap_page(pdev, ttm_dma->dma_address[i],
1526 PAGE_SIZE, DMA_BIDIRECTIONAL);
1527 ttm_dma->dma_address[i] = 0;
1529 ttm_pool_unpopulate(ttm);
1533 ttm_dma->dma_address[i] = addr;
1539 nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
1541 struct ttm_dma_tt *ttm_dma = (void *)ttm;
1542 struct nouveau_drm *drm;
1543 struct nvkm_device *device;
1544 struct drm_device *dev;
1545 struct device *pdev;
1547 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1552 drm = nouveau_bdev(ttm->bdev);
1553 device = nvxx_device(&drm->device);
1558 * Objects matching this condition have been marked as force_coherent,
1559 * so use the DMA API for them.
1561 if (!nvxx_device(&drm->device)->func->cpu_coherent &&
1562 ttm->caching_state == tt_uncached) {
1563 ttm_dma_unpopulate(ttm_dma, dev->dev);
1568 if (drm->agp.stat == ENABLED) {
1569 ttm_agp_tt_unpopulate(ttm);
1574 #ifdef CONFIG_SWIOTLB
1575 if (swiotlb_nr_tbl()) {
1576 ttm_dma_unpopulate((void *)ttm, dev->dev);
1581 for (i = 0; i < ttm->num_pages; i++) {
1582 if (ttm_dma->dma_address[i]) {
1583 dma_unmap_page(pdev, ttm_dma->dma_address[i], PAGE_SIZE,
1588 ttm_pool_unpopulate(ttm);
1592 nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool exclusive)
1594 struct reservation_object *resv = nvbo->bo.resv;
1597 reservation_object_add_excl_fence(resv, &fence->base);
1599 reservation_object_add_shared_fence(resv, &fence->base);
1602 struct ttm_bo_driver nouveau_bo_driver = {
1603 .ttm_tt_create = &nouveau_ttm_tt_create,
1604 .ttm_tt_populate = &nouveau_ttm_tt_populate,
1605 .ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate,
1606 .invalidate_caches = nouveau_bo_invalidate_caches,
1607 .init_mem_type = nouveau_bo_init_mem_type,
1608 .evict_flags = nouveau_bo_evict_flags,
1609 .move_notify = nouveau_bo_move_ntfy,
1610 .move = nouveau_bo_move,
1611 .verify_access = nouveau_bo_verify_access,
1612 .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
1613 .io_mem_reserve = &nouveau_ttm_io_mem_reserve,
1614 .io_mem_free = &nouveau_ttm_io_mem_free,
1618 nouveau_bo_vma_find(struct nouveau_bo *nvbo, struct nvkm_vm *vm)
1620 struct nvkm_vma *vma;
1621 list_for_each_entry(vma, &nvbo->vma_list, head) {
1630 nouveau_bo_vma_add(struct nouveau_bo *nvbo, struct nvkm_vm *vm,
1631 struct nvkm_vma *vma)
1633 const u32 size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
1636 ret = nvkm_vm_get(vm, size, nvbo->page_shift,
1637 NV_MEM_ACCESS_RW, vma);
1641 if ( nvbo->bo.mem.mem_type != TTM_PL_SYSTEM &&
1642 (nvbo->bo.mem.mem_type == TTM_PL_VRAM ||
1643 nvbo->page_shift != vma->vm->mmu->lpg_shift))
1644 nvkm_vm_map(vma, nvbo->bo.mem.mm_node);
1646 list_add_tail(&vma->head, &nvbo->vma_list);
1652 nouveau_bo_vma_del(struct nouveau_bo *nvbo, struct nvkm_vma *vma)
1655 if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM)
1658 list_del(&vma->head);