1 /**************************************************************************
3 * Copyright © 2012 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 #include "vmwgfx_drv.h"
31 * If we set up the screen target otable, screen objects stop working.
34 #define VMW_OTABLE_SETUP_SUB ((VMWGFX_ENABLE_SCREEN_TARGET_OTABLE) ? 0 : 1)
37 * Currently the MOB interface does not support 64-bit page frame numbers.
38 * This might change in the future to be similar to the GMR2 interface
39 * when virtual machines support memory beyond 16TB.
42 #define VMW_PPN_SIZE 4
45 * struct vmw_mob - Structure containing page table and metadata for a
46 * Guest Memory OBject.
48 * @num_pages Number of pages that make up the page table.
49 * @pt_level The indirection level of the page table. 0-2.
50 * @pt_root_page DMA address of the level 0 page of the page table.
53 struct ttm_buffer_object *pt_bo;
54 unsigned long num_pages;
56 dma_addr_t pt_root_page;
61 * struct vmw_otable - Guest Memory OBject table metadata
63 * @size: Size of the table (page-aligned).
64 * @page_table: Pointer to a struct vmw_mob holding the page table.
68 struct vmw_mob *page_table;
71 static int vmw_mob_pt_populate(struct vmw_private *dev_priv,
73 static void vmw_mob_pt_setup(struct vmw_mob *mob,
74 struct vmw_piter data_iter,
75 unsigned long num_data_pages);
78 * vmw_setup_otable_base - Issue an object table base setup command to
81 * @dev_priv: Pointer to a device private structure
82 * @type: Type of object table base
83 * @offset Start of table offset into dev_priv::otable_bo
84 * @otable Pointer to otable metadata;
86 * This function returns -ENOMEM if it fails to reserve fifo space,
87 * and may block waiting for fifo space.
89 static int vmw_setup_otable_base(struct vmw_private *dev_priv,
92 struct vmw_otable *otable)
95 SVGA3dCmdHeader header;
96 SVGA3dCmdSetOTableBase body;
99 const struct vmw_sg_table *vsgt;
100 struct vmw_piter iter;
103 BUG_ON(otable->page_table != NULL);
105 vsgt = vmw_bo_sg_table(dev_priv->otable_bo);
106 vmw_piter_start(&iter, vsgt, offset >> PAGE_SHIFT);
107 WARN_ON(!vmw_piter_next(&iter));
109 mob = vmw_mob_create(otable->size >> PAGE_SHIFT);
110 if (unlikely(mob == NULL)) {
111 DRM_ERROR("Failed creating OTable page table.\n");
115 if (otable->size <= PAGE_SIZE) {
116 mob->pt_level = SVGA3D_MOBFMT_PTDEPTH_0;
117 mob->pt_root_page = vmw_piter_dma_addr(&iter);
118 } else if (vsgt->num_regions == 1) {
119 mob->pt_level = SVGA3D_MOBFMT_RANGE;
120 mob->pt_root_page = vmw_piter_dma_addr(&iter);
122 ret = vmw_mob_pt_populate(dev_priv, mob);
123 if (unlikely(ret != 0))
124 goto out_no_populate;
126 vmw_mob_pt_setup(mob, iter, otable->size >> PAGE_SHIFT);
129 cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
130 if (unlikely(cmd == NULL)) {
131 DRM_ERROR("Failed reserving FIFO space for OTable setup.\n");
135 memset(cmd, 0, sizeof(*cmd));
136 cmd->header.id = SVGA_3D_CMD_SET_OTABLE_BASE;
137 cmd->header.size = sizeof(cmd->body);
138 cmd->body.type = type;
139 cmd->body.baseAddress = mob->pt_root_page >> PAGE_SHIFT;
140 cmd->body.sizeInBytes = otable->size;
141 cmd->body.validSizeInBytes = 0;
142 cmd->body.ptDepth = mob->pt_level;
144 vmw_fifo_commit(dev_priv, sizeof(*cmd));
145 otable->page_table = mob;
151 vmw_mob_destroy(mob);
156 * vmw_takedown_otable_base - Issue an object table base takedown command
159 * @dev_priv: Pointer to a device private structure
160 * @type: Type of object table base
163 static void vmw_takedown_otable_base(struct vmw_private *dev_priv,
165 struct vmw_otable *otable)
168 SVGA3dCmdHeader header;
169 SVGA3dCmdSetOTableBase body;
171 struct ttm_buffer_object *bo = otable->page_table->pt_bo;
173 if (otable->page_table == NULL)
176 cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
177 if (unlikely(cmd == NULL))
178 DRM_ERROR("Failed reserving FIFO space for OTable setup.\n");
180 memset(cmd, 0, sizeof(*cmd));
181 cmd->header.id = SVGA_3D_CMD_SET_OTABLE_BASE;
182 cmd->header.size = sizeof(cmd->body);
183 cmd->body.type = type;
184 cmd->body.baseAddress = 0;
185 cmd->body.sizeInBytes = 0;
186 cmd->body.validSizeInBytes = 0;
187 cmd->body.ptDepth = SVGA3D_MOBFMT_INVALID;
188 vmw_fifo_commit(dev_priv, sizeof(*cmd));
193 ret = ttm_bo_reserve(bo, false, true, false, false);
196 vmw_fence_single_bo(bo, NULL);
197 ttm_bo_unreserve(bo);
200 vmw_mob_destroy(otable->page_table);
201 otable->page_table = NULL;
205 * vmw_otables_setup - Set up guest backed memory object tables
207 * @dev_priv: Pointer to a device private structure
209 * Takes care of the device guest backed surface
210 * initialization, by setting up the guest backed memory object tables.
211 * Returns 0 on success and various error codes on failure. A succesful return
212 * means the object tables can be taken down using the vmw_otables_takedown
215 int vmw_otables_setup(struct vmw_private *dev_priv)
217 unsigned long offset;
218 unsigned long bo_size;
219 struct vmw_otable *otables;
223 otables = kzalloc(SVGA_OTABLE_DX9_MAX * sizeof(*otables),
225 if (unlikely(otables == NULL)) {
226 DRM_ERROR("Failed to allocate space for otable "
231 otables[SVGA_OTABLE_MOB].size =
232 VMWGFX_NUM_MOB * SVGA3D_OTABLE_MOB_ENTRY_SIZE;
233 otables[SVGA_OTABLE_SURFACE].size =
234 VMWGFX_NUM_GB_SURFACE * SVGA3D_OTABLE_SURFACE_ENTRY_SIZE;
235 otables[SVGA_OTABLE_CONTEXT].size =
236 VMWGFX_NUM_GB_CONTEXT * SVGA3D_OTABLE_CONTEXT_ENTRY_SIZE;
237 otables[SVGA_OTABLE_SHADER].size =
238 VMWGFX_NUM_GB_SHADER * SVGA3D_OTABLE_SHADER_ENTRY_SIZE;
239 otables[SVGA_OTABLE_SCREEN_TARGET].size =
240 VMWGFX_NUM_GB_SCREEN_TARGET *
241 SVGA3D_OTABLE_SCREEN_TARGET_ENTRY_SIZE;
244 for (i = 0; i < SVGA_OTABLE_DX9_MAX; ++i) {
246 (otables[i].size + PAGE_SIZE - 1) & PAGE_MASK;
247 bo_size += otables[i].size;
250 ret = ttm_bo_create(&dev_priv->bdev, bo_size,
252 &vmw_sys_ne_placement,
254 &dev_priv->otable_bo);
256 if (unlikely(ret != 0))
259 ret = ttm_bo_reserve(dev_priv->otable_bo, false, true, false, false);
261 ret = vmw_bo_driver.ttm_tt_populate(dev_priv->otable_bo->ttm);
262 if (unlikely(ret != 0))
264 ret = vmw_bo_map_dma(dev_priv->otable_bo);
265 if (unlikely(ret != 0))
268 ttm_bo_unreserve(dev_priv->otable_bo);
271 for (i = 0; i < SVGA_OTABLE_DX9_MAX - VMW_OTABLE_SETUP_SUB; ++i) {
272 ret = vmw_setup_otable_base(dev_priv, i, offset,
274 if (unlikely(ret != 0))
276 offset += otables[i].size;
279 dev_priv->otables = otables;
283 ttm_bo_unreserve(dev_priv->otable_bo);
285 for (i = 0; i < SVGA_OTABLE_DX9_MAX - VMW_OTABLE_SETUP_SUB; ++i)
286 vmw_takedown_otable_base(dev_priv, i, &otables[i]);
288 ttm_bo_unref(&dev_priv->otable_bo);
296 * vmw_otables_takedown - Take down guest backed memory object tables
298 * @dev_priv: Pointer to a device private structure
300 * Take down the Guest Memory Object tables.
302 void vmw_otables_takedown(struct vmw_private *dev_priv)
305 struct ttm_buffer_object *bo = dev_priv->otable_bo;
308 for (i = 0; i < SVGA_OTABLE_DX9_MAX - VMW_OTABLE_SETUP_SUB; ++i)
309 vmw_takedown_otable_base(dev_priv, i,
310 &dev_priv->otables[i]);
312 ret = ttm_bo_reserve(bo, false, true, false, false);
315 vmw_fence_single_bo(bo, NULL);
316 ttm_bo_unreserve(bo);
318 ttm_bo_unref(&dev_priv->otable_bo);
319 kfree(dev_priv->otables);
320 dev_priv->otables = NULL;
325 * vmw_mob_calculate_pt_pages - Calculate the number of page table pages
326 * needed for a guest backed memory object.
328 * @data_pages: Number of data pages in the memory object buffer.
330 static unsigned long vmw_mob_calculate_pt_pages(unsigned long data_pages)
332 unsigned long data_size = data_pages * PAGE_SIZE;
333 unsigned long tot_size = 0;
335 while (likely(data_size > PAGE_SIZE)) {
336 data_size = DIV_ROUND_UP(data_size, PAGE_SIZE);
337 data_size *= VMW_PPN_SIZE;
338 tot_size += (data_size + PAGE_SIZE - 1) & PAGE_MASK;
341 return tot_size >> PAGE_SHIFT;
345 * vmw_mob_create - Create a mob, but don't populate it.
347 * @data_pages: Number of data pages of the underlying buffer object.
349 struct vmw_mob *vmw_mob_create(unsigned long data_pages)
351 struct vmw_mob *mob = kzalloc(sizeof(*mob), GFP_KERNEL);
353 if (unlikely(mob == NULL))
356 mob->num_pages = vmw_mob_calculate_pt_pages(data_pages);
362 * vmw_mob_pt_populate - Populate the mob pagetable
364 * @mob: Pointer to the mob the pagetable of which we want to
367 * This function allocates memory to be used for the pagetable, and
368 * adjusts TTM memory accounting accordingly. Returns ENOMEM if
369 * memory resources aren't sufficient and may cause TTM buffer objects
370 * to be swapped out by using the TTM memory accounting function.
372 static int vmw_mob_pt_populate(struct vmw_private *dev_priv,
376 BUG_ON(mob->pt_bo != NULL);
378 ret = ttm_bo_create(&dev_priv->bdev, mob->num_pages * PAGE_SIZE,
380 &vmw_sys_ne_placement,
381 0, false, NULL, &mob->pt_bo);
382 if (unlikely(ret != 0))
385 ret = ttm_bo_reserve(mob->pt_bo, false, true, false, false);
388 ret = vmw_bo_driver.ttm_tt_populate(mob->pt_bo->ttm);
389 if (unlikely(ret != 0))
391 ret = vmw_bo_map_dma(mob->pt_bo);
392 if (unlikely(ret != 0))
395 ttm_bo_unreserve(mob->pt_bo);
400 ttm_bo_unreserve(mob->pt_bo);
401 ttm_bo_unref(&mob->pt_bo);
408 * vmw_mob_build_pt - Build a pagetable
410 * @data_addr: Array of DMA addresses to the underlying buffer
411 * object's data pages.
412 * @num_data_pages: Number of buffer object data pages.
413 * @pt_pages: Array of page pointers to the page table pages.
415 * Returns the number of page table pages actually used.
416 * Uses atomic kmaps of highmem pages to avoid TLB thrashing.
418 static unsigned long vmw_mob_build_pt(struct vmw_piter *data_iter,
419 unsigned long num_data_pages,
420 struct vmw_piter *pt_iter)
422 unsigned long pt_size = num_data_pages * VMW_PPN_SIZE;
423 unsigned long num_pt_pages = DIV_ROUND_UP(pt_size, PAGE_SIZE);
424 unsigned long pt_page;
425 uint32_t *addr, *save_addr;
429 for (pt_page = 0; pt_page < num_pt_pages; ++pt_page) {
430 page = vmw_piter_page(pt_iter);
432 save_addr = addr = kmap_atomic(page);
434 for (i = 0; i < PAGE_SIZE / VMW_PPN_SIZE; ++i) {
435 u32 tmp = vmw_piter_dma_addr(data_iter) >> PAGE_SHIFT;
437 if (unlikely(--num_data_pages == 0))
439 WARN_ON(!vmw_piter_next(data_iter));
441 kunmap_atomic(save_addr);
442 vmw_piter_next(pt_iter);
449 * vmw_mob_build_pt - Set up a multilevel mob pagetable
451 * @mob: Pointer to a mob whose page table needs setting up.
452 * @data_addr Array of DMA addresses to the buffer object's data
454 * @num_data_pages: Number of buffer object data pages.
456 * Uses tail recursion to set up a multilevel mob page table.
458 static void vmw_mob_pt_setup(struct vmw_mob *mob,
459 struct vmw_piter data_iter,
460 unsigned long num_data_pages)
462 unsigned long num_pt_pages = 0;
463 struct ttm_buffer_object *bo = mob->pt_bo;
464 struct vmw_piter save_pt_iter;
465 struct vmw_piter pt_iter;
466 const struct vmw_sg_table *vsgt;
469 ret = ttm_bo_reserve(bo, false, true, false, 0);
472 vsgt = vmw_bo_sg_table(bo);
473 vmw_piter_start(&pt_iter, vsgt, 0);
474 BUG_ON(!vmw_piter_next(&pt_iter));
476 while (likely(num_data_pages > 1)) {
478 BUG_ON(mob->pt_level > 2);
479 save_pt_iter = pt_iter;
480 num_pt_pages = vmw_mob_build_pt(&data_iter, num_data_pages,
482 data_iter = save_pt_iter;
483 num_data_pages = num_pt_pages;
486 mob->pt_root_page = vmw_piter_dma_addr(&save_pt_iter);
487 ttm_bo_unreserve(bo);
491 * vmw_mob_destroy - Destroy a mob, unpopulating first if necessary.
493 * @mob: Pointer to a mob to destroy.
495 void vmw_mob_destroy(struct vmw_mob *mob)
498 ttm_bo_unref(&mob->pt_bo);
503 * vmw_mob_unbind - Hide a mob from the device.
505 * @dev_priv: Pointer to a device private.
506 * @mob_id: Device id of the mob to unbind.
508 void vmw_mob_unbind(struct vmw_private *dev_priv,
512 SVGA3dCmdHeader header;
513 SVGA3dCmdDestroyGBMob body;
516 struct ttm_buffer_object *bo = mob->pt_bo;
519 ret = ttm_bo_reserve(bo, false, true, false, 0);
521 * Noone else should be using this buffer.
526 cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
527 if (unlikely(cmd == NULL)) {
528 DRM_ERROR("Failed reserving FIFO space for Memory "
529 "Object unbinding.\n");
531 cmd->header.id = SVGA_3D_CMD_DESTROY_GB_MOB;
532 cmd->header.size = sizeof(cmd->body);
533 cmd->body.mobid = mob->id;
534 vmw_fifo_commit(dev_priv, sizeof(*cmd));
536 vmw_fence_single_bo(bo, NULL);
537 ttm_bo_unreserve(bo);
539 vmw_3d_resource_dec(dev_priv, false);
543 * vmw_mob_bind - Make a mob visible to the device after first
544 * populating it if necessary.
546 * @dev_priv: Pointer to a device private.
547 * @mob: Pointer to the mob we're making visible.
548 * @data_addr: Array of DMA addresses to the data pages of the underlying
550 * @num_data_pages: Number of data pages of the underlying buffer
552 * @mob_id: Device id of the mob to bind
554 * This function is intended to be interfaced with the ttm_tt backend
557 int vmw_mob_bind(struct vmw_private *dev_priv,
559 const struct vmw_sg_table *vsgt,
560 unsigned long num_data_pages,
564 bool pt_set_up = false;
565 struct vmw_piter data_iter;
567 SVGA3dCmdHeader header;
568 SVGA3dCmdDefineGBMob body;
572 vmw_piter_start(&data_iter, vsgt, 0);
573 if (unlikely(!vmw_piter_next(&data_iter)))
576 if (likely(num_data_pages == 1)) {
577 mob->pt_level = SVGA3D_MOBFMT_PTDEPTH_0;
578 mob->pt_root_page = vmw_piter_dma_addr(&data_iter);
579 } else if (vsgt->num_regions == 1) {
580 mob->pt_level = SVGA3D_MOBFMT_RANGE;
581 mob->pt_root_page = vmw_piter_dma_addr(&data_iter);
582 } else if (unlikely(mob->pt_bo == NULL)) {
583 ret = vmw_mob_pt_populate(dev_priv, mob);
584 if (unlikely(ret != 0))
587 vmw_mob_pt_setup(mob, data_iter, num_data_pages);
591 (void) vmw_3d_resource_inc(dev_priv, false);
593 cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
594 if (unlikely(cmd == NULL)) {
595 DRM_ERROR("Failed reserving FIFO space for Memory "
596 "Object binding.\n");
597 goto out_no_cmd_space;
600 cmd->header.id = SVGA_3D_CMD_DEFINE_GB_MOB;
601 cmd->header.size = sizeof(cmd->body);
602 cmd->body.mobid = mob_id;
603 cmd->body.ptDepth = mob->pt_level;
604 cmd->body.base = mob->pt_root_page >> PAGE_SHIFT;
605 cmd->body.sizeInBytes = num_data_pages * PAGE_SIZE;
607 vmw_fifo_commit(dev_priv, sizeof(*cmd));
612 vmw_3d_resource_dec(dev_priv, false);
614 ttm_bo_unref(&mob->pt_bo);