1 /**************************************************************************
3 * Copyright © 2012 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 #include "vmwgfx_drv.h"
31 * If we set up the screen target otable, screen objects stop working.
34 #define VMW_OTABLE_SETUP_SUB ((VMWGFX_ENABLE_SCREEN_TARGET_OTABLE) ? 0 : 1)
37 #define VMW_PPN_SIZE 8
38 #define VMW_MOBFMT_PTDEPTH_0 SVGA3D_MOBFMT_PTDEPTH64_0
39 #define VMW_MOBFMT_PTDEPTH_1 SVGA3D_MOBFMT_PTDEPTH64_1
40 #define VMW_MOBFMT_PTDEPTH_2 SVGA3D_MOBFMT_PTDEPTH64_2
42 #define VMW_PPN_SIZE 4
43 #define VMW_MOBFMT_PTDEPTH_0 SVGA3D_MOBFMT_PTDEPTH_0
44 #define VMW_MOBFMT_PTDEPTH_1 SVGA3D_MOBFMT_PTDEPTH_1
45 #define VMW_MOBFMT_PTDEPTH_2 SVGA3D_MOBFMT_PTDEPTH_2
49 * struct vmw_mob - Structure containing page table and metadata for a
50 * Guest Memory OBject.
52 * @num_pages Number of pages that make up the page table.
53 * @pt_level The indirection level of the page table. 0-2.
54 * @pt_root_page DMA address of the level 0 page of the page table.
57 struct ttm_buffer_object
*pt_bo
;
58 unsigned long num_pages
;
60 dma_addr_t pt_root_page
;
65 * struct vmw_otable - Guest Memory OBject table metadata
67 * @size: Size of the table (page-aligned).
68 * @page_table: Pointer to a struct vmw_mob holding the page table.
72 struct vmw_mob
*page_table
;
75 static int vmw_mob_pt_populate(struct vmw_private
*dev_priv
,
77 static void vmw_mob_pt_setup(struct vmw_mob
*mob
,
78 struct vmw_piter data_iter
,
79 unsigned long num_data_pages
);
82 * vmw_setup_otable_base - Issue an object table base setup command to
85 * @dev_priv: Pointer to a device private structure
86 * @type: Type of object table base
87 * @offset Start of table offset into dev_priv::otable_bo
88 * @otable Pointer to otable metadata;
90 * This function returns -ENOMEM if it fails to reserve fifo space,
91 * and may block waiting for fifo space.
93 static int vmw_setup_otable_base(struct vmw_private
*dev_priv
,
96 struct vmw_otable
*otable
)
99 SVGA3dCmdHeader header
;
100 SVGA3dCmdSetOTableBase64 body
;
103 const struct vmw_sg_table
*vsgt
;
104 struct vmw_piter iter
;
107 BUG_ON(otable
->page_table
!= NULL
);
109 vsgt
= vmw_bo_sg_table(dev_priv
->otable_bo
);
110 vmw_piter_start(&iter
, vsgt
, offset
>> PAGE_SHIFT
);
111 WARN_ON(!vmw_piter_next(&iter
));
113 mob
= vmw_mob_create(otable
->size
>> PAGE_SHIFT
);
114 if (unlikely(mob
== NULL
)) {
115 DRM_ERROR("Failed creating OTable page table.\n");
119 if (otable
->size
<= PAGE_SIZE
) {
120 mob
->pt_level
= VMW_MOBFMT_PTDEPTH_0
;
121 mob
->pt_root_page
= vmw_piter_dma_addr(&iter
);
122 } else if (vsgt
->num_regions
== 1) {
123 mob
->pt_level
= SVGA3D_MOBFMT_RANGE
;
124 mob
->pt_root_page
= vmw_piter_dma_addr(&iter
);
126 ret
= vmw_mob_pt_populate(dev_priv
, mob
);
127 if (unlikely(ret
!= 0))
128 goto out_no_populate
;
130 vmw_mob_pt_setup(mob
, iter
, otable
->size
>> PAGE_SHIFT
);
131 mob
->pt_level
+= VMW_MOBFMT_PTDEPTH_1
- SVGA3D_MOBFMT_PTDEPTH_1
;
134 cmd
= vmw_fifo_reserve(dev_priv
, sizeof(*cmd
));
135 if (unlikely(cmd
== NULL
)) {
136 DRM_ERROR("Failed reserving FIFO space for OTable setup.\n");
141 memset(cmd
, 0, sizeof(*cmd
));
142 cmd
->header
.id
= SVGA_3D_CMD_SET_OTABLE_BASE64
;
143 cmd
->header
.size
= sizeof(cmd
->body
);
144 cmd
->body
.type
= type
;
145 cmd
->body
.baseAddress
= cpu_to_le64(mob
->pt_root_page
>> PAGE_SHIFT
);
146 cmd
->body
.sizeInBytes
= otable
->size
;
147 cmd
->body
.validSizeInBytes
= 0;
148 cmd
->body
.ptDepth
= mob
->pt_level
;
151 * The device doesn't support this, But the otable size is
152 * determined at compile-time, so this BUG shouldn't trigger
155 BUG_ON(mob
->pt_level
== VMW_MOBFMT_PTDEPTH_2
);
157 vmw_fifo_commit(dev_priv
, sizeof(*cmd
));
158 otable
->page_table
= mob
;
164 vmw_mob_destroy(mob
);
169 * vmw_takedown_otable_base - Issue an object table base takedown command
172 * @dev_priv: Pointer to a device private structure
173 * @type: Type of object table base
176 static void vmw_takedown_otable_base(struct vmw_private
*dev_priv
,
178 struct vmw_otable
*otable
)
181 SVGA3dCmdHeader header
;
182 SVGA3dCmdSetOTableBase body
;
184 struct ttm_buffer_object
*bo
;
186 if (otable
->page_table
== NULL
)
189 bo
= otable
->page_table
->pt_bo
;
190 cmd
= vmw_fifo_reserve(dev_priv
, sizeof(*cmd
));
191 if (unlikely(cmd
== NULL
)) {
192 DRM_ERROR("Failed reserving FIFO space for OTable "
195 memset(cmd
, 0, sizeof(*cmd
));
196 cmd
->header
.id
= SVGA_3D_CMD_SET_OTABLE_BASE
;
197 cmd
->header
.size
= sizeof(cmd
->body
);
198 cmd
->body
.type
= type
;
199 cmd
->body
.baseAddress
= 0;
200 cmd
->body
.sizeInBytes
= 0;
201 cmd
->body
.validSizeInBytes
= 0;
202 cmd
->body
.ptDepth
= SVGA3D_MOBFMT_INVALID
;
203 vmw_fifo_commit(dev_priv
, sizeof(*cmd
));
209 ret
= ttm_bo_reserve(bo
, false, true, false, NULL
);
212 vmw_fence_single_bo(bo
, NULL
);
213 ttm_bo_unreserve(bo
);
216 vmw_mob_destroy(otable
->page_table
);
217 otable
->page_table
= NULL
;
221 * vmw_otables_setup - Set up guest backed memory object tables
223 * @dev_priv: Pointer to a device private structure
225 * Takes care of the device guest backed surface
226 * initialization, by setting up the guest backed memory object tables.
227 * Returns 0 on success and various error codes on failure. A succesful return
228 * means the object tables can be taken down using the vmw_otables_takedown
231 int vmw_otables_setup(struct vmw_private
*dev_priv
)
233 unsigned long offset
;
234 unsigned long bo_size
;
235 struct vmw_otable
*otables
;
239 otables
= kzalloc(SVGA_OTABLE_DX9_MAX
* sizeof(*otables
),
241 if (unlikely(otables
== NULL
)) {
242 DRM_ERROR("Failed to allocate space for otable "
247 otables
[SVGA_OTABLE_MOB
].size
=
248 VMWGFX_NUM_MOB
* SVGA3D_OTABLE_MOB_ENTRY_SIZE
;
249 otables
[SVGA_OTABLE_SURFACE
].size
=
250 VMWGFX_NUM_GB_SURFACE
* SVGA3D_OTABLE_SURFACE_ENTRY_SIZE
;
251 otables
[SVGA_OTABLE_CONTEXT
].size
=
252 VMWGFX_NUM_GB_CONTEXT
* SVGA3D_OTABLE_CONTEXT_ENTRY_SIZE
;
253 otables
[SVGA_OTABLE_SHADER
].size
=
254 VMWGFX_NUM_GB_SHADER
* SVGA3D_OTABLE_SHADER_ENTRY_SIZE
;
255 otables
[SVGA_OTABLE_SCREEN_TARGET
].size
=
256 VMWGFX_NUM_GB_SCREEN_TARGET
*
257 SVGA3D_OTABLE_SCREEN_TARGET_ENTRY_SIZE
;
260 for (i
= 0; i
< SVGA_OTABLE_DX9_MAX
; ++i
) {
262 (otables
[i
].size
+ PAGE_SIZE
- 1) & PAGE_MASK
;
263 bo_size
+= otables
[i
].size
;
266 ret
= ttm_bo_create(&dev_priv
->bdev
, bo_size
,
268 &vmw_sys_ne_placement
,
270 &dev_priv
->otable_bo
);
272 if (unlikely(ret
!= 0))
275 ret
= ttm_bo_reserve(dev_priv
->otable_bo
, false, true, false, NULL
);
277 ret
= vmw_bo_driver
.ttm_tt_populate(dev_priv
->otable_bo
->ttm
);
278 if (unlikely(ret
!= 0))
280 ret
= vmw_bo_map_dma(dev_priv
->otable_bo
);
281 if (unlikely(ret
!= 0))
284 ttm_bo_unreserve(dev_priv
->otable_bo
);
287 for (i
= 0; i
< SVGA_OTABLE_DX9_MAX
- VMW_OTABLE_SETUP_SUB
; ++i
) {
288 ret
= vmw_setup_otable_base(dev_priv
, i
, offset
,
290 if (unlikely(ret
!= 0))
292 offset
+= otables
[i
].size
;
295 dev_priv
->otables
= otables
;
299 ttm_bo_unreserve(dev_priv
->otable_bo
);
301 for (i
= 0; i
< SVGA_OTABLE_DX9_MAX
- VMW_OTABLE_SETUP_SUB
; ++i
)
302 vmw_takedown_otable_base(dev_priv
, i
, &otables
[i
]);
304 ttm_bo_unref(&dev_priv
->otable_bo
);
312 * vmw_otables_takedown - Take down guest backed memory object tables
314 * @dev_priv: Pointer to a device private structure
316 * Take down the Guest Memory Object tables.
318 void vmw_otables_takedown(struct vmw_private
*dev_priv
)
321 struct ttm_buffer_object
*bo
= dev_priv
->otable_bo
;
324 for (i
= 0; i
< SVGA_OTABLE_DX9_MAX
- VMW_OTABLE_SETUP_SUB
; ++i
)
325 vmw_takedown_otable_base(dev_priv
, i
,
326 &dev_priv
->otables
[i
]);
328 ret
= ttm_bo_reserve(bo
, false, true, false, NULL
);
331 vmw_fence_single_bo(bo
, NULL
);
332 ttm_bo_unreserve(bo
);
334 ttm_bo_unref(&dev_priv
->otable_bo
);
335 kfree(dev_priv
->otables
);
336 dev_priv
->otables
= NULL
;
341 * vmw_mob_calculate_pt_pages - Calculate the number of page table pages
342 * needed for a guest backed memory object.
344 * @data_pages: Number of data pages in the memory object buffer.
346 static unsigned long vmw_mob_calculate_pt_pages(unsigned long data_pages
)
348 unsigned long data_size
= data_pages
* PAGE_SIZE
;
349 unsigned long tot_size
= 0;
351 while (likely(data_size
> PAGE_SIZE
)) {
352 data_size
= DIV_ROUND_UP(data_size
, PAGE_SIZE
);
353 data_size
*= VMW_PPN_SIZE
;
354 tot_size
+= (data_size
+ PAGE_SIZE
- 1) & PAGE_MASK
;
357 return tot_size
>> PAGE_SHIFT
;
361 * vmw_mob_create - Create a mob, but don't populate it.
363 * @data_pages: Number of data pages of the underlying buffer object.
365 struct vmw_mob
*vmw_mob_create(unsigned long data_pages
)
367 struct vmw_mob
*mob
= kzalloc(sizeof(*mob
), GFP_KERNEL
);
369 if (unlikely(mob
== NULL
))
372 mob
->num_pages
= vmw_mob_calculate_pt_pages(data_pages
);
378 * vmw_mob_pt_populate - Populate the mob pagetable
380 * @mob: Pointer to the mob the pagetable of which we want to
383 * This function allocates memory to be used for the pagetable, and
384 * adjusts TTM memory accounting accordingly. Returns ENOMEM if
385 * memory resources aren't sufficient and may cause TTM buffer objects
386 * to be swapped out by using the TTM memory accounting function.
388 static int vmw_mob_pt_populate(struct vmw_private
*dev_priv
,
392 BUG_ON(mob
->pt_bo
!= NULL
);
394 ret
= ttm_bo_create(&dev_priv
->bdev
, mob
->num_pages
* PAGE_SIZE
,
396 &vmw_sys_ne_placement
,
397 0, false, NULL
, &mob
->pt_bo
);
398 if (unlikely(ret
!= 0))
401 ret
= ttm_bo_reserve(mob
->pt_bo
, false, true, false, NULL
);
404 ret
= vmw_bo_driver
.ttm_tt_populate(mob
->pt_bo
->ttm
);
405 if (unlikely(ret
!= 0))
407 ret
= vmw_bo_map_dma(mob
->pt_bo
);
408 if (unlikely(ret
!= 0))
411 ttm_bo_unreserve(mob
->pt_bo
);
416 ttm_bo_unreserve(mob
->pt_bo
);
417 ttm_bo_unref(&mob
->pt_bo
);
423 * vmw_mob_assign_ppn - Assign a value to a page table entry
425 * @addr: Pointer to pointer to page table entry.
426 * @val: The page table entry
428 * Assigns a value to a page table entry pointed to by *@addr and increments
429 * *@addr according to the page table entry size.
431 #if (VMW_PPN_SIZE == 8)
432 static void vmw_mob_assign_ppn(__le32
**addr
, dma_addr_t val
)
434 *((__le64
*) *addr
) = cpu_to_le64(val
>> PAGE_SHIFT
);
438 static void vmw_mob_assign_ppn(__le32
**addr
, dma_addr_t val
)
440 *(*addr
)++ = cpu_to_le32(val
>> PAGE_SHIFT
);
445 * vmw_mob_build_pt - Build a pagetable
447 * @data_addr: Array of DMA addresses to the underlying buffer
448 * object's data pages.
449 * @num_data_pages: Number of buffer object data pages.
450 * @pt_pages: Array of page pointers to the page table pages.
452 * Returns the number of page table pages actually used.
453 * Uses atomic kmaps of highmem pages to avoid TLB thrashing.
455 static unsigned long vmw_mob_build_pt(struct vmw_piter
*data_iter
,
456 unsigned long num_data_pages
,
457 struct vmw_piter
*pt_iter
)
459 unsigned long pt_size
= num_data_pages
* VMW_PPN_SIZE
;
460 unsigned long num_pt_pages
= DIV_ROUND_UP(pt_size
, PAGE_SIZE
);
461 unsigned long pt_page
;
462 __le32
*addr
, *save_addr
;
466 for (pt_page
= 0; pt_page
< num_pt_pages
; ++pt_page
) {
467 page
= vmw_piter_page(pt_iter
);
469 save_addr
= addr
= kmap_atomic(page
);
471 for (i
= 0; i
< PAGE_SIZE
/ VMW_PPN_SIZE
; ++i
) {
472 vmw_mob_assign_ppn(&addr
,
473 vmw_piter_dma_addr(data_iter
));
474 if (unlikely(--num_data_pages
== 0))
476 WARN_ON(!vmw_piter_next(data_iter
));
478 kunmap_atomic(save_addr
);
479 vmw_piter_next(pt_iter
);
486 * vmw_mob_build_pt - Set up a multilevel mob pagetable
488 * @mob: Pointer to a mob whose page table needs setting up.
489 * @data_addr Array of DMA addresses to the buffer object's data
491 * @num_data_pages: Number of buffer object data pages.
493 * Uses tail recursion to set up a multilevel mob page table.
495 static void vmw_mob_pt_setup(struct vmw_mob
*mob
,
496 struct vmw_piter data_iter
,
497 unsigned long num_data_pages
)
499 unsigned long num_pt_pages
= 0;
500 struct ttm_buffer_object
*bo
= mob
->pt_bo
;
501 struct vmw_piter save_pt_iter
;
502 struct vmw_piter pt_iter
;
503 const struct vmw_sg_table
*vsgt
;
506 ret
= ttm_bo_reserve(bo
, false, true, false, NULL
);
509 vsgt
= vmw_bo_sg_table(bo
);
510 vmw_piter_start(&pt_iter
, vsgt
, 0);
511 BUG_ON(!vmw_piter_next(&pt_iter
));
513 while (likely(num_data_pages
> 1)) {
515 BUG_ON(mob
->pt_level
> 2);
516 save_pt_iter
= pt_iter
;
517 num_pt_pages
= vmw_mob_build_pt(&data_iter
, num_data_pages
,
519 data_iter
= save_pt_iter
;
520 num_data_pages
= num_pt_pages
;
523 mob
->pt_root_page
= vmw_piter_dma_addr(&save_pt_iter
);
524 ttm_bo_unreserve(bo
);
528 * vmw_mob_destroy - Destroy a mob, unpopulating first if necessary.
530 * @mob: Pointer to a mob to destroy.
532 void vmw_mob_destroy(struct vmw_mob
*mob
)
535 ttm_bo_unref(&mob
->pt_bo
);
540 * vmw_mob_unbind - Hide a mob from the device.
542 * @dev_priv: Pointer to a device private.
543 * @mob_id: Device id of the mob to unbind.
545 void vmw_mob_unbind(struct vmw_private
*dev_priv
,
549 SVGA3dCmdHeader header
;
550 SVGA3dCmdDestroyGBMob body
;
553 struct ttm_buffer_object
*bo
= mob
->pt_bo
;
556 ret
= ttm_bo_reserve(bo
, false, true, false, NULL
);
558 * Noone else should be using this buffer.
563 cmd
= vmw_fifo_reserve(dev_priv
, sizeof(*cmd
));
564 if (unlikely(cmd
== NULL
)) {
565 DRM_ERROR("Failed reserving FIFO space for Memory "
566 "Object unbinding.\n");
568 cmd
->header
.id
= SVGA_3D_CMD_DESTROY_GB_MOB
;
569 cmd
->header
.size
= sizeof(cmd
->body
);
570 cmd
->body
.mobid
= mob
->id
;
571 vmw_fifo_commit(dev_priv
, sizeof(*cmd
));
574 vmw_fence_single_bo(bo
, NULL
);
575 ttm_bo_unreserve(bo
);
577 vmw_3d_resource_dec(dev_priv
, false);
581 * vmw_mob_bind - Make a mob visible to the device after first
582 * populating it if necessary.
584 * @dev_priv: Pointer to a device private.
585 * @mob: Pointer to the mob we're making visible.
586 * @data_addr: Array of DMA addresses to the data pages of the underlying
588 * @num_data_pages: Number of data pages of the underlying buffer
590 * @mob_id: Device id of the mob to bind
592 * This function is intended to be interfaced with the ttm_tt backend
595 int vmw_mob_bind(struct vmw_private
*dev_priv
,
597 const struct vmw_sg_table
*vsgt
,
598 unsigned long num_data_pages
,
602 bool pt_set_up
= false;
603 struct vmw_piter data_iter
;
605 SVGA3dCmdHeader header
;
606 SVGA3dCmdDefineGBMob64 body
;
610 vmw_piter_start(&data_iter
, vsgt
, 0);
611 if (unlikely(!vmw_piter_next(&data_iter
)))
614 if (likely(num_data_pages
== 1)) {
615 mob
->pt_level
= VMW_MOBFMT_PTDEPTH_0
;
616 mob
->pt_root_page
= vmw_piter_dma_addr(&data_iter
);
617 } else if (vsgt
->num_regions
== 1) {
618 mob
->pt_level
= SVGA3D_MOBFMT_RANGE
;
619 mob
->pt_root_page
= vmw_piter_dma_addr(&data_iter
);
620 } else if (unlikely(mob
->pt_bo
== NULL
)) {
621 ret
= vmw_mob_pt_populate(dev_priv
, mob
);
622 if (unlikely(ret
!= 0))
625 vmw_mob_pt_setup(mob
, data_iter
, num_data_pages
);
627 mob
->pt_level
+= VMW_MOBFMT_PTDEPTH_1
- SVGA3D_MOBFMT_PTDEPTH_1
;
630 (void) vmw_3d_resource_inc(dev_priv
, false);
632 cmd
= vmw_fifo_reserve(dev_priv
, sizeof(*cmd
));
633 if (unlikely(cmd
== NULL
)) {
634 DRM_ERROR("Failed reserving FIFO space for Memory "
635 "Object binding.\n");
636 goto out_no_cmd_space
;
639 cmd
->header
.id
= SVGA_3D_CMD_DEFINE_GB_MOB64
;
640 cmd
->header
.size
= sizeof(cmd
->body
);
641 cmd
->body
.mobid
= mob_id
;
642 cmd
->body
.ptDepth
= mob
->pt_level
;
643 cmd
->body
.base
= cpu_to_le64(mob
->pt_root_page
>> PAGE_SHIFT
);
644 cmd
->body
.sizeInBytes
= num_data_pages
* PAGE_SIZE
;
646 vmw_fifo_commit(dev_priv
, sizeof(*cmd
));
651 vmw_3d_resource_dec(dev_priv
, false);
653 ttm_bo_unref(&mob
->pt_bo
);