2 * Copyright 2007 Dave Airlied
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
25 * Authors: Dave Airlied <airlied@linux.ie>
26 * Ben Skeggs <darktama@iinet.net.au>
27 * Jeremy Kolb <jkolb@brandeis.edu>
30 #include <linux/dma-mapping.h>
31 #include <linux/swiotlb.h>
33 #include "nouveau_drv.h"
34 #include "nouveau_dma.h"
35 #include "nouveau_fence.h"
37 #include "nouveau_bo.h"
38 #include "nouveau_ttm.h"
39 #include "nouveau_gem.h"
42 * NV10-NV40 tiling helpers
46 nv10_bo_update_tile_region(struct drm_device
*dev
, struct nouveau_drm_tile
*reg
,
47 u32 addr
, u32 size
, u32 pitch
, u32 flags
)
49 struct nouveau_drm
*drm
= nouveau_drm(dev
);
50 int i
= reg
- drm
->tile
.reg
;
51 struct nvkm_device
*device
= nvxx_device(&drm
->device
);
52 struct nvkm_fb
*fb
= device
->fb
;
53 struct nvkm_fb_tile
*tile
= &fb
->tile
.region
[i
];
55 nouveau_fence_unref(®
->fence
);
58 nvkm_fb_tile_fini(fb
, i
, tile
);
61 nvkm_fb_tile_init(fb
, i
, addr
, size
, pitch
, flags
, tile
);
63 nvkm_fb_tile_prog(fb
, i
, tile
);
66 static struct nouveau_drm_tile
*
67 nv10_bo_get_tile_region(struct drm_device
*dev
, int i
)
69 struct nouveau_drm
*drm
= nouveau_drm(dev
);
70 struct nouveau_drm_tile
*tile
= &drm
->tile
.reg
[i
];
72 spin_lock(&drm
->tile
.lock
);
75 (!tile
->fence
|| nouveau_fence_done(tile
->fence
)))
80 spin_unlock(&drm
->tile
.lock
);
85 nv10_bo_put_tile_region(struct drm_device
*dev
, struct nouveau_drm_tile
*tile
,
88 struct nouveau_drm
*drm
= nouveau_drm(dev
);
91 spin_lock(&drm
->tile
.lock
);
92 tile
->fence
= (struct nouveau_fence
*)fence_get(fence
);
94 spin_unlock(&drm
->tile
.lock
);
98 static struct nouveau_drm_tile
*
99 nv10_bo_set_tiling(struct drm_device
*dev
, u32 addr
,
100 u32 size
, u32 pitch
, u32 flags
)
102 struct nouveau_drm
*drm
= nouveau_drm(dev
);
103 struct nvkm_fb
*fb
= nvxx_fb(&drm
->device
);
104 struct nouveau_drm_tile
*tile
, *found
= NULL
;
107 for (i
= 0; i
< fb
->tile
.regions
; i
++) {
108 tile
= nv10_bo_get_tile_region(dev
, i
);
110 if (pitch
&& !found
) {
114 } else if (tile
&& fb
->tile
.region
[i
].pitch
) {
115 /* Kill an unused tile region. */
116 nv10_bo_update_tile_region(dev
, tile
, 0, 0, 0, 0);
119 nv10_bo_put_tile_region(dev
, tile
, NULL
);
123 nv10_bo_update_tile_region(dev
, found
, addr
, size
,
129 nouveau_bo_del_ttm(struct ttm_buffer_object
*bo
)
131 struct nouveau_drm
*drm
= nouveau_bdev(bo
->bdev
);
132 struct drm_device
*dev
= drm
->dev
;
133 struct nouveau_bo
*nvbo
= nouveau_bo(bo
);
135 if (unlikely(nvbo
->gem
.filp
))
136 DRM_ERROR("bo %p still attached to GEM object\n", bo
);
137 WARN_ON(nvbo
->pin_refcnt
> 0);
138 nv10_bo_put_tile_region(dev
, nvbo
->tile
, NULL
);
143 nouveau_bo_fixup_align(struct nouveau_bo
*nvbo
, u32 flags
,
144 int *align
, int *size
)
146 struct nouveau_drm
*drm
= nouveau_bdev(nvbo
->bo
.bdev
);
147 struct nvif_device
*device
= &drm
->device
;
149 if (device
->info
.family
< NV_DEVICE_INFO_V0_TESLA
) {
150 if (nvbo
->tile_mode
) {
151 if (device
->info
.chipset
>= 0x40) {
153 *size
= roundup(*size
, 64 * nvbo
->tile_mode
);
155 } else if (device
->info
.chipset
>= 0x30) {
157 *size
= roundup(*size
, 64 * nvbo
->tile_mode
);
159 } else if (device
->info
.chipset
>= 0x20) {
161 *size
= roundup(*size
, 64 * nvbo
->tile_mode
);
163 } else if (device
->info
.chipset
>= 0x10) {
165 *size
= roundup(*size
, 32 * nvbo
->tile_mode
);
169 *size
= roundup(*size
, (1 << nvbo
->page_shift
));
170 *align
= max((1 << nvbo
->page_shift
), *align
);
173 *size
= roundup(*size
, PAGE_SIZE
);
177 nouveau_bo_new(struct drm_device
*dev
, int size
, int align
,
178 uint32_t flags
, uint32_t tile_mode
, uint32_t tile_flags
,
179 struct sg_table
*sg
, struct reservation_object
*robj
,
180 struct nouveau_bo
**pnvbo
)
182 struct nouveau_drm
*drm
= nouveau_drm(dev
);
183 struct nouveau_bo
*nvbo
;
186 int type
= ttm_bo_type_device
;
191 lpg_shift
= drm
->client
.vm
->mmu
->lpg_shift
;
192 max_size
= INT_MAX
& ~((1 << lpg_shift
) - 1);
194 if (size
<= 0 || size
> max_size
) {
195 NV_WARN(drm
, "skipped size %x\n", (u32
)size
);
200 type
= ttm_bo_type_sg
;
202 nvbo
= kzalloc(sizeof(struct nouveau_bo
), GFP_KERNEL
);
205 INIT_LIST_HEAD(&nvbo
->head
);
206 INIT_LIST_HEAD(&nvbo
->entry
);
207 INIT_LIST_HEAD(&nvbo
->vma_list
);
208 nvbo
->tile_mode
= tile_mode
;
209 nvbo
->tile_flags
= tile_flags
;
210 nvbo
->bo
.bdev
= &drm
->ttm
.bdev
;
212 nvbo
->force_coherent
= flags
& TTM_PL_FLAG_UNCACHED
;
214 nvbo
->page_shift
= 12;
215 if (drm
->client
.vm
) {
216 if (!(flags
& TTM_PL_FLAG_TT
) && size
> 256 * 1024)
217 nvbo
->page_shift
= drm
->client
.vm
->mmu
->lpg_shift
;
220 nouveau_bo_fixup_align(nvbo
, flags
, &align
, &size
);
221 nvbo
->bo
.mem
.num_pages
= size
>> PAGE_SHIFT
;
222 nouveau_bo_placement_set(nvbo
, flags
, 0);
224 acc_size
= ttm_bo_dma_acc_size(&drm
->ttm
.bdev
, size
,
225 sizeof(struct nouveau_bo
));
227 ret
= ttm_bo_init(&drm
->ttm
.bdev
, &nvbo
->bo
, size
,
228 type
, &nvbo
->placement
,
229 align
>> PAGE_SHIFT
, false, NULL
, acc_size
, sg
,
230 robj
, nouveau_bo_del_ttm
);
232 /* ttm will call nouveau_bo_del_ttm if it fails.. */
241 set_placement_list(struct ttm_place
*pl
, unsigned *n
, uint32_t type
, uint32_t flags
)
245 if (type
& TTM_PL_FLAG_VRAM
)
246 pl
[(*n
)++].flags
= TTM_PL_FLAG_VRAM
| flags
;
247 if (type
& TTM_PL_FLAG_TT
)
248 pl
[(*n
)++].flags
= TTM_PL_FLAG_TT
| flags
;
249 if (type
& TTM_PL_FLAG_SYSTEM
)
250 pl
[(*n
)++].flags
= TTM_PL_FLAG_SYSTEM
| flags
;
254 set_placement_range(struct nouveau_bo
*nvbo
, uint32_t type
)
256 struct nouveau_drm
*drm
= nouveau_bdev(nvbo
->bo
.bdev
);
257 u32 vram_pages
= drm
->device
.info
.ram_size
>> PAGE_SHIFT
;
258 unsigned i
, fpfn
, lpfn
;
260 if (drm
->device
.info
.family
== NV_DEVICE_INFO_V0_CELSIUS
&&
261 nvbo
->tile_mode
&& (type
& TTM_PL_FLAG_VRAM
) &&
262 nvbo
->bo
.mem
.num_pages
< vram_pages
/ 4) {
264 * Make sure that the color and depth buffers are handled
265 * by independent memory controller units. Up to a 9x
266 * speed up when alpha-blending and depth-test are enabled
269 if (nvbo
->tile_flags
& NOUVEAU_GEM_TILE_ZETA
) {
270 fpfn
= vram_pages
/ 2;
274 lpfn
= vram_pages
/ 2;
276 for (i
= 0; i
< nvbo
->placement
.num_placement
; ++i
) {
277 nvbo
->placements
[i
].fpfn
= fpfn
;
278 nvbo
->placements
[i
].lpfn
= lpfn
;
280 for (i
= 0; i
< nvbo
->placement
.num_busy_placement
; ++i
) {
281 nvbo
->busy_placements
[i
].fpfn
= fpfn
;
282 nvbo
->busy_placements
[i
].lpfn
= lpfn
;
288 nouveau_bo_placement_set(struct nouveau_bo
*nvbo
, uint32_t type
, uint32_t busy
)
290 struct ttm_placement
*pl
= &nvbo
->placement
;
291 uint32_t flags
= (nvbo
->force_coherent
? TTM_PL_FLAG_UNCACHED
:
292 TTM_PL_MASK_CACHING
) |
293 (nvbo
->pin_refcnt
? TTM_PL_FLAG_NO_EVICT
: 0);
295 pl
->placement
= nvbo
->placements
;
296 set_placement_list(nvbo
->placements
, &pl
->num_placement
,
299 pl
->busy_placement
= nvbo
->busy_placements
;
300 set_placement_list(nvbo
->busy_placements
, &pl
->num_busy_placement
,
303 set_placement_range(nvbo
, type
);
307 nouveau_bo_pin(struct nouveau_bo
*nvbo
, uint32_t memtype
, bool contig
)
309 struct nouveau_drm
*drm
= nouveau_bdev(nvbo
->bo
.bdev
);
310 struct ttm_buffer_object
*bo
= &nvbo
->bo
;
311 bool force
= false, evict
= false;
314 ret
= ttm_bo_reserve(bo
, false, false, NULL
);
318 if (drm
->device
.info
.family
>= NV_DEVICE_INFO_V0_TESLA
&&
319 memtype
== TTM_PL_FLAG_VRAM
&& contig
) {
320 if (nvbo
->tile_flags
& NOUVEAU_GEM_TILE_NONCONTIG
) {
321 if (bo
->mem
.mem_type
== TTM_PL_VRAM
) {
322 struct nvkm_mem
*mem
= bo
->mem
.mm_node
;
323 if (!list_is_singular(&mem
->regions
))
326 nvbo
->tile_flags
&= ~NOUVEAU_GEM_TILE_NONCONTIG
;
331 if (nvbo
->pin_refcnt
) {
332 if (!(memtype
& (1 << bo
->mem
.mem_type
)) || evict
) {
333 NV_ERROR(drm
, "bo %p pinned elsewhere: "
334 "0x%08x vs 0x%08x\n", bo
,
335 1 << bo
->mem
.mem_type
, memtype
);
343 nouveau_bo_placement_set(nvbo
, TTM_PL_FLAG_TT
, 0);
344 ret
= nouveau_bo_validate(nvbo
, false, false);
350 nouveau_bo_placement_set(nvbo
, memtype
, 0);
352 /* drop pin_refcnt temporarily, so we don't trip the assertion
353 * in nouveau_bo_move() that makes sure we're not trying to
354 * move a pinned buffer
357 ret
= nouveau_bo_validate(nvbo
, false, false);
362 switch (bo
->mem
.mem_type
) {
364 drm
->gem
.vram_available
-= bo
->mem
.size
;
367 drm
->gem
.gart_available
-= bo
->mem
.size
;
375 nvbo
->tile_flags
|= NOUVEAU_GEM_TILE_NONCONTIG
;
376 ttm_bo_unreserve(bo
);
381 nouveau_bo_unpin(struct nouveau_bo
*nvbo
)
383 struct nouveau_drm
*drm
= nouveau_bdev(nvbo
->bo
.bdev
);
384 struct ttm_buffer_object
*bo
= &nvbo
->bo
;
387 ret
= ttm_bo_reserve(bo
, false, false, NULL
);
391 ref
= --nvbo
->pin_refcnt
;
392 WARN_ON_ONCE(ref
< 0);
396 nouveau_bo_placement_set(nvbo
, bo
->mem
.placement
, 0);
398 ret
= nouveau_bo_validate(nvbo
, false, false);
400 switch (bo
->mem
.mem_type
) {
402 drm
->gem
.vram_available
+= bo
->mem
.size
;
405 drm
->gem
.gart_available
+= bo
->mem
.size
;
413 ttm_bo_unreserve(bo
);
418 nouveau_bo_map(struct nouveau_bo
*nvbo
)
422 ret
= ttm_bo_reserve(&nvbo
->bo
, false, false, NULL
);
426 ret
= ttm_bo_kmap(&nvbo
->bo
, 0, nvbo
->bo
.mem
.num_pages
, &nvbo
->kmap
);
428 ttm_bo_unreserve(&nvbo
->bo
);
433 nouveau_bo_unmap(struct nouveau_bo
*nvbo
)
438 ttm_bo_kunmap(&nvbo
->kmap
);
442 nouveau_bo_sync_for_device(struct nouveau_bo
*nvbo
)
444 struct nouveau_drm
*drm
= nouveau_bdev(nvbo
->bo
.bdev
);
445 struct nvkm_device
*device
= nvxx_device(&drm
->device
);
446 struct ttm_dma_tt
*ttm_dma
= (struct ttm_dma_tt
*)nvbo
->bo
.ttm
;
452 /* Don't waste time looping if the object is coherent */
453 if (nvbo
->force_coherent
)
456 for (i
= 0; i
< ttm_dma
->ttm
.num_pages
; i
++)
457 dma_sync_single_for_device(device
->dev
, ttm_dma
->dma_address
[i
],
458 PAGE_SIZE
, DMA_TO_DEVICE
);
462 nouveau_bo_sync_for_cpu(struct nouveau_bo
*nvbo
)
464 struct nouveau_drm
*drm
= nouveau_bdev(nvbo
->bo
.bdev
);
465 struct nvkm_device
*device
= nvxx_device(&drm
->device
);
466 struct ttm_dma_tt
*ttm_dma
= (struct ttm_dma_tt
*)nvbo
->bo
.ttm
;
472 /* Don't waste time looping if the object is coherent */
473 if (nvbo
->force_coherent
)
476 for (i
= 0; i
< ttm_dma
->ttm
.num_pages
; i
++)
477 dma_sync_single_for_cpu(device
->dev
, ttm_dma
->dma_address
[i
],
478 PAGE_SIZE
, DMA_FROM_DEVICE
);
482 nouveau_bo_validate(struct nouveau_bo
*nvbo
, bool interruptible
,
487 ret
= ttm_bo_validate(&nvbo
->bo
, &nvbo
->placement
,
488 interruptible
, no_wait_gpu
);
492 nouveau_bo_sync_for_device(nvbo
);
498 nouveau_bo_wr16(struct nouveau_bo
*nvbo
, unsigned index
, u16 val
)
501 u16
*mem
= ttm_kmap_obj_virtual(&nvbo
->kmap
, &is_iomem
);
506 iowrite16_native(val
, (void __force __iomem
*)mem
);
512 nouveau_bo_rd32(struct nouveau_bo
*nvbo
, unsigned index
)
515 u32
*mem
= ttm_kmap_obj_virtual(&nvbo
->kmap
, &is_iomem
);
520 return ioread32_native((void __force __iomem
*)mem
);
526 nouveau_bo_wr32(struct nouveau_bo
*nvbo
, unsigned index
, u32 val
)
529 u32
*mem
= ttm_kmap_obj_virtual(&nvbo
->kmap
, &is_iomem
);
534 iowrite32_native(val
, (void __force __iomem
*)mem
);
539 static struct ttm_tt
*
540 nouveau_ttm_tt_create(struct ttm_bo_device
*bdev
, unsigned long size
,
541 uint32_t page_flags
, struct page
*dummy_read
)
543 #if IS_ENABLED(CONFIG_AGP)
544 struct nouveau_drm
*drm
= nouveau_bdev(bdev
);
546 if (drm
->agp
.bridge
) {
547 return ttm_agp_tt_create(bdev
, drm
->agp
.bridge
, size
,
548 page_flags
, dummy_read
);
552 return nouveau_sgdma_create_ttm(bdev
, size
, page_flags
, dummy_read
);
556 nouveau_bo_invalidate_caches(struct ttm_bo_device
*bdev
, uint32_t flags
)
558 /* We'll do this from user space. */
563 nouveau_bo_init_mem_type(struct ttm_bo_device
*bdev
, uint32_t type
,
564 struct ttm_mem_type_manager
*man
)
566 struct nouveau_drm
*drm
= nouveau_bdev(bdev
);
570 man
->flags
= TTM_MEMTYPE_FLAG_MAPPABLE
;
571 man
->available_caching
= TTM_PL_MASK_CACHING
;
572 man
->default_caching
= TTM_PL_FLAG_CACHED
;
575 man
->flags
= TTM_MEMTYPE_FLAG_FIXED
|
576 TTM_MEMTYPE_FLAG_MAPPABLE
;
577 man
->available_caching
= TTM_PL_FLAG_UNCACHED
|
579 man
->default_caching
= TTM_PL_FLAG_WC
;
581 if (drm
->device
.info
.family
>= NV_DEVICE_INFO_V0_TESLA
) {
582 /* Some BARs do not support being ioremapped WC */
583 if (nvxx_bar(&drm
->device
)->iomap_uncached
) {
584 man
->available_caching
= TTM_PL_FLAG_UNCACHED
;
585 man
->default_caching
= TTM_PL_FLAG_UNCACHED
;
588 man
->func
= &nouveau_vram_manager
;
589 man
->io_reserve_fastpath
= false;
590 man
->use_io_reserve_lru
= true;
592 man
->func
= &ttm_bo_manager_func
;
596 if (drm
->device
.info
.family
>= NV_DEVICE_INFO_V0_TESLA
)
597 man
->func
= &nouveau_gart_manager
;
599 if (!drm
->agp
.bridge
)
600 man
->func
= &nv04_gart_manager
;
602 man
->func
= &ttm_bo_manager_func
;
604 if (drm
->agp
.bridge
) {
605 man
->flags
= TTM_MEMTYPE_FLAG_MAPPABLE
;
606 man
->available_caching
= TTM_PL_FLAG_UNCACHED
|
608 man
->default_caching
= TTM_PL_FLAG_WC
;
610 man
->flags
= TTM_MEMTYPE_FLAG_MAPPABLE
|
611 TTM_MEMTYPE_FLAG_CMA
;
612 man
->available_caching
= TTM_PL_MASK_CACHING
;
613 man
->default_caching
= TTM_PL_FLAG_CACHED
;
624 nouveau_bo_evict_flags(struct ttm_buffer_object
*bo
, struct ttm_placement
*pl
)
626 struct nouveau_bo
*nvbo
= nouveau_bo(bo
);
628 switch (bo
->mem
.mem_type
) {
630 nouveau_bo_placement_set(nvbo
, TTM_PL_FLAG_TT
,
634 nouveau_bo_placement_set(nvbo
, TTM_PL_FLAG_SYSTEM
, 0);
638 *pl
= nvbo
->placement
;
643 nve0_bo_move_init(struct nouveau_channel
*chan
, u32 handle
)
645 int ret
= RING_SPACE(chan
, 2);
647 BEGIN_NVC0(chan
, NvSubCopy
, 0x0000, 1);
648 OUT_RING (chan
, handle
& 0x0000ffff);
655 nve0_bo_move_copy(struct nouveau_channel
*chan
, struct ttm_buffer_object
*bo
,
656 struct ttm_mem_reg
*old_mem
, struct ttm_mem_reg
*new_mem
)
658 struct nvkm_mem
*node
= old_mem
->mm_node
;
659 int ret
= RING_SPACE(chan
, 10);
661 BEGIN_NVC0(chan
, NvSubCopy
, 0x0400, 8);
662 OUT_RING (chan
, upper_32_bits(node
->vma
[0].offset
));
663 OUT_RING (chan
, lower_32_bits(node
->vma
[0].offset
));
664 OUT_RING (chan
, upper_32_bits(node
->vma
[1].offset
));
665 OUT_RING (chan
, lower_32_bits(node
->vma
[1].offset
));
666 OUT_RING (chan
, PAGE_SIZE
);
667 OUT_RING (chan
, PAGE_SIZE
);
668 OUT_RING (chan
, PAGE_SIZE
);
669 OUT_RING (chan
, new_mem
->num_pages
);
670 BEGIN_IMC0(chan
, NvSubCopy
, 0x0300, 0x0386);
676 nvc0_bo_move_init(struct nouveau_channel
*chan
, u32 handle
)
678 int ret
= RING_SPACE(chan
, 2);
680 BEGIN_NVC0(chan
, NvSubCopy
, 0x0000, 1);
681 OUT_RING (chan
, handle
);
687 nvc0_bo_move_copy(struct nouveau_channel
*chan
, struct ttm_buffer_object
*bo
,
688 struct ttm_mem_reg
*old_mem
, struct ttm_mem_reg
*new_mem
)
690 struct nvkm_mem
*node
= old_mem
->mm_node
;
691 u64 src_offset
= node
->vma
[0].offset
;
692 u64 dst_offset
= node
->vma
[1].offset
;
693 u32 page_count
= new_mem
->num_pages
;
696 page_count
= new_mem
->num_pages
;
698 int line_count
= (page_count
> 8191) ? 8191 : page_count
;
700 ret
= RING_SPACE(chan
, 11);
704 BEGIN_NVC0(chan
, NvSubCopy
, 0x030c, 8);
705 OUT_RING (chan
, upper_32_bits(src_offset
));
706 OUT_RING (chan
, lower_32_bits(src_offset
));
707 OUT_RING (chan
, upper_32_bits(dst_offset
));
708 OUT_RING (chan
, lower_32_bits(dst_offset
));
709 OUT_RING (chan
, PAGE_SIZE
);
710 OUT_RING (chan
, PAGE_SIZE
);
711 OUT_RING (chan
, PAGE_SIZE
);
712 OUT_RING (chan
, line_count
);
713 BEGIN_NVC0(chan
, NvSubCopy
, 0x0300, 1);
714 OUT_RING (chan
, 0x00000110);
716 page_count
-= line_count
;
717 src_offset
+= (PAGE_SIZE
* line_count
);
718 dst_offset
+= (PAGE_SIZE
* line_count
);
725 nvc0_bo_move_m2mf(struct nouveau_channel
*chan
, struct ttm_buffer_object
*bo
,
726 struct ttm_mem_reg
*old_mem
, struct ttm_mem_reg
*new_mem
)
728 struct nvkm_mem
*node
= old_mem
->mm_node
;
729 u64 src_offset
= node
->vma
[0].offset
;
730 u64 dst_offset
= node
->vma
[1].offset
;
731 u32 page_count
= new_mem
->num_pages
;
734 page_count
= new_mem
->num_pages
;
736 int line_count
= (page_count
> 2047) ? 2047 : page_count
;
738 ret
= RING_SPACE(chan
, 12);
742 BEGIN_NVC0(chan
, NvSubCopy
, 0x0238, 2);
743 OUT_RING (chan
, upper_32_bits(dst_offset
));
744 OUT_RING (chan
, lower_32_bits(dst_offset
));
745 BEGIN_NVC0(chan
, NvSubCopy
, 0x030c, 6);
746 OUT_RING (chan
, upper_32_bits(src_offset
));
747 OUT_RING (chan
, lower_32_bits(src_offset
));
748 OUT_RING (chan
, PAGE_SIZE
); /* src_pitch */
749 OUT_RING (chan
, PAGE_SIZE
); /* dst_pitch */
750 OUT_RING (chan
, PAGE_SIZE
); /* line_length */
751 OUT_RING (chan
, line_count
);
752 BEGIN_NVC0(chan
, NvSubCopy
, 0x0300, 1);
753 OUT_RING (chan
, 0x00100110);
755 page_count
-= line_count
;
756 src_offset
+= (PAGE_SIZE
* line_count
);
757 dst_offset
+= (PAGE_SIZE
* line_count
);
764 nva3_bo_move_copy(struct nouveau_channel
*chan
, struct ttm_buffer_object
*bo
,
765 struct ttm_mem_reg
*old_mem
, struct ttm_mem_reg
*new_mem
)
767 struct nvkm_mem
*node
= old_mem
->mm_node
;
768 u64 src_offset
= node
->vma
[0].offset
;
769 u64 dst_offset
= node
->vma
[1].offset
;
770 u32 page_count
= new_mem
->num_pages
;
773 page_count
= new_mem
->num_pages
;
775 int line_count
= (page_count
> 8191) ? 8191 : page_count
;
777 ret
= RING_SPACE(chan
, 11);
781 BEGIN_NV04(chan
, NvSubCopy
, 0x030c, 8);
782 OUT_RING (chan
, upper_32_bits(src_offset
));
783 OUT_RING (chan
, lower_32_bits(src_offset
));
784 OUT_RING (chan
, upper_32_bits(dst_offset
));
785 OUT_RING (chan
, lower_32_bits(dst_offset
));
786 OUT_RING (chan
, PAGE_SIZE
);
787 OUT_RING (chan
, PAGE_SIZE
);
788 OUT_RING (chan
, PAGE_SIZE
);
789 OUT_RING (chan
, line_count
);
790 BEGIN_NV04(chan
, NvSubCopy
, 0x0300, 1);
791 OUT_RING (chan
, 0x00000110);
793 page_count
-= line_count
;
794 src_offset
+= (PAGE_SIZE
* line_count
);
795 dst_offset
+= (PAGE_SIZE
* line_count
);
802 nv98_bo_move_exec(struct nouveau_channel
*chan
, struct ttm_buffer_object
*bo
,
803 struct ttm_mem_reg
*old_mem
, struct ttm_mem_reg
*new_mem
)
805 struct nvkm_mem
*node
= old_mem
->mm_node
;
806 int ret
= RING_SPACE(chan
, 7);
808 BEGIN_NV04(chan
, NvSubCopy
, 0x0320, 6);
809 OUT_RING (chan
, upper_32_bits(node
->vma
[0].offset
));
810 OUT_RING (chan
, lower_32_bits(node
->vma
[0].offset
));
811 OUT_RING (chan
, upper_32_bits(node
->vma
[1].offset
));
812 OUT_RING (chan
, lower_32_bits(node
->vma
[1].offset
));
813 OUT_RING (chan
, 0x00000000 /* COPY */);
814 OUT_RING (chan
, new_mem
->num_pages
<< PAGE_SHIFT
);
820 nv84_bo_move_exec(struct nouveau_channel
*chan
, struct ttm_buffer_object
*bo
,
821 struct ttm_mem_reg
*old_mem
, struct ttm_mem_reg
*new_mem
)
823 struct nvkm_mem
*node
= old_mem
->mm_node
;
824 int ret
= RING_SPACE(chan
, 7);
826 BEGIN_NV04(chan
, NvSubCopy
, 0x0304, 6);
827 OUT_RING (chan
, new_mem
->num_pages
<< PAGE_SHIFT
);
828 OUT_RING (chan
, upper_32_bits(node
->vma
[0].offset
));
829 OUT_RING (chan
, lower_32_bits(node
->vma
[0].offset
));
830 OUT_RING (chan
, upper_32_bits(node
->vma
[1].offset
));
831 OUT_RING (chan
, lower_32_bits(node
->vma
[1].offset
));
832 OUT_RING (chan
, 0x00000000 /* MODE_COPY, QUERY_NONE */);
838 nv50_bo_move_init(struct nouveau_channel
*chan
, u32 handle
)
840 int ret
= RING_SPACE(chan
, 6);
842 BEGIN_NV04(chan
, NvSubCopy
, 0x0000, 1);
843 OUT_RING (chan
, handle
);
844 BEGIN_NV04(chan
, NvSubCopy
, 0x0180, 3);
845 OUT_RING (chan
, chan
->drm
->ntfy
.handle
);
846 OUT_RING (chan
, chan
->vram
.handle
);
847 OUT_RING (chan
, chan
->vram
.handle
);
854 nv50_bo_move_m2mf(struct nouveau_channel
*chan
, struct ttm_buffer_object
*bo
,
855 struct ttm_mem_reg
*old_mem
, struct ttm_mem_reg
*new_mem
)
857 struct nvkm_mem
*node
= old_mem
->mm_node
;
858 u64 length
= (new_mem
->num_pages
<< PAGE_SHIFT
);
859 u64 src_offset
= node
->vma
[0].offset
;
860 u64 dst_offset
= node
->vma
[1].offset
;
861 int src_tiled
= !!node
->memtype
;
862 int dst_tiled
= !!((struct nvkm_mem
*)new_mem
->mm_node
)->memtype
;
866 u32 amount
, stride
, height
;
868 ret
= RING_SPACE(chan
, 18 + 6 * (src_tiled
+ dst_tiled
));
872 amount
= min(length
, (u64
)(4 * 1024 * 1024));
874 height
= amount
/ stride
;
877 BEGIN_NV04(chan
, NvSubCopy
, 0x0200, 7);
880 OUT_RING (chan
, stride
);
881 OUT_RING (chan
, height
);
886 BEGIN_NV04(chan
, NvSubCopy
, 0x0200, 1);
890 BEGIN_NV04(chan
, NvSubCopy
, 0x021c, 7);
893 OUT_RING (chan
, stride
);
894 OUT_RING (chan
, height
);
899 BEGIN_NV04(chan
, NvSubCopy
, 0x021c, 1);
903 BEGIN_NV04(chan
, NvSubCopy
, 0x0238, 2);
904 OUT_RING (chan
, upper_32_bits(src_offset
));
905 OUT_RING (chan
, upper_32_bits(dst_offset
));
906 BEGIN_NV04(chan
, NvSubCopy
, 0x030c, 8);
907 OUT_RING (chan
, lower_32_bits(src_offset
));
908 OUT_RING (chan
, lower_32_bits(dst_offset
));
909 OUT_RING (chan
, stride
);
910 OUT_RING (chan
, stride
);
911 OUT_RING (chan
, stride
);
912 OUT_RING (chan
, height
);
913 OUT_RING (chan
, 0x00000101);
914 OUT_RING (chan
, 0x00000000);
915 BEGIN_NV04(chan
, NvSubCopy
, NV_MEMORY_TO_MEMORY_FORMAT_NOP
, 1);
919 src_offset
+= amount
;
920 dst_offset
+= amount
;
927 nv04_bo_move_init(struct nouveau_channel
*chan
, u32 handle
)
929 int ret
= RING_SPACE(chan
, 4);
931 BEGIN_NV04(chan
, NvSubCopy
, 0x0000, 1);
932 OUT_RING (chan
, handle
);
933 BEGIN_NV04(chan
, NvSubCopy
, 0x0180, 1);
934 OUT_RING (chan
, chan
->drm
->ntfy
.handle
);
940 static inline uint32_t
941 nouveau_bo_mem_ctxdma(struct ttm_buffer_object
*bo
,
942 struct nouveau_channel
*chan
, struct ttm_mem_reg
*mem
)
944 if (mem
->mem_type
== TTM_PL_TT
)
946 return chan
->vram
.handle
;
950 nv04_bo_move_m2mf(struct nouveau_channel
*chan
, struct ttm_buffer_object
*bo
,
951 struct ttm_mem_reg
*old_mem
, struct ttm_mem_reg
*new_mem
)
953 u32 src_offset
= old_mem
->start
<< PAGE_SHIFT
;
954 u32 dst_offset
= new_mem
->start
<< PAGE_SHIFT
;
955 u32 page_count
= new_mem
->num_pages
;
958 ret
= RING_SPACE(chan
, 3);
962 BEGIN_NV04(chan
, NvSubCopy
, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE
, 2);
963 OUT_RING (chan
, nouveau_bo_mem_ctxdma(bo
, chan
, old_mem
));
964 OUT_RING (chan
, nouveau_bo_mem_ctxdma(bo
, chan
, new_mem
));
966 page_count
= new_mem
->num_pages
;
968 int line_count
= (page_count
> 2047) ? 2047 : page_count
;
970 ret
= RING_SPACE(chan
, 11);
974 BEGIN_NV04(chan
, NvSubCopy
,
975 NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN
, 8);
976 OUT_RING (chan
, src_offset
);
977 OUT_RING (chan
, dst_offset
);
978 OUT_RING (chan
, PAGE_SIZE
); /* src_pitch */
979 OUT_RING (chan
, PAGE_SIZE
); /* dst_pitch */
980 OUT_RING (chan
, PAGE_SIZE
); /* line_length */
981 OUT_RING (chan
, line_count
);
982 OUT_RING (chan
, 0x00000101);
983 OUT_RING (chan
, 0x00000000);
984 BEGIN_NV04(chan
, NvSubCopy
, NV_MEMORY_TO_MEMORY_FORMAT_NOP
, 1);
987 page_count
-= line_count
;
988 src_offset
+= (PAGE_SIZE
* line_count
);
989 dst_offset
+= (PAGE_SIZE
* line_count
);
996 nouveau_bo_move_prep(struct nouveau_drm
*drm
, struct ttm_buffer_object
*bo
,
997 struct ttm_mem_reg
*mem
)
999 struct nvkm_mem
*old_node
= bo
->mem
.mm_node
;
1000 struct nvkm_mem
*new_node
= mem
->mm_node
;
1001 u64 size
= (u64
)mem
->num_pages
<< PAGE_SHIFT
;
1004 ret
= nvkm_vm_get(drm
->client
.vm
, size
, old_node
->page_shift
,
1005 NV_MEM_ACCESS_RW
, &old_node
->vma
[0]);
1009 ret
= nvkm_vm_get(drm
->client
.vm
, size
, new_node
->page_shift
,
1010 NV_MEM_ACCESS_RW
, &old_node
->vma
[1]);
1012 nvkm_vm_put(&old_node
->vma
[0]);
1016 nvkm_vm_map(&old_node
->vma
[0], old_node
);
1017 nvkm_vm_map(&old_node
->vma
[1], new_node
);
1022 nouveau_bo_move_m2mf(struct ttm_buffer_object
*bo
, int evict
, bool intr
,
1023 bool no_wait_gpu
, struct ttm_mem_reg
*new_mem
)
1025 struct nouveau_drm
*drm
= nouveau_bdev(bo
->bdev
);
1026 struct nouveau_channel
*chan
= drm
->ttm
.chan
;
1027 struct nouveau_cli
*cli
= (void *)chan
->user
.client
;
1028 struct nouveau_fence
*fence
;
1031 /* create temporary vmas for the transfer and attach them to the
1032 * old nvkm_mem node, these will get cleaned up after ttm has
1033 * destroyed the ttm_mem_reg
1035 if (drm
->device
.info
.family
>= NV_DEVICE_INFO_V0_TESLA
) {
1036 ret
= nouveau_bo_move_prep(drm
, bo
, new_mem
);
1041 mutex_lock_nested(&cli
->mutex
, SINGLE_DEPTH_NESTING
);
1042 ret
= nouveau_fence_sync(nouveau_bo(bo
), chan
, true, intr
);
1044 ret
= drm
->ttm
.move(chan
, bo
, &bo
->mem
, new_mem
);
1046 ret
= nouveau_fence_new(chan
, false, &fence
);
1048 ret
= ttm_bo_move_accel_cleanup(bo
,
1052 nouveau_fence_unref(&fence
);
1056 mutex_unlock(&cli
->mutex
);
1061 nouveau_bo_move_init(struct nouveau_drm
*drm
)
1063 static const struct {
1067 int (*exec
)(struct nouveau_channel
*,
1068 struct ttm_buffer_object
*,
1069 struct ttm_mem_reg
*, struct ttm_mem_reg
*);
1070 int (*init
)(struct nouveau_channel
*, u32 handle
);
1072 { "COPY", 4, 0xc1b5, nve0_bo_move_copy
, nve0_bo_move_init
},
1073 { "GRCE", 0, 0xc1b5, nve0_bo_move_copy
, nvc0_bo_move_init
},
1074 { "COPY", 4, 0xc0b5, nve0_bo_move_copy
, nve0_bo_move_init
},
1075 { "GRCE", 0, 0xc0b5, nve0_bo_move_copy
, nvc0_bo_move_init
},
1076 { "COPY", 4, 0xb0b5, nve0_bo_move_copy
, nve0_bo_move_init
},
1077 { "GRCE", 0, 0xb0b5, nve0_bo_move_copy
, nvc0_bo_move_init
},
1078 { "COPY", 4, 0xa0b5, nve0_bo_move_copy
, nve0_bo_move_init
},
1079 { "GRCE", 0, 0xa0b5, nve0_bo_move_copy
, nvc0_bo_move_init
},
1080 { "COPY1", 5, 0x90b8, nvc0_bo_move_copy
, nvc0_bo_move_init
},
1081 { "COPY0", 4, 0x90b5, nvc0_bo_move_copy
, nvc0_bo_move_init
},
1082 { "COPY", 0, 0x85b5, nva3_bo_move_copy
, nv50_bo_move_init
},
1083 { "CRYPT", 0, 0x74c1, nv84_bo_move_exec
, nv50_bo_move_init
},
1084 { "M2MF", 0, 0x9039, nvc0_bo_move_m2mf
, nvc0_bo_move_init
},
1085 { "M2MF", 0, 0x5039, nv50_bo_move_m2mf
, nv50_bo_move_init
},
1086 { "M2MF", 0, 0x0039, nv04_bo_move_m2mf
, nv04_bo_move_init
},
1088 { "CRYPT", 0, 0x88b4, nv98_bo_move_exec
, nv50_bo_move_init
},
1089 }, *mthd
= _methods
;
1090 const char *name
= "CPU";
1094 struct nouveau_channel
*chan
;
1099 chan
= drm
->channel
;
1103 ret
= nvif_object_init(&chan
->user
,
1104 mthd
->oclass
| (mthd
->engine
<< 16),
1105 mthd
->oclass
, NULL
, 0,
1108 ret
= mthd
->init(chan
, drm
->ttm
.copy
.handle
);
1110 nvif_object_fini(&drm
->ttm
.copy
);
1114 drm
->ttm
.move
= mthd
->exec
;
1115 drm
->ttm
.chan
= chan
;
1119 } while ((++mthd
)->exec
);
1121 NV_INFO(drm
, "MM: using %s for buffer copies\n", name
);
1125 nouveau_bo_move_flipd(struct ttm_buffer_object
*bo
, bool evict
, bool intr
,
1126 bool no_wait_gpu
, struct ttm_mem_reg
*new_mem
)
1128 struct ttm_place placement_memtype
= {
1131 .flags
= TTM_PL_FLAG_TT
| TTM_PL_MASK_CACHING
1133 struct ttm_placement placement
;
1134 struct ttm_mem_reg tmp_mem
;
1137 placement
.num_placement
= placement
.num_busy_placement
= 1;
1138 placement
.placement
= placement
.busy_placement
= &placement_memtype
;
1141 tmp_mem
.mm_node
= NULL
;
1142 ret
= ttm_bo_mem_space(bo
, &placement
, &tmp_mem
, intr
, no_wait_gpu
);
1146 ret
= ttm_tt_bind(bo
->ttm
, &tmp_mem
);
1150 ret
= nouveau_bo_move_m2mf(bo
, true, intr
, no_wait_gpu
, &tmp_mem
);
1154 ret
= ttm_bo_move_ttm(bo
, intr
, no_wait_gpu
, new_mem
);
1156 ttm_bo_mem_put(bo
, &tmp_mem
);
1161 nouveau_bo_move_flips(struct ttm_buffer_object
*bo
, bool evict
, bool intr
,
1162 bool no_wait_gpu
, struct ttm_mem_reg
*new_mem
)
1164 struct ttm_place placement_memtype
= {
1167 .flags
= TTM_PL_FLAG_TT
| TTM_PL_MASK_CACHING
1169 struct ttm_placement placement
;
1170 struct ttm_mem_reg tmp_mem
;
1173 placement
.num_placement
= placement
.num_busy_placement
= 1;
1174 placement
.placement
= placement
.busy_placement
= &placement_memtype
;
1177 tmp_mem
.mm_node
= NULL
;
1178 ret
= ttm_bo_mem_space(bo
, &placement
, &tmp_mem
, intr
, no_wait_gpu
);
1182 ret
= ttm_bo_move_ttm(bo
, intr
, no_wait_gpu
, &tmp_mem
);
1186 ret
= nouveau_bo_move_m2mf(bo
, true, intr
, no_wait_gpu
, new_mem
);
1191 ttm_bo_mem_put(bo
, &tmp_mem
);
1196 nouveau_bo_move_ntfy(struct ttm_buffer_object
*bo
, struct ttm_mem_reg
*new_mem
)
1198 struct nouveau_bo
*nvbo
= nouveau_bo(bo
);
1199 struct nvkm_vma
*vma
;
1201 /* ttm can now (stupidly) pass the driver bos it didn't create... */
1202 if (bo
->destroy
!= nouveau_bo_del_ttm
)
1205 list_for_each_entry(vma
, &nvbo
->vma_list
, head
) {
1206 if (new_mem
&& new_mem
->mem_type
!= TTM_PL_SYSTEM
&&
1207 (new_mem
->mem_type
== TTM_PL_VRAM
||
1208 nvbo
->page_shift
!= vma
->vm
->mmu
->lpg_shift
)) {
1209 nvkm_vm_map(vma
, new_mem
->mm_node
);
1217 nouveau_bo_vm_bind(struct ttm_buffer_object
*bo
, struct ttm_mem_reg
*new_mem
,
1218 struct nouveau_drm_tile
**new_tile
)
1220 struct nouveau_drm
*drm
= nouveau_bdev(bo
->bdev
);
1221 struct drm_device
*dev
= drm
->dev
;
1222 struct nouveau_bo
*nvbo
= nouveau_bo(bo
);
1223 u64 offset
= new_mem
->start
<< PAGE_SHIFT
;
1226 if (new_mem
->mem_type
!= TTM_PL_VRAM
)
1229 if (drm
->device
.info
.family
>= NV_DEVICE_INFO_V0_CELSIUS
) {
1230 *new_tile
= nv10_bo_set_tiling(dev
, offset
, new_mem
->size
,
1239 nouveau_bo_vm_cleanup(struct ttm_buffer_object
*bo
,
1240 struct nouveau_drm_tile
*new_tile
,
1241 struct nouveau_drm_tile
**old_tile
)
1243 struct nouveau_drm
*drm
= nouveau_bdev(bo
->bdev
);
1244 struct drm_device
*dev
= drm
->dev
;
1245 struct fence
*fence
= reservation_object_get_excl(bo
->resv
);
1247 nv10_bo_put_tile_region(dev
, *old_tile
, fence
);
1248 *old_tile
= new_tile
;
1252 nouveau_bo_move(struct ttm_buffer_object
*bo
, bool evict
, bool intr
,
1253 bool no_wait_gpu
, struct ttm_mem_reg
*new_mem
)
1255 struct nouveau_drm
*drm
= nouveau_bdev(bo
->bdev
);
1256 struct nouveau_bo
*nvbo
= nouveau_bo(bo
);
1257 struct ttm_mem_reg
*old_mem
= &bo
->mem
;
1258 struct nouveau_drm_tile
*new_tile
= NULL
;
1261 ret
= ttm_bo_wait(bo
, intr
, no_wait_gpu
);
1265 if (nvbo
->pin_refcnt
)
1266 NV_WARN(drm
, "Moving pinned object %p!\n", nvbo
);
1268 if (drm
->device
.info
.family
< NV_DEVICE_INFO_V0_TESLA
) {
1269 ret
= nouveau_bo_vm_bind(bo
, new_mem
, &new_tile
);
1275 if (old_mem
->mem_type
== TTM_PL_SYSTEM
&& !bo
->ttm
) {
1276 BUG_ON(bo
->mem
.mm_node
!= NULL
);
1278 new_mem
->mm_node
= NULL
;
1282 /* Hardware assisted copy. */
1283 if (drm
->ttm
.move
) {
1284 if (new_mem
->mem_type
== TTM_PL_SYSTEM
)
1285 ret
= nouveau_bo_move_flipd(bo
, evict
, intr
,
1286 no_wait_gpu
, new_mem
);
1287 else if (old_mem
->mem_type
== TTM_PL_SYSTEM
)
1288 ret
= nouveau_bo_move_flips(bo
, evict
, intr
,
1289 no_wait_gpu
, new_mem
);
1291 ret
= nouveau_bo_move_m2mf(bo
, evict
, intr
,
1292 no_wait_gpu
, new_mem
);
1297 /* Fallback to software copy. */
1298 ret
= ttm_bo_wait(bo
, intr
, no_wait_gpu
);
1300 ret
= ttm_bo_move_memcpy(bo
, intr
, no_wait_gpu
, new_mem
);
1303 if (drm
->device
.info
.family
< NV_DEVICE_INFO_V0_TESLA
) {
1305 nouveau_bo_vm_cleanup(bo
, NULL
, &new_tile
);
1307 nouveau_bo_vm_cleanup(bo
, new_tile
, &nvbo
->tile
);
1314 nouveau_bo_verify_access(struct ttm_buffer_object
*bo
, struct file
*filp
)
1316 struct nouveau_bo
*nvbo
= nouveau_bo(bo
);
1318 return drm_vma_node_verify_access(&nvbo
->gem
.vma_node
, filp
);
1322 nouveau_ttm_io_mem_reserve(struct ttm_bo_device
*bdev
, struct ttm_mem_reg
*mem
)
1324 struct ttm_mem_type_manager
*man
= &bdev
->man
[mem
->mem_type
];
1325 struct nouveau_drm
*drm
= nouveau_bdev(bdev
);
1326 struct nvkm_device
*device
= nvxx_device(&drm
->device
);
1327 struct nvkm_mem
*node
= mem
->mm_node
;
1330 mem
->bus
.addr
= NULL
;
1331 mem
->bus
.offset
= 0;
1332 mem
->bus
.size
= mem
->num_pages
<< PAGE_SHIFT
;
1334 mem
->bus
.is_iomem
= false;
1335 if (!(man
->flags
& TTM_MEMTYPE_FLAG_MAPPABLE
))
1337 switch (mem
->mem_type
) {
1342 #if IS_ENABLED(CONFIG_AGP)
1343 if (drm
->agp
.bridge
) {
1344 mem
->bus
.offset
= mem
->start
<< PAGE_SHIFT
;
1345 mem
->bus
.base
= drm
->agp
.base
;
1346 mem
->bus
.is_iomem
= !drm
->agp
.cma
;
1349 if (drm
->device
.info
.family
< NV_DEVICE_INFO_V0_TESLA
|| !node
->memtype
)
1352 /* fallthrough, tiled memory */
1354 mem
->bus
.offset
= mem
->start
<< PAGE_SHIFT
;
1355 mem
->bus
.base
= device
->func
->resource_addr(device
, 1);
1356 mem
->bus
.is_iomem
= true;
1357 if (drm
->device
.info
.family
>= NV_DEVICE_INFO_V0_TESLA
) {
1358 struct nvkm_bar
*bar
= nvxx_bar(&drm
->device
);
1359 int page_shift
= 12;
1360 if (drm
->device
.info
.family
>= NV_DEVICE_INFO_V0_FERMI
)
1361 page_shift
= node
->page_shift
;
1363 ret
= nvkm_bar_umap(bar
, node
->size
<< 12, page_shift
,
1368 nvkm_vm_map(&node
->bar_vma
, node
);
1369 mem
->bus
.offset
= node
->bar_vma
.offset
;
1379 nouveau_ttm_io_mem_free(struct ttm_bo_device
*bdev
, struct ttm_mem_reg
*mem
)
1381 struct nvkm_mem
*node
= mem
->mm_node
;
1383 if (!node
->bar_vma
.node
)
1386 nvkm_vm_unmap(&node
->bar_vma
);
1387 nvkm_vm_put(&node
->bar_vma
);
1391 nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object
*bo
)
1393 struct nouveau_drm
*drm
= nouveau_bdev(bo
->bdev
);
1394 struct nouveau_bo
*nvbo
= nouveau_bo(bo
);
1395 struct nvkm_device
*device
= nvxx_device(&drm
->device
);
1396 u32 mappable
= device
->func
->resource_size(device
, 1) >> PAGE_SHIFT
;
1399 /* as long as the bo isn't in vram, and isn't tiled, we've got
1400 * nothing to do here.
1402 if (bo
->mem
.mem_type
!= TTM_PL_VRAM
) {
1403 if (drm
->device
.info
.family
< NV_DEVICE_INFO_V0_TESLA
||
1404 !nouveau_bo_tile_layout(nvbo
))
1407 if (bo
->mem
.mem_type
== TTM_PL_SYSTEM
) {
1408 nouveau_bo_placement_set(nvbo
, TTM_PL_TT
, 0);
1410 ret
= nouveau_bo_validate(nvbo
, false, false);
1417 /* make sure bo is in mappable vram */
1418 if (drm
->device
.info
.family
>= NV_DEVICE_INFO_V0_TESLA
||
1419 bo
->mem
.start
+ bo
->mem
.num_pages
< mappable
)
1422 for (i
= 0; i
< nvbo
->placement
.num_placement
; ++i
) {
1423 nvbo
->placements
[i
].fpfn
= 0;
1424 nvbo
->placements
[i
].lpfn
= mappable
;
1427 for (i
= 0; i
< nvbo
->placement
.num_busy_placement
; ++i
) {
1428 nvbo
->busy_placements
[i
].fpfn
= 0;
1429 nvbo
->busy_placements
[i
].lpfn
= mappable
;
1432 nouveau_bo_placement_set(nvbo
, TTM_PL_FLAG_VRAM
, 0);
1433 return nouveau_bo_validate(nvbo
, false, false);
1437 nouveau_ttm_tt_populate(struct ttm_tt
*ttm
)
1439 struct ttm_dma_tt
*ttm_dma
= (void *)ttm
;
1440 struct nouveau_drm
*drm
;
1441 struct nvkm_device
*device
;
1442 struct drm_device
*dev
;
1443 struct device
*pdev
;
1446 bool slave
= !!(ttm
->page_flags
& TTM_PAGE_FLAG_SG
);
1448 if (ttm
->state
!= tt_unpopulated
)
1451 if (slave
&& ttm
->sg
) {
1452 /* make userspace faulting work */
1453 drm_prime_sg_to_page_addr_arrays(ttm
->sg
, ttm
->pages
,
1454 ttm_dma
->dma_address
, ttm
->num_pages
);
1455 ttm
->state
= tt_unbound
;
1459 drm
= nouveau_bdev(ttm
->bdev
);
1460 device
= nvxx_device(&drm
->device
);
1464 #if IS_ENABLED(CONFIG_AGP)
1465 if (drm
->agp
.bridge
) {
1466 return ttm_agp_tt_populate(ttm
);
1470 #if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86)
1471 if (swiotlb_nr_tbl()) {
1472 return ttm_dma_populate((void *)ttm
, dev
->dev
);
1476 r
= ttm_pool_populate(ttm
);
1481 for (i
= 0; i
< ttm
->num_pages
; i
++) {
1484 addr
= dma_map_page(pdev
, ttm
->pages
[i
], 0, PAGE_SIZE
,
1487 if (dma_mapping_error(pdev
, addr
)) {
1489 dma_unmap_page(pdev
, ttm_dma
->dma_address
[i
],
1490 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
1491 ttm_dma
->dma_address
[i
] = 0;
1493 ttm_pool_unpopulate(ttm
);
1497 ttm_dma
->dma_address
[i
] = addr
;
1503 nouveau_ttm_tt_unpopulate(struct ttm_tt
*ttm
)
1505 struct ttm_dma_tt
*ttm_dma
= (void *)ttm
;
1506 struct nouveau_drm
*drm
;
1507 struct nvkm_device
*device
;
1508 struct drm_device
*dev
;
1509 struct device
*pdev
;
1511 bool slave
= !!(ttm
->page_flags
& TTM_PAGE_FLAG_SG
);
1516 drm
= nouveau_bdev(ttm
->bdev
);
1517 device
= nvxx_device(&drm
->device
);
1521 #if IS_ENABLED(CONFIG_AGP)
1522 if (drm
->agp
.bridge
) {
1523 ttm_agp_tt_unpopulate(ttm
);
1528 #if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86)
1529 if (swiotlb_nr_tbl()) {
1530 ttm_dma_unpopulate((void *)ttm
, dev
->dev
);
1535 for (i
= 0; i
< ttm
->num_pages
; i
++) {
1536 if (ttm_dma
->dma_address
[i
]) {
1537 dma_unmap_page(pdev
, ttm_dma
->dma_address
[i
], PAGE_SIZE
,
1542 ttm_pool_unpopulate(ttm
);
1546 nouveau_bo_fence(struct nouveau_bo
*nvbo
, struct nouveau_fence
*fence
, bool exclusive
)
1548 struct reservation_object
*resv
= nvbo
->bo
.resv
;
1551 reservation_object_add_excl_fence(resv
, &fence
->base
);
1553 reservation_object_add_shared_fence(resv
, &fence
->base
);
1556 struct ttm_bo_driver nouveau_bo_driver
= {
1557 .ttm_tt_create
= &nouveau_ttm_tt_create
,
1558 .ttm_tt_populate
= &nouveau_ttm_tt_populate
,
1559 .ttm_tt_unpopulate
= &nouveau_ttm_tt_unpopulate
,
1560 .invalidate_caches
= nouveau_bo_invalidate_caches
,
1561 .init_mem_type
= nouveau_bo_init_mem_type
,
1562 .evict_flags
= nouveau_bo_evict_flags
,
1563 .move_notify
= nouveau_bo_move_ntfy
,
1564 .move
= nouveau_bo_move
,
1565 .verify_access
= nouveau_bo_verify_access
,
1566 .fault_reserve_notify
= &nouveau_ttm_fault_reserve_notify
,
1567 .io_mem_reserve
= &nouveau_ttm_io_mem_reserve
,
1568 .io_mem_free
= &nouveau_ttm_io_mem_free
,
1569 .lru_tail
= &ttm_bo_default_lru_tail
,
1570 .swap_lru_tail
= &ttm_bo_default_swap_lru_tail
,
1574 nouveau_bo_vma_find(struct nouveau_bo
*nvbo
, struct nvkm_vm
*vm
)
1576 struct nvkm_vma
*vma
;
1577 list_for_each_entry(vma
, &nvbo
->vma_list
, head
) {
1586 nouveau_bo_vma_add(struct nouveau_bo
*nvbo
, struct nvkm_vm
*vm
,
1587 struct nvkm_vma
*vma
)
1589 const u32 size
= nvbo
->bo
.mem
.num_pages
<< PAGE_SHIFT
;
1592 ret
= nvkm_vm_get(vm
, size
, nvbo
->page_shift
,
1593 NV_MEM_ACCESS_RW
, vma
);
1597 if ( nvbo
->bo
.mem
.mem_type
!= TTM_PL_SYSTEM
&&
1598 (nvbo
->bo
.mem
.mem_type
== TTM_PL_VRAM
||
1599 nvbo
->page_shift
!= vma
->vm
->mmu
->lpg_shift
))
1600 nvkm_vm_map(vma
, nvbo
->bo
.mem
.mm_node
);
1602 list_add_tail(&vma
->head
, &nvbo
->vma_list
);
1608 nouveau_bo_vma_del(struct nouveau_bo
*nvbo
, struct nvkm_vma
*vma
)
1611 if (nvbo
->bo
.mem
.mem_type
!= TTM_PL_SYSTEM
)
1614 list_del(&vma
->head
);