2 * Copyright (C) 2007 Ben Skeggs.
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sublicense, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial
16 * portions of the Software.
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
30 #include "nouveau_drv.h"
32 struct nv50_instmem_priv
{
33 uint32_t save1700
[5]; /* 0x1700->0x1710 */
35 struct nouveau_gpuobj_ref
*pramin_pt
;
36 struct nouveau_gpuobj_ref
*pramin_bar
;
37 struct nouveau_gpuobj_ref
*fb_bar
;
40 #define NV50_INSTMEM_PAGE_SHIFT 12
41 #define NV50_INSTMEM_PAGE_SIZE (1 << NV50_INSTMEM_PAGE_SHIFT)
42 #define NV50_INSTMEM_PT_SIZE(a) (((a) >> 12) << 3)
44 /*NOTE: - Assumes 0x1700 already covers the correct MiB of PRAMIN
46 #define BAR0_WI32(g, o, v) do { \
48 if ((g)->im_backing) { \
49 offset = (g)->im_backing_start; \
51 offset = chan->ramin->gpuobj->im_backing_start; \
52 offset += (g)->im_pramin->start; \
55 nv_wr32(dev, NV_RAMIN + (offset & 0xfffff), (v)); \
59 nv50_instmem_init(struct drm_device
*dev
)
61 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
62 struct nouveau_channel
*chan
;
63 uint32_t c_offset
, c_size
, c_ramfc
, c_vmpd
, c_base
, pt_size
;
64 uint32_t save_nv001700
;
66 struct nv50_instmem_priv
*priv
;
69 priv
= kzalloc(sizeof(*priv
), GFP_KERNEL
);
72 dev_priv
->engine
.instmem
.priv
= priv
;
74 /* Save state, will restore at takedown. */
75 for (i
= 0x1700; i
<= 0x1710; i
+= 4)
76 priv
->save1700
[(i
-0x1700)/4] = nv_rd32(dev
, i
);
78 /* Reserve the last MiB of VRAM, we should probably try to avoid
79 * setting up the below tables over the top of the VBIOS image at
82 dev_priv
->ramin_rsvd_vram
= 1 << 20;
83 c_offset
= dev_priv
->vram_size
- dev_priv
->ramin_rsvd_vram
;
85 c_vmpd
= ((dev_priv
->chipset
& 0xf0) == 0x50) ? 0x1400 : 0x200;
86 c_ramfc
= ((dev_priv
->chipset
& 0xf0) == 0x50) ? 0x0 : 0x20;
87 c_base
= c_vmpd
+ 0x4000;
88 pt_size
= NV50_INSTMEM_PT_SIZE(dev_priv
->ramin_size
);
90 NV_DEBUG(dev
, " Rsvd VRAM base: 0x%08x\n", c_offset
);
91 NV_DEBUG(dev
, " VBIOS image: 0x%08x\n",
92 (nv_rd32(dev
, 0x619f04) & ~0xff) << 8);
93 NV_DEBUG(dev
, " Aperture size: %d MiB\n", dev_priv
->ramin_size
>> 20);
94 NV_DEBUG(dev
, " PT size: %d KiB\n", pt_size
>> 10);
96 /* Determine VM layout, we need to do this first to make sure
97 * we allocate enough memory for all the page tables.
99 dev_priv
->vm_gart_base
= roundup(NV50_VM_BLOCK
, NV50_VM_BLOCK
);
100 dev_priv
->vm_gart_size
= NV50_VM_BLOCK
;
102 dev_priv
->vm_vram_base
= dev_priv
->vm_gart_base
+ dev_priv
->vm_gart_size
;
103 dev_priv
->vm_vram_size
= dev_priv
->vram_size
;
104 if (dev_priv
->vm_vram_size
> NV50_VM_MAX_VRAM
)
105 dev_priv
->vm_vram_size
= NV50_VM_MAX_VRAM
;
106 dev_priv
->vm_vram_size
= roundup(dev_priv
->vm_vram_size
, NV50_VM_BLOCK
);
107 dev_priv
->vm_vram_pt_nr
= dev_priv
->vm_vram_size
/ NV50_VM_BLOCK
;
109 dev_priv
->vm_end
= dev_priv
->vm_vram_base
+ dev_priv
->vm_vram_size
;
111 NV_DEBUG(dev
, "NV50VM: GART 0x%016llx-0x%016llx\n",
112 dev_priv
->vm_gart_base
,
113 dev_priv
->vm_gart_base
+ dev_priv
->vm_gart_size
- 1);
114 NV_DEBUG(dev
, "NV50VM: VRAM 0x%016llx-0x%016llx\n",
115 dev_priv
->vm_vram_base
,
116 dev_priv
->vm_vram_base
+ dev_priv
->vm_vram_size
- 1);
118 c_size
+= dev_priv
->vm_vram_pt_nr
* (NV50_VM_BLOCK
/ 65536 * 8);
120 /* Map BAR0 PRAMIN aperture over the memory we want to use */
121 save_nv001700
= nv_rd32(dev
, NV50_PUNK_BAR0_PRAMIN
);
122 nv_wr32(dev
, NV50_PUNK_BAR0_PRAMIN
, (c_offset
>> 16));
124 /* Create a fake channel, and use it as our "dummy" channels 0/127.
125 * The main reason for creating a channel is so we can use the gpuobj
126 * code. However, it's probably worth noting that NVIDIA also setup
127 * their channels 0/127 with the same values they configure here.
128 * So, there may be some other reason for doing this.
130 * Have to create the entire channel manually, as the real channel
131 * creation code assumes we have PRAMIN access, and we don't until
134 chan
= kzalloc(sizeof(*chan
), GFP_KERNEL
);
139 chan
->file_priv
= (struct drm_file
*)-2;
140 dev_priv
->fifos
[0] = dev_priv
->fifos
[127] = chan
;
142 INIT_LIST_HEAD(&chan
->ramht_refs
);
144 /* Channel's PRAMIN object + heap */
145 ret
= nouveau_gpuobj_new_fake(dev
, 0, c_offset
, c_size
, 0,
150 if (drm_mm_init(&chan
->ramin_heap
, c_base
, c_size
- c_base
))
153 /* RAMFC + zero channel's PRAMIN up to start of VM pagedir */
154 ret
= nouveau_gpuobj_new_fake(dev
, c_ramfc
, c_offset
+ c_ramfc
,
155 0x4000, 0, NULL
, &chan
->ramfc
);
159 for (i
= 0; i
< c_vmpd
; i
+= 4)
160 BAR0_WI32(chan
->ramin
->gpuobj
, i
, 0);
162 /* VM page directory */
163 ret
= nouveau_gpuobj_new_fake(dev
, c_vmpd
, c_offset
+ c_vmpd
,
164 0x4000, 0, &chan
->vm_pd
, NULL
);
167 for (i
= 0; i
< 0x4000; i
+= 8) {
168 BAR0_WI32(chan
->vm_pd
, i
+ 0x00, 0x00000000);
169 BAR0_WI32(chan
->vm_pd
, i
+ 0x04, 0x00000000);
172 /* PRAMIN page table, cheat and map into VM at 0x0000000000.
173 * We map the entire fake channel into the start of the PRAMIN BAR
175 ret
= nouveau_gpuobj_new_ref(dev
, chan
, NULL
, 0, pt_size
, 0x1000,
176 0, &priv
->pramin_pt
);
181 if (dev_priv
->vram_sys_base
) {
182 v
+= dev_priv
->vram_sys_base
;
187 while (v
< dev_priv
->vram_sys_base
+ c_offset
+ c_size
) {
188 BAR0_WI32(priv
->pramin_pt
->gpuobj
, i
+ 0, lower_32_bits(v
));
189 BAR0_WI32(priv
->pramin_pt
->gpuobj
, i
+ 4, upper_32_bits(v
));
194 while (i
< pt_size
) {
195 BAR0_WI32(priv
->pramin_pt
->gpuobj
, i
+ 0, 0x00000000);
196 BAR0_WI32(priv
->pramin_pt
->gpuobj
, i
+ 4, 0x00000000);
200 BAR0_WI32(chan
->vm_pd
, 0x00, priv
->pramin_pt
->instance
| 0x63);
201 BAR0_WI32(chan
->vm_pd
, 0x04, 0x00000000);
203 /* VRAM page table(s), mapped into VM at +1GiB */
204 for (i
= 0; i
< dev_priv
->vm_vram_pt_nr
; i
++) {
205 ret
= nouveau_gpuobj_new_ref(dev
, chan
, NULL
, 0,
206 NV50_VM_BLOCK
/65536*8, 0, 0,
207 &chan
->vm_vram_pt
[i
]);
209 NV_ERROR(dev
, "Error creating VRAM page tables: %d\n",
211 dev_priv
->vm_vram_pt_nr
= i
;
214 dev_priv
->vm_vram_pt
[i
] = chan
->vm_vram_pt
[i
]->gpuobj
;
216 for (v
= 0; v
< dev_priv
->vm_vram_pt
[i
]->im_pramin
->size
;
218 BAR0_WI32(dev_priv
->vm_vram_pt
[i
], v
, 0);
220 BAR0_WI32(chan
->vm_pd
, 0x10 + (i
*8),
221 chan
->vm_vram_pt
[i
]->instance
| 0x61);
222 BAR0_WI32(chan
->vm_pd
, 0x14 + (i
*8), 0);
225 /* DMA object for PRAMIN BAR */
226 ret
= nouveau_gpuobj_new_ref(dev
, chan
, chan
, 0, 6*4, 16, 0,
230 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x00, 0x7fc00000);
231 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x04, dev_priv
->ramin_size
- 1);
232 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x08, 0x00000000);
233 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x0c, 0x00000000);
234 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x10, 0x00000000);
235 BAR0_WI32(priv
->pramin_bar
->gpuobj
, 0x14, 0x00000000);
237 /* DMA object for FB BAR */
238 ret
= nouveau_gpuobj_new_ref(dev
, chan
, chan
, 0, 6*4, 16, 0,
242 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x00, 0x7fc00000);
243 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x04, 0x40000000 +
244 pci_resource_len(dev
->pdev
, 1) - 1);
245 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x08, 0x40000000);
246 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x0c, 0x00000000);
247 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x10, 0x00000000);
248 BAR0_WI32(priv
->fb_bar
->gpuobj
, 0x14, 0x00000000);
250 /* Poke the relevant regs, and pray it works :) */
251 nv_wr32(dev
, NV50_PUNK_BAR_CFG_BASE
, (chan
->ramin
->instance
>> 12));
252 nv_wr32(dev
, NV50_PUNK_UNK1710
, 0);
253 nv_wr32(dev
, NV50_PUNK_BAR_CFG_BASE
, (chan
->ramin
->instance
>> 12) |
254 NV50_PUNK_BAR_CFG_BASE_VALID
);
255 nv_wr32(dev
, NV50_PUNK_BAR1_CTXDMA
, (priv
->fb_bar
->instance
>> 4) |
256 NV50_PUNK_BAR1_CTXDMA_VALID
);
257 nv_wr32(dev
, NV50_PUNK_BAR3_CTXDMA
, (priv
->pramin_bar
->instance
>> 4) |
258 NV50_PUNK_BAR3_CTXDMA_VALID
);
260 for (i
= 0; i
< 8; i
++)
261 nv_wr32(dev
, 0x1900 + (i
*4), 0);
263 /* Assume that praying isn't enough, check that we can re-read the
264 * entire fake channel back from the PRAMIN BAR */
265 for (i
= 0; i
< c_size
; i
+= 4) {
266 if (nv_rd32(dev
, NV_RAMIN
+ i
) != nv_ri32(dev
, i
)) {
267 NV_ERROR(dev
, "Error reading back PRAMIN at 0x%08x\n",
273 nv_wr32(dev
, NV50_PUNK_BAR0_PRAMIN
, save_nv001700
);
275 /* Global PRAMIN heap */
276 if (drm_mm_init(&dev_priv
->ramin_heap
, c_size
, dev_priv
->ramin_size
- c_size
)) {
277 NV_ERROR(dev
, "Failed to init RAMIN heap\n");
280 /*XXX: incorrect, but needed to make hash func "work" */
281 dev_priv
->ramht_offset
= 0x10000;
282 dev_priv
->ramht_bits
= 9;
283 dev_priv
->ramht_size
= (1 << dev_priv
->ramht_bits
) * 8;
288 nv50_instmem_takedown(struct drm_device
*dev
)
290 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
291 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
292 struct nouveau_channel
*chan
= dev_priv
->fifos
[0];
300 /* Restore state from before init */
301 for (i
= 0x1700; i
<= 0x1710; i
+= 4)
302 nv_wr32(dev
, i
, priv
->save1700
[(i
- 0x1700) / 4]);
304 nouveau_gpuobj_ref_del(dev
, &priv
->fb_bar
);
305 nouveau_gpuobj_ref_del(dev
, &priv
->pramin_bar
);
306 nouveau_gpuobj_ref_del(dev
, &priv
->pramin_pt
);
308 /* Destroy dummy channel */
310 for (i
= 0; i
< dev_priv
->vm_vram_pt_nr
; i
++) {
311 nouveau_gpuobj_ref_del(dev
, &chan
->vm_vram_pt
[i
]);
312 dev_priv
->vm_vram_pt
[i
] = NULL
;
314 dev_priv
->vm_vram_pt_nr
= 0;
316 nouveau_gpuobj_del(dev
, &chan
->vm_pd
);
317 nouveau_gpuobj_ref_del(dev
, &chan
->ramfc
);
318 nouveau_gpuobj_ref_del(dev
, &chan
->ramin
);
319 drm_mm_takedown(&chan
->ramin_heap
);
321 dev_priv
->fifos
[0] = dev_priv
->fifos
[127] = NULL
;
325 dev_priv
->engine
.instmem
.priv
= NULL
;
330 nv50_instmem_suspend(struct drm_device
*dev
)
332 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
333 struct nouveau_channel
*chan
= dev_priv
->fifos
[0];
334 struct nouveau_gpuobj
*ramin
= chan
->ramin
->gpuobj
;
337 ramin
->im_backing_suspend
= vmalloc(ramin
->im_pramin
->size
);
338 if (!ramin
->im_backing_suspend
)
341 for (i
= 0; i
< ramin
->im_pramin
->size
; i
+= 4)
342 ramin
->im_backing_suspend
[i
/4] = nv_ri32(dev
, i
);
347 nv50_instmem_resume(struct drm_device
*dev
)
349 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
350 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
351 struct nouveau_channel
*chan
= dev_priv
->fifos
[0];
352 struct nouveau_gpuobj
*ramin
= chan
->ramin
->gpuobj
;
355 nv_wr32(dev
, NV50_PUNK_BAR0_PRAMIN
, (ramin
->im_backing_start
>> 16));
356 for (i
= 0; i
< ramin
->im_pramin
->size
; i
+= 4)
357 BAR0_WI32(ramin
, i
, ramin
->im_backing_suspend
[i
/4]);
358 vfree(ramin
->im_backing_suspend
);
359 ramin
->im_backing_suspend
= NULL
;
361 /* Poke the relevant regs, and pray it works :) */
362 nv_wr32(dev
, NV50_PUNK_BAR_CFG_BASE
, (chan
->ramin
->instance
>> 12));
363 nv_wr32(dev
, NV50_PUNK_UNK1710
, 0);
364 nv_wr32(dev
, NV50_PUNK_BAR_CFG_BASE
, (chan
->ramin
->instance
>> 12) |
365 NV50_PUNK_BAR_CFG_BASE_VALID
);
366 nv_wr32(dev
, NV50_PUNK_BAR1_CTXDMA
, (priv
->fb_bar
->instance
>> 4) |
367 NV50_PUNK_BAR1_CTXDMA_VALID
);
368 nv_wr32(dev
, NV50_PUNK_BAR3_CTXDMA
, (priv
->pramin_bar
->instance
>> 4) |
369 NV50_PUNK_BAR3_CTXDMA_VALID
);
371 for (i
= 0; i
< 8; i
++)
372 nv_wr32(dev
, 0x1900 + (i
*4), 0);
376 nv50_instmem_populate(struct drm_device
*dev
, struct nouveau_gpuobj
*gpuobj
,
381 if (gpuobj
->im_backing
)
384 *sz
= ALIGN(*sz
, NV50_INSTMEM_PAGE_SIZE
);
388 ret
= nouveau_bo_new(dev
, NULL
, *sz
, 0, TTM_PL_FLAG_VRAM
, 0, 0x0000,
389 true, false, &gpuobj
->im_backing
);
391 NV_ERROR(dev
, "error getting PRAMIN backing pages: %d\n", ret
);
395 ret
= nouveau_bo_pin(gpuobj
->im_backing
, TTM_PL_FLAG_VRAM
);
397 NV_ERROR(dev
, "error pinning PRAMIN backing VRAM: %d\n", ret
);
398 nouveau_bo_ref(NULL
, &gpuobj
->im_backing
);
402 gpuobj
->im_backing_start
= gpuobj
->im_backing
->bo
.mem
.mm_node
->start
;
403 gpuobj
->im_backing_start
<<= PAGE_SHIFT
;
409 nv50_instmem_clear(struct drm_device
*dev
, struct nouveau_gpuobj
*gpuobj
)
411 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
413 if (gpuobj
&& gpuobj
->im_backing
) {
414 if (gpuobj
->im_bound
)
415 dev_priv
->engine
.instmem
.unbind(dev
, gpuobj
);
416 nouveau_bo_unpin(gpuobj
->im_backing
);
417 nouveau_bo_ref(NULL
, &gpuobj
->im_backing
);
418 gpuobj
->im_backing
= NULL
;
423 nv50_instmem_bind(struct drm_device
*dev
, struct nouveau_gpuobj
*gpuobj
)
425 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
426 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
427 struct nouveau_gpuobj
*pramin_pt
= priv
->pramin_pt
->gpuobj
;
428 uint32_t pte
, pte_end
;
431 if (!gpuobj
->im_backing
|| !gpuobj
->im_pramin
|| gpuobj
->im_bound
)
434 NV_DEBUG(dev
, "st=0x%lx sz=0x%lx\n",
435 gpuobj
->im_pramin
->start
, gpuobj
->im_pramin
->size
);
437 pte
= (gpuobj
->im_pramin
->start
>> 12) << 1;
438 pte_end
= ((gpuobj
->im_pramin
->size
>> 12) << 1) + pte
;
439 vram
= gpuobj
->im_backing_start
;
441 NV_DEBUG(dev
, "pramin=0x%lx, pte=%d, pte_end=%d\n",
442 gpuobj
->im_pramin
->start
, pte
, pte_end
);
443 NV_DEBUG(dev
, "first vram page: 0x%08x\n", gpuobj
->im_backing_start
);
446 if (dev_priv
->vram_sys_base
) {
447 vram
+= dev_priv
->vram_sys_base
;
451 while (pte
< pte_end
) {
452 nv_wo32(dev
, pramin_pt
, pte
++, lower_32_bits(vram
));
453 nv_wo32(dev
, pramin_pt
, pte
++, upper_32_bits(vram
));
454 vram
+= NV50_INSTMEM_PAGE_SIZE
;
456 dev_priv
->engine
.instmem
.flush(dev
);
458 nv50_vm_flush(dev
, 4);
459 nv50_vm_flush(dev
, 6);
461 gpuobj
->im_bound
= 1;
466 nv50_instmem_unbind(struct drm_device
*dev
, struct nouveau_gpuobj
*gpuobj
)
468 struct drm_nouveau_private
*dev_priv
= dev
->dev_private
;
469 struct nv50_instmem_priv
*priv
= dev_priv
->engine
.instmem
.priv
;
470 uint32_t pte
, pte_end
;
472 if (gpuobj
->im_bound
== 0)
475 pte
= (gpuobj
->im_pramin
->start
>> 12) << 1;
476 pte_end
= ((gpuobj
->im_pramin
->size
>> 12) << 1) + pte
;
478 while (pte
< pte_end
) {
479 nv_wo32(dev
, priv
->pramin_pt
->gpuobj
, pte
++, 0x00000000);
480 nv_wo32(dev
, priv
->pramin_pt
->gpuobj
, pte
++, 0x00000000);
482 dev_priv
->engine
.instmem
.flush(dev
);
484 gpuobj
->im_bound
= 0;
489 nv50_instmem_flush(struct drm_device
*dev
)
491 nv_wr32(dev
, 0x00330c, 0x00000001);
492 if (!nv_wait(0x00330c, 0x00000002, 0x00000000))
493 NV_ERROR(dev
, "PRAMIN flush timeout\n");
497 nv84_instmem_flush(struct drm_device
*dev
)
499 nv_wr32(dev
, 0x070000, 0x00000001);
500 if (!nv_wait(0x070000, 0x00000002, 0x00000000))
501 NV_ERROR(dev
, "PRAMIN flush timeout\n");
505 nv50_vm_flush(struct drm_device
*dev
, int engine
)
507 nv_wr32(dev
, 0x100c80, (engine
<< 16) | 1);
508 if (!nv_wait(0x100c80, 0x00000001, 0x00000000))
509 NV_ERROR(dev
, "vm flush timeout: engine %d\n", engine
);