2 * Copyright 2012 Red Hat Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
28 #include <core/client.h>
29 #include <core/handle.h>
30 #include <core/ramht.h>
31 #include <subdev/instmem.h>
32 #include <subdev/timer.h>
33 #include <engine/sw.h>
35 static const struct nv04_fifo_ramfc
37 { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT
},
38 { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET
},
39 { 16, 0, 0x08, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE
},
40 { 16, 16, 0x08, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT
},
41 { 32, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_STATE
},
42 { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_FETCH
},
43 { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_ENGINE
},
44 { 32, 0, 0x18, 0, NV04_PFIFO_CACHE1_PULL1
},
49 nv04_fifo_pause(struct nvkm_fifo
*base
, unsigned long *pflags
)
50 __acquires(fifo
->base
.lock
)
52 struct nv04_fifo
*fifo
= nv04_fifo(base
);
53 struct nvkm_device
*device
= fifo
->base
.engine
.subdev
.device
;
56 spin_lock_irqsave(&fifo
->base
.lock
, flags
);
59 nvkm_wr32(device
, NV03_PFIFO_CACHES
, 0x00000000);
60 nvkm_mask(device
, NV04_PFIFO_CACHE1_PULL0
, 0x00000001, 0x00000000);
62 /* in some cases the puller may be left in an inconsistent state
63 * if you try to stop it while it's busy translating handles.
64 * sometimes you get a CACHE_ERROR, sometimes it just fails
65 * silently; sending incorrect instance offsets to PGRAPH after
66 * it's started up again.
68 * to avoid this, we invalidate the most recently calculated
71 nvkm_msec(device
, 2000,
72 u32 tmp
= nvkm_rd32(device
, NV04_PFIFO_CACHE1_PULL0
);
73 if (!(tmp
& NV04_PFIFO_CACHE1_PULL0_HASH_BUSY
))
77 if (nvkm_rd32(device
, NV04_PFIFO_CACHE1_PULL0
) &
78 NV04_PFIFO_CACHE1_PULL0_HASH_FAILED
)
79 nvkm_wr32(device
, NV03_PFIFO_INTR_0
, NV_PFIFO_INTR_CACHE_ERROR
);
81 nvkm_wr32(device
, NV04_PFIFO_CACHE1_HASH
, 0x00000000);
85 nv04_fifo_start(struct nvkm_fifo
*base
, unsigned long *pflags
)
86 __releases(fifo
->base
.lock
)
88 struct nv04_fifo
*fifo
= nv04_fifo(base
);
89 struct nvkm_device
*device
= fifo
->base
.engine
.subdev
.device
;
90 unsigned long flags
= *pflags
;
92 nvkm_mask(device
, NV04_PFIFO_CACHE1_PULL0
, 0x00000001, 0x00000001);
93 nvkm_wr32(device
, NV03_PFIFO_CACHES
, 0x00000001);
95 spin_unlock_irqrestore(&fifo
->base
.lock
, flags
);
99 nv_dma_state_err(u32 state
)
101 static const char * const desc
[] = {
102 "NONE", "CALL_SUBR_ACTIVE", "INVALID_MTHD", "RET_SUBR_INACTIVE",
103 "INVALID_CMD", "IB_EMPTY"/* NV50+ */, "MEM_FAULT", "UNK"
105 return desc
[(state
>> 29) & 0x7];
109 nv04_fifo_swmthd(struct nvkm_device
*device
, u32 chid
, u32 addr
, u32 data
)
111 struct nvkm_sw
*sw
= device
->sw
;
112 const int subc
= (addr
& 0x0000e000) >> 13;
113 const int mthd
= (addr
& 0x00001ffc);
114 const u32 mask
= 0x0000000f << (subc
* 4);
115 u32 engine
= nvkm_rd32(device
, 0x003280);
116 bool handled
= false;
119 case 0x0000 ... 0x0000: /* subchannel's engine -> software */
120 nvkm_wr32(device
, 0x003280, (engine
&= ~mask
));
121 case 0x0180 ... 0x01fc: /* handle -> instance */
122 data
= nvkm_rd32(device
, 0x003258) & 0x0000ffff;
123 case 0x0100 ... 0x017c:
124 case 0x0200 ... 0x1ffc: /* pass method down to sw */
125 if (!(engine
& mask
) && sw
)
126 handled
= nvkm_sw_mthd(sw
, chid
, subc
, mthd
, data
);
136 nv04_fifo_cache_error(struct nv04_fifo
*fifo
, u32 chid
, u32 get
)
138 struct nvkm_subdev
*subdev
= &fifo
->base
.engine
.subdev
;
139 struct nvkm_device
*device
= subdev
->device
;
140 struct nvkm_fifo_chan
*chan
;
142 u32 pull0
= nvkm_rd32(device
, 0x003250);
146 /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before wrapping on my
147 * G80 chips, but CACHE1 isn't big enough for this much data.. Tests
148 * show that it wraps around to the start at GET=0x800.. No clue as to
151 ptr
= (get
& 0x7ff) >> 2;
153 if (device
->card_type
< NV_40
) {
154 mthd
= nvkm_rd32(device
, NV04_PFIFO_CACHE1_METHOD(ptr
));
155 data
= nvkm_rd32(device
, NV04_PFIFO_CACHE1_DATA(ptr
));
157 mthd
= nvkm_rd32(device
, NV40_PFIFO_CACHE1_METHOD(ptr
));
158 data
= nvkm_rd32(device
, NV40_PFIFO_CACHE1_DATA(ptr
));
161 if (!(pull0
& 0x00000100) ||
162 !nv04_fifo_swmthd(device
, chid
, mthd
, data
)) {
163 chan
= nvkm_fifo_chan_chid(&fifo
->base
, chid
, &flags
);
164 nvkm_error(subdev
, "CACHE_ERROR - "
165 "ch %d [%s] subc %d mthd %04x data %08x\n",
166 chid
, chan
? chan
->object
.client
->name
: "unknown",
167 (mthd
>> 13) & 7, mthd
& 0x1ffc, data
);
168 nvkm_fifo_chan_put(&fifo
->base
, flags
, &chan
);
171 nvkm_wr32(device
, NV04_PFIFO_CACHE1_DMA_PUSH
, 0);
172 nvkm_wr32(device
, NV03_PFIFO_INTR_0
, NV_PFIFO_INTR_CACHE_ERROR
);
174 nvkm_wr32(device
, NV03_PFIFO_CACHE1_PUSH0
,
175 nvkm_rd32(device
, NV03_PFIFO_CACHE1_PUSH0
) & ~1);
176 nvkm_wr32(device
, NV03_PFIFO_CACHE1_GET
, get
+ 4);
177 nvkm_wr32(device
, NV03_PFIFO_CACHE1_PUSH0
,
178 nvkm_rd32(device
, NV03_PFIFO_CACHE1_PUSH0
) | 1);
179 nvkm_wr32(device
, NV04_PFIFO_CACHE1_HASH
, 0);
181 nvkm_wr32(device
, NV04_PFIFO_CACHE1_DMA_PUSH
,
182 nvkm_rd32(device
, NV04_PFIFO_CACHE1_DMA_PUSH
) | 1);
183 nvkm_wr32(device
, NV04_PFIFO_CACHE1_PULL0
, 1);
187 nv04_fifo_dma_pusher(struct nv04_fifo
*fifo
, u32 chid
)
189 struct nvkm_subdev
*subdev
= &fifo
->base
.engine
.subdev
;
190 struct nvkm_device
*device
= subdev
->device
;
191 u32 dma_get
= nvkm_rd32(device
, 0x003244);
192 u32 dma_put
= nvkm_rd32(device
, 0x003240);
193 u32 push
= nvkm_rd32(device
, 0x003220);
194 u32 state
= nvkm_rd32(device
, 0x003228);
195 struct nvkm_fifo_chan
*chan
;
199 chan
= nvkm_fifo_chan_chid(&fifo
->base
, chid
, &flags
);
200 name
= chan
? chan
->object
.client
->name
: "unknown";
201 if (device
->card_type
== NV_50
) {
202 u32 ho_get
= nvkm_rd32(device
, 0x003328);
203 u32 ho_put
= nvkm_rd32(device
, 0x003320);
204 u32 ib_get
= nvkm_rd32(device
, 0x003334);
205 u32 ib_put
= nvkm_rd32(device
, 0x003330);
207 nvkm_error(subdev
, "DMA_PUSHER - "
208 "ch %d [%s] get %02x%08x put %02x%08x ib_get %08x "
209 "ib_put %08x state %08x (err: %s) push %08x\n",
210 chid
, name
, ho_get
, dma_get
, ho_put
, dma_put
,
211 ib_get
, ib_put
, state
, nv_dma_state_err(state
),
214 /* METHOD_COUNT, in DMA_STATE on earlier chipsets */
215 nvkm_wr32(device
, 0x003364, 0x00000000);
216 if (dma_get
!= dma_put
|| ho_get
!= ho_put
) {
217 nvkm_wr32(device
, 0x003244, dma_put
);
218 nvkm_wr32(device
, 0x003328, ho_put
);
220 if (ib_get
!= ib_put
)
221 nvkm_wr32(device
, 0x003334, ib_put
);
223 nvkm_error(subdev
, "DMA_PUSHER - ch %d [%s] get %08x put %08x "
224 "state %08x (err: %s) push %08x\n",
225 chid
, name
, dma_get
, dma_put
, state
,
226 nv_dma_state_err(state
), push
);
228 if (dma_get
!= dma_put
)
229 nvkm_wr32(device
, 0x003244, dma_put
);
231 nvkm_fifo_chan_put(&fifo
->base
, flags
, &chan
);
233 nvkm_wr32(device
, 0x003228, 0x00000000);
234 nvkm_wr32(device
, 0x003220, 0x00000001);
235 nvkm_wr32(device
, 0x002100, NV_PFIFO_INTR_DMA_PUSHER
);
239 nv04_fifo_intr(struct nvkm_fifo
*base
)
241 struct nv04_fifo
*fifo
= nv04_fifo(base
);
242 struct nvkm_subdev
*subdev
= &fifo
->base
.engine
.subdev
;
243 struct nvkm_device
*device
= subdev
->device
;
244 u32 mask
= nvkm_rd32(device
, NV03_PFIFO_INTR_EN_0
);
245 u32 stat
= nvkm_rd32(device
, NV03_PFIFO_INTR_0
) & mask
;
246 u32 reassign
, chid
, get
, sem
;
248 reassign
= nvkm_rd32(device
, NV03_PFIFO_CACHES
) & 1;
249 nvkm_wr32(device
, NV03_PFIFO_CACHES
, 0);
251 chid
= nvkm_rd32(device
, NV03_PFIFO_CACHE1_PUSH1
) & (fifo
->base
.nr
- 1);
252 get
= nvkm_rd32(device
, NV03_PFIFO_CACHE1_GET
);
254 if (stat
& NV_PFIFO_INTR_CACHE_ERROR
) {
255 nv04_fifo_cache_error(fifo
, chid
, get
);
256 stat
&= ~NV_PFIFO_INTR_CACHE_ERROR
;
259 if (stat
& NV_PFIFO_INTR_DMA_PUSHER
) {
260 nv04_fifo_dma_pusher(fifo
, chid
);
261 stat
&= ~NV_PFIFO_INTR_DMA_PUSHER
;
264 if (stat
& NV_PFIFO_INTR_SEMAPHORE
) {
265 stat
&= ~NV_PFIFO_INTR_SEMAPHORE
;
266 nvkm_wr32(device
, NV03_PFIFO_INTR_0
, NV_PFIFO_INTR_SEMAPHORE
);
268 sem
= nvkm_rd32(device
, NV10_PFIFO_CACHE1_SEMAPHORE
);
269 nvkm_wr32(device
, NV10_PFIFO_CACHE1_SEMAPHORE
, sem
| 0x1);
271 nvkm_wr32(device
, NV03_PFIFO_CACHE1_GET
, get
+ 4);
272 nvkm_wr32(device
, NV04_PFIFO_CACHE1_PULL0
, 1);
275 if (device
->card_type
== NV_50
) {
276 if (stat
& 0x00000010) {
278 nvkm_wr32(device
, 0x002100, 0x00000010);
281 if (stat
& 0x40000000) {
282 nvkm_wr32(device
, 0x002100, 0x40000000);
283 nvkm_fifo_uevent(&fifo
->base
);
289 nvkm_warn(subdev
, "intr %08x\n", stat
);
290 nvkm_mask(device
, NV03_PFIFO_INTR_EN_0
, stat
, 0x00000000);
291 nvkm_wr32(device
, NV03_PFIFO_INTR_0
, stat
);
294 nvkm_wr32(device
, NV03_PFIFO_CACHES
, reassign
);
298 nv04_fifo_init(struct nvkm_fifo
*base
)
300 struct nv04_fifo
*fifo
= nv04_fifo(base
);
301 struct nvkm_device
*device
= fifo
->base
.engine
.subdev
.device
;
302 struct nvkm_instmem
*imem
= device
->imem
;
303 struct nvkm_ramht
*ramht
= imem
->ramht
;
304 struct nvkm_memory
*ramro
= imem
->ramro
;
305 struct nvkm_memory
*ramfc
= imem
->ramfc
;
307 nvkm_wr32(device
, NV04_PFIFO_DELAY_0
, 0x000000ff);
308 nvkm_wr32(device
, NV04_PFIFO_DMA_TIMESLICE
, 0x0101ffff);
310 nvkm_wr32(device
, NV03_PFIFO_RAMHT
, (0x03 << 24) /* search 128 */ |
311 ((ramht
->bits
- 9) << 16) |
312 (ramht
->gpuobj
->addr
>> 8));
313 nvkm_wr32(device
, NV03_PFIFO_RAMRO
, nvkm_memory_addr(ramro
) >> 8);
314 nvkm_wr32(device
, NV03_PFIFO_RAMFC
, nvkm_memory_addr(ramfc
) >> 8);
316 nvkm_wr32(device
, NV03_PFIFO_CACHE1_PUSH1
, fifo
->base
.nr
- 1);
318 nvkm_wr32(device
, NV03_PFIFO_INTR_0
, 0xffffffff);
319 nvkm_wr32(device
, NV03_PFIFO_INTR_EN_0
, 0xffffffff);
321 nvkm_wr32(device
, NV03_PFIFO_CACHE1_PUSH0
, 1);
322 nvkm_wr32(device
, NV04_PFIFO_CACHE1_PULL0
, 1);
323 nvkm_wr32(device
, NV03_PFIFO_CACHES
, 1);
327 nv04_fifo_new_(const struct nvkm_fifo_func
*func
, struct nvkm_device
*device
,
328 int index
, int nr
, const struct nv04_fifo_ramfc
*ramfc
,
329 struct nvkm_fifo
**pfifo
)
331 struct nv04_fifo
*fifo
;
334 if (!(fifo
= kzalloc(sizeof(*fifo
), GFP_KERNEL
)))
337 *pfifo
= &fifo
->base
;
339 ret
= nvkm_fifo_ctor(func
, device
, index
, nr
, &fifo
->base
);
343 set_bit(nr
- 1, fifo
->base
.mask
); /* inactive channel */
347 static const struct nvkm_fifo_func
349 .init
= nv04_fifo_init
,
350 .intr
= nv04_fifo_intr
,
351 .pause
= nv04_fifo_pause
,
352 .start
= nv04_fifo_start
,
354 &nv04_fifo_dma_oclass
,
360 nv04_fifo_new(struct nvkm_device
*device
, int index
, struct nvkm_fifo
**pfifo
)
362 return nv04_fifo_new_(&nv04_fifo
, device
, index
, 16,
363 nv04_fifo_ramfc
, pfifo
);