Commit | Line | Data |
---|---|---|
5132f377 | 1 | /* |
ebb945a9 | 2 | * Copyright 2012 Red Hat Inc. |
5132f377 BS |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | * Authors: Ben Skeggs | |
23 | */ | |
24 | ||
ebb945a9 BS |
25 | #include <core/client.h> |
26 | #include <core/handle.h> | |
27 | #include <core/namedb.h> | |
28 | #include <core/gpuobj.h> | |
29 | #include <core/engctx.h> | |
9bd2ddba | 30 | #include <core/event.h> |
bbf8906b BS |
31 | #include <nvif/unpack.h> |
32 | #include <nvif/class.h> | |
ebb945a9 | 33 | #include <core/enum.h> |
5132f377 | 34 | |
ebb945a9 BS |
35 | #include <subdev/timer.h> |
36 | #include <subdev/bar.h> | |
52225551 | 37 | #include <subdev/fb.h> |
ebb945a9 | 38 | #include <subdev/vm.h> |
5132f377 | 39 | |
ebb945a9 | 40 | #include <engine/dmaobj.h> |
a763951a BS |
41 | |
42 | #include "nve0.h" | |
5132f377 | 43 | |
507ceb15 | 44 | #define _(a,b) { (a), ((1ULL << (a)) | (b)) } |
dbff2dee | 45 | static const struct { |
507ceb15 MP |
46 | u64 subdev; |
47 | u64 mask; | |
dbff2dee | 48 | } fifo_engine[] = { |
48506d17 BS |
49 | _(NVDEV_ENGINE_GR , (1ULL << NVDEV_ENGINE_SW) | |
50 | (1ULL << NVDEV_ENGINE_COPY2)), | |
dbff2dee BS |
51 | _(NVDEV_ENGINE_VP , 0), |
52 | _(NVDEV_ENGINE_PPP , 0), | |
53 | _(NVDEV_ENGINE_BSP , 0), | |
54 | _(NVDEV_ENGINE_COPY0 , 0), | |
55 | _(NVDEV_ENGINE_COPY1 , 0), | |
56 | _(NVDEV_ENGINE_VENC , 0), | |
57 | }; | |
58 | #undef _ | |
59 | #define FIFO_ENGINE_NR ARRAY_SIZE(fifo_engine) | |
60 | ||
ebb945a9 | 61 | struct nve0_fifo_engn { |
f82c44a7 BS |
62 | struct nouveau_gpuobj *runlist[2]; |
63 | int cur_runlist; | |
138b873f | 64 | wait_queue_head_t wait; |
5132f377 BS |
65 | }; |
66 | ||
67 | struct nve0_fifo_priv { | |
ebb945a9 | 68 | struct nouveau_fifo base; |
98d1e317 BS |
69 | |
70 | struct work_struct fault; | |
71 | u64 mask; | |
72 | ||
dbff2dee | 73 | struct nve0_fifo_engn engine[FIFO_ENGINE_NR]; |
5132f377 BS |
74 | struct { |
75 | struct nouveau_gpuobj *mem; | |
76 | struct nouveau_vma bar; | |
77 | } user; | |
78 | int spoon_nr; | |
79 | }; | |
80 | ||
ebb945a9 BS |
81 | struct nve0_fifo_base { |
82 | struct nouveau_fifo_base base; | |
83 | struct nouveau_gpuobj *pgd; | |
84 | struct nouveau_vm *vm; | |
85 | }; | |
86 | ||
5132f377 | 87 | struct nve0_fifo_chan { |
c420b2dc | 88 | struct nouveau_fifo_chan base; |
5132f377 | 89 | u32 engine; |
87032e11 BS |
90 | enum { |
91 | STOPPED, | |
92 | RUNNING, | |
93 | KILLED | |
94 | } state; | |
5132f377 BS |
95 | }; |
96 | ||
ebb945a9 BS |
97 | /******************************************************************************* |
98 | * FIFO channel objects | |
99 | ******************************************************************************/ | |
100 | ||
5132f377 | 101 | static void |
f82c44a7 | 102 | nve0_fifo_runlist_update(struct nve0_fifo_priv *priv, u32 engine) |
5132f377 | 103 | { |
ebb945a9 BS |
104 | struct nouveau_bar *bar = nouveau_bar(priv); |
105 | struct nve0_fifo_engn *engn = &priv->engine[engine]; | |
5132f377 | 106 | struct nouveau_gpuobj *cur; |
ebb945a9 | 107 | int i, p; |
5132f377 | 108 | |
c2e3259b | 109 | mutex_lock(&nv_subdev(priv)->mutex); |
f82c44a7 BS |
110 | cur = engn->runlist[engn->cur_runlist]; |
111 | engn->cur_runlist = !engn->cur_runlist; | |
5132f377 | 112 | |
ebb945a9 | 113 | for (i = 0, p = 0; i < priv->base.max; i++) { |
87032e11 BS |
114 | struct nve0_fifo_chan *chan = (void *)priv->base.channel[i]; |
115 | if (chan && chan->state == RUNNING && chan->engine == engine) { | |
116 | nv_wo32(cur, p + 0, i); | |
117 | nv_wo32(cur, p + 4, 0x00000000); | |
118 | p += 8; | |
119 | } | |
5132f377 | 120 | } |
ebb945a9 | 121 | bar->flush(bar); |
5132f377 | 122 | |
ebb945a9 BS |
123 | nv_wr32(priv, 0x002270, cur->addr >> 12); |
124 | nv_wr32(priv, 0x002274, (engine << 20) | (p >> 3)); | |
87032e11 | 125 | |
5c0633e6 BS |
126 | if (wait_event_timeout(engn->wait, !(nv_rd32(priv, 0x002284 + |
127 | (engine * 0x08)) & 0x00100000), | |
128 | msecs_to_jiffies(2000)) == 0) | |
f82c44a7 | 129 | nv_error(priv, "runlist %d update timeout\n", engine); |
c2e3259b | 130 | mutex_unlock(&nv_subdev(priv)->mutex); |
5132f377 BS |
131 | } |
132 | ||
c420b2dc | 133 | static int |
ebb945a9 BS |
134 | nve0_fifo_context_attach(struct nouveau_object *parent, |
135 | struct nouveau_object *object) | |
5132f377 | 136 | { |
ebb945a9 BS |
137 | struct nouveau_bar *bar = nouveau_bar(parent); |
138 | struct nve0_fifo_base *base = (void *)parent->parent; | |
139 | struct nouveau_engctx *ectx = (void *)object; | |
140 | u32 addr; | |
141 | int ret; | |
142 | ||
143 | switch (nv_engidx(object->engine)) { | |
01672ef4 | 144 | case NVDEV_ENGINE_SW : |
448a4532 | 145 | return 0; |
dbff2dee | 146 | case NVDEV_ENGINE_COPY0: |
01672ef4 BS |
147 | case NVDEV_ENGINE_COPY1: |
148 | case NVDEV_ENGINE_COPY2: | |
448a4532 | 149 | nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12; |
01672ef4 BS |
150 | return 0; |
151 | case NVDEV_ENGINE_GR : addr = 0x0210; break; | |
b2f04fc6 | 152 | case NVDEV_ENGINE_BSP : addr = 0x0270; break; |
a7416d0d | 153 | case NVDEV_ENGINE_VP : addr = 0x0250; break; |
f3295b3c | 154 | case NVDEV_ENGINE_PPP : addr = 0x0260; break; |
ebb945a9 BS |
155 | default: |
156 | return -EINVAL; | |
5132f377 BS |
157 | } |
158 | ||
ebb945a9 BS |
159 | if (!ectx->vma.node) { |
160 | ret = nouveau_gpuobj_map_vm(nv_gpuobj(ectx), base->vm, | |
161 | NV_MEM_ACCESS_RW, &ectx->vma); | |
162 | if (ret) | |
163 | return ret; | |
4c2d4222 BS |
164 | |
165 | nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12; | |
ebb945a9 BS |
166 | } |
167 | ||
168 | nv_wo32(base, addr + 0x00, lower_32_bits(ectx->vma.offset) | 4); | |
169 | nv_wo32(base, addr + 0x04, upper_32_bits(ectx->vma.offset)); | |
170 | bar->flush(bar); | |
171 | return 0; | |
5132f377 BS |
172 | } |
173 | ||
ebb945a9 BS |
174 | static int |
175 | nve0_fifo_context_detach(struct nouveau_object *parent, bool suspend, | |
176 | struct nouveau_object *object) | |
5132f377 | 177 | { |
ebb945a9 BS |
178 | struct nouveau_bar *bar = nouveau_bar(parent); |
179 | struct nve0_fifo_priv *priv = (void *)parent->engine; | |
180 | struct nve0_fifo_base *base = (void *)parent->parent; | |
181 | struct nve0_fifo_chan *chan = (void *)parent; | |
182 | u32 addr; | |
183 | ||
184 | switch (nv_engidx(object->engine)) { | |
185 | case NVDEV_ENGINE_SW : return 0; | |
dbff2dee | 186 | case NVDEV_ENGINE_COPY0: |
01672ef4 BS |
187 | case NVDEV_ENGINE_COPY1: |
188 | case NVDEV_ENGINE_COPY2: addr = 0x0000; break; | |
189 | case NVDEV_ENGINE_GR : addr = 0x0210; break; | |
b2f04fc6 | 190 | case NVDEV_ENGINE_BSP : addr = 0x0270; break; |
a7416d0d | 191 | case NVDEV_ENGINE_VP : addr = 0x0250; break; |
f3295b3c | 192 | case NVDEV_ENGINE_PPP : addr = 0x0260; break; |
ebb945a9 BS |
193 | default: |
194 | return -EINVAL; | |
195 | } | |
196 | ||
ebb945a9 BS |
197 | nv_wr32(priv, 0x002634, chan->base.chid); |
198 | if (!nv_wait(priv, 0x002634, 0xffffffff, chan->base.chid)) { | |
93260d3c MS |
199 | nv_error(priv, "channel %d [%s] kick timeout\n", |
200 | chan->base.chid, nouveau_client_name(chan)); | |
ebb945a9 BS |
201 | if (suspend) |
202 | return -EBUSY; | |
5132f377 BS |
203 | } |
204 | ||
01672ef4 BS |
205 | if (addr) { |
206 | nv_wo32(base, addr + 0x00, 0x00000000); | |
207 | nv_wo32(base, addr + 0x04, 0x00000000); | |
208 | bar->flush(bar); | |
209 | } | |
210 | ||
ebb945a9 | 211 | return 0; |
5132f377 BS |
212 | } |
213 | ||
214 | static int | |
ebb945a9 BS |
215 | nve0_fifo_chan_ctor(struct nouveau_object *parent, |
216 | struct nouveau_object *engine, | |
217 | struct nouveau_oclass *oclass, void *data, u32 size, | |
218 | struct nouveau_object **pobject) | |
5132f377 | 219 | { |
bbf8906b BS |
220 | union { |
221 | struct kepler_channel_gpfifo_a_v0 v0; | |
222 | } *args = data; | |
ebb945a9 BS |
223 | struct nouveau_bar *bar = nouveau_bar(parent); |
224 | struct nve0_fifo_priv *priv = (void *)engine; | |
225 | struct nve0_fifo_base *base = (void *)parent; | |
226 | struct nve0_fifo_chan *chan; | |
ebb945a9 BS |
227 | u64 usermem, ioffset, ilength; |
228 | int ret, i; | |
229 | ||
bbf8906b BS |
230 | nv_ioctl(parent, "create channel gpfifo size %d\n", size); |
231 | if (nvif_unpack(args->v0, 0, 0, false)) { | |
232 | nv_ioctl(parent, "create channel gpfifo vers %d pushbuf %08x " | |
233 | "ioffset %016llx ilength %08x engine %08x\n", | |
234 | args->v0.version, args->v0.pushbuf, args->v0.ioffset, | |
235 | args->v0.ilength, args->v0.engine); | |
236 | } else | |
237 | return ret; | |
ebb945a9 | 238 | |
dbff2dee | 239 | for (i = 0; i < FIFO_ENGINE_NR; i++) { |
bbf8906b | 240 | if (args->v0.engine & (1 << i)) { |
dbff2dee | 241 | if (nouveau_engine(parent, fifo_engine[i].subdev)) { |
bbf8906b | 242 | args->v0.engine = (1 << i); |
dbff2dee BS |
243 | break; |
244 | } | |
245 | } | |
246 | } | |
247 | ||
56fbd2b6 | 248 | if (i == FIFO_ENGINE_NR) { |
bbf8906b | 249 | nv_error(priv, "unsupported engines 0x%08x\n", args->v0.engine); |
dbff2dee | 250 | return -ENODEV; |
56fbd2b6 | 251 | } |
dbff2dee | 252 | |
ebb945a9 BS |
253 | ret = nouveau_fifo_channel_create(parent, engine, oclass, 1, |
254 | priv->user.bar.offset, 0x200, | |
bbf8906b | 255 | args->v0.pushbuf, |
dbff2dee | 256 | fifo_engine[i].mask, &chan); |
ebb945a9 BS |
257 | *pobject = nv_object(chan); |
258 | if (ret) | |
259 | return ret; | |
260 | ||
bbf8906b BS |
261 | args->v0.chid = chan->base.chid; |
262 | ||
ebb945a9 BS |
263 | nv_parent(chan)->context_attach = nve0_fifo_context_attach; |
264 | nv_parent(chan)->context_detach = nve0_fifo_context_detach; | |
dbff2dee | 265 | chan->engine = i; |
ebb945a9 BS |
266 | |
267 | usermem = chan->base.chid * 0x200; | |
bbf8906b BS |
268 | ioffset = args->v0.ioffset; |
269 | ilength = order_base_2(args->v0.ilength / 8); | |
ebb945a9 BS |
270 | |
271 | for (i = 0; i < 0x200; i += 4) | |
272 | nv_wo32(priv->user.mem, usermem + i, 0x00000000); | |
273 | ||
274 | nv_wo32(base, 0x08, lower_32_bits(priv->user.mem->addr + usermem)); | |
275 | nv_wo32(base, 0x0c, upper_32_bits(priv->user.mem->addr + usermem)); | |
276 | nv_wo32(base, 0x10, 0x0000face); | |
277 | nv_wo32(base, 0x30, 0xfffff902); | |
278 | nv_wo32(base, 0x48, lower_32_bits(ioffset)); | |
279 | nv_wo32(base, 0x4c, upper_32_bits(ioffset) | (ilength << 16)); | |
280 | nv_wo32(base, 0x84, 0x20400000); | |
281 | nv_wo32(base, 0x94, 0x30000001); | |
282 | nv_wo32(base, 0x9c, 0x00000100); | |
283 | nv_wo32(base, 0xac, 0x0000001f); | |
284 | nv_wo32(base, 0xe8, chan->base.chid); | |
285 | nv_wo32(base, 0xb8, 0xf8000000); | |
286 | nv_wo32(base, 0xf8, 0x10003080); /* 0x002310 */ | |
287 | nv_wo32(base, 0xfc, 0x10000010); /* 0x002350 */ | |
288 | bar->flush(bar); | |
289 | return 0; | |
290 | } | |
5132f377 | 291 | |
ebb945a9 BS |
292 | static int |
293 | nve0_fifo_chan_init(struct nouveau_object *object) | |
294 | { | |
295 | struct nouveau_gpuobj *base = nv_gpuobj(object->parent); | |
296 | struct nve0_fifo_priv *priv = (void *)object->engine; | |
297 | struct nve0_fifo_chan *chan = (void *)object; | |
298 | u32 chid = chan->base.chid; | |
299 | int ret; | |
5132f377 | 300 | |
ebb945a9 BS |
301 | ret = nouveau_fifo_channel_init(&chan->base); |
302 | if (ret) | |
303 | return ret; | |
5132f377 | 304 | |
dbff2dee | 305 | nv_mask(priv, 0x800004 + (chid * 8), 0x000f0000, chan->engine << 16); |
ebb945a9 | 306 | nv_wr32(priv, 0x800000 + (chid * 8), 0x80000000 | base->addr >> 12); |
87032e11 BS |
307 | |
308 | if (chan->state == STOPPED && (chan->state = RUNNING) == RUNNING) { | |
309 | nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400); | |
310 | nve0_fifo_runlist_update(priv, chan->engine); | |
311 | nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400); | |
312 | } | |
313 | ||
ebb945a9 BS |
314 | return 0; |
315 | } | |
5132f377 | 316 | |
ebb945a9 BS |
317 | static int |
318 | nve0_fifo_chan_fini(struct nouveau_object *object, bool suspend) | |
319 | { | |
320 | struct nve0_fifo_priv *priv = (void *)object->engine; | |
321 | struct nve0_fifo_chan *chan = (void *)object; | |
322 | u32 chid = chan->base.chid; | |
5132f377 | 323 | |
87032e11 BS |
324 | if (chan->state == RUNNING && (chan->state = STOPPED) == STOPPED) { |
325 | nv_mask(priv, 0x800004 + (chid * 8), 0x00000800, 0x00000800); | |
326 | nve0_fifo_runlist_update(priv, chan->engine); | |
327 | } | |
5132f377 | 328 | |
87032e11 | 329 | nv_wr32(priv, 0x800000 + (chid * 8), 0x00000000); |
ebb945a9 BS |
330 | return nouveau_fifo_channel_fini(&chan->base, suspend); |
331 | } | |
5132f377 | 332 | |
ebb945a9 BS |
333 | static struct nouveau_ofuncs |
334 | nve0_fifo_ofuncs = { | |
335 | .ctor = nve0_fifo_chan_ctor, | |
336 | .dtor = _nouveau_fifo_channel_dtor, | |
337 | .init = nve0_fifo_chan_init, | |
338 | .fini = nve0_fifo_chan_fini, | |
6c6ae061 | 339 | .map = _nouveau_fifo_channel_map, |
ebb945a9 BS |
340 | .rd32 = _nouveau_fifo_channel_rd32, |
341 | .wr32 = _nouveau_fifo_channel_wr32, | |
867920f8 | 342 | .ntfy = _nouveau_fifo_channel_ntfy |
ebb945a9 | 343 | }; |
5132f377 | 344 | |
ebb945a9 BS |
345 | static struct nouveau_oclass |
346 | nve0_fifo_sclass[] = { | |
bbf8906b | 347 | { KEPLER_CHANNEL_GPFIFO_A, &nve0_fifo_ofuncs }, |
ebb945a9 BS |
348 | {} |
349 | }; | |
350 | ||
351 | /******************************************************************************* | |
352 | * FIFO context - instmem heap and vm setup | |
353 | ******************************************************************************/ | |
5132f377 | 354 | |
c420b2dc | 355 | static int |
ebb945a9 BS |
356 | nve0_fifo_context_ctor(struct nouveau_object *parent, |
357 | struct nouveau_object *engine, | |
358 | struct nouveau_oclass *oclass, void *data, u32 size, | |
359 | struct nouveau_object **pobject) | |
c420b2dc | 360 | { |
ebb945a9 BS |
361 | struct nve0_fifo_base *base; |
362 | int ret; | |
c420b2dc | 363 | |
ebb945a9 BS |
364 | ret = nouveau_fifo_context_create(parent, engine, oclass, NULL, 0x1000, |
365 | 0x1000, NVOBJ_FLAG_ZERO_ALLOC, &base); | |
366 | *pobject = nv_object(base); | |
367 | if (ret) | |
368 | return ret; | |
c420b2dc | 369 | |
f50c8054 BS |
370 | ret = nouveau_gpuobj_new(nv_object(base), NULL, 0x10000, 0x1000, 0, |
371 | &base->pgd); | |
ebb945a9 BS |
372 | if (ret) |
373 | return ret; | |
374 | ||
375 | nv_wo32(base, 0x0200, lower_32_bits(base->pgd->addr)); | |
376 | nv_wo32(base, 0x0204, upper_32_bits(base->pgd->addr)); | |
377 | nv_wo32(base, 0x0208, 0xffffffff); | |
378 | nv_wo32(base, 0x020c, 0x000000ff); | |
379 | ||
380 | ret = nouveau_vm_ref(nouveau_client(parent)->vm, &base->vm, base->pgd); | |
381 | if (ret) | |
382 | return ret; | |
c420b2dc | 383 | |
c420b2dc BS |
384 | return 0; |
385 | } | |
386 | ||
ebb945a9 BS |
387 | static void |
388 | nve0_fifo_context_dtor(struct nouveau_object *object) | |
389 | { | |
390 | struct nve0_fifo_base *base = (void *)object; | |
391 | nouveau_vm_ref(NULL, &base->vm, base->pgd); | |
392 | nouveau_gpuobj_ref(NULL, &base->pgd); | |
393 | nouveau_fifo_context_destroy(&base->base); | |
394 | } | |
395 | ||
396 | static struct nouveau_oclass | |
397 | nve0_fifo_cclass = { | |
398 | .handle = NV_ENGCTX(FIFO, 0xe0), | |
399 | .ofuncs = &(struct nouveau_ofuncs) { | |
400 | .ctor = nve0_fifo_context_ctor, | |
401 | .dtor = nve0_fifo_context_dtor, | |
402 | .init = _nouveau_fifo_context_init, | |
403 | .fini = _nouveau_fifo_context_fini, | |
404 | .rd32 = _nouveau_fifo_context_rd32, | |
405 | .wr32 = _nouveau_fifo_context_wr32, | |
406 | }, | |
407 | }; | |
408 | ||
409 | /******************************************************************************* | |
410 | * PFIFO engine | |
411 | ******************************************************************************/ | |
412 | ||
98d1e317 BS |
413 | static inline int |
414 | nve0_fifo_engidx(struct nve0_fifo_priv *priv, u32 engn) | |
415 | { | |
416 | switch (engn) { | |
417 | case NVDEV_ENGINE_GR : | |
418 | case NVDEV_ENGINE_COPY2: engn = 0; break; | |
419 | case NVDEV_ENGINE_BSP : engn = 1; break; | |
420 | case NVDEV_ENGINE_PPP : engn = 2; break; | |
421 | case NVDEV_ENGINE_VP : engn = 3; break; | |
422 | case NVDEV_ENGINE_COPY0: engn = 4; break; | |
423 | case NVDEV_ENGINE_COPY1: engn = 5; break; | |
424 | case NVDEV_ENGINE_VENC : engn = 6; break; | |
425 | default: | |
426 | return -1; | |
427 | } | |
428 | ||
429 | return engn; | |
430 | } | |
431 | ||
129dcca7 BS |
432 | static inline struct nouveau_engine * |
433 | nve0_fifo_engine(struct nve0_fifo_priv *priv, u32 engn) | |
434 | { | |
435 | if (engn >= ARRAY_SIZE(fifo_engine)) | |
436 | return NULL; | |
437 | return nouveau_engine(priv, fifo_engine[engn].subdev); | |
438 | } | |
439 | ||
98d1e317 BS |
440 | static void |
441 | nve0_fifo_recover_work(struct work_struct *work) | |
442 | { | |
443 | struct nve0_fifo_priv *priv = container_of(work, typeof(*priv), fault); | |
444 | struct nouveau_object *engine; | |
445 | unsigned long flags; | |
446 | u32 engn, engm = 0; | |
447 | u64 mask, todo; | |
448 | ||
449 | spin_lock_irqsave(&priv->base.lock, flags); | |
450 | mask = priv->mask; | |
451 | priv->mask = 0ULL; | |
452 | spin_unlock_irqrestore(&priv->base.lock, flags); | |
453 | ||
454 | for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn)) | |
455 | engm |= 1 << nve0_fifo_engidx(priv, engn); | |
456 | nv_mask(priv, 0x002630, engm, engm); | |
457 | ||
458 | for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn)) { | |
459 | if ((engine = (void *)nouveau_engine(priv, engn))) { | |
460 | nv_ofuncs(engine)->fini(engine, false); | |
461 | WARN_ON(nv_ofuncs(engine)->init(engine)); | |
462 | } | |
463 | nve0_fifo_runlist_update(priv, nve0_fifo_engidx(priv, engn)); | |
464 | } | |
465 | ||
466 | nv_wr32(priv, 0x00262c, engm); | |
467 | nv_mask(priv, 0x002630, engm, 0x00000000); | |
468 | } | |
469 | ||
470 | static void | |
471 | nve0_fifo_recover(struct nve0_fifo_priv *priv, struct nouveau_engine *engine, | |
472 | struct nve0_fifo_chan *chan) | |
473 | { | |
98d1e317 BS |
474 | u32 chid = chan->base.chid; |
475 | unsigned long flags; | |
476 | ||
477 | nv_error(priv, "%s engine fault on channel %d, recovering...\n", | |
478 | nv_subdev(engine)->name, chid); | |
479 | ||
480 | nv_mask(priv, 0x800004 + (chid * 0x08), 0x00000800, 0x00000800); | |
481 | chan->state = KILLED; | |
482 | ||
483 | spin_lock_irqsave(&priv->base.lock, flags); | |
ec0e5542 | 484 | priv->mask |= 1ULL << nv_engidx(engine); |
98d1e317 BS |
485 | spin_unlock_irqrestore(&priv->base.lock, flags); |
486 | schedule_work(&priv->fault); | |
487 | } | |
488 | ||
3d61b967 BS |
489 | static int |
490 | nve0_fifo_swmthd(struct nve0_fifo_priv *priv, u32 chid, u32 mthd, u32 data) | |
491 | { | |
492 | struct nve0_fifo_chan *chan = NULL; | |
493 | struct nouveau_handle *bind; | |
494 | unsigned long flags; | |
495 | int ret = -EINVAL; | |
496 | ||
497 | spin_lock_irqsave(&priv->base.lock, flags); | |
498 | if (likely(chid >= priv->base.min && chid <= priv->base.max)) | |
499 | chan = (void *)priv->base.channel[chid]; | |
500 | if (unlikely(!chan)) | |
501 | goto out; | |
502 | ||
503 | bind = nouveau_namedb_get_class(nv_namedb(chan), 0x906e); | |
504 | if (likely(bind)) { | |
505 | if (!mthd || !nv_call(bind->object, mthd, data)) | |
506 | ret = 0; | |
507 | nouveau_namedb_put(bind); | |
508 | } | |
509 | ||
510 | out: | |
511 | spin_unlock_irqrestore(&priv->base.lock, flags); | |
512 | return ret; | |
513 | } | |
514 | ||
56b2f68c BS |
515 | static const struct nouveau_enum |
516 | nve0_fifo_bind_reason[] = { | |
517 | { 0x01, "BIND_NOT_UNBOUND" }, | |
518 | { 0x02, "SNOOP_WITHOUT_BAR1" }, | |
519 | { 0x03, "UNBIND_WHILE_RUNNING" }, | |
520 | { 0x05, "INVALID_RUNLIST" }, | |
521 | { 0x06, "INVALID_CTX_TGT" }, | |
522 | { 0x0b, "UNBIND_WHILE_PARKED" }, | |
523 | {} | |
524 | }; | |
525 | ||
526 | static void | |
527 | nve0_fifo_intr_bind(struct nve0_fifo_priv *priv) | |
528 | { | |
529 | u32 intr = nv_rd32(priv, 0x00252c); | |
530 | u32 code = intr & 0x000000ff; | |
531 | const struct nouveau_enum *en; | |
532 | char enunk[6] = ""; | |
533 | ||
534 | en = nouveau_enum_find(nve0_fifo_bind_reason, code); | |
535 | if (!en) | |
536 | snprintf(enunk, sizeof(enunk), "UNK%02x", code); | |
537 | ||
538 | nv_error(priv, "BIND_ERROR [ %s ]\n", en ? en->name : enunk); | |
539 | } | |
540 | ||
0a7760e0 BS |
541 | static const struct nouveau_enum |
542 | nve0_fifo_sched_reason[] = { | |
e9fb9805 BS |
543 | { 0x0a, "CTXSW_TIMEOUT" }, |
544 | {} | |
545 | }; | |
546 | ||
129dcca7 BS |
547 | static void |
548 | nve0_fifo_intr_sched_ctxsw(struct nve0_fifo_priv *priv) | |
549 | { | |
550 | struct nouveau_engine *engine; | |
551 | struct nve0_fifo_chan *chan; | |
552 | u32 engn; | |
553 | ||
554 | for (engn = 0; engn < ARRAY_SIZE(fifo_engine); engn++) { | |
555 | u32 stat = nv_rd32(priv, 0x002640 + (engn * 0x04)); | |
556 | u32 busy = (stat & 0x80000000); | |
557 | u32 next = (stat & 0x07ff0000) >> 16; | |
558 | u32 chsw = (stat & 0x00008000); | |
559 | u32 save = (stat & 0x00004000); | |
560 | u32 load = (stat & 0x00002000); | |
561 | u32 prev = (stat & 0x000007ff); | |
562 | u32 chid = load ? next : prev; | |
563 | (void)save; | |
564 | ||
565 | if (busy && chsw) { | |
566 | if (!(chan = (void *)priv->base.channel[chid])) | |
567 | continue; | |
568 | if (!(engine = nve0_fifo_engine(priv, engn))) | |
569 | continue; | |
570 | nve0_fifo_recover(priv, engine, chan); | |
571 | } | |
572 | } | |
573 | } | |
574 | ||
885f3ced BS |
575 | static void |
576 | nve0_fifo_intr_sched(struct nve0_fifo_priv *priv) | |
577 | { | |
578 | u32 intr = nv_rd32(priv, 0x00254c); | |
579 | u32 code = intr & 0x000000ff; | |
0a7760e0 BS |
580 | const struct nouveau_enum *en; |
581 | char enunk[6] = ""; | |
582 | ||
583 | en = nouveau_enum_find(nve0_fifo_sched_reason, code); | |
584 | if (!en) | |
585 | snprintf(enunk, sizeof(enunk), "UNK%02x", code); | |
586 | ||
587 | nv_error(priv, "SCHED_ERROR [ %s ]\n", en ? en->name : enunk); | |
129dcca7 BS |
588 | |
589 | switch (code) { | |
590 | case 0x0a: | |
591 | nve0_fifo_intr_sched_ctxsw(priv); | |
592 | break; | |
593 | default: | |
594 | break; | |
595 | } | |
885f3ced BS |
596 | } |
597 | ||
598 | static void | |
599 | nve0_fifo_intr_chsw(struct nve0_fifo_priv *priv) | |
600 | { | |
601 | u32 stat = nv_rd32(priv, 0x00256c); | |
602 | nv_error(priv, "CHSW_ERROR 0x%08x\n", stat); | |
603 | nv_wr32(priv, 0x00256c, stat); | |
604 | } | |
605 | ||
606 | static void | |
607 | nve0_fifo_intr_dropped_fault(struct nve0_fifo_priv *priv) | |
608 | { | |
609 | u32 stat = nv_rd32(priv, 0x00259c); | |
610 | nv_error(priv, "DROPPED_MMU_FAULT 0x%08x\n", stat); | |
611 | } | |
612 | ||
613 | static const struct nouveau_enum | |
614 | nve0_fifo_fault_engine[] = { | |
e1b6b14a | 615 | { 0x00, "GR", NULL, NVDEV_ENGINE_GR }, |
885f3ced | 616 | { 0x03, "IFB", NULL, NVDEV_ENGINE_IFB }, |
cb1567c2 BS |
617 | { 0x04, "BAR1", NULL, NVDEV_SUBDEV_BAR }, |
618 | { 0x05, "BAR3", NULL, NVDEV_SUBDEV_INSTMEM }, | |
e1b6b14a BS |
619 | { 0x07, "PBDMA0", NULL, NVDEV_ENGINE_FIFO }, |
620 | { 0x08, "PBDMA1", NULL, NVDEV_ENGINE_FIFO }, | |
621 | { 0x09, "PBDMA2", NULL, NVDEV_ENGINE_FIFO }, | |
622 | { 0x10, "MSVLD", NULL, NVDEV_ENGINE_BSP }, | |
623 | { 0x11, "MSPPP", NULL, NVDEV_ENGINE_PPP }, | |
624 | { 0x13, "PERF" }, | |
625 | { 0x14, "MSPDEC", NULL, NVDEV_ENGINE_VP }, | |
626 | { 0x15, "CE0", NULL, NVDEV_ENGINE_COPY0 }, | |
627 | { 0x16, "CE1", NULL, NVDEV_ENGINE_COPY1 }, | |
628 | { 0x17, "PMU" }, | |
629 | { 0x19, "MSENC", NULL, NVDEV_ENGINE_VENC }, | |
630 | { 0x1b, "CE2", NULL, NVDEV_ENGINE_COPY2 }, | |
5132f377 BS |
631 | {} |
632 | }; | |
633 | ||
885f3ced BS |
634 | static const struct nouveau_enum |
635 | nve0_fifo_fault_reason[] = { | |
e1b6b14a BS |
636 | { 0x00, "PDE" }, |
637 | { 0x01, "PDE_SIZE" }, | |
638 | { 0x02, "PTE" }, | |
639 | { 0x03, "VA_LIMIT_VIOLATION" }, | |
640 | { 0x04, "UNBOUND_INST_BLOCK" }, | |
641 | { 0x05, "PRIV_VIOLATION" }, | |
642 | { 0x06, "RO_VIOLATION" }, | |
643 | { 0x07, "WO_VIOLATION" }, | |
644 | { 0x08, "PITCH_MASK_VIOLATION" }, | |
645 | { 0x09, "WORK_CREATION" }, | |
646 | { 0x0a, "UNSUPPORTED_APERTURE" }, | |
647 | { 0x0b, "COMPRESSION_FAILURE" }, | |
648 | { 0x0c, "UNSUPPORTED_KIND" }, | |
649 | { 0x0d, "REGION_VIOLATION" }, | |
650 | { 0x0e, "BOTH_PTES_VALID" }, | |
651 | { 0x0f, "INFO_TYPE_POISONED" }, | |
5132f377 BS |
652 | {} |
653 | }; | |
654 | ||
885f3ced BS |
655 | static const struct nouveau_enum |
656 | nve0_fifo_fault_hubclient[] = { | |
e1b6b14a BS |
657 | { 0x00, "VIP" }, |
658 | { 0x01, "CE0" }, | |
659 | { 0x02, "CE1" }, | |
660 | { 0x03, "DNISO" }, | |
661 | { 0x04, "FE" }, | |
662 | { 0x05, "FECS" }, | |
663 | { 0x06, "HOST" }, | |
664 | { 0x07, "HOST_CPU" }, | |
665 | { 0x08, "HOST_CPU_NB" }, | |
666 | { 0x09, "ISO" }, | |
667 | { 0x0a, "MMU" }, | |
668 | { 0x0b, "MSPDEC" }, | |
669 | { 0x0c, "MSPPP" }, | |
670 | { 0x0d, "MSVLD" }, | |
671 | { 0x0e, "NISO" }, | |
672 | { 0x0f, "P2P" }, | |
673 | { 0x10, "PD" }, | |
674 | { 0x11, "PERF" }, | |
675 | { 0x12, "PMU" }, | |
676 | { 0x13, "RASTERTWOD" }, | |
677 | { 0x14, "SCC" }, | |
678 | { 0x15, "SCC_NB" }, | |
679 | { 0x16, "SEC" }, | |
680 | { 0x17, "SSYNC" }, | |
681 | { 0x18, "GR_COPY" }, | |
682 | { 0x19, "CE2" }, | |
683 | { 0x1a, "XV" }, | |
684 | { 0x1b, "MMU_NB" }, | |
685 | { 0x1c, "MSENC" }, | |
686 | { 0x1d, "DFALCON" }, | |
687 | { 0x1e, "SKED" }, | |
688 | { 0x1f, "AFALCON" }, | |
5132f377 BS |
689 | {} |
690 | }; | |
691 | ||
885f3ced BS |
692 | static const struct nouveau_enum |
693 | nve0_fifo_fault_gpcclient[] = { | |
e1b6b14a BS |
694 | { 0x00, "L1_0" }, { 0x01, "T1_0" }, { 0x02, "PE_0" }, |
695 | { 0x03, "L1_1" }, { 0x04, "T1_1" }, { 0x05, "PE_1" }, | |
696 | { 0x06, "L1_2" }, { 0x07, "T1_2" }, { 0x08, "PE_2" }, | |
697 | { 0x09, "L1_3" }, { 0x0a, "T1_3" }, { 0x0b, "PE_3" }, | |
698 | { 0x0c, "RAST" }, | |
699 | { 0x0d, "GCC" }, | |
700 | { 0x0e, "GPCCS" }, | |
701 | { 0x0f, "PROP_0" }, | |
702 | { 0x10, "PROP_1" }, | |
703 | { 0x11, "PROP_2" }, | |
704 | { 0x12, "PROP_3" }, | |
705 | { 0x13, "L1_4" }, { 0x14, "T1_4" }, { 0x15, "PE_4" }, | |
706 | { 0x16, "L1_5" }, { 0x17, "T1_5" }, { 0x18, "PE_5" }, | |
707 | { 0x19, "L1_6" }, { 0x1a, "T1_6" }, { 0x1b, "PE_6" }, | |
708 | { 0x1c, "L1_7" }, { 0x1d, "T1_7" }, { 0x1e, "PE_7" }, | |
709 | { 0x1f, "GPM" }, | |
710 | { 0x20, "LTP_UTLB_0" }, | |
711 | { 0x21, "LTP_UTLB_1" }, | |
712 | { 0x22, "LTP_UTLB_2" }, | |
713 | { 0x23, "LTP_UTLB_3" }, | |
714 | { 0x24, "GPC_RGG_UTLB" }, | |
5132f377 BS |
715 | {} |
716 | }; | |
717 | ||
e9fb9805 BS |
718 | static void |
719 | nve0_fifo_intr_fault(struct nve0_fifo_priv *priv, int unit) | |
5132f377 | 720 | { |
885f3ced BS |
721 | u32 inst = nv_rd32(priv, 0x002800 + (unit * 0x10)); |
722 | u32 valo = nv_rd32(priv, 0x002804 + (unit * 0x10)); | |
723 | u32 vahi = nv_rd32(priv, 0x002808 + (unit * 0x10)); | |
724 | u32 stat = nv_rd32(priv, 0x00280c + (unit * 0x10)); | |
725 | u32 gpc = (stat & 0x1f000000) >> 24; | |
5132f377 | 726 | u32 client = (stat & 0x00001f00) >> 8; |
885f3ced BS |
727 | u32 write = (stat & 0x00000080); |
728 | u32 hub = (stat & 0x00000040); | |
729 | u32 reason = (stat & 0x0000000f); | |
98d1e317 BS |
730 | struct nouveau_object *engctx = NULL, *object; |
731 | struct nouveau_engine *engine = NULL; | |
885f3ced BS |
732 | const struct nouveau_enum *er, *eu, *ec; |
733 | char erunk[6] = ""; | |
734 | char euunk[6] = ""; | |
735 | char ecunk[6] = ""; | |
736 | char gpcid[3] = ""; | |
737 | ||
738 | er = nouveau_enum_find(nve0_fifo_fault_reason, reason); | |
739 | if (!er) | |
740 | snprintf(erunk, sizeof(erunk), "UNK%02X", reason); | |
741 | ||
742 | eu = nouveau_enum_find(nve0_fifo_fault_engine, unit); | |
743 | if (eu) { | |
744 | switch (eu->data2) { | |
745 | case NVDEV_SUBDEV_BAR: | |
cb1567c2 | 746 | nv_mask(priv, 0x001704, 0x00000000, 0x00000000); |
885f3ced BS |
747 | break; |
748 | case NVDEV_SUBDEV_INSTMEM: | |
cb1567c2 | 749 | nv_mask(priv, 0x001714, 0x00000000, 0x00000000); |
885f3ced BS |
750 | break; |
751 | case NVDEV_ENGINE_IFB: | |
752 | nv_mask(priv, 0x001718, 0x00000000, 0x00000000); | |
753 | break; | |
754 | default: | |
755 | engine = nouveau_engine(priv, eu->data2); | |
756 | if (engine) | |
cb1567c2 | 757 | engctx = nouveau_engctx_get(engine, inst); |
885f3ced | 758 | break; |
cb1567c2 | 759 | } |
885f3ced BS |
760 | } else { |
761 | snprintf(euunk, sizeof(euunk), "UNK%02x", unit); | |
93260d3c | 762 | } |
885f3ced BS |
763 | |
764 | if (hub) { | |
765 | ec = nouveau_enum_find(nve0_fifo_fault_hubclient, client); | |
766 | } else { | |
767 | ec = nouveau_enum_find(nve0_fifo_fault_gpcclient, client); | |
768 | snprintf(gpcid, sizeof(gpcid), "%d", gpc); | |
769 | } | |
770 | ||
771 | if (!ec) | |
772 | snprintf(ecunk, sizeof(ecunk), "UNK%02x", client); | |
773 | ||
774 | nv_error(priv, "%s fault at 0x%010llx [%s] from %s/%s%s%s%s on " | |
775 | "channel 0x%010llx [%s]\n", write ? "write" : "read", | |
776 | (u64)vahi << 32 | valo, er ? er->name : erunk, | |
777 | eu ? eu->name : euunk, hub ? "" : "GPC", gpcid, hub ? "" : "/", | |
778 | ec ? ec->name : ecunk, (u64)inst << 12, | |
779 | nouveau_client_name(engctx)); | |
93260d3c | 780 | |
98d1e317 BS |
781 | object = engctx; |
782 | while (object) { | |
783 | switch (nv_mclass(object)) { | |
bbf8906b | 784 | case KEPLER_CHANNEL_GPFIFO_A: |
98d1e317 BS |
785 | nve0_fifo_recover(priv, engine, (void *)object); |
786 | break; | |
787 | } | |
788 | object = object->parent; | |
789 | } | |
790 | ||
93260d3c | 791 | nouveau_engctx_put(engctx); |
5132f377 BS |
792 | } |
793 | ||
70b2cc8e | 794 | static const struct nouveau_bitfield nve0_fifo_pbdma_intr_0[] = { |
3d61b967 BS |
795 | { 0x00000001, "MEMREQ" }, |
796 | { 0x00000002, "MEMACK_TIMEOUT" }, | |
797 | { 0x00000004, "MEMACK_EXTRA" }, | |
798 | { 0x00000008, "MEMDAT_TIMEOUT" }, | |
799 | { 0x00000010, "MEMDAT_EXTRA" }, | |
800 | { 0x00000020, "MEMFLUSH" }, | |
801 | { 0x00000040, "MEMOP" }, | |
802 | { 0x00000080, "LBCONNECT" }, | |
803 | { 0x00000100, "LBREQ" }, | |
804 | { 0x00000200, "LBACK_TIMEOUT" }, | |
805 | { 0x00000400, "LBACK_EXTRA" }, | |
806 | { 0x00000800, "LBDAT_TIMEOUT" }, | |
807 | { 0x00001000, "LBDAT_EXTRA" }, | |
808 | { 0x00002000, "GPFIFO" }, | |
809 | { 0x00004000, "GPPTR" }, | |
810 | { 0x00008000, "GPENTRY" }, | |
811 | { 0x00010000, "GPCRC" }, | |
812 | { 0x00020000, "PBPTR" }, | |
813 | { 0x00040000, "PBENTRY" }, | |
814 | { 0x00080000, "PBCRC" }, | |
815 | { 0x00100000, "XBARCONNECT" }, | |
816 | { 0x00200000, "METHOD" }, | |
817 | { 0x00400000, "METHODCRC" }, | |
818 | { 0x00800000, "DEVICE" }, | |
819 | { 0x02000000, "SEMAPHORE" }, | |
820 | { 0x04000000, "ACQUIRE" }, | |
821 | { 0x08000000, "PRI" }, | |
822 | { 0x20000000, "NO_CTXSW_SEG" }, | |
823 | { 0x40000000, "PBSEG" }, | |
824 | { 0x80000000, "SIGNATURE" }, | |
825 | {} | |
826 | }; | |
e2b34fa0 | 827 | |
5132f377 | 828 | static void |
70b2cc8e | 829 | nve0_fifo_intr_pbdma_0(struct nve0_fifo_priv *priv, int unit) |
5132f377 | 830 | { |
70b2cc8e BS |
831 | u32 mask = nv_rd32(priv, 0x04010c + (unit * 0x2000)); |
832 | u32 stat = nv_rd32(priv, 0x040108 + (unit * 0x2000)) & mask; | |
ebb945a9 BS |
833 | u32 addr = nv_rd32(priv, 0x0400c0 + (unit * 0x2000)); |
834 | u32 data = nv_rd32(priv, 0x0400c4 + (unit * 0x2000)); | |
835 | u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0xfff; | |
836 | u32 subc = (addr & 0x00070000) >> 16; | |
5132f377 | 837 | u32 mthd = (addr & 0x00003ffc); |
e2b34fa0 BS |
838 | u32 show = stat; |
839 | ||
ebb945a9 BS |
840 | if (stat & 0x00800000) { |
841 | if (!nve0_fifo_swmthd(priv, chid, mthd, data)) | |
842 | show &= ~0x00800000; | |
70b2cc8e | 843 | nv_wr32(priv, 0x0400c0 + (unit * 0x2000), 0x80600008); |
ebb945a9 BS |
844 | } |
845 | ||
e2b34fa0 | 846 | if (show) { |
39b05542 | 847 | nv_error(priv, "PBDMA%d:", unit); |
70b2cc8e | 848 | nouveau_bitfield_print(nve0_fifo_pbdma_intr_0, show); |
f533da10 | 849 | pr_cont("\n"); |
93260d3c | 850 | nv_error(priv, |
39b05542 | 851 | "PBDMA%d: ch %d [%s] subc %d mthd 0x%04x data 0x%08x\n", |
93260d3c MS |
852 | unit, chid, |
853 | nouveau_client_name_for_fifo_chid(&priv->base, chid), | |
854 | subc, mthd, data); | |
e2b34fa0 | 855 | } |
5132f377 | 856 | |
ebb945a9 | 857 | nv_wr32(priv, 0x040108 + (unit * 0x2000), stat); |
5132f377 BS |
858 | } |
859 | ||
70b2cc8e BS |
860 | static const struct nouveau_bitfield nve0_fifo_pbdma_intr_1[] = { |
861 | { 0x00000001, "HCE_RE_ILLEGAL_OP" }, | |
862 | { 0x00000002, "HCE_RE_ALIGNB" }, | |
863 | { 0x00000004, "HCE_PRIV" }, | |
864 | { 0x00000008, "HCE_ILLEGAL_MTHD" }, | |
865 | { 0x00000010, "HCE_ILLEGAL_CLASS" }, | |
866 | {} | |
867 | }; | |
868 | ||
869 | static void | |
870 | nve0_fifo_intr_pbdma_1(struct nve0_fifo_priv *priv, int unit) | |
871 | { | |
872 | u32 mask = nv_rd32(priv, 0x04014c + (unit * 0x2000)); | |
873 | u32 stat = nv_rd32(priv, 0x040148 + (unit * 0x2000)) & mask; | |
874 | u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0xfff; | |
875 | ||
876 | if (stat) { | |
877 | nv_error(priv, "PBDMA%d:", unit); | |
878 | nouveau_bitfield_print(nve0_fifo_pbdma_intr_1, stat); | |
879 | pr_cont("\n"); | |
880 | nv_error(priv, "PBDMA%d: ch %d %08x %08x\n", unit, chid, | |
881 | nv_rd32(priv, 0x040150 + (unit * 0x2000)), | |
882 | nv_rd32(priv, 0x040154 + (unit * 0x2000))); | |
883 | } | |
884 | ||
885 | nv_wr32(priv, 0x040148 + (unit * 0x2000), stat); | |
886 | } | |
887 | ||
138b873f BS |
888 | static void |
889 | nve0_fifo_intr_runlist(struct nve0_fifo_priv *priv) | |
890 | { | |
891 | u32 mask = nv_rd32(priv, 0x002a00); | |
892 | while (mask) { | |
893 | u32 engn = __ffs(mask); | |
894 | wake_up(&priv->engine[engn].wait); | |
895 | nv_wr32(priv, 0x002a00, 1 << engn); | |
896 | mask &= ~(1 << engn); | |
897 | } | |
898 | } | |
899 | ||
c074bdbc BS |
900 | static void |
901 | nve0_fifo_intr_engine(struct nve0_fifo_priv *priv) | |
902 | { | |
867920f8 | 903 | nouveau_fifo_uevent(&priv->base); |
c074bdbc BS |
904 | } |
905 | ||
5132f377 | 906 | static void |
ebb945a9 | 907 | nve0_fifo_intr(struct nouveau_subdev *subdev) |
5132f377 | 908 | { |
ebb945a9 BS |
909 | struct nve0_fifo_priv *priv = (void *)subdev; |
910 | u32 mask = nv_rd32(priv, 0x002140); | |
911 | u32 stat = nv_rd32(priv, 0x002100) & mask; | |
5132f377 | 912 | |
e9fb9805 | 913 | if (stat & 0x00000001) { |
56b2f68c | 914 | nve0_fifo_intr_bind(priv); |
e9fb9805 BS |
915 | nv_wr32(priv, 0x002100, 0x00000001); |
916 | stat &= ~0x00000001; | |
917 | } | |
918 | ||
919 | if (stat & 0x00000010) { | |
920 | nv_error(priv, "PIO_ERROR\n"); | |
921 | nv_wr32(priv, 0x002100, 0x00000010); | |
922 | stat &= ~0x00000010; | |
923 | } | |
924 | ||
5132f377 | 925 | if (stat & 0x00000100) { |
e9fb9805 | 926 | nve0_fifo_intr_sched(priv); |
ebb945a9 | 927 | nv_wr32(priv, 0x002100, 0x00000100); |
5132f377 BS |
928 | stat &= ~0x00000100; |
929 | } | |
930 | ||
e9fb9805 BS |
931 | if (stat & 0x00010000) { |
932 | nve0_fifo_intr_chsw(priv); | |
933 | nv_wr32(priv, 0x002100, 0x00010000); | |
934 | stat &= ~0x00010000; | |
935 | } | |
936 | ||
937 | if (stat & 0x00800000) { | |
938 | nv_error(priv, "FB_FLUSH_TIMEOUT\n"); | |
939 | nv_wr32(priv, 0x002100, 0x00800000); | |
940 | stat &= ~0x00800000; | |
941 | } | |
942 | ||
943 | if (stat & 0x01000000) { | |
944 | nv_error(priv, "LB_ERROR\n"); | |
945 | nv_wr32(priv, 0x002100, 0x01000000); | |
946 | stat &= ~0x01000000; | |
947 | } | |
948 | ||
949 | if (stat & 0x08000000) { | |
950 | nve0_fifo_intr_dropped_fault(priv); | |
951 | nv_wr32(priv, 0x002100, 0x08000000); | |
952 | stat &= ~0x08000000; | |
953 | } | |
954 | ||
5132f377 | 955 | if (stat & 0x10000000) { |
885f3ced BS |
956 | u32 mask = nv_rd32(priv, 0x00259c); |
957 | while (mask) { | |
958 | u32 unit = __ffs(mask); | |
959 | nve0_fifo_intr_fault(priv, unit); | |
960 | nv_wr32(priv, 0x00259c, (1 << unit)); | |
961 | mask &= ~(1 << unit); | |
5132f377 | 962 | } |
5132f377 BS |
963 | stat &= ~0x10000000; |
964 | } | |
965 | ||
966 | if (stat & 0x20000000) { | |
39b05542 | 967 | u32 mask = nv_rd32(priv, 0x0025a0); |
3d61b967 BS |
968 | while (mask) { |
969 | u32 unit = __ffs(mask); | |
70b2cc8e BS |
970 | nve0_fifo_intr_pbdma_0(priv, unit); |
971 | nve0_fifo_intr_pbdma_1(priv, unit); | |
3d61b967 BS |
972 | nv_wr32(priv, 0x0025a0, (1 << unit)); |
973 | mask &= ~(1 << unit); | |
5132f377 | 974 | } |
5132f377 BS |
975 | stat &= ~0x20000000; |
976 | } | |
977 | ||
978 | if (stat & 0x40000000) { | |
138b873f | 979 | nve0_fifo_intr_runlist(priv); |
5132f377 BS |
980 | stat &= ~0x40000000; |
981 | } | |
982 | ||
9bd2ddba | 983 | if (stat & 0x80000000) { |
9bd2ddba | 984 | nv_wr32(priv, 0x002100, 0x80000000); |
19a10828 | 985 | nve0_fifo_intr_engine(priv); |
9bd2ddba BS |
986 | stat &= ~0x80000000; |
987 | } | |
988 | ||
5132f377 | 989 | if (stat) { |
7a42f492 BS |
990 | nv_error(priv, "INTR 0x%08x\n", stat); |
991 | nv_mask(priv, 0x002140, stat, 0x00000000); | |
ebb945a9 | 992 | nv_wr32(priv, 0x002100, stat); |
5132f377 BS |
993 | } |
994 | } | |
c420b2dc | 995 | |
9bd2ddba | 996 | static void |
79ca2770 | 997 | nve0_fifo_uevent_init(struct nvkm_event *event, int type, int index) |
9bd2ddba | 998 | { |
79ca2770 BS |
999 | struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent); |
1000 | nv_mask(fifo, 0x002140, 0x80000000, 0x80000000); | |
9bd2ddba BS |
1001 | } |
1002 | ||
1003 | static void | |
79ca2770 | 1004 | nve0_fifo_uevent_fini(struct nvkm_event *event, int type, int index) |
9bd2ddba | 1005 | { |
79ca2770 BS |
1006 | struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent); |
1007 | nv_mask(fifo, 0x002140, 0x80000000, 0x00000000); | |
9bd2ddba BS |
1008 | } |
1009 | ||
79ca2770 BS |
1010 | static const struct nvkm_event_func |
1011 | nve0_fifo_uevent_func = { | |
867920f8 | 1012 | .ctor = nouveau_fifo_uevent_ctor, |
79ca2770 BS |
1013 | .init = nve0_fifo_uevent_init, |
1014 | .fini = nve0_fifo_uevent_fini, | |
1015 | }; | |
1016 | ||
649ec925 BS |
1017 | int |
1018 | nve0_fifo_fini(struct nouveau_object *object, bool suspend) | |
1019 | { | |
1020 | struct nve0_fifo_priv *priv = (void *)object; | |
1021 | int ret; | |
1022 | ||
1023 | ret = nouveau_fifo_fini(&priv->base, suspend); | |
1024 | if (ret) | |
1025 | return ret; | |
1026 | ||
1027 | /* allow mmu fault interrupts, even when we're not using fifo */ | |
1028 | nv_mask(priv, 0x002140, 0x10000000, 0x10000000); | |
1029 | return 0; | |
1030 | } | |
1031 | ||
a763951a BS |
1032 | int |
1033 | nve0_fifo_init(struct nouveau_object *object) | |
1034 | { | |
1035 | struct nve0_fifo_priv *priv = (void *)object; | |
1036 | int ret, i; | |
1037 | ||
1038 | ret = nouveau_fifo_init(&priv->base); | |
1039 | if (ret) | |
1040 | return ret; | |
1041 | ||
39b05542 | 1042 | /* enable all available PBDMA units */ |
a763951a BS |
1043 | nv_wr32(priv, 0x000204, 0xffffffff); |
1044 | priv->spoon_nr = hweight32(nv_rd32(priv, 0x000204)); | |
39b05542 | 1045 | nv_debug(priv, "%d PBDMA unit(s)\n", priv->spoon_nr); |
a763951a | 1046 | |
39b05542 | 1047 | /* PBDMA[n] */ |
a763951a BS |
1048 | for (i = 0; i < priv->spoon_nr; i++) { |
1049 | nv_mask(priv, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000); | |
1050 | nv_wr32(priv, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */ | |
1051 | nv_wr32(priv, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */ | |
1052 | } | |
1053 | ||
70b2cc8e BS |
1054 | /* PBDMA[n].HCE */ |
1055 | for (i = 0; i < priv->spoon_nr; i++) { | |
1056 | nv_wr32(priv, 0x040148 + (i * 0x2000), 0xffffffff); /* INTR */ | |
1057 | nv_wr32(priv, 0x04014c + (i * 0x2000), 0xffffffff); /* INTREN */ | |
1058 | } | |
1059 | ||
a763951a BS |
1060 | nv_wr32(priv, 0x002254, 0x10000000 | priv->user.bar.offset >> 12); |
1061 | ||
a763951a | 1062 | nv_wr32(priv, 0x002100, 0xffffffff); |
138b873f | 1063 | nv_wr32(priv, 0x002140, 0x7fffffff); |
a763951a BS |
1064 | return 0; |
1065 | } | |
1066 | ||
1067 | void | |
1068 | nve0_fifo_dtor(struct nouveau_object *object) | |
1069 | { | |
1070 | struct nve0_fifo_priv *priv = (void *)object; | |
1071 | int i; | |
1072 | ||
1073 | nouveau_gpuobj_unmap(&priv->user.bar); | |
1074 | nouveau_gpuobj_ref(NULL, &priv->user.mem); | |
1075 | ||
1076 | for (i = 0; i < FIFO_ENGINE_NR; i++) { | |
f82c44a7 BS |
1077 | nouveau_gpuobj_ref(NULL, &priv->engine[i].runlist[1]); |
1078 | nouveau_gpuobj_ref(NULL, &priv->engine[i].runlist[0]); | |
a763951a BS |
1079 | } |
1080 | ||
1081 | nouveau_fifo_destroy(&priv->base); | |
1082 | } | |
1083 | ||
1084 | int | |
ebb945a9 BS |
1085 | nve0_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine, |
1086 | struct nouveau_oclass *oclass, void *data, u32 size, | |
1087 | struct nouveau_object **pobject) | |
1088 | { | |
a763951a | 1089 | struct nve0_fifo_impl *impl = (void *)oclass; |
ebb945a9 | 1090 | struct nve0_fifo_priv *priv; |
8d6f585d | 1091 | int ret, i; |
ebb945a9 | 1092 | |
a763951a BS |
1093 | ret = nouveau_fifo_create(parent, engine, oclass, 0, |
1094 | impl->channels - 1, &priv); | |
ebb945a9 BS |
1095 | *pobject = nv_object(priv); |
1096 | if (ret) | |
1097 | return ret; | |
1098 | ||
98d1e317 BS |
1099 | INIT_WORK(&priv->fault, nve0_fifo_recover_work); |
1100 | ||
8d6f585d BS |
1101 | for (i = 0; i < FIFO_ENGINE_NR; i++) { |
1102 | ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x8000, 0x1000, | |
f82c44a7 | 1103 | 0, &priv->engine[i].runlist[0]); |
8d6f585d BS |
1104 | if (ret) |
1105 | return ret; | |
1106 | ||
1107 | ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x8000, 0x1000, | |
f82c44a7 | 1108 | 0, &priv->engine[i].runlist[1]); |
8d6f585d BS |
1109 | if (ret) |
1110 | return ret; | |
138b873f BS |
1111 | |
1112 | init_waitqueue_head(&priv->engine[i].wait); | |
8d6f585d BS |
1113 | } |
1114 | ||
02f0b8c8 AC |
1115 | ret = nouveau_gpuobj_new(nv_object(priv), NULL, impl->channels * 0x200, |
1116 | 0x1000, NVOBJ_FLAG_ZERO_ALLOC, &priv->user.mem); | |
ebb945a9 BS |
1117 | if (ret) |
1118 | return ret; | |
1119 | ||
1120 | ret = nouveau_gpuobj_map(priv->user.mem, NV_MEM_ACCESS_RW, | |
1121 | &priv->user.bar); | |
1122 | if (ret) | |
1123 | return ret; | |
1124 | ||
79ca2770 BS |
1125 | ret = nvkm_event_init(&nve0_fifo_uevent_func, 1, 1, &priv->base.uevent); |
1126 | if (ret) | |
1127 | return ret; | |
9bd2ddba | 1128 | |
ebb945a9 BS |
1129 | nv_subdev(priv)->unit = 0x00000100; |
1130 | nv_subdev(priv)->intr = nve0_fifo_intr; | |
1131 | nv_engine(priv)->cclass = &nve0_fifo_cclass; | |
1132 | nv_engine(priv)->sclass = nve0_fifo_sclass; | |
1133 | return 0; | |
1134 | } | |
1135 | ||
16c4f227 | 1136 | struct nouveau_oclass * |
a763951a BS |
1137 | nve0_fifo_oclass = &(struct nve0_fifo_impl) { |
1138 | .base.handle = NV_ENGINE(FIFO, 0xe0), | |
1139 | .base.ofuncs = &(struct nouveau_ofuncs) { | |
ebb945a9 BS |
1140 | .ctor = nve0_fifo_ctor, |
1141 | .dtor = nve0_fifo_dtor, | |
1142 | .init = nve0_fifo_init, | |
649ec925 | 1143 | .fini = nve0_fifo_fini, |
ebb945a9 | 1144 | }, |
a763951a BS |
1145 | .channels = 4096, |
1146 | }.base; |