projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
drm/i915: Move instruction state invalidation from execbuffer to flush
[deliverable/linux.git] / drivers / gpu / drm / i915 / intel_ringbuffer.c
1 /*
2 * Copyright © 2008-2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 * Zou Nan hai <nanhai.zou@intel.com>
26 * Xiang Hai hao<haihao.xiang@intel.com>
27 *
28 */
29
30 #include "drmP.h"
31 #include "drm.h"
32 #include "i915_drv.h"
33 #include "i915_drm.h"
34 #include "i915_trace.h"
35 #include "intel_drv.h"
36
37 static u32 i915_gem_get_seqno(struct drm_device *dev)
38 {
39 drm_i915_private_t *dev_priv = dev->dev_private;
40 u32 seqno;
41
42 seqno = dev_priv->next_seqno;
43
44 /* reserve 0 for non-seqno */
45 if (++dev_priv->next_seqno == 0)
46 dev_priv->next_seqno = 1;
47
48 return seqno;
49 }
50
51 static void
52 render_ring_flush(struct intel_ring_buffer *ring,
53 u32 invalidate_domains,
54 u32 flush_domains)
55 {
56 struct drm_device *dev = ring->dev;
57 drm_i915_private_t *dev_priv = dev->dev_private;
58 u32 cmd;
59
60 #if WATCH_EXEC
61 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
62 invalidate_domains, flush_domains);
63 #endif
64
65 trace_i915_gem_request_flush(dev, dev_priv->next_seqno,
66 invalidate_domains, flush_domains);
67
68 if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
69 /*
70 * read/write caches:
71 *
72 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
73 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
74 * also flushed at 2d versus 3d pipeline switches.
75 *
76 * read-only caches:
77 *
78 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
79 * MI_READ_FLUSH is set, and is always flushed on 965.
80 *
81 * I915_GEM_DOMAIN_COMMAND may not exist?
82 *
83 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
84 * invalidated when MI_EXE_FLUSH is set.
85 *
86 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
87 * invalidated with every MI_FLUSH.
88 *
89 * TLBs:
90 *
91 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
92 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
93 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
94 * are flushed at any MI_FLUSH.
95 */
96
97 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
98 if ((invalidate_domains|flush_domains) &
99 I915_GEM_DOMAIN_RENDER)
100 cmd &= ~MI_NO_WRITE_FLUSH;
101 if (INTEL_INFO(dev)->gen < 4) {
102 /*
103 * On the 965, the sampler cache always gets flushed
104 * and this bit is reserved.
105 */
106 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
107 cmd |= MI_READ_FLUSH;
108 }
109 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
110 cmd |= MI_EXE_FLUSH;
111
112 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
113 (IS_G4X(dev) || IS_GEN5(dev)))
114 cmd |= MI_INVALIDATE_ISP;
115
116 #if WATCH_EXEC
117 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
118 #endif
119 if (intel_ring_begin(ring, 2) == 0) {
120 intel_ring_emit(ring, cmd);
121 intel_ring_emit(ring, MI_NOOP);
122 intel_ring_advance(ring);
123 }
124 }
125 }
126
127 static void ring_write_tail(struct intel_ring_buffer *ring,
128 u32 value)
129 {
130 drm_i915_private_t *dev_priv = ring->dev->dev_private;
131 I915_WRITE_TAIL(ring, value);
132 }
133
134 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
135 {
136 drm_i915_private_t *dev_priv = ring->dev->dev_private;
137 u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
138 RING_ACTHD(ring->mmio_base) : ACTHD;
139
140 return I915_READ(acthd_reg);
141 }
142
143 static int init_ring_common(struct intel_ring_buffer *ring)
144 {
145 drm_i915_private_t *dev_priv = ring->dev->dev_private;
146 struct drm_i915_gem_object *obj = ring->obj;
147 u32 head;
148
149 /* Stop the ring if it's running. */
150 I915_WRITE_CTL(ring, 0);
151 I915_WRITE_HEAD(ring, 0);
152 ring->write_tail(ring, 0);
153
154 /* Initialize the ring. */
155 I915_WRITE_START(ring, obj->gtt_offset);
156 head = I915_READ_HEAD(ring) & HEAD_ADDR;
157
158 /* G45 ring initialization fails to reset head to zero */
159 if (head != 0) {
160 DRM_ERROR("%s head not reset to zero "
161 "ctl %08x head %08x tail %08x start %08x\n",
162 ring->name,
163 I915_READ_CTL(ring),
164 I915_READ_HEAD(ring),
165 I915_READ_TAIL(ring),
166 I915_READ_START(ring));
167
168 I915_WRITE_HEAD(ring, 0);
169
170 DRM_ERROR("%s head forced to zero "
171 "ctl %08x head %08x tail %08x start %08x\n",
172 ring->name,
173 I915_READ_CTL(ring),
174 I915_READ_HEAD(ring),
175 I915_READ_TAIL(ring),
176 I915_READ_START(ring));
177 }
178
179 I915_WRITE_CTL(ring,
180 ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
181 | RING_REPORT_64K | RING_VALID);
182
183 /* If the head is still not zero, the ring is dead */
184 if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
185 I915_READ_START(ring) != obj->gtt_offset ||
186 (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
187 DRM_ERROR("%s initialization failed "
188 "ctl %08x head %08x tail %08x start %08x\n",
189 ring->name,
190 I915_READ_CTL(ring),
191 I915_READ_HEAD(ring),
192 I915_READ_TAIL(ring),
193 I915_READ_START(ring));
194 return -EIO;
195 }
196
197 if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
198 i915_kernel_lost_context(ring->dev);
199 else {
200 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
201 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
202 ring->space = ring->head - (ring->tail + 8);
203 if (ring->space < 0)
204 ring->space += ring->size;
205 }
206 return 0;
207 }
208
209 /*
210 * 965+ support PIPE_CONTROL commands, which provide finer grained control
211 * over cache flushing.
212 */
213 struct pipe_control {
214 struct drm_i915_gem_object *obj;
215 volatile u32 *cpu_page;
216 u32 gtt_offset;
217 };
218
219 static int
220 init_pipe_control(struct intel_ring_buffer *ring)
221 {
222 struct pipe_control *pc;
223 struct drm_i915_gem_object *obj;
224 int ret;
225
226 if (ring->private)
227 return 0;
228
229 pc = kmalloc(sizeof(*pc), GFP_KERNEL);
230 if (!pc)
231 return -ENOMEM;
232
233 obj = i915_gem_alloc_object(ring->dev, 4096);
234 if (obj == NULL) {
235 DRM_ERROR("Failed to allocate seqno page\n");
236 ret = -ENOMEM;
237 goto err;
238 }
239 obj->agp_type = AGP_USER_CACHED_MEMORY;
240
241 ret = i915_gem_object_pin(obj, 4096, true);
242 if (ret)
243 goto err_unref;
244
245 pc->gtt_offset = obj->gtt_offset;
246 pc->cpu_page = kmap(obj->pages[0]);
247 if (pc->cpu_page == NULL)
248 goto err_unpin;
249
250 pc->obj = obj;
251 ring->private = pc;
252 return 0;
253
254 err_unpin:
255 i915_gem_object_unpin(obj);
256 err_unref:
257 drm_gem_object_unreference(&obj->base);
258 err:
259 kfree(pc);
260 return ret;
261 }
262
263 static void
264 cleanup_pipe_control(struct intel_ring_buffer *ring)
265 {
266 struct pipe_control *pc = ring->private;
267 struct drm_i915_gem_object *obj;
268
269 if (!ring->private)
270 return;
271
272 obj = pc->obj;
273 kunmap(obj->pages[0]);
274 i915_gem_object_unpin(obj);
275 drm_gem_object_unreference(&obj->base);
276
277 kfree(pc);
278 ring->private = NULL;
279 }
280
281 static int init_render_ring(struct intel_ring_buffer *ring)
282 {
283 struct drm_device *dev = ring->dev;
284 int ret = init_ring_common(ring);
285
286 if (INTEL_INFO(dev)->gen > 3) {
287 drm_i915_private_t *dev_priv = dev->dev_private;
288 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
289 if (IS_GEN6(dev))
290 mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
291 I915_WRITE(MI_MODE, mode);
292 }
293
294 if (HAS_PIPE_CONTROL(dev)) {
295 ret = init_pipe_control(ring);
296 if (ret)
297 return ret;
298 }
299
300 return ret;
301 }
302
303 static void render_ring_cleanup(struct intel_ring_buffer *ring)
304 {
305 if (!ring->private)
306 return;
307
308 cleanup_pipe_control(ring);
309 }
310
311 #define PIPE_CONTROL_FLUSH(ring__, addr__) \
312 do { \
313 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \
314 PIPE_CONTROL_DEPTH_STALL | 2); \
315 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
316 intel_ring_emit(ring__, 0); \
317 intel_ring_emit(ring__, 0); \
318 } while (0)
319
320 /**
321 * Creates a new sequence number, emitting a write of it to the status page
322 * plus an interrupt, which will trigger i915_user_interrupt_handler.
323 *
324 * Must be called with struct_lock held.
325 *
326 * Returned sequence numbers are nonzero on success.
327 */
328 static int
329 render_ring_add_request(struct intel_ring_buffer *ring,
330 u32 *result)
331 {
332 struct drm_device *dev = ring->dev;
333 u32 seqno = i915_gem_get_seqno(dev);
334 struct pipe_control *pc = ring->private;
335 int ret;
336
337 if (IS_GEN6(dev)) {
338 ret = intel_ring_begin(ring, 6);
339 if (ret)
340 return ret;
341
342 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | 3);
343 intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE |
344 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH |
345 PIPE_CONTROL_NOTIFY);
346 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
347 intel_ring_emit(ring, seqno);
348 intel_ring_emit(ring, 0);
349 intel_ring_emit(ring, 0);
350 } else if (HAS_PIPE_CONTROL(dev)) {
351 u32 scratch_addr = pc->gtt_offset + 128;
352
353 /*
354 * Workaround qword write incoherence by flushing the
355 * PIPE_NOTIFY buffers out to memory before requesting
356 * an interrupt.
357 */
358 ret = intel_ring_begin(ring, 32);
359 if (ret)
360 return ret;
361
362 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
363 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
364 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
365 intel_ring_emit(ring, seqno);
366 intel_ring_emit(ring, 0);
367 PIPE_CONTROL_FLUSH(ring, scratch_addr);
368 scratch_addr += 128; /* write to separate cachelines */
369 PIPE_CONTROL_FLUSH(ring, scratch_addr);
370 scratch_addr += 128;
371 PIPE_CONTROL_FLUSH(ring, scratch_addr);
372 scratch_addr += 128;
373 PIPE_CONTROL_FLUSH(ring, scratch_addr);
374 scratch_addr += 128;
375 PIPE_CONTROL_FLUSH(ring, scratch_addr);
376 scratch_addr += 128;
377 PIPE_CONTROL_FLUSH(ring, scratch_addr);
378 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
379 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
380 PIPE_CONTROL_NOTIFY);
381 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
382 intel_ring_emit(ring, seqno);
383 intel_ring_emit(ring, 0);
384 } else {
385 ret = intel_ring_begin(ring, 4);
386 if (ret)
387 return ret;
388
389 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
390 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
391 intel_ring_emit(ring, seqno);
392
393 intel_ring_emit(ring, MI_USER_INTERRUPT);
394 }
395
396 intel_ring_advance(ring);
397 *result = seqno;
398 return 0;
399 }
400
401 static u32
402 render_ring_get_seqno(struct intel_ring_buffer *ring)
403 {
404 struct drm_device *dev = ring->dev;
405 if (HAS_PIPE_CONTROL(dev)) {
406 struct pipe_control *pc = ring->private;
407 return pc->cpu_page[0];
408 } else
409 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
410 }
411
412 static void
413 render_ring_get_user_irq(struct intel_ring_buffer *ring)
414 {
415 struct drm_device *dev = ring->dev;
416 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
417 unsigned long irqflags;
418
419 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
420 if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
421 if (HAS_PCH_SPLIT(dev))
422 ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
423 else
424 i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
425 }
426 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
427 }
428
429 static void
430 render_ring_put_user_irq(struct intel_ring_buffer *ring)
431 {
432 struct drm_device *dev = ring->dev;
433 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
434 unsigned long irqflags;
435
436 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
437 BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
438 if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
439 if (HAS_PCH_SPLIT(dev))
440 ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
441 else
442 i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
443 }
444 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
445 }
446
447 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
448 {
449 drm_i915_private_t *dev_priv = ring->dev->dev_private;
450 u32 mmio = IS_GEN6(ring->dev) ?
451 RING_HWS_PGA_GEN6(ring->mmio_base) :
452 RING_HWS_PGA(ring->mmio_base);
453 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
454 POSTING_READ(mmio);
455 }
456
457 static void
458 bsd_ring_flush(struct intel_ring_buffer *ring,
459 u32 invalidate_domains,
460 u32 flush_domains)
461 {
462 if (intel_ring_begin(ring, 2) == 0) {
463 intel_ring_emit(ring, MI_FLUSH);
464 intel_ring_emit(ring, MI_NOOP);
465 intel_ring_advance(ring);
466 }
467 }
468
469 static int
470 ring_add_request(struct intel_ring_buffer *ring,
471 u32 *result)
472 {
473 u32 seqno;
474 int ret;
475
476 ret = intel_ring_begin(ring, 4);
477 if (ret)
478 return ret;
479
480 seqno = i915_gem_get_seqno(ring->dev);
481
482 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
483 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
484 intel_ring_emit(ring, seqno);
485 intel_ring_emit(ring, MI_USER_INTERRUPT);
486 intel_ring_advance(ring);
487
488 DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
489 *result = seqno;
490 return 0;
491 }
492
493 static void
494 bsd_ring_get_user_irq(struct intel_ring_buffer *ring)
495 {
496 /* do nothing */
497 }
498 static void
499 bsd_ring_put_user_irq(struct intel_ring_buffer *ring)
500 {
501 /* do nothing */
502 }
503
504 static u32
505 ring_status_page_get_seqno(struct intel_ring_buffer *ring)
506 {
507 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
508 }
509
510 static int
511 ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
512 struct drm_i915_gem_execbuffer2 *exec,
513 struct drm_clip_rect *cliprects,
514 uint64_t exec_offset)
515 {
516 uint32_t exec_start;
517 int ret;
518
519 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
520
521 ret = intel_ring_begin(ring, 2);
522 if (ret)
523 return ret;
524
525 intel_ring_emit(ring,
526 MI_BATCH_BUFFER_START |
527 (2 << 6) |
528 MI_BATCH_NON_SECURE_I965);
529 intel_ring_emit(ring, exec_start);
530 intel_ring_advance(ring);
531
532 return 0;
533 }
534
535 static int
536 render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
537 struct drm_i915_gem_execbuffer2 *exec,
538 struct drm_clip_rect *cliprects,
539 uint64_t exec_offset)
540 {
541 struct drm_device *dev = ring->dev;
542 drm_i915_private_t *dev_priv = dev->dev_private;
543 int nbox = exec->num_cliprects;
544 uint32_t exec_start, exec_len;
545 int i, count, ret;
546
547 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
548 exec_len = (uint32_t) exec->batch_len;
549
550 trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
551
552 count = nbox ? nbox : 1;
553 for (i = 0; i < count; i++) {
554 if (i < nbox) {
555 ret = i915_emit_box(dev, cliprects, i,
556 exec->DR1, exec->DR4);
557 if (ret)
558 return ret;
559 }
560
561 if (IS_I830(dev) || IS_845G(dev)) {
562 ret = intel_ring_begin(ring, 4);
563 if (ret)
564 return ret;
565
566 intel_ring_emit(ring, MI_BATCH_BUFFER);
567 intel_ring_emit(ring, exec_start | MI_BATCH_NON_SECURE);
568 intel_ring_emit(ring, exec_start + exec_len - 4);
569 intel_ring_emit(ring, 0);
570 } else {
571 ret = intel_ring_begin(ring, 2);
572 if (ret)
573 return ret;
574
575 if (INTEL_INFO(dev)->gen >= 4) {
576 intel_ring_emit(ring,
577 MI_BATCH_BUFFER_START | (2 << 6)
578 | MI_BATCH_NON_SECURE_I965);
579 intel_ring_emit(ring, exec_start);
580 } else {
581 intel_ring_emit(ring, MI_BATCH_BUFFER_START
582 | (2 << 6));
583 intel_ring_emit(ring, exec_start |
584 MI_BATCH_NON_SECURE);
585 }
586 }
587 intel_ring_advance(ring);
588 }
589
590 return 0;
591 }
592
593 static void cleanup_status_page(struct intel_ring_buffer *ring)
594 {
595 drm_i915_private_t *dev_priv = ring->dev->dev_private;
596 struct drm_i915_gem_object *obj;
597
598 obj = ring->status_page.obj;
599 if (obj == NULL)
600 return;
601
602 kunmap(obj->pages[0]);
603 i915_gem_object_unpin(obj);
604 drm_gem_object_unreference(&obj->base);
605 ring->status_page.obj = NULL;
606
607 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
608 }
609
610 static int init_status_page(struct intel_ring_buffer *ring)
611 {
612 struct drm_device *dev = ring->dev;
613 drm_i915_private_t *dev_priv = dev->dev_private;
614 struct drm_i915_gem_object *obj;
615 int ret;
616
617 obj = i915_gem_alloc_object(dev, 4096);
618 if (obj == NULL) {
619 DRM_ERROR("Failed to allocate status page\n");
620 ret = -ENOMEM;
621 goto err;
622 }
623 obj->agp_type = AGP_USER_CACHED_MEMORY;
624
625 ret = i915_gem_object_pin(obj, 4096, true);
626 if (ret != 0) {
627 goto err_unref;
628 }
629
630 ring->status_page.gfx_addr = obj->gtt_offset;
631 ring->status_page.page_addr = kmap(obj->pages[0]);
632 if (ring->status_page.page_addr == NULL) {
633 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
634 goto err_unpin;
635 }
636 ring->status_page.obj = obj;
637 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
638
639 intel_ring_setup_status_page(ring);
640 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
641 ring->name, ring->status_page.gfx_addr);
642
643 return 0;
644
645 err_unpin:
646 i915_gem_object_unpin(obj);
647 err_unref:
648 drm_gem_object_unreference(&obj->base);
649 err:
650 return ret;
651 }
652
653 int intel_init_ring_buffer(struct drm_device *dev,
654 struct intel_ring_buffer *ring)
655 {
656 struct drm_i915_gem_object *obj;
657 int ret;
658
659 ring->dev = dev;
660 INIT_LIST_HEAD(&ring->active_list);
661 INIT_LIST_HEAD(&ring->request_list);
662 INIT_LIST_HEAD(&ring->gpu_write_list);
663
664 if (I915_NEED_GFX_HWS(dev)) {
665 ret = init_status_page(ring);
666 if (ret)
667 return ret;
668 }
669
670 obj = i915_gem_alloc_object(dev, ring->size);
671 if (obj == NULL) {
672 DRM_ERROR("Failed to allocate ringbuffer\n");
673 ret = -ENOMEM;
674 goto err_hws;
675 }
676
677 ring->obj = obj;
678
679 ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
680 if (ret)
681 goto err_unref;
682
683 ring->map.size = ring->size;
684 ring->map.offset = dev->agp->base + obj->gtt_offset;
685 ring->map.type = 0;
686 ring->map.flags = 0;
687 ring->map.mtrr = 0;
688
689 drm_core_ioremap_wc(&ring->map, dev);
690 if (ring->map.handle == NULL) {
691 DRM_ERROR("Failed to map ringbuffer.\n");
692 ret = -EINVAL;
693 goto err_unpin;
694 }
695
696 ring->virtual_start = ring->map.handle;
697 ret = ring->init(ring);
698 if (ret)
699 goto err_unmap;
700
701 return 0;
702
703 err_unmap:
704 drm_core_ioremapfree(&ring->map, dev);
705 err_unpin:
706 i915_gem_object_unpin(obj);
707 err_unref:
708 drm_gem_object_unreference(&obj->base);
709 ring->obj = NULL;
710 err_hws:
711 cleanup_status_page(ring);
712 return ret;
713 }
714
715 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
716 {
717 struct drm_i915_private *dev_priv;
718 int ret;
719
720 if (ring->obj == NULL)
721 return;
722
723 /* Disable the ring buffer. The ring must be idle at this point */
724 dev_priv = ring->dev->dev_private;
725 ret = intel_wait_ring_buffer(ring, ring->size - 8);
726 I915_WRITE_CTL(ring, 0);
727
728 drm_core_ioremapfree(&ring->map, ring->dev);
729
730 i915_gem_object_unpin(ring->obj);
731 drm_gem_object_unreference(&ring->obj->base);
732 ring->obj = NULL;
733
734 if (ring->cleanup)
735 ring->cleanup(ring);
736
737 cleanup_status_page(ring);
738 }
739
740 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
741 {
742 unsigned int *virt;
743 int rem;
744 rem = ring->size - ring->tail;
745
746 if (ring->space < rem) {
747 int ret = intel_wait_ring_buffer(ring, rem);
748 if (ret)
749 return ret;
750 }
751
752 virt = (unsigned int *)(ring->virtual_start + ring->tail);
753 rem /= 8;
754 while (rem--) {
755 *virt++ = MI_NOOP;
756 *virt++ = MI_NOOP;
757 }
758
759 ring->tail = 0;
760 ring->space = ring->head - 8;
761
762 return 0;
763 }
764
765 int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
766 {
767 struct drm_device *dev = ring->dev;
768 struct drm_i915_private *dev_priv = dev->dev_private;
769 unsigned long end;
770 u32 head;
771
772 head = intel_read_status_page(ring, 4);
773 if (head) {
774 ring->head = head & HEAD_ADDR;
775 ring->space = ring->head - (ring->tail + 8);
776 if (ring->space < 0)
777 ring->space += ring->size;
778 if (ring->space >= n)
779 return 0;
780 }
781
782 trace_i915_ring_wait_begin (dev);
783 end = jiffies + 3 * HZ;
784 do {
785 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
786 ring->space = ring->head - (ring->tail + 8);
787 if (ring->space < 0)
788 ring->space += ring->size;
789 if (ring->space >= n) {
790 trace_i915_ring_wait_end(dev);
791 return 0;
792 }
793
794 if (dev->primary->master) {
795 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
796 if (master_priv->sarea_priv)
797 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
798 }
799
800 msleep(1);
801 if (atomic_read(&dev_priv->mm.wedged))
802 return -EAGAIN;
803 } while (!time_after(jiffies, end));
804 trace_i915_ring_wait_end (dev);
805 return -EBUSY;
806 }
807
808 int intel_ring_begin(struct intel_ring_buffer *ring,
809 int num_dwords)
810 {
811 int n = 4*num_dwords;
812 int ret;
813
814 if (unlikely(ring->tail + n > ring->size)) {
815 ret = intel_wrap_ring_buffer(ring);
816 if (unlikely(ret))
817 return ret;
818 }
819
820 if (unlikely(ring->space < n)) {
821 ret = intel_wait_ring_buffer(ring, n);
822 if (unlikely(ret))
823 return ret;
824 }
825
826 ring->space -= n;
827 return 0;
828 }
829
830 void intel_ring_advance(struct intel_ring_buffer *ring)
831 {
832 ring->tail &= ring->size - 1;
833 ring->write_tail(ring, ring->tail);
834 }
835
836 static const struct intel_ring_buffer render_ring = {
837 .name = "render ring",
838 .id = RING_RENDER,
839 .mmio_base = RENDER_RING_BASE,
840 .size = 32 * PAGE_SIZE,
841 .init = init_render_ring,
842 .write_tail = ring_write_tail,
843 .flush = render_ring_flush,
844 .add_request = render_ring_add_request,
845 .get_seqno = render_ring_get_seqno,
846 .user_irq_get = render_ring_get_user_irq,
847 .user_irq_put = render_ring_put_user_irq,
848 .dispatch_execbuffer = render_ring_dispatch_execbuffer,
849 .cleanup = render_ring_cleanup,
850 };
851
852 /* ring buffer for bit-stream decoder */
853
854 static const struct intel_ring_buffer bsd_ring = {
855 .name = "bsd ring",
856 .id = RING_BSD,
857 .mmio_base = BSD_RING_BASE,
858 .size = 32 * PAGE_SIZE,
859 .init = init_ring_common,
860 .write_tail = ring_write_tail,
861 .flush = bsd_ring_flush,
862 .add_request = ring_add_request,
863 .get_seqno = ring_status_page_get_seqno,
864 .user_irq_get = bsd_ring_get_user_irq,
865 .user_irq_put = bsd_ring_put_user_irq,
866 .dispatch_execbuffer = ring_dispatch_execbuffer,
867 };
868
869
870 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
871 u32 value)
872 {
873 drm_i915_private_t *dev_priv = ring->dev->dev_private;
874
875 /* Every tail move must follow the sequence below */
876 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
877 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
878 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
879 I915_WRITE(GEN6_BSD_RNCID, 0x0);
880
881 if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
882 GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
883 50))
884 DRM_ERROR("timed out waiting for IDLE Indicator\n");
885
886 I915_WRITE_TAIL(ring, value);
887 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
888 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
889 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
890 }
891
892 static void gen6_ring_flush(struct intel_ring_buffer *ring,
893 u32 invalidate_domains,
894 u32 flush_domains)
895 {
896 if (intel_ring_begin(ring, 4) == 0) {
897 intel_ring_emit(ring, MI_FLUSH_DW);
898 intel_ring_emit(ring, 0);
899 intel_ring_emit(ring, 0);
900 intel_ring_emit(ring, 0);
901 intel_ring_advance(ring);
902 }
903 }
904
905 static int
906 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
907 struct drm_i915_gem_execbuffer2 *exec,
908 struct drm_clip_rect *cliprects,
909 uint64_t exec_offset)
910 {
911 uint32_t exec_start;
912 int ret;
913
914 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
915
916 ret = intel_ring_begin(ring, 2);
917 if (ret)
918 return ret;
919
920 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
921 /* bit0-7 is the length on GEN6+ */
922 intel_ring_emit(ring, exec_start);
923 intel_ring_advance(ring);
924
925 return 0;
926 }
927
928 /* ring buffer for Video Codec for Gen6+ */
929 static const struct intel_ring_buffer gen6_bsd_ring = {
930 .name = "gen6 bsd ring",
931 .id = RING_BSD,
932 .mmio_base = GEN6_BSD_RING_BASE,
933 .size = 32 * PAGE_SIZE,
934 .init = init_ring_common,
935 .write_tail = gen6_bsd_ring_write_tail,
936 .flush = gen6_ring_flush,
937 .add_request = ring_add_request,
938 .get_seqno = ring_status_page_get_seqno,
939 .user_irq_get = bsd_ring_get_user_irq,
940 .user_irq_put = bsd_ring_put_user_irq,
941 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
942 };
943
944 /* Blitter support (SandyBridge+) */
945
946 static void
947 blt_ring_get_user_irq(struct intel_ring_buffer *ring)
948 {
949 /* do nothing */
950 }
951 static void
952 blt_ring_put_user_irq(struct intel_ring_buffer *ring)
953 {
954 /* do nothing */
955 }
956
957
958 /* Workaround for some stepping of SNB,
959 * each time when BLT engine ring tail moved,
960 * the first command in the ring to be parsed
961 * should be MI_BATCH_BUFFER_START
962 */
963 #define NEED_BLT_WORKAROUND(dev) \
964 (IS_GEN6(dev) && (dev->pdev->revision < 8))
965
966 static inline struct drm_i915_gem_object *
967 to_blt_workaround(struct intel_ring_buffer *ring)
968 {
969 return ring->private;
970 }
971
972 static int blt_ring_init(struct intel_ring_buffer *ring)
973 {
974 if (NEED_BLT_WORKAROUND(ring->dev)) {
975 struct drm_i915_gem_object *obj;
976 u32 *ptr;
977 int ret;
978
979 obj = i915_gem_alloc_object(ring->dev, 4096);
980 if (obj == NULL)
981 return -ENOMEM;
982
983 ret = i915_gem_object_pin(obj, 4096, true);
984 if (ret) {
985 drm_gem_object_unreference(&obj->base);
986 return ret;
987 }
988
989 ptr = kmap(obj->pages[0]);
990 *ptr++ = MI_BATCH_BUFFER_END;
991 *ptr++ = MI_NOOP;
992 kunmap(obj->pages[0]);
993
994 ret = i915_gem_object_set_to_gtt_domain(obj, false);
995 if (ret) {
996 i915_gem_object_unpin(obj);
997 drm_gem_object_unreference(&obj->base);
998 return ret;
999 }
1000
1001 ring->private = obj;
1002 }
1003
1004 return init_ring_common(ring);
1005 }
1006
1007 static int blt_ring_begin(struct intel_ring_buffer *ring,
1008 int num_dwords)
1009 {
1010 if (ring->private) {
1011 int ret = intel_ring_begin(ring, num_dwords+2);
1012 if (ret)
1013 return ret;
1014
1015 intel_ring_emit(ring, MI_BATCH_BUFFER_START);
1016 intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
1017
1018 return 0;
1019 } else
1020 return intel_ring_begin(ring, 4);
1021 }
1022
1023 static void blt_ring_flush(struct intel_ring_buffer *ring,
1024 u32 invalidate_domains,
1025 u32 flush_domains)
1026 {
1027 if (blt_ring_begin(ring, 4) == 0) {
1028 intel_ring_emit(ring, MI_FLUSH_DW);
1029 intel_ring_emit(ring, 0);
1030 intel_ring_emit(ring, 0);
1031 intel_ring_emit(ring, 0);
1032 intel_ring_advance(ring);
1033 }
1034 }
1035
1036 static int
1037 blt_ring_add_request(struct intel_ring_buffer *ring,
1038 u32 *result)
1039 {
1040 u32 seqno;
1041 int ret;
1042
1043 ret = blt_ring_begin(ring, 4);
1044 if (ret)
1045 return ret;
1046
1047 seqno = i915_gem_get_seqno(ring->dev);
1048
1049 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
1050 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1051 intel_ring_emit(ring, seqno);
1052 intel_ring_emit(ring, MI_USER_INTERRUPT);
1053 intel_ring_advance(ring);
1054
1055 DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
1056 *result = seqno;
1057 return 0;
1058 }
1059
1060 static void blt_ring_cleanup(struct intel_ring_buffer *ring)
1061 {
1062 if (!ring->private)
1063 return;
1064
1065 i915_gem_object_unpin(ring->private);
1066 drm_gem_object_unreference(ring->private);
1067 ring->private = NULL;
1068 }
1069
1070 static const struct intel_ring_buffer gen6_blt_ring = {
1071 .name = "blt ring",
1072 .id = RING_BLT,
1073 .mmio_base = BLT_RING_BASE,
1074 .size = 32 * PAGE_SIZE,
1075 .init = blt_ring_init,
1076 .write_tail = ring_write_tail,
1077 .flush = blt_ring_flush,
1078 .add_request = blt_ring_add_request,
1079 .get_seqno = ring_status_page_get_seqno,
1080 .user_irq_get = blt_ring_get_user_irq,
1081 .user_irq_put = blt_ring_put_user_irq,
1082 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
1083 .cleanup = blt_ring_cleanup,
1084 };
1085
1086 int intel_init_render_ring_buffer(struct drm_device *dev)
1087 {
1088 drm_i915_private_t *dev_priv = dev->dev_private;
1089
1090 dev_priv->render_ring = render_ring;
1091
1092 if (!I915_NEED_GFX_HWS(dev)) {
1093 dev_priv->render_ring.status_page.page_addr
1094 = dev_priv->status_page_dmah->vaddr;
1095 memset(dev_priv->render_ring.status_page.page_addr,
1096 0, PAGE_SIZE);
1097 }
1098
1099 return intel_init_ring_buffer(dev, &dev_priv->render_ring);
1100 }
1101
1102 int intel_init_bsd_ring_buffer(struct drm_device *dev)
1103 {
1104 drm_i915_private_t *dev_priv = dev->dev_private;
1105
1106 if (IS_GEN6(dev))
1107 dev_priv->bsd_ring = gen6_bsd_ring;
1108 else
1109 dev_priv->bsd_ring = bsd_ring;
1110
1111 return intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
1112 }
1113
1114 int intel_init_blt_ring_buffer(struct drm_device *dev)
1115 {
1116 drm_i915_private_t *dev_priv = dev->dev_private;
1117
1118 dev_priv->blt_ring = gen6_blt_ring;
1119
1120 return intel_init_ring_buffer(dev, &dev_priv->blt_ring);
1121 }
Linux kernel - Deliverables
This page took 0.099906 seconds and 5 git commands to generate.