projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
drm/i915: Implement GPU semaphores for inter-ring synchronisation on SNB
[deliverable/linux.git] / drivers / gpu / drm / i915 / intel_ringbuffer.c
1 /*
2 * Copyright © 2008-2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 * Zou Nan hai <nanhai.zou@intel.com>
26 * Xiang Hai hao<haihao.xiang@intel.com>
27 *
28 */
29
30 #include "drmP.h"
31 #include "drm.h"
32 #include "i915_drv.h"
33 #include "i915_drm.h"
34 #include "i915_trace.h"
35 #include "intel_drv.h"
36
37 static u32 i915_gem_get_seqno(struct drm_device *dev)
38 {
39 drm_i915_private_t *dev_priv = dev->dev_private;
40 u32 seqno;
41
42 seqno = dev_priv->next_seqno;
43
44 /* reserve 0 for non-seqno */
45 if (++dev_priv->next_seqno == 0)
46 dev_priv->next_seqno = 1;
47
48 return seqno;
49 }
50
51 static void
52 render_ring_flush(struct intel_ring_buffer *ring,
53 u32 invalidate_domains,
54 u32 flush_domains)
55 {
56 struct drm_device *dev = ring->dev;
57 drm_i915_private_t *dev_priv = dev->dev_private;
58 u32 cmd;
59
60 #if WATCH_EXEC
61 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
62 invalidate_domains, flush_domains);
63 #endif
64
65 trace_i915_gem_request_flush(dev, dev_priv->next_seqno,
66 invalidate_domains, flush_domains);
67
68 if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
69 /*
70 * read/write caches:
71 *
72 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
73 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
74 * also flushed at 2d versus 3d pipeline switches.
75 *
76 * read-only caches:
77 *
78 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
79 * MI_READ_FLUSH is set, and is always flushed on 965.
80 *
81 * I915_GEM_DOMAIN_COMMAND may not exist?
82 *
83 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
84 * invalidated when MI_EXE_FLUSH is set.
85 *
86 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
87 * invalidated with every MI_FLUSH.
88 *
89 * TLBs:
90 *
91 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
92 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
93 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
94 * are flushed at any MI_FLUSH.
95 */
96
97 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
98 if ((invalidate_domains|flush_domains) &
99 I915_GEM_DOMAIN_RENDER)
100 cmd &= ~MI_NO_WRITE_FLUSH;
101 if (INTEL_INFO(dev)->gen < 4) {
102 /*
103 * On the 965, the sampler cache always gets flushed
104 * and this bit is reserved.
105 */
106 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
107 cmd |= MI_READ_FLUSH;
108 }
109 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
110 cmd |= MI_EXE_FLUSH;
111
112 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
113 (IS_G4X(dev) || IS_GEN5(dev)))
114 cmd |= MI_INVALIDATE_ISP;
115
116 #if WATCH_EXEC
117 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
118 #endif
119 if (intel_ring_begin(ring, 2) == 0) {
120 intel_ring_emit(ring, cmd);
121 intel_ring_emit(ring, MI_NOOP);
122 intel_ring_advance(ring);
123 }
124 }
125 }
126
127 static void ring_write_tail(struct intel_ring_buffer *ring,
128 u32 value)
129 {
130 drm_i915_private_t *dev_priv = ring->dev->dev_private;
131 I915_WRITE_TAIL(ring, value);
132 }
133
134 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
135 {
136 drm_i915_private_t *dev_priv = ring->dev->dev_private;
137 u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
138 RING_ACTHD(ring->mmio_base) : ACTHD;
139
140 return I915_READ(acthd_reg);
141 }
142
143 static int init_ring_common(struct intel_ring_buffer *ring)
144 {
145 drm_i915_private_t *dev_priv = ring->dev->dev_private;
146 struct drm_i915_gem_object *obj = ring->obj;
147 u32 head;
148
149 /* Stop the ring if it's running. */
150 I915_WRITE_CTL(ring, 0);
151 I915_WRITE_HEAD(ring, 0);
152 ring->write_tail(ring, 0);
153
154 /* Initialize the ring. */
155 I915_WRITE_START(ring, obj->gtt_offset);
156 head = I915_READ_HEAD(ring) & HEAD_ADDR;
157
158 /* G45 ring initialization fails to reset head to zero */
159 if (head != 0) {
160 DRM_ERROR("%s head not reset to zero "
161 "ctl %08x head %08x tail %08x start %08x\n",
162 ring->name,
163 I915_READ_CTL(ring),
164 I915_READ_HEAD(ring),
165 I915_READ_TAIL(ring),
166 I915_READ_START(ring));
167
168 I915_WRITE_HEAD(ring, 0);
169
170 DRM_ERROR("%s head forced to zero "
171 "ctl %08x head %08x tail %08x start %08x\n",
172 ring->name,
173 I915_READ_CTL(ring),
174 I915_READ_HEAD(ring),
175 I915_READ_TAIL(ring),
176 I915_READ_START(ring));
177 }
178
179 I915_WRITE_CTL(ring,
180 ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
181 | RING_REPORT_64K | RING_VALID);
182
183 /* If the head is still not zero, the ring is dead */
184 if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
185 I915_READ_START(ring) != obj->gtt_offset ||
186 (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
187 DRM_ERROR("%s initialization failed "
188 "ctl %08x head %08x tail %08x start %08x\n",
189 ring->name,
190 I915_READ_CTL(ring),
191 I915_READ_HEAD(ring),
192 I915_READ_TAIL(ring),
193 I915_READ_START(ring));
194 return -EIO;
195 }
196
197 if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
198 i915_kernel_lost_context(ring->dev);
199 else {
200 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
201 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
202 ring->space = ring->head - (ring->tail + 8);
203 if (ring->space < 0)
204 ring->space += ring->size;
205 }
206
207 return 0;
208 }
209
210 /*
211 * 965+ support PIPE_CONTROL commands, which provide finer grained control
212 * over cache flushing.
213 */
214 struct pipe_control {
215 struct drm_i915_gem_object *obj;
216 volatile u32 *cpu_page;
217 u32 gtt_offset;
218 };
219
220 static int
221 init_pipe_control(struct intel_ring_buffer *ring)
222 {
223 struct pipe_control *pc;
224 struct drm_i915_gem_object *obj;
225 int ret;
226
227 if (ring->private)
228 return 0;
229
230 pc = kmalloc(sizeof(*pc), GFP_KERNEL);
231 if (!pc)
232 return -ENOMEM;
233
234 obj = i915_gem_alloc_object(ring->dev, 4096);
235 if (obj == NULL) {
236 DRM_ERROR("Failed to allocate seqno page\n");
237 ret = -ENOMEM;
238 goto err;
239 }
240 obj->agp_type = AGP_USER_CACHED_MEMORY;
241
242 ret = i915_gem_object_pin(obj, 4096, true);
243 if (ret)
244 goto err_unref;
245
246 pc->gtt_offset = obj->gtt_offset;
247 pc->cpu_page = kmap(obj->pages[0]);
248 if (pc->cpu_page == NULL)
249 goto err_unpin;
250
251 pc->obj = obj;
252 ring->private = pc;
253 return 0;
254
255 err_unpin:
256 i915_gem_object_unpin(obj);
257 err_unref:
258 drm_gem_object_unreference(&obj->base);
259 err:
260 kfree(pc);
261 return ret;
262 }
263
264 static void
265 cleanup_pipe_control(struct intel_ring_buffer *ring)
266 {
267 struct pipe_control *pc = ring->private;
268 struct drm_i915_gem_object *obj;
269
270 if (!ring->private)
271 return;
272
273 obj = pc->obj;
274 kunmap(obj->pages[0]);
275 i915_gem_object_unpin(obj);
276 drm_gem_object_unreference(&obj->base);
277
278 kfree(pc);
279 ring->private = NULL;
280 }
281
282 static int init_render_ring(struct intel_ring_buffer *ring)
283 {
284 struct drm_device *dev = ring->dev;
285 struct drm_i915_private *dev_priv = dev->dev_private;
286 int ret = init_ring_common(ring);
287
288 if (INTEL_INFO(dev)->gen > 3) {
289 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
290 if (IS_GEN6(dev))
291 mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
292 I915_WRITE(MI_MODE, mode);
293 }
294
295 if (INTEL_INFO(dev)->gen >= 6) {
296 } else if (HAS_PIPE_CONTROL(dev)) {
297 ret = init_pipe_control(ring);
298 if (ret)
299 return ret;
300 }
301
302 return ret;
303 }
304
305 static void render_ring_cleanup(struct intel_ring_buffer *ring)
306 {
307 if (!ring->private)
308 return;
309
310 cleanup_pipe_control(ring);
311 }
312
313 static void
314 update_semaphore(struct intel_ring_buffer *ring, int i, u32 seqno)
315 {
316 struct drm_device *dev = ring->dev;
317 struct drm_i915_private *dev_priv = dev->dev_private;
318 int id;
319
320 /*
321 * cs -> 1 = vcs, 0 = bcs
322 * vcs -> 1 = bcs, 0 = cs,
323 * bcs -> 1 = cs, 0 = vcs.
324 */
325 id = ring - dev_priv->ring;
326 id += 2 - i;
327 id %= 3;
328
329 intel_ring_emit(ring,
330 MI_SEMAPHORE_MBOX |
331 MI_SEMAPHORE_REGISTER |
332 MI_SEMAPHORE_UPDATE);
333 intel_ring_emit(ring, seqno);
334 intel_ring_emit(ring,
335 RING_SYNC_0(dev_priv->ring[id].mmio_base) + 4*i);
336 }
337
338 static int
339 gen6_add_request(struct intel_ring_buffer *ring,
340 u32 *result)
341 {
342 u32 seqno;
343 int ret;
344
345 ret = intel_ring_begin(ring, 10);
346 if (ret)
347 return ret;
348
349 seqno = i915_gem_get_seqno(ring->dev);
350 update_semaphore(ring, 0, seqno);
351 update_semaphore(ring, 1, seqno);
352
353 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
354 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
355 intel_ring_emit(ring, seqno);
356 intel_ring_emit(ring, MI_USER_INTERRUPT);
357 intel_ring_advance(ring);
358
359 *result = seqno;
360 return 0;
361 }
362
363 int
364 intel_ring_sync(struct intel_ring_buffer *ring,
365 struct intel_ring_buffer *to,
366 u32 seqno)
367 {
368 int ret;
369
370 ret = intel_ring_begin(ring, 4);
371 if (ret)
372 return ret;
373
374 intel_ring_emit(ring,
375 MI_SEMAPHORE_MBOX |
376 MI_SEMAPHORE_REGISTER |
377 intel_ring_sync_index(ring, to) << 17 |
378 MI_SEMAPHORE_COMPARE);
379 intel_ring_emit(ring, seqno);
380 intel_ring_emit(ring, 0);
381 intel_ring_emit(ring, MI_NOOP);
382 intel_ring_advance(ring);
383
384 return 0;
385 }
386
387 #define PIPE_CONTROL_FLUSH(ring__, addr__) \
388 do { \
389 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \
390 PIPE_CONTROL_DEPTH_STALL | 2); \
391 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
392 intel_ring_emit(ring__, 0); \
393 intel_ring_emit(ring__, 0); \
394 } while (0)
395
396 static int
397 pc_render_add_request(struct intel_ring_buffer *ring,
398 u32 *result)
399 {
400 struct drm_device *dev = ring->dev;
401 u32 seqno = i915_gem_get_seqno(dev);
402 struct pipe_control *pc = ring->private;
403 u32 scratch_addr = pc->gtt_offset + 128;
404 int ret;
405
406 /*
407 * Workaround qword write incoherence by flushing the
408 * PIPE_NOTIFY buffers out to memory before requesting
409 * an interrupt.
410 */
411 ret = intel_ring_begin(ring, 32);
412 if (ret)
413 return ret;
414
415 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
416 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
417 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
418 intel_ring_emit(ring, seqno);
419 intel_ring_emit(ring, 0);
420 PIPE_CONTROL_FLUSH(ring, scratch_addr);
421 scratch_addr += 128; /* write to separate cachelines */
422 PIPE_CONTROL_FLUSH(ring, scratch_addr);
423 scratch_addr += 128;
424 PIPE_CONTROL_FLUSH(ring, scratch_addr);
425 scratch_addr += 128;
426 PIPE_CONTROL_FLUSH(ring, scratch_addr);
427 scratch_addr += 128;
428 PIPE_CONTROL_FLUSH(ring, scratch_addr);
429 scratch_addr += 128;
430 PIPE_CONTROL_FLUSH(ring, scratch_addr);
431 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
432 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
433 PIPE_CONTROL_NOTIFY);
434 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
435 intel_ring_emit(ring, seqno);
436 intel_ring_emit(ring, 0);
437 intel_ring_advance(ring);
438
439 *result = seqno;
440 return 0;
441 }
442
443 static int
444 render_ring_add_request(struct intel_ring_buffer *ring,
445 u32 *result)
446 {
447 struct drm_device *dev = ring->dev;
448 u32 seqno = i915_gem_get_seqno(dev);
449 int ret;
450
451 ret = intel_ring_begin(ring, 4);
452 if (ret)
453 return ret;
454
455 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
456 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
457 intel_ring_emit(ring, seqno);
458 intel_ring_emit(ring, MI_USER_INTERRUPT);
459 intel_ring_advance(ring);
460
461 *result = seqno;
462 return 0;
463 }
464
465 static u32
466 ring_get_seqno(struct intel_ring_buffer *ring)
467 {
468 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
469 }
470
471 static u32
472 pc_render_get_seqno(struct intel_ring_buffer *ring)
473 {
474 struct pipe_control *pc = ring->private;
475 return pc->cpu_page[0];
476 }
477
478 static void
479 render_ring_get_irq(struct intel_ring_buffer *ring)
480 {
481 struct drm_device *dev = ring->dev;
482
483 if (dev->irq_enabled && ++ring->irq_refcount == 1) {
484 drm_i915_private_t *dev_priv = dev->dev_private;
485 unsigned long irqflags;
486
487 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
488
489 if (HAS_PCH_SPLIT(dev))
490 ironlake_enable_graphics_irq(dev_priv,
491 GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
492 else
493 i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
494
495 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
496 }
497 }
498
499 static void
500 render_ring_put_irq(struct intel_ring_buffer *ring)
501 {
502 struct drm_device *dev = ring->dev;
503
504 BUG_ON(dev->irq_enabled && ring->irq_refcount == 0);
505 if (dev->irq_enabled && --ring->irq_refcount == 0) {
506 drm_i915_private_t *dev_priv = dev->dev_private;
507 unsigned long irqflags;
508
509 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
510 if (HAS_PCH_SPLIT(dev))
511 ironlake_disable_graphics_irq(dev_priv,
512 GT_USER_INTERRUPT |
513 GT_PIPE_NOTIFY);
514 else
515 i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
516 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
517 }
518 }
519
520 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
521 {
522 drm_i915_private_t *dev_priv = ring->dev->dev_private;
523 u32 mmio = IS_GEN6(ring->dev) ?
524 RING_HWS_PGA_GEN6(ring->mmio_base) :
525 RING_HWS_PGA(ring->mmio_base);
526 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
527 POSTING_READ(mmio);
528 }
529
530 static void
531 bsd_ring_flush(struct intel_ring_buffer *ring,
532 u32 invalidate_domains,
533 u32 flush_domains)
534 {
535 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
536 return;
537
538 if (intel_ring_begin(ring, 2) == 0) {
539 intel_ring_emit(ring, MI_FLUSH);
540 intel_ring_emit(ring, MI_NOOP);
541 intel_ring_advance(ring);
542 }
543 }
544
545 static int
546 ring_add_request(struct intel_ring_buffer *ring,
547 u32 *result)
548 {
549 u32 seqno;
550 int ret;
551
552 ret = intel_ring_begin(ring, 4);
553 if (ret)
554 return ret;
555
556 seqno = i915_gem_get_seqno(ring->dev);
557
558 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
559 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
560 intel_ring_emit(ring, seqno);
561 intel_ring_emit(ring, MI_USER_INTERRUPT);
562 intel_ring_advance(ring);
563
564 DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
565 *result = seqno;
566 return 0;
567 }
568
569 static void
570 ring_get_irq(struct intel_ring_buffer *ring, u32 flag)
571 {
572 struct drm_device *dev = ring->dev;
573
574 if (dev->irq_enabled && ++ring->irq_refcount == 1) {
575 drm_i915_private_t *dev_priv = dev->dev_private;
576 unsigned long irqflags;
577
578 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
579 ironlake_enable_graphics_irq(dev_priv, flag);
580 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
581 }
582 }
583
584 static void
585 ring_put_irq(struct intel_ring_buffer *ring, u32 flag)
586 {
587 struct drm_device *dev = ring->dev;
588
589 if (dev->irq_enabled && --ring->irq_refcount == 0) {
590 drm_i915_private_t *dev_priv = dev->dev_private;
591 unsigned long irqflags;
592
593 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
594 ironlake_disable_graphics_irq(dev_priv, flag);
595 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
596 }
597 }
598
599
600 static void
601 bsd_ring_get_irq(struct intel_ring_buffer *ring)
602 {
603 ring_get_irq(ring, GT_BSD_USER_INTERRUPT);
604 }
605 static void
606 bsd_ring_put_irq(struct intel_ring_buffer *ring)
607 {
608 ring_put_irq(ring, GT_BSD_USER_INTERRUPT);
609 }
610
611 static int
612 ring_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
613 {
614 int ret;
615
616 ret = intel_ring_begin(ring, 2);
617 if (ret)
618 return ret;
619
620 intel_ring_emit(ring,
621 MI_BATCH_BUFFER_START | (2 << 6) |
622 MI_BATCH_NON_SECURE_I965);
623 intel_ring_emit(ring, offset);
624 intel_ring_advance(ring);
625
626 return 0;
627 }
628
629 static int
630 render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
631 u32 offset, u32 len)
632 {
633 struct drm_device *dev = ring->dev;
634 drm_i915_private_t *dev_priv = dev->dev_private;
635 int ret;
636
637 trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
638
639 if (IS_I830(dev) || IS_845G(dev)) {
640 ret = intel_ring_begin(ring, 4);
641 if (ret)
642 return ret;
643
644 intel_ring_emit(ring, MI_BATCH_BUFFER);
645 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
646 intel_ring_emit(ring, offset + len - 8);
647 intel_ring_emit(ring, 0);
648 } else {
649 ret = intel_ring_begin(ring, 2);
650 if (ret)
651 return ret;
652
653 if (INTEL_INFO(dev)->gen >= 4) {
654 intel_ring_emit(ring,
655 MI_BATCH_BUFFER_START | (2 << 6) |
656 MI_BATCH_NON_SECURE_I965);
657 intel_ring_emit(ring, offset);
658 } else {
659 intel_ring_emit(ring,
660 MI_BATCH_BUFFER_START | (2 << 6));
661 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
662 }
663 }
664 intel_ring_advance(ring);
665
666 return 0;
667 }
668
669 static void cleanup_status_page(struct intel_ring_buffer *ring)
670 {
671 drm_i915_private_t *dev_priv = ring->dev->dev_private;
672 struct drm_i915_gem_object *obj;
673
674 obj = ring->status_page.obj;
675 if (obj == NULL)
676 return;
677
678 kunmap(obj->pages[0]);
679 i915_gem_object_unpin(obj);
680 drm_gem_object_unreference(&obj->base);
681 ring->status_page.obj = NULL;
682
683 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
684 }
685
686 static int init_status_page(struct intel_ring_buffer *ring)
687 {
688 struct drm_device *dev = ring->dev;
689 drm_i915_private_t *dev_priv = dev->dev_private;
690 struct drm_i915_gem_object *obj;
691 int ret;
692
693 obj = i915_gem_alloc_object(dev, 4096);
694 if (obj == NULL) {
695 DRM_ERROR("Failed to allocate status page\n");
696 ret = -ENOMEM;
697 goto err;
698 }
699 obj->agp_type = AGP_USER_CACHED_MEMORY;
700
701 ret = i915_gem_object_pin(obj, 4096, true);
702 if (ret != 0) {
703 goto err_unref;
704 }
705
706 ring->status_page.gfx_addr = obj->gtt_offset;
707 ring->status_page.page_addr = kmap(obj->pages[0]);
708 if (ring->status_page.page_addr == NULL) {
709 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
710 goto err_unpin;
711 }
712 ring->status_page.obj = obj;
713 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
714
715 intel_ring_setup_status_page(ring);
716 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
717 ring->name, ring->status_page.gfx_addr);
718
719 return 0;
720
721 err_unpin:
722 i915_gem_object_unpin(obj);
723 err_unref:
724 drm_gem_object_unreference(&obj->base);
725 err:
726 return ret;
727 }
728
729 int intel_init_ring_buffer(struct drm_device *dev,
730 struct intel_ring_buffer *ring)
731 {
732 struct drm_i915_gem_object *obj;
733 int ret;
734
735 ring->dev = dev;
736 INIT_LIST_HEAD(&ring->active_list);
737 INIT_LIST_HEAD(&ring->request_list);
738 INIT_LIST_HEAD(&ring->gpu_write_list);
739
740 if (I915_NEED_GFX_HWS(dev)) {
741 ret = init_status_page(ring);
742 if (ret)
743 return ret;
744 }
745
746 obj = i915_gem_alloc_object(dev, ring->size);
747 if (obj == NULL) {
748 DRM_ERROR("Failed to allocate ringbuffer\n");
749 ret = -ENOMEM;
750 goto err_hws;
751 }
752
753 ring->obj = obj;
754
755 ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
756 if (ret)
757 goto err_unref;
758
759 ring->map.size = ring->size;
760 ring->map.offset = dev->agp->base + obj->gtt_offset;
761 ring->map.type = 0;
762 ring->map.flags = 0;
763 ring->map.mtrr = 0;
764
765 drm_core_ioremap_wc(&ring->map, dev);
766 if (ring->map.handle == NULL) {
767 DRM_ERROR("Failed to map ringbuffer.\n");
768 ret = -EINVAL;
769 goto err_unpin;
770 }
771
772 ring->virtual_start = ring->map.handle;
773 ret = ring->init(ring);
774 if (ret)
775 goto err_unmap;
776
777 return 0;
778
779 err_unmap:
780 drm_core_ioremapfree(&ring->map, dev);
781 err_unpin:
782 i915_gem_object_unpin(obj);
783 err_unref:
784 drm_gem_object_unreference(&obj->base);
785 ring->obj = NULL;
786 err_hws:
787 cleanup_status_page(ring);
788 return ret;
789 }
790
791 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
792 {
793 struct drm_i915_private *dev_priv;
794 int ret;
795
796 if (ring->obj == NULL)
797 return;
798
799 /* Disable the ring buffer. The ring must be idle at this point */
800 dev_priv = ring->dev->dev_private;
801 ret = intel_wait_ring_buffer(ring, ring->size - 8);
802 I915_WRITE_CTL(ring, 0);
803
804 drm_core_ioremapfree(&ring->map, ring->dev);
805
806 i915_gem_object_unpin(ring->obj);
807 drm_gem_object_unreference(&ring->obj->base);
808 ring->obj = NULL;
809
810 if (ring->cleanup)
811 ring->cleanup(ring);
812
813 cleanup_status_page(ring);
814 }
815
816 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
817 {
818 unsigned int *virt;
819 int rem;
820 rem = ring->size - ring->tail;
821
822 if (ring->space < rem) {
823 int ret = intel_wait_ring_buffer(ring, rem);
824 if (ret)
825 return ret;
826 }
827
828 virt = (unsigned int *)(ring->virtual_start + ring->tail);
829 rem /= 8;
830 while (rem--) {
831 *virt++ = MI_NOOP;
832 *virt++ = MI_NOOP;
833 }
834
835 ring->tail = 0;
836 ring->space = ring->head - 8;
837
838 return 0;
839 }
840
841 int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
842 {
843 struct drm_device *dev = ring->dev;
844 struct drm_i915_private *dev_priv = dev->dev_private;
845 unsigned long end;
846 u32 head;
847
848 head = intel_read_status_page(ring, 4);
849 if (head) {
850 ring->head = head & HEAD_ADDR;
851 ring->space = ring->head - (ring->tail + 8);
852 if (ring->space < 0)
853 ring->space += ring->size;
854 if (ring->space >= n)
855 return 0;
856 }
857
858 trace_i915_ring_wait_begin (dev);
859 end = jiffies + 3 * HZ;
860 do {
861 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
862 ring->space = ring->head - (ring->tail + 8);
863 if (ring->space < 0)
864 ring->space += ring->size;
865 if (ring->space >= n) {
866 trace_i915_ring_wait_end(dev);
867 return 0;
868 }
869
870 if (dev->primary->master) {
871 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
872 if (master_priv->sarea_priv)
873 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
874 }
875
876 msleep(1);
877 if (atomic_read(&dev_priv->mm.wedged))
878 return -EAGAIN;
879 } while (!time_after(jiffies, end));
880 trace_i915_ring_wait_end (dev);
881 return -EBUSY;
882 }
883
884 int intel_ring_begin(struct intel_ring_buffer *ring,
885 int num_dwords)
886 {
887 int n = 4*num_dwords;
888 int ret;
889
890 if (unlikely(ring->tail + n > ring->size)) {
891 ret = intel_wrap_ring_buffer(ring);
892 if (unlikely(ret))
893 return ret;
894 }
895
896 if (unlikely(ring->space < n)) {
897 ret = intel_wait_ring_buffer(ring, n);
898 if (unlikely(ret))
899 return ret;
900 }
901
902 ring->space -= n;
903 return 0;
904 }
905
906 void intel_ring_advance(struct intel_ring_buffer *ring)
907 {
908 ring->tail &= ring->size - 1;
909 ring->write_tail(ring, ring->tail);
910 }
911
912 static const struct intel_ring_buffer render_ring = {
913 .name = "render ring",
914 .id = RING_RENDER,
915 .mmio_base = RENDER_RING_BASE,
916 .size = 32 * PAGE_SIZE,
917 .init = init_render_ring,
918 .write_tail = ring_write_tail,
919 .flush = render_ring_flush,
920 .add_request = render_ring_add_request,
921 .get_seqno = ring_get_seqno,
922 .irq_get = render_ring_get_irq,
923 .irq_put = render_ring_put_irq,
924 .dispatch_execbuffer = render_ring_dispatch_execbuffer,
925 .cleanup = render_ring_cleanup,
926 };
927
928 /* ring buffer for bit-stream decoder */
929
930 static const struct intel_ring_buffer bsd_ring = {
931 .name = "bsd ring",
932 .id = RING_BSD,
933 .mmio_base = BSD_RING_BASE,
934 .size = 32 * PAGE_SIZE,
935 .init = init_ring_common,
936 .write_tail = ring_write_tail,
937 .flush = bsd_ring_flush,
938 .add_request = ring_add_request,
939 .get_seqno = ring_get_seqno,
940 .irq_get = bsd_ring_get_irq,
941 .irq_put = bsd_ring_put_irq,
942 .dispatch_execbuffer = ring_dispatch_execbuffer,
943 };
944
945
946 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
947 u32 value)
948 {
949 drm_i915_private_t *dev_priv = ring->dev->dev_private;
950
951 /* Every tail move must follow the sequence below */
952 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
953 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
954 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
955 I915_WRITE(GEN6_BSD_RNCID, 0x0);
956
957 if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
958 GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
959 50))
960 DRM_ERROR("timed out waiting for IDLE Indicator\n");
961
962 I915_WRITE_TAIL(ring, value);
963 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
964 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
965 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
966 }
967
968 static void gen6_ring_flush(struct intel_ring_buffer *ring,
969 u32 invalidate_domains,
970 u32 flush_domains)
971 {
972 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
973 return;
974
975 if (intel_ring_begin(ring, 4) == 0) {
976 intel_ring_emit(ring, MI_FLUSH_DW);
977 intel_ring_emit(ring, 0);
978 intel_ring_emit(ring, 0);
979 intel_ring_emit(ring, 0);
980 intel_ring_advance(ring);
981 }
982 }
983
984 static int
985 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
986 u32 offset, u32 len)
987 {
988 int ret;
989
990 ret = intel_ring_begin(ring, 2);
991 if (ret)
992 return ret;
993
994 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
995 /* bit0-7 is the length on GEN6+ */
996 intel_ring_emit(ring, offset);
997 intel_ring_advance(ring);
998
999 return 0;
1000 }
1001
1002 static void
1003 gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
1004 {
1005 ring_get_irq(ring, GT_GEN6_BSD_USER_INTERRUPT);
1006 }
1007
1008 static void
1009 gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
1010 {
1011 ring_put_irq(ring, GT_GEN6_BSD_USER_INTERRUPT);
1012 }
1013
1014 /* ring buffer for Video Codec for Gen6+ */
1015 static const struct intel_ring_buffer gen6_bsd_ring = {
1016 .name = "gen6 bsd ring",
1017 .id = RING_BSD,
1018 .mmio_base = GEN6_BSD_RING_BASE,
1019 .size = 32 * PAGE_SIZE,
1020 .init = init_ring_common,
1021 .write_tail = gen6_bsd_ring_write_tail,
1022 .flush = gen6_ring_flush,
1023 .add_request = gen6_add_request,
1024 .get_seqno = ring_get_seqno,
1025 .irq_get = gen6_bsd_ring_get_irq,
1026 .irq_put = gen6_bsd_ring_put_irq,
1027 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
1028 };
1029
1030 /* Blitter support (SandyBridge+) */
1031
1032 static void
1033 blt_ring_get_irq(struct intel_ring_buffer *ring)
1034 {
1035 ring_get_irq(ring, GT_BLT_USER_INTERRUPT);
1036 }
1037
1038 static void
1039 blt_ring_put_irq(struct intel_ring_buffer *ring)
1040 {
1041 ring_put_irq(ring, GT_BLT_USER_INTERRUPT);
1042 }
1043
1044
1045 /* Workaround for some stepping of SNB,
1046 * each time when BLT engine ring tail moved,
1047 * the first command in the ring to be parsed
1048 * should be MI_BATCH_BUFFER_START
1049 */
1050 #define NEED_BLT_WORKAROUND(dev) \
1051 (IS_GEN6(dev) && (dev->pdev->revision < 8))
1052
1053 static inline struct drm_i915_gem_object *
1054 to_blt_workaround(struct intel_ring_buffer *ring)
1055 {
1056 return ring->private;
1057 }
1058
1059 static int blt_ring_init(struct intel_ring_buffer *ring)
1060 {
1061 if (NEED_BLT_WORKAROUND(ring->dev)) {
1062 struct drm_i915_gem_object *obj;
1063 u32 *ptr;
1064 int ret;
1065
1066 obj = i915_gem_alloc_object(ring->dev, 4096);
1067 if (obj == NULL)
1068 return -ENOMEM;
1069
1070 ret = i915_gem_object_pin(obj, 4096, true);
1071 if (ret) {
1072 drm_gem_object_unreference(&obj->base);
1073 return ret;
1074 }
1075
1076 ptr = kmap(obj->pages[0]);
1077 *ptr++ = MI_BATCH_BUFFER_END;
1078 *ptr++ = MI_NOOP;
1079 kunmap(obj->pages[0]);
1080
1081 ret = i915_gem_object_set_to_gtt_domain(obj, false);
1082 if (ret) {
1083 i915_gem_object_unpin(obj);
1084 drm_gem_object_unreference(&obj->base);
1085 return ret;
1086 }
1087
1088 ring->private = obj;
1089 }
1090
1091 return init_ring_common(ring);
1092 }
1093
1094 static int blt_ring_begin(struct intel_ring_buffer *ring,
1095 int num_dwords)
1096 {
1097 if (ring->private) {
1098 int ret = intel_ring_begin(ring, num_dwords+2);
1099 if (ret)
1100 return ret;
1101
1102 intel_ring_emit(ring, MI_BATCH_BUFFER_START);
1103 intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
1104
1105 return 0;
1106 } else
1107 return intel_ring_begin(ring, 4);
1108 }
1109
1110 static void blt_ring_flush(struct intel_ring_buffer *ring,
1111 u32 invalidate_domains,
1112 u32 flush_domains)
1113 {
1114 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
1115 return;
1116
1117 if (blt_ring_begin(ring, 4) == 0) {
1118 intel_ring_emit(ring, MI_FLUSH_DW);
1119 intel_ring_emit(ring, 0);
1120 intel_ring_emit(ring, 0);
1121 intel_ring_emit(ring, 0);
1122 intel_ring_advance(ring);
1123 }
1124 }
1125
1126 static void blt_ring_cleanup(struct intel_ring_buffer *ring)
1127 {
1128 if (!ring->private)
1129 return;
1130
1131 i915_gem_object_unpin(ring->private);
1132 drm_gem_object_unreference(ring->private);
1133 ring->private = NULL;
1134 }
1135
1136 static const struct intel_ring_buffer gen6_blt_ring = {
1137 .name = "blt ring",
1138 .id = RING_BLT,
1139 .mmio_base = BLT_RING_BASE,
1140 .size = 32 * PAGE_SIZE,
1141 .init = blt_ring_init,
1142 .write_tail = ring_write_tail,
1143 .flush = blt_ring_flush,
1144 .add_request = gen6_add_request,
1145 .get_seqno = ring_get_seqno,
1146 .irq_get = blt_ring_get_irq,
1147 .irq_put = blt_ring_put_irq,
1148 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
1149 .cleanup = blt_ring_cleanup,
1150 };
1151
1152 int intel_init_render_ring_buffer(struct drm_device *dev)
1153 {
1154 drm_i915_private_t *dev_priv = dev->dev_private;
1155 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1156
1157 *ring = render_ring;
1158 if (INTEL_INFO(dev)->gen >= 6) {
1159 ring->add_request = gen6_add_request;
1160 } else if (HAS_PIPE_CONTROL(dev)) {
1161 ring->add_request = pc_render_add_request;
1162 ring->get_seqno = pc_render_get_seqno;
1163 }
1164
1165 if (!I915_NEED_GFX_HWS(dev)) {
1166 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1167 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1168 }
1169
1170 return intel_init_ring_buffer(dev, ring);
1171 }
1172
1173 int intel_init_bsd_ring_buffer(struct drm_device *dev)
1174 {
1175 drm_i915_private_t *dev_priv = dev->dev_private;
1176 struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
1177
1178 if (IS_GEN6(dev))
1179 *ring = gen6_bsd_ring;
1180 else
1181 *ring = bsd_ring;
1182
1183 return intel_init_ring_buffer(dev, ring);
1184 }
1185
1186 int intel_init_blt_ring_buffer(struct drm_device *dev)
1187 {
1188 drm_i915_private_t *dev_priv = dev->dev_private;
1189 struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
1190
1191 *ring = gen6_blt_ring;
1192
1193 return intel_init_ring_buffer(dev, ring);
1194 }
Linux kernel - Deliverables
This page took 0.072724 seconds and 5 git commands to generate.