projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915/ringbuffer: Simplify the ring irq refcounting
[deliverable/linux.git] / drivers / gpu / drm / i915 / intel_ringbuffer.c
CommitLineData
62fdfeaf
EA		 1/*
2 * Copyright © 2008-2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 * Zou Nan hai <nanhai.zou@intel.com>
26 * Xiang Hai hao<haihao.xiang@intel.com>
27 *
28 */
29
30#include "drmP.h"
31#include "drm.h"
62fdfeaf 32#include "i915_drv.h"
8187a2b7 33#include "i915_drm.h"
62fdfeaf 34#include "i915_trace.h"
881f47b6 35#include "intel_drv.h"
62fdfeaf 36
6f392d54
CW		 37static u32 i915_gem_get_seqno(struct drm_device *dev)
38{
39 drm_i915_private_t *dev_priv = dev->dev_private;
40 u32 seqno;
41
42 seqno = dev_priv->next_seqno;
43
44 /* reserve 0 for non-seqno */
45 if (++dev_priv->next_seqno == 0)
46 dev_priv->next_seqno = 1;
47
48 return seqno;
49}
50
b72f3acb 51static int
78501eac 52render_ring_flush(struct intel_ring_buffer *ring,
ab6f8e32
CW		 53 u32 invalidate_domains,
54 u32 flush_domains)
62fdfeaf 55{
78501eac 56 struct drm_device *dev = ring->dev;
6f392d54
CW		 57 drm_i915_private_t *dev_priv = dev->dev_private;
58 u32 cmd;
b72f3acb 59 int ret;
6f392d54 60
62fdfeaf
EA		 61#if WATCH_EXEC
62 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
63 invalidate_domains, flush_domains);
64#endif
6f392d54
CW		 65
66 trace_i915_gem_request_flush(dev, dev_priv->next_seqno,
62fdfeaf
EA		 67 invalidate_domains, flush_domains);
68
62fdfeaf
EA		 69 if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
70 /*
71 * read/write caches:
72 *
73 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
74 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
75 * also flushed at 2d versus 3d pipeline switches.
76 *
77 * read-only caches:
78 *
79 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
80 * MI_READ_FLUSH is set, and is always flushed on 965.
81 *
82 * I915_GEM_DOMAIN_COMMAND may not exist?
83 *
84 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
85 * invalidated when MI_EXE_FLUSH is set.
86 *
87 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
88 * invalidated with every MI_FLUSH.
89 *
90 * TLBs:
91 *
92 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
93 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
94 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
95 * are flushed at any MI_FLUSH.
96 */
97
98 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
99 if ((invalidate_domains|flush_domains) &
100 I915_GEM_DOMAIN_RENDER)
101 cmd &= ~MI_NO_WRITE_FLUSH;
a6c45cf0 102 if (INTEL_INFO(dev)->gen < 4) {
62fdfeaf
EA		 103 /*
104 * On the 965, the sampler cache always gets flushed
105 * and this bit is reserved.
106 */
107 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
108 cmd |= MI_READ_FLUSH;
109 }
110 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
111 cmd |= MI_EXE_FLUSH;
112
70eac33e
CW		 113 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
114 (IS_G4X(dev) || IS_GEN5(dev)))
115 cmd |= MI_INVALIDATE_ISP;
116
62fdfeaf
EA		 117#if WATCH_EXEC
118 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
119#endif
b72f3acb
CW		 120 ret = intel_ring_begin(ring, 2);
121 if (ret)
122 return ret;
123
124 intel_ring_emit(ring, cmd);
125 intel_ring_emit(ring, MI_NOOP);
126 intel_ring_advance(ring);
62fdfeaf 127 }
b72f3acb
CW		 128
129 return 0;
8187a2b7
ZN		 130}
131
78501eac 132static void ring_write_tail(struct intel_ring_buffer *ring,
297b0c5b 133 u32 value)
d46eefa2 134{
78501eac 135 drm_i915_private_t *dev_priv = ring->dev->dev_private;
297b0c5b 136 I915_WRITE_TAIL(ring, value);
d46eefa2
XH		 137}
138
78501eac 139u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
8187a2b7 140{
78501eac
CW		 141 drm_i915_private_t *dev_priv = ring->dev->dev_private;
142 u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
3d281d8c 143 RING_ACTHD(ring->mmio_base) : ACTHD;
8187a2b7
ZN		 144
145 return I915_READ(acthd_reg);
146}
147
78501eac 148static int init_ring_common(struct intel_ring_buffer *ring)
8187a2b7 149{
78501eac 150 drm_i915_private_t *dev_priv = ring->dev->dev_private;
05394f39 151 struct drm_i915_gem_object *obj = ring->obj;
8187a2b7 152 u32 head;
8187a2b7
ZN		 153
154 /* Stop the ring if it's running. */
7f2ab699 155 I915_WRITE_CTL(ring, 0);
570ef608 156 I915_WRITE_HEAD(ring, 0);
78501eac 157 ring->write_tail(ring, 0);
8187a2b7
ZN		 158
159 /* Initialize the ring. */
05394f39 160 I915_WRITE_START(ring, obj->gtt_offset);
570ef608 161 head = I915_READ_HEAD(ring) & HEAD_ADDR;
8187a2b7
ZN		 162
163 /* G45 ring initialization fails to reset head to zero */
164 if (head != 0) {
6fd0d56e
CW		 165 DRM_DEBUG_KMS("%s head not reset to zero "
166 "ctl %08x head %08x tail %08x start %08x\n",
167 ring->name,
168 I915_READ_CTL(ring),
169 I915_READ_HEAD(ring),
170 I915_READ_TAIL(ring),
171 I915_READ_START(ring));
8187a2b7 172
570ef608 173 I915_WRITE_HEAD(ring, 0);
8187a2b7 174
6fd0d56e
CW		 175 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
176 DRM_ERROR("failed to set %s head to zero "
177 "ctl %08x head %08x tail %08x start %08x\n",
178 ring->name,
179 I915_READ_CTL(ring),
180 I915_READ_HEAD(ring),
181 I915_READ_TAIL(ring),
182 I915_READ_START(ring));
183 }
8187a2b7
ZN		 184 }
185
7f2ab699 186 I915_WRITE_CTL(ring,
ae69b42a 187 ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
6aa56062 188 | RING_REPORT_64K | RING_VALID);
8187a2b7 189
8187a2b7 190 /* If the head is still not zero, the ring is dead */
176f28eb 191 if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
05394f39 192 I915_READ_START(ring) != obj->gtt_offset ||
176f28eb 193 (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
e74cfed5
CW		 194 DRM_ERROR("%s initialization failed "
195 "ctl %08x head %08x tail %08x start %08x\n",
196 ring->name,
197 I915_READ_CTL(ring),
198 I915_READ_HEAD(ring),
199 I915_READ_TAIL(ring),
200 I915_READ_START(ring));
201 return -EIO;
8187a2b7
ZN		 202 }
203
78501eac
CW		 204 if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
205 i915_kernel_lost_context(ring->dev);
8187a2b7 206 else {
570ef608 207 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
870e86dd 208 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
8187a2b7
ZN		 209 ring->space = ring->head - (ring->tail + 8);
210 if (ring->space < 0)
211 ring->space += ring->size;
212 }
1ec14ad3 213
8187a2b7
ZN		 214 return 0;
215}
216
c6df541c
CW		 217/*
218 * 965+ support PIPE_CONTROL commands, which provide finer grained control
219 * over cache flushing.
220 */
221struct pipe_control {
222 struct drm_i915_gem_object *obj;
223 volatile u32 *cpu_page;
224 u32 gtt_offset;
225};
226
227static int
228init_pipe_control(struct intel_ring_buffer *ring)
229{
230 struct pipe_control *pc;
231 struct drm_i915_gem_object *obj;
232 int ret;
233
234 if (ring->private)
235 return 0;
236
237 pc = kmalloc(sizeof(*pc), GFP_KERNEL);
238 if (!pc)
239 return -ENOMEM;
240
241 obj = i915_gem_alloc_object(ring->dev, 4096);
242 if (obj == NULL) {
243 DRM_ERROR("Failed to allocate seqno page\n");
244 ret = -ENOMEM;
245 goto err;
246 }
247 obj->agp_type = AGP_USER_CACHED_MEMORY;
248
249 ret = i915_gem_object_pin(obj, 4096, true);
250 if (ret)
251 goto err_unref;
252
253 pc->gtt_offset = obj->gtt_offset;
254 pc->cpu_page = kmap(obj->pages[0]);
255 if (pc->cpu_page == NULL)
256 goto err_unpin;
257
258 pc->obj = obj;
259 ring->private = pc;
260 return 0;
261
262err_unpin:
263 i915_gem_object_unpin(obj);
264err_unref:
265 drm_gem_object_unreference(&obj->base);
266err:
267 kfree(pc);
268 return ret;
269}
270
271static void
272cleanup_pipe_control(struct intel_ring_buffer *ring)
273{
274 struct pipe_control *pc = ring->private;
275 struct drm_i915_gem_object *obj;
276
277 if (!ring->private)
278 return;
279
280 obj = pc->obj;
281 kunmap(obj->pages[0]);
282 i915_gem_object_unpin(obj);
283 drm_gem_object_unreference(&obj->base);
284
285 kfree(pc);
286 ring->private = NULL;
287}
288
78501eac 289static int init_render_ring(struct intel_ring_buffer *ring)
8187a2b7 290{
78501eac 291 struct drm_device *dev = ring->dev;
1ec14ad3 292 struct drm_i915_private *dev_priv = dev->dev_private;
78501eac 293 int ret = init_ring_common(ring);
a69ffdbf 294
a6c45cf0 295 if (INTEL_INFO(dev)->gen > 3) {
78501eac 296 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
a69ffdbf
ZW		 297 if (IS_GEN6(dev))
298 mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
299 I915_WRITE(MI_MODE, mode);
8187a2b7 300 }
78501eac 301
c6df541c
CW		 302 if (INTEL_INFO(dev)->gen >= 6) {
303 } else if (IS_GEN5(dev)) {
304 ret = init_pipe_control(ring);
305 if (ret)
306 return ret;
307 }
308
8187a2b7
ZN		 309 return ret;
310}
311
c6df541c
CW		 312static void render_ring_cleanup(struct intel_ring_buffer *ring)
313{
314 if (!ring->private)
315 return;
316
317 cleanup_pipe_control(ring);
318}
319
1ec14ad3
CW		 320static void
321update_semaphore(struct intel_ring_buffer *ring, int i, u32 seqno)
322{
323 struct drm_device *dev = ring->dev;
324 struct drm_i915_private *dev_priv = dev->dev_private;
325 int id;
326
327 /*
328 * cs -> 1 = vcs, 0 = bcs
329 * vcs -> 1 = bcs, 0 = cs,
330 * bcs -> 1 = cs, 0 = vcs.
331 */
332 id = ring - dev_priv->ring;
333 id += 2 - i;
334 id %= 3;
335
336 intel_ring_emit(ring,
337 MI_SEMAPHORE_MBOX |
338 MI_SEMAPHORE_REGISTER |
339 MI_SEMAPHORE_UPDATE);
340 intel_ring_emit(ring, seqno);
341 intel_ring_emit(ring,
342 RING_SYNC_0(dev_priv->ring[id].mmio_base) + 4*i);
343}
344
345static int
346gen6_add_request(struct intel_ring_buffer *ring,
347 u32 *result)
348{
349 u32 seqno;
350 int ret;
351
352 ret = intel_ring_begin(ring, 10);
353 if (ret)
354 return ret;
355
356 seqno = i915_gem_get_seqno(ring->dev);
357 update_semaphore(ring, 0, seqno);
358 update_semaphore(ring, 1, seqno);
359
360 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
361 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
362 intel_ring_emit(ring, seqno);
363 intel_ring_emit(ring, MI_USER_INTERRUPT);
364 intel_ring_advance(ring);
365
366 *result = seqno;
367 return 0;
368}
369
370int
371intel_ring_sync(struct intel_ring_buffer *ring,
372 struct intel_ring_buffer *to,
373 u32 seqno)
374{
375 int ret;
376
377 ret = intel_ring_begin(ring, 4);
378 if (ret)
379 return ret;
380
381 intel_ring_emit(ring,
382 MI_SEMAPHORE_MBOX |
383 MI_SEMAPHORE_REGISTER |
384 intel_ring_sync_index(ring, to) << 17 |
385 MI_SEMAPHORE_COMPARE);
386 intel_ring_emit(ring, seqno);
387 intel_ring_emit(ring, 0);
388 intel_ring_emit(ring, MI_NOOP);
389 intel_ring_advance(ring);
390
391 return 0;
392}
393
c6df541c
CW		 394#define PIPE_CONTROL_FLUSH(ring__, addr__) \
395do { \
396 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \
397 PIPE_CONTROL_DEPTH_STALL | 2); \
398 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
399 intel_ring_emit(ring__, 0); \
400 intel_ring_emit(ring__, 0); \
401} while (0)
402
403static int
404pc_render_add_request(struct intel_ring_buffer *ring,
405 u32 *result)
406{
407 struct drm_device *dev = ring->dev;
408 u32 seqno = i915_gem_get_seqno(dev);
409 struct pipe_control *pc = ring->private;
410 u32 scratch_addr = pc->gtt_offset + 128;
411 int ret;
412
413 /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
414 * incoherent with writes to memory, i.e. completely fubar,
415 * so we need to use PIPE_NOTIFY instead.
416 *
417 * However, we also need to workaround the qword write
418 * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
419 * memory before requesting an interrupt.
420 */
421 ret = intel_ring_begin(ring, 32);
422 if (ret)
423 return ret;
424
425 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
426 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
427 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
428 intel_ring_emit(ring, seqno);
429 intel_ring_emit(ring, 0);
430 PIPE_CONTROL_FLUSH(ring, scratch_addr);
431 scratch_addr += 128; /* write to separate cachelines */
432 PIPE_CONTROL_FLUSH(ring, scratch_addr);
433 scratch_addr += 128;
434 PIPE_CONTROL_FLUSH(ring, scratch_addr);
435 scratch_addr += 128;
436 PIPE_CONTROL_FLUSH(ring, scratch_addr);
437 scratch_addr += 128;
438 PIPE_CONTROL_FLUSH(ring, scratch_addr);
439 scratch_addr += 128;
440 PIPE_CONTROL_FLUSH(ring, scratch_addr);
441 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
442 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
443 PIPE_CONTROL_NOTIFY);
444 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
445 intel_ring_emit(ring, seqno);
446 intel_ring_emit(ring, 0);
447 intel_ring_advance(ring);
448
449 *result = seqno;
450 return 0;
451}
452
1ec14ad3
CW		 453static int
454render_ring_add_request(struct intel_ring_buffer *ring,
455 u32 *result)
456{
457 struct drm_device *dev = ring->dev;
458 u32 seqno = i915_gem_get_seqno(dev);
459 int ret;
3cce469c 460
1ec14ad3
CW		 461 ret = intel_ring_begin(ring, 4);
462 if (ret)
463 return ret;
3cce469c 464
1ec14ad3
CW		 465 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
466 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
467 intel_ring_emit(ring, seqno);
468 intel_ring_emit(ring, MI_USER_INTERRUPT);
3cce469c 469 intel_ring_advance(ring);
1ec14ad3 470
3cce469c
CW		 471 *result = seqno;
472 return 0;
62fdfeaf
EA		 473}
474
8187a2b7 475static u32
1ec14ad3 476ring_get_seqno(struct intel_ring_buffer *ring)
8187a2b7 477{
1ec14ad3
CW		 478 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
479}
480
c6df541c
CW		 481static u32
482pc_render_get_seqno(struct intel_ring_buffer *ring)
483{
484 struct pipe_control *pc = ring->private;
485 return pc->cpu_page[0];
486}
487
0f46832f
CW		 488static void
489ironlake_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
490{
491 dev_priv->gt_irq_mask &= ~mask;
492 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
493 POSTING_READ(GTIMR);
494}
495
496static void
497ironlake_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
498{
499 dev_priv->gt_irq_mask |= mask;
500 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
501 POSTING_READ(GTIMR);
502}
503
504static void
505i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
506{
507 dev_priv->irq_mask &= ~mask;
508 I915_WRITE(IMR, dev_priv->irq_mask);
509 POSTING_READ(IMR);
510}
511
512static void
513i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
514{
515 dev_priv->irq_mask |= mask;
516 I915_WRITE(IMR, dev_priv->irq_mask);
517 POSTING_READ(IMR);
518}
519
b13c2b96 520static bool
1ec14ad3 521render_ring_get_irq(struct intel_ring_buffer *ring)
62fdfeaf 522{
78501eac 523 struct drm_device *dev = ring->dev;
01a03331 524 drm_i915_private_t *dev_priv = dev->dev_private;
62fdfeaf 525
b13c2b96
CW		 526 if (!dev->irq_enabled)
527 return false;
528
01a03331
CW		 529 spin_lock(&dev_priv->irq_lock);
530 if (ring->irq_refcount++ == 0) {
62fdfeaf 531 if (HAS_PCH_SPLIT(dev))
0f46832f
CW		 532 ironlake_enable_irq(dev_priv,
533 GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
62fdfeaf
EA		 534 else
535 i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
536 }
01a03331 537 spin_unlock(&dev_priv->irq_lock);
b13c2b96
CW		 538
539 return true;
62fdfeaf
EA		 540}
541
8187a2b7 542static void
1ec14ad3 543render_ring_put_irq(struct intel_ring_buffer *ring)
62fdfeaf 544{
78501eac 545 struct drm_device *dev = ring->dev;
01a03331 546 drm_i915_private_t *dev_priv = dev->dev_private;
62fdfeaf 547
01a03331
CW		 548 spin_lock(&dev_priv->irq_lock);
549 if (--ring->irq_refcount == 0) {
62fdfeaf 550 if (HAS_PCH_SPLIT(dev))
0f46832f
CW		 551 ironlake_disable_irq(dev_priv,
552 GT_USER_INTERRUPT |
553 GT_PIPE_NOTIFY);
62fdfeaf
EA		 554 else
555 i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
556 }
01a03331 557 spin_unlock(&dev_priv->irq_lock);
62fdfeaf
EA		 558}
559
78501eac 560void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
8187a2b7 561{
78501eac
CW		 562 drm_i915_private_t *dev_priv = ring->dev->dev_private;
563 u32 mmio = IS_GEN6(ring->dev) ?
564 RING_HWS_PGA_GEN6(ring->mmio_base) :
565 RING_HWS_PGA(ring->mmio_base);
566 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
567 POSTING_READ(mmio);
8187a2b7
ZN		 568}
569
b72f3acb 570static int
78501eac
CW		 571bsd_ring_flush(struct intel_ring_buffer *ring,
572 u32 invalidate_domains,
573 u32 flush_domains)
d1b851fc 574{
b72f3acb
CW		 575 int ret;
576
1ec14ad3 577 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
b72f3acb 578 return 0;
1ec14ad3 579
b72f3acb
CW		 580 ret = intel_ring_begin(ring, 2);
581 if (ret)
582 return ret;
583
584 intel_ring_emit(ring, MI_FLUSH);
585 intel_ring_emit(ring, MI_NOOP);
586 intel_ring_advance(ring);
587 return 0;
d1b851fc
ZN		 588}
589
3cce469c 590static int
78501eac 591ring_add_request(struct intel_ring_buffer *ring,
3cce469c 592 u32 *result)
d1b851fc
ZN		 593{
594 u32 seqno;
3cce469c
CW		 595 int ret;
596
597 ret = intel_ring_begin(ring, 4);
598 if (ret)
599 return ret;
6f392d54 600
78501eac 601 seqno = i915_gem_get_seqno(ring->dev);
6f392d54 602
3cce469c
CW		 603 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
604 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
605 intel_ring_emit(ring, seqno);
606 intel_ring_emit(ring, MI_USER_INTERRUPT);
607 intel_ring_advance(ring);
d1b851fc
ZN		 608
609 DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
3cce469c
CW		 610 *result = seqno;
611 return 0;
d1b851fc
ZN		 612}
613
b13c2b96 614static bool
1ec14ad3 615ring_get_irq(struct intel_ring_buffer *ring, u32 flag)
d1b851fc 616{
1ec14ad3 617 struct drm_device *dev = ring->dev;
01a03331 618 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 619
b13c2b96
CW		 620 if (!dev->irq_enabled)
621 return false;
622
01a03331
CW		 623 spin_lock(&dev_priv->irq_lock);
624 if (ring->irq_refcount++ == 0)
0f46832f 625 ironlake_enable_irq(dev_priv, flag);
01a03331 626 spin_unlock(&dev_priv->irq_lock);
b13c2b96
CW		 627
628 return true;
d1b851fc 629}
1ec14ad3 630
d1b851fc 631static void
1ec14ad3 632ring_put_irq(struct intel_ring_buffer *ring, u32 flag)
d1b851fc 633{
1ec14ad3 634 struct drm_device *dev = ring->dev;
01a03331 635 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 636
01a03331
CW		 637 spin_lock(&dev_priv->irq_lock);
638 if (--ring->irq_refcount == 0)
0f46832f 639 ironlake_disable_irq(dev_priv, flag);
01a03331 640 spin_unlock(&dev_priv->irq_lock);
0f46832f
CW		 641}
642
643static bool
644gen6_ring_get_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
645{
646 struct drm_device *dev = ring->dev;
01a03331 647 drm_i915_private_t *dev_priv = dev->dev_private;
0f46832f
CW		 648
649 if (!dev->irq_enabled)
650 return false;
651
01a03331
CW		 652 spin_lock(&dev_priv->irq_lock);
653 if (ring->irq_refcount++ == 0) {
0f46832f
CW		 654 ring->irq_mask &= ~rflag;
655 I915_WRITE_IMR(ring, ring->irq_mask);
656 ironlake_enable_irq(dev_priv, gflag);
0f46832f 657 }
01a03331 658 spin_unlock(&dev_priv->irq_lock);
0f46832f
CW		 659
660 return true;
661}
662
663static void
664gen6_ring_put_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
665{
666 struct drm_device *dev = ring->dev;
01a03331 667 drm_i915_private_t *dev_priv = dev->dev_private;
0f46832f 668
01a03331
CW		 669 spin_lock(&dev_priv->irq_lock);
670 if (--ring->irq_refcount == 0) {
0f46832f
CW		 671 ring->irq_mask |= rflag;
672 I915_WRITE_IMR(ring, ring->irq_mask);
673 ironlake_disable_irq(dev_priv, gflag);
1ec14ad3 674 }
01a03331 675 spin_unlock(&dev_priv->irq_lock);
d1b851fc
ZN		 676}
677
b13c2b96 678static bool
1ec14ad3 679bsd_ring_get_irq(struct intel_ring_buffer *ring)
d1b851fc 680{
b13c2b96 681 return ring_get_irq(ring, GT_BSD_USER_INTERRUPT);
1ec14ad3
CW		 682}
683static void
684bsd_ring_put_irq(struct intel_ring_buffer *ring)
685{
b13c2b96 686 ring_put_irq(ring, GT_BSD_USER_INTERRUPT);
d1b851fc
ZN		 687}
688
689static int
c4e7a414 690ring_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
d1b851fc 691{
e1f99ce6 692 int ret;
78501eac 693
e1f99ce6
CW		 694 ret = intel_ring_begin(ring, 2);
695 if (ret)
696 return ret;
697
78501eac 698 intel_ring_emit(ring,
c4e7a414 699 MI_BATCH_BUFFER_START | (2 << 6) |
78501eac 700 MI_BATCH_NON_SECURE_I965);
c4e7a414 701 intel_ring_emit(ring, offset);
78501eac
CW		 702 intel_ring_advance(ring);
703
d1b851fc
ZN		 704 return 0;
705}
706
8187a2b7 707static int
78501eac 708render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
c4e7a414 709 u32 offset, u32 len)
62fdfeaf 710{
78501eac 711 struct drm_device *dev = ring->dev;
62fdfeaf 712 drm_i915_private_t *dev_priv = dev->dev_private;
c4e7a414 713 int ret;
62fdfeaf 714
6f392d54 715 trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
62fdfeaf 716
c4e7a414
CW		 717 if (IS_I830(dev) || IS_845G(dev)) {
718 ret = intel_ring_begin(ring, 4);
719 if (ret)
720 return ret;
62fdfeaf 721
c4e7a414
CW		 722 intel_ring_emit(ring, MI_BATCH_BUFFER);
723 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
724 intel_ring_emit(ring, offset + len - 8);
725 intel_ring_emit(ring, 0);
726 } else {
727 ret = intel_ring_begin(ring, 2);
728 if (ret)
729 return ret;
e1f99ce6 730
c4e7a414
CW		 731 if (INTEL_INFO(dev)->gen >= 4) {
732 intel_ring_emit(ring,
733 MI_BATCH_BUFFER_START | (2 << 6) |
734 MI_BATCH_NON_SECURE_I965);
735 intel_ring_emit(ring, offset);
62fdfeaf 736 } else {
c4e7a414
CW		 737 intel_ring_emit(ring,
738 MI_BATCH_BUFFER_START | (2 << 6));
739 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
62fdfeaf
EA		 740 }
741 }
c4e7a414 742 intel_ring_advance(ring);
62fdfeaf 743
62fdfeaf
EA		 744 return 0;
745}
746
78501eac 747static void cleanup_status_page(struct intel_ring_buffer *ring)
62fdfeaf 748{
78501eac 749 drm_i915_private_t *dev_priv = ring->dev->dev_private;
05394f39 750 struct drm_i915_gem_object *obj;
62fdfeaf 751
8187a2b7
ZN		 752 obj = ring->status_page.obj;
753 if (obj == NULL)
62fdfeaf 754 return;
62fdfeaf 755
05394f39 756 kunmap(obj->pages[0]);
62fdfeaf 757 i915_gem_object_unpin(obj);
05394f39 758 drm_gem_object_unreference(&obj->base);
8187a2b7 759 ring->status_page.obj = NULL;
62fdfeaf
EA		 760
761 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
62fdfeaf
EA		 762}
763
78501eac 764static int init_status_page(struct intel_ring_buffer *ring)
62fdfeaf 765{
78501eac 766 struct drm_device *dev = ring->dev;
62fdfeaf 767 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39 768 struct drm_i915_gem_object *obj;
62fdfeaf
EA		 769 int ret;
770
62fdfeaf
EA		 771 obj = i915_gem_alloc_object(dev, 4096);
772 if (obj == NULL) {
773 DRM_ERROR("Failed to allocate status page\n");
774 ret = -ENOMEM;
775 goto err;
776 }
05394f39 777 obj->agp_type = AGP_USER_CACHED_MEMORY;
62fdfeaf 778
75e9e915 779 ret = i915_gem_object_pin(obj, 4096, true);
62fdfeaf 780 if (ret != 0) {
62fdfeaf
EA		 781 goto err_unref;
782 }
783
05394f39
CW		 784 ring->status_page.gfx_addr = obj->gtt_offset;
785 ring->status_page.page_addr = kmap(obj->pages[0]);
8187a2b7 786 if (ring->status_page.page_addr == NULL) {
62fdfeaf 787 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
62fdfeaf
EA		 788 goto err_unpin;
789 }
8187a2b7
ZN		 790 ring->status_page.obj = obj;
791 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
62fdfeaf 792
78501eac 793 intel_ring_setup_status_page(ring);
8187a2b7
ZN		 794 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
795 ring->name, ring->status_page.gfx_addr);
62fdfeaf
EA		 796
797 return 0;
798
799err_unpin:
800 i915_gem_object_unpin(obj);
801err_unref:
05394f39 802 drm_gem_object_unreference(&obj->base);
62fdfeaf 803err:
8187a2b7 804 return ret;
62fdfeaf
EA		 805}
806
8187a2b7 807int intel_init_ring_buffer(struct drm_device *dev,
ab6f8e32 808 struct intel_ring_buffer *ring)
62fdfeaf 809{
05394f39 810 struct drm_i915_gem_object *obj;
dd785e35
CW		 811 int ret;
812
8187a2b7 813 ring->dev = dev;
23bc5982
CW		 814 INIT_LIST_HEAD(&ring->active_list);
815 INIT_LIST_HEAD(&ring->request_list);
64193406 816 INIT_LIST_HEAD(&ring->gpu_write_list);
0f46832f 817 ring->irq_mask = ~0;
62fdfeaf 818
8187a2b7 819 if (I915_NEED_GFX_HWS(dev)) {
78501eac 820 ret = init_status_page(ring);
8187a2b7
ZN		 821 if (ret)
822 return ret;
823 }
62fdfeaf 824
8187a2b7 825 obj = i915_gem_alloc_object(dev, ring->size);
62fdfeaf
EA		 826 if (obj == NULL) {
827 DRM_ERROR("Failed to allocate ringbuffer\n");
8187a2b7 828 ret = -ENOMEM;
dd785e35 829 goto err_hws;
62fdfeaf 830 }
62fdfeaf 831
05394f39 832 ring->obj = obj;
8187a2b7 833
75e9e915 834 ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
dd785e35
CW		 835 if (ret)
836 goto err_unref;
62fdfeaf 837
8187a2b7 838 ring->map.size = ring->size;
05394f39 839 ring->map.offset = dev->agp->base + obj->gtt_offset;
62fdfeaf
EA		 840 ring->map.type = 0;
841 ring->map.flags = 0;
842 ring->map.mtrr = 0;
843
844 drm_core_ioremap_wc(&ring->map, dev);
845 if (ring->map.handle == NULL) {
846 DRM_ERROR("Failed to map ringbuffer.\n");
8187a2b7 847 ret = -EINVAL;
dd785e35 848 goto err_unpin;
62fdfeaf
EA		 849 }
850
8187a2b7 851 ring->virtual_start = ring->map.handle;
78501eac 852 ret = ring->init(ring);
dd785e35
CW		 853 if (ret)
854 goto err_unmap;
62fdfeaf 855
55249baa
CW		 856 /* Workaround an erratum on the i830 which causes a hang if
857 * the TAIL pointer points to within the last 2 cachelines
858 * of the buffer.
859 */
860 ring->effective_size = ring->size;
861 if (IS_I830(ring->dev))
862 ring->effective_size -= 128;
863
c584fe47 864 return 0;
dd785e35
CW		 865
866err_unmap:
867 drm_core_ioremapfree(&ring->map, dev);
868err_unpin:
869 i915_gem_object_unpin(obj);
870err_unref:
05394f39
CW		 871 drm_gem_object_unreference(&obj->base);
872 ring->obj = NULL;
dd785e35 873err_hws:
78501eac 874 cleanup_status_page(ring);
8187a2b7 875 return ret;
62fdfeaf
EA		 876}
877
78501eac 878void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
62fdfeaf 879{
33626e6a
CW		 880 struct drm_i915_private *dev_priv;
881 int ret;
882
05394f39 883 if (ring->obj == NULL)
62fdfeaf
EA		 884 return;
885
33626e6a
CW		 886 /* Disable the ring buffer. The ring must be idle at this point */
887 dev_priv = ring->dev->dev_private;
888 ret = intel_wait_ring_buffer(ring, ring->size - 8);
889 I915_WRITE_CTL(ring, 0);
890
78501eac 891 drm_core_ioremapfree(&ring->map, ring->dev);
62fdfeaf 892
05394f39
CW		 893 i915_gem_object_unpin(ring->obj);
894 drm_gem_object_unreference(&ring->obj->base);
895 ring->obj = NULL;
78501eac 896
8d19215b
ZN		 897 if (ring->cleanup)
898 ring->cleanup(ring);
899
78501eac 900 cleanup_status_page(ring);
62fdfeaf
EA		 901}
902
78501eac 903static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
62fdfeaf 904{
8187a2b7 905 unsigned int *virt;
55249baa 906 int rem = ring->size - ring->tail;
62fdfeaf 907
8187a2b7 908 if (ring->space < rem) {
78501eac 909 int ret = intel_wait_ring_buffer(ring, rem);
62fdfeaf
EA		 910 if (ret)
911 return ret;
912 }
62fdfeaf 913
8187a2b7 914 virt = (unsigned int *)(ring->virtual_start + ring->tail);
1741dd4a
CW		 915 rem /= 8;
916 while (rem--) {
62fdfeaf 917 *virt++ = MI_NOOP;
1741dd4a
CW		 918 *virt++ = MI_NOOP;
919 }
62fdfeaf 920
8187a2b7 921 ring->tail = 0;
43ed340a 922 ring->space = ring->head - 8;
62fdfeaf
EA		 923
924 return 0;
925}
926
78501eac 927int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
62fdfeaf 928{
78501eac 929 struct drm_device *dev = ring->dev;
cae5852d 930 struct drm_i915_private *dev_priv = dev->dev_private;
78501eac 931 unsigned long end;
6aa56062
CW		 932 u32 head;
933
62fdfeaf 934 trace_i915_ring_wait_begin (dev);
8187a2b7
ZN		 935 end = jiffies + 3 * HZ;
936 do {
8c0a6bfe
CW		 937 /* If the reported head position has wrapped or hasn't advanced,
938 * fallback to the slow and accurate path.
939 */
940 head = intel_read_status_page(ring, 4);
941 if (head < ring->actual_head)
942 head = I915_READ_HEAD(ring);
943 ring->actual_head = head;
944 ring->head = head & HEAD_ADDR;
62fdfeaf
EA		 945 ring->space = ring->head - (ring->tail + 8);
946 if (ring->space < 0)
8187a2b7 947 ring->space += ring->size;
62fdfeaf 948 if (ring->space >= n) {
78501eac 949 trace_i915_ring_wait_end(dev);
62fdfeaf
EA		 950 return 0;
951 }
952
953 if (dev->primary->master) {
954 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
955 if (master_priv->sarea_priv)
956 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
957 }
d1b851fc 958
e60a0b10 959 msleep(1);
f4e0b29b
CW		 960 if (atomic_read(&dev_priv->mm.wedged))
961 return -EAGAIN;
8187a2b7
ZN		 962 } while (!time_after(jiffies, end));
963 trace_i915_ring_wait_end (dev);
964 return -EBUSY;
965}
62fdfeaf 966
e1f99ce6
CW		 967int intel_ring_begin(struct intel_ring_buffer *ring,
968 int num_dwords)
8187a2b7 969{
be26a10b 970 int n = 4*num_dwords;
e1f99ce6 971 int ret;
78501eac 972
55249baa 973 if (unlikely(ring->tail + n > ring->effective_size)) {
e1f99ce6
CW		 974 ret = intel_wrap_ring_buffer(ring);
975 if (unlikely(ret))
976 return ret;
977 }
78501eac 978
e1f99ce6
CW		 979 if (unlikely(ring->space < n)) {
980 ret = intel_wait_ring_buffer(ring, n);
981 if (unlikely(ret))
982 return ret;
983 }
d97ed339
CW		 984
985 ring->space -= n;
e1f99ce6 986 return 0;
8187a2b7 987}
62fdfeaf 988
78501eac 989void intel_ring_advance(struct intel_ring_buffer *ring)
8187a2b7 990{
d97ed339 991 ring->tail &= ring->size - 1;
78501eac 992 ring->write_tail(ring, ring->tail);
8187a2b7 993}
62fdfeaf 994
e070868e 995static const struct intel_ring_buffer render_ring = {
8187a2b7 996 .name = "render ring",
9220434a 997 .id = RING_RENDER,
333e9fe9 998 .mmio_base = RENDER_RING_BASE,
8187a2b7 999 .size = 32 * PAGE_SIZE,
8187a2b7 1000 .init = init_render_ring,
297b0c5b 1001 .write_tail = ring_write_tail,
8187a2b7
ZN		 1002 .flush = render_ring_flush,
1003 .add_request = render_ring_add_request,
1ec14ad3
CW		 1004 .get_seqno = ring_get_seqno,
1005 .irq_get = render_ring_get_irq,
1006 .irq_put = render_ring_put_irq,
78501eac 1007 .dispatch_execbuffer = render_ring_dispatch_execbuffer,
c6df541c 1008 .cleanup = render_ring_cleanup,
8187a2b7 1009};
d1b851fc
ZN		 1010
1011/* ring buffer for bit-stream decoder */
1012
e070868e 1013static const struct intel_ring_buffer bsd_ring = {
d1b851fc 1014 .name = "bsd ring",
9220434a 1015 .id = RING_BSD,
333e9fe9 1016 .mmio_base = BSD_RING_BASE,
d1b851fc 1017 .size = 32 * PAGE_SIZE,
78501eac 1018 .init = init_ring_common,
297b0c5b 1019 .write_tail = ring_write_tail,
d1b851fc 1020 .flush = bsd_ring_flush,
549f7365 1021 .add_request = ring_add_request,
1ec14ad3
CW		 1022 .get_seqno = ring_get_seqno,
1023 .irq_get = bsd_ring_get_irq,
1024 .irq_put = bsd_ring_put_irq,
78501eac 1025 .dispatch_execbuffer = ring_dispatch_execbuffer,
d1b851fc 1026};
5c1143bb 1027
881f47b6 1028
78501eac 1029static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
297b0c5b 1030 u32 value)
881f47b6 1031{
78501eac 1032 drm_i915_private_t *dev_priv = ring->dev->dev_private;
881f47b6
XH		 1033
1034 /* Every tail move must follow the sequence below */
1035 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1036 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1037 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
1038 I915_WRITE(GEN6_BSD_RNCID, 0x0);
1039
1040 if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1041 GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
1042 50))
1043 DRM_ERROR("timed out waiting for IDLE Indicator\n");
1044
870e86dd 1045 I915_WRITE_TAIL(ring, value);
881f47b6
XH		 1046 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1047 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1048 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
1049}
1050
b72f3acb
CW		 1051static int gen6_ring_flush(struct intel_ring_buffer *ring,
1052 u32 invalidate_domains,
1053 u32 flush_domains)
881f47b6 1054{
b72f3acb
CW		 1055 int ret;
1056
1ec14ad3 1057 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
b72f3acb 1058 return 0;
1ec14ad3 1059
b72f3acb
CW		 1060 ret = intel_ring_begin(ring, 4);
1061 if (ret)
1062 return ret;
1063
1064 intel_ring_emit(ring, MI_FLUSH_DW);
1065 intel_ring_emit(ring, 0);
1066 intel_ring_emit(ring, 0);
1067 intel_ring_emit(ring, 0);
1068 intel_ring_advance(ring);
1069 return 0;
881f47b6
XH		 1070}
1071
1072static int
78501eac 1073gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
c4e7a414 1074 u32 offset, u32 len)
881f47b6 1075{
e1f99ce6 1076 int ret;
ab6f8e32 1077
e1f99ce6
CW		 1078 ret = intel_ring_begin(ring, 2);
1079 if (ret)
1080 return ret;
1081
78501eac 1082 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
ab6f8e32 1083 /* bit0-7 is the length on GEN6+ */
c4e7a414 1084 intel_ring_emit(ring, offset);
78501eac 1085 intel_ring_advance(ring);
ab6f8e32 1086
881f47b6
XH		 1087 return 0;
1088}
1089
0f46832f
CW		 1090static bool
1091gen6_render_ring_get_irq(struct intel_ring_buffer *ring)
1092{
1093 return gen6_ring_get_irq(ring,
1094 GT_USER_INTERRUPT,
1095 GEN6_RENDER_USER_INTERRUPT);
1096}
1097
1098static void
1099gen6_render_ring_put_irq(struct intel_ring_buffer *ring)
1100{
1101 return gen6_ring_put_irq(ring,
1102 GT_USER_INTERRUPT,
1103 GEN6_RENDER_USER_INTERRUPT);
1104}
1105
b13c2b96 1106static bool
1ec14ad3
CW		 1107gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
1108{
0f46832f
CW		 1109 return gen6_ring_get_irq(ring,
1110 GT_GEN6_BSD_USER_INTERRUPT,
1111 GEN6_BSD_USER_INTERRUPT);
1ec14ad3
CW		 1112}
1113
1114static void
1115gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
1116{
0f46832f
CW		 1117 return gen6_ring_put_irq(ring,
1118 GT_GEN6_BSD_USER_INTERRUPT,
1119 GEN6_BSD_USER_INTERRUPT);
1ec14ad3
CW		 1120}
1121
881f47b6 1122/* ring buffer for Video Codec for Gen6+ */
e070868e 1123static const struct intel_ring_buffer gen6_bsd_ring = {
1ec14ad3
CW		 1124 .name = "gen6 bsd ring",
1125 .id = RING_BSD,
1126 .mmio_base = GEN6_BSD_RING_BASE,
1127 .size = 32 * PAGE_SIZE,
1128 .init = init_ring_common,
1129 .write_tail = gen6_bsd_ring_write_tail,
1130 .flush = gen6_ring_flush,
1131 .add_request = gen6_add_request,
1132 .get_seqno = ring_get_seqno,
1133 .irq_get = gen6_bsd_ring_get_irq,
1134 .irq_put = gen6_bsd_ring_put_irq,
1135 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
549f7365
CW		 1136};
1137
1138/* Blitter support (SandyBridge+) */
1139
b13c2b96 1140static bool
1ec14ad3 1141blt_ring_get_irq(struct intel_ring_buffer *ring)
549f7365 1142{
0f46832f
CW		 1143 return gen6_ring_get_irq(ring,
1144 GT_BLT_USER_INTERRUPT,
1145 GEN6_BLITTER_USER_INTERRUPT);
549f7365 1146}
1ec14ad3 1147
549f7365 1148static void
1ec14ad3 1149blt_ring_put_irq(struct intel_ring_buffer *ring)
549f7365 1150{
0f46832f
CW		 1151 gen6_ring_put_irq(ring,
1152 GT_BLT_USER_INTERRUPT,
1153 GEN6_BLITTER_USER_INTERRUPT);
549f7365
CW		 1154}
1155
8d19215b
ZN		 1156
1157/* Workaround for some stepping of SNB,
1158 * each time when BLT engine ring tail moved,
1159 * the first command in the ring to be parsed
1160 * should be MI_BATCH_BUFFER_START
1161 */
1162#define NEED_BLT_WORKAROUND(dev) \
1163 (IS_GEN6(dev) && (dev->pdev->revision < 8))
1164
1165static inline struct drm_i915_gem_object *
1166to_blt_workaround(struct intel_ring_buffer *ring)
1167{
1168 return ring->private;
1169}
1170
1171static int blt_ring_init(struct intel_ring_buffer *ring)
1172{
1173 if (NEED_BLT_WORKAROUND(ring->dev)) {
1174 struct drm_i915_gem_object *obj;
27153f72 1175 u32 *ptr;
8d19215b
ZN		 1176 int ret;
1177
05394f39 1178 obj = i915_gem_alloc_object(ring->dev, 4096);
8d19215b
ZN		 1179 if (obj == NULL)
1180 return -ENOMEM;
1181
05394f39 1182 ret = i915_gem_object_pin(obj, 4096, true);
8d19215b
ZN		 1183 if (ret) {
1184 drm_gem_object_unreference(&obj->base);
1185 return ret;
1186 }
1187
1188 ptr = kmap(obj->pages[0]);
27153f72
CW		 1189 *ptr++ = MI_BATCH_BUFFER_END;
1190 *ptr++ = MI_NOOP;
8d19215b
ZN		 1191 kunmap(obj->pages[0]);
1192
05394f39 1193 ret = i915_gem_object_set_to_gtt_domain(obj, false);
8d19215b 1194 if (ret) {
05394f39 1195 i915_gem_object_unpin(obj);
8d19215b
ZN		 1196 drm_gem_object_unreference(&obj->base);
1197 return ret;
1198 }
1199
1200 ring->private = obj;
1201 }
1202
1203 return init_ring_common(ring);
1204}
1205
1206static int blt_ring_begin(struct intel_ring_buffer *ring,
1207 int num_dwords)
1208{
1209 if (ring->private) {
1210 int ret = intel_ring_begin(ring, num_dwords+2);
1211 if (ret)
1212 return ret;
1213
1214 intel_ring_emit(ring, MI_BATCH_BUFFER_START);
1215 intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
1216
1217 return 0;
1218 } else
1219 return intel_ring_begin(ring, 4);
1220}
1221
b72f3acb 1222static int blt_ring_flush(struct intel_ring_buffer *ring,
8d19215b
ZN		 1223 u32 invalidate_domains,
1224 u32 flush_domains)
1225{
b72f3acb
CW		 1226 int ret;
1227
1ec14ad3 1228 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
b72f3acb 1229 return 0;
1ec14ad3 1230
b72f3acb
CW		 1231 ret = blt_ring_begin(ring, 4);
1232 if (ret)
1233 return ret;
1234
1235 intel_ring_emit(ring, MI_FLUSH_DW);
1236 intel_ring_emit(ring, 0);
1237 intel_ring_emit(ring, 0);
1238 intel_ring_emit(ring, 0);
1239 intel_ring_advance(ring);
1240 return 0;
8d19215b
ZN		 1241}
1242
8d19215b
ZN		 1243static void blt_ring_cleanup(struct intel_ring_buffer *ring)
1244{
1245 if (!ring->private)
1246 return;
1247
1248 i915_gem_object_unpin(ring->private);
1249 drm_gem_object_unreference(ring->private);
1250 ring->private = NULL;
1251}
1252
549f7365
CW		 1253static const struct intel_ring_buffer gen6_blt_ring = {
1254 .name = "blt ring",
1255 .id = RING_BLT,
1256 .mmio_base = BLT_RING_BASE,
1257 .size = 32 * PAGE_SIZE,
8d19215b 1258 .init = blt_ring_init,
297b0c5b 1259 .write_tail = ring_write_tail,
8d19215b 1260 .flush = blt_ring_flush,
1ec14ad3
CW		 1261 .add_request = gen6_add_request,
1262 .get_seqno = ring_get_seqno,
1263 .irq_get = blt_ring_get_irq,
1264 .irq_put = blt_ring_put_irq,
78501eac 1265 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
8d19215b 1266 .cleanup = blt_ring_cleanup,
881f47b6
XH		 1267};
1268
5c1143bb
XH		 1269int intel_init_render_ring_buffer(struct drm_device *dev)
1270{
1271 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 1272 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
5c1143bb 1273
1ec14ad3
CW		 1274 *ring = render_ring;
1275 if (INTEL_INFO(dev)->gen >= 6) {
1276 ring->add_request = gen6_add_request;
0f46832f
CW		 1277 ring->irq_get = gen6_render_ring_get_irq;
1278 ring->irq_put = gen6_render_ring_put_irq;
c6df541c
CW		 1279 } else if (IS_GEN5(dev)) {
1280 ring->add_request = pc_render_add_request;
1281 ring->get_seqno = pc_render_get_seqno;
1ec14ad3 1282 }
5c1143bb
XH		 1283
1284 if (!I915_NEED_GFX_HWS(dev)) {
1ec14ad3
CW		 1285 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1286 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
5c1143bb
XH		 1287 }
1288
1ec14ad3 1289 return intel_init_ring_buffer(dev, ring);
5c1143bb
XH		 1290}
1291
1292int intel_init_bsd_ring_buffer(struct drm_device *dev)
1293{
1294 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 1295 struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
5c1143bb 1296
881f47b6 1297 if (IS_GEN6(dev))
1ec14ad3 1298 *ring = gen6_bsd_ring;
881f47b6 1299 else
1ec14ad3 1300 *ring = bsd_ring;
5c1143bb 1301
1ec14ad3 1302 return intel_init_ring_buffer(dev, ring);
5c1143bb 1303}
549f7365
CW		 1304
1305int intel_init_blt_ring_buffer(struct drm_device *dev)
1306{
1307 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 1308 struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
549f7365 1309
1ec14ad3 1310 *ring = gen6_blt_ring;
549f7365 1311
1ec14ad3 1312 return intel_init_ring_buffer(dev, ring);
549f7365 1313}
Linux kernel - Deliverables
This page took 0.143573 seconds and 5 git commands to generate.