projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: page flip support for Ivy Bridge
[deliverable/linux.git] / drivers / gpu / drm / i915 / intel_ringbuffer.c
CommitLineData
62fdfeaf
EA		 1/*
2 * Copyright © 2008-2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 * Zou Nan hai <nanhai.zou@intel.com>
26 * Xiang Hai hao<haihao.xiang@intel.com>
27 *
28 */
29
30#include "drmP.h"
31#include "drm.h"
62fdfeaf 32#include "i915_drv.h"
8187a2b7 33#include "i915_drm.h"
62fdfeaf 34#include "i915_trace.h"
881f47b6 35#include "intel_drv.h"
62fdfeaf 36
c7dca47b
CW		 37static inline int ring_space(struct intel_ring_buffer *ring)
38{
39 int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
40 if (space < 0)
41 space += ring->size;
42 return space;
43}
44
6f392d54
CW		 45static u32 i915_gem_get_seqno(struct drm_device *dev)
46{
47 drm_i915_private_t *dev_priv = dev->dev_private;
48 u32 seqno;
49
50 seqno = dev_priv->next_seqno;
51
52 /* reserve 0 for non-seqno */
53 if (++dev_priv->next_seqno == 0)
54 dev_priv->next_seqno = 1;
55
56 return seqno;
57}
58
b72f3acb 59static int
78501eac 60render_ring_flush(struct intel_ring_buffer *ring,
ab6f8e32
CW		 61 u32 invalidate_domains,
62 u32 flush_domains)
62fdfeaf 63{
78501eac 64 struct drm_device *dev = ring->dev;
6f392d54 65 u32 cmd;
b72f3acb 66 int ret;
6f392d54 67
36d527de
CW		 68 /*
69 * read/write caches:
70 *
71 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
72 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
73 * also flushed at 2d versus 3d pipeline switches.
74 *
75 * read-only caches:
76 *
77 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
78 * MI_READ_FLUSH is set, and is always flushed on 965.
79 *
80 * I915_GEM_DOMAIN_COMMAND may not exist?
81 *
82 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
83 * invalidated when MI_EXE_FLUSH is set.
84 *
85 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
86 * invalidated with every MI_FLUSH.
87 *
88 * TLBs:
89 *
90 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
91 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
92 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
93 * are flushed at any MI_FLUSH.
94 */
95
96 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
97 if ((invalidate_domains|flush_domains) &
98 I915_GEM_DOMAIN_RENDER)
99 cmd &= ~MI_NO_WRITE_FLUSH;
100 if (INTEL_INFO(dev)->gen < 4) {
62fdfeaf 101 /*
36d527de
CW		 102 * On the 965, the sampler cache always gets flushed
103 * and this bit is reserved.
62fdfeaf 104 */
36d527de
CW		 105 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
106 cmd |= MI_READ_FLUSH;
107 }
108 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
109 cmd |= MI_EXE_FLUSH;
62fdfeaf 110
36d527de
CW		 111 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
112 (IS_G4X(dev) || IS_GEN5(dev)))
113 cmd |= MI_INVALIDATE_ISP;
70eac33e 114
36d527de
CW		 115 ret = intel_ring_begin(ring, 2);
116 if (ret)
117 return ret;
b72f3acb 118
36d527de
CW		 119 intel_ring_emit(ring, cmd);
120 intel_ring_emit(ring, MI_NOOP);
121 intel_ring_advance(ring);
b72f3acb
CW		 122
123 return 0;
8187a2b7
ZN		 124}
125
78501eac 126static void ring_write_tail(struct intel_ring_buffer *ring,
297b0c5b 127 u32 value)
d46eefa2 128{
78501eac 129 drm_i915_private_t *dev_priv = ring->dev->dev_private;
297b0c5b 130 I915_WRITE_TAIL(ring, value);
d46eefa2
XH		 131}
132
78501eac 133u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
8187a2b7 134{
78501eac
CW		 135 drm_i915_private_t *dev_priv = ring->dev->dev_private;
136 u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
3d281d8c 137 RING_ACTHD(ring->mmio_base) : ACTHD;
8187a2b7
ZN		 138
139 return I915_READ(acthd_reg);
140}
141
78501eac 142static int init_ring_common(struct intel_ring_buffer *ring)
8187a2b7 143{
78501eac 144 drm_i915_private_t *dev_priv = ring->dev->dev_private;
05394f39 145 struct drm_i915_gem_object *obj = ring->obj;
8187a2b7 146 u32 head;
8187a2b7
ZN		 147
148 /* Stop the ring if it's running. */
7f2ab699 149 I915_WRITE_CTL(ring, 0);
570ef608 150 I915_WRITE_HEAD(ring, 0);
78501eac 151 ring->write_tail(ring, 0);
8187a2b7
ZN		 152
153 /* Initialize the ring. */
05394f39 154 I915_WRITE_START(ring, obj->gtt_offset);
570ef608 155 head = I915_READ_HEAD(ring) & HEAD_ADDR;
8187a2b7
ZN		 156
157 /* G45 ring initialization fails to reset head to zero */
158 if (head != 0) {
6fd0d56e
CW		 159 DRM_DEBUG_KMS("%s head not reset to zero "
160 "ctl %08x head %08x tail %08x start %08x\n",
161 ring->name,
162 I915_READ_CTL(ring),
163 I915_READ_HEAD(ring),
164 I915_READ_TAIL(ring),
165 I915_READ_START(ring));
8187a2b7 166
570ef608 167 I915_WRITE_HEAD(ring, 0);
8187a2b7 168
6fd0d56e
CW		 169 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
170 DRM_ERROR("failed to set %s head to zero "
171 "ctl %08x head %08x tail %08x start %08x\n",
172 ring->name,
173 I915_READ_CTL(ring),
174 I915_READ_HEAD(ring),
175 I915_READ_TAIL(ring),
176 I915_READ_START(ring));
177 }
8187a2b7
ZN		 178 }
179
7f2ab699 180 I915_WRITE_CTL(ring,
ae69b42a 181 ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
6aa56062 182 | RING_REPORT_64K | RING_VALID);
8187a2b7 183
8187a2b7 184 /* If the head is still not zero, the ring is dead */
176f28eb 185 if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
05394f39 186 I915_READ_START(ring) != obj->gtt_offset ||
176f28eb 187 (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
e74cfed5
CW		 188 DRM_ERROR("%s initialization failed "
189 "ctl %08x head %08x tail %08x start %08x\n",
190 ring->name,
191 I915_READ_CTL(ring),
192 I915_READ_HEAD(ring),
193 I915_READ_TAIL(ring),
194 I915_READ_START(ring));
195 return -EIO;
8187a2b7
ZN		 196 }
197
78501eac
CW		 198 if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
199 i915_kernel_lost_context(ring->dev);
8187a2b7 200 else {
c7dca47b 201 ring->head = I915_READ_HEAD(ring);
870e86dd 202 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
c7dca47b 203 ring->space = ring_space(ring);
8187a2b7 204 }
1ec14ad3 205
8187a2b7
ZN		 206 return 0;
207}
208
c6df541c
CW		 209/*
210 * 965+ support PIPE_CONTROL commands, which provide finer grained control
211 * over cache flushing.
212 */
213struct pipe_control {
214 struct drm_i915_gem_object *obj;
215 volatile u32 *cpu_page;
216 u32 gtt_offset;
217};
218
219static int
220init_pipe_control(struct intel_ring_buffer *ring)
221{
222 struct pipe_control *pc;
223 struct drm_i915_gem_object *obj;
224 int ret;
225
226 if (ring->private)
227 return 0;
228
229 pc = kmalloc(sizeof(*pc), GFP_KERNEL);
230 if (!pc)
231 return -ENOMEM;
232
233 obj = i915_gem_alloc_object(ring->dev, 4096);
234 if (obj == NULL) {
235 DRM_ERROR("Failed to allocate seqno page\n");
236 ret = -ENOMEM;
237 goto err;
238 }
93dfb40c 239 obj->cache_level = I915_CACHE_LLC;
c6df541c
CW		 240
241 ret = i915_gem_object_pin(obj, 4096, true);
242 if (ret)
243 goto err_unref;
244
245 pc->gtt_offset = obj->gtt_offset;
246 pc->cpu_page = kmap(obj->pages[0]);
247 if (pc->cpu_page == NULL)
248 goto err_unpin;
249
250 pc->obj = obj;
251 ring->private = pc;
252 return 0;
253
254err_unpin:
255 i915_gem_object_unpin(obj);
256err_unref:
257 drm_gem_object_unreference(&obj->base);
258err:
259 kfree(pc);
260 return ret;
261}
262
263static void
264cleanup_pipe_control(struct intel_ring_buffer *ring)
265{
266 struct pipe_control *pc = ring->private;
267 struct drm_i915_gem_object *obj;
268
269 if (!ring->private)
270 return;
271
272 obj = pc->obj;
273 kunmap(obj->pages[0]);
274 i915_gem_object_unpin(obj);
275 drm_gem_object_unreference(&obj->base);
276
277 kfree(pc);
278 ring->private = NULL;
279}
280
78501eac 281static int init_render_ring(struct intel_ring_buffer *ring)
8187a2b7 282{
78501eac 283 struct drm_device *dev = ring->dev;
1ec14ad3 284 struct drm_i915_private *dev_priv = dev->dev_private;
78501eac 285 int ret = init_ring_common(ring);
a69ffdbf 286
a6c45cf0 287 if (INTEL_INFO(dev)->gen > 3) {
78501eac 288 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
a69ffdbf
ZW		 289 if (IS_GEN6(dev))
290 mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
291 I915_WRITE(MI_MODE, mode);
8187a2b7 292 }
78501eac 293
c6df541c
CW		 294 if (INTEL_INFO(dev)->gen >= 6) {
295 } else if (IS_GEN5(dev)) {
296 ret = init_pipe_control(ring);
297 if (ret)
298 return ret;
299 }
300
8187a2b7
ZN		 301 return ret;
302}
303
c6df541c
CW		 304static void render_ring_cleanup(struct intel_ring_buffer *ring)
305{
306 if (!ring->private)
307 return;
308
309 cleanup_pipe_control(ring);
310}
311
1ec14ad3
CW		 312static void
313update_semaphore(struct intel_ring_buffer *ring, int i, u32 seqno)
314{
315 struct drm_device *dev = ring->dev;
316 struct drm_i915_private *dev_priv = dev->dev_private;
317 int id;
318
319 /*
320 * cs -> 1 = vcs, 0 = bcs
321 * vcs -> 1 = bcs, 0 = cs,
322 * bcs -> 1 = cs, 0 = vcs.
323 */
324 id = ring - dev_priv->ring;
325 id += 2 - i;
326 id %= 3;
327
328 intel_ring_emit(ring,
329 MI_SEMAPHORE_MBOX |
330 MI_SEMAPHORE_REGISTER |
331 MI_SEMAPHORE_UPDATE);
332 intel_ring_emit(ring, seqno);
333 intel_ring_emit(ring,
334 RING_SYNC_0(dev_priv->ring[id].mmio_base) + 4*i);
335}
336
337static int
338gen6_add_request(struct intel_ring_buffer *ring,
339 u32 *result)
340{
341 u32 seqno;
342 int ret;
343
344 ret = intel_ring_begin(ring, 10);
345 if (ret)
346 return ret;
347
348 seqno = i915_gem_get_seqno(ring->dev);
349 update_semaphore(ring, 0, seqno);
350 update_semaphore(ring, 1, seqno);
351
352 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
353 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
354 intel_ring_emit(ring, seqno);
355 intel_ring_emit(ring, MI_USER_INTERRUPT);
356 intel_ring_advance(ring);
357
358 *result = seqno;
359 return 0;
360}
361
362int
363intel_ring_sync(struct intel_ring_buffer *ring,
364 struct intel_ring_buffer *to,
365 u32 seqno)
366{
367 int ret;
368
369 ret = intel_ring_begin(ring, 4);
370 if (ret)
371 return ret;
372
373 intel_ring_emit(ring,
374 MI_SEMAPHORE_MBOX |
375 MI_SEMAPHORE_REGISTER |
376 intel_ring_sync_index(ring, to) << 17 |
377 MI_SEMAPHORE_COMPARE);
378 intel_ring_emit(ring, seqno);
379 intel_ring_emit(ring, 0);
380 intel_ring_emit(ring, MI_NOOP);
381 intel_ring_advance(ring);
382
383 return 0;
384}
385
c6df541c
CW		 386#define PIPE_CONTROL_FLUSH(ring__, addr__) \
387do { \
388 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \
389 PIPE_CONTROL_DEPTH_STALL | 2); \
390 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
391 intel_ring_emit(ring__, 0); \
392 intel_ring_emit(ring__, 0); \
393} while (0)
394
395static int
396pc_render_add_request(struct intel_ring_buffer *ring,
397 u32 *result)
398{
399 struct drm_device *dev = ring->dev;
400 u32 seqno = i915_gem_get_seqno(dev);
401 struct pipe_control *pc = ring->private;
402 u32 scratch_addr = pc->gtt_offset + 128;
403 int ret;
404
405 /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
406 * incoherent with writes to memory, i.e. completely fubar,
407 * so we need to use PIPE_NOTIFY instead.
408 *
409 * However, we also need to workaround the qword write
410 * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
411 * memory before requesting an interrupt.
412 */
413 ret = intel_ring_begin(ring, 32);
414 if (ret)
415 return ret;
416
417 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
418 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
419 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
420 intel_ring_emit(ring, seqno);
421 intel_ring_emit(ring, 0);
422 PIPE_CONTROL_FLUSH(ring, scratch_addr);
423 scratch_addr += 128; /* write to separate cachelines */
424 PIPE_CONTROL_FLUSH(ring, scratch_addr);
425 scratch_addr += 128;
426 PIPE_CONTROL_FLUSH(ring, scratch_addr);
427 scratch_addr += 128;
428 PIPE_CONTROL_FLUSH(ring, scratch_addr);
429 scratch_addr += 128;
430 PIPE_CONTROL_FLUSH(ring, scratch_addr);
431 scratch_addr += 128;
432 PIPE_CONTROL_FLUSH(ring, scratch_addr);
433 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
434 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
435 PIPE_CONTROL_NOTIFY);
436 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
437 intel_ring_emit(ring, seqno);
438 intel_ring_emit(ring, 0);
439 intel_ring_advance(ring);
440
441 *result = seqno;
442 return 0;
443}
444
1ec14ad3
CW		 445static int
446render_ring_add_request(struct intel_ring_buffer *ring,
447 u32 *result)
448{
449 struct drm_device *dev = ring->dev;
450 u32 seqno = i915_gem_get_seqno(dev);
451 int ret;
3cce469c 452
1ec14ad3
CW		 453 ret = intel_ring_begin(ring, 4);
454 if (ret)
455 return ret;
3cce469c 456
1ec14ad3
CW		 457 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
458 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
459 intel_ring_emit(ring, seqno);
460 intel_ring_emit(ring, MI_USER_INTERRUPT);
3cce469c 461 intel_ring_advance(ring);
1ec14ad3 462
3cce469c
CW		 463 *result = seqno;
464 return 0;
62fdfeaf
EA		 465}
466
8187a2b7 467static u32
1ec14ad3 468ring_get_seqno(struct intel_ring_buffer *ring)
8187a2b7 469{
1ec14ad3
CW		 470 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
471}
472
c6df541c
CW		 473static u32
474pc_render_get_seqno(struct intel_ring_buffer *ring)
475{
476 struct pipe_control *pc = ring->private;
477 return pc->cpu_page[0];
478}
479
0f46832f
CW		 480static void
481ironlake_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
482{
483 dev_priv->gt_irq_mask &= ~mask;
484 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
485 POSTING_READ(GTIMR);
486}
487
488static void
489ironlake_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
490{
491 dev_priv->gt_irq_mask |= mask;
492 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
493 POSTING_READ(GTIMR);
494}
495
496static void
497i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
498{
499 dev_priv->irq_mask &= ~mask;
500 I915_WRITE(IMR, dev_priv->irq_mask);
501 POSTING_READ(IMR);
502}
503
504static void
505i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
506{
507 dev_priv->irq_mask |= mask;
508 I915_WRITE(IMR, dev_priv->irq_mask);
509 POSTING_READ(IMR);
510}
511
b13c2b96 512static bool
1ec14ad3 513render_ring_get_irq(struct intel_ring_buffer *ring)
62fdfeaf 514{
78501eac 515 struct drm_device *dev = ring->dev;
01a03331 516 drm_i915_private_t *dev_priv = dev->dev_private;
62fdfeaf 517
b13c2b96
CW		 518 if (!dev->irq_enabled)
519 return false;
520
0dc79fb2 521 spin_lock(&ring->irq_lock);
01a03331 522 if (ring->irq_refcount++ == 0) {
62fdfeaf 523 if (HAS_PCH_SPLIT(dev))
0f46832f
CW		 524 ironlake_enable_irq(dev_priv,
525 GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
62fdfeaf
EA		 526 else
527 i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
528 }
0dc79fb2 529 spin_unlock(&ring->irq_lock);
b13c2b96
CW		 530
531 return true;
62fdfeaf
EA		 532}
533
8187a2b7 534static void
1ec14ad3 535render_ring_put_irq(struct intel_ring_buffer *ring)
62fdfeaf 536{
78501eac 537 struct drm_device *dev = ring->dev;
01a03331 538 drm_i915_private_t *dev_priv = dev->dev_private;
62fdfeaf 539
0dc79fb2 540 spin_lock(&ring->irq_lock);
01a03331 541 if (--ring->irq_refcount == 0) {
62fdfeaf 542 if (HAS_PCH_SPLIT(dev))
0f46832f
CW		 543 ironlake_disable_irq(dev_priv,
544 GT_USER_INTERRUPT |
545 GT_PIPE_NOTIFY);
62fdfeaf
EA		 546 else
547 i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
548 }
0dc79fb2 549 spin_unlock(&ring->irq_lock);
62fdfeaf
EA		 550}
551
78501eac 552void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
8187a2b7 553{
78501eac
CW		 554 drm_i915_private_t *dev_priv = ring->dev->dev_private;
555 u32 mmio = IS_GEN6(ring->dev) ?
556 RING_HWS_PGA_GEN6(ring->mmio_base) :
557 RING_HWS_PGA(ring->mmio_base);
558 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
559 POSTING_READ(mmio);
8187a2b7
ZN		 560}
561
b72f3acb 562static int
78501eac
CW		 563bsd_ring_flush(struct intel_ring_buffer *ring,
564 u32 invalidate_domains,
565 u32 flush_domains)
d1b851fc 566{
b72f3acb
CW		 567 int ret;
568
b72f3acb
CW		 569 ret = intel_ring_begin(ring, 2);
570 if (ret)
571 return ret;
572
573 intel_ring_emit(ring, MI_FLUSH);
574 intel_ring_emit(ring, MI_NOOP);
575 intel_ring_advance(ring);
576 return 0;
d1b851fc
ZN		 577}
578
3cce469c 579static int
78501eac 580ring_add_request(struct intel_ring_buffer *ring,
3cce469c 581 u32 *result)
d1b851fc
ZN		 582{
583 u32 seqno;
3cce469c
CW		 584 int ret;
585
586 ret = intel_ring_begin(ring, 4);
587 if (ret)
588 return ret;
6f392d54 589
78501eac 590 seqno = i915_gem_get_seqno(ring->dev);
6f392d54 591
3cce469c
CW		 592 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
593 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
594 intel_ring_emit(ring, seqno);
595 intel_ring_emit(ring, MI_USER_INTERRUPT);
596 intel_ring_advance(ring);
d1b851fc 597
3cce469c
CW		 598 *result = seqno;
599 return 0;
d1b851fc
ZN		 600}
601
b13c2b96 602static bool
1ec14ad3 603ring_get_irq(struct intel_ring_buffer *ring, u32 flag)
d1b851fc 604{
1ec14ad3 605 struct drm_device *dev = ring->dev;
01a03331 606 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 607
b13c2b96
CW		 608 if (!dev->irq_enabled)
609 return false;
610
0dc79fb2 611 spin_lock(&ring->irq_lock);
01a03331 612 if (ring->irq_refcount++ == 0)
0f46832f 613 ironlake_enable_irq(dev_priv, flag);
0dc79fb2 614 spin_unlock(&ring->irq_lock);
b13c2b96
CW		 615
616 return true;
d1b851fc 617}
1ec14ad3 618
d1b851fc 619static void
1ec14ad3 620ring_put_irq(struct intel_ring_buffer *ring, u32 flag)
d1b851fc 621{
1ec14ad3 622 struct drm_device *dev = ring->dev;
01a03331 623 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 624
0dc79fb2 625 spin_lock(&ring->irq_lock);
01a03331 626 if (--ring->irq_refcount == 0)
0f46832f 627 ironlake_disable_irq(dev_priv, flag);
0dc79fb2 628 spin_unlock(&ring->irq_lock);
0f46832f
CW		 629}
630
631static bool
632gen6_ring_get_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
633{
634 struct drm_device *dev = ring->dev;
01a03331 635 drm_i915_private_t *dev_priv = dev->dev_private;
0f46832f
CW		 636
637 if (!dev->irq_enabled)
638 return false;
639
0dc79fb2 640 spin_lock(&ring->irq_lock);
01a03331 641 if (ring->irq_refcount++ == 0) {
0f46832f
CW		 642 ring->irq_mask &= ~rflag;
643 I915_WRITE_IMR(ring, ring->irq_mask);
644 ironlake_enable_irq(dev_priv, gflag);
0f46832f 645 }
0dc79fb2 646 spin_unlock(&ring->irq_lock);
0f46832f
CW		 647
648 return true;
649}
650
651static void
652gen6_ring_put_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
653{
654 struct drm_device *dev = ring->dev;
01a03331 655 drm_i915_private_t *dev_priv = dev->dev_private;
0f46832f 656
0dc79fb2 657 spin_lock(&ring->irq_lock);
01a03331 658 if (--ring->irq_refcount == 0) {
0f46832f
CW		 659 ring->irq_mask |= rflag;
660 I915_WRITE_IMR(ring, ring->irq_mask);
661 ironlake_disable_irq(dev_priv, gflag);
1ec14ad3 662 }
0dc79fb2 663 spin_unlock(&ring->irq_lock);
d1b851fc
ZN		 664}
665
b13c2b96 666static bool
1ec14ad3 667bsd_ring_get_irq(struct intel_ring_buffer *ring)
d1b851fc 668{
b13c2b96 669 return ring_get_irq(ring, GT_BSD_USER_INTERRUPT);
1ec14ad3
CW		 670}
671static void
672bsd_ring_put_irq(struct intel_ring_buffer *ring)
673{
b13c2b96 674 ring_put_irq(ring, GT_BSD_USER_INTERRUPT);
d1b851fc
ZN		 675}
676
677static int
c4e7a414 678ring_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
d1b851fc 679{
e1f99ce6 680 int ret;
78501eac 681
e1f99ce6
CW		 682 ret = intel_ring_begin(ring, 2);
683 if (ret)
684 return ret;
685
78501eac 686 intel_ring_emit(ring,
c4e7a414 687 MI_BATCH_BUFFER_START | (2 << 6) |
78501eac 688 MI_BATCH_NON_SECURE_I965);
c4e7a414 689 intel_ring_emit(ring, offset);
78501eac
CW		 690 intel_ring_advance(ring);
691
d1b851fc
ZN		 692 return 0;
693}
694
8187a2b7 695static int
78501eac 696render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
c4e7a414 697 u32 offset, u32 len)
62fdfeaf 698{
78501eac 699 struct drm_device *dev = ring->dev;
c4e7a414 700 int ret;
62fdfeaf 701
c4e7a414
CW		 702 if (IS_I830(dev) || IS_845G(dev)) {
703 ret = intel_ring_begin(ring, 4);
704 if (ret)
705 return ret;
62fdfeaf 706
c4e7a414
CW		 707 intel_ring_emit(ring, MI_BATCH_BUFFER);
708 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
709 intel_ring_emit(ring, offset + len - 8);
710 intel_ring_emit(ring, 0);
711 } else {
712 ret = intel_ring_begin(ring, 2);
713 if (ret)
714 return ret;
e1f99ce6 715
c4e7a414
CW		 716 if (INTEL_INFO(dev)->gen >= 4) {
717 intel_ring_emit(ring,
718 MI_BATCH_BUFFER_START | (2 << 6) |
719 MI_BATCH_NON_SECURE_I965);
720 intel_ring_emit(ring, offset);
62fdfeaf 721 } else {
c4e7a414
CW		 722 intel_ring_emit(ring,
723 MI_BATCH_BUFFER_START | (2 << 6));
724 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
62fdfeaf
EA		 725 }
726 }
c4e7a414 727 intel_ring_advance(ring);
62fdfeaf 728
62fdfeaf
EA		 729 return 0;
730}
731
78501eac 732static void cleanup_status_page(struct intel_ring_buffer *ring)
62fdfeaf 733{
78501eac 734 drm_i915_private_t *dev_priv = ring->dev->dev_private;
05394f39 735 struct drm_i915_gem_object *obj;
62fdfeaf 736
8187a2b7
ZN		 737 obj = ring->status_page.obj;
738 if (obj == NULL)
62fdfeaf 739 return;
62fdfeaf 740
05394f39 741 kunmap(obj->pages[0]);
62fdfeaf 742 i915_gem_object_unpin(obj);
05394f39 743 drm_gem_object_unreference(&obj->base);
8187a2b7 744 ring->status_page.obj = NULL;
62fdfeaf
EA		 745
746 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
62fdfeaf
EA		 747}
748
78501eac 749static int init_status_page(struct intel_ring_buffer *ring)
62fdfeaf 750{
78501eac 751 struct drm_device *dev = ring->dev;
62fdfeaf 752 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39 753 struct drm_i915_gem_object *obj;
62fdfeaf
EA		 754 int ret;
755
62fdfeaf
EA		 756 obj = i915_gem_alloc_object(dev, 4096);
757 if (obj == NULL) {
758 DRM_ERROR("Failed to allocate status page\n");
759 ret = -ENOMEM;
760 goto err;
761 }
93dfb40c 762 obj->cache_level = I915_CACHE_LLC;
62fdfeaf 763
75e9e915 764 ret = i915_gem_object_pin(obj, 4096, true);
62fdfeaf 765 if (ret != 0) {
62fdfeaf
EA		 766 goto err_unref;
767 }
768
05394f39
CW		 769 ring->status_page.gfx_addr = obj->gtt_offset;
770 ring->status_page.page_addr = kmap(obj->pages[0]);
8187a2b7 771 if (ring->status_page.page_addr == NULL) {
62fdfeaf 772 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
62fdfeaf
EA		 773 goto err_unpin;
774 }
8187a2b7
ZN		 775 ring->status_page.obj = obj;
776 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
62fdfeaf 777
78501eac 778 intel_ring_setup_status_page(ring);
8187a2b7
ZN		 779 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
780 ring->name, ring->status_page.gfx_addr);
62fdfeaf
EA		 781
782 return 0;
783
784err_unpin:
785 i915_gem_object_unpin(obj);
786err_unref:
05394f39 787 drm_gem_object_unreference(&obj->base);
62fdfeaf 788err:
8187a2b7 789 return ret;
62fdfeaf
EA		 790}
791
8187a2b7 792int intel_init_ring_buffer(struct drm_device *dev,
ab6f8e32 793 struct intel_ring_buffer *ring)
62fdfeaf 794{
05394f39 795 struct drm_i915_gem_object *obj;
dd785e35
CW		 796 int ret;
797
8187a2b7 798 ring->dev = dev;
23bc5982
CW		 799 INIT_LIST_HEAD(&ring->active_list);
800 INIT_LIST_HEAD(&ring->request_list);
64193406 801 INIT_LIST_HEAD(&ring->gpu_write_list);
0dc79fb2 802
b259f673 803 init_waitqueue_head(&ring->irq_queue);
0dc79fb2 804 spin_lock_init(&ring->irq_lock);
0f46832f 805 ring->irq_mask = ~0;
62fdfeaf 806
8187a2b7 807 if (I915_NEED_GFX_HWS(dev)) {
78501eac 808 ret = init_status_page(ring);
8187a2b7
ZN		 809 if (ret)
810 return ret;
811 }
62fdfeaf 812
8187a2b7 813 obj = i915_gem_alloc_object(dev, ring->size);
62fdfeaf
EA		 814 if (obj == NULL) {
815 DRM_ERROR("Failed to allocate ringbuffer\n");
8187a2b7 816 ret = -ENOMEM;
dd785e35 817 goto err_hws;
62fdfeaf 818 }
62fdfeaf 819
05394f39 820 ring->obj = obj;
8187a2b7 821
75e9e915 822 ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
dd785e35
CW		 823 if (ret)
824 goto err_unref;
62fdfeaf 825
8187a2b7 826 ring->map.size = ring->size;
05394f39 827 ring->map.offset = dev->agp->base + obj->gtt_offset;
62fdfeaf
EA		 828 ring->map.type = 0;
829 ring->map.flags = 0;
830 ring->map.mtrr = 0;
831
832 drm_core_ioremap_wc(&ring->map, dev);
833 if (ring->map.handle == NULL) {
834 DRM_ERROR("Failed to map ringbuffer.\n");
8187a2b7 835 ret = -EINVAL;
dd785e35 836 goto err_unpin;
62fdfeaf
EA		 837 }
838
8187a2b7 839 ring->virtual_start = ring->map.handle;
78501eac 840 ret = ring->init(ring);
dd785e35
CW		 841 if (ret)
842 goto err_unmap;
62fdfeaf 843
55249baa
CW		 844 /* Workaround an erratum on the i830 which causes a hang if
845 * the TAIL pointer points to within the last 2 cachelines
846 * of the buffer.
847 */
848 ring->effective_size = ring->size;
849 if (IS_I830(ring->dev))
850 ring->effective_size -= 128;
851
c584fe47 852 return 0;
dd785e35
CW		 853
854err_unmap:
855 drm_core_ioremapfree(&ring->map, dev);
856err_unpin:
857 i915_gem_object_unpin(obj);
858err_unref:
05394f39
CW		 859 drm_gem_object_unreference(&obj->base);
860 ring->obj = NULL;
dd785e35 861err_hws:
78501eac 862 cleanup_status_page(ring);
8187a2b7 863 return ret;
62fdfeaf
EA		 864}
865
78501eac 866void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
62fdfeaf 867{
33626e6a
CW		 868 struct drm_i915_private *dev_priv;
869 int ret;
870
05394f39 871 if (ring->obj == NULL)
62fdfeaf
EA		 872 return;
873
33626e6a
CW		 874 /* Disable the ring buffer. The ring must be idle at this point */
875 dev_priv = ring->dev->dev_private;
96f298aa 876 ret = intel_wait_ring_idle(ring);
29ee3991
CW		 877 if (ret)
878 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
879 ring->name, ret);
880
33626e6a
CW		 881 I915_WRITE_CTL(ring, 0);
882
78501eac 883 drm_core_ioremapfree(&ring->map, ring->dev);
62fdfeaf 884
05394f39
CW		 885 i915_gem_object_unpin(ring->obj);
886 drm_gem_object_unreference(&ring->obj->base);
887 ring->obj = NULL;
78501eac 888
8d19215b
ZN		 889 if (ring->cleanup)
890 ring->cleanup(ring);
891
78501eac 892 cleanup_status_page(ring);
62fdfeaf
EA		 893}
894
78501eac 895static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
62fdfeaf 896{
8187a2b7 897 unsigned int *virt;
55249baa 898 int rem = ring->size - ring->tail;
62fdfeaf 899
8187a2b7 900 if (ring->space < rem) {
78501eac 901 int ret = intel_wait_ring_buffer(ring, rem);
62fdfeaf
EA		 902 if (ret)
903 return ret;
904 }
62fdfeaf 905
8187a2b7 906 virt = (unsigned int *)(ring->virtual_start + ring->tail);
1741dd4a
CW		 907 rem /= 8;
908 while (rem--) {
62fdfeaf 909 *virt++ = MI_NOOP;
1741dd4a
CW		 910 *virt++ = MI_NOOP;
911 }
62fdfeaf 912
8187a2b7 913 ring->tail = 0;
c7dca47b 914 ring->space = ring_space(ring);
62fdfeaf
EA		 915
916 return 0;
917}
918
78501eac 919int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
62fdfeaf 920{
78501eac 921 struct drm_device *dev = ring->dev;
cae5852d 922 struct drm_i915_private *dev_priv = dev->dev_private;
78501eac 923 unsigned long end;
6aa56062
CW		 924 u32 head;
925
c7dca47b
CW		 926 /* If the reported head position has wrapped or hasn't advanced,
927 * fallback to the slow and accurate path.
928 */
929 head = intel_read_status_page(ring, 4);
930 if (head > ring->head) {
931 ring->head = head;
932 ring->space = ring_space(ring);
933 if (ring->space >= n)
934 return 0;
935 }
936
db53a302 937 trace_i915_ring_wait_begin(ring);
8187a2b7
ZN		 938 end = jiffies + 3 * HZ;
939 do {
c7dca47b
CW		 940 ring->head = I915_READ_HEAD(ring);
941 ring->space = ring_space(ring);
62fdfeaf 942 if (ring->space >= n) {
db53a302 943 trace_i915_ring_wait_end(ring);
62fdfeaf
EA		 944 return 0;
945 }
946
947 if (dev->primary->master) {
948 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
949 if (master_priv->sarea_priv)
950 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
951 }
d1b851fc 952
e60a0b10 953 msleep(1);
f4e0b29b
CW		 954 if (atomic_read(&dev_priv->mm.wedged))
955 return -EAGAIN;
8187a2b7 956 } while (!time_after(jiffies, end));
db53a302 957 trace_i915_ring_wait_end(ring);
8187a2b7
ZN		 958 return -EBUSY;
959}
62fdfeaf 960
e1f99ce6
CW		 961int intel_ring_begin(struct intel_ring_buffer *ring,
962 int num_dwords)
8187a2b7 963{
21dd3734 964 struct drm_i915_private *dev_priv = ring->dev->dev_private;
be26a10b 965 int n = 4*num_dwords;
e1f99ce6 966 int ret;
78501eac 967
21dd3734
CW		 968 if (unlikely(atomic_read(&dev_priv->mm.wedged)))
969 return -EIO;
970
55249baa 971 if (unlikely(ring->tail + n > ring->effective_size)) {
e1f99ce6
CW		 972 ret = intel_wrap_ring_buffer(ring);
973 if (unlikely(ret))
974 return ret;
975 }
78501eac 976
e1f99ce6
CW		 977 if (unlikely(ring->space < n)) {
978 ret = intel_wait_ring_buffer(ring, n);
979 if (unlikely(ret))
980 return ret;
981 }
d97ed339
CW		 982
983 ring->space -= n;
e1f99ce6 984 return 0;
8187a2b7 985}
62fdfeaf 986
78501eac 987void intel_ring_advance(struct intel_ring_buffer *ring)
8187a2b7 988{
d97ed339 989 ring->tail &= ring->size - 1;
78501eac 990 ring->write_tail(ring, ring->tail);
8187a2b7 991}
62fdfeaf 992
e070868e 993static const struct intel_ring_buffer render_ring = {
8187a2b7 994 .name = "render ring",
9220434a 995 .id = RING_RENDER,
333e9fe9 996 .mmio_base = RENDER_RING_BASE,
8187a2b7 997 .size = 32 * PAGE_SIZE,
8187a2b7 998 .init = init_render_ring,
297b0c5b 999 .write_tail = ring_write_tail,
8187a2b7
ZN		 1000 .flush = render_ring_flush,
1001 .add_request = render_ring_add_request,
1ec14ad3
CW		 1002 .get_seqno = ring_get_seqno,
1003 .irq_get = render_ring_get_irq,
1004 .irq_put = render_ring_put_irq,
78501eac 1005 .dispatch_execbuffer = render_ring_dispatch_execbuffer,
c6df541c 1006 .cleanup = render_ring_cleanup,
8187a2b7 1007};
d1b851fc
ZN		 1008
1009/* ring buffer for bit-stream decoder */
1010
e070868e 1011static const struct intel_ring_buffer bsd_ring = {
d1b851fc 1012 .name = "bsd ring",
9220434a 1013 .id = RING_BSD,
333e9fe9 1014 .mmio_base = BSD_RING_BASE,
d1b851fc 1015 .size = 32 * PAGE_SIZE,
78501eac 1016 .init = init_ring_common,
297b0c5b 1017 .write_tail = ring_write_tail,
d1b851fc 1018 .flush = bsd_ring_flush,
549f7365 1019 .add_request = ring_add_request,
1ec14ad3
CW		 1020 .get_seqno = ring_get_seqno,
1021 .irq_get = bsd_ring_get_irq,
1022 .irq_put = bsd_ring_put_irq,
78501eac 1023 .dispatch_execbuffer = ring_dispatch_execbuffer,
d1b851fc 1024};
5c1143bb 1025
881f47b6 1026
78501eac 1027static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
297b0c5b 1028 u32 value)
881f47b6 1029{
78501eac 1030 drm_i915_private_t *dev_priv = ring->dev->dev_private;
881f47b6
XH		 1031
1032 /* Every tail move must follow the sequence below */
1033 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1034 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1035 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
1036 I915_WRITE(GEN6_BSD_RNCID, 0x0);
1037
1038 if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1039 GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
1040 50))
1041 DRM_ERROR("timed out waiting for IDLE Indicator\n");
1042
870e86dd 1043 I915_WRITE_TAIL(ring, value);
881f47b6
XH		 1044 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1045 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1046 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
1047}
1048
b72f3acb 1049static int gen6_ring_flush(struct intel_ring_buffer *ring,
71a77e07 1050 u32 invalidate, u32 flush)
881f47b6 1051{
71a77e07 1052 uint32_t cmd;
b72f3acb
CW		 1053 int ret;
1054
b72f3acb
CW		 1055 ret = intel_ring_begin(ring, 4);
1056 if (ret)
1057 return ret;
1058
71a77e07
CW		 1059 cmd = MI_FLUSH_DW;
1060 if (invalidate & I915_GEM_GPU_DOMAINS)
1061 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
1062 intel_ring_emit(ring, cmd);
b72f3acb
CW		 1063 intel_ring_emit(ring, 0);
1064 intel_ring_emit(ring, 0);
71a77e07 1065 intel_ring_emit(ring, MI_NOOP);
b72f3acb
CW		 1066 intel_ring_advance(ring);
1067 return 0;
881f47b6
XH		 1068}
1069
1070static int
78501eac 1071gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
c4e7a414 1072 u32 offset, u32 len)
881f47b6 1073{
e1f99ce6 1074 int ret;
ab6f8e32 1075
e1f99ce6
CW		 1076 ret = intel_ring_begin(ring, 2);
1077 if (ret)
1078 return ret;
1079
78501eac 1080 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
ab6f8e32 1081 /* bit0-7 is the length on GEN6+ */
c4e7a414 1082 intel_ring_emit(ring, offset);
78501eac 1083 intel_ring_advance(ring);
ab6f8e32 1084
881f47b6
XH		 1085 return 0;
1086}
1087
0f46832f
CW		 1088static bool
1089gen6_render_ring_get_irq(struct intel_ring_buffer *ring)
1090{
1091 return gen6_ring_get_irq(ring,
1092 GT_USER_INTERRUPT,
1093 GEN6_RENDER_USER_INTERRUPT);
1094}
1095
1096static void
1097gen6_render_ring_put_irq(struct intel_ring_buffer *ring)
1098{
1099 return gen6_ring_put_irq(ring,
1100 GT_USER_INTERRUPT,
1101 GEN6_RENDER_USER_INTERRUPT);
1102}
1103
b13c2b96 1104static bool
1ec14ad3
CW		 1105gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
1106{
0f46832f
CW		 1107 return gen6_ring_get_irq(ring,
1108 GT_GEN6_BSD_USER_INTERRUPT,
1109 GEN6_BSD_USER_INTERRUPT);
1ec14ad3
CW		 1110}
1111
1112static void
1113gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
1114{
0f46832f
CW		 1115 return gen6_ring_put_irq(ring,
1116 GT_GEN6_BSD_USER_INTERRUPT,
1117 GEN6_BSD_USER_INTERRUPT);
1ec14ad3
CW		 1118}
1119
881f47b6 1120/* ring buffer for Video Codec for Gen6+ */
e070868e 1121static const struct intel_ring_buffer gen6_bsd_ring = {
1ec14ad3
CW		 1122 .name = "gen6 bsd ring",
1123 .id = RING_BSD,
1124 .mmio_base = GEN6_BSD_RING_BASE,
1125 .size = 32 * PAGE_SIZE,
1126 .init = init_ring_common,
1127 .write_tail = gen6_bsd_ring_write_tail,
1128 .flush = gen6_ring_flush,
1129 .add_request = gen6_add_request,
1130 .get_seqno = ring_get_seqno,
1131 .irq_get = gen6_bsd_ring_get_irq,
1132 .irq_put = gen6_bsd_ring_put_irq,
1133 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
549f7365
CW		 1134};
1135
1136/* Blitter support (SandyBridge+) */
1137
b13c2b96 1138static bool
1ec14ad3 1139blt_ring_get_irq(struct intel_ring_buffer *ring)
549f7365 1140{
0f46832f
CW		 1141 return gen6_ring_get_irq(ring,
1142 GT_BLT_USER_INTERRUPT,
1143 GEN6_BLITTER_USER_INTERRUPT);
549f7365 1144}
1ec14ad3 1145
549f7365 1146static void
1ec14ad3 1147blt_ring_put_irq(struct intel_ring_buffer *ring)
549f7365 1148{
0f46832f
CW		 1149 gen6_ring_put_irq(ring,
1150 GT_BLT_USER_INTERRUPT,
1151 GEN6_BLITTER_USER_INTERRUPT);
549f7365
CW		 1152}
1153
8d19215b
ZN		 1154
1155/* Workaround for some stepping of SNB,
1156 * each time when BLT engine ring tail moved,
1157 * the first command in the ring to be parsed
1158 * should be MI_BATCH_BUFFER_START
1159 */
1160#define NEED_BLT_WORKAROUND(dev) \
1161 (IS_GEN6(dev) && (dev->pdev->revision < 8))
1162
1163static inline struct drm_i915_gem_object *
1164to_blt_workaround(struct intel_ring_buffer *ring)
1165{
1166 return ring->private;
1167}
1168
1169static int blt_ring_init(struct intel_ring_buffer *ring)
1170{
1171 if (NEED_BLT_WORKAROUND(ring->dev)) {
1172 struct drm_i915_gem_object *obj;
27153f72 1173 u32 *ptr;
8d19215b
ZN		 1174 int ret;
1175
05394f39 1176 obj = i915_gem_alloc_object(ring->dev, 4096);
8d19215b
ZN		 1177 if (obj == NULL)
1178 return -ENOMEM;
1179
05394f39 1180 ret = i915_gem_object_pin(obj, 4096, true);
8d19215b
ZN		 1181 if (ret) {
1182 drm_gem_object_unreference(&obj->base);
1183 return ret;
1184 }
1185
1186 ptr = kmap(obj->pages[0]);
27153f72
CW		 1187 *ptr++ = MI_BATCH_BUFFER_END;
1188 *ptr++ = MI_NOOP;
8d19215b
ZN		 1189 kunmap(obj->pages[0]);
1190
05394f39 1191 ret = i915_gem_object_set_to_gtt_domain(obj, false);
8d19215b 1192 if (ret) {
05394f39 1193 i915_gem_object_unpin(obj);
8d19215b
ZN		 1194 drm_gem_object_unreference(&obj->base);
1195 return ret;
1196 }
1197
1198 ring->private = obj;
1199 }
1200
1201 return init_ring_common(ring);
1202}
1203
1204static int blt_ring_begin(struct intel_ring_buffer *ring,
1205 int num_dwords)
1206{
1207 if (ring->private) {
1208 int ret = intel_ring_begin(ring, num_dwords+2);
1209 if (ret)
1210 return ret;
1211
1212 intel_ring_emit(ring, MI_BATCH_BUFFER_START);
1213 intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
1214
1215 return 0;
1216 } else
1217 return intel_ring_begin(ring, 4);
1218}
1219
b72f3acb 1220static int blt_ring_flush(struct intel_ring_buffer *ring,
71a77e07 1221 u32 invalidate, u32 flush)
8d19215b 1222{
71a77e07 1223 uint32_t cmd;
b72f3acb
CW		 1224 int ret;
1225
b72f3acb
CW		 1226 ret = blt_ring_begin(ring, 4);
1227 if (ret)
1228 return ret;
1229
71a77e07
CW		 1230 cmd = MI_FLUSH_DW;
1231 if (invalidate & I915_GEM_DOMAIN_RENDER)
1232 cmd |= MI_INVALIDATE_TLB;
1233 intel_ring_emit(ring, cmd);
b72f3acb
CW		 1234 intel_ring_emit(ring, 0);
1235 intel_ring_emit(ring, 0);
71a77e07 1236 intel_ring_emit(ring, MI_NOOP);
b72f3acb
CW		 1237 intel_ring_advance(ring);
1238 return 0;
8d19215b
ZN		 1239}
1240
8d19215b
ZN		 1241static void blt_ring_cleanup(struct intel_ring_buffer *ring)
1242{
1243 if (!ring->private)
1244 return;
1245
1246 i915_gem_object_unpin(ring->private);
1247 drm_gem_object_unreference(ring->private);
1248 ring->private = NULL;
1249}
1250
549f7365
CW		 1251static const struct intel_ring_buffer gen6_blt_ring = {
1252 .name = "blt ring",
1253 .id = RING_BLT,
1254 .mmio_base = BLT_RING_BASE,
1255 .size = 32 * PAGE_SIZE,
8d19215b 1256 .init = blt_ring_init,
297b0c5b 1257 .write_tail = ring_write_tail,
8d19215b 1258 .flush = blt_ring_flush,
1ec14ad3
CW		 1259 .add_request = gen6_add_request,
1260 .get_seqno = ring_get_seqno,
1261 .irq_get = blt_ring_get_irq,
1262 .irq_put = blt_ring_put_irq,
78501eac 1263 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
8d19215b 1264 .cleanup = blt_ring_cleanup,
881f47b6
XH		 1265};
1266
5c1143bb
XH		 1267int intel_init_render_ring_buffer(struct drm_device *dev)
1268{
1269 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 1270 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
5c1143bb 1271
1ec14ad3
CW		 1272 *ring = render_ring;
1273 if (INTEL_INFO(dev)->gen >= 6) {
1274 ring->add_request = gen6_add_request;
0f46832f
CW		 1275 ring->irq_get = gen6_render_ring_get_irq;
1276 ring->irq_put = gen6_render_ring_put_irq;
c6df541c
CW		 1277 } else if (IS_GEN5(dev)) {
1278 ring->add_request = pc_render_add_request;
1279 ring->get_seqno = pc_render_get_seqno;
1ec14ad3 1280 }
5c1143bb
XH		 1281
1282 if (!I915_NEED_GFX_HWS(dev)) {
1ec14ad3
CW		 1283 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1284 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
5c1143bb
XH		 1285 }
1286
1ec14ad3 1287 return intel_init_ring_buffer(dev, ring);
5c1143bb
XH		 1288}
1289
e8616b6c
CW		 1290int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
1291{
1292 drm_i915_private_t *dev_priv = dev->dev_private;
1293 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1294
1295 *ring = render_ring;
1296 if (INTEL_INFO(dev)->gen >= 6) {
1297 ring->add_request = gen6_add_request;
1298 ring->irq_get = gen6_render_ring_get_irq;
1299 ring->irq_put = gen6_render_ring_put_irq;
1300 } else if (IS_GEN5(dev)) {
1301 ring->add_request = pc_render_add_request;
1302 ring->get_seqno = pc_render_get_seqno;
1303 }
1304
1305 ring->dev = dev;
1306 INIT_LIST_HEAD(&ring->active_list);
1307 INIT_LIST_HEAD(&ring->request_list);
1308 INIT_LIST_HEAD(&ring->gpu_write_list);
1309
1310 ring->size = size;
1311 ring->effective_size = ring->size;
1312 if (IS_I830(ring->dev))
1313 ring->effective_size -= 128;
1314
1315 ring->map.offset = start;
1316 ring->map.size = size;
1317 ring->map.type = 0;
1318 ring->map.flags = 0;
1319 ring->map.mtrr = 0;
1320
1321 drm_core_ioremap_wc(&ring->map, dev);
1322 if (ring->map.handle == NULL) {
1323 DRM_ERROR("can not ioremap virtual address for"
1324 " ring buffer\n");
1325 return -ENOMEM;
1326 }
1327
1328 ring->virtual_start = (void __force __iomem *)ring->map.handle;
1329 return 0;
1330}
1331
5c1143bb
XH		 1332int intel_init_bsd_ring_buffer(struct drm_device *dev)
1333{
1334 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 1335 struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
5c1143bb 1336
881f47b6 1337 if (IS_GEN6(dev))
1ec14ad3 1338 *ring = gen6_bsd_ring;
881f47b6 1339 else
1ec14ad3 1340 *ring = bsd_ring;
5c1143bb 1341
1ec14ad3 1342 return intel_init_ring_buffer(dev, ring);
5c1143bb 1343}
549f7365
CW		 1344
1345int intel_init_blt_ring_buffer(struct drm_device *dev)
1346{
1347 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 1348 struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
549f7365 1349
1ec14ad3 1350 *ring = gen6_blt_ring;
549f7365 1351
1ec14ad3 1352 return intel_init_ring_buffer(dev, ring);
549f7365 1353}
Linux kernel - Deliverables
This page took 0.212962 seconds and 5 git commands to generate.