projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
drm/i915: don't use HPD_PORT_A as an alias for HPD_NONE
[deliverable/linux.git] / drivers / gpu / drm / i915 / i915_irq.c
1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
2 */
3 /*
4 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 */
28
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
31 #include <linux/sysrq.h>
32 #include <linux/slab.h>
33 #include <linux/circ_buf.h>
34 #include <drm/drmP.h>
35 #include <drm/i915_drm.h>
36 #include "i915_drv.h"
37 #include "i915_trace.h"
38 #include "intel_drv.h"
39
40 /**
41 * DOC: interrupt handling
42 *
43 * These functions provide the basic support for enabling and disabling the
44 * interrupt handling support. There's a lot more functionality in i915_irq.c
45 * and related files, but that will be described in separate chapters.
46 */
47
48 static const u32 hpd_ibx[HPD_NUM_PINS] = {
49 [HPD_CRT] = SDE_CRT_HOTPLUG,
50 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
51 [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
52 [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
53 [HPD_PORT_D] = SDE_PORTD_HOTPLUG
54 };
55
56 static const u32 hpd_cpt[HPD_NUM_PINS] = {
57 [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
58 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
59 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
60 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
61 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
62 };
63
64 static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
65 [HPD_CRT] = CRT_HOTPLUG_INT_EN,
66 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
67 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
68 [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
69 [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
70 [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
71 };
72
73 static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
74 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
75 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
76 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
77 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
78 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
79 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
80 };
81
82 static const u32 hpd_status_i915[HPD_NUM_PINS] = {
83 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
84 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
85 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
86 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
87 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
88 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
89 };
90
91 /* BXT hpd list */
92 static const u32 hpd_bxt[HPD_NUM_PINS] = {
93 [HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
94 [HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
95 };
96
97 /* IIR can theoretically queue up two events. Be paranoid. */
98 #define GEN8_IRQ_RESET_NDX(type, which) do { \
99 I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
100 POSTING_READ(GEN8_##type##_IMR(which)); \
101 I915_WRITE(GEN8_##type##_IER(which), 0); \
102 I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
103 POSTING_READ(GEN8_##type##_IIR(which)); \
104 I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
105 POSTING_READ(GEN8_##type##_IIR(which)); \
106 } while (0)
107
108 #define GEN5_IRQ_RESET(type) do { \
109 I915_WRITE(type##IMR, 0xffffffff); \
110 POSTING_READ(type##IMR); \
111 I915_WRITE(type##IER, 0); \
112 I915_WRITE(type##IIR, 0xffffffff); \
113 POSTING_READ(type##IIR); \
114 I915_WRITE(type##IIR, 0xffffffff); \
115 POSTING_READ(type##IIR); \
116 } while (0)
117
118 /*
119 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
120 */
121 #define GEN5_ASSERT_IIR_IS_ZERO(reg) do { \
122 u32 val = I915_READ(reg); \
123 if (val) { \
124 WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", \
125 (reg), val); \
126 I915_WRITE((reg), 0xffffffff); \
127 POSTING_READ(reg); \
128 I915_WRITE((reg), 0xffffffff); \
129 POSTING_READ(reg); \
130 } \
131 } while (0)
132
133 #define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
134 GEN5_ASSERT_IIR_IS_ZERO(GEN8_##type##_IIR(which)); \
135 I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
136 I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
137 POSTING_READ(GEN8_##type##_IMR(which)); \
138 } while (0)
139
140 #define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
141 GEN5_ASSERT_IIR_IS_ZERO(type##IIR); \
142 I915_WRITE(type##IER, (ier_val)); \
143 I915_WRITE(type##IMR, (imr_val)); \
144 POSTING_READ(type##IMR); \
145 } while (0)
146
147 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
148
149 /* For display hotplug interrupt */
150 void
151 ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
152 {
153 assert_spin_locked(&dev_priv->irq_lock);
154
155 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
156 return;
157
158 if ((dev_priv->irq_mask & mask) != 0) {
159 dev_priv->irq_mask &= ~mask;
160 I915_WRITE(DEIMR, dev_priv->irq_mask);
161 POSTING_READ(DEIMR);
162 }
163 }
164
165 void
166 ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
167 {
168 assert_spin_locked(&dev_priv->irq_lock);
169
170 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
171 return;
172
173 if ((dev_priv->irq_mask & mask) != mask) {
174 dev_priv->irq_mask |= mask;
175 I915_WRITE(DEIMR, dev_priv->irq_mask);
176 POSTING_READ(DEIMR);
177 }
178 }
179
180 /**
181 * ilk_update_gt_irq - update GTIMR
182 * @dev_priv: driver private
183 * @interrupt_mask: mask of interrupt bits to update
184 * @enabled_irq_mask: mask of interrupt bits to enable
185 */
186 static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
187 uint32_t interrupt_mask,
188 uint32_t enabled_irq_mask)
189 {
190 assert_spin_locked(&dev_priv->irq_lock);
191
192 WARN_ON(enabled_irq_mask & ~interrupt_mask);
193
194 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
195 return;
196
197 dev_priv->gt_irq_mask &= ~interrupt_mask;
198 dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
199 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
200 POSTING_READ(GTIMR);
201 }
202
203 void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
204 {
205 ilk_update_gt_irq(dev_priv, mask, mask);
206 }
207
208 void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
209 {
210 ilk_update_gt_irq(dev_priv, mask, 0);
211 }
212
213 static u32 gen6_pm_iir(struct drm_i915_private *dev_priv)
214 {
215 return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
216 }
217
218 static u32 gen6_pm_imr(struct drm_i915_private *dev_priv)
219 {
220 return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
221 }
222
223 static u32 gen6_pm_ier(struct drm_i915_private *dev_priv)
224 {
225 return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
226 }
227
228 /**
229 * snb_update_pm_irq - update GEN6_PMIMR
230 * @dev_priv: driver private
231 * @interrupt_mask: mask of interrupt bits to update
232 * @enabled_irq_mask: mask of interrupt bits to enable
233 */
234 static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
235 uint32_t interrupt_mask,
236 uint32_t enabled_irq_mask)
237 {
238 uint32_t new_val;
239
240 WARN_ON(enabled_irq_mask & ~interrupt_mask);
241
242 assert_spin_locked(&dev_priv->irq_lock);
243
244 new_val = dev_priv->pm_irq_mask;
245 new_val &= ~interrupt_mask;
246 new_val |= (~enabled_irq_mask & interrupt_mask);
247
248 if (new_val != dev_priv->pm_irq_mask) {
249 dev_priv->pm_irq_mask = new_val;
250 I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
251 POSTING_READ(gen6_pm_imr(dev_priv));
252 }
253 }
254
255 void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
256 {
257 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
258 return;
259
260 snb_update_pm_irq(dev_priv, mask, mask);
261 }
262
263 static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
264 uint32_t mask)
265 {
266 snb_update_pm_irq(dev_priv, mask, 0);
267 }
268
269 void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
270 {
271 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
272 return;
273
274 __gen6_disable_pm_irq(dev_priv, mask);
275 }
276
277 void gen6_reset_rps_interrupts(struct drm_device *dev)
278 {
279 struct drm_i915_private *dev_priv = dev->dev_private;
280 uint32_t reg = gen6_pm_iir(dev_priv);
281
282 spin_lock_irq(&dev_priv->irq_lock);
283 I915_WRITE(reg, dev_priv->pm_rps_events);
284 I915_WRITE(reg, dev_priv->pm_rps_events);
285 POSTING_READ(reg);
286 dev_priv->rps.pm_iir = 0;
287 spin_unlock_irq(&dev_priv->irq_lock);
288 }
289
290 void gen6_enable_rps_interrupts(struct drm_device *dev)
291 {
292 struct drm_i915_private *dev_priv = dev->dev_private;
293
294 spin_lock_irq(&dev_priv->irq_lock);
295
296 WARN_ON(dev_priv->rps.pm_iir);
297 WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
298 dev_priv->rps.interrupts_enabled = true;
299 I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) |
300 dev_priv->pm_rps_events);
301 gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
302
303 spin_unlock_irq(&dev_priv->irq_lock);
304 }
305
306 u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
307 {
308 /*
309 * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
310 * if GEN6_PM_UP_EI_EXPIRED is masked.
311 *
312 * TODO: verify if this can be reproduced on VLV,CHV.
313 */
314 if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
315 mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;
316
317 if (INTEL_INFO(dev_priv)->gen >= 8)
318 mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;
319
320 return mask;
321 }
322
323 void gen6_disable_rps_interrupts(struct drm_device *dev)
324 {
325 struct drm_i915_private *dev_priv = dev->dev_private;
326
327 spin_lock_irq(&dev_priv->irq_lock);
328 dev_priv->rps.interrupts_enabled = false;
329 spin_unlock_irq(&dev_priv->irq_lock);
330
331 cancel_work_sync(&dev_priv->rps.work);
332
333 spin_lock_irq(&dev_priv->irq_lock);
334
335 I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
336
337 __gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
338 I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
339 ~dev_priv->pm_rps_events);
340
341 spin_unlock_irq(&dev_priv->irq_lock);
342
343 synchronize_irq(dev->irq);
344 }
345
346 /**
347 * ibx_display_interrupt_update - update SDEIMR
348 * @dev_priv: driver private
349 * @interrupt_mask: mask of interrupt bits to update
350 * @enabled_irq_mask: mask of interrupt bits to enable
351 */
352 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
353 uint32_t interrupt_mask,
354 uint32_t enabled_irq_mask)
355 {
356 uint32_t sdeimr = I915_READ(SDEIMR);
357 sdeimr &= ~interrupt_mask;
358 sdeimr |= (~enabled_irq_mask & interrupt_mask);
359
360 WARN_ON(enabled_irq_mask & ~interrupt_mask);
361
362 assert_spin_locked(&dev_priv->irq_lock);
363
364 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
365 return;
366
367 I915_WRITE(SDEIMR, sdeimr);
368 POSTING_READ(SDEIMR);
369 }
370
371 static void
372 __i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
373 u32 enable_mask, u32 status_mask)
374 {
375 u32 reg = PIPESTAT(pipe);
376 u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
377
378 assert_spin_locked(&dev_priv->irq_lock);
379 WARN_ON(!intel_irqs_enabled(dev_priv));
380
381 if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
382 status_mask & ~PIPESTAT_INT_STATUS_MASK,
383 "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
384 pipe_name(pipe), enable_mask, status_mask))
385 return;
386
387 if ((pipestat & enable_mask) == enable_mask)
388 return;
389
390 dev_priv->pipestat_irq_mask[pipe] |= status_mask;
391
392 /* Enable the interrupt, clear any pending status */
393 pipestat |= enable_mask | status_mask;
394 I915_WRITE(reg, pipestat);
395 POSTING_READ(reg);
396 }
397
398 static void
399 __i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
400 u32 enable_mask, u32 status_mask)
401 {
402 u32 reg = PIPESTAT(pipe);
403 u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
404
405 assert_spin_locked(&dev_priv->irq_lock);
406 WARN_ON(!intel_irqs_enabled(dev_priv));
407
408 if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
409 status_mask & ~PIPESTAT_INT_STATUS_MASK,
410 "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
411 pipe_name(pipe), enable_mask, status_mask))
412 return;
413
414 if ((pipestat & enable_mask) == 0)
415 return;
416
417 dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
418
419 pipestat &= ~enable_mask;
420 I915_WRITE(reg, pipestat);
421 POSTING_READ(reg);
422 }
423
424 static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
425 {
426 u32 enable_mask = status_mask << 16;
427
428 /*
429 * On pipe A we don't support the PSR interrupt yet,
430 * on pipe B and C the same bit MBZ.
431 */
432 if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
433 return 0;
434 /*
435 * On pipe B and C we don't support the PSR interrupt yet, on pipe
436 * A the same bit is for perf counters which we don't use either.
437 */
438 if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
439 return 0;
440
441 enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
442 SPRITE0_FLIP_DONE_INT_EN_VLV |
443 SPRITE1_FLIP_DONE_INT_EN_VLV);
444 if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
445 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
446 if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
447 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
448
449 return enable_mask;
450 }
451
452 void
453 i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
454 u32 status_mask)
455 {
456 u32 enable_mask;
457
458 if (IS_VALLEYVIEW(dev_priv->dev))
459 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
460 status_mask);
461 else
462 enable_mask = status_mask << 16;
463 __i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
464 }
465
466 void
467 i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
468 u32 status_mask)
469 {
470 u32 enable_mask;
471
472 if (IS_VALLEYVIEW(dev_priv->dev))
473 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
474 status_mask);
475 else
476 enable_mask = status_mask << 16;
477 __i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
478 }
479
480 /**
481 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
482 */
483 static void i915_enable_asle_pipestat(struct drm_device *dev)
484 {
485 struct drm_i915_private *dev_priv = dev->dev_private;
486
487 if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
488 return;
489
490 spin_lock_irq(&dev_priv->irq_lock);
491
492 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
493 if (INTEL_INFO(dev)->gen >= 4)
494 i915_enable_pipestat(dev_priv, PIPE_A,
495 PIPE_LEGACY_BLC_EVENT_STATUS);
496
497 spin_unlock_irq(&dev_priv->irq_lock);
498 }
499
500 /*
501 * This timing diagram depicts the video signal in and
502 * around the vertical blanking period.
503 *
504 * Assumptions about the fictitious mode used in this example:
505 * vblank_start >= 3
506 * vsync_start = vblank_start + 1
507 * vsync_end = vblank_start + 2
508 * vtotal = vblank_start + 3
509 *
510 * start of vblank:
511 * latch double buffered registers
512 * increment frame counter (ctg+)
513 * generate start of vblank interrupt (gen4+)
514 * |
515 * | frame start:
516 * | generate frame start interrupt (aka. vblank interrupt) (gmch)
517 * | may be shifted forward 1-3 extra lines via PIPECONF
518 * | |
519 * | | start of vsync:
520 * | | generate vsync interrupt
521 * | | |
522 * ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx
523 * . \hs/ . \hs/ \hs/ \hs/ . \hs/
524 * ----va---> <-----------------vb--------------------> <--------va-------------
525 * | | <----vs-----> |
526 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
527 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
528 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
529 * | | |
530 * last visible pixel first visible pixel
531 * | increment frame counter (gen3/4)
532 * pixel counter = vblank_start * htotal pixel counter = 0 (gen3/4)
533 *
534 * x = horizontal active
535 * _ = horizontal blanking
536 * hs = horizontal sync
537 * va = vertical active
538 * vb = vertical blanking
539 * vs = vertical sync
540 * vbs = vblank_start (number)
541 *
542 * Summary:
543 * - most events happen at the start of horizontal sync
544 * - frame start happens at the start of horizontal blank, 1-4 lines
545 * (depending on PIPECONF settings) after the start of vblank
546 * - gen3/4 pixel and frame counter are synchronized with the start
547 * of horizontal active on the first line of vertical active
548 */
549
550 static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe)
551 {
552 /* Gen2 doesn't have a hardware frame counter */
553 return 0;
554 }
555
556 /* Called from drm generic code, passed a 'crtc', which
557 * we use as a pipe index
558 */
559 static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
560 {
561 struct drm_i915_private *dev_priv = dev->dev_private;
562 unsigned long high_frame;
563 unsigned long low_frame;
564 u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
565 struct intel_crtc *intel_crtc =
566 to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
567 const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
568
569 htotal = mode->crtc_htotal;
570 hsync_start = mode->crtc_hsync_start;
571 vbl_start = mode->crtc_vblank_start;
572 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
573 vbl_start = DIV_ROUND_UP(vbl_start, 2);
574
575 /* Convert to pixel count */
576 vbl_start *= htotal;
577
578 /* Start of vblank event occurs at start of hsync */
579 vbl_start -= htotal - hsync_start;
580
581 high_frame = PIPEFRAME(pipe);
582 low_frame = PIPEFRAMEPIXEL(pipe);
583
584 /*
585 * High & low register fields aren't synchronized, so make sure
586 * we get a low value that's stable across two reads of the high
587 * register.
588 */
589 do {
590 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
591 low = I915_READ(low_frame);
592 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
593 } while (high1 != high2);
594
595 high1 >>= PIPE_FRAME_HIGH_SHIFT;
596 pixel = low & PIPE_PIXEL_MASK;
597 low >>= PIPE_FRAME_LOW_SHIFT;
598
599 /*
600 * The frame counter increments at beginning of active.
601 * Cook up a vblank counter by also checking the pixel
602 * counter against vblank start.
603 */
604 return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
605 }
606
607 static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
608 {
609 struct drm_i915_private *dev_priv = dev->dev_private;
610 int reg = PIPE_FRMCOUNT_GM45(pipe);
611
612 return I915_READ(reg);
613 }
614
615 /* raw reads, only for fast reads of display block, no need for forcewake etc. */
616 #define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
617
618 static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
619 {
620 struct drm_device *dev = crtc->base.dev;
621 struct drm_i915_private *dev_priv = dev->dev_private;
622 const struct drm_display_mode *mode = &crtc->base.hwmode;
623 enum pipe pipe = crtc->pipe;
624 int position, vtotal;
625
626 vtotal = mode->crtc_vtotal;
627 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
628 vtotal /= 2;
629
630 if (IS_GEN2(dev))
631 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
632 else
633 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
634
635 /*
636 * See update_scanline_offset() for the details on the
637 * scanline_offset adjustment.
638 */
639 return (position + crtc->scanline_offset) % vtotal;
640 }
641
642 static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
643 unsigned int flags, int *vpos, int *hpos,
644 ktime_t *stime, ktime_t *etime)
645 {
646 struct drm_i915_private *dev_priv = dev->dev_private;
647 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
648 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
649 const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
650 int position;
651 int vbl_start, vbl_end, hsync_start, htotal, vtotal;
652 bool in_vbl = true;
653 int ret = 0;
654 unsigned long irqflags;
655
656 if (WARN_ON(!mode->crtc_clock)) {
657 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
658 "pipe %c\n", pipe_name(pipe));
659 return 0;
660 }
661
662 htotal = mode->crtc_htotal;
663 hsync_start = mode->crtc_hsync_start;
664 vtotal = mode->crtc_vtotal;
665 vbl_start = mode->crtc_vblank_start;
666 vbl_end = mode->crtc_vblank_end;
667
668 if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
669 vbl_start = DIV_ROUND_UP(vbl_start, 2);
670 vbl_end /= 2;
671 vtotal /= 2;
672 }
673
674 ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
675
676 /*
677 * Lock uncore.lock, as we will do multiple timing critical raw
678 * register reads, potentially with preemption disabled, so the
679 * following code must not block on uncore.lock.
680 */
681 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
682
683 /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
684
685 /* Get optional system timestamp before query. */
686 if (stime)
687 *stime = ktime_get();
688
689 if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
690 /* No obvious pixelcount register. Only query vertical
691 * scanout position from Display scan line register.
692 */
693 position = __intel_get_crtc_scanline(intel_crtc);
694 } else {
695 /* Have access to pixelcount since start of frame.
696 * We can split this into vertical and horizontal
697 * scanout position.
698 */
699 position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
700
701 /* convert to pixel counts */
702 vbl_start *= htotal;
703 vbl_end *= htotal;
704 vtotal *= htotal;
705
706 /*
707 * In interlaced modes, the pixel counter counts all pixels,
708 * so one field will have htotal more pixels. In order to avoid
709 * the reported position from jumping backwards when the pixel
710 * counter is beyond the length of the shorter field, just
711 * clamp the position the length of the shorter field. This
712 * matches how the scanline counter based position works since
713 * the scanline counter doesn't count the two half lines.
714 */
715 if (position >= vtotal)
716 position = vtotal - 1;
717
718 /*
719 * Start of vblank interrupt is triggered at start of hsync,
720 * just prior to the first active line of vblank. However we
721 * consider lines to start at the leading edge of horizontal
722 * active. So, should we get here before we've crossed into
723 * the horizontal active of the first line in vblank, we would
724 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
725 * always add htotal-hsync_start to the current pixel position.
726 */
727 position = (position + htotal - hsync_start) % vtotal;
728 }
729
730 /* Get optional system timestamp after query. */
731 if (etime)
732 *etime = ktime_get();
733
734 /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
735
736 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
737
738 in_vbl = position >= vbl_start && position < vbl_end;
739
740 /*
741 * While in vblank, position will be negative
742 * counting up towards 0 at vbl_end. And outside
743 * vblank, position will be positive counting
744 * up since vbl_end.
745 */
746 if (position >= vbl_start)
747 position -= vbl_end;
748 else
749 position += vtotal - vbl_end;
750
751 if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
752 *vpos = position;
753 *hpos = 0;
754 } else {
755 *vpos = position / htotal;
756 *hpos = position - (*vpos * htotal);
757 }
758
759 /* In vblank? */
760 if (in_vbl)
761 ret |= DRM_SCANOUTPOS_IN_VBLANK;
762
763 return ret;
764 }
765
766 int intel_get_crtc_scanline(struct intel_crtc *crtc)
767 {
768 struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
769 unsigned long irqflags;
770 int position;
771
772 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
773 position = __intel_get_crtc_scanline(crtc);
774 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
775
776 return position;
777 }
778
779 static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
780 int *max_error,
781 struct timeval *vblank_time,
782 unsigned flags)
783 {
784 struct drm_crtc *crtc;
785
786 if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
787 DRM_ERROR("Invalid crtc %d\n", pipe);
788 return -EINVAL;
789 }
790
791 /* Get drm_crtc to timestamp: */
792 crtc = intel_get_crtc_for_pipe(dev, pipe);
793 if (crtc == NULL) {
794 DRM_ERROR("Invalid crtc %d\n", pipe);
795 return -EINVAL;
796 }
797
798 if (!crtc->hwmode.crtc_clock) {
799 DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
800 return -EBUSY;
801 }
802
803 /* Helper routine in DRM core does all the work: */
804 return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
805 vblank_time, flags,
806 crtc,
807 &crtc->hwmode);
808 }
809
810 static void ironlake_rps_change_irq_handler(struct drm_device *dev)
811 {
812 struct drm_i915_private *dev_priv = dev->dev_private;
813 u32 busy_up, busy_down, max_avg, min_avg;
814 u8 new_delay;
815
816 spin_lock(&mchdev_lock);
817
818 I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
819
820 new_delay = dev_priv->ips.cur_delay;
821
822 I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
823 busy_up = I915_READ(RCPREVBSYTUPAVG);
824 busy_down = I915_READ(RCPREVBSYTDNAVG);
825 max_avg = I915_READ(RCBMAXAVG);
826 min_avg = I915_READ(RCBMINAVG);
827
828 /* Handle RCS change request from hw */
829 if (busy_up > max_avg) {
830 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
831 new_delay = dev_priv->ips.cur_delay - 1;
832 if (new_delay < dev_priv->ips.max_delay)
833 new_delay = dev_priv->ips.max_delay;
834 } else if (busy_down < min_avg) {
835 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
836 new_delay = dev_priv->ips.cur_delay + 1;
837 if (new_delay > dev_priv->ips.min_delay)
838 new_delay = dev_priv->ips.min_delay;
839 }
840
841 if (ironlake_set_drps(dev, new_delay))
842 dev_priv->ips.cur_delay = new_delay;
843
844 spin_unlock(&mchdev_lock);
845
846 return;
847 }
848
849 static void notify_ring(struct intel_engine_cs *ring)
850 {
851 if (!intel_ring_initialized(ring))
852 return;
853
854 trace_i915_gem_request_notify(ring);
855
856 wake_up_all(&ring->irq_queue);
857 }
858
859 static void vlv_c0_read(struct drm_i915_private *dev_priv,
860 struct intel_rps_ei *ei)
861 {
862 ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
863 ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
864 ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
865 }
866
867 static bool vlv_c0_above(struct drm_i915_private *dev_priv,
868 const struct intel_rps_ei *old,
869 const struct intel_rps_ei *now,
870 int threshold)
871 {
872 u64 time, c0;
873
874 if (old->cz_clock == 0)
875 return false;
876
877 time = now->cz_clock - old->cz_clock;
878 time *= threshold * dev_priv->mem_freq;
879
880 /* Workload can be split between render + media, e.g. SwapBuffers
881 * being blitted in X after being rendered in mesa. To account for
882 * this we need to combine both engines into our activity counter.
883 */
884 c0 = now->render_c0 - old->render_c0;
885 c0 += now->media_c0 - old->media_c0;
886 c0 *= 100 * VLV_CZ_CLOCK_TO_MILLI_SEC * 4 / 1000;
887
888 return c0 >= time;
889 }
890
891 void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
892 {
893 vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
894 dev_priv->rps.up_ei = dev_priv->rps.down_ei;
895 }
896
897 static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
898 {
899 struct intel_rps_ei now;
900 u32 events = 0;
901
902 if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
903 return 0;
904
905 vlv_c0_read(dev_priv, &now);
906 if (now.cz_clock == 0)
907 return 0;
908
909 if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
910 if (!vlv_c0_above(dev_priv,
911 &dev_priv->rps.down_ei, &now,
912 dev_priv->rps.down_threshold))
913 events |= GEN6_PM_RP_DOWN_THRESHOLD;
914 dev_priv->rps.down_ei = now;
915 }
916
917 if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
918 if (vlv_c0_above(dev_priv,
919 &dev_priv->rps.up_ei, &now,
920 dev_priv->rps.up_threshold))
921 events |= GEN6_PM_RP_UP_THRESHOLD;
922 dev_priv->rps.up_ei = now;
923 }
924
925 return events;
926 }
927
928 static bool any_waiters(struct drm_i915_private *dev_priv)
929 {
930 struct intel_engine_cs *ring;
931 int i;
932
933 for_each_ring(ring, dev_priv, i)
934 if (ring->irq_refcount)
935 return true;
936
937 return false;
938 }
939
940 static void gen6_pm_rps_work(struct work_struct *work)
941 {
942 struct drm_i915_private *dev_priv =
943 container_of(work, struct drm_i915_private, rps.work);
944 bool client_boost;
945 int new_delay, adj, min, max;
946 u32 pm_iir;
947
948 spin_lock_irq(&dev_priv->irq_lock);
949 /* Speed up work cancelation during disabling rps interrupts. */
950 if (!dev_priv->rps.interrupts_enabled) {
951 spin_unlock_irq(&dev_priv->irq_lock);
952 return;
953 }
954 pm_iir = dev_priv->rps.pm_iir;
955 dev_priv->rps.pm_iir = 0;
956 /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
957 gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
958 client_boost = dev_priv->rps.client_boost;
959 dev_priv->rps.client_boost = false;
960 spin_unlock_irq(&dev_priv->irq_lock);
961
962 /* Make sure we didn't queue anything we're not going to process. */
963 WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
964
965 if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
966 return;
967
968 mutex_lock(&dev_priv->rps.hw_lock);
969
970 pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);
971
972 adj = dev_priv->rps.last_adj;
973 new_delay = dev_priv->rps.cur_freq;
974 min = dev_priv->rps.min_freq_softlimit;
975 max = dev_priv->rps.max_freq_softlimit;
976
977 if (client_boost) {
978 new_delay = dev_priv->rps.max_freq_softlimit;
979 adj = 0;
980 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
981 if (adj > 0)
982 adj *= 2;
983 else /* CHV needs even encode values */
984 adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
985 /*
986 * For better performance, jump directly
987 * to RPe if we're below it.
988 */
989 if (new_delay < dev_priv->rps.efficient_freq - adj) {
990 new_delay = dev_priv->rps.efficient_freq;
991 adj = 0;
992 }
993 } else if (any_waiters(dev_priv)) {
994 adj = 0;
995 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
996 if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
997 new_delay = dev_priv->rps.efficient_freq;
998 else
999 new_delay = dev_priv->rps.min_freq_softlimit;
1000 adj = 0;
1001 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1002 if (adj < 0)
1003 adj *= 2;
1004 else /* CHV needs even encode values */
1005 adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1006 } else { /* unknown event */
1007 adj = 0;
1008 }
1009
1010 dev_priv->rps.last_adj = adj;
1011
1012 /* sysfs frequency interfaces may have snuck in while servicing the
1013 * interrupt
1014 */
1015 new_delay += adj;
1016 new_delay = clamp_t(int, new_delay, min, max);
1017
1018 intel_set_rps(dev_priv->dev, new_delay);
1019
1020 mutex_unlock(&dev_priv->rps.hw_lock);
1021 }
1022
1023
1024 /**
1025 * ivybridge_parity_work - Workqueue called when a parity error interrupt
1026 * occurred.
1027 * @work: workqueue struct
1028 *
1029 * Doesn't actually do anything except notify userspace. As a consequence of
1030 * this event, userspace should try to remap the bad rows since statistically
1031 * it is likely the same row is more likely to go bad again.
1032 */
1033 static void ivybridge_parity_work(struct work_struct *work)
1034 {
1035 struct drm_i915_private *dev_priv =
1036 container_of(work, struct drm_i915_private, l3_parity.error_work);
1037 u32 error_status, row, bank, subbank;
1038 char *parity_event[6];
1039 uint32_t misccpctl;
1040 uint8_t slice = 0;
1041
1042 /* We must turn off DOP level clock gating to access the L3 registers.
1043 * In order to prevent a get/put style interface, acquire struct mutex
1044 * any time we access those registers.
1045 */
1046 mutex_lock(&dev_priv->dev->struct_mutex);
1047
1048 /* If we've screwed up tracking, just let the interrupt fire again */
1049 if (WARN_ON(!dev_priv->l3_parity.which_slice))
1050 goto out;
1051
1052 misccpctl = I915_READ(GEN7_MISCCPCTL);
1053 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
1054 POSTING_READ(GEN7_MISCCPCTL);
1055
1056 while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1057 u32 reg;
1058
1059 slice--;
1060 if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
1061 break;
1062
1063 dev_priv->l3_parity.which_slice &= ~(1<<slice);
1064
1065 reg = GEN7_L3CDERRST1 + (slice * 0x200);
1066
1067 error_status = I915_READ(reg);
1068 row = GEN7_PARITY_ERROR_ROW(error_status);
1069 bank = GEN7_PARITY_ERROR_BANK(error_status);
1070 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
1071
1072 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
1073 POSTING_READ(reg);
1074
1075 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
1076 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
1077 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
1078 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
1079 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
1080 parity_event[5] = NULL;
1081
1082 kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
1083 KOBJ_CHANGE, parity_event);
1084
1085 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
1086 slice, row, bank, subbank);
1087
1088 kfree(parity_event[4]);
1089 kfree(parity_event[3]);
1090 kfree(parity_event[2]);
1091 kfree(parity_event[1]);
1092 }
1093
1094 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1095
1096 out:
1097 WARN_ON(dev_priv->l3_parity.which_slice);
1098 spin_lock_irq(&dev_priv->irq_lock);
1099 gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
1100 spin_unlock_irq(&dev_priv->irq_lock);
1101
1102 mutex_unlock(&dev_priv->dev->struct_mutex);
1103 }
1104
1105 static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
1106 {
1107 struct drm_i915_private *dev_priv = dev->dev_private;
1108
1109 if (!HAS_L3_DPF(dev))
1110 return;
1111
1112 spin_lock(&dev_priv->irq_lock);
1113 gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
1114 spin_unlock(&dev_priv->irq_lock);
1115
1116 iir &= GT_PARITY_ERROR(dev);
1117 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
1118 dev_priv->l3_parity.which_slice |= 1 << 1;
1119
1120 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
1121 dev_priv->l3_parity.which_slice |= 1 << 0;
1122
1123 queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1124 }
1125
1126 static void ilk_gt_irq_handler(struct drm_device *dev,
1127 struct drm_i915_private *dev_priv,
1128 u32 gt_iir)
1129 {
1130 if (gt_iir &
1131 (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1132 notify_ring(&dev_priv->ring[RCS]);
1133 if (gt_iir & ILK_BSD_USER_INTERRUPT)
1134 notify_ring(&dev_priv->ring[VCS]);
1135 }
1136
1137 static void snb_gt_irq_handler(struct drm_device *dev,
1138 struct drm_i915_private *dev_priv,
1139 u32 gt_iir)
1140 {
1141
1142 if (gt_iir &
1143 (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1144 notify_ring(&dev_priv->ring[RCS]);
1145 if (gt_iir & GT_BSD_USER_INTERRUPT)
1146 notify_ring(&dev_priv->ring[VCS]);
1147 if (gt_iir & GT_BLT_USER_INTERRUPT)
1148 notify_ring(&dev_priv->ring[BCS]);
1149
1150 if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
1151 GT_BSD_CS_ERROR_INTERRUPT |
1152 GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
1153 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1154
1155 if (gt_iir & GT_PARITY_ERROR(dev))
1156 ivybridge_parity_error_irq_handler(dev, gt_iir);
1157 }
1158
1159 static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
1160 u32 master_ctl)
1161 {
1162 irqreturn_t ret = IRQ_NONE;
1163
1164 if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1165 u32 tmp = I915_READ_FW(GEN8_GT_IIR(0));
1166 if (tmp) {
1167 I915_WRITE_FW(GEN8_GT_IIR(0), tmp);
1168 ret = IRQ_HANDLED;
1169
1170 if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
1171 intel_lrc_irq_handler(&dev_priv->ring[RCS]);
1172 if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
1173 notify_ring(&dev_priv->ring[RCS]);
1174
1175 if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
1176 intel_lrc_irq_handler(&dev_priv->ring[BCS]);
1177 if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
1178 notify_ring(&dev_priv->ring[BCS]);
1179 } else
1180 DRM_ERROR("The master control interrupt lied (GT0)!\n");
1181 }
1182
1183 if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1184 u32 tmp = I915_READ_FW(GEN8_GT_IIR(1));
1185 if (tmp) {
1186 I915_WRITE_FW(GEN8_GT_IIR(1), tmp);
1187 ret = IRQ_HANDLED;
1188
1189 if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
1190 intel_lrc_irq_handler(&dev_priv->ring[VCS]);
1191 if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
1192 notify_ring(&dev_priv->ring[VCS]);
1193
1194 if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
1195 intel_lrc_irq_handler(&dev_priv->ring[VCS2]);
1196 if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
1197 notify_ring(&dev_priv->ring[VCS2]);
1198 } else
1199 DRM_ERROR("The master control interrupt lied (GT1)!\n");
1200 }
1201
1202 if (master_ctl & GEN8_GT_VECS_IRQ) {
1203 u32 tmp = I915_READ_FW(GEN8_GT_IIR(3));
1204 if (tmp) {
1205 I915_WRITE_FW(GEN8_GT_IIR(3), tmp);
1206 ret = IRQ_HANDLED;
1207
1208 if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
1209 intel_lrc_irq_handler(&dev_priv->ring[VECS]);
1210 if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
1211 notify_ring(&dev_priv->ring[VECS]);
1212 } else
1213 DRM_ERROR("The master control interrupt lied (GT3)!\n");
1214 }
1215
1216 if (master_ctl & GEN8_GT_PM_IRQ) {
1217 u32 tmp = I915_READ_FW(GEN8_GT_IIR(2));
1218 if (tmp & dev_priv->pm_rps_events) {
1219 I915_WRITE_FW(GEN8_GT_IIR(2),
1220 tmp & dev_priv->pm_rps_events);
1221 ret = IRQ_HANDLED;
1222 gen6_rps_irq_handler(dev_priv, tmp);
1223 } else
1224 DRM_ERROR("The master control interrupt lied (PM)!\n");
1225 }
1226
1227 return ret;
1228 }
1229
1230 static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1231 {
1232 switch (port) {
1233 case PORT_B:
1234 return val & PORTB_HOTPLUG_LONG_DETECT;
1235 case PORT_C:
1236 return val & PORTC_HOTPLUG_LONG_DETECT;
1237 case PORT_D:
1238 return val & PORTD_HOTPLUG_LONG_DETECT;
1239 default:
1240 return false;
1241 }
1242 }
1243
1244 static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1245 {
1246 switch (port) {
1247 case PORT_B:
1248 return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1249 case PORT_C:
1250 return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1251 case PORT_D:
1252 return val & PORTD_HOTPLUG_INT_LONG_PULSE;
1253 default:
1254 return false;
1255 }
1256 }
1257
1258 /* Get a bit mask of pins that have triggered, and which ones may be long. */
1259 static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1260 u32 hotplug_trigger, u32 dig_hotplug_reg,
1261 const u32 hpd[HPD_NUM_PINS],
1262 bool long_pulse_detect(enum port port, u32 val))
1263 {
1264 enum port port;
1265 int i;
1266
1267 *pin_mask = 0;
1268 *long_mask = 0;
1269
1270 for_each_hpd_pin(i) {
1271 if ((hpd[i] & hotplug_trigger) == 0)
1272 continue;
1273
1274 *pin_mask |= BIT(i);
1275
1276 if (!intel_hpd_pin_to_port(i, &port))
1277 continue;
1278
1279 if (long_pulse_detect(port, dig_hotplug_reg))
1280 *long_mask |= BIT(i);
1281 }
1282
1283 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
1284 hotplug_trigger, dig_hotplug_reg, *pin_mask);
1285
1286 }
1287
1288 static void gmbus_irq_handler(struct drm_device *dev)
1289 {
1290 struct drm_i915_private *dev_priv = dev->dev_private;
1291
1292 wake_up_all(&dev_priv->gmbus_wait_queue);
1293 }
1294
1295 static void dp_aux_irq_handler(struct drm_device *dev)
1296 {
1297 struct drm_i915_private *dev_priv = dev->dev_private;
1298
1299 wake_up_all(&dev_priv->gmbus_wait_queue);
1300 }
1301
1302 #if defined(CONFIG_DEBUG_FS)
1303 static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1304 uint32_t crc0, uint32_t crc1,
1305 uint32_t crc2, uint32_t crc3,
1306 uint32_t crc4)
1307 {
1308 struct drm_i915_private *dev_priv = dev->dev_private;
1309 struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
1310 struct intel_pipe_crc_entry *entry;
1311 int head, tail;
1312
1313 spin_lock(&pipe_crc->lock);
1314
1315 if (!pipe_crc->entries) {
1316 spin_unlock(&pipe_crc->lock);
1317 DRM_DEBUG_KMS("spurious interrupt\n");
1318 return;
1319 }
1320
1321 head = pipe_crc->head;
1322 tail = pipe_crc->tail;
1323
1324 if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1325 spin_unlock(&pipe_crc->lock);
1326 DRM_ERROR("CRC buffer overflowing\n");
1327 return;
1328 }
1329
1330 entry = &pipe_crc->entries[head];
1331
1332 entry->frame = dev->driver->get_vblank_counter(dev, pipe);
1333 entry->crc[0] = crc0;
1334 entry->crc[1] = crc1;
1335 entry->crc[2] = crc2;
1336 entry->crc[3] = crc3;
1337 entry->crc[4] = crc4;
1338
1339 head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1340 pipe_crc->head = head;
1341
1342 spin_unlock(&pipe_crc->lock);
1343
1344 wake_up_interruptible(&pipe_crc->wq);
1345 }
1346 #else
1347 static inline void
1348 display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1349 uint32_t crc0, uint32_t crc1,
1350 uint32_t crc2, uint32_t crc3,
1351 uint32_t crc4) {}
1352 #endif
1353
1354
1355 static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1356 {
1357 struct drm_i915_private *dev_priv = dev->dev_private;
1358
1359 display_pipe_crc_irq_handler(dev, pipe,
1360 I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1361 0, 0, 0, 0);
1362 }
1363
1364 static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1365 {
1366 struct drm_i915_private *dev_priv = dev->dev_private;
1367
1368 display_pipe_crc_irq_handler(dev, pipe,
1369 I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1370 I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1371 I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1372 I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
1373 I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1374 }
1375
1376 static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1377 {
1378 struct drm_i915_private *dev_priv = dev->dev_private;
1379 uint32_t res1, res2;
1380
1381 if (INTEL_INFO(dev)->gen >= 3)
1382 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
1383 else
1384 res1 = 0;
1385
1386 if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
1387 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
1388 else
1389 res2 = 0;
1390
1391 display_pipe_crc_irq_handler(dev, pipe,
1392 I915_READ(PIPE_CRC_RES_RED(pipe)),
1393 I915_READ(PIPE_CRC_RES_GREEN(pipe)),
1394 I915_READ(PIPE_CRC_RES_BLUE(pipe)),
1395 res1, res2);
1396 }
1397
1398 /* The RPS events need forcewake, so we add them to a work queue and mask their
1399 * IMR bits until the work is done. Other interrupts can be processed without
1400 * the work queue. */
1401 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1402 {
1403 if (pm_iir & dev_priv->pm_rps_events) {
1404 spin_lock(&dev_priv->irq_lock);
1405 gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
1406 if (dev_priv->rps.interrupts_enabled) {
1407 dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
1408 queue_work(dev_priv->wq, &dev_priv->rps.work);
1409 }
1410 spin_unlock(&dev_priv->irq_lock);
1411 }
1412
1413 if (INTEL_INFO(dev_priv)->gen >= 8)
1414 return;
1415
1416 if (HAS_VEBOX(dev_priv->dev)) {
1417 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1418 notify_ring(&dev_priv->ring[VECS]);
1419
1420 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1421 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1422 }
1423 }
1424
1425 static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
1426 {
1427 if (!drm_handle_vblank(dev, pipe))
1428 return false;
1429
1430 return true;
1431 }
1432
1433 static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
1434 {
1435 struct drm_i915_private *dev_priv = dev->dev_private;
1436 u32 pipe_stats[I915_MAX_PIPES] = { };
1437 int pipe;
1438
1439 spin_lock(&dev_priv->irq_lock);
1440 for_each_pipe(dev_priv, pipe) {
1441 int reg;
1442 u32 mask, iir_bit = 0;
1443
1444 /*
1445 * PIPESTAT bits get signalled even when the interrupt is
1446 * disabled with the mask bits, and some of the status bits do
1447 * not generate interrupts at all (like the underrun bit). Hence
1448 * we need to be careful that we only handle what we want to
1449 * handle.
1450 */
1451
1452 /* fifo underruns are filterered in the underrun handler. */
1453 mask = PIPE_FIFO_UNDERRUN_STATUS;
1454
1455 switch (pipe) {
1456 case PIPE_A:
1457 iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1458 break;
1459 case PIPE_B:
1460 iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1461 break;
1462 case PIPE_C:
1463 iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
1464 break;
1465 }
1466 if (iir & iir_bit)
1467 mask |= dev_priv->pipestat_irq_mask[pipe];
1468
1469 if (!mask)
1470 continue;
1471
1472 reg = PIPESTAT(pipe);
1473 mask |= PIPESTAT_INT_ENABLE_MASK;
1474 pipe_stats[pipe] = I915_READ(reg) & mask;
1475
1476 /*
1477 * Clear the PIPE*STAT regs before the IIR
1478 */
1479 if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
1480 PIPESTAT_INT_STATUS_MASK))
1481 I915_WRITE(reg, pipe_stats[pipe]);
1482 }
1483 spin_unlock(&dev_priv->irq_lock);
1484
1485 for_each_pipe(dev_priv, pipe) {
1486 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
1487 intel_pipe_handle_vblank(dev, pipe))
1488 intel_check_page_flip(dev, pipe);
1489
1490 if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
1491 intel_prepare_page_flip(dev, pipe);
1492 intel_finish_page_flip(dev, pipe);
1493 }
1494
1495 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1496 i9xx_pipe_crc_irq_handler(dev, pipe);
1497
1498 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1499 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1500 }
1501
1502 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1503 gmbus_irq_handler(dev);
1504 }
1505
1506 static void i9xx_hpd_irq_handler(struct drm_device *dev)
1507 {
1508 struct drm_i915_private *dev_priv = dev->dev_private;
1509 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
1510 u32 pin_mask, long_mask;
1511
1512 if (!hotplug_status)
1513 return;
1514
1515 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1516 /*
1517 * Make sure hotplug status is cleared before we clear IIR, or else we
1518 * may miss hotplug events.
1519 */
1520 POSTING_READ(PORT_HOTPLUG_STAT);
1521
1522 if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
1523 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1524
1525 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1526 hotplug_trigger, hpd_status_g4x,
1527 i9xx_port_hotplug_long_detect);
1528 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1529
1530 if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1531 dp_aux_irq_handler(dev);
1532 } else {
1533 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1534
1535 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1536 hotplug_trigger, hpd_status_g4x,
1537 i9xx_port_hotplug_long_detect);
1538 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1539 }
1540 }
1541
1542 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
1543 {
1544 struct drm_device *dev = arg;
1545 struct drm_i915_private *dev_priv = dev->dev_private;
1546 u32 iir, gt_iir, pm_iir;
1547 irqreturn_t ret = IRQ_NONE;
1548
1549 if (!intel_irqs_enabled(dev_priv))
1550 return IRQ_NONE;
1551
1552 while (true) {
1553 /* Find, clear, then process each source of interrupt */
1554
1555 gt_iir = I915_READ(GTIIR);
1556 if (gt_iir)
1557 I915_WRITE(GTIIR, gt_iir);
1558
1559 pm_iir = I915_READ(GEN6_PMIIR);
1560 if (pm_iir)
1561 I915_WRITE(GEN6_PMIIR, pm_iir);
1562
1563 iir = I915_READ(VLV_IIR);
1564 if (iir) {
1565 /* Consume port before clearing IIR or we'll miss events */
1566 if (iir & I915_DISPLAY_PORT_INTERRUPT)
1567 i9xx_hpd_irq_handler(dev);
1568 I915_WRITE(VLV_IIR, iir);
1569 }
1570
1571 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1572 goto out;
1573
1574 ret = IRQ_HANDLED;
1575
1576 if (gt_iir)
1577 snb_gt_irq_handler(dev, dev_priv, gt_iir);
1578 if (pm_iir)
1579 gen6_rps_irq_handler(dev_priv, pm_iir);
1580 /* Call regardless, as some status bits might not be
1581 * signalled in iir */
1582 valleyview_pipestat_irq_handler(dev, iir);
1583 }
1584
1585 out:
1586 return ret;
1587 }
1588
1589 static irqreturn_t cherryview_irq_handler(int irq, void *arg)
1590 {
1591 struct drm_device *dev = arg;
1592 struct drm_i915_private *dev_priv = dev->dev_private;
1593 u32 master_ctl, iir;
1594 irqreturn_t ret = IRQ_NONE;
1595
1596 if (!intel_irqs_enabled(dev_priv))
1597 return IRQ_NONE;
1598
1599 for (;;) {
1600 master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
1601 iir = I915_READ(VLV_IIR);
1602
1603 if (master_ctl == 0 && iir == 0)
1604 break;
1605
1606 ret = IRQ_HANDLED;
1607
1608 I915_WRITE(GEN8_MASTER_IRQ, 0);
1609
1610 /* Find, clear, then process each source of interrupt */
1611
1612 if (iir) {
1613 /* Consume port before clearing IIR or we'll miss events */
1614 if (iir & I915_DISPLAY_PORT_INTERRUPT)
1615 i9xx_hpd_irq_handler(dev);
1616 I915_WRITE(VLV_IIR, iir);
1617 }
1618
1619 gen8_gt_irq_handler(dev_priv, master_ctl);
1620
1621 /* Call regardless, as some status bits might not be
1622 * signalled in iir */
1623 valleyview_pipestat_irq_handler(dev, iir);
1624
1625 I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
1626 POSTING_READ(GEN8_MASTER_IRQ);
1627 }
1628
1629 return ret;
1630 }
1631
1632 static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1633 {
1634 struct drm_i915_private *dev_priv = dev->dev_private;
1635 int pipe;
1636 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1637
1638 if (hotplug_trigger) {
1639 u32 dig_hotplug_reg, pin_mask, long_mask;
1640
1641 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1642 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1643
1644 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1645 dig_hotplug_reg, hpd_ibx,
1646 pch_port_hotplug_long_detect);
1647 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1648 }
1649
1650 if (pch_iir & SDE_AUDIO_POWER_MASK) {
1651 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1652 SDE_AUDIO_POWER_SHIFT);
1653 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1654 port_name(port));
1655 }
1656
1657 if (pch_iir & SDE_AUX_MASK)
1658 dp_aux_irq_handler(dev);
1659
1660 if (pch_iir & SDE_GMBUS)
1661 gmbus_irq_handler(dev);
1662
1663 if (pch_iir & SDE_AUDIO_HDCP_MASK)
1664 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
1665
1666 if (pch_iir & SDE_AUDIO_TRANS_MASK)
1667 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
1668
1669 if (pch_iir & SDE_POISON)
1670 DRM_ERROR("PCH poison interrupt\n");
1671
1672 if (pch_iir & SDE_FDI_MASK)
1673 for_each_pipe(dev_priv, pipe)
1674 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
1675 pipe_name(pipe),
1676 I915_READ(FDI_RX_IIR(pipe)));
1677
1678 if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1679 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
1680
1681 if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1682 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
1683
1684 if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1685 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1686
1687 if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1688 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1689 }
1690
1691 static void ivb_err_int_handler(struct drm_device *dev)
1692 {
1693 struct drm_i915_private *dev_priv = dev->dev_private;
1694 u32 err_int = I915_READ(GEN7_ERR_INT);
1695 enum pipe pipe;
1696
1697 if (err_int & ERR_INT_POISON)
1698 DRM_ERROR("Poison interrupt\n");
1699
1700 for_each_pipe(dev_priv, pipe) {
1701 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
1702 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1703
1704 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
1705 if (IS_IVYBRIDGE(dev))
1706 ivb_pipe_crc_irq_handler(dev, pipe);
1707 else
1708 hsw_pipe_crc_irq_handler(dev, pipe);
1709 }
1710 }
1711
1712 I915_WRITE(GEN7_ERR_INT, err_int);
1713 }
1714
1715 static void cpt_serr_int_handler(struct drm_device *dev)
1716 {
1717 struct drm_i915_private *dev_priv = dev->dev_private;
1718 u32 serr_int = I915_READ(SERR_INT);
1719
1720 if (serr_int & SERR_INT_POISON)
1721 DRM_ERROR("PCH poison interrupt\n");
1722
1723 if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1724 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1725
1726 if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1727 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1728
1729 if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1730 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
1731
1732 I915_WRITE(SERR_INT, serr_int);
1733 }
1734
1735 static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
1736 {
1737 struct drm_i915_private *dev_priv = dev->dev_private;
1738 int pipe;
1739 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1740
1741 if (hotplug_trigger) {
1742 u32 dig_hotplug_reg, pin_mask, long_mask;
1743
1744 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1745 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1746
1747 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1748 dig_hotplug_reg, hpd_cpt,
1749 pch_port_hotplug_long_detect);
1750 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1751 }
1752
1753 if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1754 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1755 SDE_AUDIO_POWER_SHIFT_CPT);
1756 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
1757 port_name(port));
1758 }
1759
1760 if (pch_iir & SDE_AUX_MASK_CPT)
1761 dp_aux_irq_handler(dev);
1762
1763 if (pch_iir & SDE_GMBUS_CPT)
1764 gmbus_irq_handler(dev);
1765
1766 if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1767 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
1768
1769 if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1770 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
1771
1772 if (pch_iir & SDE_FDI_MASK_CPT)
1773 for_each_pipe(dev_priv, pipe)
1774 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
1775 pipe_name(pipe),
1776 I915_READ(FDI_RX_IIR(pipe)));
1777
1778 if (pch_iir & SDE_ERROR_CPT)
1779 cpt_serr_int_handler(dev);
1780 }
1781
1782 static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
1783 {
1784 struct drm_i915_private *dev_priv = dev->dev_private;
1785 enum pipe pipe;
1786
1787 if (de_iir & DE_AUX_CHANNEL_A)
1788 dp_aux_irq_handler(dev);
1789
1790 if (de_iir & DE_GSE)
1791 intel_opregion_asle_intr(dev);
1792
1793 if (de_iir & DE_POISON)
1794 DRM_ERROR("Poison interrupt\n");
1795
1796 for_each_pipe(dev_priv, pipe) {
1797 if (de_iir & DE_PIPE_VBLANK(pipe) &&
1798 intel_pipe_handle_vblank(dev, pipe))
1799 intel_check_page_flip(dev, pipe);
1800
1801 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
1802 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1803
1804 if (de_iir & DE_PIPE_CRC_DONE(pipe))
1805 i9xx_pipe_crc_irq_handler(dev, pipe);
1806
1807 /* plane/pipes map 1:1 on ilk+ */
1808 if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
1809 intel_prepare_page_flip(dev, pipe);
1810 intel_finish_page_flip_plane(dev, pipe);
1811 }
1812 }
1813
1814 /* check event from PCH */
1815 if (de_iir & DE_PCH_EVENT) {
1816 u32 pch_iir = I915_READ(SDEIIR);
1817
1818 if (HAS_PCH_CPT(dev))
1819 cpt_irq_handler(dev, pch_iir);
1820 else
1821 ibx_irq_handler(dev, pch_iir);
1822
1823 /* should clear PCH hotplug event before clear CPU irq */
1824 I915_WRITE(SDEIIR, pch_iir);
1825 }
1826
1827 if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
1828 ironlake_rps_change_irq_handler(dev);
1829 }
1830
1831 static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
1832 {
1833 struct drm_i915_private *dev_priv = dev->dev_private;
1834 enum pipe pipe;
1835
1836 if (de_iir & DE_ERR_INT_IVB)
1837 ivb_err_int_handler(dev);
1838
1839 if (de_iir & DE_AUX_CHANNEL_A_IVB)
1840 dp_aux_irq_handler(dev);
1841
1842 if (de_iir & DE_GSE_IVB)
1843 intel_opregion_asle_intr(dev);
1844
1845 for_each_pipe(dev_priv, pipe) {
1846 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
1847 intel_pipe_handle_vblank(dev, pipe))
1848 intel_check_page_flip(dev, pipe);
1849
1850 /* plane/pipes map 1:1 on ilk+ */
1851 if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
1852 intel_prepare_page_flip(dev, pipe);
1853 intel_finish_page_flip_plane(dev, pipe);
1854 }
1855 }
1856
1857 /* check event from PCH */
1858 if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
1859 u32 pch_iir = I915_READ(SDEIIR);
1860
1861 cpt_irq_handler(dev, pch_iir);
1862
1863 /* clear PCH hotplug event before clear CPU irq */
1864 I915_WRITE(SDEIIR, pch_iir);
1865 }
1866 }
1867
1868 /*
1869 * To handle irqs with the minimum potential races with fresh interrupts, we:
1870 * 1 - Disable Master Interrupt Control.
1871 * 2 - Find the source(s) of the interrupt.
1872 * 3 - Clear the Interrupt Identity bits (IIR).
1873 * 4 - Process the interrupt(s) that had bits set in the IIRs.
1874 * 5 - Re-enable Master Interrupt Control.
1875 */
1876 static irqreturn_t ironlake_irq_handler(int irq, void *arg)
1877 {
1878 struct drm_device *dev = arg;
1879 struct drm_i915_private *dev_priv = dev->dev_private;
1880 u32 de_iir, gt_iir, de_ier, sde_ier = 0;
1881 irqreturn_t ret = IRQ_NONE;
1882
1883 if (!intel_irqs_enabled(dev_priv))
1884 return IRQ_NONE;
1885
1886 /* We get interrupts on unclaimed registers, so check for this before we
1887 * do any I915_{READ,WRITE}. */
1888 intel_uncore_check_errors(dev);
1889
1890 /* disable master interrupt before clearing iir */
1891 de_ier = I915_READ(DEIER);
1892 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
1893 POSTING_READ(DEIER);
1894
1895 /* Disable south interrupts. We'll only write to SDEIIR once, so further
1896 * interrupts will will be stored on its back queue, and then we'll be
1897 * able to process them after we restore SDEIER (as soon as we restore
1898 * it, we'll get an interrupt if SDEIIR still has something to process
1899 * due to its back queue). */
1900 if (!HAS_PCH_NOP(dev)) {
1901 sde_ier = I915_READ(SDEIER);
1902 I915_WRITE(SDEIER, 0);
1903 POSTING_READ(SDEIER);
1904 }
1905
1906 /* Find, clear, then process each source of interrupt */
1907
1908 gt_iir = I915_READ(GTIIR);
1909 if (gt_iir) {
1910 I915_WRITE(GTIIR, gt_iir);
1911 ret = IRQ_HANDLED;
1912 if (INTEL_INFO(dev)->gen >= 6)
1913 snb_gt_irq_handler(dev, dev_priv, gt_iir);
1914 else
1915 ilk_gt_irq_handler(dev, dev_priv, gt_iir);
1916 }
1917
1918 de_iir = I915_READ(DEIIR);
1919 if (de_iir) {
1920 I915_WRITE(DEIIR, de_iir);
1921 ret = IRQ_HANDLED;
1922 if (INTEL_INFO(dev)->gen >= 7)
1923 ivb_display_irq_handler(dev, de_iir);
1924 else
1925 ilk_display_irq_handler(dev, de_iir);
1926 }
1927
1928 if (INTEL_INFO(dev)->gen >= 6) {
1929 u32 pm_iir = I915_READ(GEN6_PMIIR);
1930 if (pm_iir) {
1931 I915_WRITE(GEN6_PMIIR, pm_iir);
1932 ret = IRQ_HANDLED;
1933 gen6_rps_irq_handler(dev_priv, pm_iir);
1934 }
1935 }
1936
1937 I915_WRITE(DEIER, de_ier);
1938 POSTING_READ(DEIER);
1939 if (!HAS_PCH_NOP(dev)) {
1940 I915_WRITE(SDEIER, sde_ier);
1941 POSTING_READ(SDEIER);
1942 }
1943
1944 return ret;
1945 }
1946
1947 static void bxt_hpd_handler(struct drm_device *dev, uint32_t iir_status)
1948 {
1949 struct drm_i915_private *dev_priv = dev->dev_private;
1950 u32 hp_control, hp_trigger;
1951 u32 pin_mask, long_mask;
1952
1953 /* Get the status */
1954 hp_trigger = iir_status & BXT_DE_PORT_HOTPLUG_MASK;
1955 hp_control = I915_READ(BXT_HOTPLUG_CTL);
1956
1957 /* Hotplug not enabled ? */
1958 if (!(hp_control & BXT_HOTPLUG_CTL_MASK)) {
1959 DRM_ERROR("Interrupt when HPD disabled\n");
1960 return;
1961 }
1962
1963 /* Clear sticky bits in hpd status */
1964 I915_WRITE(BXT_HOTPLUG_CTL, hp_control);
1965
1966 intel_get_hpd_pins(&pin_mask, &long_mask, hp_trigger, hp_control,
1967 hpd_bxt, pch_port_hotplug_long_detect);
1968 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1969 }
1970
1971 static irqreturn_t gen8_irq_handler(int irq, void *arg)
1972 {
1973 struct drm_device *dev = arg;
1974 struct drm_i915_private *dev_priv = dev->dev_private;
1975 u32 master_ctl;
1976 irqreturn_t ret = IRQ_NONE;
1977 uint32_t tmp = 0;
1978 enum pipe pipe;
1979 u32 aux_mask = GEN8_AUX_CHANNEL_A;
1980
1981 if (!intel_irqs_enabled(dev_priv))
1982 return IRQ_NONE;
1983
1984 if (IS_GEN9(dev))
1985 aux_mask |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
1986 GEN9_AUX_CHANNEL_D;
1987
1988 master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
1989 master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
1990 if (!master_ctl)
1991 return IRQ_NONE;
1992
1993 I915_WRITE_FW(GEN8_MASTER_IRQ, 0);
1994
1995 /* Find, clear, then process each source of interrupt */
1996
1997 ret = gen8_gt_irq_handler(dev_priv, master_ctl);
1998
1999 if (master_ctl & GEN8_DE_MISC_IRQ) {
2000 tmp = I915_READ(GEN8_DE_MISC_IIR);
2001 if (tmp) {
2002 I915_WRITE(GEN8_DE_MISC_IIR, tmp);
2003 ret = IRQ_HANDLED;
2004 if (tmp & GEN8_DE_MISC_GSE)
2005 intel_opregion_asle_intr(dev);
2006 else
2007 DRM_ERROR("Unexpected DE Misc interrupt\n");
2008 }
2009 else
2010 DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2011 }
2012
2013 if (master_ctl & GEN8_DE_PORT_IRQ) {
2014 tmp = I915_READ(GEN8_DE_PORT_IIR);
2015 if (tmp) {
2016 bool found = false;
2017
2018 I915_WRITE(GEN8_DE_PORT_IIR, tmp);
2019 ret = IRQ_HANDLED;
2020
2021 if (tmp & aux_mask) {
2022 dp_aux_irq_handler(dev);
2023 found = true;
2024 }
2025
2026 if (IS_BROXTON(dev) && tmp & BXT_DE_PORT_HOTPLUG_MASK) {
2027 bxt_hpd_handler(dev, tmp);
2028 found = true;
2029 }
2030
2031 if (IS_BROXTON(dev) && (tmp & BXT_DE_PORT_GMBUS)) {
2032 gmbus_irq_handler(dev);
2033 found = true;
2034 }
2035
2036 if (!found)
2037 DRM_ERROR("Unexpected DE Port interrupt\n");
2038 }
2039 else
2040 DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2041 }
2042
2043 for_each_pipe(dev_priv, pipe) {
2044 uint32_t pipe_iir, flip_done = 0, fault_errors = 0;
2045
2046 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2047 continue;
2048
2049 pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
2050 if (pipe_iir) {
2051 ret = IRQ_HANDLED;
2052 I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
2053
2054 if (pipe_iir & GEN8_PIPE_VBLANK &&
2055 intel_pipe_handle_vblank(dev, pipe))
2056 intel_check_page_flip(dev, pipe);
2057
2058 if (IS_GEN9(dev))
2059 flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
2060 else
2061 flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;
2062
2063 if (flip_done) {
2064 intel_prepare_page_flip(dev, pipe);
2065 intel_finish_page_flip_plane(dev, pipe);
2066 }
2067
2068 if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
2069 hsw_pipe_crc_irq_handler(dev, pipe);
2070
2071 if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN)
2072 intel_cpu_fifo_underrun_irq_handler(dev_priv,
2073 pipe);
2074
2075
2076 if (IS_GEN9(dev))
2077 fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
2078 else
2079 fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2080
2081 if (fault_errors)
2082 DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
2083 pipe_name(pipe),
2084 pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
2085 } else
2086 DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
2087 }
2088
2089 if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) &&
2090 master_ctl & GEN8_DE_PCH_IRQ) {
2091 /*
2092 * FIXME(BDW): Assume for now that the new interrupt handling
2093 * scheme also closed the SDE interrupt handling race we've seen
2094 * on older pch-split platforms. But this needs testing.
2095 */
2096 u32 pch_iir = I915_READ(SDEIIR);
2097 if (pch_iir) {
2098 I915_WRITE(SDEIIR, pch_iir);
2099 ret = IRQ_HANDLED;
2100 cpt_irq_handler(dev, pch_iir);
2101 } else
2102 DRM_ERROR("The master control interrupt lied (SDE)!\n");
2103
2104 }
2105
2106 I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2107 POSTING_READ_FW(GEN8_MASTER_IRQ);
2108
2109 return ret;
2110 }
2111
2112 static void i915_error_wake_up(struct drm_i915_private *dev_priv,
2113 bool reset_completed)
2114 {
2115 struct intel_engine_cs *ring;
2116 int i;
2117
2118 /*
2119 * Notify all waiters for GPU completion events that reset state has
2120 * been changed, and that they need to restart their wait after
2121 * checking for potential errors (and bail out to drop locks if there is
2122 * a gpu reset pending so that i915_error_work_func can acquire them).
2123 */
2124
2125 /* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
2126 for_each_ring(ring, dev_priv, i)
2127 wake_up_all(&ring->irq_queue);
2128
2129 /* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
2130 wake_up_all(&dev_priv->pending_flip_queue);
2131
2132 /*
2133 * Signal tasks blocked in i915_gem_wait_for_error that the pending
2134 * reset state is cleared.
2135 */
2136 if (reset_completed)
2137 wake_up_all(&dev_priv->gpu_error.reset_queue);
2138 }
2139
2140 /**
2141 * i915_reset_and_wakeup - do process context error handling work
2142 *
2143 * Fire an error uevent so userspace can see that a hang or error
2144 * was detected.
2145 */
2146 static void i915_reset_and_wakeup(struct drm_device *dev)
2147 {
2148 struct drm_i915_private *dev_priv = to_i915(dev);
2149 struct i915_gpu_error *error = &dev_priv->gpu_error;
2150 char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
2151 char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
2152 char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2153 int ret;
2154
2155 kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
2156
2157 /*
2158 * Note that there's only one work item which does gpu resets, so we
2159 * need not worry about concurrent gpu resets potentially incrementing
2160 * error->reset_counter twice. We only need to take care of another
2161 * racing irq/hangcheck declaring the gpu dead for a second time. A
2162 * quick check for that is good enough: schedule_work ensures the
2163 * correct ordering between hang detection and this work item, and since
2164 * the reset in-progress bit is only ever set by code outside of this
2165 * work we don't need to worry about any other races.
2166 */
2167 if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
2168 DRM_DEBUG_DRIVER("resetting chip\n");
2169 kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
2170 reset_event);
2171
2172 /*
2173 * In most cases it's guaranteed that we get here with an RPM
2174 * reference held, for example because there is a pending GPU
2175 * request that won't finish until the reset is done. This
2176 * isn't the case at least when we get here by doing a
2177 * simulated reset via debugs, so get an RPM reference.
2178 */
2179 intel_runtime_pm_get(dev_priv);
2180
2181 intel_prepare_reset(dev);
2182
2183 /*
2184 * All state reset _must_ be completed before we update the
2185 * reset counter, for otherwise waiters might miss the reset
2186 * pending state and not properly drop locks, resulting in
2187 * deadlocks with the reset work.
2188 */
2189 ret = i915_reset(dev);
2190
2191 intel_finish_reset(dev);
2192
2193 intel_runtime_pm_put(dev_priv);
2194
2195 if (ret == 0) {
2196 /*
2197 * After all the gem state is reset, increment the reset
2198 * counter and wake up everyone waiting for the reset to
2199 * complete.
2200 *
2201 * Since unlock operations are a one-sided barrier only,
2202 * we need to insert a barrier here to order any seqno
2203 * updates before
2204 * the counter increment.
2205 */
2206 smp_mb__before_atomic();
2207 atomic_inc(&dev_priv->gpu_error.reset_counter);
2208
2209 kobject_uevent_env(&dev->primary->kdev->kobj,
2210 KOBJ_CHANGE, reset_done_event);
2211 } else {
2212 atomic_set_mask(I915_WEDGED, &error->reset_counter);
2213 }
2214
2215 /*
2216 * Note: The wake_up also serves as a memory barrier so that
2217 * waiters see the update value of the reset counter atomic_t.
2218 */
2219 i915_error_wake_up(dev_priv, true);
2220 }
2221 }
2222
2223 static void i915_report_and_clear_eir(struct drm_device *dev)
2224 {
2225 struct drm_i915_private *dev_priv = dev->dev_private;
2226 uint32_t instdone[I915_NUM_INSTDONE_REG];
2227 u32 eir = I915_READ(EIR);
2228 int pipe, i;
2229
2230 if (!eir)
2231 return;
2232
2233 pr_err("render error detected, EIR: 0x%08x\n", eir);
2234
2235 i915_get_extra_instdone(dev, instdone);
2236
2237 if (IS_G4X(dev)) {
2238 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
2239 u32 ipeir = I915_READ(IPEIR_I965);
2240
2241 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2242 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2243 for (i = 0; i < ARRAY_SIZE(instdone); i++)
2244 pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2245 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
2246 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2247 I915_WRITE(IPEIR_I965, ipeir);
2248 POSTING_READ(IPEIR_I965);
2249 }
2250 if (eir & GM45_ERROR_PAGE_TABLE) {
2251 u32 pgtbl_err = I915_READ(PGTBL_ER);
2252 pr_err("page table error\n");
2253 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
2254 I915_WRITE(PGTBL_ER, pgtbl_err);
2255 POSTING_READ(PGTBL_ER);
2256 }
2257 }
2258
2259 if (!IS_GEN2(dev)) {
2260 if (eir & I915_ERROR_PAGE_TABLE) {
2261 u32 pgtbl_err = I915_READ(PGTBL_ER);
2262 pr_err("page table error\n");
2263 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
2264 I915_WRITE(PGTBL_ER, pgtbl_err);
2265 POSTING_READ(PGTBL_ER);
2266 }
2267 }
2268
2269 if (eir & I915_ERROR_MEMORY_REFRESH) {
2270 pr_err("memory refresh error:\n");
2271 for_each_pipe(dev_priv, pipe)
2272 pr_err("pipe %c stat: 0x%08x\n",
2273 pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
2274 /* pipestat has already been acked */
2275 }
2276 if (eir & I915_ERROR_INSTRUCTION) {
2277 pr_err("instruction error\n");
2278 pr_err(" INSTPM: 0x%08x\n", I915_READ(INSTPM));
2279 for (i = 0; i < ARRAY_SIZE(instdone); i++)
2280 pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2281 if (INTEL_INFO(dev)->gen < 4) {
2282 u32 ipeir = I915_READ(IPEIR);
2283
2284 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR));
2285 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR));
2286 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD));
2287 I915_WRITE(IPEIR, ipeir);
2288 POSTING_READ(IPEIR);
2289 } else {
2290 u32 ipeir = I915_READ(IPEIR_I965);
2291
2292 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2293 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2294 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
2295 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2296 I915_WRITE(IPEIR_I965, ipeir);
2297 POSTING_READ(IPEIR_I965);
2298 }
2299 }
2300
2301 I915_WRITE(EIR, eir);
2302 POSTING_READ(EIR);
2303 eir = I915_READ(EIR);
2304 if (eir) {
2305 /*
2306 * some errors might have become stuck,
2307 * mask them.
2308 */
2309 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
2310 I915_WRITE(EMR, I915_READ(EMR) | eir);
2311 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2312 }
2313 }
2314
2315 /**
2316 * i915_handle_error - handle a gpu error
2317 * @dev: drm device
2318 *
2319 * Do some basic checking of regsiter state at error time and
2320 * dump it to the syslog. Also call i915_capture_error_state() to make
2321 * sure we get a record and make it available in debugfs. Fire a uevent
2322 * so userspace knows something bad happened (should trigger collection
2323 * of a ring dump etc.).
2324 */
2325 void i915_handle_error(struct drm_device *dev, bool wedged,
2326 const char *fmt, ...)
2327 {
2328 struct drm_i915_private *dev_priv = dev->dev_private;
2329 va_list args;
2330 char error_msg[80];
2331
2332 va_start(args, fmt);
2333 vscnprintf(error_msg, sizeof(error_msg), fmt, args);
2334 va_end(args);
2335
2336 i915_capture_error_state(dev, wedged, error_msg);
2337 i915_report_and_clear_eir(dev);
2338
2339 if (wedged) {
2340 atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
2341 &dev_priv->gpu_error.reset_counter);
2342
2343 /*
2344 * Wakeup waiting processes so that the reset function
2345 * i915_reset_and_wakeup doesn't deadlock trying to grab
2346 * various locks. By bumping the reset counter first, the woken
2347 * processes will see a reset in progress and back off,
2348 * releasing their locks and then wait for the reset completion.
2349 * We must do this for _all_ gpu waiters that might hold locks
2350 * that the reset work needs to acquire.
2351 *
2352 * Note: The wake_up serves as the required memory barrier to
2353 * ensure that the waiters see the updated value of the reset
2354 * counter atomic_t.
2355 */
2356 i915_error_wake_up(dev_priv, false);
2357 }
2358
2359 i915_reset_and_wakeup(dev);
2360 }
2361
2362 /* Called from drm generic code, passed 'crtc' which
2363 * we use as a pipe index
2364 */
2365 static int i915_enable_vblank(struct drm_device *dev, int pipe)
2366 {
2367 struct drm_i915_private *dev_priv = dev->dev_private;
2368 unsigned long irqflags;
2369
2370 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2371 if (INTEL_INFO(dev)->gen >= 4)
2372 i915_enable_pipestat(dev_priv, pipe,
2373 PIPE_START_VBLANK_INTERRUPT_STATUS);
2374 else
2375 i915_enable_pipestat(dev_priv, pipe,
2376 PIPE_VBLANK_INTERRUPT_STATUS);
2377 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2378
2379 return 0;
2380 }
2381
2382 static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
2383 {
2384 struct drm_i915_private *dev_priv = dev->dev_private;
2385 unsigned long irqflags;
2386 uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2387 DE_PIPE_VBLANK(pipe);
2388
2389 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2390 ironlake_enable_display_irq(dev_priv, bit);
2391 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2392
2393 return 0;
2394 }
2395
2396 static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
2397 {
2398 struct drm_i915_private *dev_priv = dev->dev_private;
2399 unsigned long irqflags;
2400
2401 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2402 i915_enable_pipestat(dev_priv, pipe,
2403 PIPE_START_VBLANK_INTERRUPT_STATUS);
2404 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2405
2406 return 0;
2407 }
2408
2409 static int gen8_enable_vblank(struct drm_device *dev, int pipe)
2410 {
2411 struct drm_i915_private *dev_priv = dev->dev_private;
2412 unsigned long irqflags;
2413
2414 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2415 dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
2416 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2417 POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2418 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2419 return 0;
2420 }
2421
2422 /* Called from drm generic code, passed 'crtc' which
2423 * we use as a pipe index
2424 */
2425 static void i915_disable_vblank(struct drm_device *dev, int pipe)
2426 {
2427 struct drm_i915_private *dev_priv = dev->dev_private;
2428 unsigned long irqflags;
2429
2430 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2431 i915_disable_pipestat(dev_priv, pipe,
2432 PIPE_VBLANK_INTERRUPT_STATUS |
2433 PIPE_START_VBLANK_INTERRUPT_STATUS);
2434 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2435 }
2436
2437 static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
2438 {
2439 struct drm_i915_private *dev_priv = dev->dev_private;
2440 unsigned long irqflags;
2441 uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2442 DE_PIPE_VBLANK(pipe);
2443
2444 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2445 ironlake_disable_display_irq(dev_priv, bit);
2446 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2447 }
2448
2449 static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
2450 {
2451 struct drm_i915_private *dev_priv = dev->dev_private;
2452 unsigned long irqflags;
2453
2454 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2455 i915_disable_pipestat(dev_priv, pipe,
2456 PIPE_START_VBLANK_INTERRUPT_STATUS);
2457 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2458 }
2459
2460 static void gen8_disable_vblank(struct drm_device *dev, int pipe)
2461 {
2462 struct drm_i915_private *dev_priv = dev->dev_private;
2463 unsigned long irqflags;
2464
2465 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2466 dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
2467 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2468 POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2469 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2470 }
2471
2472 static bool
2473 ring_idle(struct intel_engine_cs *ring, u32 seqno)
2474 {
2475 return (list_empty(&ring->request_list) ||
2476 i915_seqno_passed(seqno, ring->last_submitted_seqno));
2477 }
2478
2479 static bool
2480 ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
2481 {
2482 if (INTEL_INFO(dev)->gen >= 8) {
2483 return (ipehr >> 23) == 0x1c;
2484 } else {
2485 ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
2486 return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
2487 MI_SEMAPHORE_REGISTER);
2488 }
2489 }
2490
2491 static struct intel_engine_cs *
2492 semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
2493 {
2494 struct drm_i915_private *dev_priv = ring->dev->dev_private;
2495 struct intel_engine_cs *signaller;
2496 int i;
2497
2498 if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
2499 for_each_ring(signaller, dev_priv, i) {
2500 if (ring == signaller)
2501 continue;
2502
2503 if (offset == signaller->semaphore.signal_ggtt[ring->id])
2504 return signaller;
2505 }
2506 } else {
2507 u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
2508
2509 for_each_ring(signaller, dev_priv, i) {
2510 if(ring == signaller)
2511 continue;
2512
2513 if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
2514 return signaller;
2515 }
2516 }
2517
2518 DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
2519 ring->id, ipehr, offset);
2520
2521 return NULL;
2522 }
2523
2524 static struct intel_engine_cs *
2525 semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
2526 {
2527 struct drm_i915_private *dev_priv = ring->dev->dev_private;
2528 u32 cmd, ipehr, head;
2529 u64 offset = 0;
2530 int i, backwards;
2531
2532 ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
2533 if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
2534 return NULL;
2535
2536 /*
2537 * HEAD is likely pointing to the dword after the actual command,
2538 * so scan backwards until we find the MBOX. But limit it to just 3
2539 * or 4 dwords depending on the semaphore wait command size.
2540 * Note that we don't care about ACTHD here since that might
2541 * point at at batch, and semaphores are always emitted into the
2542 * ringbuffer itself.
2543 */
2544 head = I915_READ_HEAD(ring) & HEAD_ADDR;
2545 backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
2546
2547 for (i = backwards; i; --i) {
2548 /*
2549 * Be paranoid and presume the hw has gone off into the wild -
2550 * our ring is smaller than what the hardware (and hence
2551 * HEAD_ADDR) allows. Also handles wrap-around.
2552 */
2553 head &= ring->buffer->size - 1;
2554
2555 /* This here seems to blow up */
2556 cmd = ioread32(ring->buffer->virtual_start + head);
2557 if (cmd == ipehr)
2558 break;
2559
2560 head -= 4;
2561 }
2562
2563 if (!i)
2564 return NULL;
2565
2566 *seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
2567 if (INTEL_INFO(ring->dev)->gen >= 8) {
2568 offset = ioread32(ring->buffer->virtual_start + head + 12);
2569 offset <<= 32;
2570 offset = ioread32(ring->buffer->virtual_start + head + 8);
2571 }
2572 return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
2573 }
2574
2575 static int semaphore_passed(struct intel_engine_cs *ring)
2576 {
2577 struct drm_i915_private *dev_priv = ring->dev->dev_private;
2578 struct intel_engine_cs *signaller;
2579 u32 seqno;
2580
2581 ring->hangcheck.deadlock++;
2582
2583 signaller = semaphore_waits_for(ring, &seqno);
2584 if (signaller == NULL)
2585 return -1;
2586
2587 /* Prevent pathological recursion due to driver bugs */
2588 if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
2589 return -1;
2590
2591 if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
2592 return 1;
2593
2594 /* cursory check for an unkickable deadlock */
2595 if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
2596 semaphore_passed(signaller) < 0)
2597 return -1;
2598
2599 return 0;
2600 }
2601
2602 static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
2603 {
2604 struct intel_engine_cs *ring;
2605 int i;
2606
2607 for_each_ring(ring, dev_priv, i)
2608 ring->hangcheck.deadlock = 0;
2609 }
2610
2611 static enum intel_ring_hangcheck_action
2612 ring_stuck(struct intel_engine_cs *ring, u64 acthd)
2613 {
2614 struct drm_device *dev = ring->dev;
2615 struct drm_i915_private *dev_priv = dev->dev_private;
2616 u32 tmp;
2617
2618 if (acthd != ring->hangcheck.acthd) {
2619 if (acthd > ring->hangcheck.max_acthd) {
2620 ring->hangcheck.max_acthd = acthd;
2621 return HANGCHECK_ACTIVE;
2622 }
2623
2624 return HANGCHECK_ACTIVE_LOOP;
2625 }
2626
2627 if (IS_GEN2(dev))
2628 return HANGCHECK_HUNG;
2629
2630 /* Is the chip hanging on a WAIT_FOR_EVENT?
2631 * If so we can simply poke the RB_WAIT bit
2632 * and break the hang. This should work on
2633 * all but the second generation chipsets.
2634 */
2635 tmp = I915_READ_CTL(ring);
2636 if (tmp & RING_WAIT) {
2637 i915_handle_error(dev, false,
2638 "Kicking stuck wait on %s",
2639 ring->name);
2640 I915_WRITE_CTL(ring, tmp);
2641 return HANGCHECK_KICK;
2642 }
2643
2644 if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
2645 switch (semaphore_passed(ring)) {
2646 default:
2647 return HANGCHECK_HUNG;
2648 case 1:
2649 i915_handle_error(dev, false,
2650 "Kicking stuck semaphore on %s",
2651 ring->name);
2652 I915_WRITE_CTL(ring, tmp);
2653 return HANGCHECK_KICK;
2654 case 0:
2655 return HANGCHECK_WAIT;
2656 }
2657 }
2658
2659 return HANGCHECK_HUNG;
2660 }
2661
2662 /*
2663 * This is called when the chip hasn't reported back with completed
2664 * batchbuffers in a long time. We keep track per ring seqno progress and
2665 * if there are no progress, hangcheck score for that ring is increased.
2666 * Further, acthd is inspected to see if the ring is stuck. On stuck case
2667 * we kick the ring. If we see no progress on three subsequent calls
2668 * we assume chip is wedged and try to fix it by resetting the chip.
2669 */
2670 static void i915_hangcheck_elapsed(struct work_struct *work)
2671 {
2672 struct drm_i915_private *dev_priv =
2673 container_of(work, typeof(*dev_priv),
2674 gpu_error.hangcheck_work.work);
2675 struct drm_device *dev = dev_priv->dev;
2676 struct intel_engine_cs *ring;
2677 int i;
2678 int busy_count = 0, rings_hung = 0;
2679 bool stuck[I915_NUM_RINGS] = { 0 };
2680 #define BUSY 1
2681 #define KICK 5
2682 #define HUNG 20
2683
2684 if (!i915.enable_hangcheck)
2685 return;
2686
2687 for_each_ring(ring, dev_priv, i) {
2688 u64 acthd;
2689 u32 seqno;
2690 bool busy = true;
2691
2692 semaphore_clear_deadlocks(dev_priv);
2693
2694 seqno = ring->get_seqno(ring, false);
2695 acthd = intel_ring_get_active_head(ring);
2696
2697 if (ring->hangcheck.seqno == seqno) {
2698 if (ring_idle(ring, seqno)) {
2699 ring->hangcheck.action = HANGCHECK_IDLE;
2700
2701 if (waitqueue_active(&ring->irq_queue)) {
2702 /* Issue a wake-up to catch stuck h/w. */
2703 if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
2704 if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
2705 DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
2706 ring->name);
2707 else
2708 DRM_INFO("Fake missed irq on %s\n",
2709 ring->name);
2710 wake_up_all(&ring->irq_queue);
2711 }
2712 /* Safeguard against driver failure */
2713 ring->hangcheck.score += BUSY;
2714 } else
2715 busy = false;
2716 } else {
2717 /* We always increment the hangcheck score
2718 * if the ring is busy and still processing
2719 * the same request, so that no single request
2720 * can run indefinitely (such as a chain of
2721 * batches). The only time we do not increment
2722 * the hangcheck score on this ring, if this
2723 * ring is in a legitimate wait for another
2724 * ring. In that case the waiting ring is a
2725 * victim and we want to be sure we catch the
2726 * right culprit. Then every time we do kick
2727 * the ring, add a small increment to the
2728 * score so that we can catch a batch that is
2729 * being repeatedly kicked and so responsible
2730 * for stalling the machine.
2731 */
2732 ring->hangcheck.action = ring_stuck(ring,
2733 acthd);
2734
2735 switch (ring->hangcheck.action) {
2736 case HANGCHECK_IDLE:
2737 case HANGCHECK_WAIT:
2738 case HANGCHECK_ACTIVE:
2739 break;
2740 case HANGCHECK_ACTIVE_LOOP:
2741 ring->hangcheck.score += BUSY;
2742 break;
2743 case HANGCHECK_KICK:
2744 ring->hangcheck.score += KICK;
2745 break;
2746 case HANGCHECK_HUNG:
2747 ring->hangcheck.score += HUNG;
2748 stuck[i] = true;
2749 break;
2750 }
2751 }
2752 } else {
2753 ring->hangcheck.action = HANGCHECK_ACTIVE;
2754
2755 /* Gradually reduce the count so that we catch DoS
2756 * attempts across multiple batches.
2757 */
2758 if (ring->hangcheck.score > 0)
2759 ring->hangcheck.score--;
2760
2761 ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
2762 }
2763
2764 ring->hangcheck.seqno = seqno;
2765 ring->hangcheck.acthd = acthd;
2766 busy_count += busy;
2767 }
2768
2769 for_each_ring(ring, dev_priv, i) {
2770 if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
2771 DRM_INFO("%s on %s\n",
2772 stuck[i] ? "stuck" : "no progress",
2773 ring->name);
2774 rings_hung++;
2775 }
2776 }
2777
2778 if (rings_hung)
2779 return i915_handle_error(dev, true, "Ring hung");
2780
2781 if (busy_count)
2782 /* Reset timer case chip hangs without another request
2783 * being added */
2784 i915_queue_hangcheck(dev);
2785 }
2786
2787 void i915_queue_hangcheck(struct drm_device *dev)
2788 {
2789 struct i915_gpu_error *e = &to_i915(dev)->gpu_error;
2790
2791 if (!i915.enable_hangcheck)
2792 return;
2793
2794 /* Don't continually defer the hangcheck so that it is always run at
2795 * least once after work has been scheduled on any ring. Otherwise,
2796 * we will ignore a hung ring if a second ring is kept busy.
2797 */
2798
2799 queue_delayed_work(e->hangcheck_wq, &e->hangcheck_work,
2800 round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES));
2801 }
2802
2803 static void ibx_irq_reset(struct drm_device *dev)
2804 {
2805 struct drm_i915_private *dev_priv = dev->dev_private;
2806
2807 if (HAS_PCH_NOP(dev))
2808 return;
2809
2810 GEN5_IRQ_RESET(SDE);
2811
2812 if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
2813 I915_WRITE(SERR_INT, 0xffffffff);
2814 }
2815
2816 /*
2817 * SDEIER is also touched by the interrupt handler to work around missed PCH
2818 * interrupts. Hence we can't update it after the interrupt handler is enabled -
2819 * instead we unconditionally enable all PCH interrupt sources here, but then
2820 * only unmask them as needed with SDEIMR.
2821 *
2822 * This function needs to be called before interrupts are enabled.
2823 */
2824 static void ibx_irq_pre_postinstall(struct drm_device *dev)
2825 {
2826 struct drm_i915_private *dev_priv = dev->dev_private;
2827
2828 if (HAS_PCH_NOP(dev))
2829 return;
2830
2831 WARN_ON(I915_READ(SDEIER) != 0);
2832 I915_WRITE(SDEIER, 0xffffffff);
2833 POSTING_READ(SDEIER);
2834 }
2835
2836 static void gen5_gt_irq_reset(struct drm_device *dev)
2837 {
2838 struct drm_i915_private *dev_priv = dev->dev_private;
2839
2840 GEN5_IRQ_RESET(GT);
2841 if (INTEL_INFO(dev)->gen >= 6)
2842 GEN5_IRQ_RESET(GEN6_PM);
2843 }
2844
2845 /* drm_dma.h hooks
2846 */
2847 static void ironlake_irq_reset(struct drm_device *dev)
2848 {
2849 struct drm_i915_private *dev_priv = dev->dev_private;
2850
2851 I915_WRITE(HWSTAM, 0xffffffff);
2852
2853 GEN5_IRQ_RESET(DE);
2854 if (IS_GEN7(dev))
2855 I915_WRITE(GEN7_ERR_INT, 0xffffffff);
2856
2857 gen5_gt_irq_reset(dev);
2858
2859 ibx_irq_reset(dev);
2860 }
2861
2862 static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
2863 {
2864 enum pipe pipe;
2865
2866 I915_WRITE(PORT_HOTPLUG_EN, 0);
2867 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2868
2869 for_each_pipe(dev_priv, pipe)
2870 I915_WRITE(PIPESTAT(pipe), 0xffff);
2871
2872 GEN5_IRQ_RESET(VLV_);
2873 }
2874
2875 static void valleyview_irq_preinstall(struct drm_device *dev)
2876 {
2877 struct drm_i915_private *dev_priv = dev->dev_private;
2878
2879 /* VLV magic */
2880 I915_WRITE(VLV_IMR, 0);
2881 I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
2882 I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
2883 I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
2884
2885 gen5_gt_irq_reset(dev);
2886
2887 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
2888
2889 vlv_display_irq_reset(dev_priv);
2890 }
2891
2892 static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
2893 {
2894 GEN8_IRQ_RESET_NDX(GT, 0);
2895 GEN8_IRQ_RESET_NDX(GT, 1);
2896 GEN8_IRQ_RESET_NDX(GT, 2);
2897 GEN8_IRQ_RESET_NDX(GT, 3);
2898 }
2899
2900 static void gen8_irq_reset(struct drm_device *dev)
2901 {
2902 struct drm_i915_private *dev_priv = dev->dev_private;
2903 int pipe;
2904
2905 I915_WRITE(GEN8_MASTER_IRQ, 0);
2906 POSTING_READ(GEN8_MASTER_IRQ);
2907
2908 gen8_gt_irq_reset(dev_priv);
2909
2910 for_each_pipe(dev_priv, pipe)
2911 if (intel_display_power_is_enabled(dev_priv,
2912 POWER_DOMAIN_PIPE(pipe)))
2913 GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
2914
2915 GEN5_IRQ_RESET(GEN8_DE_PORT_);
2916 GEN5_IRQ_RESET(GEN8_DE_MISC_);
2917 GEN5_IRQ_RESET(GEN8_PCU_);
2918
2919 if (HAS_PCH_SPLIT(dev))
2920 ibx_irq_reset(dev);
2921 }
2922
2923 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
2924 unsigned int pipe_mask)
2925 {
2926 uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
2927
2928 spin_lock_irq(&dev_priv->irq_lock);
2929 if (pipe_mask & 1 << PIPE_A)
2930 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_A,
2931 dev_priv->de_irq_mask[PIPE_A],
2932 ~dev_priv->de_irq_mask[PIPE_A] | extra_ier);
2933 if (pipe_mask & 1 << PIPE_B)
2934 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B,
2935 dev_priv->de_irq_mask[PIPE_B],
2936 ~dev_priv->de_irq_mask[PIPE_B] | extra_ier);
2937 if (pipe_mask & 1 << PIPE_C)
2938 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C,
2939 dev_priv->de_irq_mask[PIPE_C],
2940 ~dev_priv->de_irq_mask[PIPE_C] | extra_ier);
2941 spin_unlock_irq(&dev_priv->irq_lock);
2942 }
2943
2944 static void cherryview_irq_preinstall(struct drm_device *dev)
2945 {
2946 struct drm_i915_private *dev_priv = dev->dev_private;
2947
2948 I915_WRITE(GEN8_MASTER_IRQ, 0);
2949 POSTING_READ(GEN8_MASTER_IRQ);
2950
2951 gen8_gt_irq_reset(dev_priv);
2952
2953 GEN5_IRQ_RESET(GEN8_PCU_);
2954
2955 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
2956
2957 vlv_display_irq_reset(dev_priv);
2958 }
2959
2960 static void ibx_hpd_irq_setup(struct drm_device *dev)
2961 {
2962 struct drm_i915_private *dev_priv = dev->dev_private;
2963 struct intel_encoder *intel_encoder;
2964 u32 hotplug_irqs, hotplug, enabled_irqs = 0;
2965
2966 if (HAS_PCH_IBX(dev)) {
2967 hotplug_irqs = SDE_HOTPLUG_MASK;
2968 for_each_intel_encoder(dev, intel_encoder)
2969 if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED)
2970 enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
2971 } else {
2972 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
2973 for_each_intel_encoder(dev, intel_encoder)
2974 if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED)
2975 enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
2976 }
2977
2978 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
2979
2980 /*
2981 * Enable digital hotplug on the PCH, and configure the DP short pulse
2982 * duration to 2ms (which is the minimum in the Display Port spec)
2983 *
2984 * This register is the same on all known PCH chips.
2985 */
2986 hotplug = I915_READ(PCH_PORT_HOTPLUG);
2987 hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
2988 hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
2989 hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
2990 hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
2991 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
2992 }
2993
2994 static void bxt_hpd_irq_setup(struct drm_device *dev)
2995 {
2996 struct drm_i915_private *dev_priv = dev->dev_private;
2997 struct intel_encoder *intel_encoder;
2998 u32 hotplug_port = 0;
2999 u32 hotplug_ctrl;
3000
3001 /* Now, enable HPD */
3002 for_each_intel_encoder(dev, intel_encoder) {
3003 if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state
3004 == HPD_ENABLED)
3005 hotplug_port |= hpd_bxt[intel_encoder->hpd_pin];
3006 }
3007
3008 /* Mask all HPD control bits */
3009 hotplug_ctrl = I915_READ(BXT_HOTPLUG_CTL) & ~BXT_HOTPLUG_CTL_MASK;
3010
3011 /* Enable requested port in hotplug control */
3012 /* TODO: implement (short) HPD support on port A */
3013 WARN_ON_ONCE(hotplug_port & BXT_DE_PORT_HP_DDIA);
3014 if (hotplug_port & BXT_DE_PORT_HP_DDIB)
3015 hotplug_ctrl |= BXT_DDIB_HPD_ENABLE;
3016 if (hotplug_port & BXT_DE_PORT_HP_DDIC)
3017 hotplug_ctrl |= BXT_DDIC_HPD_ENABLE;
3018 I915_WRITE(BXT_HOTPLUG_CTL, hotplug_ctrl);
3019
3020 /* Unmask DDI hotplug in IMR */
3021 hotplug_ctrl = I915_READ(GEN8_DE_PORT_IMR) & ~hotplug_port;
3022 I915_WRITE(GEN8_DE_PORT_IMR, hotplug_ctrl);
3023
3024 /* Enable DDI hotplug in IER */
3025 hotplug_ctrl = I915_READ(GEN8_DE_PORT_IER) | hotplug_port;
3026 I915_WRITE(GEN8_DE_PORT_IER, hotplug_ctrl);
3027 POSTING_READ(GEN8_DE_PORT_IER);
3028 }
3029
3030 static void ibx_irq_postinstall(struct drm_device *dev)
3031 {
3032 struct drm_i915_private *dev_priv = dev->dev_private;
3033 u32 mask;
3034
3035 if (HAS_PCH_NOP(dev))
3036 return;
3037
3038 if (HAS_PCH_IBX(dev))
3039 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3040 else
3041 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3042
3043 GEN5_ASSERT_IIR_IS_ZERO(SDEIIR);
3044 I915_WRITE(SDEIMR, ~mask);
3045 }
3046
3047 static void gen5_gt_irq_postinstall(struct drm_device *dev)
3048 {
3049 struct drm_i915_private *dev_priv = dev->dev_private;
3050 u32 pm_irqs, gt_irqs;
3051
3052 pm_irqs = gt_irqs = 0;
3053
3054 dev_priv->gt_irq_mask = ~0;
3055 if (HAS_L3_DPF(dev)) {
3056 /* L3 parity interrupt is always unmasked. */
3057 dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
3058 gt_irqs |= GT_PARITY_ERROR(dev);
3059 }
3060
3061 gt_irqs |= GT_RENDER_USER_INTERRUPT;
3062 if (IS_GEN5(dev)) {
3063 gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
3064 ILK_BSD_USER_INTERRUPT;
3065 } else {
3066 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
3067 }
3068
3069 GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3070
3071 if (INTEL_INFO(dev)->gen >= 6) {
3072 /*
3073 * RPS interrupts will get enabled/disabled on demand when RPS
3074 * itself is enabled/disabled.
3075 */
3076 if (HAS_VEBOX(dev))
3077 pm_irqs |= PM_VEBOX_USER_INTERRUPT;
3078
3079 dev_priv->pm_irq_mask = 0xffffffff;
3080 GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
3081 }
3082 }
3083
3084 static int ironlake_irq_postinstall(struct drm_device *dev)
3085 {
3086 struct drm_i915_private *dev_priv = dev->dev_private;
3087 u32 display_mask, extra_mask;
3088
3089 if (INTEL_INFO(dev)->gen >= 7) {
3090 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3091 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
3092 DE_PLANEB_FLIP_DONE_IVB |
3093 DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3094 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3095 DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB);
3096 } else {
3097 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3098 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3099 DE_AUX_CHANNEL_A |
3100 DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
3101 DE_POISON);
3102 extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
3103 DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN;
3104 }
3105
3106 dev_priv->irq_mask = ~display_mask;
3107
3108 I915_WRITE(HWSTAM, 0xeffe);
3109
3110 ibx_irq_pre_postinstall(dev);
3111
3112 GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3113
3114 gen5_gt_irq_postinstall(dev);
3115
3116 ibx_irq_postinstall(dev);
3117
3118 if (IS_IRONLAKE_M(dev)) {
3119 /* Enable PCU event interrupts
3120 *
3121 * spinlocking not required here for correctness since interrupt
3122 * setup is guaranteed to run in single-threaded context. But we
3123 * need it to make the assert_spin_locked happy. */
3124 spin_lock_irq(&dev_priv->irq_lock);
3125 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
3126 spin_unlock_irq(&dev_priv->irq_lock);
3127 }
3128
3129 return 0;
3130 }
3131
3132 static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
3133 {
3134 u32 pipestat_mask;
3135 u32 iir_mask;
3136 enum pipe pipe;
3137
3138 pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3139 PIPE_FIFO_UNDERRUN_STATUS;
3140
3141 for_each_pipe(dev_priv, pipe)
3142 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3143 POSTING_READ(PIPESTAT(PIPE_A));
3144
3145 pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3146 PIPE_CRC_DONE_INTERRUPT_STATUS;
3147
3148 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3149 for_each_pipe(dev_priv, pipe)
3150 i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
3151
3152 iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3153 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3154 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3155 if (IS_CHERRYVIEW(dev_priv))
3156 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3157 dev_priv->irq_mask &= ~iir_mask;
3158
3159 I915_WRITE(VLV_IIR, iir_mask);
3160 I915_WRITE(VLV_IIR, iir_mask);
3161 I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3162 I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3163 POSTING_READ(VLV_IMR);
3164 }
3165
3166 static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
3167 {
3168 u32 pipestat_mask;
3169 u32 iir_mask;
3170 enum pipe pipe;
3171
3172 iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3173 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3174 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3175 if (IS_CHERRYVIEW(dev_priv))
3176 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3177
3178 dev_priv->irq_mask |= iir_mask;
3179 I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3180 I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3181 I915_WRITE(VLV_IIR, iir_mask);
3182 I915_WRITE(VLV_IIR, iir_mask);
3183 POSTING_READ(VLV_IIR);
3184
3185 pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3186 PIPE_CRC_DONE_INTERRUPT_STATUS;
3187
3188 i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3189 for_each_pipe(dev_priv, pipe)
3190 i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
3191
3192 pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3193 PIPE_FIFO_UNDERRUN_STATUS;
3194
3195 for_each_pipe(dev_priv, pipe)
3196 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3197 POSTING_READ(PIPESTAT(PIPE_A));
3198 }
3199
3200 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3201 {
3202 assert_spin_locked(&dev_priv->irq_lock);
3203
3204 if (dev_priv->display_irqs_enabled)
3205 return;
3206
3207 dev_priv->display_irqs_enabled = true;
3208
3209 if (intel_irqs_enabled(dev_priv))
3210 valleyview_display_irqs_install(dev_priv);
3211 }
3212
3213 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3214 {
3215 assert_spin_locked(&dev_priv->irq_lock);
3216
3217 if (!dev_priv->display_irqs_enabled)
3218 return;
3219
3220 dev_priv->display_irqs_enabled = false;
3221
3222 if (intel_irqs_enabled(dev_priv))
3223 valleyview_display_irqs_uninstall(dev_priv);
3224 }
3225
3226 static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
3227 {
3228 dev_priv->irq_mask = ~0;
3229
3230 I915_WRITE(PORT_HOTPLUG_EN, 0);
3231 POSTING_READ(PORT_HOTPLUG_EN);
3232
3233 I915_WRITE(VLV_IIR, 0xffffffff);
3234 I915_WRITE(VLV_IIR, 0xffffffff);
3235 I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3236 I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3237 POSTING_READ(VLV_IMR);
3238
3239 /* Interrupt setup is already guaranteed to be single-threaded, this is
3240 * just to make the assert_spin_locked check happy. */
3241 spin_lock_irq(&dev_priv->irq_lock);
3242 if (dev_priv->display_irqs_enabled)
3243 valleyview_display_irqs_install(dev_priv);
3244 spin_unlock_irq(&dev_priv->irq_lock);
3245 }
3246
3247 static int valleyview_irq_postinstall(struct drm_device *dev)
3248 {
3249 struct drm_i915_private *dev_priv = dev->dev_private;
3250
3251 vlv_display_irq_postinstall(dev_priv);
3252
3253 gen5_gt_irq_postinstall(dev);
3254
3255 /* ack & enable invalid PTE error interrupts */
3256 #if 0 /* FIXME: add support to irq handler for checking these bits */
3257 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3258 I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
3259 #endif
3260
3261 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3262
3263 return 0;
3264 }
3265
3266 static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
3267 {
3268 /* These are interrupts we'll toggle with the ring mask register */
3269 uint32_t gt_interrupts[] = {
3270 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3271 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3272 GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
3273 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
3274 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3275 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3276 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3277 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
3278 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3279 0,
3280 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
3281 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3282 };
3283
3284 dev_priv->pm_irq_mask = 0xffffffff;
3285 GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
3286 GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3287 /*
3288 * RPS interrupts will get enabled/disabled on demand when RPS itself
3289 * is enabled/disabled.
3290 */
3291 GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
3292 GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3293 }
3294
3295 static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3296 {
3297 uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
3298 uint32_t de_pipe_enables;
3299 int pipe;
3300 u32 de_port_en = GEN8_AUX_CHANNEL_A;
3301
3302 if (IS_GEN9(dev_priv)) {
3303 de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
3304 GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3305 de_port_en |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
3306 GEN9_AUX_CHANNEL_D;
3307
3308 if (IS_BROXTON(dev_priv))
3309 de_port_en |= BXT_DE_PORT_GMBUS;
3310 } else
3311 de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
3312 GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3313
3314 de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
3315 GEN8_PIPE_FIFO_UNDERRUN;
3316
3317 dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
3318 dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
3319 dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3320
3321 for_each_pipe(dev_priv, pipe)
3322 if (intel_display_power_is_enabled(dev_priv,
3323 POWER_DOMAIN_PIPE(pipe)))
3324 GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
3325 dev_priv->de_irq_mask[pipe],
3326 de_pipe_enables);
3327
3328 GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_en, de_port_en);
3329 }
3330
3331 static int gen8_irq_postinstall(struct drm_device *dev)
3332 {
3333 struct drm_i915_private *dev_priv = dev->dev_private;
3334
3335 if (HAS_PCH_SPLIT(dev))
3336 ibx_irq_pre_postinstall(dev);
3337
3338 gen8_gt_irq_postinstall(dev_priv);
3339 gen8_de_irq_postinstall(dev_priv);
3340
3341 if (HAS_PCH_SPLIT(dev))
3342 ibx_irq_postinstall(dev);
3343
3344 I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
3345 POSTING_READ(GEN8_MASTER_IRQ);
3346
3347 return 0;
3348 }
3349
3350 static int cherryview_irq_postinstall(struct drm_device *dev)
3351 {
3352 struct drm_i915_private *dev_priv = dev->dev_private;
3353
3354 vlv_display_irq_postinstall(dev_priv);
3355
3356 gen8_gt_irq_postinstall(dev_priv);
3357
3358 I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
3359 POSTING_READ(GEN8_MASTER_IRQ);
3360
3361 return 0;
3362 }
3363
3364 static void gen8_irq_uninstall(struct drm_device *dev)
3365 {
3366 struct drm_i915_private *dev_priv = dev->dev_private;
3367
3368 if (!dev_priv)
3369 return;
3370
3371 gen8_irq_reset(dev);
3372 }
3373
3374 static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
3375 {
3376 /* Interrupt setup is already guaranteed to be single-threaded, this is
3377 * just to make the assert_spin_locked check happy. */
3378 spin_lock_irq(&dev_priv->irq_lock);
3379 if (dev_priv->display_irqs_enabled)
3380 valleyview_display_irqs_uninstall(dev_priv);
3381 spin_unlock_irq(&dev_priv->irq_lock);
3382
3383 vlv_display_irq_reset(dev_priv);
3384
3385 dev_priv->irq_mask = ~0;
3386 }
3387
3388 static void valleyview_irq_uninstall(struct drm_device *dev)
3389 {
3390 struct drm_i915_private *dev_priv = dev->dev_private;
3391
3392 if (!dev_priv)
3393 return;
3394
3395 I915_WRITE(VLV_MASTER_IER, 0);
3396
3397 gen5_gt_irq_reset(dev);
3398
3399 I915_WRITE(HWSTAM, 0xffffffff);
3400
3401 vlv_display_irq_uninstall(dev_priv);
3402 }
3403
3404 static void cherryview_irq_uninstall(struct drm_device *dev)
3405 {
3406 struct drm_i915_private *dev_priv = dev->dev_private;
3407
3408 if (!dev_priv)
3409 return;
3410
3411 I915_WRITE(GEN8_MASTER_IRQ, 0);
3412 POSTING_READ(GEN8_MASTER_IRQ);
3413
3414 gen8_gt_irq_reset(dev_priv);
3415
3416 GEN5_IRQ_RESET(GEN8_PCU_);
3417
3418 vlv_display_irq_uninstall(dev_priv);
3419 }
3420
3421 static void ironlake_irq_uninstall(struct drm_device *dev)
3422 {
3423 struct drm_i915_private *dev_priv = dev->dev_private;
3424
3425 if (!dev_priv)
3426 return;
3427
3428 ironlake_irq_reset(dev);
3429 }
3430
3431 static void i8xx_irq_preinstall(struct drm_device * dev)
3432 {
3433 struct drm_i915_private *dev_priv = dev->dev_private;
3434 int pipe;
3435
3436 for_each_pipe(dev_priv, pipe)
3437 I915_WRITE(PIPESTAT(pipe), 0);
3438 I915_WRITE16(IMR, 0xffff);
3439 I915_WRITE16(IER, 0x0);
3440 POSTING_READ16(IER);
3441 }
3442
3443 static int i8xx_irq_postinstall(struct drm_device *dev)
3444 {
3445 struct drm_i915_private *dev_priv = dev->dev_private;
3446
3447 I915_WRITE16(EMR,
3448 ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3449
3450 /* Unmask the interrupts that we always want on. */
3451 dev_priv->irq_mask =
3452 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3453 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3454 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3455 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3456 I915_WRITE16(IMR, dev_priv->irq_mask);
3457
3458 I915_WRITE16(IER,
3459 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3460 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3461 I915_USER_INTERRUPT);
3462 POSTING_READ16(IER);
3463
3464 /* Interrupt setup is already guaranteed to be single-threaded, this is
3465 * just to make the assert_spin_locked check happy. */
3466 spin_lock_irq(&dev_priv->irq_lock);
3467 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3468 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3469 spin_unlock_irq(&dev_priv->irq_lock);
3470
3471 return 0;
3472 }
3473
3474 /*
3475 * Returns true when a page flip has completed.
3476 */
3477 static bool i8xx_handle_vblank(struct drm_device *dev,
3478 int plane, int pipe, u32 iir)
3479 {
3480 struct drm_i915_private *dev_priv = dev->dev_private;
3481 u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3482
3483 if (!intel_pipe_handle_vblank(dev, pipe))
3484 return false;
3485
3486 if ((iir & flip_pending) == 0)
3487 goto check_page_flip;
3488
3489 /* We detect FlipDone by looking for the change in PendingFlip from '1'
3490 * to '0' on the following vblank, i.e. IIR has the Pendingflip
3491 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3492 * the flip is completed (no longer pending). Since this doesn't raise
3493 * an interrupt per se, we watch for the change at vblank.
3494 */
3495 if (I915_READ16(ISR) & flip_pending)
3496 goto check_page_flip;
3497
3498 intel_prepare_page_flip(dev, plane);
3499 intel_finish_page_flip(dev, pipe);
3500 return true;
3501
3502 check_page_flip:
3503 intel_check_page_flip(dev, pipe);
3504 return false;
3505 }
3506
3507 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
3508 {
3509 struct drm_device *dev = arg;
3510 struct drm_i915_private *dev_priv = dev->dev_private;
3511 u16 iir, new_iir;
3512 u32 pipe_stats[2];
3513 int pipe;
3514 u16 flip_mask =
3515 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3516 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3517
3518 if (!intel_irqs_enabled(dev_priv))
3519 return IRQ_NONE;
3520
3521 iir = I915_READ16(IIR);
3522 if (iir == 0)
3523 return IRQ_NONE;
3524
3525 while (iir & ~flip_mask) {
3526 /* Can't rely on pipestat interrupt bit in iir as it might
3527 * have been cleared after the pipestat interrupt was received.
3528 * It doesn't set the bit in iir again, but it still produces
3529 * interrupts (for non-MSI).
3530 */
3531 spin_lock(&dev_priv->irq_lock);
3532 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3533 DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3534
3535 for_each_pipe(dev_priv, pipe) {
3536 int reg = PIPESTAT(pipe);
3537 pipe_stats[pipe] = I915_READ(reg);
3538
3539 /*
3540 * Clear the PIPE*STAT regs before the IIR
3541 */
3542 if (pipe_stats[pipe] & 0x8000ffff)
3543 I915_WRITE(reg, pipe_stats[pipe]);
3544 }
3545 spin_unlock(&dev_priv->irq_lock);
3546
3547 I915_WRITE16(IIR, iir & ~flip_mask);
3548 new_iir = I915_READ16(IIR); /* Flush posted writes */
3549
3550 if (iir & I915_USER_INTERRUPT)
3551 notify_ring(&dev_priv->ring[RCS]);
3552
3553 for_each_pipe(dev_priv, pipe) {
3554 int plane = pipe;
3555 if (HAS_FBC(dev))
3556 plane = !plane;
3557
3558 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3559 i8xx_handle_vblank(dev, plane, pipe, iir))
3560 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3561
3562 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3563 i9xx_pipe_crc_irq_handler(dev, pipe);
3564
3565 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3566 intel_cpu_fifo_underrun_irq_handler(dev_priv,
3567 pipe);
3568 }
3569
3570 iir = new_iir;
3571 }
3572
3573 return IRQ_HANDLED;
3574 }
3575
3576 static void i8xx_irq_uninstall(struct drm_device * dev)
3577 {
3578 struct drm_i915_private *dev_priv = dev->dev_private;
3579 int pipe;
3580
3581 for_each_pipe(dev_priv, pipe) {
3582 /* Clear enable bits; then clear status bits */
3583 I915_WRITE(PIPESTAT(pipe), 0);
3584 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3585 }
3586 I915_WRITE16(IMR, 0xffff);
3587 I915_WRITE16(IER, 0x0);
3588 I915_WRITE16(IIR, I915_READ16(IIR));
3589 }
3590
3591 static void i915_irq_preinstall(struct drm_device * dev)
3592 {
3593 struct drm_i915_private *dev_priv = dev->dev_private;
3594 int pipe;
3595
3596 if (I915_HAS_HOTPLUG(dev)) {
3597 I915_WRITE(PORT_HOTPLUG_EN, 0);
3598 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3599 }
3600
3601 I915_WRITE16(HWSTAM, 0xeffe);
3602 for_each_pipe(dev_priv, pipe)
3603 I915_WRITE(PIPESTAT(pipe), 0);
3604 I915_WRITE(IMR, 0xffffffff);
3605 I915_WRITE(IER, 0x0);
3606 POSTING_READ(IER);
3607 }
3608
3609 static int i915_irq_postinstall(struct drm_device *dev)
3610 {
3611 struct drm_i915_private *dev_priv = dev->dev_private;
3612 u32 enable_mask;
3613
3614 I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3615
3616 /* Unmask the interrupts that we always want on. */
3617 dev_priv->irq_mask =
3618 ~(I915_ASLE_INTERRUPT |
3619 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3620 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3621 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3622 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3623
3624 enable_mask =
3625 I915_ASLE_INTERRUPT |
3626 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3627 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3628 I915_USER_INTERRUPT;
3629
3630 if (I915_HAS_HOTPLUG(dev)) {
3631 I915_WRITE(PORT_HOTPLUG_EN, 0);
3632 POSTING_READ(PORT_HOTPLUG_EN);
3633
3634 /* Enable in IER... */
3635 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
3636 /* and unmask in IMR */
3637 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
3638 }
3639
3640 I915_WRITE(IMR, dev_priv->irq_mask);
3641 I915_WRITE(IER, enable_mask);
3642 POSTING_READ(IER);
3643
3644 i915_enable_asle_pipestat(dev);
3645
3646 /* Interrupt setup is already guaranteed to be single-threaded, this is
3647 * just to make the assert_spin_locked check happy. */
3648 spin_lock_irq(&dev_priv->irq_lock);
3649 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3650 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3651 spin_unlock_irq(&dev_priv->irq_lock);
3652
3653 return 0;
3654 }
3655
3656 /*
3657 * Returns true when a page flip has completed.
3658 */
3659 static bool i915_handle_vblank(struct drm_device *dev,
3660 int plane, int pipe, u32 iir)
3661 {
3662 struct drm_i915_private *dev_priv = dev->dev_private;
3663 u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3664
3665 if (!intel_pipe_handle_vblank(dev, pipe))
3666 return false;
3667
3668 if ((iir & flip_pending) == 0)
3669 goto check_page_flip;
3670
3671 /* We detect FlipDone by looking for the change in PendingFlip from '1'
3672 * to '0' on the following vblank, i.e. IIR has the Pendingflip
3673 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3674 * the flip is completed (no longer pending). Since this doesn't raise
3675 * an interrupt per se, we watch for the change at vblank.
3676 */
3677 if (I915_READ(ISR) & flip_pending)
3678 goto check_page_flip;
3679
3680 intel_prepare_page_flip(dev, plane);
3681 intel_finish_page_flip(dev, pipe);
3682 return true;
3683
3684 check_page_flip:
3685 intel_check_page_flip(dev, pipe);
3686 return false;
3687 }
3688
3689 static irqreturn_t i915_irq_handler(int irq, void *arg)
3690 {
3691 struct drm_device *dev = arg;
3692 struct drm_i915_private *dev_priv = dev->dev_private;
3693 u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
3694 u32 flip_mask =
3695 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3696 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3697 int pipe, ret = IRQ_NONE;
3698
3699 if (!intel_irqs_enabled(dev_priv))
3700 return IRQ_NONE;
3701
3702 iir = I915_READ(IIR);
3703 do {
3704 bool irq_received = (iir & ~flip_mask) != 0;
3705 bool blc_event = false;
3706
3707 /* Can't rely on pipestat interrupt bit in iir as it might
3708 * have been cleared after the pipestat interrupt was received.
3709 * It doesn't set the bit in iir again, but it still produces
3710 * interrupts (for non-MSI).
3711 */
3712 spin_lock(&dev_priv->irq_lock);
3713 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3714 DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3715
3716 for_each_pipe(dev_priv, pipe) {
3717 int reg = PIPESTAT(pipe);
3718 pipe_stats[pipe] = I915_READ(reg);
3719
3720 /* Clear the PIPE*STAT regs before the IIR */
3721 if (pipe_stats[pipe] & 0x8000ffff) {
3722 I915_WRITE(reg, pipe_stats[pipe]);
3723 irq_received = true;
3724 }
3725 }
3726 spin_unlock(&dev_priv->irq_lock);
3727
3728 if (!irq_received)
3729 break;
3730
3731 /* Consume port. Then clear IIR or we'll miss events */
3732 if (I915_HAS_HOTPLUG(dev) &&
3733 iir & I915_DISPLAY_PORT_INTERRUPT)
3734 i9xx_hpd_irq_handler(dev);
3735
3736 I915_WRITE(IIR, iir & ~flip_mask);
3737 new_iir = I915_READ(IIR); /* Flush posted writes */
3738
3739 if (iir & I915_USER_INTERRUPT)
3740 notify_ring(&dev_priv->ring[RCS]);
3741
3742 for_each_pipe(dev_priv, pipe) {
3743 int plane = pipe;
3744 if (HAS_FBC(dev))
3745 plane = !plane;
3746
3747 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3748 i915_handle_vblank(dev, plane, pipe, iir))
3749 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3750
3751 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
3752 blc_event = true;
3753
3754 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3755 i9xx_pipe_crc_irq_handler(dev, pipe);
3756
3757 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3758 intel_cpu_fifo_underrun_irq_handler(dev_priv,
3759 pipe);
3760 }
3761
3762 if (blc_event || (iir & I915_ASLE_INTERRUPT))
3763 intel_opregion_asle_intr(dev);
3764
3765 /* With MSI, interrupts are only generated when iir
3766 * transitions from zero to nonzero. If another bit got
3767 * set while we were handling the existing iir bits, then
3768 * we would never get another interrupt.
3769 *
3770 * This is fine on non-MSI as well, as if we hit this path
3771 * we avoid exiting the interrupt handler only to generate
3772 * another one.
3773 *
3774 * Note that for MSI this could cause a stray interrupt report
3775 * if an interrupt landed in the time between writing IIR and
3776 * the posting read. This should be rare enough to never
3777 * trigger the 99% of 100,000 interrupts test for disabling
3778 * stray interrupts.
3779 */
3780 ret = IRQ_HANDLED;
3781 iir = new_iir;
3782 } while (iir & ~flip_mask);
3783
3784 return ret;
3785 }
3786
3787 static void i915_irq_uninstall(struct drm_device * dev)
3788 {
3789 struct drm_i915_private *dev_priv = dev->dev_private;
3790 int pipe;
3791
3792 if (I915_HAS_HOTPLUG(dev)) {
3793 I915_WRITE(PORT_HOTPLUG_EN, 0);
3794 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3795 }
3796
3797 I915_WRITE16(HWSTAM, 0xffff);
3798 for_each_pipe(dev_priv, pipe) {
3799 /* Clear enable bits; then clear status bits */
3800 I915_WRITE(PIPESTAT(pipe), 0);
3801 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3802 }
3803 I915_WRITE(IMR, 0xffffffff);
3804 I915_WRITE(IER, 0x0);
3805
3806 I915_WRITE(IIR, I915_READ(IIR));
3807 }
3808
3809 static void i965_irq_preinstall(struct drm_device * dev)
3810 {
3811 struct drm_i915_private *dev_priv = dev->dev_private;
3812 int pipe;
3813
3814 I915_WRITE(PORT_HOTPLUG_EN, 0);
3815 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3816
3817 I915_WRITE(HWSTAM, 0xeffe);
3818 for_each_pipe(dev_priv, pipe)
3819 I915_WRITE(PIPESTAT(pipe), 0);
3820 I915_WRITE(IMR, 0xffffffff);
3821 I915_WRITE(IER, 0x0);
3822 POSTING_READ(IER);
3823 }
3824
3825 static int i965_irq_postinstall(struct drm_device *dev)
3826 {
3827 struct drm_i915_private *dev_priv = dev->dev_private;
3828 u32 enable_mask;
3829 u32 error_mask;
3830
3831 /* Unmask the interrupts that we always want on. */
3832 dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
3833 I915_DISPLAY_PORT_INTERRUPT |
3834 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3835 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3836 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3837 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
3838 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
3839
3840 enable_mask = ~dev_priv->irq_mask;
3841 enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3842 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3843 enable_mask |= I915_USER_INTERRUPT;
3844
3845 if (IS_G4X(dev))
3846 enable_mask |= I915_BSD_USER_INTERRUPT;
3847
3848 /* Interrupt setup is already guaranteed to be single-threaded, this is
3849 * just to make the assert_spin_locked check happy. */
3850 spin_lock_irq(&dev_priv->irq_lock);
3851 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3852 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3853 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3854 spin_unlock_irq(&dev_priv->irq_lock);
3855
3856 /*
3857 * Enable some error detection, note the instruction error mask
3858 * bit is reserved, so we leave it masked.
3859 */
3860 if (IS_G4X(dev)) {
3861 error_mask = ~(GM45_ERROR_PAGE_TABLE |
3862 GM45_ERROR_MEM_PRIV |
3863 GM45_ERROR_CP_PRIV |
3864 I915_ERROR_MEMORY_REFRESH);
3865 } else {
3866 error_mask = ~(I915_ERROR_PAGE_TABLE |
3867 I915_ERROR_MEMORY_REFRESH);
3868 }
3869 I915_WRITE(EMR, error_mask);
3870
3871 I915_WRITE(IMR, dev_priv->irq_mask);
3872 I915_WRITE(IER, enable_mask);
3873 POSTING_READ(IER);
3874
3875 I915_WRITE(PORT_HOTPLUG_EN, 0);
3876 POSTING_READ(PORT_HOTPLUG_EN);
3877
3878 i915_enable_asle_pipestat(dev);
3879
3880 return 0;
3881 }
3882
3883 static void i915_hpd_irq_setup(struct drm_device *dev)
3884 {
3885 struct drm_i915_private *dev_priv = dev->dev_private;
3886 struct intel_encoder *intel_encoder;
3887 u32 hotplug_en;
3888
3889 assert_spin_locked(&dev_priv->irq_lock);
3890
3891 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
3892 hotplug_en &= ~HOTPLUG_INT_EN_MASK;
3893 /* Note HDMI and DP share hotplug bits */
3894 /* enable bits are the same for all generations */
3895 for_each_intel_encoder(dev, intel_encoder)
3896 if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED)
3897 hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
3898 /* Programming the CRT detection parameters tends
3899 to generate a spurious hotplug event about three
3900 seconds later. So just do it once.
3901 */
3902 if (IS_G4X(dev))
3903 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
3904 hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
3905 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
3906
3907 /* Ignore TV since it's buggy */
3908 I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
3909 }
3910
3911 static irqreturn_t i965_irq_handler(int irq, void *arg)
3912 {
3913 struct drm_device *dev = arg;
3914 struct drm_i915_private *dev_priv = dev->dev_private;
3915 u32 iir, new_iir;
3916 u32 pipe_stats[I915_MAX_PIPES];
3917 int ret = IRQ_NONE, pipe;
3918 u32 flip_mask =
3919 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3920 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3921
3922 if (!intel_irqs_enabled(dev_priv))
3923 return IRQ_NONE;
3924
3925 iir = I915_READ(IIR);
3926
3927 for (;;) {
3928 bool irq_received = (iir & ~flip_mask) != 0;
3929 bool blc_event = false;
3930
3931 /* Can't rely on pipestat interrupt bit in iir as it might
3932 * have been cleared after the pipestat interrupt was received.
3933 * It doesn't set the bit in iir again, but it still produces
3934 * interrupts (for non-MSI).
3935 */
3936 spin_lock(&dev_priv->irq_lock);
3937 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3938 DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3939
3940 for_each_pipe(dev_priv, pipe) {
3941 int reg = PIPESTAT(pipe);
3942 pipe_stats[pipe] = I915_READ(reg);
3943
3944 /*
3945 * Clear the PIPE*STAT regs before the IIR
3946 */
3947 if (pipe_stats[pipe] & 0x8000ffff) {
3948 I915_WRITE(reg, pipe_stats[pipe]);
3949 irq_received = true;
3950 }
3951 }
3952 spin_unlock(&dev_priv->irq_lock);
3953
3954 if (!irq_received)
3955 break;
3956
3957 ret = IRQ_HANDLED;
3958
3959 /* Consume port. Then clear IIR or we'll miss events */
3960 if (iir & I915_DISPLAY_PORT_INTERRUPT)
3961 i9xx_hpd_irq_handler(dev);
3962
3963 I915_WRITE(IIR, iir & ~flip_mask);
3964 new_iir = I915_READ(IIR); /* Flush posted writes */
3965
3966 if (iir & I915_USER_INTERRUPT)
3967 notify_ring(&dev_priv->ring[RCS]);
3968 if (iir & I915_BSD_USER_INTERRUPT)
3969 notify_ring(&dev_priv->ring[VCS]);
3970
3971 for_each_pipe(dev_priv, pipe) {
3972 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
3973 i915_handle_vblank(dev, pipe, pipe, iir))
3974 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
3975
3976 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
3977 blc_event = true;
3978
3979 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3980 i9xx_pipe_crc_irq_handler(dev, pipe);
3981
3982 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3983 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
3984 }
3985
3986 if (blc_event || (iir & I915_ASLE_INTERRUPT))
3987 intel_opregion_asle_intr(dev);
3988
3989 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
3990 gmbus_irq_handler(dev);
3991
3992 /* With MSI, interrupts are only generated when iir
3993 * transitions from zero to nonzero. If another bit got
3994 * set while we were handling the existing iir bits, then
3995 * we would never get another interrupt.
3996 *
3997 * This is fine on non-MSI as well, as if we hit this path
3998 * we avoid exiting the interrupt handler only to generate
3999 * another one.
4000 *
4001 * Note that for MSI this could cause a stray interrupt report
4002 * if an interrupt landed in the time between writing IIR and
4003 * the posting read. This should be rare enough to never
4004 * trigger the 99% of 100,000 interrupts test for disabling
4005 * stray interrupts.
4006 */
4007 iir = new_iir;
4008 }
4009
4010 return ret;
4011 }
4012
4013 static void i965_irq_uninstall(struct drm_device * dev)
4014 {
4015 struct drm_i915_private *dev_priv = dev->dev_private;
4016 int pipe;
4017
4018 if (!dev_priv)
4019 return;
4020
4021 I915_WRITE(PORT_HOTPLUG_EN, 0);
4022 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4023
4024 I915_WRITE(HWSTAM, 0xffffffff);
4025 for_each_pipe(dev_priv, pipe)
4026 I915_WRITE(PIPESTAT(pipe), 0);
4027 I915_WRITE(IMR, 0xffffffff);
4028 I915_WRITE(IER, 0x0);
4029
4030 for_each_pipe(dev_priv, pipe)
4031 I915_WRITE(PIPESTAT(pipe),
4032 I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
4033 I915_WRITE(IIR, I915_READ(IIR));
4034 }
4035
4036 /**
4037 * intel_irq_init - initializes irq support
4038 * @dev_priv: i915 device instance
4039 *
4040 * This function initializes all the irq support including work items, timers
4041 * and all the vtables. It does not setup the interrupt itself though.
4042 */
4043 void intel_irq_init(struct drm_i915_private *dev_priv)
4044 {
4045 struct drm_device *dev = dev_priv->dev;
4046
4047 intel_hpd_init_work(dev_priv);
4048
4049 INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4050 INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4051
4052 /* Let's track the enabled rps events */
4053 if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
4054 /* WaGsvRC0ResidencyMethod:vlv */
4055 dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED;
4056 else
4057 dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4058
4059 INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
4060 i915_hangcheck_elapsed);
4061
4062 pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
4063
4064 if (IS_GEN2(dev_priv)) {
4065 dev->max_vblank_count = 0;
4066 dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
4067 } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4068 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4069 dev->driver->get_vblank_counter = gm45_get_vblank_counter;
4070 } else {
4071 dev->driver->get_vblank_counter = i915_get_vblank_counter;
4072 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4073 }
4074
4075 /*
4076 * Opt out of the vblank disable timer on everything except gen2.
4077 * Gen2 doesn't have a hardware frame counter and so depends on
4078 * vblank interrupts to produce sane vblank seuquence numbers.
4079 */
4080 if (!IS_GEN2(dev_priv))
4081 dev->vblank_disable_immediate = true;
4082
4083 dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
4084 dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4085
4086 if (IS_CHERRYVIEW(dev_priv)) {
4087 dev->driver->irq_handler = cherryview_irq_handler;
4088 dev->driver->irq_preinstall = cherryview_irq_preinstall;
4089 dev->driver->irq_postinstall = cherryview_irq_postinstall;
4090 dev->driver->irq_uninstall = cherryview_irq_uninstall;
4091 dev->driver->enable_vblank = valleyview_enable_vblank;
4092 dev->driver->disable_vblank = valleyview_disable_vblank;
4093 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4094 } else if (IS_VALLEYVIEW(dev_priv)) {
4095 dev->driver->irq_handler = valleyview_irq_handler;
4096 dev->driver->irq_preinstall = valleyview_irq_preinstall;
4097 dev->driver->irq_postinstall = valleyview_irq_postinstall;
4098 dev->driver->irq_uninstall = valleyview_irq_uninstall;
4099 dev->driver->enable_vblank = valleyview_enable_vblank;
4100 dev->driver->disable_vblank = valleyview_disable_vblank;
4101 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4102 } else if (INTEL_INFO(dev_priv)->gen >= 8) {
4103 dev->driver->irq_handler = gen8_irq_handler;
4104 dev->driver->irq_preinstall = gen8_irq_reset;
4105 dev->driver->irq_postinstall = gen8_irq_postinstall;
4106 dev->driver->irq_uninstall = gen8_irq_uninstall;
4107 dev->driver->enable_vblank = gen8_enable_vblank;
4108 dev->driver->disable_vblank = gen8_disable_vblank;
4109 if (HAS_PCH_SPLIT(dev))
4110 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
4111 else
4112 dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4113 } else if (HAS_PCH_SPLIT(dev)) {
4114 dev->driver->irq_handler = ironlake_irq_handler;
4115 dev->driver->irq_preinstall = ironlake_irq_reset;
4116 dev->driver->irq_postinstall = ironlake_irq_postinstall;
4117 dev->driver->irq_uninstall = ironlake_irq_uninstall;
4118 dev->driver->enable_vblank = ironlake_enable_vblank;
4119 dev->driver->disable_vblank = ironlake_disable_vblank;
4120 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
4121 } else {
4122 if (INTEL_INFO(dev_priv)->gen == 2) {
4123 dev->driver->irq_preinstall = i8xx_irq_preinstall;
4124 dev->driver->irq_postinstall = i8xx_irq_postinstall;
4125 dev->driver->irq_handler = i8xx_irq_handler;
4126 dev->driver->irq_uninstall = i8xx_irq_uninstall;
4127 } else if (INTEL_INFO(dev_priv)->gen == 3) {
4128 dev->driver->irq_preinstall = i915_irq_preinstall;
4129 dev->driver->irq_postinstall = i915_irq_postinstall;
4130 dev->driver->irq_uninstall = i915_irq_uninstall;
4131 dev->driver->irq_handler = i915_irq_handler;
4132 } else {
4133 dev->driver->irq_preinstall = i965_irq_preinstall;
4134 dev->driver->irq_postinstall = i965_irq_postinstall;
4135 dev->driver->irq_uninstall = i965_irq_uninstall;
4136 dev->driver->irq_handler = i965_irq_handler;
4137 }
4138 if (I915_HAS_HOTPLUG(dev_priv))
4139 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4140 dev->driver->enable_vblank = i915_enable_vblank;
4141 dev->driver->disable_vblank = i915_disable_vblank;
4142 }
4143 }
4144
4145 /**
4146 * intel_irq_install - enables the hardware interrupt
4147 * @dev_priv: i915 device instance
4148 *
4149 * This function enables the hardware interrupt handling, but leaves the hotplug
4150 * handling still disabled. It is called after intel_irq_init().
4151 *
4152 * In the driver load and resume code we need working interrupts in a few places
4153 * but don't want to deal with the hassle of concurrent probe and hotplug
4154 * workers. Hence the split into this two-stage approach.
4155 */
4156 int intel_irq_install(struct drm_i915_private *dev_priv)
4157 {
4158 /*
4159 * We enable some interrupt sources in our postinstall hooks, so mark
4160 * interrupts as enabled _before_ actually enabling them to avoid
4161 * special cases in our ordering checks.
4162 */
4163 dev_priv->pm.irqs_enabled = true;
4164
4165 return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
4166 }
4167
4168 /**
4169 * intel_irq_uninstall - finilizes all irq handling
4170 * @dev_priv: i915 device instance
4171 *
4172 * This stops interrupt and hotplug handling and unregisters and frees all
4173 * resources acquired in the init functions.
4174 */
4175 void intel_irq_uninstall(struct drm_i915_private *dev_priv)
4176 {
4177 drm_irq_uninstall(dev_priv->dev);
4178 intel_hpd_cancel_work(dev_priv);
4179 dev_priv->pm.irqs_enabled = false;
4180 }
4181
4182 /**
4183 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
4184 * @dev_priv: i915 device instance
4185 *
4186 * This function is used to disable interrupts at runtime, both in the runtime
4187 * pm and the system suspend/resume code.
4188 */
4189 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4190 {
4191 dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
4192 dev_priv->pm.irqs_enabled = false;
4193 synchronize_irq(dev_priv->dev->irq);
4194 }
4195
4196 /**
4197 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
4198 * @dev_priv: i915 device instance
4199 *
4200 * This function is used to enable interrupts at runtime, both in the runtime
4201 * pm and the system suspend/resume code.
4202 */
4203 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4204 {
4205 dev_priv->pm.irqs_enabled = true;
4206 dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
4207 dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
4208 }
Linux kernel - Deliverables
This page took 0.234943 seconds and 6 git commands to generate.