projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: properly clear IIR at irq_uninstall on Gen5+
[deliverable/linux.git] / drivers / gpu / drm / i915 / i915_irq.c
CommitLineData
0d6aa60b 1/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
1da177e4 2 */
0d6aa60b 3/*
1da177e4
LT		 4 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
5 * All Rights Reserved.
bc54fd1a
DA		 6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
0d6aa60b 27 */
1da177e4 28
a70491cc
JP		 29#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
63eeaf38 31#include <linux/sysrq.h>
5a0e3ad6 32#include <linux/slab.h>
b2c88f5b 33#include <linux/circ_buf.h>
760285e7
DH		 34#include <drm/drmP.h>
35#include <drm/i915_drm.h>
1da177e4 36#include "i915_drv.h"
1c5d22f7 37#include "i915_trace.h"
79e53945 38#include "intel_drv.h"
1da177e4 39
e5868a31
EE		 40static const u32 hpd_ibx[] = {
41 [HPD_CRT] = SDE_CRT_HOTPLUG,
42 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
43 [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
44 [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
45 [HPD_PORT_D] = SDE_PORTD_HOTPLUG
46};
47
48static const u32 hpd_cpt[] = {
49 [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
73c352a2 50 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
e5868a31
EE		 51 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
52 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
53 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
54};
55
56static const u32 hpd_mask_i915[] = {
57 [HPD_CRT] = CRT_HOTPLUG_INT_EN,
58 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
59 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
60 [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
61 [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
62 [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
63};
64
704cfb87 65static const u32 hpd_status_g4x[] = {
e5868a31
EE		 66 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
67 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
68 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
69 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
70 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
71 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
72};
73
e5868a31
EE		 74static const u32 hpd_status_i915[] = { /* i915 and valleyview are the same */
75 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
76 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
77 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
78 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
79 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
80 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
81};
82
5c502442 83/* IIR can theoretically queue up two events. Be paranoid. */
f86f3fb0 84#define GEN8_IRQ_RESET_NDX(type, which) do { \
5c502442
PZ		 85 I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
86 POSTING_READ(GEN8_##type##_IMR(which)); \
87 I915_WRITE(GEN8_##type##_IER(which), 0); \
88 I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
89 POSTING_READ(GEN8_##type##_IIR(which)); \
90 I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
91 POSTING_READ(GEN8_##type##_IIR(which)); \
92} while (0)
93
f86f3fb0 94#define GEN5_IRQ_RESET(type) do { \
a9d356a6 95 I915_WRITE(type##IMR, 0xffffffff); \
5c502442 96 POSTING_READ(type##IMR); \
a9d356a6 97 I915_WRITE(type##IER, 0); \
5c502442
PZ		 98 I915_WRITE(type##IIR, 0xffffffff); \
99 POSTING_READ(type##IIR); \
100 I915_WRITE(type##IIR, 0xffffffff); \
101 POSTING_READ(type##IIR); \
a9d356a6
PZ		 102} while (0)
103
036a4a7d 104/* For display hotplug interrupt */
995b6762 105static void
2d1013dd 106ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
036a4a7d 107{
4bc9d430
DV		 108 assert_spin_locked(&dev_priv->irq_lock);
109
5d584b2e 110 if (dev_priv->pm.irqs_disabled) {
c67a470b 111 WARN(1, "IRQs disabled\n");
5d584b2e 112 dev_priv->pm.regsave.deimr &= ~mask;
c67a470b
PZ		 113 return;
114 }
115
1ec14ad3
CW		 116 if ((dev_priv->irq_mask & mask) != 0) {
117 dev_priv->irq_mask &= ~mask;
118 I915_WRITE(DEIMR, dev_priv->irq_mask);
3143a2bf 119 POSTING_READ(DEIMR);
036a4a7d
ZW		 120 }
121}
122
0ff9800a 123static void
2d1013dd 124ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
036a4a7d 125{
4bc9d430
DV		 126 assert_spin_locked(&dev_priv->irq_lock);
127
5d584b2e 128 if (dev_priv->pm.irqs_disabled) {
c67a470b 129 WARN(1, "IRQs disabled\n");
5d584b2e 130 dev_priv->pm.regsave.deimr |= mask;
c67a470b
PZ		 131 return;
132 }
133
1ec14ad3
CW		 134 if ((dev_priv->irq_mask & mask) != mask) {
135 dev_priv->irq_mask |= mask;
136 I915_WRITE(DEIMR, dev_priv->irq_mask);
3143a2bf 137 POSTING_READ(DEIMR);
036a4a7d
ZW		 138 }
139}
140
43eaea13
PZ		 141/**
142 * ilk_update_gt_irq - update GTIMR
143 * @dev_priv: driver private
144 * @interrupt_mask: mask of interrupt bits to update
145 * @enabled_irq_mask: mask of interrupt bits to enable
146 */
147static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
148 uint32_t interrupt_mask,
149 uint32_t enabled_irq_mask)
150{
151 assert_spin_locked(&dev_priv->irq_lock);
152
5d584b2e 153 if (dev_priv->pm.irqs_disabled) {
c67a470b 154 WARN(1, "IRQs disabled\n");
5d584b2e
PZ		 155 dev_priv->pm.regsave.gtimr &= ~interrupt_mask;
156 dev_priv->pm.regsave.gtimr |= (~enabled_irq_mask &
c67a470b
PZ		 157 interrupt_mask);
158 return;
159 }
160
43eaea13
PZ		 161 dev_priv->gt_irq_mask &= ~interrupt_mask;
162 dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
163 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
164 POSTING_READ(GTIMR);
165}
166
167void ilk_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
168{
169 ilk_update_gt_irq(dev_priv, mask, mask);
170}
171
172void ilk_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
173{
174 ilk_update_gt_irq(dev_priv, mask, 0);
175}
176
edbfdb45
PZ		 177/**
178 * snb_update_pm_irq - update GEN6_PMIMR
179 * @dev_priv: driver private
180 * @interrupt_mask: mask of interrupt bits to update
181 * @enabled_irq_mask: mask of interrupt bits to enable
182 */
183static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
184 uint32_t interrupt_mask,
185 uint32_t enabled_irq_mask)
186{
605cd25b 187 uint32_t new_val;
edbfdb45
PZ		 188
189 assert_spin_locked(&dev_priv->irq_lock);
190
5d584b2e 191 if (dev_priv->pm.irqs_disabled) {
c67a470b 192 WARN(1, "IRQs disabled\n");
5d584b2e
PZ		 193 dev_priv->pm.regsave.gen6_pmimr &= ~interrupt_mask;
194 dev_priv->pm.regsave.gen6_pmimr |= (~enabled_irq_mask &
c67a470b
PZ		 195 interrupt_mask);
196 return;
197 }
198
605cd25b 199 new_val = dev_priv->pm_irq_mask;
f52ecbcf
PZ		 200 new_val &= ~interrupt_mask;
201 new_val |= (~enabled_irq_mask & interrupt_mask);
202
605cd25b
PZ		 203 if (new_val != dev_priv->pm_irq_mask) {
204 dev_priv->pm_irq_mask = new_val;
205 I915_WRITE(GEN6_PMIMR, dev_priv->pm_irq_mask);
f52ecbcf
PZ		 206 POSTING_READ(GEN6_PMIMR);
207 }
edbfdb45
PZ		 208}
209
210void snb_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
211{
212 snb_update_pm_irq(dev_priv, mask, mask);
213}
214
215void snb_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
216{
217 snb_update_pm_irq(dev_priv, mask, 0);
218}
219
8664281b
PZ		 220static bool ivb_can_enable_err_int(struct drm_device *dev)
221{
222 struct drm_i915_private *dev_priv = dev->dev_private;
223 struct intel_crtc *crtc;
224 enum pipe pipe;
225
4bc9d430
DV		 226 assert_spin_locked(&dev_priv->irq_lock);
227
8664281b
PZ		 228 for_each_pipe(pipe) {
229 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
230
231 if (crtc->cpu_fifo_underrun_disabled)
232 return false;
233 }
234
235 return true;
236}
237
238static bool cpt_can_enable_serr_int(struct drm_device *dev)
239{
240 struct drm_i915_private *dev_priv = dev->dev_private;
241 enum pipe pipe;
242 struct intel_crtc *crtc;
243
fee884ed
DV		 244 assert_spin_locked(&dev_priv->irq_lock);
245
8664281b
PZ		 246 for_each_pipe(pipe) {
247 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
248
249 if (crtc->pch_fifo_underrun_disabled)
250 return false;
251 }
252
253 return true;
254}
255
2d9d2b0b
VS		 256static void i9xx_clear_fifo_underrun(struct drm_device *dev, enum pipe pipe)
257{
258 struct drm_i915_private *dev_priv = dev->dev_private;
259 u32 reg = PIPESTAT(pipe);
260 u32 pipestat = I915_READ(reg) & 0x7fff0000;
261
262 assert_spin_locked(&dev_priv->irq_lock);
263
264 I915_WRITE(reg, pipestat | PIPE_FIFO_UNDERRUN_STATUS);
265 POSTING_READ(reg);
266}
267
8664281b
PZ		 268static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev,
269 enum pipe pipe, bool enable)
270{
271 struct drm_i915_private *dev_priv = dev->dev_private;
272 uint32_t bit = (pipe == PIPE_A) ? DE_PIPEA_FIFO_UNDERRUN :
273 DE_PIPEB_FIFO_UNDERRUN;
274
275 if (enable)
276 ironlake_enable_display_irq(dev_priv, bit);
277 else
278 ironlake_disable_display_irq(dev_priv, bit);
279}
280
281static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev,
7336df65 282 enum pipe pipe, bool enable)
8664281b
PZ		 283{
284 struct drm_i915_private *dev_priv = dev->dev_private;
8664281b 285 if (enable) {
7336df65
DV		 286 I915_WRITE(GEN7_ERR_INT, ERR_INT_FIFO_UNDERRUN(pipe));
287
8664281b
PZ		 288 if (!ivb_can_enable_err_int(dev))
289 return;
290
8664281b
PZ		 291 ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
292 } else {
7336df65
DV		 293 bool was_enabled = !(I915_READ(DEIMR) & DE_ERR_INT_IVB);
294
295 /* Change the state _after_ we've read out the current one. */
8664281b 296 ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
7336df65
DV		 297
298 if (!was_enabled &&
299 (I915_READ(GEN7_ERR_INT) & ERR_INT_FIFO_UNDERRUN(pipe))) {
300 DRM_DEBUG_KMS("uncleared fifo underrun on pipe %c\n",
301 pipe_name(pipe));
302 }
8664281b
PZ		 303 }
304}
305
38d83c96
DV		 306static void broadwell_set_fifo_underrun_reporting(struct drm_device *dev,
307 enum pipe pipe, bool enable)
308{
309 struct drm_i915_private *dev_priv = dev->dev_private;
310
311 assert_spin_locked(&dev_priv->irq_lock);
312
313 if (enable)
314 dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_FIFO_UNDERRUN;
315 else
316 dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_FIFO_UNDERRUN;
317 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
318 POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
319}
320
fee884ed
DV		 321/**
322 * ibx_display_interrupt_update - update SDEIMR
323 * @dev_priv: driver private
324 * @interrupt_mask: mask of interrupt bits to update
325 * @enabled_irq_mask: mask of interrupt bits to enable
326 */
327static void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
328 uint32_t interrupt_mask,
329 uint32_t enabled_irq_mask)
330{
331 uint32_t sdeimr = I915_READ(SDEIMR);
332 sdeimr &= ~interrupt_mask;
333 sdeimr |= (~enabled_irq_mask & interrupt_mask);
334
335 assert_spin_locked(&dev_priv->irq_lock);
336
5d584b2e 337 if (dev_priv->pm.irqs_disabled &&
c67a470b
PZ		 338 (interrupt_mask & SDE_HOTPLUG_MASK_CPT)) {
339 WARN(1, "IRQs disabled\n");
5d584b2e
PZ		 340 dev_priv->pm.regsave.sdeimr &= ~interrupt_mask;
341 dev_priv->pm.regsave.sdeimr |= (~enabled_irq_mask &
c67a470b
PZ		 342 interrupt_mask);
343 return;
344 }
345
fee884ed
DV		 346 I915_WRITE(SDEIMR, sdeimr);
347 POSTING_READ(SDEIMR);
348}
349#define ibx_enable_display_interrupt(dev_priv, bits) \
350 ibx_display_interrupt_update((dev_priv), (bits), (bits))
351#define ibx_disable_display_interrupt(dev_priv, bits) \
352 ibx_display_interrupt_update((dev_priv), (bits), 0)
353
de28075d
DV		 354static void ibx_set_fifo_underrun_reporting(struct drm_device *dev,
355 enum transcoder pch_transcoder,
8664281b
PZ		 356 bool enable)
357{
8664281b 358 struct drm_i915_private *dev_priv = dev->dev_private;
de28075d
DV		 359 uint32_t bit = (pch_transcoder == TRANSCODER_A) ?
360 SDE_TRANSA_FIFO_UNDER : SDE_TRANSB_FIFO_UNDER;
8664281b
PZ		 361
362 if (enable)
fee884ed 363 ibx_enable_display_interrupt(dev_priv, bit);
8664281b 364 else
fee884ed 365 ibx_disable_display_interrupt(dev_priv, bit);
8664281b
PZ		 366}
367
368static void cpt_set_fifo_underrun_reporting(struct drm_device *dev,
369 enum transcoder pch_transcoder,
370 bool enable)
371{
372 struct drm_i915_private *dev_priv = dev->dev_private;
373
374 if (enable) {
1dd246fb
DV		 375 I915_WRITE(SERR_INT,
376 SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder));
377
8664281b
PZ		 378 if (!cpt_can_enable_serr_int(dev))
379 return;
380
fee884ed 381 ibx_enable_display_interrupt(dev_priv, SDE_ERROR_CPT);
8664281b 382 } else {
1dd246fb
DV		 383 uint32_t tmp = I915_READ(SERR_INT);
384 bool was_enabled = !(I915_READ(SDEIMR) & SDE_ERROR_CPT);
385
386 /* Change the state _after_ we've read out the current one. */
fee884ed 387 ibx_disable_display_interrupt(dev_priv, SDE_ERROR_CPT);
1dd246fb
DV		 388
389 if (!was_enabled &&
390 (tmp & SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder))) {
391 DRM_DEBUG_KMS("uncleared pch fifo underrun on pch transcoder %c\n",
392 transcoder_name(pch_transcoder));
393 }
8664281b 394 }
8664281b
PZ		 395}
396
397/**
398 * intel_set_cpu_fifo_underrun_reporting - enable/disable FIFO underrun messages
399 * @dev: drm device
400 * @pipe: pipe
401 * @enable: true if we want to report FIFO underrun errors, false otherwise
402 *
403 * This function makes us disable or enable CPU fifo underruns for a specific
404 * pipe. Notice that on some Gens (e.g. IVB, HSW), disabling FIFO underrun
405 * reporting for one pipe may also disable all the other CPU error interruts for
406 * the other pipes, due to the fact that there's just one interrupt mask/enable
407 * bit for all the pipes.
408 *
409 * Returns the previous state of underrun reporting.
410 */
f88d42f1
ID		 411bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
412 enum pipe pipe, bool enable)
8664281b
PZ		 413{
414 struct drm_i915_private *dev_priv = dev->dev_private;
415 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
416 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8664281b
PZ		 417 bool ret;
418
77961eb9
ID		 419 assert_spin_locked(&dev_priv->irq_lock);
420
8664281b
PZ		 421 ret = !intel_crtc->cpu_fifo_underrun_disabled;
422
423 if (enable == ret)
424 goto done;
425
426 intel_crtc->cpu_fifo_underrun_disabled = !enable;
427
2d9d2b0b
VS		 428 if (enable && (INTEL_INFO(dev)->gen < 5 || IS_VALLEYVIEW(dev)))
429 i9xx_clear_fifo_underrun(dev, pipe);
430 else if (IS_GEN5(dev) || IS_GEN6(dev))
8664281b
PZ		 431 ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
432 else if (IS_GEN7(dev))
7336df65 433 ivybridge_set_fifo_underrun_reporting(dev, pipe, enable);
38d83c96
DV		 434 else if (IS_GEN8(dev))
435 broadwell_set_fifo_underrun_reporting(dev, pipe, enable);
8664281b
PZ		 436
437done:
f88d42f1
ID		 438 return ret;
439}
440
441bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
442 enum pipe pipe, bool enable)
443{
444 struct drm_i915_private *dev_priv = dev->dev_private;
445 unsigned long flags;
446 bool ret;
447
448 spin_lock_irqsave(&dev_priv->irq_lock, flags);
449 ret = __intel_set_cpu_fifo_underrun_reporting(dev, pipe, enable);
8664281b 450 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
f88d42f1 451
8664281b
PZ		 452 return ret;
453}
454
91d181dd
ID		 455static bool __cpu_fifo_underrun_reporting_enabled(struct drm_device *dev,
456 enum pipe pipe)
457{
458 struct drm_i915_private *dev_priv = dev->dev_private;
459 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
460 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
461
462 return !intel_crtc->cpu_fifo_underrun_disabled;
463}
464
8664281b
PZ		 465/**
466 * intel_set_pch_fifo_underrun_reporting - enable/disable FIFO underrun messages
467 * @dev: drm device
468 * @pch_transcoder: the PCH transcoder (same as pipe on IVB and older)
469 * @enable: true if we want to report FIFO underrun errors, false otherwise
470 *
471 * This function makes us disable or enable PCH fifo underruns for a specific
472 * PCH transcoder. Notice that on some PCHs (e.g. CPT/PPT), disabling FIFO
473 * underrun reporting for one transcoder may also disable all the other PCH
474 * error interruts for the other transcoders, due to the fact that there's just
475 * one interrupt mask/enable bit for all the transcoders.
476 *
477 * Returns the previous state of underrun reporting.
478 */
479bool intel_set_pch_fifo_underrun_reporting(struct drm_device *dev,
480 enum transcoder pch_transcoder,
481 bool enable)
482{
483 struct drm_i915_private *dev_priv = dev->dev_private;
de28075d
DV		 484 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder];
485 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8664281b
PZ		 486 unsigned long flags;
487 bool ret;
488
de28075d
DV		 489 /*
490 * NOTE: Pre-LPT has a fixed cpu pipe -> pch transcoder mapping, but LPT
491 * has only one pch transcoder A that all pipes can use. To avoid racy
492 * pch transcoder -> pipe lookups from interrupt code simply store the
493 * underrun statistics in crtc A. Since we never expose this anywhere
494 * nor use it outside of the fifo underrun code here using the "wrong"
495 * crtc on LPT won't cause issues.
496 */
8664281b
PZ		 497
498 spin_lock_irqsave(&dev_priv->irq_lock, flags);
499
500 ret = !intel_crtc->pch_fifo_underrun_disabled;
501
502 if (enable == ret)
503 goto done;
504
505 intel_crtc->pch_fifo_underrun_disabled = !enable;
506
507 if (HAS_PCH_IBX(dev))
de28075d 508 ibx_set_fifo_underrun_reporting(dev, pch_transcoder, enable);
8664281b
PZ		 509 else
510 cpt_set_fifo_underrun_reporting(dev, pch_transcoder, enable);
511
512done:
513 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
514 return ret;
515}
516
517
b5ea642a 518static void
755e9019
ID		 519__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
520 u32 enable_mask, u32 status_mask)
7c463586 521{
46c06a30 522 u32 reg = PIPESTAT(pipe);
755e9019 523 u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
7c463586 524
b79480ba
DV		 525 assert_spin_locked(&dev_priv->irq_lock);
526
755e9019
ID		 527 if (WARN_ON_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
528 status_mask & ~PIPESTAT_INT_STATUS_MASK))
529 return;
530
531 if ((pipestat & enable_mask) == enable_mask)
46c06a30
VS		 532 return;
533
91d181dd
ID		 534 dev_priv->pipestat_irq_mask[pipe] |= status_mask;
535
46c06a30 536 /* Enable the interrupt, clear any pending status */
755e9019 537 pipestat |= enable_mask | status_mask;
46c06a30
VS		 538 I915_WRITE(reg, pipestat);
539 POSTING_READ(reg);
7c463586
KP		 540}
541
b5ea642a 542static void
755e9019
ID		 543__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
544 u32 enable_mask, u32 status_mask)
7c463586 545{
46c06a30 546 u32 reg = PIPESTAT(pipe);
755e9019 547 u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
7c463586 548
b79480ba
DV		 549 assert_spin_locked(&dev_priv->irq_lock);
550
755e9019
ID		 551 if (WARN_ON_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
552 status_mask & ~PIPESTAT_INT_STATUS_MASK))
46c06a30
VS		 553 return;
554
755e9019
ID		 555 if ((pipestat & enable_mask) == 0)
556 return;
557
91d181dd
ID		 558 dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
559
755e9019 560 pipestat &= ~enable_mask;
46c06a30
VS		 561 I915_WRITE(reg, pipestat);
562 POSTING_READ(reg);
7c463586
KP		 563}
564
10c59c51
ID		 565static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
566{
567 u32 enable_mask = status_mask << 16;
568
569 /*
570 * On pipe A we don't support the PSR interrupt yet, on pipe B the
571 * same bit MBZ.
572 */
573 if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
574 return 0;
575
576 enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
577 SPRITE0_FLIP_DONE_INT_EN_VLV |
578 SPRITE1_FLIP_DONE_INT_EN_VLV);
579 if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
580 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
581 if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
582 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
583
584 return enable_mask;
585}
586
755e9019
ID		 587void
588i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
589 u32 status_mask)
590{
591 u32 enable_mask;
592
10c59c51
ID		 593 if (IS_VALLEYVIEW(dev_priv->dev))
594 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
595 status_mask);
596 else
597 enable_mask = status_mask << 16;
755e9019
ID		 598 __i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
599}
600
601void
602i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
603 u32 status_mask)
604{
605 u32 enable_mask;
606
10c59c51
ID		 607 if (IS_VALLEYVIEW(dev_priv->dev))
608 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
609 status_mask);
610 else
611 enable_mask = status_mask << 16;
755e9019
ID		 612 __i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
613}
614
01c66889 615/**
f49e38dd 616 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
01c66889 617 */
f49e38dd 618static void i915_enable_asle_pipestat(struct drm_device *dev)
01c66889 619{
2d1013dd 620 struct drm_i915_private *dev_priv = dev->dev_private;
1ec14ad3
CW		 621 unsigned long irqflags;
622
f49e38dd
JN		 623 if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
624 return;
625
1ec14ad3 626 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
01c66889 627
755e9019 628 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
f898780b 629 if (INTEL_INFO(dev)->gen >= 4)
3b6c42e8 630 i915_enable_pipestat(dev_priv, PIPE_A,
755e9019 631 PIPE_LEGACY_BLC_EVENT_STATUS);
1ec14ad3
CW		 632
633 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
01c66889
ZY		 634}
635
0a3e67a4
JB		 636/**
637 * i915_pipe_enabled - check if a pipe is enabled
638 * @dev: DRM device
639 * @pipe: pipe to check
640 *
641 * Reading certain registers when the pipe is disabled can hang the chip.
642 * Use this routine to make sure the PLL is running and the pipe is active
643 * before reading such registers if unsure.
644 */
645static int
646i915_pipe_enabled(struct drm_device *dev, int pipe)
647{
2d1013dd 648 struct drm_i915_private *dev_priv = dev->dev_private;
702e7a56 649
a01025af
DV		 650 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
651 /* Locking is horribly broken here, but whatever. */
652 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
653 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
71f8ba6b 654
a01025af
DV		 655 return intel_crtc->active;
656 } else {
657 return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE;
658 }
0a3e67a4
JB		 659}
660
4cdb83ec
VS		 661static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe)
662{
663 /* Gen2 doesn't have a hardware frame counter */
664 return 0;
665}
666
42f52ef8
KP		 667/* Called from drm generic code, passed a 'crtc', which
668 * we use as a pipe index
669 */
f71d4af4 670static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
0a3e67a4 671{
2d1013dd 672 struct drm_i915_private *dev_priv = dev->dev_private;
0a3e67a4
JB		 673 unsigned long high_frame;
674 unsigned long low_frame;
391f75e2 675 u32 high1, high2, low, pixel, vbl_start;
0a3e67a4
JB		 676
677 if (!i915_pipe_enabled(dev, pipe)) {
44d98a61 678 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
9db4a9c7 679 "pipe %c\n", pipe_name(pipe));
0a3e67a4
JB		 680 return 0;
681 }
682
391f75e2
VS		 683 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
684 struct intel_crtc *intel_crtc =
685 to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
686 const struct drm_display_mode *mode =
687 &intel_crtc->config.adjusted_mode;
688
689 vbl_start = mode->crtc_vblank_start * mode->crtc_htotal;
690 } else {
a2d213dd 691 enum transcoder cpu_transcoder = (enum transcoder) pipe;
391f75e2
VS		 692 u32 htotal;
693
694 htotal = ((I915_READ(HTOTAL(cpu_transcoder)) >> 16) & 0x1fff) + 1;
695 vbl_start = (I915_READ(VBLANK(cpu_transcoder)) & 0x1fff) + 1;
696
697 vbl_start *= htotal;
698 }
699
9db4a9c7
JB		 700 high_frame = PIPEFRAME(pipe);
701 low_frame = PIPEFRAMEPIXEL(pipe);
5eddb70b 702
0a3e67a4
JB		 703 /*
704 * High & low register fields aren't synchronized, so make sure
705 * we get a low value that's stable across two reads of the high
706 * register.
707 */
708 do {
5eddb70b 709 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
391f75e2 710 low = I915_READ(low_frame);
5eddb70b 711 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
0a3e67a4
JB		 712 } while (high1 != high2);
713
5eddb70b 714 high1 >>= PIPE_FRAME_HIGH_SHIFT;
391f75e2 715 pixel = low & PIPE_PIXEL_MASK;
5eddb70b 716 low >>= PIPE_FRAME_LOW_SHIFT;
391f75e2
VS		 717
718 /*
719 * The frame counter increments at beginning of active.
720 * Cook up a vblank counter by also checking the pixel
721 * counter against vblank start.
722 */
edc08d0a 723 return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
0a3e67a4
JB		 724}
725
f71d4af4 726static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
9880b7a5 727{
2d1013dd 728 struct drm_i915_private *dev_priv = dev->dev_private;
9db4a9c7 729 int reg = PIPE_FRMCOUNT_GM45(pipe);
9880b7a5
JB		 730
731 if (!i915_pipe_enabled(dev, pipe)) {
44d98a61 732 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
9db4a9c7 733 "pipe %c\n", pipe_name(pipe));
9880b7a5
JB		 734 return 0;
735 }
736
737 return I915_READ(reg);
738}
739
ad3543ed
MK		 740/* raw reads, only for fast reads of display block, no need for forcewake etc. */
741#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
ad3543ed 742
095163ba 743static bool ilk_pipe_in_vblank_locked(struct drm_device *dev, enum pipe pipe)
54ddcbd2
VS		 744{
745 struct drm_i915_private *dev_priv = dev->dev_private;
746 uint32_t status;
24302624
VS		 747 int reg;
748
749 if (INTEL_INFO(dev)->gen >= 8) {
750 status = GEN8_PIPE_VBLANK;
751 reg = GEN8_DE_PIPE_ISR(pipe);
752 } else if (INTEL_INFO(dev)->gen >= 7) {
753 status = DE_PIPE_VBLANK_IVB(pipe);
754 reg = DEISR;
54ddcbd2 755 } else {
24302624
VS		 756 status = DE_PIPE_VBLANK(pipe);
757 reg = DEISR;
54ddcbd2 758 }
ad3543ed 759
24302624 760 return __raw_i915_read32(dev_priv, reg) & status;
54ddcbd2
VS		 761}
762
f71d4af4 763static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
abca9e45
VS		 764 unsigned int flags, int *vpos, int *hpos,
765 ktime_t *stime, ktime_t *etime)
0af7e4df 766{
c2baf4b7
VS		 767 struct drm_i915_private *dev_priv = dev->dev_private;
768 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
769 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
770 const struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode;
3aa18df8 771 int position;
0af7e4df
MK		 772 int vbl_start, vbl_end, htotal, vtotal;
773 bool in_vbl = true;
774 int ret = 0;
ad3543ed 775 unsigned long irqflags;
0af7e4df 776
c2baf4b7 777 if (!intel_crtc->active) {
0af7e4df 778 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
9db4a9c7 779 "pipe %c\n", pipe_name(pipe));
0af7e4df
MK		 780 return 0;
781 }
782
c2baf4b7
VS		 783 htotal = mode->crtc_htotal;
784 vtotal = mode->crtc_vtotal;
785 vbl_start = mode->crtc_vblank_start;
786 vbl_end = mode->crtc_vblank_end;
0af7e4df 787
d31faf65
VS		 788 if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
789 vbl_start = DIV_ROUND_UP(vbl_start, 2);
790 vbl_end /= 2;
791 vtotal /= 2;
792 }
793
c2baf4b7
VS		 794 ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
795
ad3543ed
MK		 796 /*
797 * Lock uncore.lock, as we will do multiple timing critical raw
798 * register reads, potentially with preemption disabled, so the
799 * following code must not block on uncore.lock.
800 */
801 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
802
803 /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
804
805 /* Get optional system timestamp before query. */
806 if (stime)
807 *stime = ktime_get();
808
7c06b08a 809 if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
0af7e4df
MK		 810 /* No obvious pixelcount register. Only query vertical
811 * scanout position from Display scan line register.
812 */
7c06b08a 813 if (IS_GEN2(dev))
ad3543ed 814 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
7c06b08a 815 else
ad3543ed 816 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
54ddcbd2 817
fcb81823
VS		 818 if (HAS_DDI(dev)) {
819 /*
820 * On HSW HDMI outputs there seems to be a 2 line
821 * difference, whereas eDP has the normal 1 line
822 * difference that earlier platforms have. External
823 * DP is unknown. For now just check for the 2 line
824 * difference case on all output types on HSW+.
825 *
826 * This might misinterpret the scanline counter being
827 * one line too far along on eDP, but that's less
828 * dangerous than the alternative since that would lead
829 * the vblank timestamp code astray when it sees a
830 * scanline count before vblank_start during a vblank
831 * interrupt.
832 */
833 in_vbl = ilk_pipe_in_vblank_locked(dev, pipe);
834 if ((in_vbl && (position == vbl_start - 2 ||
835 position == vbl_start - 1)) ||
836 (!in_vbl && (position == vbl_end - 2 ||
837 position == vbl_end - 1)))
838 position = (position + 2) % vtotal;
839 } else if (HAS_PCH_SPLIT(dev)) {
095163ba
VS		 840 /*
841 * The scanline counter increments at the leading edge
842 * of hsync, ie. it completely misses the active portion
843 * of the line. Fix up the counter at both edges of vblank
844 * to get a more accurate picture whether we're in vblank
845 * or not.
846 */
847 in_vbl = ilk_pipe_in_vblank_locked(dev, pipe);
848 if ((in_vbl && position == vbl_start - 1) ||
849 (!in_vbl && position == vbl_end - 1))
850 position = (position + 1) % vtotal;
851 } else {
852 /*
853 * ISR vblank status bits don't work the way we'd want
854 * them to work on non-PCH platforms (for
855 * ilk_pipe_in_vblank_locked()), and there doesn't
856 * appear any other way to determine if we're currently
857 * in vblank.
858 *
859 * Instead let's assume that we're already in vblank if
860 * we got called from the vblank interrupt and the
861 * scanline counter value indicates that we're on the
862 * line just prior to vblank start. This should result
863 * in the correct answer, unless the vblank interrupt
864 * delivery really got delayed for almost exactly one
865 * full frame/field.
866 */
867 if (flags & DRM_CALLED_FROM_VBLIRQ &&
868 position == vbl_start - 1) {
869 position = (position + 1) % vtotal;
870
871 /* Signal this correction as "applied". */
872 ret |= 0x8;
873 }
874 }
0af7e4df
MK		 875 } else {
876 /* Have access to pixelcount since start of frame.
877 * We can split this into vertical and horizontal
878 * scanout position.
879 */
ad3543ed 880 position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
0af7e4df 881
3aa18df8
VS		 882 /* convert to pixel counts */
883 vbl_start *= htotal;
884 vbl_end *= htotal;
885 vtotal *= htotal;
0af7e4df
MK		 886 }
887
ad3543ed
MK		 888 /* Get optional system timestamp after query. */
889 if (etime)
890 *etime = ktime_get();
891
892 /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
893
894 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
895
3aa18df8
VS		 896 in_vbl = position >= vbl_start && position < vbl_end;
897
898 /*
899 * While in vblank, position will be negative
900 * counting up towards 0 at vbl_end. And outside
901 * vblank, position will be positive counting
902 * up since vbl_end.
903 */
904 if (position >= vbl_start)
905 position -= vbl_end;
906 else
907 position += vtotal - vbl_end;
0af7e4df 908
7c06b08a 909 if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
3aa18df8
VS		 910 *vpos = position;
911 *hpos = 0;
912 } else {
913 *vpos = position / htotal;
914 *hpos = position - (*vpos * htotal);
915 }
0af7e4df 916
0af7e4df
MK		 917 /* In vblank? */
918 if (in_vbl)
919 ret |= DRM_SCANOUTPOS_INVBL;
920
921 return ret;
922}
923
f71d4af4 924static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
0af7e4df
MK		 925 int *max_error,
926 struct timeval *vblank_time,
927 unsigned flags)
928{
4041b853 929 struct drm_crtc *crtc;
0af7e4df 930
7eb552ae 931 if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
4041b853 932 DRM_ERROR("Invalid crtc %d\n", pipe);
0af7e4df
MK		 933 return -EINVAL;
934 }
935
936 /* Get drm_crtc to timestamp: */
4041b853
CW		 937 crtc = intel_get_crtc_for_pipe(dev, pipe);
938 if (crtc == NULL) {
939 DRM_ERROR("Invalid crtc %d\n", pipe);
940 return -EINVAL;
941 }
942
943 if (!crtc->enabled) {
944 DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
945 return -EBUSY;
946 }
0af7e4df
MK		 947
948 /* Helper routine in DRM core does all the work: */
4041b853
CW		 949 return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
950 vblank_time, flags,
7da903ef
VS		 951 crtc,
952 &to_intel_crtc(crtc)->config.adjusted_mode);
0af7e4df
MK		 953}
954
67c347ff
JN		 955static bool intel_hpd_irq_event(struct drm_device *dev,
956 struct drm_connector *connector)
321a1b30
EE		 957{
958 enum drm_connector_status old_status;
959
960 WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
961 old_status = connector->status;
962
963 connector->status = connector->funcs->detect(connector, false);
67c347ff
JN		 964 if (old_status == connector->status)
965 return false;
966
967 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s\n",
321a1b30
EE		 968 connector->base.id,
969 drm_get_connector_name(connector),
67c347ff
JN		 970 drm_get_connector_status_name(old_status),
971 drm_get_connector_status_name(connector->status));
972
973 return true;
321a1b30
EE		 974}
975
5ca58282
JB		 976/*
977 * Handle hotplug events outside the interrupt handler proper.
978 */
ac4c16c5
EE		 979#define I915_REENABLE_HOTPLUG_DELAY (2*60*1000)
980
5ca58282
JB		 981static void i915_hotplug_work_func(struct work_struct *work)
982{
2d1013dd
JN		 983 struct drm_i915_private *dev_priv =
984 container_of(work, struct drm_i915_private, hotplug_work);
5ca58282 985 struct drm_device *dev = dev_priv->dev;
c31c4ba3 986 struct drm_mode_config *mode_config = &dev->mode_config;
cd569aed
EE		 987 struct intel_connector *intel_connector;
988 struct intel_encoder *intel_encoder;
989 struct drm_connector *connector;
990 unsigned long irqflags;
991 bool hpd_disabled = false;
321a1b30 992 bool changed = false;
142e2398 993 u32 hpd_event_bits;
4ef69c7a 994
52d7eced
DV		 995 /* HPD irq before everything is fully set up. */
996 if (!dev_priv->enable_hotplug_processing)
997 return;
998
a65e34c7 999 mutex_lock(&mode_config->mutex);
e67189ab
JB		 1000 DRM_DEBUG_KMS("running encoder hotplug functions\n");
1001
cd569aed 1002 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
142e2398
EE		 1003
1004 hpd_event_bits = dev_priv->hpd_event_bits;
1005 dev_priv->hpd_event_bits = 0;
cd569aed
EE		 1006 list_for_each_entry(connector, &mode_config->connector_list, head) {
1007 intel_connector = to_intel_connector(connector);
1008 intel_encoder = intel_connector->encoder;
1009 if (intel_encoder->hpd_pin > HPD_NONE &&
1010 dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_MARK_DISABLED &&
1011 connector->polled == DRM_CONNECTOR_POLL_HPD) {
1012 DRM_INFO("HPD interrupt storm detected on connector %s: "
1013 "switching from hotplug detection to polling\n",
1014 drm_get_connector_name(connector));
1015 dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark = HPD_DISABLED;
1016 connector->polled = DRM_CONNECTOR_POLL_CONNECT
1017 | DRM_CONNECTOR_POLL_DISCONNECT;
1018 hpd_disabled = true;
1019 }
142e2398
EE		 1020 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
1021 DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n",
1022 drm_get_connector_name(connector), intel_encoder->hpd_pin);
1023 }
cd569aed
EE		 1024 }
1025 /* if there were no outputs to poll, poll was disabled,
1026 * therefore make sure it's enabled when disabling HPD on
1027 * some connectors */
ac4c16c5 1028 if (hpd_disabled) {
cd569aed 1029 drm_kms_helper_poll_enable(dev);
ac4c16c5
EE		 1030 mod_timer(&dev_priv->hotplug_reenable_timer,
1031 jiffies + msecs_to_jiffies(I915_REENABLE_HOTPLUG_DELAY));
1032 }
cd569aed
EE		 1033
1034 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1035
321a1b30
EE		 1036 list_for_each_entry(connector, &mode_config->connector_list, head) {
1037 intel_connector = to_intel_connector(connector);
1038 intel_encoder = intel_connector->encoder;
1039 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
1040 if (intel_encoder->hot_plug)
1041 intel_encoder->hot_plug(intel_encoder);
1042 if (intel_hpd_irq_event(dev, connector))
1043 changed = true;
1044 }
1045 }
40ee3381
KP		 1046 mutex_unlock(&mode_config->mutex);
1047
321a1b30
EE		 1048 if (changed)
1049 drm_kms_helper_hotplug_event(dev);
5ca58282
JB		 1050}
1051
3ca1cced
VS		 1052static void intel_hpd_irq_uninstall(struct drm_i915_private *dev_priv)
1053{
1054 del_timer_sync(&dev_priv->hotplug_reenable_timer);
1055}
1056
d0ecd7e2 1057static void ironlake_rps_change_irq_handler(struct drm_device *dev)
f97108d1 1058{
2d1013dd 1059 struct drm_i915_private *dev_priv = dev->dev_private;
b5b72e89 1060 u32 busy_up, busy_down, max_avg, min_avg;
9270388e 1061 u8 new_delay;
9270388e 1062
d0ecd7e2 1063 spin_lock(&mchdev_lock);
f97108d1 1064
73edd18f
DV		 1065 I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
1066
20e4d407 1067 new_delay = dev_priv->ips.cur_delay;
9270388e 1068
7648fa99 1069 I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
b5b72e89
MG		 1070 busy_up = I915_READ(RCPREVBSYTUPAVG);
1071 busy_down = I915_READ(RCPREVBSYTDNAVG);
f97108d1
JB		 1072 max_avg = I915_READ(RCBMAXAVG);
1073 min_avg = I915_READ(RCBMINAVG);
1074
1075 /* Handle RCS change request from hw */
b5b72e89 1076 if (busy_up > max_avg) {
20e4d407
DV		 1077 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
1078 new_delay = dev_priv->ips.cur_delay - 1;
1079 if (new_delay < dev_priv->ips.max_delay)
1080 new_delay = dev_priv->ips.max_delay;
b5b72e89 1081 } else if (busy_down < min_avg) {
20e4d407
DV		 1082 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
1083 new_delay = dev_priv->ips.cur_delay + 1;
1084 if (new_delay > dev_priv->ips.min_delay)
1085 new_delay = dev_priv->ips.min_delay;
f97108d1
JB		 1086 }
1087
7648fa99 1088 if (ironlake_set_drps(dev, new_delay))
20e4d407 1089 dev_priv->ips.cur_delay = new_delay;
f97108d1 1090
d0ecd7e2 1091 spin_unlock(&mchdev_lock);
9270388e 1092
f97108d1
JB		 1093 return;
1094}
1095
549f7365
CW		 1096static void notify_ring(struct drm_device *dev,
1097 struct intel_ring_buffer *ring)
1098{
475553de
CW		 1099 if (ring->obj == NULL)
1100 return;
1101
814e9b57 1102 trace_i915_gem_request_complete(ring);
9862e600 1103
549f7365 1104 wake_up_all(&ring->irq_queue);
10cd45b6 1105 i915_queue_hangcheck(dev);
549f7365
CW		 1106}
1107
4912d041 1108static void gen6_pm_rps_work(struct work_struct *work)
3b8d8d91 1109{
2d1013dd
JN		 1110 struct drm_i915_private *dev_priv =
1111 container_of(work, struct drm_i915_private, rps.work);
edbfdb45 1112 u32 pm_iir;
dd75fdc8 1113 int new_delay, adj;
4912d041 1114
59cdb63d 1115 spin_lock_irq(&dev_priv->irq_lock);
c6a828d3
DV		 1116 pm_iir = dev_priv->rps.pm_iir;
1117 dev_priv->rps.pm_iir = 0;
4848405c 1118 /* Make sure not to corrupt PMIMR state used by ringbuffer code */
a6706b45 1119 snb_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
59cdb63d 1120 spin_unlock_irq(&dev_priv->irq_lock);
3b8d8d91 1121
60611c13 1122 /* Make sure we didn't queue anything we're not going to process. */
a6706b45 1123 WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
60611c13 1124
a6706b45 1125 if ((pm_iir & dev_priv->pm_rps_events) == 0)
3b8d8d91
JB		 1126 return;
1127
4fc688ce 1128 mutex_lock(&dev_priv->rps.hw_lock);
7b9e0ae6 1129
dd75fdc8 1130 adj = dev_priv->rps.last_adj;
7425034a 1131 if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
dd75fdc8
CW		 1132 if (adj > 0)
1133 adj *= 2;
1134 else
1135 adj = 1;
b39fb297 1136 new_delay = dev_priv->rps.cur_freq + adj;
7425034a
VS		 1137
1138 /*
1139 * For better performance, jump directly
1140 * to RPe if we're below it.
1141 */
b39fb297
BW		 1142 if (new_delay < dev_priv->rps.efficient_freq)
1143 new_delay = dev_priv->rps.efficient_freq;
dd75fdc8 1144 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
b39fb297
BW		 1145 if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
1146 new_delay = dev_priv->rps.efficient_freq;
dd75fdc8 1147 else
b39fb297 1148 new_delay = dev_priv->rps.min_freq_softlimit;
dd75fdc8
CW		 1149 adj = 0;
1150 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1151 if (adj < 0)
1152 adj *= 2;
1153 else
1154 adj = -1;
b39fb297 1155 new_delay = dev_priv->rps.cur_freq + adj;
dd75fdc8 1156 } else { /* unknown event */
b39fb297 1157 new_delay = dev_priv->rps.cur_freq;
dd75fdc8 1158 }
3b8d8d91 1159
79249636
BW		 1160 /* sysfs frequency interfaces may have snuck in while servicing the
1161 * interrupt
1162 */
1272e7b8 1163 new_delay = clamp_t(int, new_delay,
b39fb297
BW		 1164 dev_priv->rps.min_freq_softlimit,
1165 dev_priv->rps.max_freq_softlimit);
27544369 1166
b39fb297 1167 dev_priv->rps.last_adj = new_delay - dev_priv->rps.cur_freq;
dd75fdc8
CW		 1168
1169 if (IS_VALLEYVIEW(dev_priv->dev))
1170 valleyview_set_rps(dev_priv->dev, new_delay);
1171 else
1172 gen6_set_rps(dev_priv->dev, new_delay);
3b8d8d91 1173
4fc688ce 1174 mutex_unlock(&dev_priv->rps.hw_lock);
3b8d8d91
JB		 1175}
1176
e3689190
BW		 1177
1178/**
1179 * ivybridge_parity_work - Workqueue called when a parity error interrupt
1180 * occurred.
1181 * @work: workqueue struct
1182 *
1183 * Doesn't actually do anything except notify userspace. As a consequence of
1184 * this event, userspace should try to remap the bad rows since statistically
1185 * it is likely the same row is more likely to go bad again.
1186 */
1187static void ivybridge_parity_work(struct work_struct *work)
1188{
2d1013dd
JN		 1189 struct drm_i915_private *dev_priv =
1190 container_of(work, struct drm_i915_private, l3_parity.error_work);
e3689190 1191 u32 error_status, row, bank, subbank;
35a85ac6 1192 char *parity_event[6];
e3689190
BW		 1193 uint32_t misccpctl;
1194 unsigned long flags;
35a85ac6 1195 uint8_t slice = 0;
e3689190
BW		 1196
1197 /* We must turn off DOP level clock gating to access the L3 registers.
1198 * In order to prevent a get/put style interface, acquire struct mutex
1199 * any time we access those registers.
1200 */
1201 mutex_lock(&dev_priv->dev->struct_mutex);
1202
35a85ac6
BW		 1203 /* If we've screwed up tracking, just let the interrupt fire again */
1204 if (WARN_ON(!dev_priv->l3_parity.which_slice))
1205 goto out;
1206
e3689190
BW		 1207 misccpctl = I915_READ(GEN7_MISCCPCTL);
1208 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
1209 POSTING_READ(GEN7_MISCCPCTL);
1210
35a85ac6
BW		 1211 while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1212 u32 reg;
e3689190 1213
35a85ac6
BW		 1214 slice--;
1215 if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
1216 break;
e3689190 1217
35a85ac6 1218 dev_priv->l3_parity.which_slice &= ~(1<<slice);
e3689190 1219
35a85ac6 1220 reg = GEN7_L3CDERRST1 + (slice * 0x200);
e3689190 1221
35a85ac6
BW		 1222 error_status = I915_READ(reg);
1223 row = GEN7_PARITY_ERROR_ROW(error_status);
1224 bank = GEN7_PARITY_ERROR_BANK(error_status);
1225 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
1226
1227 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
1228 POSTING_READ(reg);
1229
1230 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
1231 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
1232 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
1233 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
1234 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
1235 parity_event[5] = NULL;
1236
5bdebb18 1237 kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
35a85ac6 1238 KOBJ_CHANGE, parity_event);
e3689190 1239
35a85ac6
BW		 1240 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
1241 slice, row, bank, subbank);
e3689190 1242
35a85ac6
BW		 1243 kfree(parity_event[4]);
1244 kfree(parity_event[3]);
1245 kfree(parity_event[2]);
1246 kfree(parity_event[1]);
1247 }
e3689190 1248
35a85ac6 1249 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
e3689190 1250
35a85ac6
BW		 1251out:
1252 WARN_ON(dev_priv->l3_parity.which_slice);
1253 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1254 ilk_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
1255 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1256
1257 mutex_unlock(&dev_priv->dev->struct_mutex);
e3689190
BW		 1258}
1259
35a85ac6 1260static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
e3689190 1261{
2d1013dd 1262 struct drm_i915_private *dev_priv = dev->dev_private;
e3689190 1263
040d2baa 1264 if (!HAS_L3_DPF(dev))
e3689190
BW		 1265 return;
1266
d0ecd7e2 1267 spin_lock(&dev_priv->irq_lock);
35a85ac6 1268 ilk_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
d0ecd7e2 1269 spin_unlock(&dev_priv->irq_lock);
e3689190 1270
35a85ac6
BW		 1271 iir &= GT_PARITY_ERROR(dev);
1272 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
1273 dev_priv->l3_parity.which_slice |= 1 << 1;
1274
1275 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
1276 dev_priv->l3_parity.which_slice |= 1 << 0;
1277
a4da4fa4 1278 queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
e3689190
BW		 1279}
1280
f1af8fc1
PZ		 1281static void ilk_gt_irq_handler(struct drm_device *dev,
1282 struct drm_i915_private *dev_priv,
1283 u32 gt_iir)
1284{
1285 if (gt_iir &
1286 (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1287 notify_ring(dev, &dev_priv->ring[RCS]);
1288 if (gt_iir & ILK_BSD_USER_INTERRUPT)
1289 notify_ring(dev, &dev_priv->ring[VCS]);
1290}
1291
e7b4c6b1
DV		 1292static void snb_gt_irq_handler(struct drm_device *dev,
1293 struct drm_i915_private *dev_priv,
1294 u32 gt_iir)
1295{
1296
cc609d5d
BW		 1297 if (gt_iir &
1298 (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
e7b4c6b1 1299 notify_ring(dev, &dev_priv->ring[RCS]);
cc609d5d 1300 if (gt_iir & GT_BSD_USER_INTERRUPT)
e7b4c6b1 1301 notify_ring(dev, &dev_priv->ring[VCS]);
cc609d5d 1302 if (gt_iir & GT_BLT_USER_INTERRUPT)
e7b4c6b1
DV		 1303 notify_ring(dev, &dev_priv->ring[BCS]);
1304
cc609d5d
BW		 1305 if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
1306 GT_BSD_CS_ERROR_INTERRUPT |
1307 GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) {
58174462
MK		 1308 i915_handle_error(dev, false, "GT error interrupt 0x%08x",
1309 gt_iir);
e7b4c6b1 1310 }
e3689190 1311
35a85ac6
BW		 1312 if (gt_iir & GT_PARITY_ERROR(dev))
1313 ivybridge_parity_error_irq_handler(dev, gt_iir);
e7b4c6b1
DV		 1314}
1315
abd58f01
BW		 1316static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev,
1317 struct drm_i915_private *dev_priv,
1318 u32 master_ctl)
1319{
1320 u32 rcs, bcs, vcs;
1321 uint32_t tmp = 0;
1322 irqreturn_t ret = IRQ_NONE;
1323
1324 if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1325 tmp = I915_READ(GEN8_GT_IIR(0));
1326 if (tmp) {
1327 ret = IRQ_HANDLED;
1328 rcs = tmp >> GEN8_RCS_IRQ_SHIFT;
1329 bcs = tmp >> GEN8_BCS_IRQ_SHIFT;
1330 if (rcs & GT_RENDER_USER_INTERRUPT)
1331 notify_ring(dev, &dev_priv->ring[RCS]);
1332 if (bcs & GT_RENDER_USER_INTERRUPT)
1333 notify_ring(dev, &dev_priv->ring[BCS]);
1334 I915_WRITE(GEN8_GT_IIR(0), tmp);
1335 } else
1336 DRM_ERROR("The master control interrupt lied (GT0)!\n");
1337 }
1338
1339 if (master_ctl & GEN8_GT_VCS1_IRQ) {
1340 tmp = I915_READ(GEN8_GT_IIR(1));
1341 if (tmp) {
1342 ret = IRQ_HANDLED;
1343 vcs = tmp >> GEN8_VCS1_IRQ_SHIFT;
1344 if (vcs & GT_RENDER_USER_INTERRUPT)
1345 notify_ring(dev, &dev_priv->ring[VCS]);
1346 I915_WRITE(GEN8_GT_IIR(1), tmp);
1347 } else
1348 DRM_ERROR("The master control interrupt lied (GT1)!\n");
1349 }
1350
1351 if (master_ctl & GEN8_GT_VECS_IRQ) {
1352 tmp = I915_READ(GEN8_GT_IIR(3));
1353 if (tmp) {
1354 ret = IRQ_HANDLED;
1355 vcs = tmp >> GEN8_VECS_IRQ_SHIFT;
1356 if (vcs & GT_RENDER_USER_INTERRUPT)
1357 notify_ring(dev, &dev_priv->ring[VECS]);
1358 I915_WRITE(GEN8_GT_IIR(3), tmp);
1359 } else
1360 DRM_ERROR("The master control interrupt lied (GT3)!\n");
1361 }
1362
1363 return ret;
1364}
1365
b543fb04
EE		 1366#define HPD_STORM_DETECT_PERIOD 1000
1367#define HPD_STORM_THRESHOLD 5
1368
10a504de 1369static inline void intel_hpd_irq_handler(struct drm_device *dev,
22062dba
DV		 1370 u32 hotplug_trigger,
1371 const u32 *hpd)
b543fb04 1372{
2d1013dd 1373 struct drm_i915_private *dev_priv = dev->dev_private;
b543fb04 1374 int i;
10a504de 1375 bool storm_detected = false;
b543fb04 1376
91d131d2
DV		 1377 if (!hotplug_trigger)
1378 return;
1379
cc9bd499
ID		 1380 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
1381 hotplug_trigger);
1382
b5ea2d56 1383 spin_lock(&dev_priv->irq_lock);
b543fb04 1384 for (i = 1; i < HPD_NUM_PINS; i++) {
821450c6 1385
3432087e 1386 WARN_ONCE(hpd[i] & hotplug_trigger &&
8b5565b8 1387 dev_priv->hpd_stats[i].hpd_mark == HPD_DISABLED,
cba1c073
CW		 1388 "Received HPD interrupt (0x%08x) on pin %d (0x%08x) although disabled\n",
1389 hotplug_trigger, i, hpd[i]);
b8f102e8 1390
b543fb04
EE		 1391 if (!(hpd[i] & hotplug_trigger) ||
1392 dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED)
1393 continue;
1394
bc5ead8c 1395 dev_priv->hpd_event_bits |= (1 << i);
b543fb04
EE		 1396 if (!time_in_range(jiffies, dev_priv->hpd_stats[i].hpd_last_jiffies,
1397 dev_priv->hpd_stats[i].hpd_last_jiffies
1398 + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD))) {
1399 dev_priv->hpd_stats[i].hpd_last_jiffies = jiffies;
1400 dev_priv->hpd_stats[i].hpd_cnt = 0;
b8f102e8 1401 DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: 0\n", i);
b543fb04
EE		 1402 } else if (dev_priv->hpd_stats[i].hpd_cnt > HPD_STORM_THRESHOLD) {
1403 dev_priv->hpd_stats[i].hpd_mark = HPD_MARK_DISABLED;
142e2398 1404 dev_priv->hpd_event_bits &= ~(1 << i);
b543fb04 1405 DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", i);
10a504de 1406 storm_detected = true;
b543fb04
EE		 1407 } else {
1408 dev_priv->hpd_stats[i].hpd_cnt++;
b8f102e8
EE		 1409 DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: %d\n", i,
1410 dev_priv->hpd_stats[i].hpd_cnt);
b543fb04
EE		 1411 }
1412 }
1413
10a504de
DV		 1414 if (storm_detected)
1415 dev_priv->display.hpd_irq_setup(dev);
b5ea2d56 1416 spin_unlock(&dev_priv->irq_lock);
5876fa0d 1417
645416f5
DV		 1418 /*
1419 * Our hotplug handler can grab modeset locks (by calling down into the
1420 * fb helpers). Hence it must not be run on our own dev-priv->wq work
1421 * queue for otherwise the flush_work in the pageflip code will
1422 * deadlock.
1423 */
1424 schedule_work(&dev_priv->hotplug_work);
b543fb04
EE		 1425}
1426
515ac2bb
DV		 1427static void gmbus_irq_handler(struct drm_device *dev)
1428{
2d1013dd 1429 struct drm_i915_private *dev_priv = dev->dev_private;
28c70f16 1430
28c70f16 1431 wake_up_all(&dev_priv->gmbus_wait_queue);
515ac2bb
DV		 1432}
1433
ce99c256
DV		 1434static void dp_aux_irq_handler(struct drm_device *dev)
1435{
2d1013dd 1436 struct drm_i915_private *dev_priv = dev->dev_private;
9ee32fea 1437
9ee32fea 1438 wake_up_all(&dev_priv->gmbus_wait_queue);
ce99c256
DV		 1439}
1440
8bf1e9f1 1441#if defined(CONFIG_DEBUG_FS)
277de95e
DV		 1442static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1443 uint32_t crc0, uint32_t crc1,
1444 uint32_t crc2, uint32_t crc3,
1445 uint32_t crc4)
8bf1e9f1
SH		 1446{
1447 struct drm_i915_private *dev_priv = dev->dev_private;
1448 struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
1449 struct intel_pipe_crc_entry *entry;
ac2300d4 1450 int head, tail;
b2c88f5b 1451
d538bbdf
DL		 1452 spin_lock(&pipe_crc->lock);
1453
0c912c79 1454 if (!pipe_crc->entries) {
d538bbdf 1455 spin_unlock(&pipe_crc->lock);
0c912c79
DL		 1456 DRM_ERROR("spurious interrupt\n");
1457 return;
1458 }
1459
d538bbdf
DL		 1460 head = pipe_crc->head;
1461 tail = pipe_crc->tail;
b2c88f5b
DL		 1462
1463 if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
d538bbdf 1464 spin_unlock(&pipe_crc->lock);
b2c88f5b
DL		 1465 DRM_ERROR("CRC buffer overflowing\n");
1466 return;
1467 }
1468
1469 entry = &pipe_crc->entries[head];
8bf1e9f1 1470
8bc5e955 1471 entry->frame = dev->driver->get_vblank_counter(dev, pipe);
eba94eb9
DV		 1472 entry->crc[0] = crc0;
1473 entry->crc[1] = crc1;
1474 entry->crc[2] = crc2;
1475 entry->crc[3] = crc3;
1476 entry->crc[4] = crc4;
b2c88f5b
DL		 1477
1478 head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
d538bbdf
DL		 1479 pipe_crc->head = head;
1480
1481 spin_unlock(&pipe_crc->lock);
07144428
DL		 1482
1483 wake_up_interruptible(&pipe_crc->wq);
8bf1e9f1 1484}
277de95e
DV		 1485#else
1486static inline void
1487display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1488 uint32_t crc0, uint32_t crc1,
1489 uint32_t crc2, uint32_t crc3,
1490 uint32_t crc4) {}
1491#endif
1492
eba94eb9 1493
277de95e 1494static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
5a69b89f
DV		 1495{
1496 struct drm_i915_private *dev_priv = dev->dev_private;
1497
277de95e
DV		 1498 display_pipe_crc_irq_handler(dev, pipe,
1499 I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1500 0, 0, 0, 0);
5a69b89f
DV		 1501}
1502
277de95e 1503static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
eba94eb9
DV		 1504{
1505 struct drm_i915_private *dev_priv = dev->dev_private;
1506
277de95e
DV		 1507 display_pipe_crc_irq_handler(dev, pipe,
1508 I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1509 I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1510 I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1511 I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
1512 I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
eba94eb9 1513}
5b3a856b 1514
277de95e 1515static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
5b3a856b
DV		 1516{
1517 struct drm_i915_private *dev_priv = dev->dev_private;
0b5c5ed0
DV		 1518 uint32_t res1, res2;
1519
1520 if (INTEL_INFO(dev)->gen >= 3)
1521 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
1522 else
1523 res1 = 0;
1524
1525 if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
1526 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
1527 else
1528 res2 = 0;
5b3a856b 1529
277de95e
DV		 1530 display_pipe_crc_irq_handler(dev, pipe,
1531 I915_READ(PIPE_CRC_RES_RED(pipe)),
1532 I915_READ(PIPE_CRC_RES_GREEN(pipe)),
1533 I915_READ(PIPE_CRC_RES_BLUE(pipe)),
1534 res1, res2);
5b3a856b 1535}
8bf1e9f1 1536
1403c0d4
PZ		 1537/* The RPS events need forcewake, so we add them to a work queue and mask their
1538 * IMR bits until the work is done. Other interrupts can be processed without
1539 * the work queue. */
1540static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
baf02a1f 1541{
a6706b45 1542 if (pm_iir & dev_priv->pm_rps_events) {
59cdb63d 1543 spin_lock(&dev_priv->irq_lock);
a6706b45
D		 1544 dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
1545 snb_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
59cdb63d 1546 spin_unlock(&dev_priv->irq_lock);
2adbee62
DV		 1547
1548 queue_work(dev_priv->wq, &dev_priv->rps.work);
baf02a1f 1549 }
baf02a1f 1550
1403c0d4
PZ		 1551 if (HAS_VEBOX(dev_priv->dev)) {
1552 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1553 notify_ring(dev_priv->dev, &dev_priv->ring[VECS]);
12638c57 1554
1403c0d4 1555 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) {
58174462
MK		 1556 i915_handle_error(dev_priv->dev, false,
1557 "VEBOX CS error interrupt 0x%08x",
1558 pm_iir);
1403c0d4 1559 }
12638c57 1560 }
baf02a1f
BW		 1561}
1562
c1874ed7
ID		 1563static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
1564{
1565 struct drm_i915_private *dev_priv = dev->dev_private;
91d181dd 1566 u32 pipe_stats[I915_MAX_PIPES] = { };
c1874ed7
ID		 1567 int pipe;
1568
58ead0d7 1569 spin_lock(&dev_priv->irq_lock);
c1874ed7 1570 for_each_pipe(pipe) {
91d181dd 1571 int reg;
bbb5eebf 1572 u32 mask, iir_bit = 0;
91d181dd 1573
bbb5eebf
DV		 1574 /*
1575 * PIPESTAT bits get signalled even when the interrupt is
1576 * disabled with the mask bits, and some of the status bits do
1577 * not generate interrupts at all (like the underrun bit). Hence
1578 * we need to be careful that we only handle what we want to
1579 * handle.
1580 */
1581 mask = 0;
1582 if (__cpu_fifo_underrun_reporting_enabled(dev, pipe))
1583 mask |= PIPE_FIFO_UNDERRUN_STATUS;
1584
1585 switch (pipe) {
1586 case PIPE_A:
1587 iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1588 break;
1589 case PIPE_B:
1590 iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1591 break;
1592 }
1593 if (iir & iir_bit)
1594 mask |= dev_priv->pipestat_irq_mask[pipe];
1595
1596 if (!mask)
91d181dd
ID		 1597 continue;
1598
1599 reg = PIPESTAT(pipe);
bbb5eebf
DV		 1600 mask |= PIPESTAT_INT_ENABLE_MASK;
1601 pipe_stats[pipe] = I915_READ(reg) & mask;
c1874ed7
ID		 1602
1603 /*
1604 * Clear the PIPE*STAT regs before the IIR
1605 */
91d181dd
ID		 1606 if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
1607 PIPESTAT_INT_STATUS_MASK))
c1874ed7
ID		 1608 I915_WRITE(reg, pipe_stats[pipe]);
1609 }
58ead0d7 1610 spin_unlock(&dev_priv->irq_lock);
c1874ed7
ID		 1611
1612 for_each_pipe(pipe) {
1613 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
1614 drm_handle_vblank(dev, pipe);
1615
579a9b0e 1616 if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
c1874ed7
ID		 1617 intel_prepare_page_flip(dev, pipe);
1618 intel_finish_page_flip(dev, pipe);
1619 }
1620
1621 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1622 i9xx_pipe_crc_irq_handler(dev, pipe);
1623
1624 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS &&
1625 intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
1626 DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
1627 }
1628
1629 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1630 gmbus_irq_handler(dev);
1631}
1632
16c6c56b
VS		 1633static void i9xx_hpd_irq_handler(struct drm_device *dev)
1634{
1635 struct drm_i915_private *dev_priv = dev->dev_private;
1636 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
1637
1638 if (IS_G4X(dev)) {
1639 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1640
1641 intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_g4x);
1642 } else {
1643 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1644
1645 intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
1646 }
1647
1648 if ((IS_G4X(dev) || IS_VALLEYVIEW(dev)) &&
1649 hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1650 dp_aux_irq_handler(dev);
1651
1652 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1653 /*
1654 * Make sure hotplug status is cleared before we clear IIR, or else we
1655 * may miss hotplug events.
1656 */
1657 POSTING_READ(PORT_HOTPLUG_STAT);
1658}
1659
ff1f525e 1660static irqreturn_t valleyview_irq_handler(int irq, void *arg)
7e231dbe
JB		 1661{
1662 struct drm_device *dev = (struct drm_device *) arg;
2d1013dd 1663 struct drm_i915_private *dev_priv = dev->dev_private;
7e231dbe
JB		 1664 u32 iir, gt_iir, pm_iir;
1665 irqreturn_t ret = IRQ_NONE;
7e231dbe 1666
7e231dbe
JB		 1667 while (true) {
1668 iir = I915_READ(VLV_IIR);
1669 gt_iir = I915_READ(GTIIR);
1670 pm_iir = I915_READ(GEN6_PMIIR);
1671
1672 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1673 goto out;
1674
1675 ret = IRQ_HANDLED;
1676
e7b4c6b1 1677 snb_gt_irq_handler(dev, dev_priv, gt_iir);
7e231dbe 1678
c1874ed7 1679 valleyview_pipestat_irq_handler(dev, iir);
31acc7f5 1680
7e231dbe 1681 /* Consume port. Then clear IIR or we'll miss events */
16c6c56b
VS		 1682 if (iir & I915_DISPLAY_PORT_INTERRUPT)
1683 i9xx_hpd_irq_handler(dev);
7e231dbe 1684
60611c13 1685 if (pm_iir)
d0ecd7e2 1686 gen6_rps_irq_handler(dev_priv, pm_iir);
7e231dbe
JB		 1687
1688 I915_WRITE(GTIIR, gt_iir);
1689 I915_WRITE(GEN6_PMIIR, pm_iir);
1690 I915_WRITE(VLV_IIR, iir);
1691 }
1692
1693out:
1694 return ret;
1695}
1696
23e81d69 1697static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
776ad806 1698{
2d1013dd 1699 struct drm_i915_private *dev_priv = dev->dev_private;
9db4a9c7 1700 int pipe;
b543fb04 1701 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
776ad806 1702
91d131d2
DV		 1703 intel_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
1704
cfc33bf7
VS		 1705 if (pch_iir & SDE_AUDIO_POWER_MASK) {
1706 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1707 SDE_AUDIO_POWER_SHIFT);
776ad806 1708 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
cfc33bf7
VS		 1709 port_name(port));
1710 }
776ad806 1711
ce99c256
DV		 1712 if (pch_iir & SDE_AUX_MASK)
1713 dp_aux_irq_handler(dev);
1714
776ad806 1715 if (pch_iir & SDE_GMBUS)
515ac2bb 1716 gmbus_irq_handler(dev);
776ad806
JB		 1717
1718 if (pch_iir & SDE_AUDIO_HDCP_MASK)
1719 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
1720
1721 if (pch_iir & SDE_AUDIO_TRANS_MASK)
1722 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
1723
1724 if (pch_iir & SDE_POISON)
1725 DRM_ERROR("PCH poison interrupt\n");
1726
9db4a9c7
JB		 1727 if (pch_iir & SDE_FDI_MASK)
1728 for_each_pipe(pipe)
1729 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
1730 pipe_name(pipe),
1731 I915_READ(FDI_RX_IIR(pipe)));
776ad806
JB		 1732
1733 if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1734 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
1735
1736 if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1737 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
1738
776ad806 1739 if (pch_iir & SDE_TRANSA_FIFO_UNDER)
8664281b
PZ		 1740 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
1741 false))
fc2c807b 1742 DRM_ERROR("PCH transcoder A FIFO underrun\n");
8664281b
PZ		 1743
1744 if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1745 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
1746 false))
fc2c807b 1747 DRM_ERROR("PCH transcoder B FIFO underrun\n");
8664281b
PZ		 1748}
1749
1750static void ivb_err_int_handler(struct drm_device *dev)
1751{
1752 struct drm_i915_private *dev_priv = dev->dev_private;
1753 u32 err_int = I915_READ(GEN7_ERR_INT);
5a69b89f 1754 enum pipe pipe;
8664281b 1755
de032bf4
PZ		 1756 if (err_int & ERR_INT_POISON)
1757 DRM_ERROR("Poison interrupt\n");
1758
5a69b89f
DV		 1759 for_each_pipe(pipe) {
1760 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe)) {
1761 if (intel_set_cpu_fifo_underrun_reporting(dev, pipe,
1762 false))
fc2c807b
VS		 1763 DRM_ERROR("Pipe %c FIFO underrun\n",
1764 pipe_name(pipe));
5a69b89f 1765 }
8bf1e9f1 1766
5a69b89f
DV		 1767 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
1768 if (IS_IVYBRIDGE(dev))
277de95e 1769 ivb_pipe_crc_irq_handler(dev, pipe);
5a69b89f 1770 else
277de95e 1771 hsw_pipe_crc_irq_handler(dev, pipe);
5a69b89f
DV		 1772 }
1773 }
8bf1e9f1 1774
8664281b
PZ		 1775 I915_WRITE(GEN7_ERR_INT, err_int);
1776}
1777
1778static void cpt_serr_int_handler(struct drm_device *dev)
1779{
1780 struct drm_i915_private *dev_priv = dev->dev_private;
1781 u32 serr_int = I915_READ(SERR_INT);
1782
de032bf4
PZ		 1783 if (serr_int & SERR_INT_POISON)
1784 DRM_ERROR("PCH poison interrupt\n");
1785
8664281b
PZ		 1786 if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1787 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
1788 false))
fc2c807b 1789 DRM_ERROR("PCH transcoder A FIFO underrun\n");
8664281b
PZ		 1790
1791 if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1792 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
1793 false))
fc2c807b 1794 DRM_ERROR("PCH transcoder B FIFO underrun\n");
8664281b
PZ		 1795
1796 if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1797 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_C,
1798 false))
fc2c807b 1799 DRM_ERROR("PCH transcoder C FIFO underrun\n");
8664281b
PZ		 1800
1801 I915_WRITE(SERR_INT, serr_int);
776ad806
JB		 1802}
1803
23e81d69
AJ		 1804static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
1805{
2d1013dd 1806 struct drm_i915_private *dev_priv = dev->dev_private;
23e81d69 1807 int pipe;
b543fb04 1808 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
23e81d69 1809
91d131d2
DV		 1810 intel_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
1811
cfc33bf7
VS		 1812 if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1813 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1814 SDE_AUDIO_POWER_SHIFT_CPT);
1815 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
1816 port_name(port));
1817 }
23e81d69
AJ		 1818
1819 if (pch_iir & SDE_AUX_MASK_CPT)
ce99c256 1820 dp_aux_irq_handler(dev);
23e81d69
AJ		 1821
1822 if (pch_iir & SDE_GMBUS_CPT)
515ac2bb 1823 gmbus_irq_handler(dev);
23e81d69
AJ		 1824
1825 if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1826 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
1827
1828 if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1829 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
1830
1831 if (pch_iir & SDE_FDI_MASK_CPT)
1832 for_each_pipe(pipe)
1833 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
1834 pipe_name(pipe),
1835 I915_READ(FDI_RX_IIR(pipe)));
8664281b
PZ		 1836
1837 if (pch_iir & SDE_ERROR_CPT)
1838 cpt_serr_int_handler(dev);
23e81d69
AJ		 1839}
1840
c008bc6e
PZ		 1841static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
1842{
1843 struct drm_i915_private *dev_priv = dev->dev_private;
40da17c2 1844 enum pipe pipe;
c008bc6e
PZ		 1845
1846 if (de_iir & DE_AUX_CHANNEL_A)
1847 dp_aux_irq_handler(dev);
1848
1849 if (de_iir & DE_GSE)
1850 intel_opregion_asle_intr(dev);
1851
c008bc6e
PZ		 1852 if (de_iir & DE_POISON)
1853 DRM_ERROR("Poison interrupt\n");
1854
40da17c2
DV		 1855 for_each_pipe(pipe) {
1856 if (de_iir & DE_PIPE_VBLANK(pipe))
1857 drm_handle_vblank(dev, pipe);
5b3a856b 1858
40da17c2
DV		 1859 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
1860 if (intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
fc2c807b
VS		 1861 DRM_ERROR("Pipe %c FIFO underrun\n",
1862 pipe_name(pipe));
5b3a856b 1863
40da17c2
DV		 1864 if (de_iir & DE_PIPE_CRC_DONE(pipe))
1865 i9xx_pipe_crc_irq_handler(dev, pipe);
c008bc6e 1866
40da17c2
DV		 1867 /* plane/pipes map 1:1 on ilk+ */
1868 if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
1869 intel_prepare_page_flip(dev, pipe);
1870 intel_finish_page_flip_plane(dev, pipe);
1871 }
c008bc6e
PZ		 1872 }
1873
1874 /* check event from PCH */
1875 if (de_iir & DE_PCH_EVENT) {
1876 u32 pch_iir = I915_READ(SDEIIR);
1877
1878 if (HAS_PCH_CPT(dev))
1879 cpt_irq_handler(dev, pch_iir);
1880 else
1881 ibx_irq_handler(dev, pch_iir);
1882
1883 /* should clear PCH hotplug event before clear CPU irq */
1884 I915_WRITE(SDEIIR, pch_iir);
1885 }
1886
1887 if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
1888 ironlake_rps_change_irq_handler(dev);
1889}
1890
9719fb98
PZ		 1891static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
1892{
1893 struct drm_i915_private *dev_priv = dev->dev_private;
07d27e20 1894 enum pipe pipe;
9719fb98
PZ		 1895
1896 if (de_iir & DE_ERR_INT_IVB)
1897 ivb_err_int_handler(dev);
1898
1899 if (de_iir & DE_AUX_CHANNEL_A_IVB)
1900 dp_aux_irq_handler(dev);
1901
1902 if (de_iir & DE_GSE_IVB)
1903 intel_opregion_asle_intr(dev);
1904
07d27e20
DL		 1905 for_each_pipe(pipe) {
1906 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)))
1907 drm_handle_vblank(dev, pipe);
40da17c2
DV		 1908
1909 /* plane/pipes map 1:1 on ilk+ */
07d27e20
DL		 1910 if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
1911 intel_prepare_page_flip(dev, pipe);
1912 intel_finish_page_flip_plane(dev, pipe);
9719fb98
PZ		 1913 }
1914 }
1915
1916 /* check event from PCH */
1917 if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
1918 u32 pch_iir = I915_READ(SDEIIR);
1919
1920 cpt_irq_handler(dev, pch_iir);
1921
1922 /* clear PCH hotplug event before clear CPU irq */
1923 I915_WRITE(SDEIIR, pch_iir);
1924 }
1925}
1926
f1af8fc1 1927static irqreturn_t ironlake_irq_handler(int irq, void *arg)
b1f14ad0
JB		 1928{
1929 struct drm_device *dev = (struct drm_device *) arg;
2d1013dd 1930 struct drm_i915_private *dev_priv = dev->dev_private;
f1af8fc1 1931 u32 de_iir, gt_iir, de_ier, sde_ier = 0;
0e43406b 1932 irqreturn_t ret = IRQ_NONE;
b1f14ad0 1933
8664281b
PZ		 1934 /* We get interrupts on unclaimed registers, so check for this before we
1935 * do any I915_{READ,WRITE}. */
907b28c5 1936 intel_uncore_check_errors(dev);
8664281b 1937
b1f14ad0
JB		 1938 /* disable master interrupt before clearing iir */
1939 de_ier = I915_READ(DEIER);
1940 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
23a78516 1941 POSTING_READ(DEIER);
b1f14ad0 1942
44498aea
PZ		 1943 /* Disable south interrupts. We'll only write to SDEIIR once, so further
1944 * interrupts will will be stored on its back queue, and then we'll be
1945 * able to process them after we restore SDEIER (as soon as we restore
1946 * it, we'll get an interrupt if SDEIIR still has something to process
1947 * due to its back queue). */
ab5c608b
BW		 1948 if (!HAS_PCH_NOP(dev)) {
1949 sde_ier = I915_READ(SDEIER);
1950 I915_WRITE(SDEIER, 0);
1951 POSTING_READ(SDEIER);
1952 }
44498aea 1953
b1f14ad0 1954 gt_iir = I915_READ(GTIIR);
0e43406b 1955 if (gt_iir) {
d8fc8a47 1956 if (INTEL_INFO(dev)->gen >= 6)
f1af8fc1 1957 snb_gt_irq_handler(dev, dev_priv, gt_iir);
d8fc8a47
PZ		 1958 else
1959 ilk_gt_irq_handler(dev, dev_priv, gt_iir);
0e43406b
CW		 1960 I915_WRITE(GTIIR, gt_iir);
1961 ret = IRQ_HANDLED;
b1f14ad0
JB		 1962 }
1963
0e43406b
CW		 1964 de_iir = I915_READ(DEIIR);
1965 if (de_iir) {
f1af8fc1
PZ		 1966 if (INTEL_INFO(dev)->gen >= 7)
1967 ivb_display_irq_handler(dev, de_iir);
1968 else
1969 ilk_display_irq_handler(dev, de_iir);
0e43406b
CW		 1970 I915_WRITE(DEIIR, de_iir);
1971 ret = IRQ_HANDLED;
b1f14ad0
JB		 1972 }
1973
f1af8fc1
PZ		 1974 if (INTEL_INFO(dev)->gen >= 6) {
1975 u32 pm_iir = I915_READ(GEN6_PMIIR);
1976 if (pm_iir) {
1403c0d4 1977 gen6_rps_irq_handler(dev_priv, pm_iir);
f1af8fc1
PZ		 1978 I915_WRITE(GEN6_PMIIR, pm_iir);
1979 ret = IRQ_HANDLED;
1980 }
0e43406b 1981 }
b1f14ad0 1982
b1f14ad0
JB		 1983 I915_WRITE(DEIER, de_ier);
1984 POSTING_READ(DEIER);
ab5c608b
BW		 1985 if (!HAS_PCH_NOP(dev)) {
1986 I915_WRITE(SDEIER, sde_ier);
1987 POSTING_READ(SDEIER);
1988 }
b1f14ad0
JB		 1989
1990 return ret;
1991}
1992
abd58f01
BW		 1993static irqreturn_t gen8_irq_handler(int irq, void *arg)
1994{
1995 struct drm_device *dev = arg;
1996 struct drm_i915_private *dev_priv = dev->dev_private;
1997 u32 master_ctl;
1998 irqreturn_t ret = IRQ_NONE;
1999 uint32_t tmp = 0;
c42664cc 2000 enum pipe pipe;
abd58f01 2001
abd58f01
BW		 2002 master_ctl = I915_READ(GEN8_MASTER_IRQ);
2003 master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
2004 if (!master_ctl)
2005 return IRQ_NONE;
2006
2007 I915_WRITE(GEN8_MASTER_IRQ, 0);
2008 POSTING_READ(GEN8_MASTER_IRQ);
2009
2010 ret = gen8_gt_irq_handler(dev, dev_priv, master_ctl);
2011
2012 if (master_ctl & GEN8_DE_MISC_IRQ) {
2013 tmp = I915_READ(GEN8_DE_MISC_IIR);
2014 if (tmp & GEN8_DE_MISC_GSE)
2015 intel_opregion_asle_intr(dev);
2016 else if (tmp)
2017 DRM_ERROR("Unexpected DE Misc interrupt\n");
2018 else
2019 DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2020
2021 if (tmp) {
2022 I915_WRITE(GEN8_DE_MISC_IIR, tmp);
2023 ret = IRQ_HANDLED;
2024 }
2025 }
2026
6d766f02
DV		 2027 if (master_ctl & GEN8_DE_PORT_IRQ) {
2028 tmp = I915_READ(GEN8_DE_PORT_IIR);
2029 if (tmp & GEN8_AUX_CHANNEL_A)
2030 dp_aux_irq_handler(dev);
2031 else if (tmp)
2032 DRM_ERROR("Unexpected DE Port interrupt\n");
2033 else
2034 DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2035
2036 if (tmp) {
2037 I915_WRITE(GEN8_DE_PORT_IIR, tmp);
2038 ret = IRQ_HANDLED;
2039 }
2040 }
2041
c42664cc
DV		 2042 for_each_pipe(pipe) {
2043 uint32_t pipe_iir;
abd58f01 2044
c42664cc
DV		 2045 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2046 continue;
abd58f01 2047
c42664cc
DV		 2048 pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
2049 if (pipe_iir & GEN8_PIPE_VBLANK)
2050 drm_handle_vblank(dev, pipe);
abd58f01 2051
c42664cc
DV		 2052 if (pipe_iir & GEN8_PIPE_FLIP_DONE) {
2053 intel_prepare_page_flip(dev, pipe);
2054 intel_finish_page_flip_plane(dev, pipe);
abd58f01 2055 }
c42664cc 2056
0fbe7870
DV		 2057 if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
2058 hsw_pipe_crc_irq_handler(dev, pipe);
2059
38d83c96
DV		 2060 if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN) {
2061 if (intel_set_cpu_fifo_underrun_reporting(dev, pipe,
2062 false))
fc2c807b
VS		 2063 DRM_ERROR("Pipe %c FIFO underrun\n",
2064 pipe_name(pipe));
38d83c96
DV		 2065 }
2066
30100f2b
DV		 2067 if (pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS) {
2068 DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
2069 pipe_name(pipe),
2070 pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
2071 }
c42664cc
DV		 2072
2073 if (pipe_iir) {
2074 ret = IRQ_HANDLED;
2075 I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
2076 } else
abd58f01
BW		 2077 DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
2078 }
2079
92d03a80
DV		 2080 if (!HAS_PCH_NOP(dev) && master_ctl & GEN8_DE_PCH_IRQ) {
2081 /*
2082 * FIXME(BDW): Assume for now that the new interrupt handling
2083 * scheme also closed the SDE interrupt handling race we've seen
2084 * on older pch-split platforms. But this needs testing.
2085 */
2086 u32 pch_iir = I915_READ(SDEIIR);
2087
2088 cpt_irq_handler(dev, pch_iir);
2089
2090 if (pch_iir) {
2091 I915_WRITE(SDEIIR, pch_iir);
2092 ret = IRQ_HANDLED;
2093 }
2094 }
2095
abd58f01
BW		 2096 I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2097 POSTING_READ(GEN8_MASTER_IRQ);
2098
2099 return ret;
2100}
2101
17e1df07
DV		 2102static void i915_error_wake_up(struct drm_i915_private *dev_priv,
2103 bool reset_completed)
2104{
2105 struct intel_ring_buffer *ring;
2106 int i;
2107
2108 /*
2109 * Notify all waiters for GPU completion events that reset state has
2110 * been changed, and that they need to restart their wait after
2111 * checking for potential errors (and bail out to drop locks if there is
2112 * a gpu reset pending so that i915_error_work_func can acquire them).
2113 */
2114
2115 /* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
2116 for_each_ring(ring, dev_priv, i)
2117 wake_up_all(&ring->irq_queue);
2118
2119 /* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
2120 wake_up_all(&dev_priv->pending_flip_queue);
2121
2122 /*
2123 * Signal tasks blocked in i915_gem_wait_for_error that the pending
2124 * reset state is cleared.
2125 */
2126 if (reset_completed)
2127 wake_up_all(&dev_priv->gpu_error.reset_queue);
2128}
2129
8a905236
JB		 2130/**
2131 * i915_error_work_func - do process context error handling work
2132 * @work: work struct
2133 *
2134 * Fire an error uevent so userspace can see that a hang or error
2135 * was detected.
2136 */
2137static void i915_error_work_func(struct work_struct *work)
2138{
1f83fee0
DV		 2139 struct i915_gpu_error *error = container_of(work, struct i915_gpu_error,
2140 work);
2d1013dd
JN		 2141 struct drm_i915_private *dev_priv =
2142 container_of(error, struct drm_i915_private, gpu_error);
8a905236 2143 struct drm_device *dev = dev_priv->dev;
cce723ed
BW		 2144 char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
2145 char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
2146 char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
17e1df07 2147 int ret;
8a905236 2148
5bdebb18 2149 kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
f316a42c 2150
7db0ba24
DV		 2151 /*
2152 * Note that there's only one work item which does gpu resets, so we
2153 * need not worry about concurrent gpu resets potentially incrementing
2154 * error->reset_counter twice. We only need to take care of another
2155 * racing irq/hangcheck declaring the gpu dead for a second time. A
2156 * quick check for that is good enough: schedule_work ensures the
2157 * correct ordering between hang detection and this work item, and since
2158 * the reset in-progress bit is only ever set by code outside of this
2159 * work we don't need to worry about any other races.
2160 */
2161 if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
f803aa55 2162 DRM_DEBUG_DRIVER("resetting chip\n");
5bdebb18 2163 kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
7db0ba24 2164 reset_event);
1f83fee0 2165
17e1df07
DV		 2166 /*
2167 * All state reset _must_ be completed before we update the
2168 * reset counter, for otherwise waiters might miss the reset
2169 * pending state and not properly drop locks, resulting in
2170 * deadlocks with the reset work.
2171 */
f69061be
DV		 2172 ret = i915_reset(dev);
2173
17e1df07
DV		 2174 intel_display_handle_reset(dev);
2175
f69061be
DV		 2176 if (ret == 0) {
2177 /*
2178 * After all the gem state is reset, increment the reset
2179 * counter and wake up everyone waiting for the reset to
2180 * complete.
2181 *
2182 * Since unlock operations are a one-sided barrier only,
2183 * we need to insert a barrier here to order any seqno
2184 * updates before
2185 * the counter increment.
2186 */
2187 smp_mb__before_atomic_inc();
2188 atomic_inc(&dev_priv->gpu_error.reset_counter);
2189
5bdebb18 2190 kobject_uevent_env(&dev->primary->kdev->kobj,
f69061be 2191 KOBJ_CHANGE, reset_done_event);
1f83fee0 2192 } else {
2ac0f450 2193 atomic_set_mask(I915_WEDGED, &error->reset_counter);
f316a42c 2194 }
1f83fee0 2195
17e1df07
DV		 2196 /*
2197 * Note: The wake_up also serves as a memory barrier so that
2198 * waiters see the update value of the reset counter atomic_t.
2199 */
2200 i915_error_wake_up(dev_priv, true);
f316a42c 2201 }
8a905236
JB		 2202}
2203
35aed2e6 2204static void i915_report_and_clear_eir(struct drm_device *dev)
8a905236
JB		 2205{
2206 struct drm_i915_private *dev_priv = dev->dev_private;
bd9854f9 2207 uint32_t instdone[I915_NUM_INSTDONE_REG];
8a905236 2208 u32 eir = I915_READ(EIR);
050ee91f 2209 int pipe, i;
8a905236 2210
35aed2e6
CW		 2211 if (!eir)
2212 return;
8a905236 2213
a70491cc 2214 pr_err("render error detected, EIR: 0x%08x\n", eir);
8a905236 2215
bd9854f9
BW		 2216 i915_get_extra_instdone(dev, instdone);
2217
8a905236
JB		 2218 if (IS_G4X(dev)) {
2219 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
2220 u32 ipeir = I915_READ(IPEIR_I965);
2221
a70491cc
JP		 2222 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2223 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
050ee91f
BW		 2224 for (i = 0; i < ARRAY_SIZE(instdone); i++)
2225 pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
a70491cc 2226 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
a70491cc 2227 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
8a905236 2228 I915_WRITE(IPEIR_I965, ipeir);
3143a2bf 2229 POSTING_READ(IPEIR_I965);
8a905236
JB		 2230 }
2231 if (eir & GM45_ERROR_PAGE_TABLE) {
2232 u32 pgtbl_err = I915_READ(PGTBL_ER);
a70491cc
JP		 2233 pr_err("page table error\n");
2234 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
8a905236 2235 I915_WRITE(PGTBL_ER, pgtbl_err);
3143a2bf 2236 POSTING_READ(PGTBL_ER);
8a905236
JB		 2237 }
2238 }
2239
a6c45cf0 2240 if (!IS_GEN2(dev)) {
8a905236
JB		 2241 if (eir & I915_ERROR_PAGE_TABLE) {
2242 u32 pgtbl_err = I915_READ(PGTBL_ER);
a70491cc
JP		 2243 pr_err("page table error\n");
2244 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
8a905236 2245 I915_WRITE(PGTBL_ER, pgtbl_err);
3143a2bf 2246 POSTING_READ(PGTBL_ER);
8a905236
JB		 2247 }
2248 }
2249
2250 if (eir & I915_ERROR_MEMORY_REFRESH) {
a70491cc 2251 pr_err("memory refresh error:\n");
9db4a9c7 2252 for_each_pipe(pipe)
a70491cc 2253 pr_err("pipe %c stat: 0x%08x\n",
9db4a9c7 2254 pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
8a905236
JB		 2255 /* pipestat has already been acked */
2256 }
2257 if (eir & I915_ERROR_INSTRUCTION) {
a70491cc
JP		 2258 pr_err("instruction error\n");
2259 pr_err(" INSTPM: 0x%08x\n", I915_READ(INSTPM));
050ee91f
BW		 2260 for (i = 0; i < ARRAY_SIZE(instdone); i++)
2261 pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
a6c45cf0 2262 if (INTEL_INFO(dev)->gen < 4) {
8a905236
JB		 2263 u32 ipeir = I915_READ(IPEIR);
2264
a70491cc
JP		 2265 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR));
2266 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR));
a70491cc 2267 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD));
8a905236 2268 I915_WRITE(IPEIR, ipeir);
3143a2bf 2269 POSTING_READ(IPEIR);
8a905236
JB		 2270 } else {
2271 u32 ipeir = I915_READ(IPEIR_I965);
2272
a70491cc
JP		 2273 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2274 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
a70491cc 2275 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
a70491cc 2276 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
8a905236 2277 I915_WRITE(IPEIR_I965, ipeir);
3143a2bf 2278 POSTING_READ(IPEIR_I965);
8a905236
JB		 2279 }
2280 }
2281
2282 I915_WRITE(EIR, eir);
3143a2bf 2283 POSTING_READ(EIR);
8a905236
JB		 2284 eir = I915_READ(EIR);
2285 if (eir) {
2286 /*
2287 * some errors might have become stuck,
2288 * mask them.
2289 */
2290 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
2291 I915_WRITE(EMR, I915_READ(EMR) | eir);
2292 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2293 }
35aed2e6
CW		 2294}
2295
2296/**
2297 * i915_handle_error - handle an error interrupt
2298 * @dev: drm device
2299 *
2300 * Do some basic checking of regsiter state at error interrupt time and
2301 * dump it to the syslog. Also call i915_capture_error_state() to make
2302 * sure we get a record and make it available in debugfs. Fire a uevent
2303 * so userspace knows something bad happened (should trigger collection
2304 * of a ring dump etc.).
2305 */
58174462
MK		 2306void i915_handle_error(struct drm_device *dev, bool wedged,
2307 const char *fmt, ...)
35aed2e6
CW		 2308{
2309 struct drm_i915_private *dev_priv = dev->dev_private;
58174462
MK		 2310 va_list args;
2311 char error_msg[80];
35aed2e6 2312
58174462
MK		 2313 va_start(args, fmt);
2314 vscnprintf(error_msg, sizeof(error_msg), fmt, args);
2315 va_end(args);
2316
2317 i915_capture_error_state(dev, wedged, error_msg);
35aed2e6 2318 i915_report_and_clear_eir(dev);
8a905236 2319
ba1234d1 2320 if (wedged) {
f69061be
DV		 2321 atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
2322 &dev_priv->gpu_error.reset_counter);
ba1234d1 2323
11ed50ec 2324 /*
17e1df07
DV		 2325 * Wakeup waiting processes so that the reset work function
2326 * i915_error_work_func doesn't deadlock trying to grab various
2327 * locks. By bumping the reset counter first, the woken
2328 * processes will see a reset in progress and back off,
2329 * releasing their locks and then wait for the reset completion.
2330 * We must do this for _all_ gpu waiters that might hold locks
2331 * that the reset work needs to acquire.
2332 *
2333 * Note: The wake_up serves as the required memory barrier to
2334 * ensure that the waiters see the updated value of the reset
2335 * counter atomic_t.
11ed50ec 2336 */
17e1df07 2337 i915_error_wake_up(dev_priv, false);
11ed50ec
BG		 2338 }
2339
122f46ba
DV		 2340 /*
2341 * Our reset work can grab modeset locks (since it needs to reset the
2342 * state of outstanding pagelips). Hence it must not be run on our own
2343 * dev-priv->wq work queue for otherwise the flush_work in the pageflip
2344 * code will deadlock.
2345 */
2346 schedule_work(&dev_priv->gpu_error.work);
8a905236
JB		 2347}
2348
21ad8330 2349static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, int pipe)
4e5359cd 2350{
2d1013dd 2351 struct drm_i915_private *dev_priv = dev->dev_private;
4e5359cd
SF		 2352 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
2353 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
05394f39 2354 struct drm_i915_gem_object *obj;
4e5359cd
SF		 2355 struct intel_unpin_work *work;
2356 unsigned long flags;
2357 bool stall_detected;
2358
2359 /* Ignore early vblank irqs */
2360 if (intel_crtc == NULL)
2361 return;
2362
2363 spin_lock_irqsave(&dev->event_lock, flags);
2364 work = intel_crtc->unpin_work;
2365
e7d841ca
CW		 2366 if (work == NULL ||
2367 atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE ||
2368 !work->enable_stall_check) {
4e5359cd
SF		 2369 /* Either the pending flip IRQ arrived, or we're too early. Don't check */
2370 spin_unlock_irqrestore(&dev->event_lock, flags);
2371 return;
2372 }
2373
2374 /* Potential stall - if we see that the flip has happened, assume a missed interrupt */
05394f39 2375 obj = work->pending_flip_obj;
a6c45cf0 2376 if (INTEL_INFO(dev)->gen >= 4) {
9db4a9c7 2377 int dspsurf = DSPSURF(intel_crtc->plane);
446f2545 2378 stall_detected = I915_HI_DISPBASE(I915_READ(dspsurf)) ==
f343c5f6 2379 i915_gem_obj_ggtt_offset(obj);
4e5359cd 2380 } else {
9db4a9c7 2381 int dspaddr = DSPADDR(intel_crtc->plane);
f343c5f6 2382 stall_detected = I915_READ(dspaddr) == (i915_gem_obj_ggtt_offset(obj) +
01f2c773 2383 crtc->y * crtc->fb->pitches[0] +
4e5359cd
SF		 2384 crtc->x * crtc->fb->bits_per_pixel/8);
2385 }
2386
2387 spin_unlock_irqrestore(&dev->event_lock, flags);
2388
2389 if (stall_detected) {
2390 DRM_DEBUG_DRIVER("Pageflip stall detected\n");
2391 intel_prepare_page_flip(dev, intel_crtc->plane);
2392 }
2393}
2394
42f52ef8
KP		 2395/* Called from drm generic code, passed 'crtc' which
2396 * we use as a pipe index
2397 */
f71d4af4 2398static int i915_enable_vblank(struct drm_device *dev, int pipe)
0a3e67a4 2399{
2d1013dd 2400 struct drm_i915_private *dev_priv = dev->dev_private;
e9d21d7f 2401 unsigned long irqflags;
71e0ffa5 2402
5eddb70b 2403 if (!i915_pipe_enabled(dev, pipe))
71e0ffa5 2404 return -EINVAL;
0a3e67a4 2405
1ec14ad3 2406 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
f796cf8f 2407 if (INTEL_INFO(dev)->gen >= 4)
7c463586 2408 i915_enable_pipestat(dev_priv, pipe,
755e9019 2409 PIPE_START_VBLANK_INTERRUPT_STATUS);
e9d21d7f 2410 else
7c463586 2411 i915_enable_pipestat(dev_priv, pipe,
755e9019 2412 PIPE_VBLANK_INTERRUPT_STATUS);
8692d00e
CW		 2413
2414 /* maintain vblank delivery even in deep C-states */
3d13ef2e 2415 if (INTEL_INFO(dev)->gen == 3)
6b26c86d 2416 I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_AGPBUSY_DIS));
1ec14ad3 2417 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
8692d00e 2418
0a3e67a4
JB		 2419 return 0;
2420}
2421
f71d4af4 2422static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
f796cf8f 2423{
2d1013dd 2424 struct drm_i915_private *dev_priv = dev->dev_private;
f796cf8f 2425 unsigned long irqflags;
b518421f 2426 uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
40da17c2 2427 DE_PIPE_VBLANK(pipe);
f796cf8f
JB		 2428
2429 if (!i915_pipe_enabled(dev, pipe))
2430 return -EINVAL;
2431
2432 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
b518421f 2433 ironlake_enable_display_irq(dev_priv, bit);
b1f14ad0
JB		 2434 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2435
2436 return 0;
2437}
2438
7e231dbe
JB		 2439static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
2440{
2d1013dd 2441 struct drm_i915_private *dev_priv = dev->dev_private;
7e231dbe 2442 unsigned long irqflags;
7e231dbe
JB		 2443
2444 if (!i915_pipe_enabled(dev, pipe))
2445 return -EINVAL;
2446
2447 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
31acc7f5 2448 i915_enable_pipestat(dev_priv, pipe,
755e9019 2449 PIPE_START_VBLANK_INTERRUPT_STATUS);
7e231dbe
JB		 2450 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2451
2452 return 0;
2453}
2454
abd58f01
BW		 2455static int gen8_enable_vblank(struct drm_device *dev, int pipe)
2456{
2457 struct drm_i915_private *dev_priv = dev->dev_private;
2458 unsigned long irqflags;
abd58f01
BW		 2459
2460 if (!i915_pipe_enabled(dev, pipe))
2461 return -EINVAL;
2462
2463 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
7167d7c6
DV		 2464 dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
2465 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2466 POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
abd58f01
BW		 2467 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2468 return 0;
2469}
2470
42f52ef8
KP		 2471/* Called from drm generic code, passed 'crtc' which
2472 * we use as a pipe index
2473 */
f71d4af4 2474static void i915_disable_vblank(struct drm_device *dev, int pipe)
0a3e67a4 2475{
2d1013dd 2476 struct drm_i915_private *dev_priv = dev->dev_private;
e9d21d7f 2477 unsigned long irqflags;
0a3e67a4 2478
1ec14ad3 2479 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3d13ef2e 2480 if (INTEL_INFO(dev)->gen == 3)
6b26c86d 2481 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_DIS));
8692d00e 2482
f796cf8f 2483 i915_disable_pipestat(dev_priv, pipe,
755e9019
ID		 2484 PIPE_VBLANK_INTERRUPT_STATUS |
2485 PIPE_START_VBLANK_INTERRUPT_STATUS);
f796cf8f
JB		 2486 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2487}
2488
f71d4af4 2489static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
f796cf8f 2490{
2d1013dd 2491 struct drm_i915_private *dev_priv = dev->dev_private;
f796cf8f 2492 unsigned long irqflags;
b518421f 2493 uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
40da17c2 2494 DE_PIPE_VBLANK(pipe);
f796cf8f
JB		 2495
2496 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
b518421f 2497 ironlake_disable_display_irq(dev_priv, bit);
b1f14ad0
JB		 2498 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2499}
2500
7e231dbe
JB		 2501static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
2502{
2d1013dd 2503 struct drm_i915_private *dev_priv = dev->dev_private;
7e231dbe 2504 unsigned long irqflags;
7e231dbe
JB		 2505
2506 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
31acc7f5 2507 i915_disable_pipestat(dev_priv, pipe,
755e9019 2508 PIPE_START_VBLANK_INTERRUPT_STATUS);
7e231dbe
JB		 2509 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2510}
2511
abd58f01
BW		 2512static void gen8_disable_vblank(struct drm_device *dev, int pipe)
2513{
2514 struct drm_i915_private *dev_priv = dev->dev_private;
2515 unsigned long irqflags;
abd58f01
BW		 2516
2517 if (!i915_pipe_enabled(dev, pipe))
2518 return;
2519
2520 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
7167d7c6
DV		 2521 dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
2522 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2523 POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
abd58f01
BW		 2524 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2525}
2526
893eead0
CW		 2527static u32
2528ring_last_seqno(struct intel_ring_buffer *ring)
852835f3 2529{
893eead0
CW		 2530 return list_entry(ring->request_list.prev,
2531 struct drm_i915_gem_request, list)->seqno;
2532}
2533
9107e9d2
CW		 2534static bool
2535ring_idle(struct intel_ring_buffer *ring, u32 seqno)
2536{
2537 return (list_empty(&ring->request_list) ||
2538 i915_seqno_passed(seqno, ring_last_seqno(ring)));
f65d9421
BG		 2539}
2540
a028c4b0
DV		 2541static bool
2542ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
2543{
2544 if (INTEL_INFO(dev)->gen >= 8) {
2545 /*
2546 * FIXME: gen8 semaphore support - currently we don't emit
2547 * semaphores on bdw anyway, but this needs to be addressed when
2548 * we merge that code.
2549 */
2550 return false;
2551 } else {
2552 ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
2553 return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
2554 MI_SEMAPHORE_REGISTER);
2555 }
2556}
2557
921d42ea
DV		 2558static struct intel_ring_buffer *
2559semaphore_wait_to_signaller_ring(struct intel_ring_buffer *ring, u32 ipehr)
2560{
2561 struct drm_i915_private *dev_priv = ring->dev->dev_private;
2562 struct intel_ring_buffer *signaller;
2563 int i;
2564
2565 if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
2566 /*
2567 * FIXME: gen8 semaphore support - currently we don't emit
2568 * semaphores on bdw anyway, but this needs to be addressed when
2569 * we merge that code.
2570 */
2571 return NULL;
2572 } else {
2573 u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
2574
2575 for_each_ring(signaller, dev_priv, i) {
2576 if(ring == signaller)
2577 continue;
2578
2579 if (sync_bits ==
2580 signaller->semaphore_register[ring->id])
2581 return signaller;
2582 }
2583 }
2584
2585 DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x\n",
2586 ring->id, ipehr);
2587
2588 return NULL;
2589}
2590
6274f212
CW		 2591static struct intel_ring_buffer *
2592semaphore_waits_for(struct intel_ring_buffer *ring, u32 *seqno)
a24a11e6
CW		 2593{
2594 struct drm_i915_private *dev_priv = ring->dev->dev_private;
88fe429d
DV		 2595 u32 cmd, ipehr, head;
2596 int i;
a24a11e6
CW		 2597
2598 ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
a028c4b0 2599 if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
6274f212 2600 return NULL;
a24a11e6 2601
88fe429d
DV		 2602 /*
2603 * HEAD is likely pointing to the dword after the actual command,
2604 * so scan backwards until we find the MBOX. But limit it to just 3
2605 * dwords. Note that we don't care about ACTHD here since that might
2606 * point at at batch, and semaphores are always emitted into the
2607 * ringbuffer itself.
a24a11e6 2608 */
88fe429d
DV		 2609 head = I915_READ_HEAD(ring) & HEAD_ADDR;
2610
2611 for (i = 4; i; --i) {
2612 /*
2613 * Be paranoid and presume the hw has gone off into the wild -
2614 * our ring is smaller than what the hardware (and hence
2615 * HEAD_ADDR) allows. Also handles wrap-around.
2616 */
2617 head &= ring->size - 1;
2618
2619 /* This here seems to blow up */
2620 cmd = ioread32(ring->virtual_start + head);
a24a11e6
CW		 2621 if (cmd == ipehr)
2622 break;
2623
88fe429d
DV		 2624 head -= 4;
2625 }
2626
2627 if (!i)
2628 return NULL;
a24a11e6 2629
88fe429d 2630 *seqno = ioread32(ring->virtual_start + head + 4) + 1;
921d42ea 2631 return semaphore_wait_to_signaller_ring(ring, ipehr);
a24a11e6
CW		 2632}
2633
6274f212
CW		 2634static int semaphore_passed(struct intel_ring_buffer *ring)
2635{
2636 struct drm_i915_private *dev_priv = ring->dev->dev_private;
2637 struct intel_ring_buffer *signaller;
2638 u32 seqno, ctl;
2639
2640 ring->hangcheck.deadlock = true;
2641
2642 signaller = semaphore_waits_for(ring, &seqno);
2643 if (signaller == NULL || signaller->hangcheck.deadlock)
2644 return -1;
2645
2646 /* cursory check for an unkickable deadlock */
2647 ctl = I915_READ_CTL(signaller);
2648 if (ctl & RING_WAIT_SEMAPHORE && semaphore_passed(signaller) < 0)
2649 return -1;
2650
2651 return i915_seqno_passed(signaller->get_seqno(signaller, false), seqno);
2652}
2653
2654static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
2655{
2656 struct intel_ring_buffer *ring;
2657 int i;
2658
2659 for_each_ring(ring, dev_priv, i)
2660 ring->hangcheck.deadlock = false;
2661}
2662
ad8beaea 2663static enum intel_ring_hangcheck_action
50877445 2664ring_stuck(struct intel_ring_buffer *ring, u64 acthd)
1ec14ad3
CW		 2665{
2666 struct drm_device *dev = ring->dev;
2667 struct drm_i915_private *dev_priv = dev->dev_private;
9107e9d2
CW		 2668 u32 tmp;
2669
6274f212 2670 if (ring->hangcheck.acthd != acthd)
f2f4d82f 2671 return HANGCHECK_ACTIVE;
6274f212 2672
9107e9d2 2673 if (IS_GEN2(dev))
f2f4d82f 2674 return HANGCHECK_HUNG;
9107e9d2
CW		 2675
2676 /* Is the chip hanging on a WAIT_FOR_EVENT?
2677 * If so we can simply poke the RB_WAIT bit
2678 * and break the hang. This should work on
2679 * all but the second generation chipsets.
2680 */
2681 tmp = I915_READ_CTL(ring);
1ec14ad3 2682 if (tmp & RING_WAIT) {
58174462
MK		 2683 i915_handle_error(dev, false,
2684 "Kicking stuck wait on %s",
2685 ring->name);
1ec14ad3 2686 I915_WRITE_CTL(ring, tmp);
f2f4d82f 2687 return HANGCHECK_KICK;
6274f212
CW		 2688 }
2689
2690 if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
2691 switch (semaphore_passed(ring)) {
2692 default:
f2f4d82f 2693 return HANGCHECK_HUNG;
6274f212 2694 case 1:
58174462
MK		 2695 i915_handle_error(dev, false,
2696 "Kicking stuck semaphore on %s",
2697 ring->name);
6274f212 2698 I915_WRITE_CTL(ring, tmp);
f2f4d82f 2699 return HANGCHECK_KICK;
6274f212 2700 case 0:
f2f4d82f 2701 return HANGCHECK_WAIT;
6274f212 2702 }
9107e9d2 2703 }
ed5cbb03 2704
f2f4d82f 2705 return HANGCHECK_HUNG;
ed5cbb03
MK		 2706}
2707
f65d9421
BG		 2708/**
2709 * This is called when the chip hasn't reported back with completed
05407ff8
MK		 2710 * batchbuffers in a long time. We keep track per ring seqno progress and
2711 * if there are no progress, hangcheck score for that ring is increased.
2712 * Further, acthd is inspected to see if the ring is stuck. On stuck case
2713 * we kick the ring. If we see no progress on three subsequent calls
2714 * we assume chip is wedged and try to fix it by resetting the chip.
f65d9421 2715 */
a658b5d2 2716static void i915_hangcheck_elapsed(unsigned long data)
f65d9421
BG		 2717{
2718 struct drm_device *dev = (struct drm_device *)data;
2d1013dd 2719 struct drm_i915_private *dev_priv = dev->dev_private;
b4519513 2720 struct intel_ring_buffer *ring;
b4519513 2721 int i;
05407ff8 2722 int busy_count = 0, rings_hung = 0;
9107e9d2
CW		 2723 bool stuck[I915_NUM_RINGS] = { 0 };
2724#define BUSY 1
2725#define KICK 5
2726#define HUNG 20
893eead0 2727
d330a953 2728 if (!i915.enable_hangcheck)
3e0dc6b0
BW		 2729 return;
2730
b4519513 2731 for_each_ring(ring, dev_priv, i) {
50877445
CW		 2732 u64 acthd;
2733 u32 seqno;
9107e9d2 2734 bool busy = true;
05407ff8 2735
6274f212
CW		 2736 semaphore_clear_deadlocks(dev_priv);
2737
05407ff8
MK		 2738 seqno = ring->get_seqno(ring, false);
2739 acthd = intel_ring_get_active_head(ring);
b4519513 2740
9107e9d2
CW		 2741 if (ring->hangcheck.seqno == seqno) {
2742 if (ring_idle(ring, seqno)) {
da661464
MK		 2743 ring->hangcheck.action = HANGCHECK_IDLE;
2744
9107e9d2
CW		 2745 if (waitqueue_active(&ring->irq_queue)) {
2746 /* Issue a wake-up to catch stuck h/w. */
094f9a54 2747 if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
f4adcd24
DV		 2748 if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
2749 DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
2750 ring->name);
2751 else
2752 DRM_INFO("Fake missed irq on %s\n",
2753 ring->name);
094f9a54
CW		 2754 wake_up_all(&ring->irq_queue);
2755 }
2756 /* Safeguard against driver failure */
2757 ring->hangcheck.score += BUSY;
9107e9d2
CW		 2758 } else
2759 busy = false;
05407ff8 2760 } else {
6274f212
CW		 2761 /* We always increment the hangcheck score
2762 * if the ring is busy and still processing
2763 * the same request, so that no single request
2764 * can run indefinitely (such as a chain of
2765 * batches). The only time we do not increment
2766 * the hangcheck score on this ring, if this
2767 * ring is in a legitimate wait for another
2768 * ring. In that case the waiting ring is a
2769 * victim and we want to be sure we catch the
2770 * right culprit. Then every time we do kick
2771 * the ring, add a small increment to the
2772 * score so that we can catch a batch that is
2773 * being repeatedly kicked and so responsible
2774 * for stalling the machine.
2775 */
ad8beaea
MK		 2776 ring->hangcheck.action = ring_stuck(ring,
2777 acthd);
2778
2779 switch (ring->hangcheck.action) {
da661464 2780 case HANGCHECK_IDLE:
f2f4d82f 2781 case HANGCHECK_WAIT:
6274f212 2782 break;
f2f4d82f 2783 case HANGCHECK_ACTIVE:
ea04cb31 2784 ring->hangcheck.score += BUSY;
6274f212 2785 break;
f2f4d82f 2786 case HANGCHECK_KICK:
ea04cb31 2787 ring->hangcheck.score += KICK;
6274f212 2788 break;
f2f4d82f 2789 case HANGCHECK_HUNG:
ea04cb31 2790 ring->hangcheck.score += HUNG;
6274f212
CW		 2791 stuck[i] = true;
2792 break;
2793 }
05407ff8 2794 }
9107e9d2 2795 } else {
da661464
MK		 2796 ring->hangcheck.action = HANGCHECK_ACTIVE;
2797
9107e9d2
CW		 2798 /* Gradually reduce the count so that we catch DoS
2799 * attempts across multiple batches.
2800 */
2801 if (ring->hangcheck.score > 0)
2802 ring->hangcheck.score--;
d1e61e7f
CW		 2803 }
2804
05407ff8
MK		 2805 ring->hangcheck.seqno = seqno;
2806 ring->hangcheck.acthd = acthd;
9107e9d2 2807 busy_count += busy;
893eead0 2808 }
b9201c14 2809
92cab734 2810 for_each_ring(ring, dev_priv, i) {
b6b0fac0 2811 if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
b8d88d1d
DV		 2812 DRM_INFO("%s on %s\n",
2813 stuck[i] ? "stuck" : "no progress",
2814 ring->name);
a43adf07 2815 rings_hung++;
92cab734
MK		 2816 }
2817 }
2818
05407ff8 2819 if (rings_hung)
58174462 2820 return i915_handle_error(dev, true, "Ring hung");
f65d9421 2821
05407ff8
MK		 2822 if (busy_count)
2823 /* Reset timer case chip hangs without another request
2824 * being added */
10cd45b6
MK		 2825 i915_queue_hangcheck(dev);
2826}
2827
2828void i915_queue_hangcheck(struct drm_device *dev)
2829{
2830 struct drm_i915_private *dev_priv = dev->dev_private;
d330a953 2831 if (!i915.enable_hangcheck)
10cd45b6
MK		 2832 return;
2833
2834 mod_timer(&dev_priv->gpu_error.hangcheck_timer,
2835 round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
f65d9421
BG		 2836}
2837
91738a95
PZ		 2838static void ibx_irq_preinstall(struct drm_device *dev)
2839{
2840 struct drm_i915_private *dev_priv = dev->dev_private;
2841
2842 if (HAS_PCH_NOP(dev))
2843 return;
2844
f86f3fb0 2845 GEN5_IRQ_RESET(SDE);
91738a95
PZ		 2846 /*
2847 * SDEIER is also touched by the interrupt handler to work around missed
2848 * PCH interrupts. Hence we can't update it after the interrupt handler
2849 * is enabled - instead we unconditionally enable all PCH interrupt
2850 * sources here, but then only unmask them as needed with SDEIMR.
2851 */
2852 I915_WRITE(SDEIER, 0xffffffff);
2853 POSTING_READ(SDEIER);
2854}
2855
d18ea1b5
DV		 2856static void gen5_gt_irq_preinstall(struct drm_device *dev)
2857{
2858 struct drm_i915_private *dev_priv = dev->dev_private;
2859
f86f3fb0 2860 GEN5_IRQ_RESET(GT);
a9d356a6 2861 if (INTEL_INFO(dev)->gen >= 6)
f86f3fb0 2862 GEN5_IRQ_RESET(GEN6_PM);
d18ea1b5
DV		 2863}
2864
1da177e4
LT		 2865/* drm_dma.h hooks
2866*/
f71d4af4 2867static void ironlake_irq_preinstall(struct drm_device *dev)
036a4a7d 2868{
2d1013dd 2869 struct drm_i915_private *dev_priv = dev->dev_private;
036a4a7d
ZW		 2870
2871 I915_WRITE(HWSTAM, 0xeffe);
bdfcdb63 2872
f86f3fb0 2873 GEN5_IRQ_RESET(DE);
036a4a7d 2874
d18ea1b5 2875 gen5_gt_irq_preinstall(dev);
c650156a 2876
91738a95 2877 ibx_irq_preinstall(dev);
7d99163d
BW		 2878}
2879
7e231dbe
JB		 2880static void valleyview_irq_preinstall(struct drm_device *dev)
2881{
2d1013dd 2882 struct drm_i915_private *dev_priv = dev->dev_private;
7e231dbe
JB		 2883 int pipe;
2884
7e231dbe
JB		 2885 /* VLV magic */
2886 I915_WRITE(VLV_IMR, 0);
2887 I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
2888 I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
2889 I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
2890
7e231dbe
JB		 2891 /* and GT */
2892 I915_WRITE(GTIIR, I915_READ(GTIIR));
2893 I915_WRITE(GTIIR, I915_READ(GTIIR));
d18ea1b5
DV		 2894
2895 gen5_gt_irq_preinstall(dev);
7e231dbe
JB		 2896
2897 I915_WRITE(DPINVGTT, 0xff);
2898
2899 I915_WRITE(PORT_HOTPLUG_EN, 0);
2900 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2901 for_each_pipe(pipe)
2902 I915_WRITE(PIPESTAT(pipe), 0xffff);
2903 I915_WRITE(VLV_IIR, 0xffffffff);
2904 I915_WRITE(VLV_IMR, 0xffffffff);
2905 I915_WRITE(VLV_IER, 0x0);
2906 POSTING_READ(VLV_IER);
2907}
2908
abd58f01
BW		 2909static void gen8_irq_preinstall(struct drm_device *dev)
2910{
2911 struct drm_i915_private *dev_priv = dev->dev_private;
2912 int pipe;
2913
abd58f01
BW		 2914 I915_WRITE(GEN8_MASTER_IRQ, 0);
2915 POSTING_READ(GEN8_MASTER_IRQ);
2916
f86f3fb0
PZ		 2917 GEN8_IRQ_RESET_NDX(GT, 0);
2918 GEN8_IRQ_RESET_NDX(GT, 1);
2919 GEN8_IRQ_RESET_NDX(GT, 2);
2920 GEN8_IRQ_RESET_NDX(GT, 3);
abd58f01
BW		 2921
2922 for_each_pipe(pipe) {
f86f3fb0 2923 GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
abd58f01
BW		 2924 }
2925
f86f3fb0
PZ		 2926 GEN5_IRQ_RESET(GEN8_DE_PORT_);
2927 GEN5_IRQ_RESET(GEN8_DE_MISC_);
2928 GEN5_IRQ_RESET(GEN8_PCU_);
09f2344d
JB		 2929
2930 ibx_irq_preinstall(dev);
abd58f01
BW		 2931}
2932
82a28bcf 2933static void ibx_hpd_irq_setup(struct drm_device *dev)
7fe0b973 2934{
2d1013dd 2935 struct drm_i915_private *dev_priv = dev->dev_private;
82a28bcf
DV		 2936 struct drm_mode_config *mode_config = &dev->mode_config;
2937 struct intel_encoder *intel_encoder;
fee884ed 2938 u32 hotplug_irqs, hotplug, enabled_irqs = 0;
82a28bcf
DV		 2939
2940 if (HAS_PCH_IBX(dev)) {
fee884ed 2941 hotplug_irqs = SDE_HOTPLUG_MASK;
82a28bcf 2942 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
cd569aed 2943 if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
fee884ed 2944 enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
82a28bcf 2945 } else {
fee884ed 2946 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
82a28bcf 2947 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
cd569aed 2948 if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
fee884ed 2949 enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
82a28bcf 2950 }
7fe0b973 2951
fee884ed 2952 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
82a28bcf
DV		 2953
2954 /*
2955 * Enable digital hotplug on the PCH, and configure the DP short pulse
2956 * duration to 2ms (which is the minimum in the Display Port spec)
2957 *
2958 * This register is the same on all known PCH chips.
2959 */
7fe0b973
KP		 2960 hotplug = I915_READ(PCH_PORT_HOTPLUG);
2961 hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
2962 hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
2963 hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
2964 hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
2965 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
2966}
2967
d46da437
PZ		 2968static void ibx_irq_postinstall(struct drm_device *dev)
2969{
2d1013dd 2970 struct drm_i915_private *dev_priv = dev->dev_private;
82a28bcf 2971 u32 mask;
e5868a31 2972
692a04cf
DV		 2973 if (HAS_PCH_NOP(dev))
2974 return;
2975
8664281b 2976 if (HAS_PCH_IBX(dev)) {
5c673b60 2977 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
8664281b 2978 } else {
5c673b60 2979 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
8664281b
PZ		 2980
2981 I915_WRITE(SERR_INT, I915_READ(SERR_INT));
2982 }
ab5c608b 2983
d46da437
PZ		 2984 I915_WRITE(SDEIIR, I915_READ(SDEIIR));
2985 I915_WRITE(SDEIMR, ~mask);
d46da437
PZ		 2986}
2987
0a9a8c91
DV		 2988static void gen5_gt_irq_postinstall(struct drm_device *dev)
2989{
2990 struct drm_i915_private *dev_priv = dev->dev_private;
2991 u32 pm_irqs, gt_irqs;
2992
2993 pm_irqs = gt_irqs = 0;
2994
2995 dev_priv->gt_irq_mask = ~0;
040d2baa 2996 if (HAS_L3_DPF(dev)) {
0a9a8c91 2997 /* L3 parity interrupt is always unmasked. */
35a85ac6
BW		 2998 dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
2999 gt_irqs |= GT_PARITY_ERROR(dev);
0a9a8c91
DV		 3000 }
3001
3002 gt_irqs |= GT_RENDER_USER_INTERRUPT;
3003 if (IS_GEN5(dev)) {
3004 gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
3005 ILK_BSD_USER_INTERRUPT;
3006 } else {
3007 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
3008 }
3009
3010 I915_WRITE(GTIIR, I915_READ(GTIIR));
3011 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
3012 I915_WRITE(GTIER, gt_irqs);
3013 POSTING_READ(GTIER);
3014
3015 if (INTEL_INFO(dev)->gen >= 6) {
a6706b45 3016 pm_irqs |= dev_priv->pm_rps_events;
0a9a8c91
DV		 3017
3018 if (HAS_VEBOX(dev))
3019 pm_irqs |= PM_VEBOX_USER_INTERRUPT;
3020
605cd25b 3021 dev_priv->pm_irq_mask = 0xffffffff;
0a9a8c91 3022 I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
605cd25b 3023 I915_WRITE(GEN6_PMIMR, dev_priv->pm_irq_mask);
0a9a8c91
DV		 3024 I915_WRITE(GEN6_PMIER, pm_irqs);
3025 POSTING_READ(GEN6_PMIER);
3026 }
3027}
3028
f71d4af4 3029static int ironlake_irq_postinstall(struct drm_device *dev)
036a4a7d 3030{
4bc9d430 3031 unsigned long irqflags;
2d1013dd 3032 struct drm_i915_private *dev_priv = dev->dev_private;
8e76f8dc
PZ		 3033 u32 display_mask, extra_mask;
3034
3035 if (INTEL_INFO(dev)->gen >= 7) {
3036 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3037 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
3038 DE_PLANEB_FLIP_DONE_IVB |
5c673b60 3039 DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
8e76f8dc 3040 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
5c673b60 3041 DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB);
8e76f8dc
PZ		 3042
3043 I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
3044 } else {
3045 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3046 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
5b3a856b 3047 DE_AUX_CHANNEL_A |
5b3a856b
DV		 3048 DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
3049 DE_POISON);
5c673b60
DV		 3050 extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
3051 DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN;
8e76f8dc 3052 }
036a4a7d 3053
1ec14ad3 3054 dev_priv->irq_mask = ~display_mask;
036a4a7d
ZW		 3055
3056 /* should always can generate irq */
3057 I915_WRITE(DEIIR, I915_READ(DEIIR));
1ec14ad3 3058 I915_WRITE(DEIMR, dev_priv->irq_mask);
8e76f8dc 3059 I915_WRITE(DEIER, display_mask | extra_mask);
3143a2bf 3060 POSTING_READ(DEIER);
036a4a7d 3061
0a9a8c91 3062 gen5_gt_irq_postinstall(dev);
036a4a7d 3063
d46da437 3064 ibx_irq_postinstall(dev);
7fe0b973 3065
f97108d1 3066 if (IS_IRONLAKE_M(dev)) {
6005ce42
DV		 3067 /* Enable PCU event interrupts
3068 *
3069 * spinlocking not required here for correctness since interrupt
4bc9d430
DV		 3070 * setup is guaranteed to run in single-threaded context. But we
3071 * need it to make the assert_spin_locked happy. */
3072 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
f97108d1 3073 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
4bc9d430 3074 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
f97108d1
JB		 3075 }
3076
036a4a7d
ZW		 3077 return 0;
3078}
3079
f8b79e58
ID		 3080static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
3081{
3082 u32 pipestat_mask;
3083 u32 iir_mask;
3084
3085 pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3086 PIPE_FIFO_UNDERRUN_STATUS;
3087
3088 I915_WRITE(PIPESTAT(PIPE_A), pipestat_mask);
3089 I915_WRITE(PIPESTAT(PIPE_B), pipestat_mask);
3090 POSTING_READ(PIPESTAT(PIPE_A));
3091
3092 pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3093 PIPE_CRC_DONE_INTERRUPT_STATUS;
3094
3095 i915_enable_pipestat(dev_priv, PIPE_A, pipestat_mask |
3096 PIPE_GMBUS_INTERRUPT_STATUS);
3097 i915_enable_pipestat(dev_priv, PIPE_B, pipestat_mask);
3098
3099 iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3100 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3101 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3102 dev_priv->irq_mask &= ~iir_mask;
3103
3104 I915_WRITE(VLV_IIR, iir_mask);
3105 I915_WRITE(VLV_IIR, iir_mask);
3106 I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3107 I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3108 POSTING_READ(VLV_IER);
3109}
3110
3111static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
3112{
3113 u32 pipestat_mask;
3114 u32 iir_mask;
3115
3116 iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3117 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
6c7fba04 3118 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
f8b79e58
ID		 3119
3120 dev_priv->irq_mask |= iir_mask;
3121 I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3122 I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3123 I915_WRITE(VLV_IIR, iir_mask);
3124 I915_WRITE(VLV_IIR, iir_mask);
3125 POSTING_READ(VLV_IIR);
3126
3127 pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3128 PIPE_CRC_DONE_INTERRUPT_STATUS;
3129
3130 i915_disable_pipestat(dev_priv, PIPE_A, pipestat_mask |
3131 PIPE_GMBUS_INTERRUPT_STATUS);
3132 i915_disable_pipestat(dev_priv, PIPE_B, pipestat_mask);
3133
3134 pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3135 PIPE_FIFO_UNDERRUN_STATUS;
3136 I915_WRITE(PIPESTAT(PIPE_A), pipestat_mask);
3137 I915_WRITE(PIPESTAT(PIPE_B), pipestat_mask);
3138 POSTING_READ(PIPESTAT(PIPE_A));
3139}
3140
3141void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3142{
3143 assert_spin_locked(&dev_priv->irq_lock);
3144
3145 if (dev_priv->display_irqs_enabled)
3146 return;
3147
3148 dev_priv->display_irqs_enabled = true;
3149
3150 if (dev_priv->dev->irq_enabled)
3151 valleyview_display_irqs_install(dev_priv);
3152}
3153
3154void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3155{
3156 assert_spin_locked(&dev_priv->irq_lock);
3157
3158 if (!dev_priv->display_irqs_enabled)
3159 return;
3160
3161 dev_priv->display_irqs_enabled = false;
3162
3163 if (dev_priv->dev->irq_enabled)
3164 valleyview_display_irqs_uninstall(dev_priv);
3165}
3166
7e231dbe
JB		 3167static int valleyview_irq_postinstall(struct drm_device *dev)
3168{
2d1013dd 3169 struct drm_i915_private *dev_priv = dev->dev_private;
b79480ba 3170 unsigned long irqflags;
7e231dbe 3171
f8b79e58 3172 dev_priv->irq_mask = ~0;
7e231dbe 3173
20afbda2
DV		 3174 I915_WRITE(PORT_HOTPLUG_EN, 0);
3175 POSTING_READ(PORT_HOTPLUG_EN);
3176
7e231dbe 3177 I915_WRITE(VLV_IMR, dev_priv->irq_mask);
f8b79e58 3178 I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
7e231dbe 3179 I915_WRITE(VLV_IIR, 0xffffffff);
7e231dbe
JB		 3180 POSTING_READ(VLV_IER);
3181
b79480ba
DV		 3182 /* Interrupt setup is already guaranteed to be single-threaded, this is
3183 * just to make the assert_spin_locked check happy. */
3184 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
f8b79e58
ID		 3185 if (dev_priv->display_irqs_enabled)
3186 valleyview_display_irqs_install(dev_priv);
b79480ba 3187 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
31acc7f5 3188
7e231dbe
JB		 3189 I915_WRITE(VLV_IIR, 0xffffffff);
3190 I915_WRITE(VLV_IIR, 0xffffffff);
3191
0a9a8c91 3192 gen5_gt_irq_postinstall(dev);
7e231dbe
JB		 3193
3194 /* ack & enable invalid PTE error interrupts */
3195#if 0 /* FIXME: add support to irq handler for checking these bits */
3196 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3197 I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
3198#endif
3199
3200 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
20afbda2
DV		 3201
3202 return 0;
3203}
3204
abd58f01
BW		 3205static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
3206{
3207 int i;
3208
3209 /* These are interrupts we'll toggle with the ring mask register */
3210 uint32_t gt_interrupts[] = {
3211 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3212 GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
3213 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3214 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3215 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3216 0,
3217 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3218 };
3219
3220 for (i = 0; i < ARRAY_SIZE(gt_interrupts); i++) {
3221 u32 tmp = I915_READ(GEN8_GT_IIR(i));
3222 if (tmp)
3223 DRM_ERROR("Interrupt (%d) should have been masked in pre-install 0x%08x\n",
3224 i, tmp);
3225 I915_WRITE(GEN8_GT_IMR(i), ~gt_interrupts[i]);
3226 I915_WRITE(GEN8_GT_IER(i), gt_interrupts[i]);
3227 }
3228 POSTING_READ(GEN8_GT_IER(0));
3229}
3230
3231static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3232{
3233 struct drm_device *dev = dev_priv->dev;
13b3a0a7
DV		 3234 uint32_t de_pipe_masked = GEN8_PIPE_FLIP_DONE |
3235 GEN8_PIPE_CDCLK_CRC_DONE |
13b3a0a7 3236 GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
5c673b60
DV		 3237 uint32_t de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
3238 GEN8_PIPE_FIFO_UNDERRUN;
abd58f01 3239 int pipe;
13b3a0a7
DV		 3240 dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
3241 dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
3242 dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
abd58f01
BW		 3243
3244 for_each_pipe(pipe) {
3245 u32 tmp = I915_READ(GEN8_DE_PIPE_IIR(pipe));
3246 if (tmp)
3247 DRM_ERROR("Interrupt (%d) should have been masked in pre-install 0x%08x\n",
3248 pipe, tmp);
3249 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
3250 I915_WRITE(GEN8_DE_PIPE_IER(pipe), de_pipe_enables);
3251 }
3252 POSTING_READ(GEN8_DE_PIPE_ISR(0));
3253
6d766f02
DV		 3254 I915_WRITE(GEN8_DE_PORT_IMR, ~GEN8_AUX_CHANNEL_A);
3255 I915_WRITE(GEN8_DE_PORT_IER, GEN8_AUX_CHANNEL_A);
abd58f01
BW		 3256 POSTING_READ(GEN8_DE_PORT_IER);
3257}
3258
3259static int gen8_irq_postinstall(struct drm_device *dev)
3260{
3261 struct drm_i915_private *dev_priv = dev->dev_private;
3262
3263 gen8_gt_irq_postinstall(dev_priv);
3264 gen8_de_irq_postinstall(dev_priv);
3265
3266 ibx_irq_postinstall(dev);
3267
3268 I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
3269 POSTING_READ(GEN8_MASTER_IRQ);
3270
3271 return 0;
3272}
3273
3274static void gen8_irq_uninstall(struct drm_device *dev)
3275{
3276 struct drm_i915_private *dev_priv = dev->dev_private;
3277 int pipe;
3278
3279 if (!dev_priv)
3280 return;
3281
abd58f01
BW		 3282 I915_WRITE(GEN8_MASTER_IRQ, 0);
3283
f86f3fb0
PZ		 3284 GEN8_IRQ_RESET_NDX(GT, 0);
3285 GEN8_IRQ_RESET_NDX(GT, 1);
3286 GEN8_IRQ_RESET_NDX(GT, 2);
3287 GEN8_IRQ_RESET_NDX(GT, 3);
abd58f01 3288
f86f3fb0
PZ		 3289 for_each_pipe(pipe)
3290 GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
abd58f01 3291
f86f3fb0
PZ		 3292 GEN5_IRQ_RESET(GEN8_DE_PORT_);
3293 GEN5_IRQ_RESET(GEN8_DE_MISC_);
3294 GEN5_IRQ_RESET(GEN8_PCU_);
abd58f01
BW		 3295}
3296
7e231dbe
JB		 3297static void valleyview_irq_uninstall(struct drm_device *dev)
3298{
2d1013dd 3299 struct drm_i915_private *dev_priv = dev->dev_private;
f8b79e58 3300 unsigned long irqflags;
7e231dbe
JB		 3301 int pipe;
3302
3303 if (!dev_priv)
3304 return;
3305
3ca1cced 3306 intel_hpd_irq_uninstall(dev_priv);
ac4c16c5 3307
7e231dbe
JB		 3308 for_each_pipe(pipe)
3309 I915_WRITE(PIPESTAT(pipe), 0xffff);
3310
3311 I915_WRITE(HWSTAM, 0xffffffff);
3312 I915_WRITE(PORT_HOTPLUG_EN, 0);
3313 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
f8b79e58
ID		 3314
3315 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3316 if (dev_priv->display_irqs_enabled)
3317 valleyview_display_irqs_uninstall(dev_priv);
3318 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3319
3320 dev_priv->irq_mask = 0;
3321
7e231dbe
JB		 3322 I915_WRITE(VLV_IIR, 0xffffffff);
3323 I915_WRITE(VLV_IMR, 0xffffffff);
3324 I915_WRITE(VLV_IER, 0x0);
3325 POSTING_READ(VLV_IER);
3326}
3327
f71d4af4 3328static void ironlake_irq_uninstall(struct drm_device *dev)
036a4a7d 3329{
2d1013dd 3330 struct drm_i915_private *dev_priv = dev->dev_private;
4697995b
JB		 3331
3332 if (!dev_priv)
3333 return;
3334
3ca1cced 3335 intel_hpd_irq_uninstall(dev_priv);
ac4c16c5 3336
036a4a7d
ZW		 3337 I915_WRITE(HWSTAM, 0xffffffff);
3338
f86f3fb0 3339 GEN5_IRQ_RESET(DE);
8664281b
PZ		 3340 if (IS_GEN7(dev))
3341 I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
036a4a7d 3342
f86f3fb0 3343 GEN5_IRQ_RESET(GT);
c71ae014 3344 if (INTEL_INFO(dev)->gen >= 6)
f86f3fb0 3345 GEN5_IRQ_RESET(GEN6_PM);
192aac1f 3346
ab5c608b
BW		 3347 if (HAS_PCH_NOP(dev))
3348 return;
3349
f86f3fb0 3350 GEN5_IRQ_RESET(SDE);
8664281b
PZ		 3351 if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
3352 I915_WRITE(SERR_INT, I915_READ(SERR_INT));
036a4a7d
ZW		 3353}
3354
a266c7d5 3355static void i8xx_irq_preinstall(struct drm_device * dev)
1da177e4 3356{
2d1013dd 3357 struct drm_i915_private *dev_priv = dev->dev_private;
9db4a9c7 3358 int pipe;
91e3738e 3359
9db4a9c7
JB		 3360 for_each_pipe(pipe)
3361 I915_WRITE(PIPESTAT(pipe), 0);
a266c7d5
CW		 3362 I915_WRITE16(IMR, 0xffff);
3363 I915_WRITE16(IER, 0x0);
3364 POSTING_READ16(IER);
c2798b19
CW		 3365}
3366
3367static int i8xx_irq_postinstall(struct drm_device *dev)
3368{
2d1013dd 3369 struct drm_i915_private *dev_priv = dev->dev_private;
379ef82d 3370 unsigned long irqflags;
c2798b19 3371
c2798b19
CW		 3372 I915_WRITE16(EMR,
3373 ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3374
3375 /* Unmask the interrupts that we always want on. */
3376 dev_priv->irq_mask =
3377 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3378 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3379 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3380 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
3381 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
3382 I915_WRITE16(IMR, dev_priv->irq_mask);
3383
3384 I915_WRITE16(IER,
3385 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3386 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3387 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
3388 I915_USER_INTERRUPT);
3389 POSTING_READ16(IER);
3390
379ef82d
DV		 3391 /* Interrupt setup is already guaranteed to be single-threaded, this is
3392 * just to make the assert_spin_locked check happy. */
3393 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
755e9019
ID		 3394 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3395 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
379ef82d
DV		 3396 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3397
c2798b19
CW		 3398 return 0;
3399}
3400
90a72f87
VS		 3401/*
3402 * Returns true when a page flip has completed.
3403 */
3404static bool i8xx_handle_vblank(struct drm_device *dev,
1f1c2e24 3405 int plane, int pipe, u32 iir)
90a72f87 3406{
2d1013dd 3407 struct drm_i915_private *dev_priv = dev->dev_private;
1f1c2e24 3408 u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
90a72f87
VS		 3409
3410 if (!drm_handle_vblank(dev, pipe))
3411 return false;
3412
3413 if ((iir & flip_pending) == 0)
3414 return false;
3415
1f1c2e24 3416 intel_prepare_page_flip(dev, plane);
90a72f87
VS		 3417
3418 /* We detect FlipDone by looking for the change in PendingFlip from '1'
3419 * to '0' on the following vblank, i.e. IIR has the Pendingflip
3420 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3421 * the flip is completed (no longer pending). Since this doesn't raise
3422 * an interrupt per se, we watch for the change at vblank.
3423 */
3424 if (I915_READ16(ISR) & flip_pending)
3425 return false;
3426
3427 intel_finish_page_flip(dev, pipe);
3428
3429 return true;
3430}
3431
ff1f525e 3432static irqreturn_t i8xx_irq_handler(int irq, void *arg)
c2798b19
CW		 3433{
3434 struct drm_device *dev = (struct drm_device *) arg;
2d1013dd 3435 struct drm_i915_private *dev_priv = dev->dev_private;
c2798b19
CW		 3436 u16 iir, new_iir;
3437 u32 pipe_stats[2];
3438 unsigned long irqflags;
c2798b19
CW		 3439 int pipe;
3440 u16 flip_mask =
3441 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3442 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3443
c2798b19
CW		 3444 iir = I915_READ16(IIR);
3445 if (iir == 0)
3446 return IRQ_NONE;
3447
3448 while (iir & ~flip_mask) {
3449 /* Can't rely on pipestat interrupt bit in iir as it might
3450 * have been cleared after the pipestat interrupt was received.
3451 * It doesn't set the bit in iir again, but it still produces
3452 * interrupts (for non-MSI).
3453 */
3454 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3455 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
58174462
MK		 3456 i915_handle_error(dev, false,
3457 "Command parser error, iir 0x%08x",
3458 iir);
c2798b19
CW		 3459
3460 for_each_pipe(pipe) {
3461 int reg = PIPESTAT(pipe);
3462 pipe_stats[pipe] = I915_READ(reg);
3463
3464 /*
3465 * Clear the PIPE*STAT regs before the IIR
3466 */
2d9d2b0b 3467 if (pipe_stats[pipe] & 0x8000ffff)
c2798b19 3468 I915_WRITE(reg, pipe_stats[pipe]);
c2798b19
CW		 3469 }
3470 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3471
3472 I915_WRITE16(IIR, iir & ~flip_mask);
3473 new_iir = I915_READ16(IIR); /* Flush posted writes */
3474
d05c617e 3475 i915_update_dri1_breadcrumb(dev);
c2798b19
CW		 3476
3477 if (iir & I915_USER_INTERRUPT)
3478 notify_ring(dev, &dev_priv->ring[RCS]);
3479
4356d586 3480 for_each_pipe(pipe) {
1f1c2e24 3481 int plane = pipe;
3a77c4c4 3482 if (HAS_FBC(dev))
1f1c2e24
VS		 3483 plane = !plane;
3484
4356d586 3485 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
1f1c2e24
VS		 3486 i8xx_handle_vblank(dev, plane, pipe, iir))
3487 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
c2798b19 3488
4356d586 3489 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
277de95e 3490 i9xx_pipe_crc_irq_handler(dev, pipe);
2d9d2b0b
VS		 3491
3492 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS &&
3493 intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
fc2c807b 3494 DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
4356d586 3495 }
c2798b19
CW		 3496
3497 iir = new_iir;
3498 }
3499
3500 return IRQ_HANDLED;
3501}
3502
3503static void i8xx_irq_uninstall(struct drm_device * dev)
3504{
2d1013dd 3505 struct drm_i915_private *dev_priv = dev->dev_private;
c2798b19
CW		 3506 int pipe;
3507
c2798b19
CW		 3508 for_each_pipe(pipe) {
3509 /* Clear enable bits; then clear status bits */
3510 I915_WRITE(PIPESTAT(pipe), 0);
3511 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3512 }
3513 I915_WRITE16(IMR, 0xffff);
3514 I915_WRITE16(IER, 0x0);
3515 I915_WRITE16(IIR, I915_READ16(IIR));
3516}
3517
a266c7d5
CW		 3518static void i915_irq_preinstall(struct drm_device * dev)
3519{
2d1013dd 3520 struct drm_i915_private *dev_priv = dev->dev_private;
a266c7d5
CW		 3521 int pipe;
3522
a266c7d5
CW		 3523 if (I915_HAS_HOTPLUG(dev)) {
3524 I915_WRITE(PORT_HOTPLUG_EN, 0);
3525 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3526 }
3527
00d98ebd 3528 I915_WRITE16(HWSTAM, 0xeffe);
a266c7d5
CW		 3529 for_each_pipe(pipe)
3530 I915_WRITE(PIPESTAT(pipe), 0);
3531 I915_WRITE(IMR, 0xffffffff);
3532 I915_WRITE(IER, 0x0);
3533 POSTING_READ(IER);
3534}
3535
3536static int i915_irq_postinstall(struct drm_device *dev)
3537{
2d1013dd 3538 struct drm_i915_private *dev_priv = dev->dev_private;
38bde180 3539 u32 enable_mask;
379ef82d 3540 unsigned long irqflags;
a266c7d5 3541
38bde180
CW		 3542 I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3543
3544 /* Unmask the interrupts that we always want on. */
3545 dev_priv->irq_mask =
3546 ~(I915_ASLE_INTERRUPT |
3547 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3548 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3549 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3550 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
3551 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
3552
3553 enable_mask =
3554 I915_ASLE_INTERRUPT |
3555 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3556 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3557 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
3558 I915_USER_INTERRUPT;
3559
a266c7d5 3560 if (I915_HAS_HOTPLUG(dev)) {
20afbda2
DV		 3561 I915_WRITE(PORT_HOTPLUG_EN, 0);
3562 POSTING_READ(PORT_HOTPLUG_EN);
3563
a266c7d5
CW		 3564 /* Enable in IER... */
3565 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
3566 /* and unmask in IMR */
3567 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
3568 }
3569
a266c7d5
CW		 3570 I915_WRITE(IMR, dev_priv->irq_mask);
3571 I915_WRITE(IER, enable_mask);
3572 POSTING_READ(IER);
3573
f49e38dd 3574 i915_enable_asle_pipestat(dev);
20afbda2 3575
379ef82d
DV		 3576 /* Interrupt setup is already guaranteed to be single-threaded, this is
3577 * just to make the assert_spin_locked check happy. */
3578 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
755e9019
ID		 3579 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3580 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
379ef82d
DV		 3581 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3582
20afbda2
DV		 3583 return 0;
3584}
3585
90a72f87
VS		 3586/*
3587 * Returns true when a page flip has completed.
3588 */
3589static bool i915_handle_vblank(struct drm_device *dev,
3590 int plane, int pipe, u32 iir)
3591{
2d1013dd 3592 struct drm_i915_private *dev_priv = dev->dev_private;
90a72f87
VS		 3593 u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3594
3595 if (!drm_handle_vblank(dev, pipe))
3596 return false;
3597
3598 if ((iir & flip_pending) == 0)
3599 return false;
3600
3601 intel_prepare_page_flip(dev, plane);
3602
3603 /* We detect FlipDone by looking for the change in PendingFlip from '1'
3604 * to '0' on the following vblank, i.e. IIR has the Pendingflip
3605 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3606 * the flip is completed (no longer pending). Since this doesn't raise
3607 * an interrupt per se, we watch for the change at vblank.
3608 */
3609 if (I915_READ(ISR) & flip_pending)
3610 return false;
3611
3612 intel_finish_page_flip(dev, pipe);
3613
3614 return true;
3615}
3616
ff1f525e 3617static irqreturn_t i915_irq_handler(int irq, void *arg)
a266c7d5
CW		 3618{
3619 struct drm_device *dev = (struct drm_device *) arg;
2d1013dd 3620 struct drm_i915_private *dev_priv = dev->dev_private;
8291ee90 3621 u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
a266c7d5 3622 unsigned long irqflags;
38bde180
CW		 3623 u32 flip_mask =
3624 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3625 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
38bde180 3626 int pipe, ret = IRQ_NONE;
a266c7d5 3627
a266c7d5 3628 iir = I915_READ(IIR);
38bde180
CW		 3629 do {
3630 bool irq_received = (iir & ~flip_mask) != 0;
8291ee90 3631 bool blc_event = false;
a266c7d5
CW		 3632
3633 /* Can't rely on pipestat interrupt bit in iir as it might
3634 * have been cleared after the pipestat interrupt was received.
3635 * It doesn't set the bit in iir again, but it still produces
3636 * interrupts (for non-MSI).
3637 */
3638 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3639 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
58174462
MK		 3640 i915_handle_error(dev, false,
3641 "Command parser error, iir 0x%08x",
3642 iir);
a266c7d5
CW		 3643
3644 for_each_pipe(pipe) {
3645 int reg = PIPESTAT(pipe);
3646 pipe_stats[pipe] = I915_READ(reg);
3647
38bde180 3648 /* Clear the PIPE*STAT regs before the IIR */
a266c7d5 3649 if (pipe_stats[pipe] & 0x8000ffff) {
a266c7d5 3650 I915_WRITE(reg, pipe_stats[pipe]);
38bde180 3651 irq_received = true;
a266c7d5
CW		 3652 }
3653 }
3654 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3655
3656 if (!irq_received)
3657 break;
3658
a266c7d5 3659 /* Consume port. Then clear IIR or we'll miss events */
16c6c56b
VS		 3660 if (I915_HAS_HOTPLUG(dev) &&
3661 iir & I915_DISPLAY_PORT_INTERRUPT)
3662 i9xx_hpd_irq_handler(dev);
a266c7d5 3663
38bde180 3664 I915_WRITE(IIR, iir & ~flip_mask);
a266c7d5
CW		 3665 new_iir = I915_READ(IIR); /* Flush posted writes */
3666
a266c7d5
CW		 3667 if (iir & I915_USER_INTERRUPT)
3668 notify_ring(dev, &dev_priv->ring[RCS]);
a266c7d5 3669
a266c7d5 3670 for_each_pipe(pipe) {
38bde180 3671 int plane = pipe;
3a77c4c4 3672 if (HAS_FBC(dev))
38bde180 3673 plane = !plane;
90a72f87 3674
8291ee90 3675 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
90a72f87
VS		 3676 i915_handle_vblank(dev, plane, pipe, iir))
3677 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
a266c7d5
CW		 3678
3679 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
3680 blc_event = true;
4356d586
DV		 3681
3682 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
277de95e 3683 i9xx_pipe_crc_irq_handler(dev, pipe);
2d9d2b0b
VS		 3684
3685 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS &&
3686 intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
fc2c807b 3687 DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
a266c7d5
CW		 3688 }
3689
a266c7d5
CW		 3690 if (blc_event || (iir & I915_ASLE_INTERRUPT))
3691 intel_opregion_asle_intr(dev);
3692
3693 /* With MSI, interrupts are only generated when iir
3694 * transitions from zero to nonzero. If another bit got
3695 * set while we were handling the existing iir bits, then
3696 * we would never get another interrupt.
3697 *
3698 * This is fine on non-MSI as well, as if we hit this path
3699 * we avoid exiting the interrupt handler only to generate
3700 * another one.
3701 *
3702 * Note that for MSI this could cause a stray interrupt report
3703 * if an interrupt landed in the time between writing IIR and
3704 * the posting read. This should be rare enough to never
3705 * trigger the 99% of 100,000 interrupts test for disabling
3706 * stray interrupts.
3707 */
38bde180 3708 ret = IRQ_HANDLED;
a266c7d5 3709 iir = new_iir;
38bde180 3710 } while (iir & ~flip_mask);
a266c7d5 3711
d05c617e 3712 i915_update_dri1_breadcrumb(dev);
8291ee90 3713
a266c7d5
CW		 3714 return ret;
3715}
3716
3717static void i915_irq_uninstall(struct drm_device * dev)
3718{
2d1013dd 3719 struct drm_i915_private *dev_priv = dev->dev_private;
a266c7d5
CW		 3720 int pipe;
3721
3ca1cced 3722 intel_hpd_irq_uninstall(dev_priv);
ac4c16c5 3723
a266c7d5
CW		 3724 if (I915_HAS_HOTPLUG(dev)) {
3725 I915_WRITE(PORT_HOTPLUG_EN, 0);
3726 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3727 }
3728
00d98ebd 3729 I915_WRITE16(HWSTAM, 0xffff);
55b39755
CW		 3730 for_each_pipe(pipe) {
3731 /* Clear enable bits; then clear status bits */
a266c7d5 3732 I915_WRITE(PIPESTAT(pipe), 0);
55b39755
CW		 3733 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3734 }
a266c7d5
CW		 3735 I915_WRITE(IMR, 0xffffffff);
3736 I915_WRITE(IER, 0x0);
3737
a266c7d5
CW		 3738 I915_WRITE(IIR, I915_READ(IIR));
3739}
3740
3741static void i965_irq_preinstall(struct drm_device * dev)
3742{
2d1013dd 3743 struct drm_i915_private *dev_priv = dev->dev_private;
a266c7d5
CW		 3744 int pipe;
3745
adca4730
CW		 3746 I915_WRITE(PORT_HOTPLUG_EN, 0);
3747 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
a266c7d5
CW		 3748
3749 I915_WRITE(HWSTAM, 0xeffe);
3750 for_each_pipe(pipe)
3751 I915_WRITE(PIPESTAT(pipe), 0);
3752 I915_WRITE(IMR, 0xffffffff);
3753 I915_WRITE(IER, 0x0);
3754 POSTING_READ(IER);
3755}
3756
3757static int i965_irq_postinstall(struct drm_device *dev)
3758{
2d1013dd 3759 struct drm_i915_private *dev_priv = dev->dev_private;
bbba0a97 3760 u32 enable_mask;
a266c7d5 3761 u32 error_mask;
b79480ba 3762 unsigned long irqflags;
a266c7d5 3763
a266c7d5 3764 /* Unmask the interrupts that we always want on. */
bbba0a97 3765 dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
adca4730 3766 I915_DISPLAY_PORT_INTERRUPT |
bbba0a97
CW		 3767 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3768 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3769 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3770 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
3771 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
3772
3773 enable_mask = ~dev_priv->irq_mask;
21ad8330
VS		 3774 enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3775 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
bbba0a97
CW		 3776 enable_mask |= I915_USER_INTERRUPT;
3777
3778 if (IS_G4X(dev))
3779 enable_mask |= I915_BSD_USER_INTERRUPT;
a266c7d5 3780
b79480ba
DV		 3781 /* Interrupt setup is already guaranteed to be single-threaded, this is
3782 * just to make the assert_spin_locked check happy. */
3783 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
755e9019
ID		 3784 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3785 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3786 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
b79480ba 3787 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
a266c7d5 3788
a266c7d5
CW		 3789 /*
3790 * Enable some error detection, note the instruction error mask
3791 * bit is reserved, so we leave it masked.
3792 */
3793 if (IS_G4X(dev)) {
3794 error_mask = ~(GM45_ERROR_PAGE_TABLE |
3795 GM45_ERROR_MEM_PRIV |
3796 GM45_ERROR_CP_PRIV |
3797 I915_ERROR_MEMORY_REFRESH);
3798 } else {
3799 error_mask = ~(I915_ERROR_PAGE_TABLE |
3800 I915_ERROR_MEMORY_REFRESH);
3801 }
3802 I915_WRITE(EMR, error_mask);
3803
3804 I915_WRITE(IMR, dev_priv->irq_mask);
3805 I915_WRITE(IER, enable_mask);
3806 POSTING_READ(IER);
3807
20afbda2
DV		 3808 I915_WRITE(PORT_HOTPLUG_EN, 0);
3809 POSTING_READ(PORT_HOTPLUG_EN);
3810
f49e38dd 3811 i915_enable_asle_pipestat(dev);
20afbda2
DV		 3812
3813 return 0;
3814}
3815
bac56d5b 3816static void i915_hpd_irq_setup(struct drm_device *dev)
20afbda2 3817{
2d1013dd 3818 struct drm_i915_private *dev_priv = dev->dev_private;
e5868a31 3819 struct drm_mode_config *mode_config = &dev->mode_config;
cd569aed 3820 struct intel_encoder *intel_encoder;
20afbda2
DV		 3821 u32 hotplug_en;
3822
b5ea2d56
DV		 3823 assert_spin_locked(&dev_priv->irq_lock);
3824
bac56d5b
EE		 3825 if (I915_HAS_HOTPLUG(dev)) {
3826 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
3827 hotplug_en &= ~HOTPLUG_INT_EN_MASK;
3828 /* Note HDMI and DP share hotplug bits */
e5868a31 3829 /* enable bits are the same for all generations */
cd569aed
EE		 3830 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
3831 if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
3832 hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
bac56d5b
EE		 3833 /* Programming the CRT detection parameters tends
3834 to generate a spurious hotplug event about three
3835 seconds later. So just do it once.
3836 */
3837 if (IS_G4X(dev))
3838 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
85fc95ba 3839 hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
bac56d5b 3840 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
a266c7d5 3841
bac56d5b
EE		 3842 /* Ignore TV since it's buggy */
3843 I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
3844 }
a266c7d5
CW		 3845}
3846
ff1f525e 3847static irqreturn_t i965_irq_handler(int irq, void *arg)
a266c7d5
CW		 3848{
3849 struct drm_device *dev = (struct drm_device *) arg;
2d1013dd 3850 struct drm_i915_private *dev_priv = dev->dev_private;
a266c7d5
CW		 3851 u32 iir, new_iir;
3852 u32 pipe_stats[I915_MAX_PIPES];
a266c7d5 3853 unsigned long irqflags;
a266c7d5 3854 int ret = IRQ_NONE, pipe;
21ad8330
VS		 3855 u32 flip_mask =
3856 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3857 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
a266c7d5 3858
a266c7d5
CW		 3859 iir = I915_READ(IIR);
3860
a266c7d5 3861 for (;;) {
501e01d7 3862 bool irq_received = (iir & ~flip_mask) != 0;
2c8ba29f
CW		 3863 bool blc_event = false;
3864
a266c7d5
CW		 3865 /* Can't rely on pipestat interrupt bit in iir as it might
3866 * have been cleared after the pipestat interrupt was received.
3867 * It doesn't set the bit in iir again, but it still produces
3868 * interrupts (for non-MSI).
3869 */
3870 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3871 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
58174462
MK		 3872 i915_handle_error(dev, false,
3873 "Command parser error, iir 0x%08x",
3874 iir);
a266c7d5
CW		 3875
3876 for_each_pipe(pipe) {
3877 int reg = PIPESTAT(pipe);
3878 pipe_stats[pipe] = I915_READ(reg);
3879
3880 /*
3881 * Clear the PIPE*STAT regs before the IIR
3882 */
3883 if (pipe_stats[pipe] & 0x8000ffff) {
a266c7d5 3884 I915_WRITE(reg, pipe_stats[pipe]);
501e01d7 3885 irq_received = true;
a266c7d5
CW		 3886 }
3887 }
3888 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3889
3890 if (!irq_received)
3891 break;
3892
3893 ret = IRQ_HANDLED;
3894
3895 /* Consume port. Then clear IIR or we'll miss events */
16c6c56b
VS		 3896 if (iir & I915_DISPLAY_PORT_INTERRUPT)
3897 i9xx_hpd_irq_handler(dev);
a266c7d5 3898
21ad8330 3899 I915_WRITE(IIR, iir & ~flip_mask);
a266c7d5
CW		 3900 new_iir = I915_READ(IIR); /* Flush posted writes */
3901
a266c7d5
CW		 3902 if (iir & I915_USER_INTERRUPT)
3903 notify_ring(dev, &dev_priv->ring[RCS]);
3904 if (iir & I915_BSD_USER_INTERRUPT)
3905 notify_ring(dev, &dev_priv->ring[VCS]);
3906
a266c7d5 3907 for_each_pipe(pipe) {
2c8ba29f 3908 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
90a72f87
VS		 3909 i915_handle_vblank(dev, pipe, pipe, iir))
3910 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
a266c7d5
CW		 3911
3912 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
3913 blc_event = true;
4356d586
DV		 3914
3915 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
277de95e 3916 i9xx_pipe_crc_irq_handler(dev, pipe);
a266c7d5 3917
2d9d2b0b
VS		 3918 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS &&
3919 intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
fc2c807b 3920 DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
2d9d2b0b 3921 }
a266c7d5
CW		 3922
3923 if (blc_event || (iir & I915_ASLE_INTERRUPT))
3924 intel_opregion_asle_intr(dev);
3925
515ac2bb
DV		 3926 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
3927 gmbus_irq_handler(dev);
3928
a266c7d5
CW		 3929 /* With MSI, interrupts are only generated when iir
3930 * transitions from zero to nonzero. If another bit got
3931 * set while we were handling the existing iir bits, then
3932 * we would never get another interrupt.
3933 *
3934 * This is fine on non-MSI as well, as if we hit this path
3935 * we avoid exiting the interrupt handler only to generate
3936 * another one.
3937 *
3938 * Note that for MSI this could cause a stray interrupt report
3939 * if an interrupt landed in the time between writing IIR and
3940 * the posting read. This should be rare enough to never
3941 * trigger the 99% of 100,000 interrupts test for disabling
3942 * stray interrupts.
3943 */
3944 iir = new_iir;
3945 }
3946
d05c617e 3947 i915_update_dri1_breadcrumb(dev);
2c8ba29f 3948
a266c7d5
CW		 3949 return ret;
3950}
3951
3952static void i965_irq_uninstall(struct drm_device * dev)
3953{
2d1013dd 3954 struct drm_i915_private *dev_priv = dev->dev_private;
a266c7d5
CW		 3955 int pipe;
3956
3957 if (!dev_priv)
3958 return;
3959
3ca1cced 3960 intel_hpd_irq_uninstall(dev_priv);
ac4c16c5 3961
adca4730
CW		 3962 I915_WRITE(PORT_HOTPLUG_EN, 0);
3963 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
a266c7d5
CW		 3964
3965 I915_WRITE(HWSTAM, 0xffffffff);
3966 for_each_pipe(pipe)
3967 I915_WRITE(PIPESTAT(pipe), 0);
3968 I915_WRITE(IMR, 0xffffffff);
3969 I915_WRITE(IER, 0x0);
3970
3971 for_each_pipe(pipe)
3972 I915_WRITE(PIPESTAT(pipe),
3973 I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
3974 I915_WRITE(IIR, I915_READ(IIR));
3975}
3976
3ca1cced 3977static void intel_hpd_irq_reenable(unsigned long data)
ac4c16c5 3978{
2d1013dd 3979 struct drm_i915_private *dev_priv = (struct drm_i915_private *)data;
ac4c16c5
EE		 3980 struct drm_device *dev = dev_priv->dev;
3981 struct drm_mode_config *mode_config = &dev->mode_config;
3982 unsigned long irqflags;
3983 int i;
3984
3985 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3986 for (i = (HPD_NONE + 1); i < HPD_NUM_PINS; i++) {
3987 struct drm_connector *connector;
3988
3989 if (dev_priv->hpd_stats[i].hpd_mark != HPD_DISABLED)
3990 continue;
3991
3992 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
3993
3994 list_for_each_entry(connector, &mode_config->connector_list, head) {
3995 struct intel_connector *intel_connector = to_intel_connector(connector);
3996
3997 if (intel_connector->encoder->hpd_pin == i) {
3998 if (connector->polled != intel_connector->polled)
3999 DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n",
4000 drm_get_connector_name(connector));
4001 connector->polled = intel_connector->polled;
4002 if (!connector->polled)
4003 connector->polled = DRM_CONNECTOR_POLL_HPD;
4004 }
4005 }
4006 }
4007 if (dev_priv->display.hpd_irq_setup)
4008 dev_priv->display.hpd_irq_setup(dev);
4009 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
4010}
4011
f71d4af4
JB		 4012void intel_irq_init(struct drm_device *dev)
4013{
8b2e326d
CW		 4014 struct drm_i915_private *dev_priv = dev->dev_private;
4015
4016 INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
99584db3 4017 INIT_WORK(&dev_priv->gpu_error.work, i915_error_work_func);
c6a828d3 4018 INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
a4da4fa4 4019 INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
8b2e326d 4020
a6706b45
D		 4021 /* Let's track the enabled rps events */
4022 dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4023
99584db3
DV		 4024 setup_timer(&dev_priv->gpu_error.hangcheck_timer,
4025 i915_hangcheck_elapsed,
61bac78e 4026 (unsigned long) dev);
3ca1cced 4027 setup_timer(&dev_priv->hotplug_reenable_timer, intel_hpd_irq_reenable,
ac4c16c5 4028 (unsigned long) dev_priv);
61bac78e 4029
97a19a24 4030 pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
9ee32fea 4031
4cdb83ec
VS		 4032 if (IS_GEN2(dev)) {
4033 dev->max_vblank_count = 0;
4034 dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
4035 } else if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
f71d4af4
JB		 4036 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4037 dev->driver->get_vblank_counter = gm45_get_vblank_counter;
391f75e2
VS		 4038 } else {
4039 dev->driver->get_vblank_counter = i915_get_vblank_counter;
4040 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
f71d4af4
JB		 4041 }
4042
c2baf4b7 4043 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
c3613de9 4044 dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
c2baf4b7
VS		 4045 dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4046 }
f71d4af4 4047
7e231dbe
JB		 4048 if (IS_VALLEYVIEW(dev)) {
4049 dev->driver->irq_handler = valleyview_irq_handler;
4050 dev->driver->irq_preinstall = valleyview_irq_preinstall;
4051 dev->driver->irq_postinstall = valleyview_irq_postinstall;
4052 dev->driver->irq_uninstall = valleyview_irq_uninstall;
4053 dev->driver->enable_vblank = valleyview_enable_vblank;
4054 dev->driver->disable_vblank = valleyview_disable_vblank;
fa00abe0 4055 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
abd58f01
BW		 4056 } else if (IS_GEN8(dev)) {
4057 dev->driver->irq_handler = gen8_irq_handler;
4058 dev->driver->irq_preinstall = gen8_irq_preinstall;
4059 dev->driver->irq_postinstall = gen8_irq_postinstall;
4060 dev->driver->irq_uninstall = gen8_irq_uninstall;
4061 dev->driver->enable_vblank = gen8_enable_vblank;
4062 dev->driver->disable_vblank = gen8_disable_vblank;
4063 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
f71d4af4
JB		 4064 } else if (HAS_PCH_SPLIT(dev)) {
4065 dev->driver->irq_handler = ironlake_irq_handler;
4066 dev->driver->irq_preinstall = ironlake_irq_preinstall;
4067 dev->driver->irq_postinstall = ironlake_irq_postinstall;
4068 dev->driver->irq_uninstall = ironlake_irq_uninstall;
4069 dev->driver->enable_vblank = ironlake_enable_vblank;
4070 dev->driver->disable_vblank = ironlake_disable_vblank;
82a28bcf 4071 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
f71d4af4 4072 } else {
c2798b19
CW		 4073 if (INTEL_INFO(dev)->gen == 2) {
4074 dev->driver->irq_preinstall = i8xx_irq_preinstall;
4075 dev->driver->irq_postinstall = i8xx_irq_postinstall;
4076 dev->driver->irq_handler = i8xx_irq_handler;
4077 dev->driver->irq_uninstall = i8xx_irq_uninstall;
a266c7d5
CW		 4078 } else if (INTEL_INFO(dev)->gen == 3) {
4079 dev->driver->irq_preinstall = i915_irq_preinstall;
4080 dev->driver->irq_postinstall = i915_irq_postinstall;
4081 dev->driver->irq_uninstall = i915_irq_uninstall;
4082 dev->driver->irq_handler = i915_irq_handler;
20afbda2 4083 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
c2798b19 4084 } else {
a266c7d5
CW		 4085 dev->driver->irq_preinstall = i965_irq_preinstall;
4086 dev->driver->irq_postinstall = i965_irq_postinstall;
4087 dev->driver->irq_uninstall = i965_irq_uninstall;
4088 dev->driver->irq_handler = i965_irq_handler;
bac56d5b 4089 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
c2798b19 4090 }
f71d4af4
JB		 4091 dev->driver->enable_vblank = i915_enable_vblank;
4092 dev->driver->disable_vblank = i915_disable_vblank;
4093 }
4094}
20afbda2
DV		 4095
4096void intel_hpd_init(struct drm_device *dev)
4097{
4098 struct drm_i915_private *dev_priv = dev->dev_private;
821450c6
EE		 4099 struct drm_mode_config *mode_config = &dev->mode_config;
4100 struct drm_connector *connector;
b5ea2d56 4101 unsigned long irqflags;
821450c6 4102 int i;
20afbda2 4103
821450c6
EE		 4104 for (i = 1; i < HPD_NUM_PINS; i++) {
4105 dev_priv->hpd_stats[i].hpd_cnt = 0;
4106 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
4107 }
4108 list_for_each_entry(connector, &mode_config->connector_list, head) {
4109 struct intel_connector *intel_connector = to_intel_connector(connector);
4110 connector->polled = intel_connector->polled;
4111 if (!connector->polled && I915_HAS_HOTPLUG(dev) && intel_connector->encoder->hpd_pin > HPD_NONE)
4112 connector->polled = DRM_CONNECTOR_POLL_HPD;
4113 }
b5ea2d56
DV		 4114
4115 /* Interrupt setup is already guaranteed to be single-threaded, this is
4116 * just to make the assert_spin_locked checks happy. */
4117 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
20afbda2
DV		 4118 if (dev_priv->display.hpd_irq_setup)
4119 dev_priv->display.hpd_irq_setup(dev);
b5ea2d56 4120 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
20afbda2 4121}
c67a470b 4122
5d584b2e
PZ		 4123/* Disable interrupts so we can allow runtime PM. */
4124void hsw_runtime_pm_disable_interrupts(struct drm_device *dev)
c67a470b
PZ		 4125{
4126 struct drm_i915_private *dev_priv = dev->dev_private;
4127 unsigned long irqflags;
4128
4129 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
4130
5d584b2e
PZ		 4131 dev_priv->pm.regsave.deimr = I915_READ(DEIMR);
4132 dev_priv->pm.regsave.sdeimr = I915_READ(SDEIMR);
4133 dev_priv->pm.regsave.gtimr = I915_READ(GTIMR);
4134 dev_priv->pm.regsave.gtier = I915_READ(GTIER);
4135 dev_priv->pm.regsave.gen6_pmimr = I915_READ(GEN6_PMIMR);
c67a470b 4136
1f2d4531
PZ		 4137 ironlake_disable_display_irq(dev_priv, 0xffffffff);
4138 ibx_disable_display_interrupt(dev_priv, 0xffffffff);
c67a470b
PZ		 4139 ilk_disable_gt_irq(dev_priv, 0xffffffff);
4140 snb_disable_pm_irq(dev_priv, 0xffffffff);
4141
5d584b2e 4142 dev_priv->pm.irqs_disabled = true;
c67a470b
PZ		 4143
4144 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
4145}
4146
5d584b2e
PZ		 4147/* Restore interrupts so we can recover from runtime PM. */
4148void hsw_runtime_pm_restore_interrupts(struct drm_device *dev)
c67a470b
PZ		 4149{
4150 struct drm_i915_private *dev_priv = dev->dev_private;
4151 unsigned long irqflags;
1f2d4531 4152 uint32_t val;
c67a470b
PZ		 4153
4154 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
4155
4156 val = I915_READ(DEIMR);
1f2d4531 4157 WARN(val != 0xffffffff, "DEIMR is 0x%08x\n", val);
c67a470b 4158
1f2d4531
PZ		 4159 val = I915_READ(SDEIMR);
4160 WARN(val != 0xffffffff, "SDEIMR is 0x%08x\n", val);
c67a470b
PZ		 4161
4162 val = I915_READ(GTIMR);
1f2d4531 4163 WARN(val != 0xffffffff, "GTIMR is 0x%08x\n", val);
c67a470b
PZ		 4164
4165 val = I915_READ(GEN6_PMIMR);
1f2d4531 4166 WARN(val != 0xffffffff, "GEN6_PMIMR is 0x%08x\n", val);
c67a470b 4167
5d584b2e 4168 dev_priv->pm.irqs_disabled = false;
c67a470b 4169
5d584b2e
PZ		 4170 ironlake_enable_display_irq(dev_priv, ~dev_priv->pm.regsave.deimr);
4171 ibx_enable_display_interrupt(dev_priv, ~dev_priv->pm.regsave.sdeimr);
4172 ilk_enable_gt_irq(dev_priv, ~dev_priv->pm.regsave.gtimr);
4173 snb_enable_pm_irq(dev_priv, ~dev_priv->pm.regsave.gen6_pmimr);
4174 I915_WRITE(GTIER, dev_priv->pm.regsave.gtier);
c67a470b
PZ		 4175
4176 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
4177}
Linux kernel - Deliverables
This page took 1.820657 seconds and 5 git commands to generate.