projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
x86/irq: Store irq descriptor in vector array
[deliverable/linux.git] / arch / x86 / kernel / irq.c
1 /*
2 * Common interrupt code for 32 and 64 bit
3 */
4 #include <linux/cpu.h>
5 #include <linux/interrupt.h>
6 #include <linux/kernel_stat.h>
7 #include <linux/of.h>
8 #include <linux/seq_file.h>
9 #include <linux/smp.h>
10 #include <linux/ftrace.h>
11 #include <linux/delay.h>
12 #include <linux/export.h>
13
14 #include <asm/apic.h>
15 #include <asm/io_apic.h>
16 #include <asm/irq.h>
17 #include <asm/idle.h>
18 #include <asm/mce.h>
19 #include <asm/hw_irq.h>
20 #include <asm/desc.h>
21
22 #define CREATE_TRACE_POINTS
23 #include <asm/trace/irq_vectors.h>
24
25 DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
26 EXPORT_PER_CPU_SYMBOL(irq_stat);
27
28 DEFINE_PER_CPU(struct pt_regs *, irq_regs);
29 EXPORT_PER_CPU_SYMBOL(irq_regs);
30
31 atomic_t irq_err_count;
32
33 /* Function pointer for generic interrupt vector handling */
34 void (*x86_platform_ipi_callback)(void) = NULL;
35
36 /*
37 * 'what should we do if we get a hw irq event on an illegal vector'.
38 * each architecture has to answer this themselves.
39 */
40 void ack_bad_irq(unsigned int irq)
41 {
42 if (printk_ratelimit())
43 pr_err("unexpected IRQ trap at vector %02x\n", irq);
44
45 /*
46 * Currently unexpected vectors happen only on SMP and APIC.
47 * We _must_ ack these because every local APIC has only N
48 * irq slots per priority level, and a 'hanging, unacked' IRQ
49 * holds up an irq slot - in excessive cases (when multiple
50 * unexpected vectors occur) that might lock up the APIC
51 * completely.
52 * But only ack when the APIC is enabled -AK
53 */
54 ack_APIC_irq();
55 }
56
57 #define irq_stats(x) (&per_cpu(irq_stat, x))
58 /*
59 * /proc/interrupts printing for arch specific interrupts
60 */
61 int arch_show_interrupts(struct seq_file *p, int prec)
62 {
63 int j;
64
65 seq_printf(p, "%*s: ", prec, "NMI");
66 for_each_online_cpu(j)
67 seq_printf(p, "%10u ", irq_stats(j)->__nmi_count);
68 seq_puts(p, " Non-maskable interrupts\n");
69 #ifdef CONFIG_X86_LOCAL_APIC
70 seq_printf(p, "%*s: ", prec, "LOC");
71 for_each_online_cpu(j)
72 seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs);
73 seq_puts(p, " Local timer interrupts\n");
74
75 seq_printf(p, "%*s: ", prec, "SPU");
76 for_each_online_cpu(j)
77 seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count);
78 seq_puts(p, " Spurious interrupts\n");
79 seq_printf(p, "%*s: ", prec, "PMI");
80 for_each_online_cpu(j)
81 seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
82 seq_puts(p, " Performance monitoring interrupts\n");
83 seq_printf(p, "%*s: ", prec, "IWI");
84 for_each_online_cpu(j)
85 seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
86 seq_puts(p, " IRQ work interrupts\n");
87 seq_printf(p, "%*s: ", prec, "RTR");
88 for_each_online_cpu(j)
89 seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count);
90 seq_puts(p, " APIC ICR read retries\n");
91 #endif
92 if (x86_platform_ipi_callback) {
93 seq_printf(p, "%*s: ", prec, "PLT");
94 for_each_online_cpu(j)
95 seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis);
96 seq_puts(p, " Platform interrupts\n");
97 }
98 #ifdef CONFIG_SMP
99 seq_printf(p, "%*s: ", prec, "RES");
100 for_each_online_cpu(j)
101 seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count);
102 seq_puts(p, " Rescheduling interrupts\n");
103 seq_printf(p, "%*s: ", prec, "CAL");
104 for_each_online_cpu(j)
105 seq_printf(p, "%10u ", irq_stats(j)->irq_call_count -
106 irq_stats(j)->irq_tlb_count);
107 seq_puts(p, " Function call interrupts\n");
108 seq_printf(p, "%*s: ", prec, "TLB");
109 for_each_online_cpu(j)
110 seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count);
111 seq_puts(p, " TLB shootdowns\n");
112 #endif
113 #ifdef CONFIG_X86_THERMAL_VECTOR
114 seq_printf(p, "%*s: ", prec, "TRM");
115 for_each_online_cpu(j)
116 seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count);
117 seq_puts(p, " Thermal event interrupts\n");
118 #endif
119 #ifdef CONFIG_X86_MCE_THRESHOLD
120 seq_printf(p, "%*s: ", prec, "THR");
121 for_each_online_cpu(j)
122 seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count);
123 seq_puts(p, " Threshold APIC interrupts\n");
124 #endif
125 #ifdef CONFIG_X86_MCE_AMD
126 seq_printf(p, "%*s: ", prec, "DFR");
127 for_each_online_cpu(j)
128 seq_printf(p, "%10u ", irq_stats(j)->irq_deferred_error_count);
129 seq_puts(p, " Deferred Error APIC interrupts\n");
130 #endif
131 #ifdef CONFIG_X86_MCE
132 seq_printf(p, "%*s: ", prec, "MCE");
133 for_each_online_cpu(j)
134 seq_printf(p, "%10u ", per_cpu(mce_exception_count, j));
135 seq_puts(p, " Machine check exceptions\n");
136 seq_printf(p, "%*s: ", prec, "MCP");
137 for_each_online_cpu(j)
138 seq_printf(p, "%10u ", per_cpu(mce_poll_count, j));
139 seq_puts(p, " Machine check polls\n");
140 #endif
141 #if IS_ENABLED(CONFIG_HYPERV) || defined(CONFIG_XEN)
142 seq_printf(p, "%*s: ", prec, "HYP");
143 for_each_online_cpu(j)
144 seq_printf(p, "%10u ", irq_stats(j)->irq_hv_callback_count);
145 seq_puts(p, " Hypervisor callback interrupts\n");
146 #endif
147 seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
148 #if defined(CONFIG_X86_IO_APIC)
149 seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
150 #endif
151 #ifdef CONFIG_HAVE_KVM
152 seq_printf(p, "%*s: ", prec, "PIN");
153 for_each_online_cpu(j)
154 seq_printf(p, "%10u ", irq_stats(j)->kvm_posted_intr_ipis);
155 seq_puts(p, " Posted-interrupt notification event\n");
156
157 seq_printf(p, "%*s: ", prec, "PIW");
158 for_each_online_cpu(j)
159 seq_printf(p, "%10u ",
160 irq_stats(j)->kvm_posted_intr_wakeup_ipis);
161 seq_puts(p, " Posted-interrupt wakeup event\n");
162 #endif
163 return 0;
164 }
165
166 /*
167 * /proc/stat helpers
168 */
169 u64 arch_irq_stat_cpu(unsigned int cpu)
170 {
171 u64 sum = irq_stats(cpu)->__nmi_count;
172
173 #ifdef CONFIG_X86_LOCAL_APIC
174 sum += irq_stats(cpu)->apic_timer_irqs;
175 sum += irq_stats(cpu)->irq_spurious_count;
176 sum += irq_stats(cpu)->apic_perf_irqs;
177 sum += irq_stats(cpu)->apic_irq_work_irqs;
178 sum += irq_stats(cpu)->icr_read_retry_count;
179 #endif
180 if (x86_platform_ipi_callback)
181 sum += irq_stats(cpu)->x86_platform_ipis;
182 #ifdef CONFIG_SMP
183 sum += irq_stats(cpu)->irq_resched_count;
184 sum += irq_stats(cpu)->irq_call_count;
185 #endif
186 #ifdef CONFIG_X86_THERMAL_VECTOR
187 sum += irq_stats(cpu)->irq_thermal_count;
188 #endif
189 #ifdef CONFIG_X86_MCE_THRESHOLD
190 sum += irq_stats(cpu)->irq_threshold_count;
191 #endif
192 #ifdef CONFIG_X86_MCE
193 sum += per_cpu(mce_exception_count, cpu);
194 sum += per_cpu(mce_poll_count, cpu);
195 #endif
196 return sum;
197 }
198
199 u64 arch_irq_stat(void)
200 {
201 u64 sum = atomic_read(&irq_err_count);
202 return sum;
203 }
204
205
206 /*
207 * do_IRQ handles all normal device IRQ's (the special
208 * SMP cross-CPU interrupts have their own specific
209 * handlers).
210 */
211 __visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
212 {
213 struct pt_regs *old_regs = set_irq_regs(regs);
214 struct irq_desc * desc;
215 /* high bit used in ret_from_ code */
216 unsigned vector = ~regs->orig_ax;
217
218 entering_irq();
219
220 desc = __this_cpu_read(vector_irq[vector]);
221
222 if (!handle_irq(desc, regs)) {
223 ack_APIC_irq();
224
225 if (desc != VECTOR_RETRIGGERED) {
226 pr_emerg_ratelimited("%s: %d.%d No irq handler for vector\n",
227 __func__, smp_processor_id(),
228 vector);
229 } else {
230 __this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
231 }
232 }
233
234 exiting_irq();
235
236 set_irq_regs(old_regs);
237 return 1;
238 }
239
240 /*
241 * Handler for X86_PLATFORM_IPI_VECTOR.
242 */
243 void __smp_x86_platform_ipi(void)
244 {
245 inc_irq_stat(x86_platform_ipis);
246
247 if (x86_platform_ipi_callback)
248 x86_platform_ipi_callback();
249 }
250
251 __visible void smp_x86_platform_ipi(struct pt_regs *regs)
252 {
253 struct pt_regs *old_regs = set_irq_regs(regs);
254
255 entering_ack_irq();
256 __smp_x86_platform_ipi();
257 exiting_irq();
258 set_irq_regs(old_regs);
259 }
260
261 #ifdef CONFIG_HAVE_KVM
262 static void dummy_handler(void) {}
263 static void (*kvm_posted_intr_wakeup_handler)(void) = dummy_handler;
264
265 void kvm_set_posted_intr_wakeup_handler(void (*handler)(void))
266 {
267 if (handler)
268 kvm_posted_intr_wakeup_handler = handler;
269 else
270 kvm_posted_intr_wakeup_handler = dummy_handler;
271 }
272 EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler);
273
274 /*
275 * Handler for POSTED_INTERRUPT_VECTOR.
276 */
277 __visible void smp_kvm_posted_intr_ipi(struct pt_regs *regs)
278 {
279 struct pt_regs *old_regs = set_irq_regs(regs);
280
281 entering_ack_irq();
282 inc_irq_stat(kvm_posted_intr_ipis);
283 exiting_irq();
284 set_irq_regs(old_regs);
285 }
286
287 /*
288 * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
289 */
290 __visible void smp_kvm_posted_intr_wakeup_ipi(struct pt_regs *regs)
291 {
292 struct pt_regs *old_regs = set_irq_regs(regs);
293
294 entering_ack_irq();
295 inc_irq_stat(kvm_posted_intr_wakeup_ipis);
296 kvm_posted_intr_wakeup_handler();
297 exiting_irq();
298 set_irq_regs(old_regs);
299 }
300 #endif
301
302 __visible void smp_trace_x86_platform_ipi(struct pt_regs *regs)
303 {
304 struct pt_regs *old_regs = set_irq_regs(regs);
305
306 entering_ack_irq();
307 trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
308 __smp_x86_platform_ipi();
309 trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR);
310 exiting_irq();
311 set_irq_regs(old_regs);
312 }
313
314 EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq);
315
316 #ifdef CONFIG_HOTPLUG_CPU
317
318 /* These two declarations are only used in check_irq_vectors_for_cpu_disable()
319 * below, which is protected by stop_machine(). Putting them on the stack
320 * results in a stack frame overflow. Dynamically allocating could result in a
321 * failure so declare these two cpumasks as global.
322 */
323 static struct cpumask affinity_new, online_new;
324
325 /*
326 * This cpu is going to be removed and its vectors migrated to the remaining
327 * online cpus. Check to see if there are enough vectors in the remaining cpus.
328 * This function is protected by stop_machine().
329 */
330 int check_irq_vectors_for_cpu_disable(void)
331 {
332 unsigned int this_cpu, vector, this_count, count;
333 struct irq_desc *desc;
334 struct irq_data *data;
335 int cpu;
336
337 this_cpu = smp_processor_id();
338 cpumask_copy(&online_new, cpu_online_mask);
339 cpumask_clear_cpu(this_cpu, &online_new);
340
341 this_count = 0;
342 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
343 desc = __this_cpu_read(vector_irq[vector]);
344 if (IS_ERR_OR_NULL(desc))
345 continue;
346 /*
347 * Protect against concurrent action removal, affinity
348 * changes etc.
349 */
350 raw_spin_lock(&desc->lock);
351 data = irq_desc_get_irq_data(desc);
352 cpumask_copy(&affinity_new,
353 irq_data_get_affinity_mask(data));
354 cpumask_clear_cpu(this_cpu, &affinity_new);
355
356 /* Do not count inactive or per-cpu irqs. */
357 if (!irq_desc_has_action(desc) || irqd_is_per_cpu(data)) {
358 raw_spin_unlock(&desc->lock);
359 continue;
360 }
361
362 raw_spin_unlock(&desc->lock);
363 /*
364 * A single irq may be mapped to multiple cpu's
365 * vector_irq[] (for example IOAPIC cluster mode). In
366 * this case we have two possibilities:
367 *
368 * 1) the resulting affinity mask is empty; that is
369 * this the down'd cpu is the last cpu in the irq's
370 * affinity mask, or
371 *
372 * 2) the resulting affinity mask is no longer a
373 * subset of the online cpus but the affinity mask is
374 * not zero; that is the down'd cpu is the last online
375 * cpu in a user set affinity mask.
376 */
377 if (cpumask_empty(&affinity_new) ||
378 !cpumask_subset(&affinity_new, &online_new))
379 this_count++;
380 }
381
382 count = 0;
383 for_each_online_cpu(cpu) {
384 if (cpu == this_cpu)
385 continue;
386 /*
387 * We scan from FIRST_EXTERNAL_VECTOR to first system
388 * vector. If the vector is marked in the used vectors
389 * bitmap or an irq is assigned to it, we don't count
390 * it as available.
391 *
392 * As this is an inaccurate snapshot anyway, we can do
393 * this w/o holding vector_lock.
394 */
395 for (vector = FIRST_EXTERNAL_VECTOR;
396 vector < first_system_vector; vector++) {
397 if (!test_bit(vector, used_vectors) &&
398 IS_ERR_OR_NULL(per_cpu(vector_irq, cpu)[vector]))
399 count++;
400 }
401 }
402
403 if (count < this_count) {
404 pr_warn("CPU %d disable failed: CPU has %u vectors assigned and there are only %u available.\n",
405 this_cpu, this_count, count);
406 return -ERANGE;
407 }
408 return 0;
409 }
410
411 /* A cpu has been removed from cpu_online_mask. Reset irq affinities. */
412 void fixup_irqs(void)
413 {
414 unsigned int irq, vector;
415 static int warned;
416 struct irq_desc *desc;
417 struct irq_data *data;
418 struct irq_chip *chip;
419 int ret;
420
421 for_each_irq_desc(irq, desc) {
422 int break_affinity = 0;
423 int set_affinity = 1;
424 const struct cpumask *affinity;
425
426 if (!desc)
427 continue;
428 if (irq == 2)
429 continue;
430
431 /* interrupt's are disabled at this point */
432 raw_spin_lock(&desc->lock);
433
434 data = irq_desc_get_irq_data(desc);
435 affinity = irq_data_get_affinity_mask(data);
436 if (!irq_has_action(irq) || irqd_is_per_cpu(data) ||
437 cpumask_subset(affinity, cpu_online_mask)) {
438 raw_spin_unlock(&desc->lock);
439 continue;
440 }
441
442 /*
443 * Complete the irq move. This cpu is going down and for
444 * non intr-remapping case, we can't wait till this interrupt
445 * arrives at this cpu before completing the irq move.
446 */
447 irq_force_complete_move(irq);
448
449 if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) {
450 break_affinity = 1;
451 affinity = cpu_online_mask;
452 }
453
454 chip = irq_data_get_irq_chip(data);
455 if (!irqd_can_move_in_process_context(data) && chip->irq_mask)
456 chip->irq_mask(data);
457
458 if (chip->irq_set_affinity) {
459 ret = chip->irq_set_affinity(data, affinity, true);
460 if (ret == -ENOSPC)
461 pr_crit("IRQ %d set affinity failed because there are no available vectors. The device assigned to this IRQ is unstable.\n", irq);
462 } else {
463 if (!(warned++))
464 set_affinity = 0;
465 }
466
467 /*
468 * We unmask if the irq was not marked masked by the
469 * core code. That respects the lazy irq disable
470 * behaviour.
471 */
472 if (!irqd_can_move_in_process_context(data) &&
473 !irqd_irq_masked(data) && chip->irq_unmask)
474 chip->irq_unmask(data);
475
476 raw_spin_unlock(&desc->lock);
477
478 if (break_affinity && set_affinity)
479 pr_notice("Broke affinity for irq %i\n", irq);
480 else if (!set_affinity)
481 pr_notice("Cannot set affinity for irq %i\n", irq);
482 }
483
484 /*
485 * We can remove mdelay() and then send spuriuous interrupts to
486 * new cpu targets for all the irqs that were handled previously by
487 * this cpu. While it works, I have seen spurious interrupt messages
488 * (nothing wrong but still...).
489 *
490 * So for now, retain mdelay(1) and check the IRR and then send those
491 * interrupts to new targets as this cpu is already offlined...
492 */
493 mdelay(1);
494
495 /*
496 * We can walk the vector array of this cpu without holding
497 * vector_lock because the cpu is already marked !online, so
498 * nothing else will touch it.
499 */
500 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
501 unsigned int irr;
502
503 if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector])))
504 continue;
505
506 irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
507 if (irr & (1 << (vector % 32))) {
508 desc = __this_cpu_read(vector_irq[vector]);
509
510 raw_spin_lock(&desc->lock);
511 data = irq_desc_get_irq_data(desc);
512 chip = irq_data_get_irq_chip(data);
513 if (chip->irq_retrigger) {
514 chip->irq_retrigger(data);
515 __this_cpu_write(vector_irq[vector], VECTOR_RETRIGGERED);
516 }
517 raw_spin_unlock(&desc->lock);
518 }
519 if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED)
520 __this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
521 }
522 }
523 #endif
Linux kernel - Deliverables
This page took 0.056256 seconds and 6 git commands to generate.