4 * Xen models interrupts with abstract event channels. Because each
5 * domain gets 1024 event channels, but NR_IRQ is not that large, we
6 * must dynamically map irqs<->event channels. The event channels
7 * interface with the rest of the kernel by defining a xen interrupt
8 * chip. When an event is recieved, it is mapped to an irq and sent
9 * through the normal interrupt processing path.
11 * There are four kinds of events which can be mapped to an event
14 * 1. Inter-domain notifications. This includes all the virtual
15 * device events, since they're driven by front-ends in another domain
17 * 2. VIRQs, typically used for timers. These are per-cpu events.
19 * 4. Hardware interrupts. Not supported at present.
21 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
24 #include <linux/linkage.h>
25 #include <linux/interrupt.h>
26 #include <linux/irq.h>
27 #include <linux/module.h>
28 #include <linux/string.h>
30 #include <asm/ptrace.h>
32 #include <asm/sync_bitops.h>
33 #include <asm/xen/hypercall.h>
34 #include <asm/xen/hypervisor.h>
36 #include <xen/xen-ops.h>
37 #include <xen/events.h>
38 #include <xen/interface/xen.h>
39 #include <xen/interface/event_channel.h>
42 * This lock protects updates to the following mapping and reference-count
43 * arrays. The lock does not need to be acquired to read the mapping tables.
45 static DEFINE_SPINLOCK(irq_mapping_update_lock
);
47 /* IRQ <-> VIRQ mapping. */
48 static DEFINE_PER_CPU(int, virq_to_irq
[NR_VIRQS
]) = {[0 ... NR_VIRQS
-1] = -1};
50 /* IRQ <-> IPI mapping */
51 static DEFINE_PER_CPU(int, ipi_to_irq
[XEN_NR_IPIS
]) = {[0 ... XEN_NR_IPIS
-1] = -1};
53 /* Packed IRQ information: binding type, sub-type index, and event channel. */
56 unsigned short evtchn
;
61 static struct packed_irq irq_info
[NR_IRQS
];
72 /* Convenient shorthand for packed representation of an unbound IRQ. */
73 #define IRQ_UNBOUND mk_irq_info(IRQT_UNBOUND, 0, 0)
75 static int evtchn_to_irq
[NR_EVENT_CHANNELS
] = {
76 [0 ... NR_EVENT_CHANNELS
-1] = -1
78 static unsigned long cpu_evtchn_mask
[NR_CPUS
][NR_EVENT_CHANNELS
/BITS_PER_LONG
];
79 static u8 cpu_evtchn
[NR_EVENT_CHANNELS
];
81 /* Reference counts for bindings to IRQs. */
82 static int irq_bindcount
[NR_IRQS
];
84 /* Xen will never allocate port zero for any purpose. */
85 #define VALID_EVTCHN(chn) ((chn) != 0)
87 static struct irq_chip xen_dynamic_chip
;
89 /* Constructor for packed IRQ information. */
90 static inline struct packed_irq
mk_irq_info(u32 type
, u32 index
, u32 evtchn
)
92 return (struct packed_irq
) { evtchn
, index
, type
};
96 * Accessors for packed IRQ information.
98 static inline unsigned int evtchn_from_irq(int irq
)
100 return irq_info
[irq
].evtchn
;
103 static inline unsigned int index_from_irq(int irq
)
105 return irq_info
[irq
].index
;
108 static inline unsigned int type_from_irq(int irq
)
110 return irq_info
[irq
].type
;
113 static inline unsigned long active_evtchns(unsigned int cpu
,
114 struct shared_info
*sh
,
117 return (sh
->evtchn_pending
[idx
] &
118 cpu_evtchn_mask
[cpu
][idx
] &
119 ~sh
->evtchn_mask
[idx
]);
122 static void bind_evtchn_to_cpu(unsigned int chn
, unsigned int cpu
)
124 int irq
= evtchn_to_irq
[chn
];
128 irq_desc
[irq
].affinity
= cpumask_of_cpu(cpu
);
131 __clear_bit(chn
, cpu_evtchn_mask
[cpu_evtchn
[chn
]]);
132 __set_bit(chn
, cpu_evtchn_mask
[cpu
]);
134 cpu_evtchn
[chn
] = cpu
;
137 static void init_evtchn_cpu_bindings(void)
141 /* By default all event channels notify CPU#0. */
142 for (i
= 0; i
< NR_IRQS
; i
++)
143 irq_desc
[i
].affinity
= cpumask_of_cpu(0);
146 memset(cpu_evtchn
, 0, sizeof(cpu_evtchn
));
147 memset(cpu_evtchn_mask
[0], ~0, sizeof(cpu_evtchn_mask
[0]));
150 static inline unsigned int cpu_from_evtchn(unsigned int evtchn
)
152 return cpu_evtchn
[evtchn
];
155 static inline void clear_evtchn(int port
)
157 struct shared_info
*s
= HYPERVISOR_shared_info
;
158 sync_clear_bit(port
, &s
->evtchn_pending
[0]);
161 static inline void set_evtchn(int port
)
163 struct shared_info
*s
= HYPERVISOR_shared_info
;
164 sync_set_bit(port
, &s
->evtchn_pending
[0]);
169 * notify_remote_via_irq - send event to remote end of event channel via irq
170 * @irq: irq of event channel to send event to
172 * Unlike notify_remote_via_evtchn(), this is safe to use across
173 * save/restore. Notifications on a broken connection are silently
176 void notify_remote_via_irq(int irq
)
178 int evtchn
= evtchn_from_irq(irq
);
180 if (VALID_EVTCHN(evtchn
))
181 notify_remote_via_evtchn(evtchn
);
183 EXPORT_SYMBOL_GPL(notify_remote_via_irq
);
185 static void mask_evtchn(int port
)
187 struct shared_info
*s
= HYPERVISOR_shared_info
;
188 sync_set_bit(port
, &s
->evtchn_mask
[0]);
191 static void unmask_evtchn(int port
)
193 struct shared_info
*s
= HYPERVISOR_shared_info
;
194 unsigned int cpu
= get_cpu();
196 BUG_ON(!irqs_disabled());
198 /* Slow path (hypercall) if this is a non-local port. */
199 if (unlikely(cpu
!= cpu_from_evtchn(port
))) {
200 struct evtchn_unmask unmask
= { .port
= port
};
201 (void)HYPERVISOR_event_channel_op(EVTCHNOP_unmask
, &unmask
);
203 struct vcpu_info
*vcpu_info
= __get_cpu_var(xen_vcpu
);
205 sync_clear_bit(port
, &s
->evtchn_mask
[0]);
208 * The following is basically the equivalent of
209 * 'hw_resend_irq'. Just like a real IO-APIC we 'lose
210 * the interrupt edge' if the channel is masked.
212 if (sync_test_bit(port
, &s
->evtchn_pending
[0]) &&
213 !sync_test_and_set_bit(port
/ BITS_PER_LONG
,
214 &vcpu_info
->evtchn_pending_sel
))
215 vcpu_info
->evtchn_upcall_pending
= 1;
221 static int find_unbound_irq(void)
225 /* Only allocate from dynirq range */
226 for (irq
= 0; irq
< NR_IRQS
; irq
++)
227 if (irq_bindcount
[irq
] == 0)
231 panic("No available IRQ to bind to: increase NR_IRQS!\n");
236 int bind_evtchn_to_irq(unsigned int evtchn
)
240 spin_lock(&irq_mapping_update_lock
);
242 irq
= evtchn_to_irq
[evtchn
];
245 irq
= find_unbound_irq();
247 dynamic_irq_init(irq
);
248 set_irq_chip_and_handler_name(irq
, &xen_dynamic_chip
,
249 handle_level_irq
, "event");
251 evtchn_to_irq
[evtchn
] = irq
;
252 irq_info
[irq
] = mk_irq_info(IRQT_EVTCHN
, 0, evtchn
);
255 irq_bindcount
[irq
]++;
257 spin_unlock(&irq_mapping_update_lock
);
261 EXPORT_SYMBOL_GPL(bind_evtchn_to_irq
);
263 static int bind_ipi_to_irq(unsigned int ipi
, unsigned int cpu
)
265 struct evtchn_bind_ipi bind_ipi
;
268 spin_lock(&irq_mapping_update_lock
);
270 irq
= per_cpu(ipi_to_irq
, cpu
)[ipi
];
272 irq
= find_unbound_irq();
276 dynamic_irq_init(irq
);
277 set_irq_chip_and_handler_name(irq
, &xen_dynamic_chip
,
278 handle_level_irq
, "ipi");
281 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi
,
284 evtchn
= bind_ipi
.port
;
286 evtchn_to_irq
[evtchn
] = irq
;
287 irq_info
[irq
] = mk_irq_info(IRQT_IPI
, ipi
, evtchn
);
289 per_cpu(ipi_to_irq
, cpu
)[ipi
] = irq
;
291 bind_evtchn_to_cpu(evtchn
, cpu
);
294 irq_bindcount
[irq
]++;
297 spin_unlock(&irq_mapping_update_lock
);
302 static int bind_virq_to_irq(unsigned int virq
, unsigned int cpu
)
304 struct evtchn_bind_virq bind_virq
;
307 spin_lock(&irq_mapping_update_lock
);
309 irq
= per_cpu(virq_to_irq
, cpu
)[virq
];
312 bind_virq
.virq
= virq
;
313 bind_virq
.vcpu
= cpu
;
314 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq
,
317 evtchn
= bind_virq
.port
;
319 irq
= find_unbound_irq();
321 dynamic_irq_init(irq
);
322 set_irq_chip_and_handler_name(irq
, &xen_dynamic_chip
,
323 handle_level_irq
, "virq");
325 evtchn_to_irq
[evtchn
] = irq
;
326 irq_info
[irq
] = mk_irq_info(IRQT_VIRQ
, virq
, evtchn
);
328 per_cpu(virq_to_irq
, cpu
)[virq
] = irq
;
330 bind_evtchn_to_cpu(evtchn
, cpu
);
333 irq_bindcount
[irq
]++;
335 spin_unlock(&irq_mapping_update_lock
);
340 static void unbind_from_irq(unsigned int irq
)
342 struct evtchn_close close
;
343 int evtchn
= evtchn_from_irq(irq
);
345 spin_lock(&irq_mapping_update_lock
);
347 if ((--irq_bindcount
[irq
] == 0) && VALID_EVTCHN(evtchn
)) {
349 if (HYPERVISOR_event_channel_op(EVTCHNOP_close
, &close
) != 0)
352 switch (type_from_irq(irq
)) {
354 per_cpu(virq_to_irq
, cpu_from_evtchn(evtchn
))
355 [index_from_irq(irq
)] = -1;
361 /* Closed ports are implicitly re-bound to VCPU0. */
362 bind_evtchn_to_cpu(evtchn
, 0);
364 evtchn_to_irq
[evtchn
] = -1;
365 irq_info
[irq
] = IRQ_UNBOUND
;
367 dynamic_irq_cleanup(irq
);
370 spin_unlock(&irq_mapping_update_lock
);
373 int bind_evtchn_to_irqhandler(unsigned int evtchn
,
374 irq_handler_t handler
,
375 unsigned long irqflags
,
376 const char *devname
, void *dev_id
)
381 irq
= bind_evtchn_to_irq(evtchn
);
382 retval
= request_irq(irq
, handler
, irqflags
, devname
, dev_id
);
384 unbind_from_irq(irq
);
390 EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler
);
392 int bind_virq_to_irqhandler(unsigned int virq
, unsigned int cpu
,
393 irq_handler_t handler
,
394 unsigned long irqflags
, const char *devname
, void *dev_id
)
399 irq
= bind_virq_to_irq(virq
, cpu
);
400 retval
= request_irq(irq
, handler
, irqflags
, devname
, dev_id
);
402 unbind_from_irq(irq
);
408 EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler
);
410 int bind_ipi_to_irqhandler(enum ipi_vector ipi
,
412 irq_handler_t handler
,
413 unsigned long irqflags
,
419 irq
= bind_ipi_to_irq(ipi
, cpu
);
423 retval
= request_irq(irq
, handler
, irqflags
, devname
, dev_id
);
425 unbind_from_irq(irq
);
432 void unbind_from_irqhandler(unsigned int irq
, void *dev_id
)
434 free_irq(irq
, dev_id
);
435 unbind_from_irq(irq
);
437 EXPORT_SYMBOL_GPL(unbind_from_irqhandler
);
439 void xen_send_IPI_one(unsigned int cpu
, enum ipi_vector vector
)
441 int irq
= per_cpu(ipi_to_irq
, cpu
)[vector
];
443 notify_remote_via_irq(irq
);
446 irqreturn_t
xen_debug_interrupt(int irq
, void *dev_id
)
448 struct shared_info
*sh
= HYPERVISOR_shared_info
;
449 int cpu
= smp_processor_id();
452 static DEFINE_SPINLOCK(debug_lock
);
454 spin_lock_irqsave(&debug_lock
, flags
);
456 printk("vcpu %d\n ", cpu
);
458 for_each_online_cpu(i
) {
459 struct vcpu_info
*v
= per_cpu(xen_vcpu
, i
);
460 printk("%d: masked=%d pending=%d event_sel %08lx\n ", i
,
461 (get_irq_regs() && i
== cpu
) ? xen_irqs_disabled(get_irq_regs()) : v
->evtchn_upcall_mask
,
462 v
->evtchn_upcall_pending
,
463 v
->evtchn_pending_sel
);
465 printk("pending:\n ");
466 for(i
= ARRAY_SIZE(sh
->evtchn_pending
)-1; i
>= 0; i
--)
467 printk("%08lx%s", sh
->evtchn_pending
[i
],
468 i
% 8 == 0 ? "\n " : " ");
469 printk("\nmasks:\n ");
470 for(i
= ARRAY_SIZE(sh
->evtchn_mask
)-1; i
>= 0; i
--)
471 printk("%08lx%s", sh
->evtchn_mask
[i
],
472 i
% 8 == 0 ? "\n " : " ");
474 printk("\nunmasked:\n ");
475 for(i
= ARRAY_SIZE(sh
->evtchn_mask
)-1; i
>= 0; i
--)
476 printk("%08lx%s", sh
->evtchn_pending
[i
] & ~sh
->evtchn_mask
[i
],
477 i
% 8 == 0 ? "\n " : " ");
479 printk("\npending list:\n");
480 for(i
= 0; i
< NR_EVENT_CHANNELS
; i
++) {
481 if (sync_test_bit(i
, sh
->evtchn_pending
)) {
482 printk(" %d: event %d -> irq %d\n",
488 spin_unlock_irqrestore(&debug_lock
, flags
);
495 * Search the CPUs pending events bitmasks. For each one found, map
496 * the event number to an irq, and feed it into do_IRQ() for
499 * Xen uses a two-level bitmap to speed searching. The first level is
500 * a bitset of words which contain pending event bits. The second
501 * level is a bitset of pending events themselves.
503 void xen_evtchn_do_upcall(struct pt_regs
*regs
)
506 struct shared_info
*s
= HYPERVISOR_shared_info
;
507 struct vcpu_info
*vcpu_info
= __get_cpu_var(xen_vcpu
);
508 static DEFINE_PER_CPU(unsigned, nesting_count
);
512 unsigned long pending_words
;
514 vcpu_info
->evtchn_upcall_pending
= 0;
516 if (__get_cpu_var(nesting_count
)++)
519 #ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */
520 /* Clear master flag /before/ clearing selector flag. */
523 pending_words
= xchg(&vcpu_info
->evtchn_pending_sel
, 0);
524 while (pending_words
!= 0) {
525 unsigned long pending_bits
;
526 int word_idx
= __ffs(pending_words
);
527 pending_words
&= ~(1UL << word_idx
);
529 while ((pending_bits
= active_evtchns(cpu
, s
, word_idx
)) != 0) {
530 int bit_idx
= __ffs(pending_bits
);
531 int port
= (word_idx
* BITS_PER_LONG
) + bit_idx
;
532 int irq
= evtchn_to_irq
[port
];
535 xen_do_IRQ(irq
, regs
);
539 BUG_ON(!irqs_disabled());
541 count
= __get_cpu_var(nesting_count
);
542 __get_cpu_var(nesting_count
) = 0;
549 /* Rebind a new event channel to an existing irq. */
550 void rebind_evtchn_irq(int evtchn
, int irq
)
552 /* Make sure the irq is masked, since the new event channel
553 will also be masked. */
556 spin_lock(&irq_mapping_update_lock
);
558 /* After resume the irq<->evtchn mappings are all cleared out */
559 BUG_ON(evtchn_to_irq
[evtchn
] != -1);
560 /* Expect irq to have been bound before,
561 so the bindcount should be non-0 */
562 BUG_ON(irq_bindcount
[irq
] == 0);
564 evtchn_to_irq
[evtchn
] = irq
;
565 irq_info
[irq
] = mk_irq_info(IRQT_EVTCHN
, 0, evtchn
);
567 spin_unlock(&irq_mapping_update_lock
);
569 /* new event channels are always bound to cpu 0 */
570 irq_set_affinity(irq
, cpumask_of_cpu(0));
572 /* Unmask the event channel. */
576 /* Rebind an evtchn so that it gets delivered to a specific cpu */
577 static void rebind_irq_to_cpu(unsigned irq
, unsigned tcpu
)
579 struct evtchn_bind_vcpu bind_vcpu
;
580 int evtchn
= evtchn_from_irq(irq
);
582 if (!VALID_EVTCHN(evtchn
))
585 /* Send future instances of this interrupt to other vcpu. */
586 bind_vcpu
.port
= evtchn
;
587 bind_vcpu
.vcpu
= tcpu
;
590 * If this fails, it usually just indicates that we're dealing with a
591 * virq or IPI channel, which don't actually need to be rebound. Ignore
592 * it, but don't do the xenlinux-level rebind in that case.
594 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu
, &bind_vcpu
) >= 0)
595 bind_evtchn_to_cpu(evtchn
, tcpu
);
599 static void set_affinity_irq(unsigned irq
, cpumask_t dest
)
601 unsigned tcpu
= first_cpu(dest
);
602 rebind_irq_to_cpu(irq
, tcpu
);
605 int resend_irq_on_evtchn(unsigned int irq
)
607 int masked
, evtchn
= evtchn_from_irq(irq
);
608 struct shared_info
*s
= HYPERVISOR_shared_info
;
610 if (!VALID_EVTCHN(evtchn
))
613 masked
= sync_test_and_set_bit(evtchn
, s
->evtchn_mask
);
614 sync_set_bit(evtchn
, s
->evtchn_pending
);
616 unmask_evtchn(evtchn
);
621 static void enable_dynirq(unsigned int irq
)
623 int evtchn
= evtchn_from_irq(irq
);
625 if (VALID_EVTCHN(evtchn
))
626 unmask_evtchn(evtchn
);
629 static void disable_dynirq(unsigned int irq
)
631 int evtchn
= evtchn_from_irq(irq
);
633 if (VALID_EVTCHN(evtchn
))
637 static void ack_dynirq(unsigned int irq
)
639 int evtchn
= evtchn_from_irq(irq
);
641 move_native_irq(irq
);
643 if (VALID_EVTCHN(evtchn
))
644 clear_evtchn(evtchn
);
647 static int retrigger_dynirq(unsigned int irq
)
649 int evtchn
= evtchn_from_irq(irq
);
650 struct shared_info
*sh
= HYPERVISOR_shared_info
;
653 if (VALID_EVTCHN(evtchn
)) {
656 masked
= sync_test_and_set_bit(evtchn
, sh
->evtchn_mask
);
657 sync_set_bit(evtchn
, sh
->evtchn_pending
);
659 unmask_evtchn(evtchn
);
666 static void restore_cpu_virqs(unsigned int cpu
)
668 struct evtchn_bind_virq bind_virq
;
669 int virq
, irq
, evtchn
;
671 for (virq
= 0; virq
< NR_VIRQS
; virq
++) {
672 if ((irq
= per_cpu(virq_to_irq
, cpu
)[virq
]) == -1)
675 BUG_ON(irq_info
[irq
].type
!= IRQT_VIRQ
);
676 BUG_ON(irq_info
[irq
].index
!= virq
);
678 /* Get a new binding from Xen. */
679 bind_virq
.virq
= virq
;
680 bind_virq
.vcpu
= cpu
;
681 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq
,
684 evtchn
= bind_virq
.port
;
686 /* Record the new mapping. */
687 evtchn_to_irq
[evtchn
] = irq
;
688 irq_info
[irq
] = mk_irq_info(IRQT_VIRQ
, virq
, evtchn
);
689 bind_evtchn_to_cpu(evtchn
, cpu
);
692 unmask_evtchn(evtchn
);
696 static void restore_cpu_ipis(unsigned int cpu
)
698 struct evtchn_bind_ipi bind_ipi
;
699 int ipi
, irq
, evtchn
;
701 for (ipi
= 0; ipi
< XEN_NR_IPIS
; ipi
++) {
702 if ((irq
= per_cpu(ipi_to_irq
, cpu
)[ipi
]) == -1)
705 BUG_ON(irq_info
[irq
].type
!= IRQT_IPI
);
706 BUG_ON(irq_info
[irq
].index
!= ipi
);
708 /* Get a new binding from Xen. */
710 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi
,
713 evtchn
= bind_ipi
.port
;
715 /* Record the new mapping. */
716 evtchn_to_irq
[evtchn
] = irq
;
717 irq_info
[irq
] = mk_irq_info(IRQT_IPI
, ipi
, evtchn
);
718 bind_evtchn_to_cpu(evtchn
, cpu
);
721 unmask_evtchn(evtchn
);
726 /* Clear an irq's pending state, in preparation for polling on it */
727 void xen_clear_irq_pending(int irq
)
729 int evtchn
= evtchn_from_irq(irq
);
731 if (VALID_EVTCHN(evtchn
))
732 clear_evtchn(evtchn
);
735 /* Poll waiting for an irq to become pending. In the usual case, the
736 irq will be disabled so it won't deliver an interrupt. */
737 void xen_poll_irq(int irq
)
739 evtchn_port_t evtchn
= evtchn_from_irq(irq
);
741 if (VALID_EVTCHN(evtchn
)) {
742 struct sched_poll poll
;
746 poll
.ports
= &evtchn
;
748 if (HYPERVISOR_sched_op(SCHEDOP_poll
, &poll
) != 0)
753 void xen_irq_resume(void)
755 unsigned int cpu
, irq
, evtchn
;
757 init_evtchn_cpu_bindings();
759 /* New event-channel space is not 'live' yet. */
760 for (evtchn
= 0; evtchn
< NR_EVENT_CHANNELS
; evtchn
++)
763 /* No IRQ <-> event-channel mappings. */
764 for (irq
= 0; irq
< NR_IRQS
; irq
++)
765 irq_info
[irq
].evtchn
= 0; /* zap event-channel binding */
767 for (evtchn
= 0; evtchn
< NR_EVENT_CHANNELS
; evtchn
++)
768 evtchn_to_irq
[evtchn
] = -1;
770 for_each_possible_cpu(cpu
) {
771 restore_cpu_virqs(cpu
);
772 restore_cpu_ipis(cpu
);
776 static struct irq_chip xen_dynamic_chip __read_mostly
= {
778 .mask
= disable_dynirq
,
779 .unmask
= enable_dynirq
,
781 .set_affinity
= set_affinity_irq
,
782 .retrigger
= retrigger_dynirq
,
785 void __init
xen_init_IRQ(void)
789 init_evtchn_cpu_bindings();
791 /* No event channels are 'live' right now. */
792 for (i
= 0; i
< NR_EVENT_CHANNELS
; i
++)
795 /* Dynamic IRQ space is currently unbound. Zero the refcnts. */
796 for (i
= 0; i
< NR_IRQS
; i
++)
797 irq_bindcount
[i
] = 0;
799 irq_ctx_init(smp_processor_id());