2 * linux/kernel/time/tick-broadcast.c
4 * This file contains functions which emulate a local clock-event
5 * device via a broadcast event source.
7 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
8 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
9 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
11 * This code is licenced under the GPL version 2. For details see
12 * kernel-base/COPYING.
14 #include <linux/cpu.h>
15 #include <linux/err.h>
16 #include <linux/hrtimer.h>
17 #include <linux/interrupt.h>
18 #include <linux/percpu.h>
19 #include <linux/profile.h>
20 #include <linux/sched.h>
21 #include <linux/smp.h>
23 #include "tick-internal.h"
26 * Broadcast support for broken x86 hardware, where the local apic
27 * timer stops in C3 state.
30 static struct tick_device tick_broadcast_device
;
31 /* FIXME: Use cpumask_var_t. */
32 static DECLARE_BITMAP(tick_broadcast_mask
, NR_CPUS
);
33 static DECLARE_BITMAP(tmpmask
, NR_CPUS
);
34 static DEFINE_RAW_SPINLOCK(tick_broadcast_lock
);
35 static int tick_broadcast_force
;
37 #ifdef CONFIG_TICK_ONESHOT
38 static void tick_broadcast_clear_oneshot(int cpu
);
40 static inline void tick_broadcast_clear_oneshot(int cpu
) { }
44 * Debugging: see timer_list.c
46 struct tick_device
*tick_get_broadcast_device(void)
48 return &tick_broadcast_device
;
51 struct cpumask
*tick_get_broadcast_mask(void)
53 return to_cpumask(tick_broadcast_mask
);
57 * Start the device in periodic mode
59 static void tick_broadcast_start_periodic(struct clock_event_device
*bc
)
62 tick_setup_periodic(bc
, 1);
66 * Check, if the device can be utilized as broadcast device:
68 int tick_check_broadcast_device(struct clock_event_device
*dev
)
70 if ((dev
->features
& CLOCK_EVT_FEAT_DUMMY
) ||
71 (tick_broadcast_device
.evtdev
&&
72 tick_broadcast_device
.evtdev
->rating
>= dev
->rating
) ||
73 (dev
->features
& CLOCK_EVT_FEAT_C3STOP
))
76 clockevents_exchange_device(tick_broadcast_device
.evtdev
, dev
);
77 tick_broadcast_device
.evtdev
= dev
;
78 if (!cpumask_empty(tick_get_broadcast_mask()))
79 tick_broadcast_start_periodic(dev
);
84 * Check, if the device is the broadcast device
86 int tick_is_broadcast_device(struct clock_event_device
*dev
)
88 return (dev
&& tick_broadcast_device
.evtdev
== dev
);
91 static void err_broadcast(const struct cpumask
*mask
)
93 pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n");
96 static void tick_device_setup_broadcast_func(struct clock_event_device
*dev
)
99 dev
->broadcast
= tick_broadcast
;
100 if (!dev
->broadcast
) {
101 pr_warn_once("%s depends on broadcast, but no broadcast function available\n",
103 dev
->broadcast
= err_broadcast
;
108 * Check, if the device is disfunctional and a place holder, which
109 * needs to be handled by the broadcast device.
111 int tick_device_uses_broadcast(struct clock_event_device
*dev
, int cpu
)
116 raw_spin_lock_irqsave(&tick_broadcast_lock
, flags
);
119 * Devices might be registered with both periodic and oneshot
120 * mode disabled. This signals, that the device needs to be
121 * operated from the broadcast device and is a placeholder for
122 * the cpu local device.
124 if (!tick_device_is_functional(dev
)) {
125 dev
->event_handler
= tick_handle_periodic
;
126 tick_device_setup_broadcast_func(dev
);
127 cpumask_set_cpu(cpu
, tick_get_broadcast_mask());
128 tick_broadcast_start_periodic(tick_broadcast_device
.evtdev
);
132 * When the new device is not affected by the stop
133 * feature and the cpu is marked in the broadcast mask
134 * then clear the broadcast bit.
136 if (!(dev
->features
& CLOCK_EVT_FEAT_C3STOP
)) {
137 int cpu
= smp_processor_id();
138 cpumask_clear_cpu(cpu
, tick_get_broadcast_mask());
139 tick_broadcast_clear_oneshot(cpu
);
141 tick_device_setup_broadcast_func(dev
);
144 raw_spin_unlock_irqrestore(&tick_broadcast_lock
, flags
);
148 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
149 int tick_receive_broadcast(void)
151 struct tick_device
*td
= this_cpu_ptr(&tick_cpu_device
);
152 struct clock_event_device
*evt
= td
->evtdev
;
157 if (!evt
->event_handler
)
160 evt
->event_handler(evt
);
166 * Broadcast the event to the cpus, which are set in the mask (mangled).
168 static void tick_do_broadcast(struct cpumask
*mask
)
170 int cpu
= smp_processor_id();
171 struct tick_device
*td
;
174 * Check, if the current cpu is in the mask
176 if (cpumask_test_cpu(cpu
, mask
)) {
177 cpumask_clear_cpu(cpu
, mask
);
178 td
= &per_cpu(tick_cpu_device
, cpu
);
179 td
->evtdev
->event_handler(td
->evtdev
);
182 if (!cpumask_empty(mask
)) {
184 * It might be necessary to actually check whether the devices
185 * have different broadcast functions. For now, just use the
186 * one of the first device. This works as long as we have this
187 * misfeature only on x86 (lapic)
189 td
= &per_cpu(tick_cpu_device
, cpumask_first(mask
));
190 td
->evtdev
->broadcast(mask
);
195 * Periodic broadcast:
196 * - invoke the broadcast handlers
198 static void tick_do_periodic_broadcast(void)
200 raw_spin_lock(&tick_broadcast_lock
);
202 cpumask_and(to_cpumask(tmpmask
),
203 cpu_online_mask
, tick_get_broadcast_mask());
204 tick_do_broadcast(to_cpumask(tmpmask
));
206 raw_spin_unlock(&tick_broadcast_lock
);
210 * Event handler for periodic broadcast ticks
212 static void tick_handle_periodic_broadcast(struct clock_event_device
*dev
)
216 tick_do_periodic_broadcast();
219 * The device is in periodic mode. No reprogramming necessary:
221 if (dev
->mode
== CLOCK_EVT_MODE_PERIODIC
)
225 * Setup the next period for devices, which do not have
226 * periodic mode. We read dev->next_event first and add to it
227 * when the event already expired. clockevents_program_event()
228 * sets dev->next_event only when the event is really
229 * programmed to the device.
231 for (next
= dev
->next_event
; ;) {
232 next
= ktime_add(next
, tick_period
);
234 if (!clockevents_program_event(dev
, next
, false))
236 tick_do_periodic_broadcast();
241 * Powerstate information: The system enters/leaves a state, where
242 * affected devices might stop
244 static void tick_do_broadcast_on_off(unsigned long *reason
)
246 struct clock_event_device
*bc
, *dev
;
247 struct tick_device
*td
;
251 raw_spin_lock_irqsave(&tick_broadcast_lock
, flags
);
253 cpu
= smp_processor_id();
254 td
= &per_cpu(tick_cpu_device
, cpu
);
256 bc
= tick_broadcast_device
.evtdev
;
259 * Is the device not affected by the powerstate ?
261 if (!dev
|| !(dev
->features
& CLOCK_EVT_FEAT_C3STOP
))
264 if (!tick_device_is_functional(dev
))
267 bc_stopped
= cpumask_empty(tick_get_broadcast_mask());
270 case CLOCK_EVT_NOTIFY_BROADCAST_ON
:
271 case CLOCK_EVT_NOTIFY_BROADCAST_FORCE
:
272 if (!cpumask_test_cpu(cpu
, tick_get_broadcast_mask())) {
273 cpumask_set_cpu(cpu
, tick_get_broadcast_mask());
274 if (tick_broadcast_device
.mode
==
275 TICKDEV_MODE_PERIODIC
)
276 clockevents_shutdown(dev
);
278 if (*reason
== CLOCK_EVT_NOTIFY_BROADCAST_FORCE
)
279 tick_broadcast_force
= 1;
281 case CLOCK_EVT_NOTIFY_BROADCAST_OFF
:
282 if (!tick_broadcast_force
&&
283 cpumask_test_cpu(cpu
, tick_get_broadcast_mask())) {
284 cpumask_clear_cpu(cpu
, tick_get_broadcast_mask());
285 if (tick_broadcast_device
.mode
==
286 TICKDEV_MODE_PERIODIC
)
287 tick_setup_periodic(dev
, 0);
292 if (cpumask_empty(tick_get_broadcast_mask())) {
294 clockevents_shutdown(bc
);
295 } else if (bc_stopped
) {
296 if (tick_broadcast_device
.mode
== TICKDEV_MODE_PERIODIC
)
297 tick_broadcast_start_periodic(bc
);
299 tick_broadcast_setup_oneshot(bc
);
302 raw_spin_unlock_irqrestore(&tick_broadcast_lock
, flags
);
306 * Powerstate information: The system enters/leaves a state, where
307 * affected devices might stop.
309 void tick_broadcast_on_off(unsigned long reason
, int *oncpu
)
311 if (!cpumask_test_cpu(*oncpu
, cpu_online_mask
))
312 printk(KERN_ERR
"tick-broadcast: ignoring broadcast for "
313 "offline CPU #%d\n", *oncpu
);
315 tick_do_broadcast_on_off(&reason
);
319 * Set the periodic handler depending on broadcast on/off
321 void tick_set_periodic_handler(struct clock_event_device
*dev
, int broadcast
)
324 dev
->event_handler
= tick_handle_periodic
;
326 dev
->event_handler
= tick_handle_periodic_broadcast
;
330 * Remove a CPU from broadcasting
332 void tick_shutdown_broadcast(unsigned int *cpup
)
334 struct clock_event_device
*bc
;
336 unsigned int cpu
= *cpup
;
338 raw_spin_lock_irqsave(&tick_broadcast_lock
, flags
);
340 bc
= tick_broadcast_device
.evtdev
;
341 cpumask_clear_cpu(cpu
, tick_get_broadcast_mask());
343 if (tick_broadcast_device
.mode
== TICKDEV_MODE_PERIODIC
) {
344 if (bc
&& cpumask_empty(tick_get_broadcast_mask()))
345 clockevents_shutdown(bc
);
348 raw_spin_unlock_irqrestore(&tick_broadcast_lock
, flags
);
351 void tick_suspend_broadcast(void)
353 struct clock_event_device
*bc
;
356 raw_spin_lock_irqsave(&tick_broadcast_lock
, flags
);
358 bc
= tick_broadcast_device
.evtdev
;
360 clockevents_shutdown(bc
);
362 raw_spin_unlock_irqrestore(&tick_broadcast_lock
, flags
);
365 int tick_resume_broadcast(void)
367 struct clock_event_device
*bc
;
371 raw_spin_lock_irqsave(&tick_broadcast_lock
, flags
);
373 bc
= tick_broadcast_device
.evtdev
;
376 clockevents_set_mode(bc
, CLOCK_EVT_MODE_RESUME
);
378 switch (tick_broadcast_device
.mode
) {
379 case TICKDEV_MODE_PERIODIC
:
380 if (!cpumask_empty(tick_get_broadcast_mask()))
381 tick_broadcast_start_periodic(bc
);
382 broadcast
= cpumask_test_cpu(smp_processor_id(),
383 tick_get_broadcast_mask());
385 case TICKDEV_MODE_ONESHOT
:
386 if (!cpumask_empty(tick_get_broadcast_mask()))
387 broadcast
= tick_resume_broadcast_oneshot(bc
);
391 raw_spin_unlock_irqrestore(&tick_broadcast_lock
, flags
);
397 #ifdef CONFIG_TICK_ONESHOT
399 /* FIXME: use cpumask_var_t. */
400 static DECLARE_BITMAP(tick_broadcast_oneshot_mask
, NR_CPUS
);
403 * Exposed for debugging: see timer_list.c
405 struct cpumask
*tick_get_broadcast_oneshot_mask(void)
407 return to_cpumask(tick_broadcast_oneshot_mask
);
410 static int tick_broadcast_set_event(ktime_t expires
, int force
)
412 struct clock_event_device
*bc
= tick_broadcast_device
.evtdev
;
414 if (bc
->mode
!= CLOCK_EVT_MODE_ONESHOT
)
415 clockevents_set_mode(bc
, CLOCK_EVT_MODE_ONESHOT
);
417 return clockevents_program_event(bc
, expires
, force
);
420 int tick_resume_broadcast_oneshot(struct clock_event_device
*bc
)
422 clockevents_set_mode(bc
, CLOCK_EVT_MODE_ONESHOT
);
427 * Called from irq_enter() when idle was interrupted to reenable the
430 void tick_check_oneshot_broadcast(int cpu
)
432 if (cpumask_test_cpu(cpu
, to_cpumask(tick_broadcast_oneshot_mask
))) {
433 struct tick_device
*td
= &per_cpu(tick_cpu_device
, cpu
);
435 clockevents_set_mode(td
->evtdev
, CLOCK_EVT_MODE_ONESHOT
);
440 * Handle oneshot mode broadcasting
442 static void tick_handle_oneshot_broadcast(struct clock_event_device
*dev
)
444 struct tick_device
*td
;
445 ktime_t now
, next_event
;
448 raw_spin_lock(&tick_broadcast_lock
);
450 dev
->next_event
.tv64
= KTIME_MAX
;
451 next_event
.tv64
= KTIME_MAX
;
452 cpumask_clear(to_cpumask(tmpmask
));
454 /* Find all expired events */
455 for_each_cpu(cpu
, tick_get_broadcast_oneshot_mask()) {
456 td
= &per_cpu(tick_cpu_device
, cpu
);
457 if (td
->evtdev
->next_event
.tv64
<= now
.tv64
)
458 cpumask_set_cpu(cpu
, to_cpumask(tmpmask
));
459 else if (td
->evtdev
->next_event
.tv64
< next_event
.tv64
)
460 next_event
.tv64
= td
->evtdev
->next_event
.tv64
;
464 * Wakeup the cpus which have an expired event.
466 tick_do_broadcast(to_cpumask(tmpmask
));
469 * Two reasons for reprogram:
471 * - The global event did not expire any CPU local
472 * events. This happens in dyntick mode, as the maximum PIT
473 * delta is quite small.
475 * - There are pending events on sleeping CPUs which were not
478 if (next_event
.tv64
!= KTIME_MAX
) {
480 * Rearm the broadcast device. If event expired,
483 if (tick_broadcast_set_event(next_event
, 0))
486 raw_spin_unlock(&tick_broadcast_lock
);
490 * Powerstate information: The system enters/leaves a state, where
491 * affected devices might stop
493 void tick_broadcast_oneshot_control(unsigned long reason
)
495 struct clock_event_device
*bc
, *dev
;
496 struct tick_device
*td
;
501 * Periodic mode does not care about the enter/exit of power
504 if (tick_broadcast_device
.mode
== TICKDEV_MODE_PERIODIC
)
508 * We are called with preemtion disabled from the depth of the
509 * idle code, so we can't be moved away.
511 cpu
= smp_processor_id();
512 td
= &per_cpu(tick_cpu_device
, cpu
);
515 if (!(dev
->features
& CLOCK_EVT_FEAT_C3STOP
))
518 bc
= tick_broadcast_device
.evtdev
;
520 raw_spin_lock_irqsave(&tick_broadcast_lock
, flags
);
521 if (reason
== CLOCK_EVT_NOTIFY_BROADCAST_ENTER
) {
522 if (!cpumask_test_cpu(cpu
, tick_get_broadcast_oneshot_mask())) {
523 cpumask_set_cpu(cpu
, tick_get_broadcast_oneshot_mask());
524 clockevents_set_mode(dev
, CLOCK_EVT_MODE_SHUTDOWN
);
525 if (dev
->next_event
.tv64
< bc
->next_event
.tv64
)
526 tick_broadcast_set_event(dev
->next_event
, 1);
529 if (cpumask_test_cpu(cpu
, tick_get_broadcast_oneshot_mask())) {
530 cpumask_clear_cpu(cpu
,
531 tick_get_broadcast_oneshot_mask());
532 clockevents_set_mode(dev
, CLOCK_EVT_MODE_ONESHOT
);
533 if (dev
->next_event
.tv64
!= KTIME_MAX
)
534 tick_program_event(dev
->next_event
, 1);
537 raw_spin_unlock_irqrestore(&tick_broadcast_lock
, flags
);
541 * Reset the one shot broadcast for a cpu
543 * Called with tick_broadcast_lock held
545 static void tick_broadcast_clear_oneshot(int cpu
)
547 cpumask_clear_cpu(cpu
, tick_get_broadcast_oneshot_mask());
550 static void tick_broadcast_init_next_event(struct cpumask
*mask
,
553 struct tick_device
*td
;
556 for_each_cpu(cpu
, mask
) {
557 td
= &per_cpu(tick_cpu_device
, cpu
);
559 td
->evtdev
->next_event
= expires
;
564 * tick_broadcast_setup_oneshot - setup the broadcast device
566 void tick_broadcast_setup_oneshot(struct clock_event_device
*bc
)
568 int cpu
= smp_processor_id();
570 /* Set it up only once ! */
571 if (bc
->event_handler
!= tick_handle_oneshot_broadcast
) {
572 int was_periodic
= bc
->mode
== CLOCK_EVT_MODE_PERIODIC
;
574 bc
->event_handler
= tick_handle_oneshot_broadcast
;
576 /* Take the do_timer update */
577 tick_do_timer_cpu
= cpu
;
580 * We must be careful here. There might be other CPUs
581 * waiting for periodic broadcast. We need to set the
582 * oneshot_mask bits for those and program the
583 * broadcast device to fire.
585 cpumask_copy(to_cpumask(tmpmask
), tick_get_broadcast_mask());
586 cpumask_clear_cpu(cpu
, to_cpumask(tmpmask
));
587 cpumask_or(tick_get_broadcast_oneshot_mask(),
588 tick_get_broadcast_oneshot_mask(),
589 to_cpumask(tmpmask
));
591 if (was_periodic
&& !cpumask_empty(to_cpumask(tmpmask
))) {
592 clockevents_set_mode(bc
, CLOCK_EVT_MODE_ONESHOT
);
593 tick_broadcast_init_next_event(to_cpumask(tmpmask
),
595 tick_broadcast_set_event(tick_next_period
, 1);
597 bc
->next_event
.tv64
= KTIME_MAX
;
600 * The first cpu which switches to oneshot mode sets
601 * the bit for all other cpus which are in the general
602 * (periodic) broadcast mask. So the bit is set and
603 * would prevent the first broadcast enter after this
604 * to program the bc device.
606 tick_broadcast_clear_oneshot(cpu
);
611 * Select oneshot operating mode for the broadcast device
613 void tick_broadcast_switch_to_oneshot(void)
615 struct clock_event_device
*bc
;
618 raw_spin_lock_irqsave(&tick_broadcast_lock
, flags
);
620 tick_broadcast_device
.mode
= TICKDEV_MODE_ONESHOT
;
621 bc
= tick_broadcast_device
.evtdev
;
623 tick_broadcast_setup_oneshot(bc
);
625 raw_spin_unlock_irqrestore(&tick_broadcast_lock
, flags
);
630 * Remove a dead CPU from broadcasting
632 void tick_shutdown_broadcast_oneshot(unsigned int *cpup
)
635 unsigned int cpu
= *cpup
;
637 raw_spin_lock_irqsave(&tick_broadcast_lock
, flags
);
640 * Clear the broadcast mask flag for the dead cpu, but do not
641 * stop the broadcast device!
643 cpumask_clear_cpu(cpu
, tick_get_broadcast_oneshot_mask());
645 raw_spin_unlock_irqrestore(&tick_broadcast_lock
, flags
);
649 * Check, whether the broadcast device is in one shot mode
651 int tick_broadcast_oneshot_active(void)
653 return tick_broadcast_device
.mode
== TICKDEV_MODE_ONESHOT
;
657 * Check whether the broadcast device supports oneshot.
659 bool tick_broadcast_oneshot_available(void)
661 struct clock_event_device
*bc
= tick_broadcast_device
.evtdev
;
663 return bc
? bc
->features
& CLOCK_EVT_FEAT_ONESHOT
: false;