[deliverable/linux.git] / kernel / softirq.c

/*
 *	linux/kernel/softirq.c
 *
 *	Copyright (C) 1992 Linus Torvalds
 *
 *	Distribute under GPLv2.
 *
 *	Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
 */

#include <linux/module.h>
#include <linux/kernel_stat.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/notifier.h>
#include <linux/percpu.h>
#include <linux/cpu.h>
#include <linux/freezer.h>
#include <linux/kthread.h>
#include <linux/rcupdate.h>
#include <linux/smp.h>
#include <linux/tick.h>

#include <asm/irq.h>
/*
   - No shared variables, all the data are CPU local.
   - If a softirq needs serialization, let it serialize itself
     by its own spinlocks.
   - Even if softirq is serialized, only local cpu is marked for
     execution. Hence, we get something sort of weak cpu binding.
     Though it is still not clear, will it result in better locality
     or will not.

   Examples:
   - NET RX softirq. It is multithreaded and does not require
     any global serialization.
   - NET TX softirq. It kicks software netdevice queues, hence
     it is logically serialized per device, but this serialization
     is invisible to common code.
   - Tasklets: serialized wrt itself.
 */

#ifndef __ARCH_IRQ_STAT
irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
EXPORT_SYMBOL(irq_stat);
#endif

static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;

static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);

/*
 * we cannot loop indefinitely here to avoid userspace starvation,
 * but we also don't want to introduce a worst case 1/HZ latency
 * to the pending events, so lets the scheduler to balance
 * the softirq load for us.
 */
static inline void wakeup_softirqd(void)
{
	/* Interrupts are disabled: no need to stop preemption */
	struct task_struct *tsk = __get_cpu_var(ksoftirqd);

	if (tsk && tsk->state != TASK_RUNNING)
		wake_up_process(tsk);
}

/*
 * This one is for softirq.c-internal use,
 * where hardirqs are disabled legitimately:
 */
#ifdef CONFIG_TRACE_IRQFLAGS
static void __local_bh_disable(unsigned long ip)
{
	unsigned long flags;

	WARN_ON_ONCE(in_irq());

	raw_local_irq_save(flags);
	add_preempt_count(SOFTIRQ_OFFSET);
	/*
	 * Were softirqs turned off above:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_off(ip);
	raw_local_irq_restore(flags);
}
#else /* !CONFIG_TRACE_IRQFLAGS */
static inline void __local_bh_disable(unsigned long ip)
{
	add_preempt_count(SOFTIRQ_OFFSET);
	barrier();
}
#endif /* CONFIG_TRACE_IRQFLAGS */

void local_bh_disable(void)
{
	__local_bh_disable((unsigned long)__builtin_return_address(0));
}

EXPORT_SYMBOL(local_bh_disable);

void __local_bh_enable(void)
{
	WARN_ON_ONCE(in_irq());

	/*
	 * softirqs should never be enabled by __local_bh_enable(),
	 * it always nests inside local_bh_enable() sections:
	 */
	WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);

	sub_preempt_count(SOFTIRQ_OFFSET);
}
EXPORT_SYMBOL_GPL(__local_bh_enable);

/*
 * Special-case - softirqs can safely be enabled in
 * cond_resched_softirq(), or by __do_softirq(),
 * without processing still-pending softirqs:
 */
void _local_bh_enable(void)
{
	WARN_ON_ONCE(in_irq());
	WARN_ON_ONCE(!irqs_disabled());

	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on((unsigned long)__builtin_return_address(0));
	sub_preempt_count(SOFTIRQ_OFFSET);
}

EXPORT_SYMBOL(_local_bh_enable);

static inline void _local_bh_enable_ip(unsigned long ip)
{
	WARN_ON_ONCE(in_irq() || irqs_disabled());
#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_disable();
#endif
	/*
	 * Are softirqs going to be turned on now:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on(ip);
	/*
	 * Keep preemption disabled until we are done with
	 * softirq processing:
 	 */
 	sub_preempt_count(SOFTIRQ_OFFSET - 1);

	if (unlikely(!in_interrupt() && local_softirq_pending()))
		do_softirq();

	dec_preempt_count();
#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_enable();
#endif
	preempt_check_resched();
}

void local_bh_enable(void)
{
	_local_bh_enable_ip((unsigned long)__builtin_return_address(0));
}
EXPORT_SYMBOL(local_bh_enable);

void local_bh_enable_ip(unsigned long ip)
{
	_local_bh_enable_ip(ip);
}
EXPORT_SYMBOL(local_bh_enable_ip);

/*
 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 * and we fall back to softirqd after that.
 *
 * This number has been established via experimentation.
 * The two things to balance is latency against fairness -
 * we want to handle softirqs as soon as possible, but they
 * should not be able to lock up the box.
 */
#define MAX_SOFTIRQ_RESTART 10

asmlinkage void __do_softirq(void)
{
	struct softirq_action *h;
	__u32 pending;
	int max_restart = MAX_SOFTIRQ_RESTART;
	int cpu;

	pending = local_softirq_pending();
	account_system_vtime(current);

	__local_bh_disable((unsigned long)__builtin_return_address(0));
	trace_softirq_enter();

	cpu = smp_processor_id();
restart:
	/* Reset the pending bitmask before enabling irqs */
	set_softirq_pending(0);

	local_irq_enable();

	h = softirq_vec;

	do {
		if (pending & 1) {
			int prev_count = preempt_count();

			h->action(h);

			if (unlikely(prev_count != preempt_count())) {
				printk(KERN_ERR "huh, entered softirq %td %p"
				       "with preempt_count %08x,"
				       " exited with %08x?\n", h - softirq_vec,
				       h->action, prev_count, preempt_count());
				preempt_count() = prev_count;
			}

			rcu_bh_qsctr_inc(cpu);
		}
		h++;
		pending >>= 1;
	} while (pending);

	local_irq_disable();

	pending = local_softirq_pending();
	if (pending && --max_restart)
		goto restart;

	if (pending)
		wakeup_softirqd();

	trace_softirq_exit();

	account_system_vtime(current);
	_local_bh_enable();
}

#ifndef __ARCH_HAS_DO_SOFTIRQ

asmlinkage void do_softirq(void)
{
	__u32 pending;
	unsigned long flags;

	if (in_interrupt())
		return;

	local_irq_save(flags);

	pending = local_softirq_pending();

	if (pending)
		__do_softirq();

	local_irq_restore(flags);
}

#endif

/*
 * Enter an interrupt context.
 */
void irq_enter(void)
{
#ifdef CONFIG_NO_HZ
	int cpu = smp_processor_id();
	if (idle_cpu(cpu) && !in_interrupt())
		tick_nohz_stop_idle(cpu);
#endif
	__irq_enter();
#ifdef CONFIG_NO_HZ
	if (idle_cpu(cpu))
		tick_nohz_update_jiffies();
#endif
}

#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
# define invoke_softirq()	__do_softirq()
#else
# define invoke_softirq()	do_softirq()
#endif

/*
 * Exit an interrupt context. Process softirqs if needed and possible:
 */
void irq_exit(void)
{
	account_system_vtime(current);
	trace_hardirq_exit();
	sub_preempt_count(IRQ_EXIT_OFFSET);
	if (!in_interrupt() && local_softirq_pending())
		invoke_softirq();

#ifdef CONFIG_NO_HZ
	/* Make sure that timer wheel updates are propagated */
	if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
		tick_nohz_stop_sched_tick(0);
	rcu_irq_exit();
#endif
	preempt_enable_no_resched();
}

/*
 * This function must run with irqs disabled!
 */
inline void raise_softirq_irqoff(unsigned int nr)
{
	__raise_softirq_irqoff(nr);

	/*
	 * If we're in an interrupt or softirq, we're done
	 * (this also catches softirq-disabled code). We will
	 * actually run the softirq once we return from
	 * the irq or softirq.
	 *
	 * Otherwise we wake up ksoftirqd to make sure we
	 * schedule the softirq soon.
	 */
	if (!in_interrupt())
		wakeup_softirqd();
}

void raise_softirq(unsigned int nr)
{
	unsigned long flags;

	local_irq_save(flags);
	raise_softirq_irqoff(nr);
	local_irq_restore(flags);
}

void open_softirq(int nr, void (*action)(struct softirq_action *))
{
	softirq_vec[nr].action = action;
}

/* Tasklets */
struct tasklet_head
{
	struct tasklet_struct *head;
	struct tasklet_struct **tail;
};

static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);

void __tasklet_schedule(struct tasklet_struct *t)
{
	unsigned long flags;

	local_irq_save(flags);
	t->next = NULL;
	*__get_cpu_var(tasklet_vec).tail = t;
	__get_cpu_var(tasklet_vec).tail = &(t->next);
	raise_softirq_irqoff(TASKLET_SOFTIRQ);
	local_irq_restore(flags);
}

EXPORT_SYMBOL(__tasklet_schedule);

void __tasklet_hi_schedule(struct tasklet_struct *t)
{
	unsigned long flags;

	local_irq_save(flags);
	t->next = NULL;
	*__get_cpu_var(tasklet_hi_vec).tail = t;
	__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
	raise_softirq_irqoff(HI_SOFTIRQ);
	local_irq_restore(flags);
}

EXPORT_SYMBOL(__tasklet_hi_schedule);

static void tasklet_action(struct softirq_action *a)
{
	struct tasklet_struct *list;

	local_irq_disable();
	list = __get_cpu_var(tasklet_vec).head;
	__get_cpu_var(tasklet_vec).head = NULL;
	__get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
	local_irq_enable();

	while (list) {
		struct tasklet_struct *t = list;

		list = list->next;

		if (tasklet_trylock(t)) {
			if (!atomic_read(&t->count)) {
				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
					BUG();
				t->func(t->data);
				tasklet_unlock(t);
				continue;
			}
			tasklet_unlock(t);
		}

		local_irq_disable();
		t->next = NULL;
		*__get_cpu_var(tasklet_vec).tail = t;
		__get_cpu_var(tasklet_vec).tail = &(t->next);
		__raise_softirq_irqoff(TASKLET_SOFTIRQ);
		local_irq_enable();
	}
}

static void tasklet_hi_action(struct softirq_action *a)
{
	struct tasklet_struct *list;

	local_irq_disable();
	list = __get_cpu_var(tasklet_hi_vec).head;
	__get_cpu_var(tasklet_hi_vec).head = NULL;
	__get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
	local_irq_enable();

	while (list) {
		struct tasklet_struct *t = list;

		list = list->next;

		if (tasklet_trylock(t)) {
			if (!atomic_read(&t->count)) {
				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
					BUG();
				t->func(t->data);
				tasklet_unlock(t);
				continue;
			}
			tasklet_unlock(t);
		}

		local_irq_disable();
		t->next = NULL;
		*__get_cpu_var(tasklet_hi_vec).tail = t;
		__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
		__raise_softirq_irqoff(HI_SOFTIRQ);
		local_irq_enable();
	}
}


void tasklet_init(struct tasklet_struct *t,
		  void (*func)(unsigned long), unsigned long data)
{
	t->next = NULL;
	t->state = 0;
	atomic_set(&t->count, 0);
	t->func = func;
	t->data = data;
}

EXPORT_SYMBOL(tasklet_init);

void tasklet_kill(struct tasklet_struct *t)
{
	if (in_interrupt())
		printk("Attempt to kill tasklet from interrupt\n");

	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
		do
			yield();
		while (test_bit(TASKLET_STATE_SCHED, &t->state));
	}
	tasklet_unlock_wait(t);
	clear_bit(TASKLET_STATE_SCHED, &t->state);
}

EXPORT_SYMBOL(tasklet_kill);

void __init softirq_init(void)
{
	int cpu;

	for_each_possible_cpu(cpu) {
		per_cpu(tasklet_vec, cpu).tail =
			&per_cpu(tasklet_vec, cpu).head;
		per_cpu(tasklet_hi_vec, cpu).tail =
			&per_cpu(tasklet_hi_vec, cpu).head;
	}

	open_softirq(TASKLET_SOFTIRQ, tasklet_action);
	open_softirq(HI_SOFTIRQ, tasklet_hi_action);
}

static int ksoftirqd(void * __bind_cpu)
{
	set_current_state(TASK_INTERRUPTIBLE);

	while (!kthread_should_stop()) {
		preempt_disable();
		if (!local_softirq_pending()) {
			preempt_enable_no_resched();
			schedule();
			preempt_disable();
		}

		__set_current_state(TASK_RUNNING);

		while (local_softirq_pending()) {
			/* Preempt disable stops cpu going offline.
			   If already offline, we'll be on wrong CPU:
			   don't process */
			if (cpu_is_offline((long)__bind_cpu))
				goto wait_to_die;
			do_softirq();
			preempt_enable_no_resched();
			cond_resched();
			preempt_disable();
		}
		preempt_enable();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;

wait_to_die:
	preempt_enable();
	/* Wait for kthread_stop */
	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
		schedule();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;
}

#ifdef CONFIG_HOTPLUG_CPU
/*
 * tasklet_kill_immediate is called to remove a tasklet which can already be
 * scheduled for execution on @cpu.
 *
 * Unlike tasklet_kill, this function removes the tasklet
 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 *
 * When this function is called, @cpu must be in the CPU_DEAD state.
 */
void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
{
	struct tasklet_struct **i;

	BUG_ON(cpu_online(cpu));
	BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));

	if (!test_bit(TASKLET_STATE_SCHED, &t->state))
		return;

	/* CPU is dead, so no lock needed. */
	for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
		if (*i == t) {
			*i = t->next;
			/* If this was the tail element, move the tail ptr */
			if (*i == NULL)
				per_cpu(tasklet_vec, cpu).tail = i;
			return;
		}
	}
	BUG();
}

static void takeover_tasklets(unsigned int cpu)
{
	/* CPU is dead, so no lock needed. */
	local_irq_disable();

	/* Find end, append list for that CPU. */
	if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
		*(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
		__get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
		per_cpu(tasklet_vec, cpu).head = NULL;
		per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
	}
	raise_softirq_irqoff(TASKLET_SOFTIRQ);

	if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
		*__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
		__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
		per_cpu(tasklet_hi_vec, cpu).head = NULL;
		per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
	}
	raise_softirq_irqoff(HI_SOFTIRQ);

	local_irq_enable();
}
#endif /* CONFIG_HOTPLUG_CPU */

static int __cpuinit cpu_callback(struct notifier_block *nfb,
				  unsigned long action,
				  void *hcpu)
{
	int hotcpu = (unsigned long)hcpu;
	struct task_struct *p;

	switch (action) {
	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
		p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
		if (IS_ERR(p)) {
			printk("ksoftirqd for %i failed\n", hotcpu);
			return NOTIFY_BAD;
		}
		kthread_bind(p, hotcpu);
  		per_cpu(ksoftirqd, hotcpu) = p;
 		break;
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
		wake_up_process(per_cpu(ksoftirqd, hotcpu));
		break;
#ifdef CONFIG_HOTPLUG_CPU
	case CPU_UP_CANCELED:
	case CPU_UP_CANCELED_FROZEN:
		if (!per_cpu(ksoftirqd, hotcpu))
			break;
		/* Unbind so it can run.  Fall thru. */
		kthread_bind(per_cpu(ksoftirqd, hotcpu),
			     any_online_cpu(cpu_online_map));
	case CPU_DEAD:
	case CPU_DEAD_FROZEN: {
		struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };

		p = per_cpu(ksoftirqd, hotcpu);
		per_cpu(ksoftirqd, hotcpu) = NULL;
		sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
		kthread_stop(p);
		takeover_tasklets(hotcpu);
		break;
	}
#endif /* CONFIG_HOTPLUG_CPU */
 	}
	return NOTIFY_OK;
}

static struct notifier_block __cpuinitdata cpu_nfb = {
	.notifier_call = cpu_callback
};

static __init int spawn_ksoftirqd(void)
{
	void *cpu = (void *)(long)smp_processor_id();
	int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);

	BUG_ON(err == NOTIFY_BAD);
	cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
	register_cpu_notifier(&cpu_nfb);
	return 0;
}
early_initcall(spawn_ksoftirqd);

#ifdef CONFIG_SMP
/*
 * Call a function on all processors
 */
int on_each_cpu(void (*func) (void *info), void *info, int wait)
{
	int ret = 0;

	preempt_disable();
	ret = smp_call_function(func, info, wait);
	local_irq_disable();
	func(info);
	local_irq_enable();
	preempt_enable();
	return ret;
}
EXPORT_SYMBOL(on_each_cpu);
#endif
Commit	Line	Data
	1	/*
	2	* linux/kernel/softirq.c
	3	*
	4	* Copyright (C) 1992 Linus Torvalds
	5	*
	6	* Distribute under GPLv2.
	7	*
	8	* Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
	9	*/
	10
	11	#include <linux/module.h>
	12	#include <linux/kernel_stat.h>
	13	#include <linux/interrupt.h>
	14	#include <linux/init.h>
	15	#include <linux/mm.h>
	16	#include <linux/notifier.h>
	17	#include <linux/percpu.h>
	18	#include <linux/cpu.h>
	19	#include <linux/freezer.h>
	20	#include <linux/kthread.h>
	21	#include <linux/rcupdate.h>
	22	#include <linux/smp.h>
	23	#include <linux/tick.h>
	24
	25	#include <asm/irq.h>
	26	/*
	27	- No shared variables, all the data are CPU local.
	28	- If a softirq needs serialization, let it serialize itself
	29	by its own spinlocks.
	30	- Even if softirq is serialized, only local cpu is marked for
	31	execution. Hence, we get something sort of weak cpu binding.
	32	Though it is still not clear, will it result in better locality
	33	or will not.
	34
	35	Examples:
	36	- NET RX softirq. It is multithreaded and does not require
	37	any global serialization.
	38	- NET TX softirq. It kicks software netdevice queues, hence
	39	it is logically serialized per device, but this serialization
	40	is invisible to common code.
	41	- Tasklets: serialized wrt itself.
	42	*/
	43
	44	#ifndef __ARCH_IRQ_STAT
	45	irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
	46	EXPORT_SYMBOL(irq_stat);
	47	#endif
	48
	49	static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
	50
	51	static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
	52
	53	/*
	54	* we cannot loop indefinitely here to avoid userspace starvation,
	55	* but we also don't want to introduce a worst case 1/HZ latency
	56	* to the pending events, so lets the scheduler to balance
	57	* the softirq load for us.
	58	*/
	59	static inline void wakeup_softirqd(void)
	60	{
	61	/* Interrupts are disabled: no need to stop preemption */
	62	struct task_struct *tsk = __get_cpu_var(ksoftirqd);
	63
	64	if (tsk && tsk->state != TASK_RUNNING)
	65	wake_up_process(tsk);
	66	}
	67
	68	/*
	69	* This one is for softirq.c-internal use,
	70	* where hardirqs are disabled legitimately:
	71	*/
	72	#ifdef CONFIG_TRACE_IRQFLAGS
	73	static void __local_bh_disable(unsigned long ip)
	74	{
	75	unsigned long flags;
	76
	77	WARN_ON_ONCE(in_irq());
	78
	79	raw_local_irq_save(flags);
	80	add_preempt_count(SOFTIRQ_OFFSET);
	81	/*
	82	* Were softirqs turned off above:
	83	*/
	84	if (softirq_count() == SOFTIRQ_OFFSET)
	85	trace_softirqs_off(ip);
	86	raw_local_irq_restore(flags);
	87	}
	88	#else /* !CONFIG_TRACE_IRQFLAGS */
	89	static inline void __local_bh_disable(unsigned long ip)
	90	{
	91	add_preempt_count(SOFTIRQ_OFFSET);
	92	barrier();
	93	}
	94	#endif /* CONFIG_TRACE_IRQFLAGS */
	95
	96	void local_bh_disable(void)
	97	{
	98	__local_bh_disable((unsigned long)__builtin_return_address(0));
	99	}
	100
	101	EXPORT_SYMBOL(local_bh_disable);
	102
	103	void __local_bh_enable(void)
	104	{
	105	WARN_ON_ONCE(in_irq());
	106
	107	/*
	108	* softirqs should never be enabled by __local_bh_enable(),
	109	* it always nests inside local_bh_enable() sections:
	110	*/
	111	WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
	112
	113	sub_preempt_count(SOFTIRQ_OFFSET);
	114	}
	115	EXPORT_SYMBOL_GPL(__local_bh_enable);
	116
	117	/*
	118	* Special-case - softirqs can safely be enabled in
	119	* cond_resched_softirq(), or by __do_softirq(),
	120	* without processing still-pending softirqs:
	121	*/
	122	void _local_bh_enable(void)
	123	{
	124	WARN_ON_ONCE(in_irq());
	125	WARN_ON_ONCE(!irqs_disabled());
	126
	127	if (softirq_count() == SOFTIRQ_OFFSET)
	128	trace_softirqs_on((unsigned long)__builtin_return_address(0));
	129	sub_preempt_count(SOFTIRQ_OFFSET);
	130	}
	131
	132	EXPORT_SYMBOL(_local_bh_enable);
	133
	134	static inline void _local_bh_enable_ip(unsigned long ip)
	135	{
	136	WARN_ON_ONCE(in_irq() \|\| irqs_disabled());
	137	#ifdef CONFIG_TRACE_IRQFLAGS
	138	local_irq_disable();
	139	#endif
	140	/*
	141	* Are softirqs going to be turned on now:
	142	*/
	143	if (softirq_count() == SOFTIRQ_OFFSET)
	144	trace_softirqs_on(ip);
	145	/*
	146	* Keep preemption disabled until we are done with
	147	* softirq processing:
	148	*/
	149	sub_preempt_count(SOFTIRQ_OFFSET - 1);
	150
	151	if (unlikely(!in_interrupt() && local_softirq_pending()))
	152	do_softirq();
	153
	154	dec_preempt_count();
	155	#ifdef CONFIG_TRACE_IRQFLAGS
	156	local_irq_enable();
	157	#endif
	158	preempt_check_resched();
	159	}
	160
	161	void local_bh_enable(void)
	162	{
	163	_local_bh_enable_ip((unsigned long)__builtin_return_address(0));
	164	}
	165	EXPORT_SYMBOL(local_bh_enable);
	166
	167	void local_bh_enable_ip(unsigned long ip)
	168	{
	169	_local_bh_enable_ip(ip);
	170	}
	171	EXPORT_SYMBOL(local_bh_enable_ip);
	172
	173	/*
	174	* We restart softirq processing MAX_SOFTIRQ_RESTART times,
	175	* and we fall back to softirqd after that.
	176	*
	177	* This number has been established via experimentation.
	178	* The two things to balance is latency against fairness -
	179	* we want to handle softirqs as soon as possible, but they
	180	* should not be able to lock up the box.
	181	*/
	182	#define MAX_SOFTIRQ_RESTART 10
	183
	184	asmlinkage void __do_softirq(void)
	185	{
	186	struct softirq_action *h;
	187	__u32 pending;
	188	int max_restart = MAX_SOFTIRQ_RESTART;
	189	int cpu;
	190
	191	pending = local_softirq_pending();
	192	account_system_vtime(current);
	193
	194	__local_bh_disable((unsigned long)__builtin_return_address(0));
	195	trace_softirq_enter();
	196
	197	cpu = smp_processor_id();
	198	restart:
	199	/* Reset the pending bitmask before enabling irqs */
	200	set_softirq_pending(0);
	201
	202	local_irq_enable();
	203
	204	h = softirq_vec;
	205
	206	do {
	207	if (pending & 1) {
	208	int prev_count = preempt_count();
	209
	210	h->action(h);
	211
	212	if (unlikely(prev_count != preempt_count())) {
	213	printk(KERN_ERR "huh, entered softirq %td %p"
	214	"with preempt_count %08x,"
	215	" exited with %08x?\n", h - softirq_vec,
	216	h->action, prev_count, preempt_count());
	217	preempt_count() = prev_count;
	218	}
	219
	220	rcu_bh_qsctr_inc(cpu);
	221	}
	222	h++;
	223	pending >>= 1;
	224	} while (pending);
	225
	226	local_irq_disable();
	227
	228	pending = local_softirq_pending();
	229	if (pending && --max_restart)
	230	goto restart;
	231
	232	if (pending)
	233	wakeup_softirqd();
	234
	235	trace_softirq_exit();
	236
	237	account_system_vtime(current);
	238	_local_bh_enable();
	239	}
	240
	241	#ifndef __ARCH_HAS_DO_SOFTIRQ
	242
	243	asmlinkage void do_softirq(void)
	244	{
	245	__u32 pending;
	246	unsigned long flags;
	247
	248	if (in_interrupt())
	249	return;
	250
	251	local_irq_save(flags);
	252
	253	pending = local_softirq_pending();
	254
	255	if (pending)
	256	__do_softirq();
	257
	258	local_irq_restore(flags);
	259	}
	260
	261	#endif
	262
	263	/*
	264	* Enter an interrupt context.
	265	*/
	266	void irq_enter(void)
	267	{
	268	#ifdef CONFIG_NO_HZ
	269	int cpu = smp_processor_id();
	270	if (idle_cpu(cpu) && !in_interrupt())
	271	tick_nohz_stop_idle(cpu);
	272	#endif
	273	__irq_enter();
	274	#ifdef CONFIG_NO_HZ
	275	if (idle_cpu(cpu))
	276	tick_nohz_update_jiffies();
	277	#endif
	278	}
	279
	280	#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
	281	# define invoke_softirq() __do_softirq()
	282	#else
	283	# define invoke_softirq() do_softirq()
	284	#endif
	285
	286	/*
	287	* Exit an interrupt context. Process softirqs if needed and possible:
	288	*/
	289	void irq_exit(void)
	290	{
	291	account_system_vtime(current);
	292	trace_hardirq_exit();
	293	sub_preempt_count(IRQ_EXIT_OFFSET);
	294	if (!in_interrupt() && local_softirq_pending())
	295	invoke_softirq();
	296
	297	#ifdef CONFIG_NO_HZ
	298	/* Make sure that timer wheel updates are propagated */
	299	if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
	300	tick_nohz_stop_sched_tick(0);
	301	rcu_irq_exit();
	302	#endif
	303	preempt_enable_no_resched();
	304	}
	305
	306	/*
	307	* This function must run with irqs disabled!
	308	*/
	309	inline void raise_softirq_irqoff(unsigned int nr)
	310	{
	311	__raise_softirq_irqoff(nr);
	312
	313	/*
	314	* If we're in an interrupt or softirq, we're done
	315	* (this also catches softirq-disabled code). We will
	316	* actually run the softirq once we return from
	317	* the irq or softirq.
	318	*
	319	* Otherwise we wake up ksoftirqd to make sure we
	320	* schedule the softirq soon.
	321	*/
	322	if (!in_interrupt())
	323	wakeup_softirqd();
	324	}
	325
	326	void raise_softirq(unsigned int nr)
	327	{
	328	unsigned long flags;
	329
	330	local_irq_save(flags);
	331	raise_softirq_irqoff(nr);
	332	local_irq_restore(flags);
	333	}
	334
	335	void open_softirq(int nr, void (action)(struct softirq_action ))
	336	{
	337	softirq_vec[nr].action = action;
	338	}
	339
	340	/* Tasklets */
	341	struct tasklet_head
	342	{
	343	struct tasklet_struct *head;
	344	struct tasklet_struct **tail;
	345	};
	346
	347	static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
	348	static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
	349
	350	void __tasklet_schedule(struct tasklet_struct *t)
	351	{
	352	unsigned long flags;
	353
	354	local_irq_save(flags);
	355	t->next = NULL;
	356	*__get_cpu_var(tasklet_vec).tail = t;
	357	__get_cpu_var(tasklet_vec).tail = &(t->next);
	358	raise_softirq_irqoff(TASKLET_SOFTIRQ);
	359	local_irq_restore(flags);
	360	}
	361
	362	EXPORT_SYMBOL(__tasklet_schedule);
	363
	364	void __tasklet_hi_schedule(struct tasklet_struct *t)
	365	{
	366	unsigned long flags;
	367
	368	local_irq_save(flags);
	369	t->next = NULL;
	370	*__get_cpu_var(tasklet_hi_vec).tail = t;
	371	__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
	372	raise_softirq_irqoff(HI_SOFTIRQ);
	373	local_irq_restore(flags);
	374	}
	375
	376	EXPORT_SYMBOL(__tasklet_hi_schedule);
	377
	378	static void tasklet_action(struct softirq_action *a)
	379	{
	380	struct tasklet_struct *list;
	381
	382	local_irq_disable();
	383	list = __get_cpu_var(tasklet_vec).head;
	384	__get_cpu_var(tasklet_vec).head = NULL;
	385	__get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
	386	local_irq_enable();
	387
	388	while (list) {
	389	struct tasklet_struct *t = list;
	390
	391	list = list->next;
	392
	393	if (tasklet_trylock(t)) {
	394	if (!atomic_read(&t->count)) {
	395	if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
	396	BUG();
	397	t->func(t->data);
	398	tasklet_unlock(t);
	399	continue;
	400	}
	401	tasklet_unlock(t);
	402	}
	403
	404	local_irq_disable();
	405	t->next = NULL;
	406	*__get_cpu_var(tasklet_vec).tail = t;
	407	__get_cpu_var(tasklet_vec).tail = &(t->next);
	408	__raise_softirq_irqoff(TASKLET_SOFTIRQ);
	409	local_irq_enable();
	410	}
	411	}
	412
	413	static void tasklet_hi_action(struct softirq_action *a)
	414	{
	415	struct tasklet_struct *list;
	416
	417	local_irq_disable();
	418	list = __get_cpu_var(tasklet_hi_vec).head;
	419	__get_cpu_var(tasklet_hi_vec).head = NULL;
	420	__get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
	421	local_irq_enable();
	422
	423	while (list) {
	424	struct tasklet_struct *t = list;
	425
	426	list = list->next;
	427
	428	if (tasklet_trylock(t)) {
	429	if (!atomic_read(&t->count)) {
	430	if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
	431	BUG();
	432	t->func(t->data);
	433	tasklet_unlock(t);
	434	continue;
	435	}
	436	tasklet_unlock(t);
	437	}
	438
	439	local_irq_disable();
	440	t->next = NULL;
	441	*__get_cpu_var(tasklet_hi_vec).tail = t;
	442	__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
	443	__raise_softirq_irqoff(HI_SOFTIRQ);
	444	local_irq_enable();
	445	}
	446	}
	447
	448
	449	void tasklet_init(struct tasklet_struct *t,
	450	void (*func)(unsigned long), unsigned long data)
	451	{
	452	t->next = NULL;
	453	t->state = 0;
	454	atomic_set(&t->count, 0);
	455	t->func = func;
	456	t->data = data;
	457	}
	458
	459	EXPORT_SYMBOL(tasklet_init);
	460
	461	void tasklet_kill(struct tasklet_struct *t)
	462	{
	463	if (in_interrupt())
	464	printk("Attempt to kill tasklet from interrupt\n");
	465
	466	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
	467	do
	468	yield();
	469	while (test_bit(TASKLET_STATE_SCHED, &t->state));
	470	}
	471	tasklet_unlock_wait(t);
	472	clear_bit(TASKLET_STATE_SCHED, &t->state);
	473	}
	474
	475	EXPORT_SYMBOL(tasklet_kill);
	476
	477	void __init softirq_init(void)
	478	{
	479	int cpu;
	480
	481	for_each_possible_cpu(cpu) {
	482	per_cpu(tasklet_vec, cpu).tail =
	483	&per_cpu(tasklet_vec, cpu).head;
	484	per_cpu(tasklet_hi_vec, cpu).tail =
	485	&per_cpu(tasklet_hi_vec, cpu).head;
	486	}
	487
	488	open_softirq(TASKLET_SOFTIRQ, tasklet_action);
	489	open_softirq(HI_SOFTIRQ, tasklet_hi_action);
	490	}
	491
	492	static int ksoftirqd(void * __bind_cpu)
	493	{
	494	set_current_state(TASK_INTERRUPTIBLE);
	495
	496	while (!kthread_should_stop()) {
	497	preempt_disable();
	498	if (!local_softirq_pending()) {
	499	preempt_enable_no_resched();
	500	schedule();
	501	preempt_disable();
	502	}
	503
	504	__set_current_state(TASK_RUNNING);
	505
	506	while (local_softirq_pending()) {
	507	/* Preempt disable stops cpu going offline.
	508	If already offline, we'll be on wrong CPU:
	509	don't process */
	510	if (cpu_is_offline((long)__bind_cpu))
	511	goto wait_to_die;
	512	do_softirq();
	513	preempt_enable_no_resched();
	514	cond_resched();
	515	preempt_disable();
	516	}
	517	preempt_enable();
	518	set_current_state(TASK_INTERRUPTIBLE);
	519	}
	520	__set_current_state(TASK_RUNNING);
	521	return 0;
	522
	523	wait_to_die:
	524	preempt_enable();
	525	/* Wait for kthread_stop */
	526	set_current_state(TASK_INTERRUPTIBLE);
	527	while (!kthread_should_stop()) {
	528	schedule();
	529	set_current_state(TASK_INTERRUPTIBLE);
	530	}
	531	__set_current_state(TASK_RUNNING);
	532	return 0;
	533	}
	534
	535	#ifdef CONFIG_HOTPLUG_CPU
	536	/*
	537	* tasklet_kill_immediate is called to remove a tasklet which can already be
	538	* scheduled for execution on @cpu.
	539	*
	540	* Unlike tasklet_kill, this function removes the tasklet
	541	* _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
	542	*
	543	* When this function is called, @cpu must be in the CPU_DEAD state.
	544	*/
	545	void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
	546	{
	547	struct tasklet_struct **i;
	548
	549	BUG_ON(cpu_online(cpu));
	550	BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
	551
	552	if (!test_bit(TASKLET_STATE_SCHED, &t->state))
	553	return;
	554
	555	/* CPU is dead, so no lock needed. */
	556	for (i = &per_cpu(tasklet_vec, cpu).head; i; i = &(i)->next) {
	557	if (*i == t) {
	558	*i = t->next;
	559	/* If this was the tail element, move the tail ptr */
	560	if (*i == NULL)
	561	per_cpu(tasklet_vec, cpu).tail = i;
	562	return;
	563	}
	564	}
	565	BUG();
	566	}
	567
	568	static void takeover_tasklets(unsigned int cpu)
	569	{
	570	/* CPU is dead, so no lock needed. */
	571	local_irq_disable();
	572
	573	/* Find end, append list for that CPU. */
	574	if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
	575	*(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
	576	__get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
	577	per_cpu(tasklet_vec, cpu).head = NULL;
	578	per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
	579	}
	580	raise_softirq_irqoff(TASKLET_SOFTIRQ);
	581
	582	if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
	583	*__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
	584	__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
	585	per_cpu(tasklet_hi_vec, cpu).head = NULL;
	586	per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
	587	}
	588	raise_softirq_irqoff(HI_SOFTIRQ);
	589
	590	local_irq_enable();
	591	}
	592	#endif /* CONFIG_HOTPLUG_CPU */
	593
	594	static int __cpuinit cpu_callback(struct notifier_block *nfb,
	595	unsigned long action,
	596	void *hcpu)
	597	{
	598	int hotcpu = (unsigned long)hcpu;
	599	struct task_struct *p;
	600
	601	switch (action) {
	602	case CPU_UP_PREPARE:
	603	case CPU_UP_PREPARE_FROZEN:
	604	p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
	605	if (IS_ERR(p)) {
	606	printk("ksoftirqd for %i failed\n", hotcpu);
	607	return NOTIFY_BAD;
	608	}
	609	kthread_bind(p, hotcpu);
	610	per_cpu(ksoftirqd, hotcpu) = p;
	611	break;
	612	case CPU_ONLINE:
	613	case CPU_ONLINE_FROZEN:
	614	wake_up_process(per_cpu(ksoftirqd, hotcpu));
	615	break;
	616	#ifdef CONFIG_HOTPLUG_CPU
	617	case CPU_UP_CANCELED:
	618	case CPU_UP_CANCELED_FROZEN:
	619	if (!per_cpu(ksoftirqd, hotcpu))
	620	break;
	621	/* Unbind so it can run. Fall thru. */
	622	kthread_bind(per_cpu(ksoftirqd, hotcpu),
	623	any_online_cpu(cpu_online_map));
	624	case CPU_DEAD:
	625	case CPU_DEAD_FROZEN: {
	626	struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
	627
	628	p = per_cpu(ksoftirqd, hotcpu);
	629	per_cpu(ksoftirqd, hotcpu) = NULL;
	630	sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
	631	kthread_stop(p);
	632	takeover_tasklets(hotcpu);
	633	break;
	634	}
	635	#endif /* CONFIG_HOTPLUG_CPU */
	636	}
	637	return NOTIFY_OK;
	638	}
	639
	640	static struct notifier_block __cpuinitdata cpu_nfb = {
	641	.notifier_call = cpu_callback
	642	};
	643
	644	static __init int spawn_ksoftirqd(void)
	645	{
	646	void cpu = (void )(long)smp_processor_id();
	647	int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
	648
	649	BUG_ON(err == NOTIFY_BAD);
	650	cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
	651	register_cpu_notifier(&cpu_nfb);
	652	return 0;
	653	}
	654	early_initcall(spawn_ksoftirqd);
	655
	656	#ifdef CONFIG_SMP
	657	/*
	658	* Call a function on all processors
	659	*/
	660	int on_each_cpu(void (func) (void info), void *info, int wait)
	661	{
	662	int ret = 0;
	663
	664	preempt_disable();
	665	ret = smp_call_function(func, info, wait);
	666	local_irq_disable();
	667	func(info);
	668	local_irq_enable();
	669	preempt_enable();
	670	return ret;
	671	}
	672	EXPORT_SYMBOL(on_each_cpu);
	673	#endif