[deliverable/linux.git] / kernel / smp.c

/*
 * Generic helpers for smp ipi calls
 *
 * (C) Jens Axboe <jens.axboe@oracle.com> 2008
 *
 */
#include <linux/init.h>
#include <linux/module.h>
#include <linux/percpu.h>
#include <linux/rcupdate.h>
#include <linux/rculist.h>
#include <linux/smp.h>

static DEFINE_PER_CPU(struct call_single_queue, call_single_queue);
static LIST_HEAD(call_function_queue);
__cacheline_aligned_in_smp DEFINE_SPINLOCK(call_function_lock);

enum {
	CSD_FLAG_WAIT		= 0x01,
	CSD_FLAG_ALLOC		= 0x02,
};

struct call_function_data {
	struct call_single_data csd;
	spinlock_t lock;
	unsigned int refs;
	struct rcu_head rcu_head;
	unsigned long cpumask_bits[];
};

struct call_single_queue {
	struct list_head list;
	spinlock_t lock;
};

static int __cpuinit init_call_single_data(void)
{
	int i;

	for_each_possible_cpu(i) {
		struct call_single_queue *q = &per_cpu(call_single_queue, i);

		spin_lock_init(&q->lock);
		INIT_LIST_HEAD(&q->list);
	}
	return 0;
}
early_initcall(init_call_single_data);

static void csd_flag_wait(struct call_single_data *data)
{
	/* Wait for response */
	do {
		if (!(data->flags & CSD_FLAG_WAIT))
			break;
		cpu_relax();
	} while (1);
}

/*
 * Insert a previously allocated call_single_data element for execution
 * on the given CPU. data must already have ->func, ->info, and ->flags set.
 */
static void generic_exec_single(int cpu, struct call_single_data *data)
{
	struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);
	int wait = data->flags & CSD_FLAG_WAIT, ipi;
	unsigned long flags;

	spin_lock_irqsave(&dst->lock, flags);
	ipi = list_empty(&dst->list);
	list_add_tail(&data->list, &dst->list);
	spin_unlock_irqrestore(&dst->lock, flags);

	/*
	 * Make the list addition visible before sending the ipi.
	 */
	smp_mb();

	if (ipi)
		arch_send_call_function_single_ipi(cpu);

	if (wait)
		csd_flag_wait(data);
}

static void rcu_free_call_data(struct rcu_head *head)
{
	struct call_function_data *data;

	data = container_of(head, struct call_function_data, rcu_head);

	kfree(data);
}

/*
 * Invoked by arch to handle an IPI for call function. Must be called with
 * interrupts disabled.
 */
void generic_smp_call_function_interrupt(void)
{
	struct call_function_data *data;
	int cpu = get_cpu();

	/*
	 * It's ok to use list_for_each_rcu() here even though we may delete
	 * 'pos', since list_del_rcu() doesn't clear ->next
	 */
	rcu_read_lock();
	list_for_each_entry_rcu(data, &call_function_queue, csd.list) {
		int refs;

		if (!cpumask_test_cpu(cpu, to_cpumask(data->cpumask_bits)))
			continue;

		data->csd.func(data->csd.info);

		spin_lock(&data->lock);
		cpumask_clear_cpu(cpu, to_cpumask(data->cpumask_bits));
		WARN_ON(data->refs == 0);
		data->refs--;
		refs = data->refs;
		spin_unlock(&data->lock);

		if (refs)
			continue;

		spin_lock(&call_function_lock);
		list_del_rcu(&data->csd.list);
		spin_unlock(&call_function_lock);

		if (data->csd.flags & CSD_FLAG_WAIT) {
			/*
			 * serialize stores to data with the flag clear
			 * and wakeup
			 */
			smp_wmb();
			data->csd.flags &= ~CSD_FLAG_WAIT;
		}
		if (data->csd.flags & CSD_FLAG_ALLOC)
			call_rcu(&data->rcu_head, rcu_free_call_data);
	}
	rcu_read_unlock();

	put_cpu();
}

/*
 * Invoked by arch to handle an IPI for call function single. Must be called
 * from the arch with interrupts disabled.
 */
void generic_smp_call_function_single_interrupt(void)
{
	struct call_single_queue *q = &__get_cpu_var(call_single_queue);
	LIST_HEAD(list);

	/*
	 * Need to see other stores to list head for checking whether
	 * list is empty without holding q->lock
	 */
	smp_read_barrier_depends();
	while (!list_empty(&q->list)) {
		unsigned int data_flags;

		spin_lock(&q->lock);
		list_replace_init(&q->list, &list);
		spin_unlock(&q->lock);

		while (!list_empty(&list)) {
			struct call_single_data *data;

			data = list_entry(list.next, struct call_single_data,
						list);
			list_del(&data->list);

			/*
			 * 'data' can be invalid after this call if
			 * flags == 0 (when called through
			 * generic_exec_single(), so save them away before
			 * making the call.
			 */
			data_flags = data->flags;

			data->func(data->info);

			if (data_flags & CSD_FLAG_WAIT) {
				smp_wmb();
				data->flags &= ~CSD_FLAG_WAIT;
			} else if (data_flags & CSD_FLAG_ALLOC)
				kfree(data);
		}
		/*
		 * See comment on outer loop
		 */
		smp_read_barrier_depends();
	}
}

/*
 * smp_call_function_single - Run a function on a specific CPU
 * @func: The function to run. This must be fast and non-blocking.
 * @info: An arbitrary pointer to pass to the function.
 * @wait: If true, wait until function has completed on other CPUs.
 *
 * Returns 0 on success, else a negative status code. Note that @wait
 * will be implicitly turned on in case of allocation failures, since
 * we fall back to on-stack allocation.
 */
int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
			     int wait)
{
	struct call_single_data d;
	unsigned long flags;
	/* prevent preemption and reschedule on another processor,
	   as well as CPU removal */
	int me = get_cpu();
	int err = 0;

	/* Can deadlock when called with interrupts disabled */
	WARN_ON(irqs_disabled());

	if (cpu == me) {
		local_irq_save(flags);
		func(info);
		local_irq_restore(flags);
	} else if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) {
		struct call_single_data *data = NULL;

		if (!wait) {
			data = kmalloc(sizeof(*data), GFP_ATOMIC);
			if (data)
				data->flags = CSD_FLAG_ALLOC;
		}
		if (!data) {
			data = &d;
			data->flags = CSD_FLAG_WAIT;
		}

		data->func = func;
		data->info = info;
		generic_exec_single(cpu, data);
	} else {
		err = -ENXIO;	/* CPU not online */
	}

	put_cpu();
	return err;
}
EXPORT_SYMBOL(smp_call_function_single);

/**
 * __smp_call_function_single(): Run a function on another CPU
 * @cpu: The CPU to run on.
 * @data: Pre-allocated and setup data structure
 *
 * Like smp_call_function_single(), but allow caller to pass in a pre-allocated
 * data structure. Useful for embedding @data inside other structures, for
 * instance.
 *
 */
void __smp_call_function_single(int cpu, struct call_single_data *data)
{
	/* Can deadlock when called with interrupts disabled */
	WARN_ON((data->flags & CSD_FLAG_WAIT) && irqs_disabled());

	generic_exec_single(cpu, data);
}

/* FIXME: Shim for archs using old arch_send_call_function_ipi API. */
#ifndef arch_send_call_function_ipi_mask
#define arch_send_call_function_ipi_mask(maskp) \
	arch_send_call_function_ipi(*(maskp))
#endif

/**
 * smp_call_function_many(): Run a function on a set of other CPUs.
 * @mask: The set of cpus to run on (only runs on online subset).
 * @func: The function to run. This must be fast and non-blocking.
 * @info: An arbitrary pointer to pass to the function.
 * @wait: If true, wait (atomically) until function has completed on other CPUs.
 *
 * If @wait is true, then returns once @func has returned. Note that @wait
 * will be implicitly turned on in case of allocation failures, since
 * we fall back to on-stack allocation.
 *
 * You must not call this function with disabled interrupts or from a
 * hardware interrupt handler or from a bottom half handler. Preemption
 * must be disabled when calling this function.
 */
void smp_call_function_many(const struct cpumask *mask,
			    void (*func)(void *), void *info,
			    bool wait)
{
	struct call_function_data *data;
	unsigned long flags;
	int cpu, next_cpu;

	/* Can deadlock when called with interrupts disabled */
	WARN_ON(irqs_disabled());

	/* So, what's a CPU they want?  Ignoring this one. */
	cpu = cpumask_first_and(mask, cpu_online_mask);
	if (cpu == smp_processor_id())
		cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
	/* No online cpus?  We're done. */
	if (cpu >= nr_cpu_ids)
		return;

	/* Do we have another CPU which isn't us? */
	next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
	if (next_cpu == smp_processor_id())
		next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask);

	/* Fastpath: do that cpu by itself. */
	if (next_cpu >= nr_cpu_ids) {
		smp_call_function_single(cpu, func, info, wait);
		return;
	}

	data = kmalloc(sizeof(*data) + cpumask_size(), GFP_ATOMIC);
	if (unlikely(!data)) {
		/* Slow path. */
		for_each_online_cpu(cpu) {
			if (cpu == smp_processor_id())
				continue;
			if (cpumask_test_cpu(cpu, mask))
				smp_call_function_single(cpu, func, info, wait);
		}
		return;
	}

	spin_lock_init(&data->lock);
	data->csd.flags = CSD_FLAG_ALLOC;
	if (wait)
		data->csd.flags |= CSD_FLAG_WAIT;
	data->csd.func = func;
	data->csd.info = info;
	cpumask_and(to_cpumask(data->cpumask_bits), mask, cpu_online_mask);
	cpumask_clear_cpu(smp_processor_id(), to_cpumask(data->cpumask_bits));
	data->refs = cpumask_weight(to_cpumask(data->cpumask_bits));

	spin_lock_irqsave(&call_function_lock, flags);
	list_add_tail_rcu(&data->csd.list, &call_function_queue);
	spin_unlock_irqrestore(&call_function_lock, flags);

	/*
	 * Make the list addition visible before sending the ipi.
	 */
	smp_mb();

	/* Send a message to all CPUs in the map */
	arch_send_call_function_ipi_mask(to_cpumask(data->cpumask_bits));

	/* optionally wait for the CPUs to complete */
	if (wait)
		csd_flag_wait(&data->csd);
}
EXPORT_SYMBOL(smp_call_function_many);

/**
 * smp_call_function(): Run a function on all other CPUs.
 * @func: The function to run. This must be fast and non-blocking.
 * @info: An arbitrary pointer to pass to the function.
 * @wait: If true, wait (atomically) until function has completed on other CPUs.
 *
 * Returns 0.
 *
 * If @wait is true, then returns once @func has returned; otherwise
 * it returns just before the target cpu calls @func. In case of allocation
 * failure, @wait will be implicitly turned on.
 *
 * You must not call this function with disabled interrupts or from a
 * hardware interrupt handler or from a bottom half handler.
 */
int smp_call_function(void (*func)(void *), void *info, int wait)
{
	preempt_disable();
	smp_call_function_many(cpu_online_mask, func, info, wait);
	preempt_enable();
	return 0;
}
EXPORT_SYMBOL(smp_call_function);

void ipi_call_lock(void)
{
	spin_lock(&call_function_lock);
}

void ipi_call_unlock(void)
{
	spin_unlock(&call_function_lock);
}

void ipi_call_lock_irq(void)
{
	spin_lock_irq(&call_function_lock);
}

void ipi_call_unlock_irq(void)
{
	spin_unlock_irq(&call_function_lock);
}
Commit	Line	Data
3d442233 JA	1	/*
	2	* Generic helpers for smp ipi calls
	3	*
	4	* (C) Jens Axboe <jens.axboe@oracle.com> 2008
	5	*
	6	*/
	7	#include <linux/init.h>
	8	#include <linux/module.h>
	9	#include <linux/percpu.h>
	10	#include <linux/rcupdate.h>
59190f42	11	#include <linux/rculist.h>
3d442233 JA	12	#include <linux/smp.h>
	13
	14	static DEFINE_PER_CPU(struct call_single_queue, call_single_queue);
	15	static LIST_HEAD(call_function_queue);
	16	__cacheline_aligned_in_smp DEFINE_SPINLOCK(call_function_lock);
	17
	18	enum {
	19	CSD_FLAG_WAIT = 0x01,
	20	CSD_FLAG_ALLOC = 0x02,
	21	};
	22
	23	struct call_function_data {
	24	struct call_single_data csd;
	25	spinlock_t lock;
	26	unsigned int refs;
3d442233	27	struct rcu_head rcu_head;
54b11e6d	28	unsigned long cpumask_bits[];
3d442233 JA	29	};
	30
	31	struct call_single_queue {
	32	struct list_head list;
	33	spinlock_t lock;
	34	};
	35
7babe8db	36	static int __cpuinit init_call_single_data(void)
3d442233 JA	37	{
	38	int i;
	39
	40	for_each_possible_cpu(i) {
	41	struct call_single_queue *q = &per_cpu(call_single_queue, i);
	42
	43	spin_lock_init(&q->lock);
	44	INIT_LIST_HEAD(&q->list);
	45	}
7babe8db	46	return 0;
3d442233	47	}
7babe8db	48	early_initcall(init_call_single_data);
3d442233 JA	49
	50	static void csd_flag_wait(struct call_single_data *data)
	51	{
	52	/* Wait for response */
	53	do {
3d442233 JA	54	if (!(data->flags & CSD_FLAG_WAIT))
	55	break;
	56	cpu_relax();
	57	} while (1);
	58	}
	59
	60	/*
	61	* Insert a previously allocated call_single_data element for execution
	62	* on the given CPU. data must already have ->func, ->info, and ->flags set.
	63	*/
	64	static void generic_exec_single(int cpu, struct call_single_data *data)
	65	{
	66	struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);
	67	int wait = data->flags & CSD_FLAG_WAIT, ipi;
	68	unsigned long flags;
	69
	70	spin_lock_irqsave(&dst->lock, flags);
	71	ipi = list_empty(&dst->list);
	72	list_add_tail(&data->list, &dst->list);
	73	spin_unlock_irqrestore(&dst->lock, flags);
	74
561920a0 SS	75	/*
	76	* Make the list addition visible before sending the ipi.
	77	*/
	78	smp_mb();
	79
3d442233 JA	80	if (ipi)
	81	arch_send_call_function_single_ipi(cpu);
	82
	83	if (wait)
	84	csd_flag_wait(data);
	85	}
	86
	87	static void rcu_free_call_data(struct rcu_head *head)
	88	{
	89	struct call_function_data *data;
	90
	91	data = container_of(head, struct call_function_data, rcu_head);
	92
	93	kfree(data);
	94	}
	95
	96	/*
	97	* Invoked by arch to handle an IPI for call function. Must be called with
	98	* interrupts disabled.
	99	*/
	100	void generic_smp_call_function_interrupt(void)
	101	{
	102	struct call_function_data *data;
	103	int cpu = get_cpu();
	104
	105	/*
	106	* It's ok to use list_for_each_rcu() here even though we may delete
	107	* 'pos', since list_del_rcu() doesn't clear ->next
	108	*/
	109	rcu_read_lock();
	110	list_for_each_entry_rcu(data, &call_function_queue, csd.list) {
	111	int refs;
	112
54b11e6d	113	if (!cpumask_test_cpu(cpu, to_cpumask(data->cpumask_bits)))
3d442233 JA	114	continue;
	115
	116	data->csd.func(data->csd.info);
	117
	118	spin_lock(&data->lock);
54b11e6d	119	cpumask_clear_cpu(cpu, to_cpumask(data->cpumask_bits));
3d442233 JA	120	WARN_ON(data->refs == 0);
	121	data->refs--;
	122	refs = data->refs;
	123	spin_unlock(&data->lock);
	124
	125	if (refs)
	126	continue;
	127
	128	spin_lock(&call_function_lock);
	129	list_del_rcu(&data->csd.list);
	130	spin_unlock(&call_function_lock);
	131
	132	if (data->csd.flags & CSD_FLAG_WAIT) {
	133	/*
	134	* serialize stores to data with the flag clear
	135	* and wakeup
	136	*/
	137	smp_wmb();
	138	data->csd.flags &= ~CSD_FLAG_WAIT;
c2fc1198 NP	139	}
c2fc1198 NP	140	if (data->csd.flags & CSD_FLAG_ALLOC)
3d442233 JA	141	call_rcu(&data->rcu_head, rcu_free_call_data);
	142	}
	143	rcu_read_unlock();
	144
	145	put_cpu();
	146	}
	147
	148	/*
	149	* Invoked by arch to handle an IPI for call function single. Must be called
	150	* from the arch with interrupts disabled.
	151	*/
	152	void generic_smp_call_function_single_interrupt(void)
	153	{
	154	struct call_single_queue *q = &__get_cpu_var(call_single_queue);
	155	LIST_HEAD(list);
	156
	157	/*
	158	* Need to see other stores to list head for checking whether
	159	* list is empty without holding q->lock
	160	*/
561920a0	161	smp_read_barrier_depends();
3d442233 JA	162	while (!list_empty(&q->list)) {
	163	unsigned int data_flags;
	164
	165	spin_lock(&q->lock);
	166	list_replace_init(&q->list, &list);
	167	spin_unlock(&q->lock);
	168
	169	while (!list_empty(&list)) {
	170	struct call_single_data *data;
	171
	172	data = list_entry(list.next, struct call_single_data,
	173	list);
	174	list_del(&data->list);
	175
	176	/*
	177	* 'data' can be invalid after this call if
	178	* flags == 0 (when called through
	179	* generic_exec_single(), so save them away before
	180	* making the call.
	181	*/
	182	data_flags = data->flags;
	183
	184	data->func(data->info);
	185
	186	if (data_flags & CSD_FLAG_WAIT) {
	187	smp_wmb();
	188	data->flags &= ~CSD_FLAG_WAIT;
	189	} else if (data_flags & CSD_FLAG_ALLOC)
	190	kfree(data);
	191	}
	192	/*
	193	* See comment on outer loop
	194	*/
561920a0	195	smp_read_barrier_depends();
3d442233 JA	196	}
	197	}
	198
	199	/*
	200	* smp_call_function_single - Run a function on a specific CPU
	201	* @func: The function to run. This must be fast and non-blocking.
	202	* @info: An arbitrary pointer to pass to the function.
3d442233 JA	203	* @wait: If true, wait until function has completed on other CPUs.
	204	*
	205	* Returns 0 on success, else a negative status code. Note that @wait
	206	* will be implicitly turned on in case of allocation failures, since
	207	* we fall back to on-stack allocation.
	208	*/
	209	int smp_call_function_single(int cpu, void (func) (void info), void *info,
8691e5a8	210	int wait)
3d442233 JA	211	{
	212	struct call_single_data d;
	213	unsigned long flags;
f73be6de PA	214	/* prevent preemption and reschedule on another processor,
f73be6de PA	215	as well as CPU removal */
3d442233	216	int me = get_cpu();
f73be6de	217	int err = 0;
3d442233 JA	218
	219	/* Can deadlock when called with interrupts disabled */
	220	WARN_ON(irqs_disabled());
	221
	222	if (cpu == me) {
	223	local_irq_save(flags);
	224	func(info);
	225	local_irq_restore(flags);
4f4b6c1a	226	} else if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) {
3d442233 JA	227	struct call_single_data *data = NULL;
	228
	229	if (!wait) {
	230	data = kmalloc(sizeof(*data), GFP_ATOMIC);
	231	if (data)
	232	data->flags = CSD_FLAG_ALLOC;
	233	}
	234	if (!data) {
	235	data = &d;
	236	data->flags = CSD_FLAG_WAIT;
	237	}
	238
	239	data->func = func;
	240	data->info = info;
	241	generic_exec_single(cpu, data);
f73be6de PA	242	} else {
f73be6de PA	243	err = -ENXIO; /* CPU not online */
3d442233 JA	244	}
	245
	246	put_cpu();
f73be6de	247	return err;
3d442233 JA	248	}
	249	EXPORT_SYMBOL(smp_call_function_single);
	250
	251	/**
	252	* __smp_call_function_single(): Run a function on another CPU
	253	* @cpu: The CPU to run on.
	254	* @data: Pre-allocated and setup data structure
	255	*
	256	* Like smp_call_function_single(), but allow caller to pass in a pre-allocated
	257	* data structure. Useful for embedding @data inside other structures, for
	258	* instance.
	259	*
	260	*/
	261	void __smp_call_function_single(int cpu, struct call_single_data *data)
	262	{
	263	/* Can deadlock when called with interrupts disabled */
	264	WARN_ON((data->flags & CSD_FLAG_WAIT) && irqs_disabled());
	265
	266	generic_exec_single(cpu, data);
	267	}
	268
ce47d974 RR	269	/* FIXME: Shim for archs using old arch_send_call_function_ipi API. */
	270	#ifndef arch_send_call_function_ipi_mask
	271	#define arch_send_call_function_ipi_mask(maskp) \
	272	arch_send_call_function_ipi(*(maskp))
	273	#endif
	274
3d442233	275	/**
54b11e6d RR	276	* smp_call_function_many(): Run a function on a set of other CPUs.
54b11e6d RR	277	* @mask: The set of cpus to run on (only runs on online subset).
3d442233 JA	278	* @func: The function to run. This must be fast and non-blocking.
	279	* @info: An arbitrary pointer to pass to the function.
	280	* @wait: If true, wait (atomically) until function has completed on other CPUs.
	281	*
3d442233 JA	282	* If @wait is true, then returns once @func has returned. Note that @wait
	283	* will be implicitly turned on in case of allocation failures, since
	284	* we fall back to on-stack allocation.
	285	*
	286	* You must not call this function with disabled interrupts or from a
	287	* hardware interrupt handler or from a bottom half handler. Preemption
	288	* must be disabled when calling this function.
	289	*/
54b11e6d RR	290	void smp_call_function_many(const struct cpumask *mask,
	291	void (func)(void ), void *info,
	292	bool wait)
3d442233	293	{
54b11e6d	294	struct call_function_data *data;
3d442233	295	unsigned long flags;
54b11e6d	296	int cpu, next_cpu;
3d442233 JA	297
	298	/* Can deadlock when called with interrupts disabled */
	299	WARN_ON(irqs_disabled());
	300
54b11e6d RR	301	/* So, what's a CPU they want? Ignoring this one. */
	302	cpu = cpumask_first_and(mask, cpu_online_mask);
	303	if (cpu == smp_processor_id())
	304	cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
	305	/* No online cpus? We're done. */
	306	if (cpu >= nr_cpu_ids)
	307	return;
	308
	309	/* Do we have another CPU which isn't us? */
	310	next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
	311	if (next_cpu == smp_processor_id())
	312	next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask);
	313
	314	/* Fastpath: do that cpu by itself. */
	315	if (next_cpu >= nr_cpu_ids) {
	316	smp_call_function_single(cpu, func, info, wait);
	317	return;
3d442233 JA	318	}
3d442233 JA	319
54b11e6d RR	320	data = kmalloc(sizeof(*data) + cpumask_size(), GFP_ATOMIC);
	321	if (unlikely(!data)) {
	322	/* Slow path. */
	323	for_each_online_cpu(cpu) {
	324	if (cpu == smp_processor_id())
	325	continue;
	326	if (cpumask_test_cpu(cpu, mask))
	327	smp_call_function_single(cpu, func, info, wait);
	328	}
	329	return;
3d442233 JA	330	}
	331
	332	spin_lock_init(&data->lock);
54b11e6d RR	333	data->csd.flags = CSD_FLAG_ALLOC;
	334	if (wait)
	335	data->csd.flags \|= CSD_FLAG_WAIT;
3d442233 JA	336	data->csd.func = func;
3d442233 JA	337	data->csd.info = info;
54b11e6d RR	338	cpumask_and(to_cpumask(data->cpumask_bits), mask, cpu_online_mask);
	339	cpumask_clear_cpu(smp_processor_id(), to_cpumask(data->cpumask_bits));
	340	data->refs = cpumask_weight(to_cpumask(data->cpumask_bits));
3d442233 JA	341
	342	spin_lock_irqsave(&call_function_lock, flags);
	343	list_add_tail_rcu(&data->csd.list, &call_function_queue);
	344	spin_unlock_irqrestore(&call_function_lock, flags);
	345
561920a0 SS	346	/*
	347	* Make the list addition visible before sending the ipi.
	348	*/
	349	smp_mb();
	350
3d442233	351	/* Send a message to all CPUs in the map */
ce47d974	352	arch_send_call_function_ipi_mask(to_cpumask(data->cpumask_bits));
3d442233 JA	353
3d442233 JA	354	/* optionally wait for the CPUs to complete */
54b11e6d	355	if (wait)
3d442233	356	csd_flag_wait(&data->csd);
3d442233	357	}
54b11e6d	358	EXPORT_SYMBOL(smp_call_function_many);
3d442233 JA	359
	360	/**
	361	* smp_call_function(): Run a function on all other CPUs.
	362	* @func: The function to run. This must be fast and non-blocking.
	363	* @info: An arbitrary pointer to pass to the function.
3d442233 JA	364	* @wait: If true, wait (atomically) until function has completed on other CPUs.
3d442233 JA	365	*
54b11e6d	366	* Returns 0.
3d442233 JA	367	*
	368	* If @wait is true, then returns once @func has returned; otherwise
	369	* it returns just before the target cpu calls @func. In case of allocation
	370	* failure, @wait will be implicitly turned on.
	371	*
	372	* You must not call this function with disabled interrupts or from a
	373	* hardware interrupt handler or from a bottom half handler.
	374	*/
8691e5a8	375	int smp_call_function(void (func)(void ), void *info, int wait)
3d442233	376	{
3d442233	377	preempt_disable();
54b11e6d	378	smp_call_function_many(cpu_online_mask, func, info, wait);
3d442233	379	preempt_enable();
54b11e6d	380	return 0;
3d442233 JA	381	}
	382	EXPORT_SYMBOL(smp_call_function);
	383
	384	void ipi_call_lock(void)
	385	{
	386	spin_lock(&call_function_lock);
	387	}
	388
	389	void ipi_call_unlock(void)
	390	{
	391	spin_unlock(&call_function_lock);
	392	}
	393
	394	void ipi_call_lock_irq(void)
	395	{
	396	spin_lock_irq(&call_function_lock);
	397	}
	398
	399	void ipi_call_unlock_irq(void)
	400	{
	401	spin_unlock_irq(&call_function_lock);
	402	}