[deliverable/linux.git] / arch / arm / kernel / smp.c

/*
 *  linux/arch/arm/kernel/smp.c
 *
 *  Copyright (C) 2002 ARM Limited, All Rights Reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/config.h>
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/spinlock.h>
#include <linux/sched.h>
#include <linux/interrupt.h>
#include <linux/cache.h>
#include <linux/profile.h>
#include <linux/errno.h>
#include <linux/mm.h>
#include <linux/cpu.h>
#include <linux/smp.h>
#include <linux/seq_file.h>

#include <asm/atomic.h>
#include <asm/cacheflush.h>
#include <asm/cpu.h>
#include <asm/mmu_context.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/processor.h>
#include <asm/tlbflush.h>
#include <asm/ptrace.h>

/*
 * bitmask of present and online CPUs.
 * The present bitmask indicates that the CPU is physically present.
 * The online bitmask indicates that the CPU is up and running.
 */
cpumask_t cpu_present_mask;
cpumask_t cpu_online_map;

/*
 * as from 2.5, kernels no longer have an init_tasks structure
 * so we need some other way of telling a new secondary core
 * where to place its SVC stack
 */
struct secondary_data secondary_data;

/*
 * structures for inter-processor calls
 * - A collection of single bit ipi messages.
 */
struct ipi_data {
	spinlock_t lock;
	unsigned long ipi_count;
	unsigned long bits;
};

static DEFINE_PER_CPU(struct ipi_data, ipi_data) = {
	.lock	= SPIN_LOCK_UNLOCKED,
};

enum ipi_msg_type {
	IPI_TIMER,
	IPI_RESCHEDULE,
	IPI_CALL_FUNC,
	IPI_CPU_STOP,
};

struct smp_call_struct {
	void (*func)(void *info);
	void *info;
	int wait;
	cpumask_t pending;
	cpumask_t unfinished;
};

static struct smp_call_struct * volatile smp_call_function_data;
static DEFINE_SPINLOCK(smp_call_function_lock);

int __init __cpu_up(unsigned int cpu)
{
	struct task_struct *idle;
	pgd_t *pgd;
	pmd_t *pmd;
	int ret;

	/*
	 * Spawn a new process manually.  Grab a pointer to
	 * its task struct so we can mess with it
	 */
	idle = fork_idle(cpu);
	if (IS_ERR(idle)) {
		printk(KERN_ERR "CPU%u: fork() failed\n", cpu);
		return PTR_ERR(idle);
	}

	/*
	 * Allocate initial page tables to allow the new CPU to
	 * enable the MMU safely.  This essentially means a set
	 * of our "standard" page tables, with the addition of
	 * a 1:1 mapping for the physical address of the kernel.
	 */
	pgd = pgd_alloc(&init_mm);
	pmd = pmd_offset(pgd, PHYS_OFFSET);
	*pmd = __pmd((PHYS_OFFSET & PGDIR_MASK) |
		     PMD_TYPE_SECT | PMD_SECT_AP_WRITE);

	/*
	 * We need to tell the secondary core where to find
	 * its stack and the page tables.
	 */
	secondary_data.stack = (void *)idle->thread_info + THREAD_SIZE - 8;
	secondary_data.pgdir = virt_to_phys(pgd);
	wmb();

	/*
	 * Now bring the CPU into our world.
	 */
	ret = boot_secondary(cpu, idle);
	if (ret == 0) {
		unsigned long timeout;

		/*
		 * CPU was successfully started, wait for it
		 * to come online or time out.
		 */
		timeout = jiffies + HZ;
		while (time_before(jiffies, timeout)) {
			if (cpu_online(cpu))
				break;

			udelay(10);
			barrier();
		}

		if (!cpu_online(cpu))
			ret = -EIO;
	}

	secondary_data.stack = 0;
	secondary_data.pgdir = 0;

	*pmd_offset(pgd, PHYS_OFFSET) = __pmd(0);
	pgd_free(pgd);

	if (ret) {
		printk(KERN_CRIT "cpu_up: processor %d failed to boot\n", cpu);
		/*
		 * FIXME: We need to clean up the new idle thread. --rmk
		 */
	}

	return ret;
}

/*
 * This is the secondary CPU boot entry.  We're using this CPUs
 * idle thread stack, but a set of temporary page tables.
 */
asmlinkage void __init secondary_start_kernel(void)
{
	struct mm_struct *mm = &init_mm;
	unsigned int cpu = smp_processor_id();

	printk("CPU%u: Booted secondary processor\n", cpu);

	/*
	 * All kernel threads share the same mm context; grab a
	 * reference and switch to it.
	 */
	atomic_inc(&mm->mm_users);
	atomic_inc(&mm->mm_count);
	current->active_mm = mm;
	cpu_set(cpu, mm->cpu_vm_mask);
	cpu_switch_mm(mm->pgd, mm);
	enter_lazy_tlb(mm, current);

	cpu_init();

	/*
	 * Give the platform a chance to do its own initialisation.
	 */
	platform_secondary_init(cpu);

	/*
	 * Enable local interrupts.
	 */
	local_irq_enable();
	local_fiq_enable();

	calibrate_delay();

	smp_store_cpu_info(cpu);

	/*
	 * OK, now it's safe to let the boot CPU continue
	 */
	cpu_set(cpu, cpu_online_map);

	/*
	 * OK, it's off to the idle thread for us
	 */
	cpu_idle();
}

/*
 * Called by both boot and secondaries to move global data into
 * per-processor storage.
 */
void __init smp_store_cpu_info(unsigned int cpuid)
{
	struct cpuinfo_arm *cpu_info = &per_cpu(cpu_data, cpuid);

	cpu_info->loops_per_jiffy = loops_per_jiffy;
}

void __init smp_cpus_done(unsigned int max_cpus)
{
	int cpu;
	unsigned long bogosum = 0;

	for_each_online_cpu(cpu)
		bogosum += per_cpu(cpu_data, cpu).loops_per_jiffy;

	printk(KERN_INFO "SMP: Total of %d processors activated "
	       "(%lu.%02lu BogoMIPS).\n",
	       num_online_cpus(),
	       bogosum / (500000/HZ),
	       (bogosum / (5000/HZ)) % 100);
}

void __init smp_prepare_boot_cpu(void)
{
	unsigned int cpu = smp_processor_id();

	cpu_set(cpu, cpu_present_mask);
	cpu_set(cpu, cpu_online_map);
}

static void send_ipi_message(cpumask_t callmap, enum ipi_msg_type msg)
{
	unsigned long flags;
	unsigned int cpu;

	local_irq_save(flags);

	for_each_cpu_mask(cpu, callmap) {
		struct ipi_data *ipi = &per_cpu(ipi_data, cpu);

		spin_lock(&ipi->lock);
		ipi->bits |= 1 << msg;
		spin_unlock(&ipi->lock);
	}

	/*
	 * Call the platform specific cross-CPU call function.
	 */
	smp_cross_call(callmap);

	local_irq_restore(flags);
}

/*
 * You must not call this function with disabled interrupts, from a
 * hardware interrupt handler, nor from a bottom half handler.
 */
int smp_call_function_on_cpu(void (*func)(void *info), void *info, int retry,
                             int wait, cpumask_t callmap)
{
	struct smp_call_struct data;
	unsigned long timeout;
	int ret = 0;

	data.func = func;
	data.info = info;
	data.wait = wait;

	cpu_clear(smp_processor_id(), callmap);
	if (cpus_empty(callmap))
		goto out;

	data.pending = callmap;
	if (wait)
		data.unfinished = callmap;

	/*
	 * try to get the mutex on smp_call_function_data
	 */
	spin_lock(&smp_call_function_lock);
	smp_call_function_data = &data;

	send_ipi_message(callmap, IPI_CALL_FUNC);

	timeout = jiffies + HZ;
	while (!cpus_empty(data.pending) && time_before(jiffies, timeout))
		barrier();

	/*
	 * did we time out?
	 */
	if (!cpus_empty(data.pending)) {
		/*
		 * this may be causing our panic - report it
		 */
		printk(KERN_CRIT
		       "CPU%u: smp_call_function timeout for %p(%p)\n"
		       "      callmap %lx pending %lx, %swait\n",
		       smp_processor_id(), func, info, callmap, data.pending,
		       wait ? "" : "no ");

		/*
		 * TRACE
		 */
		timeout = jiffies + (5 * HZ);
		while (!cpus_empty(data.pending) && time_before(jiffies, timeout))
			barrier();

		if (cpus_empty(data.pending))
			printk(KERN_CRIT "     RESOLVED\n");
		else
			printk(KERN_CRIT "     STILL STUCK\n");
	}

	/*
	 * whatever happened, we're done with the data, so release it
	 */
	smp_call_function_data = NULL;
	spin_unlock(&smp_call_function_lock);

	if (!cpus_empty(data.pending)) {
		ret = -ETIMEDOUT;
		goto out;
	}

	if (wait)
		while (!cpus_empty(data.unfinished))
			barrier();
 out:

	return 0;
}

int smp_call_function(void (*func)(void *info), void *info, int retry,
                      int wait)
{
	return smp_call_function_on_cpu(func, info, retry, wait,
					cpu_online_map);
}

void show_ipi_list(struct seq_file *p)
{
	unsigned int cpu;

	seq_puts(p, "IPI:");

	for_each_online_cpu(cpu)
		seq_printf(p, " %10lu", per_cpu(ipi_data, cpu).ipi_count);

	seq_putc(p, '\n');
}

static void ipi_timer(struct pt_regs *regs)
{
	int user = user_mode(regs);

	irq_enter();
	profile_tick(CPU_PROFILING, regs);
	update_process_times(user);
	irq_exit();
}

/*
 * ipi_call_function - handle IPI from smp_call_function()
 *
 * Note that we copy data out of the cross-call structure and then
 * let the caller know that we're here and have done with their data
 */
static void ipi_call_function(unsigned int cpu)
{
	struct smp_call_struct *data = smp_call_function_data;
	void (*func)(void *info) = data->func;
	void *info = data->info;
	int wait = data->wait;

	cpu_clear(cpu, data->pending);

	func(info);

	if (wait)
		cpu_clear(cpu, data->unfinished);
}

static DEFINE_SPINLOCK(stop_lock);

/*
 * ipi_cpu_stop - handle IPI from smp_send_stop()
 */
static void ipi_cpu_stop(unsigned int cpu)
{
	spin_lock(&stop_lock);
	printk(KERN_CRIT "CPU%u: stopping\n", cpu);
	dump_stack();
	spin_unlock(&stop_lock);

	cpu_clear(cpu, cpu_online_map);

	local_fiq_disable();
	local_irq_disable();

	while (1)
		cpu_relax();
}

/*
 * Main handler for inter-processor interrupts
 *
 * For ARM, the ipimask now only identifies a single
 * category of IPI (Bit 1 IPIs have been replaced by a
 * different mechanism):
 *
 *  Bit 0 - Inter-processor function call
 */
void do_IPI(struct pt_regs *regs)
{
	unsigned int cpu = smp_processor_id();
	struct ipi_data *ipi = &per_cpu(ipi_data, cpu);

	ipi->ipi_count++;

	for (;;) {
		unsigned long msgs;

		spin_lock(&ipi->lock);
		msgs = ipi->bits;
		ipi->bits = 0;
		spin_unlock(&ipi->lock);

		if (!msgs)
			break;

		do {
			unsigned nextmsg;

			nextmsg = msgs & -msgs;
			msgs &= ~nextmsg;
			nextmsg = ffz(~nextmsg);

			switch (nextmsg) {
			case IPI_TIMER:
				ipi_timer(regs);
				break;

			case IPI_RESCHEDULE:
				/*
				 * nothing more to do - eveything is
				 * done on the interrupt return path
				 */
				break;

			case IPI_CALL_FUNC:
				ipi_call_function(cpu);
				break;

			case IPI_CPU_STOP:
				ipi_cpu_stop(cpu);
				break;

			default:
				printk(KERN_CRIT "CPU%u: Unknown IPI message 0x%x\n",
				       cpu, nextmsg);
				break;
			}
		} while (msgs);
	}
}

void smp_send_reschedule(int cpu)
{
	send_ipi_message(cpumask_of_cpu(cpu), IPI_RESCHEDULE);
}

void smp_send_timer(void)
{
	cpumask_t mask = cpu_online_map;
	cpu_clear(smp_processor_id(), mask);
	send_ipi_message(mask, IPI_TIMER);
}

void smp_send_stop(void)
{
	cpumask_t mask = cpu_online_map;
	cpu_clear(smp_processor_id(), mask);
	send_ipi_message(mask, IPI_CPU_STOP);
}

/*
 * not supported here
 */
int __init setup_profiling_timer(unsigned int multiplier)
{
	return -EINVAL;
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* linux/arch/arm/kernel/smp.c
	3	*
	4	* Copyright (C) 2002 ARM Limited, All Rights Reserved.
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 as
	8	* published by the Free Software Foundation.
	9	*/
	10	#include <linux/config.h>
	11	#include <linux/delay.h>
	12	#include <linux/init.h>
	13	#include <linux/spinlock.h>
	14	#include <linux/sched.h>
	15	#include <linux/interrupt.h>
	16	#include <linux/cache.h>
	17	#include <linux/profile.h>
	18	#include <linux/errno.h>
	19	#include <linux/mm.h>
	20	#include <linux/cpu.h>
	21	#include <linux/smp.h>
	22	#include <linux/seq_file.h>
	23
	24	#include <asm/atomic.h>
	25	#include <asm/cacheflush.h>
	26	#include <asm/cpu.h>
e65f38ed RK	27	#include <asm/mmu_context.h>
	28	#include <asm/pgtable.h>
	29	#include <asm/pgalloc.h>
1da177e4 LT	30	#include <asm/processor.h>
	31	#include <asm/tlbflush.h>
	32	#include <asm/ptrace.h>
	33
	34	/*
	35	* bitmask of present and online CPUs.
	36	* The present bitmask indicates that the CPU is physically present.
	37	* The online bitmask indicates that the CPU is up and running.
	38	*/
	39	cpumask_t cpu_present_mask;
	40	cpumask_t cpu_online_map;
	41
e65f38ed RK	42	/*
	43	* as from 2.5, kernels no longer have an init_tasks structure
	44	* so we need some other way of telling a new secondary core
	45	* where to place its SVC stack
	46	*/
	47	struct secondary_data secondary_data;
	48
1da177e4 LT	49	/*
	50	* structures for inter-processor calls
	51	* - A collection of single bit ipi messages.
	52	*/
	53	struct ipi_data {
	54	spinlock_t lock;
	55	unsigned long ipi_count;
	56	unsigned long bits;
	57	};
	58
	59	static DEFINE_PER_CPU(struct ipi_data, ipi_data) = {
	60	.lock = SPIN_LOCK_UNLOCKED,
	61	};
	62
	63	enum ipi_msg_type {
	64	IPI_TIMER,
	65	IPI_RESCHEDULE,
	66	IPI_CALL_FUNC,
	67	IPI_CPU_STOP,
	68	};
	69
	70	struct smp_call_struct {
	71	void (func)(void info);
	72	void *info;
	73	int wait;
	74	cpumask_t pending;
	75	cpumask_t unfinished;
	76	};
	77
	78	static struct smp_call_struct * volatile smp_call_function_data;
	79	static DEFINE_SPINLOCK(smp_call_function_lock);
	80
	81	int __init __cpu_up(unsigned int cpu)
	82	{
	83	struct task_struct *idle;
e65f38ed RK	84	pgd_t *pgd;
e65f38ed RK	85	pmd_t *pmd;
1da177e4 LT	86	int ret;
	87
	88	/*
	89	* Spawn a new process manually. Grab a pointer to
	90	* its task struct so we can mess with it
	91	*/
	92	idle = fork_idle(cpu);
	93	if (IS_ERR(idle)) {
	94	printk(KERN_ERR "CPU%u: fork() failed\n", cpu);
	95	return PTR_ERR(idle);
	96	}
	97
e65f38ed RK	98	/*
	99	* Allocate initial page tables to allow the new CPU to
	100	* enable the MMU safely. This essentially means a set
	101	* of our "standard" page tables, with the addition of
	102	* a 1:1 mapping for the physical address of the kernel.
	103	*/
	104	pgd = pgd_alloc(&init_mm);
	105	pmd = pmd_offset(pgd, PHYS_OFFSET);
	106	*pmd = __pmd((PHYS_OFFSET & PGDIR_MASK) \|
	107	PMD_TYPE_SECT \| PMD_SECT_AP_WRITE);
	108
	109	/*
	110	* We need to tell the secondary core where to find
	111	* its stack and the page tables.
	112	*/
	113	secondary_data.stack = (void *)idle->thread_info + THREAD_SIZE - 8;
	114	secondary_data.pgdir = virt_to_phys(pgd);
	115	wmb();
	116
1da177e4 LT	117	/*
	118	* Now bring the CPU into our world.
	119	*/
	120	ret = boot_secondary(cpu, idle);
e65f38ed RK	121	if (ret == 0) {
	122	unsigned long timeout;
	123
	124	/*
	125	* CPU was successfully started, wait for it
	126	* to come online or time out.
	127	*/
	128	timeout = jiffies + HZ;
	129	while (time_before(jiffies, timeout)) {
	130	if (cpu_online(cpu))
	131	break;
	132
	133	udelay(10);
	134	barrier();
	135	}
	136
	137	if (!cpu_online(cpu))
	138	ret = -EIO;
	139	}
	140
	141	secondary_data.stack = 0;
	142	secondary_data.pgdir = 0;
	143
	144	*pmd_offset(pgd, PHYS_OFFSET) = __pmd(0);
	145	pgd_free(pgd);
	146
1da177e4 LT	147	if (ret) {
	148	printk(KERN_CRIT "cpu_up: processor %d failed to boot\n", cpu);
	149	/*
	150	* FIXME: We need to clean up the new idle thread. --rmk
	151	*/
	152	}
	153
	154	return ret;
	155	}
	156
e65f38ed RK	157	/*
	158	* This is the secondary CPU boot entry. We're using this CPUs
	159	* idle thread stack, but a set of temporary page tables.
	160	*/
	161	asmlinkage void __init secondary_start_kernel(void)
	162	{
	163	struct mm_struct *mm = &init_mm;
	164	unsigned int cpu = smp_processor_id();
	165
	166	printk("CPU%u: Booted secondary processor\n", cpu);
	167
	168	/*
	169	* All kernel threads share the same mm context; grab a
	170	* reference and switch to it.
	171	*/
	172	atomic_inc(&mm->mm_users);
	173	atomic_inc(&mm->mm_count);
	174	current->active_mm = mm;
	175	cpu_set(cpu, mm->cpu_vm_mask);
	176	cpu_switch_mm(mm->pgd, mm);
	177	enter_lazy_tlb(mm, current);
	178
	179	cpu_init();
	180
	181	/*
	182	* Give the platform a chance to do its own initialisation.
	183	*/
	184	platform_secondary_init(cpu);
	185
	186	/*
	187	* Enable local interrupts.
	188	*/
	189	local_irq_enable();
	190	local_fiq_enable();
	191
	192	calibrate_delay();
	193
	194	smp_store_cpu_info(cpu);
	195
	196	/*
	197	* OK, now it's safe to let the boot CPU continue
	198	*/
	199	cpu_set(cpu, cpu_online_map);
	200
	201	/*
	202	* OK, it's off to the idle thread for us
	203	*/
	204	cpu_idle();
	205	}
	206
1da177e4 LT	207	/*
	208	* Called by both boot and secondaries to move global data into
	209	* per-processor storage.
	210	*/
	211	void __init smp_store_cpu_info(unsigned int cpuid)
	212	{
	213	struct cpuinfo_arm *cpu_info = &per_cpu(cpu_data, cpuid);
	214
	215	cpu_info->loops_per_jiffy = loops_per_jiffy;
	216	}
	217
	218	void __init smp_cpus_done(unsigned int max_cpus)
	219	{
	220	int cpu;
	221	unsigned long bogosum = 0;
	222
	223	for_each_online_cpu(cpu)
	224	bogosum += per_cpu(cpu_data, cpu).loops_per_jiffy;
	225
	226	printk(KERN_INFO "SMP: Total of %d processors activated "
	227	"(%lu.%02lu BogoMIPS).\n",
	228	num_online_cpus(),
	229	bogosum / (500000/HZ),
	230	(bogosum / (5000/HZ)) % 100);
	231	}
	232
	233	void __init smp_prepare_boot_cpu(void)
	234	{
	235	unsigned int cpu = smp_processor_id();
	236
	237	cpu_set(cpu, cpu_present_mask);
	238	cpu_set(cpu, cpu_online_map);
	239	}
	240
	241	static void send_ipi_message(cpumask_t callmap, enum ipi_msg_type msg)
	242	{
	243	unsigned long flags;
	244	unsigned int cpu;
	245
	246	local_irq_save(flags);
	247
	248	for_each_cpu_mask(cpu, callmap) {
	249	struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
	250
	251	spin_lock(&ipi->lock);
	252	ipi->bits \|= 1 << msg;
	253	spin_unlock(&ipi->lock);
	254	}
	255
	256	/*
	257	* Call the platform specific cross-CPU call function.
	258	*/
	259	smp_cross_call(callmap);
	260
	261	local_irq_restore(flags);
	262	}
	263
	264	/*
	265	* You must not call this function with disabled interrupts, from a
	266	* hardware interrupt handler, nor from a bottom half handler.
	267	*/
	268	int smp_call_function_on_cpu(void (func)(void info), void *info, int retry,
	269	int wait, cpumask_t callmap)
	270	{
271	struct smp_call_struct data;
272	unsigned long timeout;
273	int ret = 0;
274
275	data.func = func;
276	data.info = info;
277	data.wait = wait;
278
279	cpu_clear(smp_processor_id(), callmap);
280	if (cpus_empty(callmap))
281	goto out;
282
283	data.pending = callmap;
284	if (wait)
285	data.unfinished = callmap;
286
287	/*
288	* try to get the mutex on smp_call_function_data
289	*/
290	spin_lock(&smp_call_function_lock);
291	smp_call_function_data = &data;
292
293	send_ipi_message(callmap, IPI_CALL_FUNC);
294
295	timeout = jiffies + HZ;
296	while (!cpus_empty(data.pending) && time_before(jiffies, timeout))
297	barrier();
298
299	/*
300	* did we time out?
301	*/
302	if (!cpus_empty(data.pending)) {
303	/*
304	* this may be causing our panic - report it
305	*/
306	printk(KERN_CRIT
307	"CPU%u: smp_call_function timeout for %p(%p)\n"
308	" callmap %lx pending %lx, %swait\n",
309	smp_processor_id(), func, info, callmap, data.pending,
310	wait ? "" : "no ");
311
312	/*
313	* TRACE
314	*/
315	timeout = jiffies + (5 * HZ);
316	while (!cpus_empty(data.pending) && time_before(jiffies, timeout))
317	barrier();
318
319	if (cpus_empty(data.pending))
320	printk(KERN_CRIT " RESOLVED\n");
321	else
322	printk(KERN_CRIT " STILL STUCK\n");
323	}
324
325	/*
326	* whatever happened, we're done with the data, so release it
327	*/
328	smp_call_function_data = NULL;
329	spin_unlock(&smp_call_function_lock);
330
331	if (!cpus_empty(data.pending)) {
332	ret = -ETIMEDOUT;
333	goto out;
334	}
335
336	if (wait)
337	while (!cpus_empty(data.unfinished))
338	barrier();
339	out:
340
341	return 0;
342	}
343
344	int smp_call_function(void (func)(void info), void *info, int retry,
345	int wait)
346	{
347	return smp_call_function_on_cpu(func, info, retry, wait,
348	cpu_online_map);
349	}
350
351	void show_ipi_list(struct seq_file *p)
352	{
353	unsigned int cpu;
354
355	seq_puts(p, "IPI:");
356
357	for_each_online_cpu(cpu)
358	seq_printf(p, " %10lu", per_cpu(ipi_data, cpu).ipi_count);
359
360	seq_putc(p, '\n');
361	}
362
363	static void ipi_timer(struct pt_regs *regs)
364	{
365	int user = user_mode(regs);
366
367	irq_enter();
368	profile_tick(CPU_PROFILING, regs);
369	update_process_times(user);
370	irq_exit();
371	}
372
373	/*
374	* ipi_call_function - handle IPI from smp_call_function()
375	*
376	* Note that we copy data out of the cross-call structure and then
377	* let the caller know that we're here and have done with their data
378	*/
379	static void ipi_call_function(unsigned int cpu)
380	{
381	struct smp_call_struct *data = smp_call_function_data;
382	void (func)(void info) = data->func;
383	void *info = data->info;
384	int wait = data->wait;
385
386	cpu_clear(cpu, data->pending);
387
388	func(info);
389
390	if (wait)
391	cpu_clear(cpu, data->unfinished);
392	}
393
394	static DEFINE_SPINLOCK(stop_lock);
395
396	/*
397	* ipi_cpu_stop - handle IPI from smp_send_stop()
398	*/
399	static void ipi_cpu_stop(unsigned int cpu)
400	{
401	spin_lock(&stop_lock);
402	printk(KERN_CRIT "CPU%u: stopping\n", cpu);
403	dump_stack();
404	spin_unlock(&stop_lock);
405
406	cpu_clear(cpu, cpu_online_map);
407
408	local_fiq_disable();
409	local_irq_disable();
410
411	while (1)
412	cpu_relax();
413	}
414
415	/*
416	* Main handler for inter-processor interrupts
417	*
418	* For ARM, the ipimask now only identifies a single
419	* category of IPI (Bit 1 IPIs have been replaced by a
420	* different mechanism):
421	*
422	* Bit 0 - Inter-processor function call
423	*/
424	void do_IPI(struct pt_regs *regs)
425	{
426	unsigned int cpu = smp_processor_id();
427	struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
428
429	ipi->ipi_count++;
430
431	for (;;) {
432	unsigned long msgs;
433
434	spin_lock(&ipi->lock);
435	msgs = ipi->bits;
436	ipi->bits = 0;
437	spin_unlock(&ipi->lock);
438
439	if (!msgs)
440	break;
441
442	do {
443	unsigned nextmsg;
444
445	nextmsg = msgs & -msgs;
446	msgs &= ~nextmsg;
447	nextmsg = ffz(~nextmsg);
448
449	switch (nextmsg) {
450	case IPI_TIMER:
451	ipi_timer(regs);
452	break;
453
454	case IPI_RESCHEDULE:
455	/*
456	* nothing more to do - eveything is
457	* done on the interrupt return path
458	*/
459	break;
460
461	case IPI_CALL_FUNC:
462	ipi_call_function(cpu);
463	break;
464
465	case IPI_CPU_STOP:
466	ipi_cpu_stop(cpu);
467	break;
468
469	default:
470	printk(KERN_CRIT "CPU%u: Unknown IPI message 0x%x\n",
471	cpu, nextmsg);
472	break;
473	}
474	} while (msgs);
475	}
476	}
477
478	void smp_send_reschedule(int cpu)
479	{
480	send_ipi_message(cpumask_of_cpu(cpu), IPI_RESCHEDULE);
481	}
482
483	void smp_send_timer(void)
484	{
485	cpumask_t mask = cpu_online_map;
486	cpu_clear(smp_processor_id(), mask);
487	send_ipi_message(mask, IPI_TIMER);
488	}
489
490	void smp_send_stop(void)
491	{
492	cpumask_t mask = cpu_online_map;
493	cpu_clear(smp_processor_id(), mask);
494	send_ipi_message(mask, IPI_CPU_STOP);
495	}
496
497	/*
498	* not supported here
499	*/
500	int __init setup_profiling_timer(unsigned int multiplier)
501	{
502	return -EINVAL;
503	}