[deliverable/linux.git] / arch / x86 / xen / smp.c

/*
 * Xen SMP support
 *
 * This file implements the Xen versions of smp_ops.  SMP under Xen is
 * very straightforward.  Bringing a CPU up is simply a matter of
 * loading its initial context and setting it running.
 *
 * IPIs are handled through the Xen event mechanism.
 *
 * Because virtual CPUs can be scheduled onto any real CPU, there's no
 * useful topology information for the kernel to make use of.  As a
 * result, all CPUs are treated as if they're single-core and
 * single-threaded.
 */
#include <linux/sched.h>
#include <linux/err.h>
#include <linux/smp.h>

#include <asm/paravirt.h>
#include <asm/desc.h>
#include <asm/pgtable.h>
#include <asm/cpu.h>

#include <xen/interface/xen.h>
#include <xen/interface/vcpu.h>

#include <asm/xen/interface.h>
#include <asm/xen/hypercall.h>

#include <xen/page.h>
#include <xen/events.h>

#include "xen-ops.h"
#include "mmu.h"

cpumask_t xen_cpu_initialized_map;

static DEFINE_PER_CPU(int, resched_irq);
static DEFINE_PER_CPU(int, callfunc_irq);
static DEFINE_PER_CPU(int, callfuncsingle_irq);
static DEFINE_PER_CPU(int, debug_irq) = -1;

static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);

/*
 * Reschedule call back. Nothing to do,
 * all the work is done automatically when
 * we return from the interrupt.
 */
static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
{
#ifdef CONFIG_X86_32
	__get_cpu_var(irq_stat).irq_resched_count++;
#else
	add_pda(irq_resched_count, 1);
#endif

	return IRQ_HANDLED;
}

static __cpuinit void cpu_bringup(void)
{
	int cpu = smp_processor_id();

	cpu_init();
	touch_softlockup_watchdog();
	preempt_disable();

	xen_enable_sysenter();
	xen_enable_syscall();

	cpu = smp_processor_id();
	smp_store_cpu_info(cpu);
	cpu_data(cpu).x86_max_cores = 1;
	set_cpu_sibling_map(cpu);

	xen_setup_cpu_clockevents();

	cpu_set(cpu, cpu_online_map);
	x86_write_percpu(cpu_state, CPU_ONLINE);
	wmb();

	/* We can take interrupts now: we're officially "up". */
	local_irq_enable();

	wmb();			/* make sure everything is out */
}

static __cpuinit void cpu_bringup_and_idle(void)
{
	cpu_bringup();
	cpu_idle();
}

static int xen_smp_intr_init(unsigned int cpu)
{
	int rc;
	const char *resched_name, *callfunc_name, *debug_name;

	resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
	rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
				    cpu,
				    xen_reschedule_interrupt,
				    IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
				    resched_name,
				    NULL);
	if (rc < 0)
		goto fail;
	per_cpu(resched_irq, cpu) = rc;

	callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
				    cpu,
				    xen_call_function_interrupt,
				    IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
				    callfunc_name,
				    NULL);
	if (rc < 0)
		goto fail;
	per_cpu(callfunc_irq, cpu) = rc;

	debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
	rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt,
				     IRQF_DISABLED | IRQF_PERCPU | IRQF_NOBALANCING,
				     debug_name, NULL);
	if (rc < 0)
		goto fail;
	per_cpu(debug_irq, cpu) = rc;

	callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
				    cpu,
				    xen_call_function_single_interrupt,
				    IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
				    callfunc_name,
				    NULL);
	if (rc < 0)
		goto fail;
	per_cpu(callfuncsingle_irq, cpu) = rc;

	return 0;

 fail:
	if (per_cpu(resched_irq, cpu) >= 0)
		unbind_from_irqhandler(per_cpu(resched_irq, cpu), NULL);
	if (per_cpu(callfunc_irq, cpu) >= 0)
		unbind_from_irqhandler(per_cpu(callfunc_irq, cpu), NULL);
	if (per_cpu(debug_irq, cpu) >= 0)
		unbind_from_irqhandler(per_cpu(debug_irq, cpu), NULL);
	if (per_cpu(callfuncsingle_irq, cpu) >= 0)
		unbind_from_irqhandler(per_cpu(callfuncsingle_irq, cpu), NULL);

	return rc;
}

static void __init xen_fill_possible_map(void)
{
	int i, rc;

	for (i = 0; i < NR_CPUS; i++) {
		rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
		if (rc >= 0) {
			num_processors++;
			cpu_set(i, cpu_possible_map);
		}
	}
}

static void __init xen_smp_prepare_boot_cpu(void)
{
	BUG_ON(smp_processor_id() != 0);
	native_smp_prepare_boot_cpu();

	/* We've switched to the "real" per-cpu gdt, so make sure the
	   old memory can be recycled */
	make_lowmem_page_readwrite(&per_cpu_var(gdt_page));

	xen_setup_vcpu_info_placement();
}

static void __init xen_smp_prepare_cpus(unsigned int max_cpus)
{
	unsigned cpu;

	xen_init_lock_cpu(0);

	smp_store_cpu_info(0);
	cpu_data(0).x86_max_cores = 1;
	set_cpu_sibling_map(0);

	if (xen_smp_intr_init(0))
		BUG();

	xen_cpu_initialized_map = cpumask_of_cpu(0);

	/* Restrict the possible_map according to max_cpus. */
	while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
		for (cpu = NR_CPUS - 1; !cpu_possible(cpu); cpu--)
			continue;
		cpu_clear(cpu, cpu_possible_map);
	}

	for_each_possible_cpu (cpu) {
		struct task_struct *idle;

		if (cpu == 0)
			continue;

		idle = fork_idle(cpu);
		if (IS_ERR(idle))
			panic("failed fork for CPU %d", cpu);

		cpu_set(cpu, cpu_present_map);
	}
}

static __cpuinit int
cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
{
	struct vcpu_guest_context *ctxt;
	struct desc_struct *gdt;

	if (cpu_test_and_set(cpu, xen_cpu_initialized_map))
		return 0;

	ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
	if (ctxt == NULL)
		return -ENOMEM;

	gdt = get_cpu_gdt_table(cpu);

	ctxt->flags = VGCF_IN_KERNEL;
	ctxt->user_regs.ds = __USER_DS;
	ctxt->user_regs.es = __USER_DS;
	ctxt->user_regs.ss = __KERNEL_DS;
#ifdef CONFIG_X86_32
	ctxt->user_regs.fs = __KERNEL_PERCPU;
#endif
	ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
	ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */

	memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));

	xen_copy_trap_info(ctxt->trap_ctxt);

	ctxt->ldt_ents = 0;

	BUG_ON((unsigned long)gdt & ~PAGE_MASK);
	make_lowmem_page_readonly(gdt);

	ctxt->gdt_frames[0] = virt_to_mfn(gdt);
	ctxt->gdt_ents      = GDT_ENTRIES;

	ctxt->user_regs.cs = __KERNEL_CS;
	ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);

	ctxt->kernel_ss = __KERNEL_DS;
	ctxt->kernel_sp = idle->thread.sp0;

#ifdef CONFIG_X86_32
	ctxt->event_callback_cs     = __KERNEL_CS;
	ctxt->failsafe_callback_cs  = __KERNEL_CS;
#endif
	ctxt->event_callback_eip    = (unsigned long)xen_hypervisor_callback;
	ctxt->failsafe_callback_eip = (unsigned long)xen_failsafe_callback;

	per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
	ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(swapper_pg_dir));

	if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt))
		BUG();

	kfree(ctxt);
	return 0;
}

static int __cpuinit xen_cpu_up(unsigned int cpu)
{
	struct task_struct *idle = idle_task(cpu);
	int rc;

#ifdef CONFIG_X86_64
	/* Allocate node local memory for AP pdas */
	WARN_ON(cpu == 0);
	if (cpu > 0) {
		rc = get_local_pda(cpu);
		if (rc)
			return rc;
	}
#endif

#ifdef CONFIG_X86_32
	init_gdt(cpu);
	per_cpu(current_task, cpu) = idle;
	irq_ctx_init(cpu);
#else
	cpu_pda(cpu)->pcurrent = idle;
	clear_tsk_thread_flag(idle, TIF_FORK);
#endif
	xen_setup_timer(cpu);
	xen_init_lock_cpu(cpu);

	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;

	/* make sure interrupts start blocked */
	per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1;

	rc = cpu_initialize_context(cpu, idle);
	if (rc)
		return rc;

	if (num_online_cpus() == 1)
		alternatives_smp_switch(1);

	rc = xen_smp_intr_init(cpu);
	if (rc)
		return rc;

	rc = HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL);
	BUG_ON(rc);

	while(per_cpu(cpu_state, cpu) != CPU_ONLINE) {
		HYPERVISOR_sched_op(SCHEDOP_yield, 0);
		barrier();
	}

	return 0;
}

static void xen_smp_cpus_done(unsigned int max_cpus)
{
}

#ifdef CONFIG_HOTPLUG_CPU
int xen_cpu_disable(void)
{
	unsigned int cpu = smp_processor_id();
	if (cpu == 0)
		return -EBUSY;

	cpu_disable_common();

	load_cr3(swapper_pg_dir);
	return 0;
}

void xen_cpu_die(unsigned int cpu)
{
	while (HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) {
		current->state = TASK_UNINTERRUPTIBLE;
		schedule_timeout(HZ/10);
	}
	unbind_from_irqhandler(per_cpu(resched_irq, cpu), NULL);
	unbind_from_irqhandler(per_cpu(callfunc_irq, cpu), NULL);
	unbind_from_irqhandler(per_cpu(debug_irq, cpu), NULL);
	unbind_from_irqhandler(per_cpu(callfuncsingle_irq, cpu), NULL);
	xen_uninit_lock_cpu(cpu);
	xen_teardown_timer(cpu);

	if (num_online_cpus() == 1)
		alternatives_smp_switch(0);
}

void xen_play_dead(void)
{
	play_dead_common();
	HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL);
	cpu_bringup();
}

#else /* !CONFIG_HOTPLUG_CPU */
int xen_cpu_disable(void)
{
	return -ENOSYS;
}

void xen_cpu_die(unsigned int cpu)
{
	BUG();
}

void xen_play_dead(void)
{
	BUG();
}

#endif
static void stop_self(void *v)
{
	int cpu = smp_processor_id();

	/* make sure we're not pinning something down */
	load_cr3(swapper_pg_dir);
	/* should set up a minimal gdt */

	HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL);
	BUG();
}

static void xen_smp_send_stop(void)
{
	smp_call_function(stop_self, NULL, 0);
}

static void xen_smp_send_reschedule(int cpu)
{
	xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
}

static void xen_send_IPI_mask(cpumask_t mask, enum ipi_vector vector)
{
	unsigned cpu;

	cpus_and(mask, mask, cpu_online_map);

	for_each_cpu_mask_nr(cpu, mask)
		xen_send_IPI_one(cpu, vector);
}

static void xen_smp_send_call_function_ipi(cpumask_t mask)
{
	int cpu;

	xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);

	/* Make sure other vcpus get a chance to run if they need to. */
	for_each_cpu_mask_nr(cpu, mask) {
		if (xen_vcpu_stolen(cpu)) {
			HYPERVISOR_sched_op(SCHEDOP_yield, 0);
			break;
		}
	}
}

static void xen_smp_send_call_function_single_ipi(int cpu)
{
	xen_send_IPI_mask(cpumask_of_cpu(cpu), XEN_CALL_FUNCTION_SINGLE_VECTOR);
}

static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
{
	irq_enter();
	generic_smp_call_function_interrupt();
#ifdef CONFIG_X86_32
	__get_cpu_var(irq_stat).irq_call_count++;
#else
	add_pda(irq_call_count, 1);
#endif
	irq_exit();

	return IRQ_HANDLED;
}

static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
{
	irq_enter();
	generic_smp_call_function_single_interrupt();
#ifdef CONFIG_X86_32
	__get_cpu_var(irq_stat).irq_call_count++;
#else
	add_pda(irq_call_count, 1);
#endif
	irq_exit();

	return IRQ_HANDLED;
}

static const struct smp_ops xen_smp_ops __initdata = {
	.smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu,
	.smp_prepare_cpus = xen_smp_prepare_cpus,
	.smp_cpus_done = xen_smp_cpus_done,

	.cpu_up = xen_cpu_up,
	.cpu_die = xen_cpu_die,
	.cpu_disable = xen_cpu_disable,
	.play_dead = xen_play_dead,

	.smp_send_stop = xen_smp_send_stop,
	.smp_send_reschedule = xen_smp_send_reschedule,

	.send_call_func_ipi = xen_smp_send_call_function_ipi,
	.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
};

void __init xen_smp_init(void)
{
	smp_ops = xen_smp_ops;
	xen_fill_possible_map();
	xen_init_spinlocks();
}
Commit	Line	Data
f87e4cac JF	1	/*
	2	* Xen SMP support
	3	*
	4	* This file implements the Xen versions of smp_ops. SMP under Xen is
	5	* very straightforward. Bringing a CPU up is simply a matter of
	6	* loading its initial context and setting it running.
	7	*
	8	* IPIs are handled through the Xen event mechanism.
	9	*
	10	* Because virtual CPUs can be scheduled onto any real CPU, there's no
	11	* useful topology information for the kernel to make use of. As a
	12	* result, all CPUs are treated as if they're single-core and
	13	* single-threaded.
f87e4cac JF	14	*/
	15	#include <linux/sched.h>
	16	#include <linux/err.h>
	17	#include <linux/smp.h>
	18
	19	#include <asm/paravirt.h>
	20	#include <asm/desc.h>
	21	#include <asm/pgtable.h>
	22	#include <asm/cpu.h>
	23
	24	#include <xen/interface/xen.h>
	25	#include <xen/interface/vcpu.h>
	26
	27	#include <asm/xen/interface.h>
	28	#include <asm/xen/hypercall.h>
	29
	30	#include <xen/page.h>
	31	#include <xen/events.h>
	32
	33	#include "xen-ops.h"
	34	#include "mmu.h"
	35
0e91398f	36	cpumask_t xen_cpu_initialized_map;
f87e4cac	37
3b16cf87 JA	38	static DEFINE_PER_CPU(int, resched_irq);
	39	static DEFINE_PER_CPU(int, callfunc_irq);
	40	static DEFINE_PER_CPU(int, callfuncsingle_irq);
	41	static DEFINE_PER_CPU(int, debug_irq) = -1;
f87e4cac JF	42
f87e4cac JF	43	static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
3b16cf87	44	static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
f87e4cac JF	45
	46	/*
	47	* Reschedule call back. Nothing to do,
	48	* all the work is done automatically when
	49	* we return from the interrupt.
	50	*/
	51	static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
	52	{
38bb5ab4 JF	53	#ifdef CONFIG_X86_32
	54	__get_cpu_var(irq_stat).irq_resched_count++;
	55	#else
	56	add_pda(irq_resched_count, 1);
	57	#endif
	58
f87e4cac JF	59	return IRQ_HANDLED;
	60	}
	61
d68d82af	62	static __cpuinit void cpu_bringup(void)
f87e4cac JF	63	{
	64	int cpu = smp_processor_id();
	65
	66	cpu_init();
d68d82af	67	touch_softlockup_watchdog();
c7b75947 JF	68	preempt_disable();
c7b75947 JF	69
e2a81baf	70	xen_enable_sysenter();
6fcac6d3	71	xen_enable_syscall();
f87e4cac	72
c7b75947 JF	73	cpu = smp_processor_id();
	74	smp_store_cpu_info(cpu);
	75	cpu_data(cpu).x86_max_cores = 1;
	76	set_cpu_sibling_map(cpu);
f87e4cac JF	77
	78	xen_setup_cpu_clockevents();
	79
c7b75947 JF	80	cpu_set(cpu, cpu_online_map);
	81	x86_write_percpu(cpu_state, CPU_ONLINE);
	82	wmb();
	83
f87e4cac JF	84	/* We can take interrupts now: we're officially "up". */
	85	local_irq_enable();
	86
	87	wmb(); /* make sure everything is out */
d68d82af AN	88	}
	89
	90	static __cpuinit void cpu_bringup_and_idle(void)
	91	{
	92	cpu_bringup();
f87e4cac JF	93	cpu_idle();
	94	}
	95
	96	static int xen_smp_intr_init(unsigned int cpu)
	97	{
	98	int rc;
ee523ca1	99	const char resched_name, callfunc_name, *debug_name;
f87e4cac JF	100
	101	resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
	102	rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
	103	cpu,
	104	xen_reschedule_interrupt,
	105	IRQF_DISABLED\|IRQF_PERCPU\|IRQF_NOBALANCING,
	106	resched_name,
	107	NULL);
	108	if (rc < 0)
	109	goto fail;
	110	per_cpu(resched_irq, cpu) = rc;
	111
	112	callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
	113	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
	114	cpu,
	115	xen_call_function_interrupt,
	116	IRQF_DISABLED\|IRQF_PERCPU\|IRQF_NOBALANCING,
	117	callfunc_name,
	118	NULL);
	119	if (rc < 0)
	120	goto fail;
	121	per_cpu(callfunc_irq, cpu) = rc;
	122
ee523ca1 JF	123	debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
	124	rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt,
	125	IRQF_DISABLED \| IRQF_PERCPU \| IRQF_NOBALANCING,
	126	debug_name, NULL);
	127	if (rc < 0)
	128	goto fail;
	129	per_cpu(debug_irq, cpu) = rc;
	130
3b16cf87 JA	131	callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
	132	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
	133	cpu,
	134	xen_call_function_single_interrupt,
	135	IRQF_DISABLED\|IRQF_PERCPU\|IRQF_NOBALANCING,
	136	callfunc_name,
	137	NULL);
	138	if (rc < 0)
	139	goto fail;
	140	per_cpu(callfuncsingle_irq, cpu) = rc;
	141
f87e4cac JF	142	return 0;
	143
	144	fail:
	145	if (per_cpu(resched_irq, cpu) >= 0)
	146	unbind_from_irqhandler(per_cpu(resched_irq, cpu), NULL);
	147	if (per_cpu(callfunc_irq, cpu) >= 0)
	148	unbind_from_irqhandler(per_cpu(callfunc_irq, cpu), NULL);
ee523ca1 JF	149	if (per_cpu(debug_irq, cpu) >= 0)
ee523ca1 JF	150	unbind_from_irqhandler(per_cpu(debug_irq, cpu), NULL);
3b16cf87 JA	151	if (per_cpu(callfuncsingle_irq, cpu) >= 0)
	152	unbind_from_irqhandler(per_cpu(callfuncsingle_irq, cpu), NULL);
	153
f87e4cac JF	154	return rc;
	155	}
	156
c7b75947	157	static void __init xen_fill_possible_map(void)
f87e4cac JF	158	{
	159	int i, rc;
	160
	161	for (i = 0; i < NR_CPUS; i++) {
	162	rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
4560a294 JF	163	if (rc >= 0) {
4560a294 JF	164	num_processors++;
f87e4cac	165	cpu_set(i, cpu_possible_map);
4560a294	166	}
f87e4cac JF	167	}
	168	}
	169
a9e7062d	170	static void __init xen_smp_prepare_boot_cpu(void)
f87e4cac	171	{
f87e4cac JF	172	BUG_ON(smp_processor_id() != 0);
	173	native_smp_prepare_boot_cpu();
	174
f87e4cac JF	175	/* We've switched to the "real" per-cpu gdt, so make sure the
f87e4cac JF	176	old memory can be recycled */
c7b75947	177	make_lowmem_page_readwrite(&per_cpu_var(gdt_page));
60223a32 JF	178
60223a32 JF	179	xen_setup_vcpu_info_placement();
f87e4cac JF	180	}
f87e4cac JF	181
a9e7062d	182	static void __init xen_smp_prepare_cpus(unsigned int max_cpus)
f87e4cac JF	183	{
	184	unsigned cpu;
	185
2d9e1e2f JF	186	xen_init_lock_cpu(0);
2d9e1e2f JF	187
f87e4cac	188	smp_store_cpu_info(0);
c7b75947	189	cpu_data(0).x86_max_cores = 1;
f87e4cac JF	190	set_cpu_sibling_map(0);
	191
	192	if (xen_smp_intr_init(0))
	193	BUG();
	194
ecaa6c9d	195	xen_cpu_initialized_map = cpumask_of_cpu(0);
f87e4cac JF	196
	197	/* Restrict the possible_map according to max_cpus. */
	198	while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
7c04e64a	199	for (cpu = NR_CPUS - 1; !cpu_possible(cpu); cpu--)
f87e4cac JF	200	continue;
	201	cpu_clear(cpu, cpu_possible_map);
	202	}
	203
	204	for_each_possible_cpu (cpu) {
	205	struct task_struct *idle;
	206
	207	if (cpu == 0)
	208	continue;
	209
	210	idle = fork_idle(cpu);
	211	if (IS_ERR(idle))
	212	panic("failed fork for CPU %d", cpu);
	213
	214	cpu_set(cpu, cpu_present_map);
	215	}
f87e4cac JF	216	}
	217
	218	static __cpuinit int
	219	cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
	220	{
	221	struct vcpu_guest_context *ctxt;
c7b75947	222	struct desc_struct *gdt;
f87e4cac	223
ecaa6c9d	224	if (cpu_test_and_set(cpu, xen_cpu_initialized_map))
f87e4cac JF	225	return 0;
	226
	227	ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
	228	if (ctxt == NULL)
	229	return -ENOMEM;
	230
c7b75947 JF	231	gdt = get_cpu_gdt_table(cpu);
c7b75947 JF	232
f87e4cac JF	233	ctxt->flags = VGCF_IN_KERNEL;
	234	ctxt->user_regs.ds = __USER_DS;
	235	ctxt->user_regs.es = __USER_DS;
f87e4cac	236	ctxt->user_regs.ss = __KERNEL_DS;
c7b75947 JF	237	#ifdef CONFIG_X86_32
	238	ctxt->user_regs.fs = __KERNEL_PERCPU;
	239	#endif
f87e4cac JF	240	ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
	241	ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
	242
	243	memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
	244
	245	xen_copy_trap_info(ctxt->trap_ctxt);
	246
	247	ctxt->ldt_ents = 0;
	248
c7b75947 JF	249	BUG_ON((unsigned long)gdt & ~PAGE_MASK);
c7b75947 JF	250	make_lowmem_page_readonly(gdt);
f87e4cac	251
c7b75947 JF	252	ctxt->gdt_frames[0] = virt_to_mfn(gdt);
c7b75947 JF	253	ctxt->gdt_ents = GDT_ENTRIES;
f87e4cac JF	254
f87e4cac JF	255	ctxt->user_regs.cs = __KERNEL_CS;
faca6227	256	ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
f87e4cac JF	257
f87e4cac JF	258	ctxt->kernel_ss = __KERNEL_DS;
faca6227	259	ctxt->kernel_sp = idle->thread.sp0;
f87e4cac	260
c7b75947	261	#ifdef CONFIG_X86_32
f87e4cac	262	ctxt->event_callback_cs = __KERNEL_CS;
f87e4cac	263	ctxt->failsafe_callback_cs = __KERNEL_CS;
c7b75947 JF	264	#endif
c7b75947 JF	265	ctxt->event_callback_eip = (unsigned long)xen_hypervisor_callback;
f87e4cac JF	266	ctxt->failsafe_callback_eip = (unsigned long)xen_failsafe_callback;
	267
	268	per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
	269	ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(swapper_pg_dir));
	270
	271	if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt))
	272	BUG();
	273
	274	kfree(ctxt);
	275	return 0;
	276	}
	277
a9e7062d	278	static int __cpuinit xen_cpu_up(unsigned int cpu)
f87e4cac JF	279	{
	280	struct task_struct *idle = idle_task(cpu);
	281	int rc;
	282
c7b75947 JF	283	#ifdef CONFIG_X86_64
	284	/* Allocate node local memory for AP pdas */
	285	WARN_ON(cpu == 0);
	286	if (cpu > 0) {
	287	rc = get_local_pda(cpu);
	288	if (rc)
	289	return rc;
	290	}
	291	#endif
	292
	293	#ifdef CONFIG_X86_32
f87e4cac JF	294	init_gdt(cpu);
f87e4cac JF	295	per_cpu(current_task, cpu) = idle;
f87e4cac	296	irq_ctx_init(cpu);
c7b75947 JF	297	#else
	298	cpu_pda(cpu)->pcurrent = idle;
	299	clear_tsk_thread_flag(idle, TIF_FORK);
	300	#endif
f87e4cac	301	xen_setup_timer(cpu);
2d9e1e2f	302	xen_init_lock_cpu(cpu);
f87e4cac	303
c7b75947 JF	304	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
c7b75947 JF	305
f87e4cac JF	306	/* make sure interrupts start blocked */
	307	per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1;
	308
	309	rc = cpu_initialize_context(cpu, idle);
	310	if (rc)
	311	return rc;
	312
	313	if (num_online_cpus() == 1)
	314	alternatives_smp_switch(1);
	315
	316	rc = xen_smp_intr_init(cpu);
	317	if (rc)
	318	return rc;
	319
f87e4cac JF	320	rc = HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL);
	321	BUG_ON(rc);
	322
c7b75947 JF	323	while(per_cpu(cpu_state, cpu) != CPU_ONLINE) {
	324	HYPERVISOR_sched_op(SCHEDOP_yield, 0);
	325	barrier();
	326	}
	327
f87e4cac JF	328	return 0;
	329	}
	330
a9e7062d	331	static void xen_smp_cpus_done(unsigned int max_cpus)
f87e4cac JF	332	{
	333	}
	334
2737146b	335	#ifdef CONFIG_HOTPLUG_CPU
d68d82af AN	336	int xen_cpu_disable(void)
	337	{
	338	unsigned int cpu = smp_processor_id();
	339	if (cpu == 0)
	340	return -EBUSY;
	341
	342	cpu_disable_common();
	343
	344	load_cr3(swapper_pg_dir);
	345	return 0;
	346	}
	347
	348	void xen_cpu_die(unsigned int cpu)
	349	{
	350	while (HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) {
	351	current->state = TASK_UNINTERRUPTIBLE;
	352	schedule_timeout(HZ/10);
	353	}
	354	unbind_from_irqhandler(per_cpu(resched_irq, cpu), NULL);
	355	unbind_from_irqhandler(per_cpu(callfunc_irq, cpu), NULL);
	356	unbind_from_irqhandler(per_cpu(debug_irq, cpu), NULL);
	357	unbind_from_irqhandler(per_cpu(callfuncsingle_irq, cpu), NULL);
	358	xen_uninit_lock_cpu(cpu);
	359	xen_teardown_timer(cpu);
	360
	361	if (num_online_cpus() == 1)
	362	alternatives_smp_switch(0);
	363	}
	364
	365	void xen_play_dead(void)
	366	{
	367	play_dead_common();
	368	HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL);
	369	cpu_bringup();
	370	}
	371
2737146b AN	372	#else /* !CONFIG_HOTPLUG_CPU */
	373	int xen_cpu_disable(void)
	374	{
	375	return -ENOSYS;
	376	}
	377
	378	void xen_cpu_die(unsigned int cpu)
	379	{
	380	BUG();
	381	}
	382
	383	void xen_play_dead(void)
	384	{
	385	BUG();
	386	}
	387
	388	#endif
f87e4cac JF	389	static void stop_self(void *v)
	390	{
	391	int cpu = smp_processor_id();
	392
	393	/* make sure we're not pinning something down */
	394	load_cr3(swapper_pg_dir);
	395	/* should set up a minimal gdt */
	396
	397	HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL);
	398	BUG();
	399	}
	400
a9e7062d	401	static void xen_smp_send_stop(void)
f87e4cac	402	{
8691e5a8	403	smp_call_function(stop_self, NULL, 0);
f87e4cac JF	404	}
f87e4cac JF	405
a9e7062d	406	static void xen_smp_send_reschedule(int cpu)
f87e4cac JF	407	{
	408	xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
	409	}
	410
f87e4cac JF	411	static void xen_send_IPI_mask(cpumask_t mask, enum ipi_vector vector)
	412	{
	413	unsigned cpu;
	414
	415	cpus_and(mask, mask, cpu_online_map);
	416
334ef7a7	417	for_each_cpu_mask_nr(cpu, mask)
f87e4cac JF	418	xen_send_IPI_one(cpu, vector);
	419	}
	420
a9e7062d	421	static void xen_smp_send_call_function_ipi(cpumask_t mask)
3b16cf87 JA	422	{
	423	int cpu;
	424
	425	xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
	426
	427	/* Make sure other vcpus get a chance to run if they need to. */
82638844	428	for_each_cpu_mask_nr(cpu, mask) {
3b16cf87 JA	429	if (xen_vcpu_stolen(cpu)) {
	430	HYPERVISOR_sched_op(SCHEDOP_yield, 0);
	431	break;
	432	}
	433	}
	434	}
	435
a9e7062d	436	static void xen_smp_send_call_function_single_ipi(int cpu)
3b16cf87 JA	437	{
	438	xen_send_IPI_mask(cpumask_of_cpu(cpu), XEN_CALL_FUNCTION_SINGLE_VECTOR);
	439	}
	440
f87e4cac JF	441	static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
f87e4cac JF	442	{
f87e4cac	443	irq_enter();
3b16cf87	444	generic_smp_call_function_interrupt();
c7b75947	445	#ifdef CONFIG_X86_32
38e760a1	446	__get_cpu_var(irq_stat).irq_call_count++;
c7b75947 JF	447	#else
	448	add_pda(irq_call_count, 1);
	449	#endif
f87e4cac JF	450	irq_exit();
f87e4cac JF	451
f87e4cac JF	452	return IRQ_HANDLED;
	453	}
	454
3b16cf87	455	static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
f87e4cac	456	{
3b16cf87 JA	457	irq_enter();
3b16cf87 JA	458	generic_smp_call_function_single_interrupt();
c7b75947	459	#ifdef CONFIG_X86_32
3b16cf87	460	__get_cpu_var(irq_stat).irq_call_count++;
c7b75947 JF	461	#else
	462	add_pda(irq_call_count, 1);
	463	#endif
3b16cf87	464	irq_exit();
f87e4cac	465
3b16cf87	466	return IRQ_HANDLED;
f87e4cac	467	}
a9e7062d JF	468
	469	static const struct smp_ops xen_smp_ops __initdata = {
	470	.smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu,
	471	.smp_prepare_cpus = xen_smp_prepare_cpus,
a9e7062d JF	472	.smp_cpus_done = xen_smp_cpus_done,
a9e7062d JF	473
d68d82af AN	474	.cpu_up = xen_cpu_up,
	475	.cpu_die = xen_cpu_die,
	476	.cpu_disable = xen_cpu_disable,
	477	.play_dead = xen_play_dead,
	478
a9e7062d JF	479	.smp_send_stop = xen_smp_send_stop,
	480	.smp_send_reschedule = xen_smp_send_reschedule,
	481
	482	.send_call_func_ipi = xen_smp_send_call_function_ipi,
	483	.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
	484	};
	485
	486	void __init xen_smp_init(void)
	487	{
	488	smp_ops = xen_smp_ops;
c7b75947	489	xen_fill_possible_map();
2d9e1e2f	490	xen_init_spinlocks();
a9e7062d	491	}