2 * x86 SMP booting functions
4 * (c) 1995 Alan Cox, Building #3 <alan@redhat.com>
5 * (c) 1998, 1999, 2000 Ingo Molnar <mingo@redhat.com>
6 * Copyright 2001 Andi Kleen, SuSE Labs.
8 * Much of the core SMP work is based on previous work by Thomas Radke, to
9 * whom a great many thanks are extended.
11 * Thanks to Intel for making available several different Pentium,
12 * Pentium Pro and Pentium-II/Xeon MP machines.
13 * Original development of Linux SMP code supported by Caldera.
15 * This code is released under the GNU General Public License version 2
18 * Felix Koop : NR_CPUS used properly
19 * Jose Renau : Handle single CPU case.
20 * Alan Cox : By repeated request 8) - Total BogoMIP report.
21 * Greg Wright : Fix for kernel stacks panic.
22 * Erich Boleyn : MP v1.4 and additional changes.
23 * Matthias Sattler : Changes for 2.1 kernel map.
24 * Michel Lespinasse : Changes for 2.1 kernel map.
25 * Michael Chastain : Change trampoline.S to gnu as.
26 * Alan Cox : Dumb bug: 'B' step PPro's are fine
27 * Ingo Molnar : Added APIC timers, based on code
29 * Ingo Molnar : various cleanups and rewrites
30 * Tigran Aivazian : fixed "0.00 in /proc/uptime on SMP" bug.
31 * Maciej W. Rozycki : Bits for genuine 82489DX APICs
32 * Andi Kleen : Changed for SMP boot into long mode.
33 * Rusty Russell : Hacked into shape for new "hotplug" boot process.
34 * Andi Kleen : Converted to new state machine.
36 * Probably mostly hotplug CPU ready now.
37 * Ashok Raj : CPU hotplug support
41 #include <linux/config.h>
42 #include <linux/init.h>
45 #include <linux/kernel_stat.h>
46 #include <linux/smp_lock.h>
47 #include <linux/bootmem.h>
48 #include <linux/thread_info.h>
49 #include <linux/module.h>
51 #include <linux/delay.h>
52 #include <linux/mc146818rtc.h>
54 #include <asm/pgalloc.h>
56 #include <asm/kdebug.h>
57 #include <asm/tlbflush.h>
58 #include <asm/proto.h>
61 #include <asm/hw_irq.h>
64 /* Number of siblings per CPU package */
65 int smp_num_siblings
= 1;
66 EXPORT_SYMBOL(smp_num_siblings
);
68 /* Last level cache ID of each logical CPU */
69 u8 cpu_llc_id
[NR_CPUS
] __cpuinitdata
= {[0 ... NR_CPUS
-1] = BAD_APICID
};
70 EXPORT_SYMBOL(cpu_llc_id
);
72 /* Bitmask of currently online CPUs */
73 cpumask_t cpu_online_map __read_mostly
;
75 EXPORT_SYMBOL(cpu_online_map
);
78 * Private maps to synchronize booting between AP and BP.
79 * Probably not needed anymore, but it makes for easier debugging. -AK
81 cpumask_t cpu_callin_map
;
82 cpumask_t cpu_callout_map
;
83 EXPORT_SYMBOL(cpu_callout_map
);
85 cpumask_t cpu_possible_map
;
86 EXPORT_SYMBOL(cpu_possible_map
);
88 /* Per CPU bogomips and other parameters */
89 struct cpuinfo_x86 cpu_data
[NR_CPUS
] __cacheline_aligned
;
90 EXPORT_SYMBOL(cpu_data
);
92 /* Set when the idlers are all forked */
93 int smp_threads_ready
;
95 /* representing HT siblings of each logical CPU */
96 cpumask_t cpu_sibling_map
[NR_CPUS
] __read_mostly
;
97 EXPORT_SYMBOL(cpu_sibling_map
);
99 /* representing HT and core siblings of each logical CPU */
100 cpumask_t cpu_core_map
[NR_CPUS
] __read_mostly
;
101 EXPORT_SYMBOL(cpu_core_map
);
104 * Trampoline 80x86 program as an array.
107 extern unsigned char trampoline_data
[];
108 extern unsigned char trampoline_end
[];
110 /* State of each CPU */
111 DEFINE_PER_CPU(int, cpu_state
) = { 0 };
114 * Store all idle threads, this can be reused instead of creating
115 * a new thread. Also avoids complicated thread destroy functionality
118 struct task_struct
*idle_thread_array
[NR_CPUS
] __cpuinitdata
;
120 #define get_idle_for_cpu(x) (idle_thread_array[(x)])
121 #define set_idle_for_cpu(x,p) (idle_thread_array[(x)] = (p))
124 * Currently trivial. Write the real->protected mode
125 * bootstrap into the page concerned. The caller
126 * has made sure it's suitably aligned.
129 static unsigned long __cpuinit
setup_trampoline(void)
131 void *tramp
= __va(SMP_TRAMPOLINE_BASE
);
132 memcpy(tramp
, trampoline_data
, trampoline_end
- trampoline_data
);
133 return virt_to_phys(tramp
);
137 * The bootstrap kernel entry code has set these up. Save them for
141 static void __cpuinit
smp_store_cpu_info(int id
)
143 struct cpuinfo_x86
*c
= cpu_data
+ id
;
151 * New Funky TSC sync algorithm borrowed from IA64.
152 * Main advantage is that it doesn't reset the TSCs fully and
153 * in general looks more robust and it works better than my earlier
154 * attempts. I believe it was written by David Mosberger. Some minor
155 * adjustments for x86-64 by me -AK
157 * Original comment reproduced below.
159 * Synchronize TSC of the current (slave) CPU with the TSC of the
160 * MASTER CPU (normally the time-keeper CPU). We use a closed loop to
161 * eliminate the possibility of unaccounted-for errors (such as
162 * getting a machine check in the middle of a calibration step). The
163 * basic idea is for the slave to ask the master what itc value it has
164 * and to read its own itc before and after the master responds. Each
165 * iteration gives us three timestamps:
178 * The goal is to adjust the slave's TSC such that tm falls exactly
179 * half-way between t0 and t1. If we achieve this, the clocks are
180 * synchronized provided the interconnect between the slave and the
181 * master is symmetric. Even if the interconnect were asymmetric, we
182 * would still know that the synchronization error is smaller than the
183 * roundtrip latency (t0 - t1).
185 * When the interconnect is quiet and symmetric, this lets us
186 * synchronize the TSC to within one or two cycles. However, we can
187 * only *guarantee* that the synchronization is accurate to within a
188 * round-trip time, which is typically in the range of several hundred
189 * cycles (e.g., ~500 cycles). In practice, this means that the TSCs
190 * are usually almost perfectly synchronized, but we shouldn't assume
191 * that the accuracy is much better than half a micro second or so.
193 * [there are other errors like the latency of RDTSC and of the
194 * WRMSR. These can also account to hundreds of cycles. So it's
195 * probably worse. It claims 153 cycles error on a dual Opteron,
196 * but I suspect the numbers are actually somewhat worse -AK]
200 #define SLAVE (SMP_CACHE_BYTES/8)
202 /* Intentionally don't use cpu_relax() while TSC synchronization
203 because we don't want to go into funky power save modi or cause
204 hypervisors to schedule us away. Going to sleep would likely affect
205 latency and low latency is the primary objective here. -AK */
206 #define no_cpu_relax() barrier()
208 static __cpuinitdata
DEFINE_SPINLOCK(tsc_sync_lock
);
209 static volatile __cpuinitdata
unsigned long go
[SLAVE
+ 1];
210 static int notscsync __cpuinitdata
;
212 #undef DEBUG_TSC_SYNC
214 #define NUM_ROUNDS 64 /* magic value */
215 #define NUM_ITERS 5 /* likewise */
217 /* Callback on boot CPU */
218 static __cpuinit
void sync_master(void *arg
)
220 unsigned long flags
, i
;
224 local_irq_save(flags
);
226 for (i
= 0; i
< NUM_ROUNDS
*NUM_ITERS
; ++i
) {
233 local_irq_restore(flags
);
237 * Return the number of cycles by which our tsc differs from the tsc
238 * on the master (time-keeper) CPU. A positive number indicates our
239 * tsc is ahead of the master, negative that it is behind.
242 get_delta(long *rt
, long *master
)
244 unsigned long best_t0
= 0, best_t1
= ~0UL, best_tm
= 0;
245 unsigned long tcenter
, t0
, t1
, tm
;
248 for (i
= 0; i
< NUM_ITERS
; ++i
) {
251 while (!(tm
= go
[SLAVE
]))
256 if (t1
- t0
< best_t1
- best_t0
)
257 best_t0
= t0
, best_t1
= t1
, best_tm
= tm
;
260 *rt
= best_t1
- best_t0
;
261 *master
= best_tm
- best_t0
;
263 /* average best_t0 and best_t1 without overflow: */
264 tcenter
= (best_t0
/2 + best_t1
/2);
265 if (best_t0
% 2 + best_t1
% 2 == 2)
267 return tcenter
- best_tm
;
270 static __cpuinit
void sync_tsc(unsigned int master
)
273 long delta
, adj
, adjust_latency
= 0;
274 unsigned long flags
, rt
, master_time_stamp
, bound
;
275 #ifdef DEBUG_TSC_SYNC
276 static struct syncdebug
{
277 long rt
; /* roundtrip time */
278 long master
; /* master's timestamp */
279 long diff
; /* difference between midpoint and master's timestamp */
280 long lat
; /* estimate of tsc adjustment latency */
281 } t
[NUM_ROUNDS
] __cpuinitdata
;
284 printk(KERN_INFO
"CPU %d: Syncing TSC to CPU %u.\n",
285 smp_processor_id(), master
);
289 /* It is dangerous to broadcast IPI as cpus are coming up,
290 * as they may not be ready to accept them. So since
291 * we only need to send the ipi to the boot cpu direct
292 * the message, and avoid the race.
294 smp_call_function_single(master
, sync_master
, NULL
, 1, 0);
296 while (go
[MASTER
]) /* wait for master to be ready */
299 spin_lock_irqsave(&tsc_sync_lock
, flags
);
301 for (i
= 0; i
< NUM_ROUNDS
; ++i
) {
302 delta
= get_delta(&rt
, &master_time_stamp
);
304 done
= 1; /* let's lock on to this... */
311 adjust_latency
+= -delta
;
312 adj
= -delta
+ adjust_latency
/4;
317 wrmsrl(MSR_IA32_TSC
, t
+ adj
);
319 #ifdef DEBUG_TSC_SYNC
321 t
[i
].master
= master_time_stamp
;
323 t
[i
].lat
= adjust_latency
/4;
327 spin_unlock_irqrestore(&tsc_sync_lock
, flags
);
329 #ifdef DEBUG_TSC_SYNC
330 for (i
= 0; i
< NUM_ROUNDS
; ++i
)
331 printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n",
332 t
[i
].rt
, t
[i
].master
, t
[i
].diff
, t
[i
].lat
);
336 "CPU %d: synchronized TSC with CPU %u (last diff %ld cycles, "
337 "maxerr %lu cycles)\n",
338 smp_processor_id(), master
, delta
, rt
);
341 static void __cpuinit
tsc_sync_wait(void)
344 * When the CPU has synchronized TSCs assume the BIOS
345 * or the hardware already synced. Otherwise we could
346 * mess up a possible perfect synchronization with a
347 * not-quite-perfect algorithm.
349 if (notscsync
|| !cpu_has_tsc
|| !unsynchronized_tsc())
354 static __init
int notscsync_setup(char *s
)
359 __setup("notscsync", notscsync_setup
);
361 static atomic_t init_deasserted __cpuinitdata
;
364 * Report back to the Boot Processor.
367 void __cpuinit
smp_callin(void)
370 unsigned long timeout
;
373 * If waken up by an INIT in an 82489DX configuration
374 * we may get here before an INIT-deassert IPI reaches
375 * our local APIC. We have to wait for the IPI or we'll
376 * lock up on an APIC access.
378 while (!atomic_read(&init_deasserted
))
382 * (This works even if the APIC is not enabled.)
384 phys_id
= GET_APIC_ID(apic_read(APIC_ID
));
385 cpuid
= smp_processor_id();
386 if (cpu_isset(cpuid
, cpu_callin_map
)) {
387 panic("smp_callin: phys CPU#%d, CPU#%d already present??\n",
390 Dprintk("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid
, phys_id
);
393 * STARTUP IPIs are fragile beasts as they might sometimes
394 * trigger some glue motherboard logic. Complete APIC bus
395 * silence for 1 second, this overestimates the time the
396 * boot CPU is spending to send the up to 2 STARTUP IPIs
397 * by a factor of two. This should be enough.
401 * Waiting 2s total for startup (udelay is not yet working)
403 timeout
= jiffies
+ 2*HZ
;
404 while (time_before(jiffies
, timeout
)) {
406 * Has the boot CPU finished it's STARTUP sequence?
408 if (cpu_isset(cpuid
, cpu_callout_map
))
413 if (!time_before(jiffies
, timeout
)) {
414 panic("smp_callin: CPU%d started up but did not get a callout!\n",
419 * the boot CPU has finished the init stage and is spinning
420 * on callin_map until we finish. We are free to set up this
421 * CPU, first the APIC. (this is probably redundant on most
425 Dprintk("CALLIN, before setup_local_APIC().\n");
431 * Need to enable IRQs because it can take longer and then
432 * the NMI watchdog might kill us.
437 Dprintk("Stack at about %p\n",&cpuid
);
439 disable_APIC_timer();
442 * Save our processor parameters
444 smp_store_cpu_info(cpuid
);
447 * Allow the master to continue.
449 cpu_set(cpuid
, cpu_callin_map
);
452 /* maps the cpu to the sched domain representing multi-core */
453 cpumask_t
cpu_coregroup_map(int cpu
)
455 struct cpuinfo_x86
*c
= cpu_data
+ cpu
;
457 * For perf, we return last level cache shared map.
458 * TBD: when power saving sched policy is added, we will return
459 * cpu_core_map when power saving policy is enabled
461 return c
->llc_shared_map
;
464 /* representing cpus for which sibling maps can be computed */
465 static cpumask_t cpu_sibling_setup_map
;
467 static inline void set_cpu_sibling_map(int cpu
)
470 struct cpuinfo_x86
*c
= cpu_data
;
472 cpu_set(cpu
, cpu_sibling_setup_map
);
474 if (smp_num_siblings
> 1) {
475 for_each_cpu_mask(i
, cpu_sibling_setup_map
) {
476 if (c
[cpu
].phys_proc_id
== c
[i
].phys_proc_id
&&
477 c
[cpu
].cpu_core_id
== c
[i
].cpu_core_id
) {
478 cpu_set(i
, cpu_sibling_map
[cpu
]);
479 cpu_set(cpu
, cpu_sibling_map
[i
]);
480 cpu_set(i
, cpu_core_map
[cpu
]);
481 cpu_set(cpu
, cpu_core_map
[i
]);
482 cpu_set(i
, c
[cpu
].llc_shared_map
);
483 cpu_set(cpu
, c
[i
].llc_shared_map
);
487 cpu_set(cpu
, cpu_sibling_map
[cpu
]);
490 cpu_set(cpu
, c
[cpu
].llc_shared_map
);
492 if (current_cpu_data
.x86_max_cores
== 1) {
493 cpu_core_map
[cpu
] = cpu_sibling_map
[cpu
];
494 c
[cpu
].booted_cores
= 1;
498 for_each_cpu_mask(i
, cpu_sibling_setup_map
) {
499 if (cpu_llc_id
[cpu
] != BAD_APICID
&&
500 cpu_llc_id
[cpu
] == cpu_llc_id
[i
]) {
501 cpu_set(i
, c
[cpu
].llc_shared_map
);
502 cpu_set(cpu
, c
[i
].llc_shared_map
);
504 if (c
[cpu
].phys_proc_id
== c
[i
].phys_proc_id
) {
505 cpu_set(i
, cpu_core_map
[cpu
]);
506 cpu_set(cpu
, cpu_core_map
[i
]);
508 * Does this new cpu bringup a new core?
510 if (cpus_weight(cpu_sibling_map
[cpu
]) == 1) {
512 * for each core in package, increment
513 * the booted_cores for this new cpu
515 if (first_cpu(cpu_sibling_map
[i
]) == i
)
516 c
[cpu
].booted_cores
++;
518 * increment the core count for all
519 * the other cpus in this package
523 } else if (i
!= cpu
&& !c
[cpu
].booted_cores
)
524 c
[cpu
].booted_cores
= c
[i
].booted_cores
;
530 * Setup code on secondary processor (after comming out of the trampoline)
532 void __cpuinit
start_secondary(void)
535 * Dont put anything before smp_callin(), SMP
536 * booting is too fragile that we want to limit the
537 * things done here to the most necessary things.
543 /* otherwise gcc will move up the smp_processor_id before the cpu_init */
546 Dprintk("cpu %d: setting up apic clock\n", smp_processor_id());
547 setup_secondary_APIC_clock();
549 Dprintk("cpu %d: enabling apic timer\n", smp_processor_id());
551 if (nmi_watchdog
== NMI_IO_APIC
) {
552 disable_8259A_irq(0);
553 enable_NMI_through_LVT0(NULL
);
560 * The sibling maps must be set before turing the online map on for
563 set_cpu_sibling_map(smp_processor_id());
566 * Wait for TSC sync to not schedule things before.
567 * We still process interrupts, which could see an inconsistent
568 * time in that window unfortunately.
569 * Do this here because TSC sync has global unprotected state.
574 * We need to hold call_lock, so there is no inconsistency
575 * between the time smp_call_function() determines number of
576 * IPI receipients, and the time when the determination is made
577 * for which cpus receive the IPI in genapic_flat.c. Holding this
578 * lock helps us to not include this cpu in a currently in progress
579 * smp_call_function().
581 lock_ipi_call_lock();
584 * Allow the master to continue.
586 cpu_set(smp_processor_id(), cpu_online_map
);
587 per_cpu(cpu_state
, smp_processor_id()) = CPU_ONLINE
;
588 unlock_ipi_call_lock();
593 extern volatile unsigned long init_rsp
;
594 extern void (*initial_code
)(void);
597 static void inquire_remote_apic(int apicid
)
599 unsigned i
, regs
[] = { APIC_ID
>> 4, APIC_LVR
>> 4, APIC_SPIV
>> 4 };
600 char *names
[] = { "ID", "VERSION", "SPIV" };
603 printk(KERN_INFO
"Inquiring remote APIC #%d...\n", apicid
);
605 for (i
= 0; i
< sizeof(regs
) / sizeof(*regs
); i
++) {
606 printk("... APIC #%d %s: ", apicid
, names
[i
]);
611 apic_wait_icr_idle();
613 apic_write(APIC_ICR2
, SET_APIC_DEST_FIELD(apicid
));
614 apic_write(APIC_ICR
, APIC_DM_REMRD
| regs
[i
]);
619 status
= apic_read(APIC_ICR
) & APIC_ICR_RR_MASK
;
620 } while (status
== APIC_ICR_RR_INPROG
&& timeout
++ < 1000);
623 case APIC_ICR_RR_VALID
:
624 status
= apic_read(APIC_RRR
);
625 printk("%08x\n", status
);
635 * Kick the secondary to wake up.
637 static int __cpuinit
wakeup_secondary_via_INIT(int phys_apicid
, unsigned int start_rip
)
639 unsigned long send_status
= 0, accept_status
= 0;
640 int maxlvt
, timeout
, num_starts
, j
;
642 Dprintk("Asserting INIT.\n");
645 * Turn INIT on target chip
647 apic_write(APIC_ICR2
, SET_APIC_DEST_FIELD(phys_apicid
));
652 apic_write(APIC_ICR
, APIC_INT_LEVELTRIG
| APIC_INT_ASSERT
655 Dprintk("Waiting for send to finish...\n");
660 send_status
= apic_read(APIC_ICR
) & APIC_ICR_BUSY
;
661 } while (send_status
&& (timeout
++ < 1000));
665 Dprintk("Deasserting INIT.\n");
668 apic_write(APIC_ICR2
, SET_APIC_DEST_FIELD(phys_apicid
));
671 apic_write(APIC_ICR
, APIC_INT_LEVELTRIG
| APIC_DM_INIT
);
673 Dprintk("Waiting for send to finish...\n");
678 send_status
= apic_read(APIC_ICR
) & APIC_ICR_BUSY
;
679 } while (send_status
&& (timeout
++ < 1000));
682 atomic_set(&init_deasserted
, 1);
687 * Run STARTUP IPI loop.
689 Dprintk("#startup loops: %d.\n", num_starts
);
691 maxlvt
= get_maxlvt();
693 for (j
= 1; j
<= num_starts
; j
++) {
694 Dprintk("Sending STARTUP #%d.\n",j
);
695 apic_write(APIC_ESR
, 0);
697 Dprintk("After apic_write.\n");
704 apic_write(APIC_ICR2
, SET_APIC_DEST_FIELD(phys_apicid
));
706 /* Boot on the stack */
707 /* Kick the second */
708 apic_write(APIC_ICR
, APIC_DM_STARTUP
| (start_rip
>> 12));
711 * Give the other CPU some time to accept the IPI.
715 Dprintk("Startup point 1.\n");
717 Dprintk("Waiting for send to finish...\n");
722 send_status
= apic_read(APIC_ICR
) & APIC_ICR_BUSY
;
723 } while (send_status
&& (timeout
++ < 1000));
726 * Give the other CPU some time to accept the IPI.
730 * Due to the Pentium erratum 3AP.
733 apic_write(APIC_ESR
, 0);
735 accept_status
= (apic_read(APIC_ESR
) & 0xEF);
736 if (send_status
|| accept_status
)
739 Dprintk("After Startup.\n");
742 printk(KERN_ERR
"APIC never delivered???\n");
744 printk(KERN_ERR
"APIC delivery error (%lx).\n", accept_status
);
746 return (send_status
| accept_status
);
750 struct task_struct
*idle
;
751 struct completion done
;
755 void do_fork_idle(void *_c_idle
)
757 struct create_idle
*c_idle
= _c_idle
;
759 c_idle
->idle
= fork_idle(c_idle
->cpu
);
760 complete(&c_idle
->done
);
766 static int __cpuinit
do_boot_cpu(int cpu
, int apicid
)
768 unsigned long boot_error
;
770 unsigned long start_rip
;
771 struct create_idle c_idle
= {
773 .done
= COMPLETION_INITIALIZER(c_idle
.done
),
775 DECLARE_WORK(work
, do_fork_idle
, &c_idle
);
777 /* allocate memory for gdts of secondary cpus. Hotplug is considered */
778 if (!cpu_gdt_descr
[cpu
].address
&&
779 !(cpu_gdt_descr
[cpu
].address
= get_zeroed_page(GFP_KERNEL
))) {
780 printk(KERN_ERR
"Failed to allocate GDT for CPU %d\n", cpu
);
784 /* Allocate node local memory for AP pdas */
785 if (cpu_pda(cpu
) == &boot_cpu_pda
[cpu
]) {
786 struct x8664_pda
*newpda
, *pda
;
787 int node
= cpu_to_node(cpu
);
789 newpda
= kmalloc_node(sizeof (struct x8664_pda
), GFP_ATOMIC
,
792 memcpy(newpda
, pda
, sizeof (struct x8664_pda
));
793 cpu_pda(cpu
) = newpda
;
796 "Could not allocate node local PDA for CPU %d on node %d\n",
801 alternatives_smp_switch(1);
803 c_idle
.idle
= get_idle_for_cpu(cpu
);
806 c_idle
.idle
->thread
.rsp
= (unsigned long) (((struct pt_regs
*)
807 (THREAD_SIZE
+ task_stack_page(c_idle
.idle
))) - 1);
808 init_idle(c_idle
.idle
, cpu
);
813 * During cold boot process, keventd thread is not spun up yet.
814 * When we do cpu hot-add, we create idle threads on the fly, we should
815 * not acquire any attributes from the calling context. Hence the clean
816 * way to create kernel_threads() is to do that from keventd().
817 * We do the current_is_keventd() due to the fact that ACPI notifier
818 * was also queuing to keventd() and when the caller is already running
819 * in context of keventd(), we would end up with locking up the keventd
822 if (!keventd_up() || current_is_keventd())
823 work
.func(work
.data
);
825 schedule_work(&work
);
826 wait_for_completion(&c_idle
.done
);
829 if (IS_ERR(c_idle
.idle
)) {
830 printk("failed fork for CPU %d\n", cpu
);
831 return PTR_ERR(c_idle
.idle
);
834 set_idle_for_cpu(cpu
, c_idle
.idle
);
838 cpu_pda(cpu
)->pcurrent
= c_idle
.idle
;
840 start_rip
= setup_trampoline();
842 init_rsp
= c_idle
.idle
->thread
.rsp
;
843 per_cpu(init_tss
,cpu
).rsp0
= init_rsp
;
844 initial_code
= start_secondary
;
845 clear_tsk_thread_flag(c_idle
.idle
, TIF_FORK
);
847 printk(KERN_INFO
"Booting processor %d/%d APIC 0x%x\n", cpu
,
848 cpus_weight(cpu_present_map
),
852 * This grunge runs the startup process for
853 * the targeted processor.
856 atomic_set(&init_deasserted
, 0);
858 Dprintk("Setting warm reset code and vector.\n");
860 CMOS_WRITE(0xa, 0xf);
863 *((volatile unsigned short *) phys_to_virt(0x469)) = start_rip
>> 4;
865 *((volatile unsigned short *) phys_to_virt(0x467)) = start_rip
& 0xf;
869 * Be paranoid about clearing APIC errors.
871 apic_write(APIC_ESR
, 0);
875 * Status is now clean
880 * Starting actual IPI sequence...
882 boot_error
= wakeup_secondary_via_INIT(apicid
, start_rip
);
886 * allow APs to start initializing.
888 Dprintk("Before Callout %d.\n", cpu
);
889 cpu_set(cpu
, cpu_callout_map
);
890 Dprintk("After Callout %d.\n", cpu
);
893 * Wait 5s total for a response
895 for (timeout
= 0; timeout
< 50000; timeout
++) {
896 if (cpu_isset(cpu
, cpu_callin_map
))
897 break; /* It has booted */
901 if (cpu_isset(cpu
, cpu_callin_map
)) {
902 /* number CPUs logically, starting from 1 (BSP is 0) */
903 Dprintk("CPU has booted.\n");
906 if (*((volatile unsigned char *)phys_to_virt(SMP_TRAMPOLINE_BASE
))
908 /* trampoline started but...? */
909 printk("Stuck ??\n");
911 /* trampoline code not run */
912 printk("Not responding.\n");
914 inquire_remote_apic(apicid
);
919 cpu_clear(cpu
, cpu_callout_map
); /* was set here (do_boot_cpu()) */
920 clear_bit(cpu
, &cpu_initialized
); /* was set by cpu_init() */
921 clear_node_cpumask(cpu
); /* was set by numa_add_cpu */
922 cpu_clear(cpu
, cpu_present_map
);
923 cpu_clear(cpu
, cpu_possible_map
);
924 x86_cpu_to_apicid
[cpu
] = BAD_APICID
;
925 x86_cpu_to_log_apicid
[cpu
] = BAD_APICID
;
932 cycles_t cacheflush_time
;
933 unsigned long cache_decay_ticks
;
936 * Cleanup possible dangling ends...
938 static __cpuinit
void smp_cleanup_boot(void)
941 * Paranoid: Set warm reset code and vector here back
947 * Reset trampoline flag
949 *((volatile int *) phys_to_virt(0x467)) = 0;
953 * Fall back to non SMP mode after errors.
955 * RED-PEN audit/test this more. I bet there is more state messed up here.
957 static __init
void disable_smp(void)
959 cpu_present_map
= cpumask_of_cpu(0);
960 cpu_possible_map
= cpumask_of_cpu(0);
961 if (smp_found_config
)
962 phys_cpu_present_map
= physid_mask_of_physid(boot_cpu_id
);
964 phys_cpu_present_map
= physid_mask_of_physid(0);
965 cpu_set(0, cpu_sibling_map
[0]);
966 cpu_set(0, cpu_core_map
[0]);
969 #ifdef CONFIG_HOTPLUG_CPU
971 int additional_cpus __initdata
= -1;
974 * cpu_possible_map should be static, it cannot change as cpu's
975 * are onlined, or offlined. The reason is per-cpu data-structures
976 * are allocated by some modules at init time, and dont expect to
977 * do this dynamically on cpu arrival/departure.
978 * cpu_present_map on the other hand can change dynamically.
979 * In case when cpu_hotplug is not compiled, then we resort to current
980 * behaviour, which is cpu_possible == cpu_present.
983 * Three ways to find out the number of additional hotplug CPUs:
984 * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
985 * - The user can overwrite it with additional_cpus=NUM
986 * - Otherwise don't reserve additional CPUs.
987 * We do this because additional CPUs waste a lot of memory.
990 __init
void prefill_possible_map(void)
995 if (additional_cpus
== -1) {
996 if (disabled_cpus
> 0)
997 additional_cpus
= disabled_cpus
;
1001 possible
= num_processors
+ additional_cpus
;
1002 if (possible
> NR_CPUS
)
1005 printk(KERN_INFO
"SMP: Allowing %d CPUs, %d hotplug CPUs\n",
1007 max_t(int, possible
- num_processors
, 0));
1009 for (i
= 0; i
< possible
; i
++)
1010 cpu_set(i
, cpu_possible_map
);
1015 * Various sanity checks.
1017 static int __init
smp_sanity_check(unsigned max_cpus
)
1019 if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map
)) {
1020 printk("weird, boot CPU (#%d) not listed by the BIOS.\n",
1021 hard_smp_processor_id());
1022 physid_set(hard_smp_processor_id(), phys_cpu_present_map
);
1026 * If we couldn't find an SMP configuration at boot time,
1027 * get out of here now!
1029 if (!smp_found_config
) {
1030 printk(KERN_NOTICE
"SMP motherboard not detected.\n");
1032 if (APIC_init_uniprocessor())
1033 printk(KERN_NOTICE
"Local APIC not detected."
1034 " Using dummy APIC emulation.\n");
1039 * Should not be necessary because the MP table should list the boot
1040 * CPU too, but we do it for the sake of robustness anyway.
1042 if (!physid_isset(boot_cpu_id
, phys_cpu_present_map
)) {
1043 printk(KERN_NOTICE
"weird, boot CPU (#%d) not listed by the BIOS.\n",
1045 physid_set(hard_smp_processor_id(), phys_cpu_present_map
);
1049 * If we couldn't find a local APIC, then get out of here now!
1051 if (!cpu_has_apic
) {
1052 printk(KERN_ERR
"BIOS bug, local APIC #%d not detected!...\n",
1054 printk(KERN_ERR
"... forcing use of dummy APIC emulation. (tell your hw vendor)\n");
1060 * If SMP should be disabled, then really disable it!
1063 printk(KERN_INFO
"SMP mode deactivated, forcing use of dummy APIC emulation.\n");
1072 * Prepare for SMP bootup. The MP table or ACPI has been read
1073 * earlier. Just do some sanity checking here and enable APIC mode.
1075 void __init
smp_prepare_cpus(unsigned int max_cpus
)
1077 nmi_watchdog_default();
1078 current_cpu_data
= boot_cpu_data
;
1079 current_thread_info()->cpu
= 0; /* needed? */
1080 set_cpu_sibling_map(0);
1082 if (smp_sanity_check(max_cpus
) < 0) {
1083 printk(KERN_INFO
"SMP disabled\n");
1090 * Switch from PIC to APIC mode.
1095 if (GET_APIC_ID(apic_read(APIC_ID
)) != boot_cpu_id
) {
1096 panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
1097 GET_APIC_ID(apic_read(APIC_ID
)), boot_cpu_id
);
1098 /* Or can we switch back to PIC here? */
1102 * Now start the IO-APICs
1104 if (!skip_ioapic_setup
&& nr_ioapics
)
1110 * Set up local APIC timer on boot CPU.
1113 setup_boot_APIC_clock();
1117 * Early setup to make printk work.
1119 void __init
smp_prepare_boot_cpu(void)
1121 int me
= smp_processor_id();
1122 cpu_set(me
, cpu_online_map
);
1123 cpu_set(me
, cpu_callout_map
);
1124 per_cpu(cpu_state
, me
) = CPU_ONLINE
;
1128 * Entry point to boot a CPU.
1130 int __cpuinit
__cpu_up(unsigned int cpu
)
1133 int apicid
= cpu_present_to_apicid(cpu
);
1135 WARN_ON(irqs_disabled());
1137 Dprintk("++++++++++++++++++++=_---CPU UP %u\n", cpu
);
1139 if (apicid
== BAD_APICID
|| apicid
== boot_cpu_id
||
1140 !physid_isset(apicid
, phys_cpu_present_map
)) {
1141 printk("__cpu_up: bad cpu %d\n", cpu
);
1146 * Already booted CPU?
1148 if (cpu_isset(cpu
, cpu_callin_map
)) {
1149 Dprintk("do_boot_cpu %d Already started\n", cpu
);
1153 per_cpu(cpu_state
, cpu
) = CPU_UP_PREPARE
;
1155 err
= do_boot_cpu(cpu
, apicid
);
1157 Dprintk("do_boot_cpu failed %d\n", err
);
1161 /* Unleash the CPU! */
1162 Dprintk("waiting for cpu %d\n", cpu
);
1164 while (!cpu_isset(cpu
, cpu_online_map
))
1172 * Finish the SMP boot.
1174 void __init
smp_cpus_done(unsigned int max_cpus
)
1178 #ifdef CONFIG_X86_IO_APIC
1179 setup_ioapic_dest();
1182 check_nmi_watchdog();
1185 #ifdef CONFIG_HOTPLUG_CPU
1187 static void remove_siblinginfo(int cpu
)
1190 struct cpuinfo_x86
*c
= cpu_data
;
1192 for_each_cpu_mask(sibling
, cpu_core_map
[cpu
]) {
1193 cpu_clear(cpu
, cpu_core_map
[sibling
]);
1195 * last thread sibling in this cpu core going down
1197 if (cpus_weight(cpu_sibling_map
[cpu
]) == 1)
1198 c
[sibling
].booted_cores
--;
1201 for_each_cpu_mask(sibling
, cpu_sibling_map
[cpu
])
1202 cpu_clear(cpu
, cpu_sibling_map
[sibling
]);
1203 cpus_clear(cpu_sibling_map
[cpu
]);
1204 cpus_clear(cpu_core_map
[cpu
]);
1205 c
[cpu
].phys_proc_id
= 0;
1206 c
[cpu
].cpu_core_id
= 0;
1207 cpu_clear(cpu
, cpu_sibling_setup_map
);
1210 void remove_cpu_from_maps(void)
1212 int cpu
= smp_processor_id();
1214 cpu_clear(cpu
, cpu_callout_map
);
1215 cpu_clear(cpu
, cpu_callin_map
);
1216 clear_bit(cpu
, &cpu_initialized
); /* was set by cpu_init() */
1217 clear_node_cpumask(cpu
);
1220 int __cpu_disable(void)
1222 int cpu
= smp_processor_id();
1225 * Perhaps use cpufreq to drop frequency, but that could go
1226 * into generic code.
1228 * We won't take down the boot processor on i386 due to some
1229 * interrupts only being able to be serviced by the BSP.
1230 * Especially so if we're not using an IOAPIC -zwane
1239 * Allow any queued timer interrupts to get serviced
1240 * This is only a temporary solution until we cleanup
1241 * fixup_irqs as we do for IA64.
1246 local_irq_disable();
1247 remove_siblinginfo(cpu
);
1249 /* It's now safe to remove this processor from the online map */
1250 cpu_clear(cpu
, cpu_online_map
);
1251 remove_cpu_from_maps();
1252 fixup_irqs(cpu_online_map
);
1256 void __cpu_die(unsigned int cpu
)
1258 /* We don't do anything here: idle task is faking death itself. */
1261 for (i
= 0; i
< 10; i
++) {
1262 /* They ack this in play_dead by setting CPU_DEAD */
1263 if (per_cpu(cpu_state
, cpu
) == CPU_DEAD
) {
1264 printk ("CPU %d is now offline\n", cpu
);
1265 if (1 == num_online_cpus())
1266 alternatives_smp_switch(0);
1271 printk(KERN_ERR
"CPU %u didn't die...\n", cpu
);
1274 __init
int setup_additional_cpus(char *s
)
1276 return get_option(&s
, &additional_cpus
);
1278 __setup("additional_cpus=", setup_additional_cpus
);
1280 #else /* ... !CONFIG_HOTPLUG_CPU */
1282 int __cpu_disable(void)
1287 void __cpu_die(unsigned int cpu
)
1289 /* We said "no" in __cpu_disable */
1292 #endif /* CONFIG_HOTPLUG_CPU */