projects / deliverable / linux.git / blob
? search:


f3e9b2df4b165d9d1a31322960778a2e9ef68bf3
[deliverable/linux.git] / arch / x86 / kernel / apic / apic.c
1 /*
2 * Local APIC handling, local APIC timers
3 *
4 * (c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
5 *
6 * Fixes
7 * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
8 * thanks to Eric Gilmore
9 * and Rolf G. Tews
10 * for testing these extensively.
11 * Maciej W. Rozycki : Various updates and fixes.
12 * Mikael Pettersson : Power Management for UP-APIC.
13 * Pavel Machek and
14 * Mikael Pettersson : PM converted to driver model.
15 */
16
17 #include <linux/perf_event.h>
18 #include <linux/kernel_stat.h>
19 #include <linux/mc146818rtc.h>
20 #include <linux/acpi_pmtmr.h>
21 #include <linux/clockchips.h>
22 #include <linux/interrupt.h>
23 #include <linux/bootmem.h>
24 #include <linux/ftrace.h>
25 #include <linux/ioport.h>
26 #include <linux/export.h>
27 #include <linux/syscore_ops.h>
28 #include <linux/delay.h>
29 #include <linux/timex.h>
30 #include <linux/i8253.h>
31 #include <linux/dmar.h>
32 #include <linux/init.h>
33 #include <linux/cpu.h>
34 #include <linux/dmi.h>
35 #include <linux/smp.h>
36 #include <linux/mm.h>
37
38 #include <asm/trace/irq_vectors.h>
39 #include <asm/irq_remapping.h>
40 #include <asm/perf_event.h>
41 #include <asm/x86_init.h>
42 #include <asm/pgalloc.h>
43 #include <linux/atomic.h>
44 #include <asm/mpspec.h>
45 #include <asm/i8259.h>
46 #include <asm/proto.h>
47 #include <asm/apic.h>
48 #include <asm/io_apic.h>
49 #include <asm/desc.h>
50 #include <asm/hpet.h>
51 #include <asm/idle.h>
52 #include <asm/mtrr.h>
53 #include <asm/time.h>
54 #include <asm/smp.h>
55 #include <asm/mce.h>
56 #include <asm/tsc.h>
57 #include <asm/hypervisor.h>
58
59 unsigned int num_processors;
60
61 unsigned disabled_cpus;
62
63 /* Processor that is doing the boot up */
64 unsigned int boot_cpu_physical_apicid = -1U;
65 EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid);
66
67 /*
68 * The highest APIC ID seen during enumeration.
69 */
70 static unsigned int max_physical_apicid;
71
72 /*
73 * Bitmask of physically existing CPUs:
74 */
75 physid_mask_t phys_cpu_present_map;
76
77 /*
78 * Processor to be disabled specified by kernel parameter
79 * disable_cpu_apicid=<int>, mostly used for the kdump 2nd kernel to
80 * avoid undefined behaviour caused by sending INIT from AP to BSP.
81 */
82 static unsigned int disabled_cpu_apicid __read_mostly = BAD_APICID;
83
84 /*
85 * This variable controls which CPUs receive external NMIs. By default,
86 * external NMIs are delivered only to the BSP.
87 */
88 static int apic_extnmi = APIC_EXTNMI_BSP;
89
90 /*
91 * Map cpu index to physical APIC ID
92 */
93 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID);
94 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid, BAD_APICID);
95 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid, U32_MAX);
96 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
97 EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
98 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_acpiid);
99
100 #ifdef CONFIG_X86_32
101
102 /*
103 * On x86_32, the mapping between cpu and logical apicid may vary
104 * depending on apic in use. The following early percpu variable is
105 * used for the mapping. This is where the behaviors of x86_64 and 32
106 * actually diverge. Let's keep it ugly for now.
107 */
108 DEFINE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid, BAD_APICID);
109
110 /* Local APIC was disabled by the BIOS and enabled by the kernel */
111 static int enabled_via_apicbase;
112
113 /*
114 * Handle interrupt mode configuration register (IMCR).
115 * This register controls whether the interrupt signals
116 * that reach the BSP come from the master PIC or from the
117 * local APIC. Before entering Symmetric I/O Mode, either
118 * the BIOS or the operating system must switch out of
119 * PIC Mode by changing the IMCR.
120 */
121 static inline void imcr_pic_to_apic(void)
122 {
123 /* select IMCR register */
124 outb(0x70, 0x22);
125 /* NMI and 8259 INTR go through APIC */
126 outb(0x01, 0x23);
127 }
128
129 static inline void imcr_apic_to_pic(void)
130 {
131 /* select IMCR register */
132 outb(0x70, 0x22);
133 /* NMI and 8259 INTR go directly to BSP */
134 outb(0x00, 0x23);
135 }
136 #endif
137
138 /*
139 * Knob to control our willingness to enable the local APIC.
140 *
141 * +1=force-enable
142 */
143 static int force_enable_local_apic __initdata;
144
145 /*
146 * APIC command line parameters
147 */
148 static int __init parse_lapic(char *arg)
149 {
150 if (IS_ENABLED(CONFIG_X86_32) && !arg)
151 force_enable_local_apic = 1;
152 else if (arg && !strncmp(arg, "notscdeadline", 13))
153 setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
154 return 0;
155 }
156 early_param("lapic", parse_lapic);
157
158 #ifdef CONFIG_X86_64
159 static int apic_calibrate_pmtmr __initdata;
160 static __init int setup_apicpmtimer(char *s)
161 {
162 apic_calibrate_pmtmr = 1;
163 notsc_setup(NULL);
164 return 0;
165 }
166 __setup("apicpmtimer", setup_apicpmtimer);
167 #endif
168
169 unsigned long mp_lapic_addr;
170 int disable_apic;
171 /* Disable local APIC timer from the kernel commandline or via dmi quirk */
172 static int disable_apic_timer __initdata;
173 /* Local APIC timer works in C2 */
174 int local_apic_timer_c2_ok;
175 EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
176
177 int first_system_vector = FIRST_SYSTEM_VECTOR;
178
179 /*
180 * Debug level, exported for io_apic.c
181 */
182 unsigned int apic_verbosity;
183
184 int pic_mode;
185
186 /* Have we found an MP table */
187 int smp_found_config;
188
189 static struct resource lapic_resource = {
190 .name = "Local APIC",
191 .flags = IORESOURCE_MEM | IORESOURCE_BUSY,
192 };
193
194 unsigned int lapic_timer_frequency = 0;
195
196 static void apic_pm_activate(void);
197
198 static unsigned long apic_phys;
199
200 /*
201 * Get the LAPIC version
202 */
203 static inline int lapic_get_version(void)
204 {
205 return GET_APIC_VERSION(apic_read(APIC_LVR));
206 }
207
208 /*
209 * Check, if the APIC is integrated or a separate chip
210 */
211 static inline int lapic_is_integrated(void)
212 {
213 #ifdef CONFIG_X86_64
214 return 1;
215 #else
216 return APIC_INTEGRATED(lapic_get_version());
217 #endif
218 }
219
220 /*
221 * Check, whether this is a modern or a first generation APIC
222 */
223 static int modern_apic(void)
224 {
225 /* AMD systems use old APIC versions, so check the CPU */
226 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
227 boot_cpu_data.x86 >= 0xf)
228 return 1;
229 return lapic_get_version() >= 0x14;
230 }
231
232 /*
233 * right after this call apic become NOOP driven
234 * so apic->write/read doesn't do anything
235 */
236 static void __init apic_disable(void)
237 {
238 pr_info("APIC: switched to apic NOOP\n");
239 apic = &apic_noop;
240 }
241
242 void native_apic_wait_icr_idle(void)
243 {
244 while (apic_read(APIC_ICR) & APIC_ICR_BUSY)
245 cpu_relax();
246 }
247
248 u32 native_safe_apic_wait_icr_idle(void)
249 {
250 u32 send_status;
251 int timeout;
252
253 timeout = 0;
254 do {
255 send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
256 if (!send_status)
257 break;
258 inc_irq_stat(icr_read_retry_count);
259 udelay(100);
260 } while (timeout++ < 1000);
261
262 return send_status;
263 }
264
265 void native_apic_icr_write(u32 low, u32 id)
266 {
267 unsigned long flags;
268
269 local_irq_save(flags);
270 apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(id));
271 apic_write(APIC_ICR, low);
272 local_irq_restore(flags);
273 }
274
275 u64 native_apic_icr_read(void)
276 {
277 u32 icr1, icr2;
278
279 icr2 = apic_read(APIC_ICR2);
280 icr1 = apic_read(APIC_ICR);
281
282 return icr1 | ((u64)icr2 << 32);
283 }
284
285 #ifdef CONFIG_X86_32
286 /**
287 * get_physical_broadcast - Get number of physical broadcast IDs
288 */
289 int get_physical_broadcast(void)
290 {
291 return modern_apic() ? 0xff : 0xf;
292 }
293 #endif
294
295 /**
296 * lapic_get_maxlvt - get the maximum number of local vector table entries
297 */
298 int lapic_get_maxlvt(void)
299 {
300 unsigned int v;
301
302 v = apic_read(APIC_LVR);
303 /*
304 * - we always have APIC integrated on 64bit mode
305 * - 82489DXs do not report # of LVT entries
306 */
307 return APIC_INTEGRATED(GET_APIC_VERSION(v)) ? GET_APIC_MAXLVT(v) : 2;
308 }
309
310 /*
311 * Local APIC timer
312 */
313
314 /* Clock divisor */
315 #define APIC_DIVISOR 16
316 #define TSC_DIVISOR 8
317
318 /*
319 * This function sets up the local APIC timer, with a timeout of
320 * 'clocks' APIC bus clock. During calibration we actually call
321 * this function twice on the boot CPU, once with a bogus timeout
322 * value, second time for real. The other (noncalibrating) CPUs
323 * call this function only once, with the real, calibrated value.
324 *
325 * We do reads before writes even if unnecessary, to get around the
326 * P5 APIC double write bug.
327 */
328 static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
329 {
330 unsigned int lvtt_value, tmp_value;
331
332 lvtt_value = LOCAL_TIMER_VECTOR;
333 if (!oneshot)
334 lvtt_value |= APIC_LVT_TIMER_PERIODIC;
335 else if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
336 lvtt_value |= APIC_LVT_TIMER_TSCDEADLINE;
337
338 if (!lapic_is_integrated())
339 lvtt_value |= SET_APIC_TIMER_BASE(APIC_TIMER_BASE_DIV);
340
341 if (!irqen)
342 lvtt_value |= APIC_LVT_MASKED;
343
344 apic_write(APIC_LVTT, lvtt_value);
345
346 if (lvtt_value & APIC_LVT_TIMER_TSCDEADLINE) {
347 /*
348 * See Intel SDM: TSC-Deadline Mode chapter. In xAPIC mode,
349 * writing to the APIC LVTT and TSC_DEADLINE MSR isn't serialized.
350 * According to Intel, MFENCE can do the serialization here.
351 */
352 asm volatile("mfence" : : : "memory");
353
354 printk_once(KERN_DEBUG "TSC deadline timer enabled\n");
355 return;
356 }
357
358 /*
359 * Divide PICLK by 16
360 */
361 tmp_value = apic_read(APIC_TDCR);
362 apic_write(APIC_TDCR,
363 (tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) |
364 APIC_TDR_DIV_16);
365
366 if (!oneshot)
367 apic_write(APIC_TMICT, clocks / APIC_DIVISOR);
368 }
369
370 /*
371 * Setup extended LVT, AMD specific
372 *
373 * Software should use the LVT offsets the BIOS provides. The offsets
374 * are determined by the subsystems using it like those for MCE
375 * threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts
376 * are supported. Beginning with family 10h at least 4 offsets are
377 * available.
378 *
379 * Since the offsets must be consistent for all cores, we keep track
380 * of the LVT offsets in software and reserve the offset for the same
381 * vector also to be used on other cores. An offset is freed by
382 * setting the entry to APIC_EILVT_MASKED.
383 *
384 * If the BIOS is right, there should be no conflicts. Otherwise a
385 * "[Firmware Bug]: ..." error message is generated. However, if
386 * software does not properly determines the offsets, it is not
387 * necessarily a BIOS bug.
388 */
389
390 static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX];
391
392 static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new)
393 {
394 return (old & APIC_EILVT_MASKED)
395 || (new == APIC_EILVT_MASKED)
396 || ((new & ~APIC_EILVT_MASKED) == old);
397 }
398
399 static unsigned int reserve_eilvt_offset(int offset, unsigned int new)
400 {
401 unsigned int rsvd, vector;
402
403 if (offset >= APIC_EILVT_NR_MAX)
404 return ~0;
405
406 rsvd = atomic_read(&eilvt_offsets[offset]);
407 do {
408 vector = rsvd & ~APIC_EILVT_MASKED; /* 0: unassigned */
409 if (vector && !eilvt_entry_is_changeable(vector, new))
410 /* may not change if vectors are different */
411 return rsvd;
412 rsvd = atomic_cmpxchg(&eilvt_offsets[offset], rsvd, new);
413 } while (rsvd != new);
414
415 rsvd &= ~APIC_EILVT_MASKED;
416 if (rsvd && rsvd != vector)
417 pr_info("LVT offset %d assigned for vector 0x%02x\n",
418 offset, rsvd);
419
420 return new;
421 }
422
423 /*
424 * If mask=1, the LVT entry does not generate interrupts while mask=0
425 * enables the vector. See also the BKDGs. Must be called with
426 * preemption disabled.
427 */
428
429 int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask)
430 {
431 unsigned long reg = APIC_EILVTn(offset);
432 unsigned int new, old, reserved;
433
434 new = (mask << 16) | (msg_type << 8) | vector;
435 old = apic_read(reg);
436 reserved = reserve_eilvt_offset(offset, new);
437
438 if (reserved != new) {
439 pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
440 "vector 0x%x, but the register is already in use for "
441 "vector 0x%x on another cpu\n",
442 smp_processor_id(), reg, offset, new, reserved);
443 return -EINVAL;
444 }
445
446 if (!eilvt_entry_is_changeable(old, new)) {
447 pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
448 "vector 0x%x, but the register is already in use for "
449 "vector 0x%x on this cpu\n",
450 smp_processor_id(), reg, offset, new, old);
451 return -EBUSY;
452 }
453
454 apic_write(reg, new);
455
456 return 0;
457 }
458 EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
459
460 /*
461 * Program the next event, relative to now
462 */
463 static int lapic_next_event(unsigned long delta,
464 struct clock_event_device *evt)
465 {
466 apic_write(APIC_TMICT, delta);
467 return 0;
468 }
469
470 static int lapic_next_deadline(unsigned long delta,
471 struct clock_event_device *evt)
472 {
473 u64 tsc;
474
475 tsc = rdtsc();
476 wrmsrl(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR));
477 return 0;
478 }
479
480 static int lapic_timer_shutdown(struct clock_event_device *evt)
481 {
482 unsigned int v;
483
484 /* Lapic used as dummy for broadcast ? */
485 if (evt->features & CLOCK_EVT_FEAT_DUMMY)
486 return 0;
487
488 v = apic_read(APIC_LVTT);
489 v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
490 apic_write(APIC_LVTT, v);
491 apic_write(APIC_TMICT, 0);
492 return 0;
493 }
494
495 static inline int
496 lapic_timer_set_periodic_oneshot(struct clock_event_device *evt, bool oneshot)
497 {
498 /* Lapic used as dummy for broadcast ? */
499 if (evt->features & CLOCK_EVT_FEAT_DUMMY)
500 return 0;
501
502 __setup_APIC_LVTT(lapic_timer_frequency, oneshot, 1);
503 return 0;
504 }
505
506 static int lapic_timer_set_periodic(struct clock_event_device *evt)
507 {
508 return lapic_timer_set_periodic_oneshot(evt, false);
509 }
510
511 static int lapic_timer_set_oneshot(struct clock_event_device *evt)
512 {
513 return lapic_timer_set_periodic_oneshot(evt, true);
514 }
515
516 /*
517 * Local APIC timer broadcast function
518 */
519 static void lapic_timer_broadcast(const struct cpumask *mask)
520 {
521 #ifdef CONFIG_SMP
522 apic->send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
523 #endif
524 }
525
526
527 /*
528 * The local apic timer can be used for any function which is CPU local.
529 */
530 static struct clock_event_device lapic_clockevent = {
531 .name = "lapic",
532 .features = CLOCK_EVT_FEAT_PERIODIC |
533 CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP
534 | CLOCK_EVT_FEAT_DUMMY,
535 .shift = 32,
536 .set_state_shutdown = lapic_timer_shutdown,
537 .set_state_periodic = lapic_timer_set_periodic,
538 .set_state_oneshot = lapic_timer_set_oneshot,
539 .set_next_event = lapic_next_event,
540 .broadcast = lapic_timer_broadcast,
541 .rating = 100,
542 .irq = -1,
543 };
544 static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
545
546 /*
547 * Setup the local APIC timer for this CPU. Copy the initialized values
548 * of the boot CPU and register the clock event in the framework.
549 */
550 static void setup_APIC_timer(void)
551 {
552 struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
553
554 if (this_cpu_has(X86_FEATURE_ARAT)) {
555 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP;
556 /* Make LAPIC timer preferrable over percpu HPET */
557 lapic_clockevent.rating = 150;
558 }
559
560 memcpy(levt, &lapic_clockevent, sizeof(*levt));
561 levt->cpumask = cpumask_of(smp_processor_id());
562
563 if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) {
564 levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC |
565 CLOCK_EVT_FEAT_DUMMY);
566 levt->set_next_event = lapic_next_deadline;
567 clockevents_config_and_register(levt,
568 tsc_khz * (1000 / TSC_DIVISOR),
569 0xF, ~0UL);
570 } else
571 clockevents_register_device(levt);
572 }
573
574 /*
575 * Install the updated TSC frequency from recalibration at the TSC
576 * deadline clockevent devices.
577 */
578 static void __lapic_update_tsc_freq(void *info)
579 {
580 struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
581
582 if (!this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
583 return;
584
585 clockevents_update_freq(levt, tsc_khz * (1000 / TSC_DIVISOR));
586 }
587
588 void lapic_update_tsc_freq(void)
589 {
590 /*
591 * The clockevent device's ->mult and ->shift can both be
592 * changed. In order to avoid races, schedule the frequency
593 * update code on each CPU.
594 */
595 on_each_cpu(__lapic_update_tsc_freq, NULL, 0);
596 }
597
598 /*
599 * In this functions we calibrate APIC bus clocks to the external timer.
600 *
601 * We want to do the calibration only once since we want to have local timer
602 * irqs syncron. CPUs connected by the same APIC bus have the very same bus
603 * frequency.
604 *
605 * This was previously done by reading the PIT/HPET and waiting for a wrap
606 * around to find out, that a tick has elapsed. I have a box, where the PIT
607 * readout is broken, so it never gets out of the wait loop again. This was
608 * also reported by others.
609 *
610 * Monitoring the jiffies value is inaccurate and the clockevents
611 * infrastructure allows us to do a simple substitution of the interrupt
612 * handler.
613 *
614 * The calibration routine also uses the pm_timer when possible, as the PIT
615 * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes
616 * back to normal later in the boot process).
617 */
618
619 #define LAPIC_CAL_LOOPS (HZ/10)
620
621 static __initdata int lapic_cal_loops = -1;
622 static __initdata long lapic_cal_t1, lapic_cal_t2;
623 static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2;
624 static __initdata unsigned long lapic_cal_pm1, lapic_cal_pm2;
625 static __initdata unsigned long lapic_cal_j1, lapic_cal_j2;
626
627 /*
628 * Temporary interrupt handler.
629 */
630 static void __init lapic_cal_handler(struct clock_event_device *dev)
631 {
632 unsigned long long tsc = 0;
633 long tapic = apic_read(APIC_TMCCT);
634 unsigned long pm = acpi_pm_read_early();
635
636 if (boot_cpu_has(X86_FEATURE_TSC))
637 tsc = rdtsc();
638
639 switch (lapic_cal_loops++) {
640 case 0:
641 lapic_cal_t1 = tapic;
642 lapic_cal_tsc1 = tsc;
643 lapic_cal_pm1 = pm;
644 lapic_cal_j1 = jiffies;
645 break;
646
647 case LAPIC_CAL_LOOPS:
648 lapic_cal_t2 = tapic;
649 lapic_cal_tsc2 = tsc;
650 if (pm < lapic_cal_pm1)
651 pm += ACPI_PM_OVRRUN;
652 lapic_cal_pm2 = pm;
653 lapic_cal_j2 = jiffies;
654 break;
655 }
656 }
657
658 static int __init
659 calibrate_by_pmtimer(long deltapm, long *delta, long *deltatsc)
660 {
661 const long pm_100ms = PMTMR_TICKS_PER_SEC / 10;
662 const long pm_thresh = pm_100ms / 100;
663 unsigned long mult;
664 u64 res;
665
666 #ifndef CONFIG_X86_PM_TIMER
667 return -1;
668 #endif
669
670 apic_printk(APIC_VERBOSE, "... PM-Timer delta = %ld\n", deltapm);
671
672 /* Check, if the PM timer is available */
673 if (!deltapm)
674 return -1;
675
676 mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, 22);
677
678 if (deltapm > (pm_100ms - pm_thresh) &&
679 deltapm < (pm_100ms + pm_thresh)) {
680 apic_printk(APIC_VERBOSE, "... PM-Timer result ok\n");
681 return 0;
682 }
683
684 res = (((u64)deltapm) * mult) >> 22;
685 do_div(res, 1000000);
686 pr_warning("APIC calibration not consistent "
687 "with PM-Timer: %ldms instead of 100ms\n",(long)res);
688
689 /* Correct the lapic counter value */
690 res = (((u64)(*delta)) * pm_100ms);
691 do_div(res, deltapm);
692 pr_info("APIC delta adjusted to PM-Timer: "
693 "%lu (%ld)\n", (unsigned long)res, *delta);
694 *delta = (long)res;
695
696 /* Correct the tsc counter value */
697 if (boot_cpu_has(X86_FEATURE_TSC)) {
698 res = (((u64)(*deltatsc)) * pm_100ms);
699 do_div(res, deltapm);
700 apic_printk(APIC_VERBOSE, "TSC delta adjusted to "
701 "PM-Timer: %lu (%ld)\n",
702 (unsigned long)res, *deltatsc);
703 *deltatsc = (long)res;
704 }
705
706 return 0;
707 }
708
709 static int __init calibrate_APIC_clock(void)
710 {
711 struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
712 void (*real_handler)(struct clock_event_device *dev);
713 unsigned long deltaj;
714 long delta, deltatsc;
715 int pm_referenced = 0;
716
717 /**
718 * check if lapic timer has already been calibrated by platform
719 * specific routine, such as tsc calibration code. if so, we just fill
720 * in the clockevent structure and return.
721 */
722
723 if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) {
724 return 0;
725 } else if (lapic_timer_frequency) {
726 apic_printk(APIC_VERBOSE, "lapic timer already calibrated %d\n",
727 lapic_timer_frequency);
728 lapic_clockevent.mult = div_sc(lapic_timer_frequency/APIC_DIVISOR,
729 TICK_NSEC, lapic_clockevent.shift);
730 lapic_clockevent.max_delta_ns =
731 clockevent_delta2ns(0x7FFFFF, &lapic_clockevent);
732 lapic_clockevent.min_delta_ns =
733 clockevent_delta2ns(0xF, &lapic_clockevent);
734 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
735 return 0;
736 }
737
738 apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n"
739 "calibrating APIC timer ...\n");
740
741 local_irq_disable();
742
743 /* Replace the global interrupt handler */
744 real_handler = global_clock_event->event_handler;
745 global_clock_event->event_handler = lapic_cal_handler;
746
747 /*
748 * Setup the APIC counter to maximum. There is no way the lapic
749 * can underflow in the 100ms detection time frame
750 */
751 __setup_APIC_LVTT(0xffffffff, 0, 0);
752
753 /* Let the interrupts run */
754 local_irq_enable();
755
756 while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
757 cpu_relax();
758
759 local_irq_disable();
760
761 /* Restore the real event handler */
762 global_clock_event->event_handler = real_handler;
763
764 /* Build delta t1-t2 as apic timer counts down */
765 delta = lapic_cal_t1 - lapic_cal_t2;
766 apic_printk(APIC_VERBOSE, "... lapic delta = %ld\n", delta);
767
768 deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1);
769
770 /* we trust the PM based calibration if possible */
771 pm_referenced = !calibrate_by_pmtimer(lapic_cal_pm2 - lapic_cal_pm1,
772 &delta, &deltatsc);
773
774 /* Calculate the scaled math multiplication factor */
775 lapic_clockevent.mult = div_sc(delta, TICK_NSEC * LAPIC_CAL_LOOPS,
776 lapic_clockevent.shift);
777 lapic_clockevent.max_delta_ns =
778 clockevent_delta2ns(0x7FFFFFFF, &lapic_clockevent);
779 lapic_clockevent.min_delta_ns =
780 clockevent_delta2ns(0xF, &lapic_clockevent);
781
782 lapic_timer_frequency = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS;
783
784 apic_printk(APIC_VERBOSE, "..... delta %ld\n", delta);
785 apic_printk(APIC_VERBOSE, "..... mult: %u\n", lapic_clockevent.mult);
786 apic_printk(APIC_VERBOSE, "..... calibration result: %u\n",
787 lapic_timer_frequency);
788
789 if (boot_cpu_has(X86_FEATURE_TSC)) {
790 apic_printk(APIC_VERBOSE, "..... CPU clock speed is "
791 "%ld.%04ld MHz.\n",
792 (deltatsc / LAPIC_CAL_LOOPS) / (1000000 / HZ),
793 (deltatsc / LAPIC_CAL_LOOPS) % (1000000 / HZ));
794 }
795
796 apic_printk(APIC_VERBOSE, "..... host bus clock speed is "
797 "%u.%04u MHz.\n",
798 lapic_timer_frequency / (1000000 / HZ),
799 lapic_timer_frequency % (1000000 / HZ));
800
801 /*
802 * Do a sanity check on the APIC calibration result
803 */
804 if (lapic_timer_frequency < (1000000 / HZ)) {
805 local_irq_enable();
806 pr_warning("APIC frequency too slow, disabling apic timer\n");
807 return -1;
808 }
809
810 levt->features &= ~CLOCK_EVT_FEAT_DUMMY;
811
812 /*
813 * PM timer calibration failed or not turned on
814 * so lets try APIC timer based calibration
815 */
816 if (!pm_referenced) {
817 apic_printk(APIC_VERBOSE, "... verify APIC timer\n");
818
819 /*
820 * Setup the apic timer manually
821 */
822 levt->event_handler = lapic_cal_handler;
823 lapic_timer_set_periodic(levt);
824 lapic_cal_loops = -1;
825
826 /* Let the interrupts run */
827 local_irq_enable();
828
829 while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
830 cpu_relax();
831
832 /* Stop the lapic timer */
833 local_irq_disable();
834 lapic_timer_shutdown(levt);
835
836 /* Jiffies delta */
837 deltaj = lapic_cal_j2 - lapic_cal_j1;
838 apic_printk(APIC_VERBOSE, "... jiffies delta = %lu\n", deltaj);
839
840 /* Check, if the jiffies result is consistent */
841 if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2)
842 apic_printk(APIC_VERBOSE, "... jiffies result ok\n");
843 else
844 levt->features |= CLOCK_EVT_FEAT_DUMMY;
845 }
846 local_irq_enable();
847
848 if (levt->features & CLOCK_EVT_FEAT_DUMMY) {
849 pr_warning("APIC timer disabled due to verification failure\n");
850 return -1;
851 }
852
853 return 0;
854 }
855
856 /*
857 * Setup the boot APIC
858 *
859 * Calibrate and verify the result.
860 */
861 void __init setup_boot_APIC_clock(void)
862 {
863 /*
864 * The local apic timer can be disabled via the kernel
865 * commandline or from the CPU detection code. Register the lapic
866 * timer as a dummy clock event source on SMP systems, so the
867 * broadcast mechanism is used. On UP systems simply ignore it.
868 */
869 if (disable_apic_timer) {
870 pr_info("Disabling APIC timer\n");
871 /* No broadcast on UP ! */
872 if (num_possible_cpus() > 1) {
873 lapic_clockevent.mult = 1;
874 setup_APIC_timer();
875 }
876 return;
877 }
878
879 if (calibrate_APIC_clock()) {
880 /* No broadcast on UP ! */
881 if (num_possible_cpus() > 1)
882 setup_APIC_timer();
883 return;
884 }
885
886 /*
887 * If nmi_watchdog is set to IO_APIC, we need the
888 * PIT/HPET going. Otherwise register lapic as a dummy
889 * device.
890 */
891 lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
892
893 /* Setup the lapic or request the broadcast */
894 setup_APIC_timer();
895 }
896
897 void setup_secondary_APIC_clock(void)
898 {
899 setup_APIC_timer();
900 }
901
902 /*
903 * The guts of the apic timer interrupt
904 */
905 static void local_apic_timer_interrupt(void)
906 {
907 int cpu = smp_processor_id();
908 struct clock_event_device *evt = &per_cpu(lapic_events, cpu);
909
910 /*
911 * Normally we should not be here till LAPIC has been initialized but
912 * in some cases like kdump, its possible that there is a pending LAPIC
913 * timer interrupt from previous kernel's context and is delivered in
914 * new kernel the moment interrupts are enabled.
915 *
916 * Interrupts are enabled early and LAPIC is setup much later, hence
917 * its possible that when we get here evt->event_handler is NULL.
918 * Check for event_handler being NULL and discard the interrupt as
919 * spurious.
920 */
921 if (!evt->event_handler) {
922 pr_warning("Spurious LAPIC timer interrupt on cpu %d\n", cpu);
923 /* Switch it off */
924 lapic_timer_shutdown(evt);
925 return;
926 }
927
928 /*
929 * the NMI deadlock-detector uses this.
930 */
931 inc_irq_stat(apic_timer_irqs);
932
933 evt->event_handler(evt);
934 }
935
936 /*
937 * Local APIC timer interrupt. This is the most natural way for doing
938 * local interrupts, but local timer interrupts can be emulated by
939 * broadcast interrupts too. [in case the hw doesn't support APIC timers]
940 *
941 * [ if a single-CPU system runs an SMP kernel then we call the local
942 * interrupt as well. Thus we cannot inline the local irq ... ]
943 */
944 __visible void __irq_entry smp_apic_timer_interrupt(struct pt_regs *regs)
945 {
946 struct pt_regs *old_regs = set_irq_regs(regs);
947
948 /*
949 * NOTE! We'd better ACK the irq immediately,
950 * because timer handling can be slow.
951 *
952 * update_process_times() expects us to have done irq_enter().
953 * Besides, if we don't timer interrupts ignore the global
954 * interrupt lock, which is the WrongThing (tm) to do.
955 */
956 entering_ack_irq();
957 local_apic_timer_interrupt();
958 exiting_irq();
959
960 set_irq_regs(old_regs);
961 }
962
963 __visible void __irq_entry smp_trace_apic_timer_interrupt(struct pt_regs *regs)
964 {
965 struct pt_regs *old_regs = set_irq_regs(regs);
966
967 /*
968 * NOTE! We'd better ACK the irq immediately,
969 * because timer handling can be slow.
970 *
971 * update_process_times() expects us to have done irq_enter().
972 * Besides, if we don't timer interrupts ignore the global
973 * interrupt lock, which is the WrongThing (tm) to do.
974 */
975 entering_ack_irq();
976 trace_local_timer_entry(LOCAL_TIMER_VECTOR);
977 local_apic_timer_interrupt();
978 trace_local_timer_exit(LOCAL_TIMER_VECTOR);
979 exiting_irq();
980
981 set_irq_regs(old_regs);
982 }
983
984 int setup_profiling_timer(unsigned int multiplier)
985 {
986 return -EINVAL;
987 }
988
989 /*
990 * Local APIC start and shutdown
991 */
992
993 /**
994 * clear_local_APIC - shutdown the local APIC
995 *
996 * This is called, when a CPU is disabled and before rebooting, so the state of
997 * the local APIC has no dangling leftovers. Also used to cleanout any BIOS
998 * leftovers during boot.
999 */
1000 void clear_local_APIC(void)
1001 {
1002 int maxlvt;
1003 u32 v;
1004
1005 /* APIC hasn't been mapped yet */
1006 if (!x2apic_mode && !apic_phys)
1007 return;
1008
1009 maxlvt = lapic_get_maxlvt();
1010 /*
1011 * Masking an LVT entry can trigger a local APIC error
1012 * if the vector is zero. Mask LVTERR first to prevent this.
1013 */
1014 if (maxlvt >= 3) {
1015 v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
1016 apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
1017 }
1018 /*
1019 * Careful: we have to set masks only first to deassert
1020 * any level-triggered sources.
1021 */
1022 v = apic_read(APIC_LVTT);
1023 apic_write(APIC_LVTT, v | APIC_LVT_MASKED);
1024 v = apic_read(APIC_LVT0);
1025 apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
1026 v = apic_read(APIC_LVT1);
1027 apic_write(APIC_LVT1, v | APIC_LVT_MASKED);
1028 if (maxlvt >= 4) {
1029 v = apic_read(APIC_LVTPC);
1030 apic_write(APIC_LVTPC, v | APIC_LVT_MASKED);
1031 }
1032
1033 /* lets not touch this if we didn't frob it */
1034 #ifdef CONFIG_X86_THERMAL_VECTOR
1035 if (maxlvt >= 5) {
1036 v = apic_read(APIC_LVTTHMR);
1037 apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED);
1038 }
1039 #endif
1040 #ifdef CONFIG_X86_MCE_INTEL
1041 if (maxlvt >= 6) {
1042 v = apic_read(APIC_LVTCMCI);
1043 if (!(v & APIC_LVT_MASKED))
1044 apic_write(APIC_LVTCMCI, v | APIC_LVT_MASKED);
1045 }
1046 #endif
1047
1048 /*
1049 * Clean APIC state for other OSs:
1050 */
1051 apic_write(APIC_LVTT, APIC_LVT_MASKED);
1052 apic_write(APIC_LVT0, APIC_LVT_MASKED);
1053 apic_write(APIC_LVT1, APIC_LVT_MASKED);
1054 if (maxlvt >= 3)
1055 apic_write(APIC_LVTERR, APIC_LVT_MASKED);
1056 if (maxlvt >= 4)
1057 apic_write(APIC_LVTPC, APIC_LVT_MASKED);
1058
1059 /* Integrated APIC (!82489DX) ? */
1060 if (lapic_is_integrated()) {
1061 if (maxlvt > 3)
1062 /* Clear ESR due to Pentium errata 3AP and 11AP */
1063 apic_write(APIC_ESR, 0);
1064 apic_read(APIC_ESR);
1065 }
1066 }
1067
1068 /**
1069 * disable_local_APIC - clear and disable the local APIC
1070 */
1071 void disable_local_APIC(void)
1072 {
1073 unsigned int value;
1074
1075 /* APIC hasn't been mapped yet */
1076 if (!x2apic_mode && !apic_phys)
1077 return;
1078
1079 clear_local_APIC();
1080
1081 /*
1082 * Disable APIC (implies clearing of registers
1083 * for 82489DX!).
1084 */
1085 value = apic_read(APIC_SPIV);
1086 value &= ~APIC_SPIV_APIC_ENABLED;
1087 apic_write(APIC_SPIV, value);
1088
1089 #ifdef CONFIG_X86_32
1090 /*
1091 * When LAPIC was disabled by the BIOS and enabled by the kernel,
1092 * restore the disabled state.
1093 */
1094 if (enabled_via_apicbase) {
1095 unsigned int l, h;
1096
1097 rdmsr(MSR_IA32_APICBASE, l, h);
1098 l &= ~MSR_IA32_APICBASE_ENABLE;
1099 wrmsr(MSR_IA32_APICBASE, l, h);
1100 }
1101 #endif
1102 }
1103
1104 /*
1105 * If Linux enabled the LAPIC against the BIOS default disable it down before
1106 * re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and
1107 * not power-off. Additionally clear all LVT entries before disable_local_APIC
1108 * for the case where Linux didn't enable the LAPIC.
1109 */
1110 void lapic_shutdown(void)
1111 {
1112 unsigned long flags;
1113
1114 if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config())
1115 return;
1116
1117 local_irq_save(flags);
1118
1119 #ifdef CONFIG_X86_32
1120 if (!enabled_via_apicbase)
1121 clear_local_APIC();
1122 else
1123 #endif
1124 disable_local_APIC();
1125
1126
1127 local_irq_restore(flags);
1128 }
1129
1130 /**
1131 * sync_Arb_IDs - synchronize APIC bus arbitration IDs
1132 */
1133 void __init sync_Arb_IDs(void)
1134 {
1135 /*
1136 * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not
1137 * needed on AMD.
1138 */
1139 if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
1140 return;
1141
1142 /*
1143 * Wait for idle.
1144 */
1145 apic_wait_icr_idle();
1146
1147 apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n");
1148 apic_write(APIC_ICR, APIC_DEST_ALLINC |
1149 APIC_INT_LEVELTRIG | APIC_DM_INIT);
1150 }
1151
1152 /*
1153 * An initial setup of the virtual wire mode.
1154 */
1155 void __init init_bsp_APIC(void)
1156 {
1157 unsigned int value;
1158
1159 /*
1160 * Don't do the setup now if we have a SMP BIOS as the
1161 * through-I/O-APIC virtual wire mode might be active.
1162 */
1163 if (smp_found_config || !boot_cpu_has(X86_FEATURE_APIC))
1164 return;
1165
1166 /*
1167 * Do not trust the local APIC being empty at bootup.
1168 */
1169 clear_local_APIC();
1170
1171 /*
1172 * Enable APIC.
1173 */
1174 value = apic_read(APIC_SPIV);
1175 value &= ~APIC_VECTOR_MASK;
1176 value |= APIC_SPIV_APIC_ENABLED;
1177
1178 #ifdef CONFIG_X86_32
1179 /* This bit is reserved on P4/Xeon and should be cleared */
1180 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
1181 (boot_cpu_data.x86 == 15))
1182 value &= ~APIC_SPIV_FOCUS_DISABLED;
1183 else
1184 #endif
1185 value |= APIC_SPIV_FOCUS_DISABLED;
1186 value |= SPURIOUS_APIC_VECTOR;
1187 apic_write(APIC_SPIV, value);
1188
1189 /*
1190 * Set up the virtual wire mode.
1191 */
1192 apic_write(APIC_LVT0, APIC_DM_EXTINT);
1193 value = APIC_DM_NMI;
1194 if (!lapic_is_integrated()) /* 82489DX */
1195 value |= APIC_LVT_LEVEL_TRIGGER;
1196 if (apic_extnmi == APIC_EXTNMI_NONE)
1197 value |= APIC_LVT_MASKED;
1198 apic_write(APIC_LVT1, value);
1199 }
1200
1201 static void lapic_setup_esr(void)
1202 {
1203 unsigned int oldvalue, value, maxlvt;
1204
1205 if (!lapic_is_integrated()) {
1206 pr_info("No ESR for 82489DX.\n");
1207 return;
1208 }
1209
1210 if (apic->disable_esr) {
1211 /*
1212 * Something untraceable is creating bad interrupts on
1213 * secondary quads ... for the moment, just leave the
1214 * ESR disabled - we can't do anything useful with the
1215 * errors anyway - mbligh
1216 */
1217 pr_info("Leaving ESR disabled.\n");
1218 return;
1219 }
1220
1221 maxlvt = lapic_get_maxlvt();
1222 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
1223 apic_write(APIC_ESR, 0);
1224 oldvalue = apic_read(APIC_ESR);
1225
1226 /* enables sending errors */
1227 value = ERROR_APIC_VECTOR;
1228 apic_write(APIC_LVTERR, value);
1229
1230 /*
1231 * spec says clear errors after enabling vector.
1232 */
1233 if (maxlvt > 3)
1234 apic_write(APIC_ESR, 0);
1235 value = apic_read(APIC_ESR);
1236 if (value != oldvalue)
1237 apic_printk(APIC_VERBOSE, "ESR value before enabling "
1238 "vector: 0x%08x after: 0x%08x\n",
1239 oldvalue, value);
1240 }
1241
1242 /**
1243 * setup_local_APIC - setup the local APIC
1244 *
1245 * Used to setup local APIC while initializing BSP or bringin up APs.
1246 * Always called with preemption disabled.
1247 */
1248 void setup_local_APIC(void)
1249 {
1250 int cpu = smp_processor_id();
1251 unsigned int value, queued;
1252 int i, j, acked = 0;
1253 unsigned long long tsc = 0, ntsc;
1254 long long max_loops = cpu_khz ? cpu_khz : 1000000;
1255
1256 if (boot_cpu_has(X86_FEATURE_TSC))
1257 tsc = rdtsc();
1258
1259 if (disable_apic) {
1260 disable_ioapic_support();
1261 return;
1262 }
1263
1264 #ifdef CONFIG_X86_32
1265 /* Pound the ESR really hard over the head with a big hammer - mbligh */
1266 if (lapic_is_integrated() && apic->disable_esr) {
1267 apic_write(APIC_ESR, 0);
1268 apic_write(APIC_ESR, 0);
1269 apic_write(APIC_ESR, 0);
1270 apic_write(APIC_ESR, 0);
1271 }
1272 #endif
1273 perf_events_lapic_init();
1274
1275 /*
1276 * Double-check whether this APIC is really registered.
1277 * This is meaningless in clustered apic mode, so we skip it.
1278 */
1279 BUG_ON(!apic->apic_id_registered());
1280
1281 /*
1282 * Intel recommends to set DFR, LDR and TPR before enabling
1283 * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
1284 * document number 292116). So here it goes...
1285 */
1286 apic->init_apic_ldr();
1287
1288 #ifdef CONFIG_X86_32
1289 /*
1290 * APIC LDR is initialized. If logical_apicid mapping was
1291 * initialized during get_smp_config(), make sure it matches the
1292 * actual value.
1293 */
1294 i = early_per_cpu(x86_cpu_to_logical_apicid, cpu);
1295 WARN_ON(i != BAD_APICID && i != logical_smp_processor_id());
1296 /* always use the value from LDR */
1297 early_per_cpu(x86_cpu_to_logical_apicid, cpu) =
1298 logical_smp_processor_id();
1299 #endif
1300
1301 /*
1302 * Set Task Priority to 'accept all'. We never change this
1303 * later on.
1304 */
1305 value = apic_read(APIC_TASKPRI);
1306 value &= ~APIC_TPRI_MASK;
1307 apic_write(APIC_TASKPRI, value);
1308
1309 /*
1310 * After a crash, we no longer service the interrupts and a pending
1311 * interrupt from previous kernel might still have ISR bit set.
1312 *
1313 * Most probably by now CPU has serviced that pending interrupt and
1314 * it might not have done the ack_APIC_irq() because it thought,
1315 * interrupt came from i8259 as ExtInt. LAPIC did not get EOI so it
1316 * does not clear the ISR bit and cpu thinks it has already serivced
1317 * the interrupt. Hence a vector might get locked. It was noticed
1318 * for timer irq (vector 0x31). Issue an extra EOI to clear ISR.
1319 */
1320 do {
1321 queued = 0;
1322 for (i = APIC_ISR_NR - 1; i >= 0; i--)
1323 queued |= apic_read(APIC_IRR + i*0x10);
1324
1325 for (i = APIC_ISR_NR - 1; i >= 0; i--) {
1326 value = apic_read(APIC_ISR + i*0x10);
1327 for (j = 31; j >= 0; j--) {
1328 if (value & (1<<j)) {
1329 ack_APIC_irq();
1330 acked++;
1331 }
1332 }
1333 }
1334 if (acked > 256) {
1335 printk(KERN_ERR "LAPIC pending interrupts after %d EOI\n",
1336 acked);
1337 break;
1338 }
1339 if (queued) {
1340 if (boot_cpu_has(X86_FEATURE_TSC) && cpu_khz) {
1341 ntsc = rdtsc();
1342 max_loops = (cpu_khz << 10) - (ntsc - tsc);
1343 } else
1344 max_loops--;
1345 }
1346 } while (queued && max_loops > 0);
1347 WARN_ON(max_loops <= 0);
1348
1349 /*
1350 * Now that we are all set up, enable the APIC
1351 */
1352 value = apic_read(APIC_SPIV);
1353 value &= ~APIC_VECTOR_MASK;
1354 /*
1355 * Enable APIC
1356 */
1357 value |= APIC_SPIV_APIC_ENABLED;
1358
1359 #ifdef CONFIG_X86_32
1360 /*
1361 * Some unknown Intel IO/APIC (or APIC) errata is biting us with
1362 * certain networking cards. If high frequency interrupts are
1363 * happening on a particular IOAPIC pin, plus the IOAPIC routing
1364 * entry is masked/unmasked at a high rate as well then sooner or
1365 * later IOAPIC line gets 'stuck', no more interrupts are received
1366 * from the device. If focus CPU is disabled then the hang goes
1367 * away, oh well :-(
1368 *
1369 * [ This bug can be reproduced easily with a level-triggered
1370 * PCI Ne2000 networking cards and PII/PIII processors, dual
1371 * BX chipset. ]
1372 */
1373 /*
1374 * Actually disabling the focus CPU check just makes the hang less
1375 * frequent as it makes the interrupt distributon model be more
1376 * like LRU than MRU (the short-term load is more even across CPUs).
1377 * See also the comment in end_level_ioapic_irq(). --macro
1378 */
1379
1380 /*
1381 * - enable focus processor (bit==0)
1382 * - 64bit mode always use processor focus
1383 * so no need to set it
1384 */
1385 value &= ~APIC_SPIV_FOCUS_DISABLED;
1386 #endif
1387
1388 /*
1389 * Set spurious IRQ vector
1390 */
1391 value |= SPURIOUS_APIC_VECTOR;
1392 apic_write(APIC_SPIV, value);
1393
1394 /*
1395 * Set up LVT0, LVT1:
1396 *
1397 * set up through-local-APIC on the BP's LINT0. This is not
1398 * strictly necessary in pure symmetric-IO mode, but sometimes
1399 * we delegate interrupts to the 8259A.
1400 */
1401 /*
1402 * TODO: set up through-local-APIC from through-I/O-APIC? --macro
1403 */
1404 value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
1405 if (!cpu && (pic_mode || !value)) {
1406 value = APIC_DM_EXTINT;
1407 apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", cpu);
1408 } else {
1409 value = APIC_DM_EXTINT | APIC_LVT_MASKED;
1410 apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", cpu);
1411 }
1412 apic_write(APIC_LVT0, value);
1413
1414 /*
1415 * Only the BSP sees the LINT1 NMI signal by default. This can be
1416 * modified by apic_extnmi= boot option.
1417 */
1418 if ((!cpu && apic_extnmi != APIC_EXTNMI_NONE) ||
1419 apic_extnmi == APIC_EXTNMI_ALL)
1420 value = APIC_DM_NMI;
1421 else
1422 value = APIC_DM_NMI | APIC_LVT_MASKED;
1423 if (!lapic_is_integrated()) /* 82489DX */
1424 value |= APIC_LVT_LEVEL_TRIGGER;
1425 apic_write(APIC_LVT1, value);
1426
1427 #ifdef CONFIG_X86_MCE_INTEL
1428 /* Recheck CMCI information after local APIC is up on CPU #0 */
1429 if (!cpu)
1430 cmci_recheck();
1431 #endif
1432 }
1433
1434 static void end_local_APIC_setup(void)
1435 {
1436 lapic_setup_esr();
1437
1438 #ifdef CONFIG_X86_32
1439 {
1440 unsigned int value;
1441 /* Disable the local apic timer */
1442 value = apic_read(APIC_LVTT);
1443 value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
1444 apic_write(APIC_LVTT, value);
1445 }
1446 #endif
1447
1448 apic_pm_activate();
1449 }
1450
1451 /*
1452 * APIC setup function for application processors. Called from smpboot.c
1453 */
1454 void apic_ap_setup(void)
1455 {
1456 setup_local_APIC();
1457 end_local_APIC_setup();
1458 }
1459
1460 #ifdef CONFIG_X86_X2APIC
1461 int x2apic_mode;
1462
1463 enum {
1464 X2APIC_OFF,
1465 X2APIC_ON,
1466 X2APIC_DISABLED,
1467 };
1468 static int x2apic_state;
1469
1470 static void __x2apic_disable(void)
1471 {
1472 u64 msr;
1473
1474 if (!boot_cpu_has(X86_FEATURE_APIC))
1475 return;
1476
1477 rdmsrl(MSR_IA32_APICBASE, msr);
1478 if (!(msr & X2APIC_ENABLE))
1479 return;
1480 /* Disable xapic and x2apic first and then reenable xapic mode */
1481 wrmsrl(MSR_IA32_APICBASE, msr & ~(X2APIC_ENABLE | XAPIC_ENABLE));
1482 wrmsrl(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE);
1483 printk_once(KERN_INFO "x2apic disabled\n");
1484 }
1485
1486 static void __x2apic_enable(void)
1487 {
1488 u64 msr;
1489
1490 rdmsrl(MSR_IA32_APICBASE, msr);
1491 if (msr & X2APIC_ENABLE)
1492 return;
1493 wrmsrl(MSR_IA32_APICBASE, msr | X2APIC_ENABLE);
1494 printk_once(KERN_INFO "x2apic enabled\n");
1495 }
1496
1497 static int __init setup_nox2apic(char *str)
1498 {
1499 if (x2apic_enabled()) {
1500 int apicid = native_apic_msr_read(APIC_ID);
1501
1502 if (apicid >= 255) {
1503 pr_warning("Apicid: %08x, cannot enforce nox2apic\n",
1504 apicid);
1505 return 0;
1506 }
1507 pr_warning("x2apic already enabled.\n");
1508 __x2apic_disable();
1509 }
1510 setup_clear_cpu_cap(X86_FEATURE_X2APIC);
1511 x2apic_state = X2APIC_DISABLED;
1512 x2apic_mode = 0;
1513 return 0;
1514 }
1515 early_param("nox2apic", setup_nox2apic);
1516
1517 /* Called from cpu_init() to enable x2apic on (secondary) cpus */
1518 void x2apic_setup(void)
1519 {
1520 /*
1521 * If x2apic is not in ON state, disable it if already enabled
1522 * from BIOS.
1523 */
1524 if (x2apic_state != X2APIC_ON) {
1525 __x2apic_disable();
1526 return;
1527 }
1528 __x2apic_enable();
1529 }
1530
1531 static __init void x2apic_disable(void)
1532 {
1533 u32 x2apic_id, state = x2apic_state;
1534
1535 x2apic_mode = 0;
1536 x2apic_state = X2APIC_DISABLED;
1537
1538 if (state != X2APIC_ON)
1539 return;
1540
1541 x2apic_id = read_apic_id();
1542 if (x2apic_id >= 255)
1543 panic("Cannot disable x2apic, id: %08x\n", x2apic_id);
1544
1545 __x2apic_disable();
1546 register_lapic_address(mp_lapic_addr);
1547 }
1548
1549 static __init void x2apic_enable(void)
1550 {
1551 if (x2apic_state != X2APIC_OFF)
1552 return;
1553
1554 x2apic_mode = 1;
1555 x2apic_state = X2APIC_ON;
1556 __x2apic_enable();
1557 }
1558
1559 static __init void try_to_enable_x2apic(int remap_mode)
1560 {
1561 if (x2apic_state == X2APIC_DISABLED)
1562 return;
1563
1564 if (remap_mode != IRQ_REMAP_X2APIC_MODE) {
1565 /* IR is required if there is APIC ID > 255 even when running
1566 * under KVM
1567 */
1568 if (max_physical_apicid > 255 ||
1569 !hypervisor_x2apic_available()) {
1570 pr_info("x2apic: IRQ remapping doesn't support X2APIC mode\n");
1571 x2apic_disable();
1572 return;
1573 }
1574
1575 /*
1576 * without IR all CPUs can be addressed by IOAPIC/MSI
1577 * only in physical mode
1578 */
1579 x2apic_phys = 1;
1580 }
1581 x2apic_enable();
1582 }
1583
1584 void __init check_x2apic(void)
1585 {
1586 if (x2apic_enabled()) {
1587 pr_info("x2apic: enabled by BIOS, switching to x2apic ops\n");
1588 x2apic_mode = 1;
1589 x2apic_state = X2APIC_ON;
1590 } else if (!boot_cpu_has(X86_FEATURE_X2APIC)) {
1591 x2apic_state = X2APIC_DISABLED;
1592 }
1593 }
1594 #else /* CONFIG_X86_X2APIC */
1595 static int __init validate_x2apic(void)
1596 {
1597 if (!apic_is_x2apic_enabled())
1598 return 0;
1599 /*
1600 * Checkme: Can we simply turn off x2apic here instead of panic?
1601 */
1602 panic("BIOS has enabled x2apic but kernel doesn't support x2apic, please disable x2apic in BIOS.\n");
1603 }
1604 early_initcall(validate_x2apic);
1605
1606 static inline void try_to_enable_x2apic(int remap_mode) { }
1607 static inline void __x2apic_enable(void) { }
1608 #endif /* !CONFIG_X86_X2APIC */
1609
1610 static int __init try_to_enable_IR(void)
1611 {
1612 #ifdef CONFIG_X86_IO_APIC
1613 if (!x2apic_enabled() && skip_ioapic_setup) {
1614 pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n");
1615 return -1;
1616 }
1617 #endif
1618 return irq_remapping_enable();
1619 }
1620
1621 void __init enable_IR_x2apic(void)
1622 {
1623 unsigned long flags;
1624 int ret, ir_stat;
1625
1626 if (skip_ioapic_setup)
1627 return;
1628
1629 ir_stat = irq_remapping_prepare();
1630 if (ir_stat < 0 && !x2apic_supported())
1631 return;
1632
1633 ret = save_ioapic_entries();
1634 if (ret) {
1635 pr_info("Saving IO-APIC state failed: %d\n", ret);
1636 return;
1637 }
1638
1639 local_irq_save(flags);
1640 legacy_pic->mask_all();
1641 mask_ioapic_entries();
1642
1643 /* If irq_remapping_prepare() succeeded, try to enable it */
1644 if (ir_stat >= 0)
1645 ir_stat = try_to_enable_IR();
1646 /* ir_stat contains the remap mode or an error code */
1647 try_to_enable_x2apic(ir_stat);
1648
1649 if (ir_stat < 0)
1650 restore_ioapic_entries();
1651 legacy_pic->restore_mask();
1652 local_irq_restore(flags);
1653 }
1654
1655 #ifdef CONFIG_X86_64
1656 /*
1657 * Detect and enable local APICs on non-SMP boards.
1658 * Original code written by Keir Fraser.
1659 * On AMD64 we trust the BIOS - if it says no APIC it is likely
1660 * not correctly set up (usually the APIC timer won't work etc.)
1661 */
1662 static int __init detect_init_APIC(void)
1663 {
1664 if (!boot_cpu_has(X86_FEATURE_APIC)) {
1665 pr_info("No local APIC present\n");
1666 return -1;
1667 }
1668
1669 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
1670 return 0;
1671 }
1672 #else
1673
1674 static int __init apic_verify(void)
1675 {
1676 u32 features, h, l;
1677
1678 /*
1679 * The APIC feature bit should now be enabled
1680 * in `cpuid'
1681 */
1682 features = cpuid_edx(1);
1683 if (!(features & (1 << X86_FEATURE_APIC))) {
1684 pr_warning("Could not enable APIC!\n");
1685 return -1;
1686 }
1687 set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
1688 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
1689
1690 /* The BIOS may have set up the APIC at some other address */
1691 if (boot_cpu_data.x86 >= 6) {
1692 rdmsr(MSR_IA32_APICBASE, l, h);
1693 if (l & MSR_IA32_APICBASE_ENABLE)
1694 mp_lapic_addr = l & MSR_IA32_APICBASE_BASE;
1695 }
1696
1697 pr_info("Found and enabled local APIC!\n");
1698 return 0;
1699 }
1700
1701 int __init apic_force_enable(unsigned long addr)
1702 {
1703 u32 h, l;
1704
1705 if (disable_apic)
1706 return -1;
1707
1708 /*
1709 * Some BIOSes disable the local APIC in the APIC_BASE
1710 * MSR. This can only be done in software for Intel P6 or later
1711 * and AMD K7 (Model > 1) or later.
1712 */
1713 if (boot_cpu_data.x86 >= 6) {
1714 rdmsr(MSR_IA32_APICBASE, l, h);
1715 if (!(l & MSR_IA32_APICBASE_ENABLE)) {
1716 pr_info("Local APIC disabled by BIOS -- reenabling.\n");
1717 l &= ~MSR_IA32_APICBASE_BASE;
1718 l |= MSR_IA32_APICBASE_ENABLE | addr;
1719 wrmsr(MSR_IA32_APICBASE, l, h);
1720 enabled_via_apicbase = 1;
1721 }
1722 }
1723 return apic_verify();
1724 }
1725
1726 /*
1727 * Detect and initialize APIC
1728 */
1729 static int __init detect_init_APIC(void)
1730 {
1731 /* Disabled by kernel option? */
1732 if (disable_apic)
1733 return -1;
1734
1735 switch (boot_cpu_data.x86_vendor) {
1736 case X86_VENDOR_AMD:
1737 if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) ||
1738 (boot_cpu_data.x86 >= 15))
1739 break;
1740 goto no_apic;
1741 case X86_VENDOR_INTEL:
1742 if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 ||
1743 (boot_cpu_data.x86 == 5 && boot_cpu_has(X86_FEATURE_APIC)))
1744 break;
1745 goto no_apic;
1746 default:
1747 goto no_apic;
1748 }
1749
1750 if (!boot_cpu_has(X86_FEATURE_APIC)) {
1751 /*
1752 * Over-ride BIOS and try to enable the local APIC only if
1753 * "lapic" specified.
1754 */
1755 if (!force_enable_local_apic) {
1756 pr_info("Local APIC disabled by BIOS -- "
1757 "you can enable it with \"lapic\"\n");
1758 return -1;
1759 }
1760 if (apic_force_enable(APIC_DEFAULT_PHYS_BASE))
1761 return -1;
1762 } else {
1763 if (apic_verify())
1764 return -1;
1765 }
1766
1767 apic_pm_activate();
1768
1769 return 0;
1770
1771 no_apic:
1772 pr_info("No local APIC present or hardware disabled\n");
1773 return -1;
1774 }
1775 #endif
1776
1777 /**
1778 * init_apic_mappings - initialize APIC mappings
1779 */
1780 void __init init_apic_mappings(void)
1781 {
1782 unsigned int new_apicid;
1783
1784 if (x2apic_mode) {
1785 boot_cpu_physical_apicid = read_apic_id();
1786 return;
1787 }
1788
1789 /* If no local APIC can be found return early */
1790 if (!smp_found_config && detect_init_APIC()) {
1791 /* lets NOP'ify apic operations */
1792 pr_info("APIC: disable apic facility\n");
1793 apic_disable();
1794 } else {
1795 apic_phys = mp_lapic_addr;
1796
1797 /*
1798 * acpi lapic path already maps that address in
1799 * acpi_register_lapic_address()
1800 */
1801 if (!acpi_lapic && !smp_found_config)
1802 register_lapic_address(apic_phys);
1803 }
1804
1805 /*
1806 * Fetch the APIC ID of the BSP in case we have a
1807 * default configuration (or the MP table is broken).
1808 */
1809 new_apicid = read_apic_id();
1810 if (boot_cpu_physical_apicid != new_apicid) {
1811 boot_cpu_physical_apicid = new_apicid;
1812 /*
1813 * yeah -- we lie about apic_version
1814 * in case if apic was disabled via boot option
1815 * but it's not a problem for SMP compiled kernel
1816 * since smp_sanity_check is prepared for such a case
1817 * and disable smp mode
1818 */
1819 apic_version[new_apicid] =
1820 GET_APIC_VERSION(apic_read(APIC_LVR));
1821 }
1822 }
1823
1824 void __init register_lapic_address(unsigned long address)
1825 {
1826 mp_lapic_addr = address;
1827
1828 if (!x2apic_mode) {
1829 set_fixmap_nocache(FIX_APIC_BASE, address);
1830 apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n",
1831 APIC_BASE, mp_lapic_addr);
1832 }
1833 if (boot_cpu_physical_apicid == -1U) {
1834 boot_cpu_physical_apicid = read_apic_id();
1835 apic_version[boot_cpu_physical_apicid] =
1836 GET_APIC_VERSION(apic_read(APIC_LVR));
1837 }
1838 }
1839
1840 int apic_version[MAX_LOCAL_APIC];
1841
1842 /*
1843 * Local APIC interrupts
1844 */
1845
1846 /*
1847 * This interrupt should _never_ happen with our APIC/SMP architecture
1848 */
1849 static void __smp_spurious_interrupt(u8 vector)
1850 {
1851 u32 v;
1852
1853 /*
1854 * Check if this really is a spurious interrupt and ACK it
1855 * if it is a vectored one. Just in case...
1856 * Spurious interrupts should not be ACKed.
1857 */
1858 v = apic_read(APIC_ISR + ((vector & ~0x1f) >> 1));
1859 if (v & (1 << (vector & 0x1f)))
1860 ack_APIC_irq();
1861
1862 inc_irq_stat(irq_spurious_count);
1863
1864 /* see sw-dev-man vol 3, chapter 7.4.13.5 */
1865 pr_info("spurious APIC interrupt through vector %02x on CPU#%d, "
1866 "should never happen.\n", vector, smp_processor_id());
1867 }
1868
1869 __visible void smp_spurious_interrupt(struct pt_regs *regs)
1870 {
1871 entering_irq();
1872 __smp_spurious_interrupt(~regs->orig_ax);
1873 exiting_irq();
1874 }
1875
1876 __visible void smp_trace_spurious_interrupt(struct pt_regs *regs)
1877 {
1878 u8 vector = ~regs->orig_ax;
1879
1880 entering_irq();
1881 trace_spurious_apic_entry(vector);
1882 __smp_spurious_interrupt(vector);
1883 trace_spurious_apic_exit(vector);
1884 exiting_irq();
1885 }
1886
1887 /*
1888 * This interrupt should never happen with our APIC/SMP architecture
1889 */
1890 static void __smp_error_interrupt(struct pt_regs *regs)
1891 {
1892 u32 v;
1893 u32 i = 0;
1894 static const char * const error_interrupt_reason[] = {
1895 "Send CS error", /* APIC Error Bit 0 */
1896 "Receive CS error", /* APIC Error Bit 1 */
1897 "Send accept error", /* APIC Error Bit 2 */
1898 "Receive accept error", /* APIC Error Bit 3 */
1899 "Redirectable IPI", /* APIC Error Bit 4 */
1900 "Send illegal vector", /* APIC Error Bit 5 */
1901 "Received illegal vector", /* APIC Error Bit 6 */
1902 "Illegal register address", /* APIC Error Bit 7 */
1903 };
1904
1905 /* First tickle the hardware, only then report what went on. -- REW */
1906 if (lapic_get_maxlvt() > 3) /* Due to the Pentium erratum 3AP. */
1907 apic_write(APIC_ESR, 0);
1908 v = apic_read(APIC_ESR);
1909 ack_APIC_irq();
1910 atomic_inc(&irq_err_count);
1911
1912 apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x",
1913 smp_processor_id(), v);
1914
1915 v &= 0xff;
1916 while (v) {
1917 if (v & 0x1)
1918 apic_printk(APIC_DEBUG, KERN_CONT " : %s", error_interrupt_reason[i]);
1919 i++;
1920 v >>= 1;
1921 }
1922
1923 apic_printk(APIC_DEBUG, KERN_CONT "\n");
1924
1925 }
1926
1927 __visible void smp_error_interrupt(struct pt_regs *regs)
1928 {
1929 entering_irq();
1930 __smp_error_interrupt(regs);
1931 exiting_irq();
1932 }
1933
1934 __visible void smp_trace_error_interrupt(struct pt_regs *regs)
1935 {
1936 entering_irq();
1937 trace_error_apic_entry(ERROR_APIC_VECTOR);
1938 __smp_error_interrupt(regs);
1939 trace_error_apic_exit(ERROR_APIC_VECTOR);
1940 exiting_irq();
1941 }
1942
1943 /**
1944 * connect_bsp_APIC - attach the APIC to the interrupt system
1945 */
1946 static void __init connect_bsp_APIC(void)
1947 {
1948 #ifdef CONFIG_X86_32
1949 if (pic_mode) {
1950 /*
1951 * Do not trust the local APIC being empty at bootup.
1952 */
1953 clear_local_APIC();
1954 /*
1955 * PIC mode, enable APIC mode in the IMCR, i.e. connect BSP's
1956 * local APIC to INT and NMI lines.
1957 */
1958 apic_printk(APIC_VERBOSE, "leaving PIC mode, "
1959 "enabling APIC mode.\n");
1960 imcr_pic_to_apic();
1961 }
1962 #endif
1963 }
1964
1965 /**
1966 * disconnect_bsp_APIC - detach the APIC from the interrupt system
1967 * @virt_wire_setup: indicates, whether virtual wire mode is selected
1968 *
1969 * Virtual wire mode is necessary to deliver legacy interrupts even when the
1970 * APIC is disabled.
1971 */
1972 void disconnect_bsp_APIC(int virt_wire_setup)
1973 {
1974 unsigned int value;
1975
1976 #ifdef CONFIG_X86_32
1977 if (pic_mode) {
1978 /*
1979 * Put the board back into PIC mode (has an effect only on
1980 * certain older boards). Note that APIC interrupts, including
1981 * IPIs, won't work beyond this point! The only exception are
1982 * INIT IPIs.
1983 */
1984 apic_printk(APIC_VERBOSE, "disabling APIC mode, "
1985 "entering PIC mode.\n");
1986 imcr_apic_to_pic();
1987 return;
1988 }
1989 #endif
1990
1991 /* Go back to Virtual Wire compatibility mode */
1992
1993 /* For the spurious interrupt use vector F, and enable it */
1994 value = apic_read(APIC_SPIV);
1995 value &= ~APIC_VECTOR_MASK;
1996 value |= APIC_SPIV_APIC_ENABLED;
1997 value |= 0xf;
1998 apic_write(APIC_SPIV, value);
1999
2000 if (!virt_wire_setup) {
2001 /*
2002 * For LVT0 make it edge triggered, active high,
2003 * external and enabled
2004 */
2005 value = apic_read(APIC_LVT0);
2006 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2007 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2008 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2009 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2010 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
2011 apic_write(APIC_LVT0, value);
2012 } else {
2013 /* Disable LVT0 */
2014 apic_write(APIC_LVT0, APIC_LVT_MASKED);
2015 }
2016
2017 /*
2018 * For LVT1 make it edge triggered, active high,
2019 * nmi and enabled
2020 */
2021 value = apic_read(APIC_LVT1);
2022 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2023 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2024 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2025 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2026 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
2027 apic_write(APIC_LVT1, value);
2028 }
2029
2030 int generic_processor_info(int apicid, int version)
2031 {
2032 int cpu, max = nr_cpu_ids;
2033 bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid,
2034 phys_cpu_present_map);
2035
2036 /*
2037 * boot_cpu_physical_apicid is designed to have the apicid
2038 * returned by read_apic_id(), i.e, the apicid of the
2039 * currently booting-up processor. However, on some platforms,
2040 * it is temporarily modified by the apicid reported as BSP
2041 * through MP table. Concretely:
2042 *
2043 * - arch/x86/kernel/mpparse.c: MP_processor_info()
2044 * - arch/x86/mm/amdtopology.c: amd_numa_init()
2045 *
2046 * This function is executed with the modified
2047 * boot_cpu_physical_apicid. So, disabled_cpu_apicid kernel
2048 * parameter doesn't work to disable APs on kdump 2nd kernel.
2049 *
2050 * Since fixing handling of boot_cpu_physical_apicid requires
2051 * another discussion and tests on each platform, we leave it
2052 * for now and here we use read_apic_id() directly in this
2053 * function, generic_processor_info().
2054 */
2055 if (disabled_cpu_apicid != BAD_APICID &&
2056 disabled_cpu_apicid != read_apic_id() &&
2057 disabled_cpu_apicid == apicid) {
2058 int thiscpu = num_processors + disabled_cpus;
2059
2060 pr_warning("APIC: Disabling requested cpu."
2061 " Processor %d/0x%x ignored.\n",
2062 thiscpu, apicid);
2063
2064 disabled_cpus++;
2065 return -ENODEV;
2066 }
2067
2068 /*
2069 * If boot cpu has not been detected yet, then only allow upto
2070 * nr_cpu_ids - 1 processors and keep one slot free for boot cpu
2071 */
2072 if (!boot_cpu_detected && num_processors >= nr_cpu_ids - 1 &&
2073 apicid != boot_cpu_physical_apicid) {
2074 int thiscpu = max + disabled_cpus - 1;
2075
2076 pr_warning(
2077 "APIC: NR_CPUS/possible_cpus limit of %i almost"
2078 " reached. Keeping one slot for boot cpu."
2079 " Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
2080
2081 disabled_cpus++;
2082 return -ENODEV;
2083 }
2084
2085 if (num_processors >= nr_cpu_ids) {
2086 int thiscpu = max + disabled_cpus;
2087
2088 pr_warning(
2089 "APIC: NR_CPUS/possible_cpus limit of %i reached."
2090 " Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
2091
2092 disabled_cpus++;
2093 return -EINVAL;
2094 }
2095
2096 if (apicid == boot_cpu_physical_apicid) {
2097 /*
2098 * x86_bios_cpu_apicid is required to have processors listed
2099 * in same order as logical cpu numbers. Hence the first
2100 * entry is BSP, and so on.
2101 * boot_cpu_init() already hold bit 0 in cpu_present_mask
2102 * for BSP.
2103 */
2104 cpu = 0;
2105 } else
2106 cpu = cpumask_next_zero(-1, cpu_present_mask);
2107
2108 /*
2109 * This can happen on physical hotplug. The sanity check at boot time
2110 * is done from native_smp_prepare_cpus() after num_possible_cpus() is
2111 * established.
2112 */
2113 if (topology_update_package_map(apicid, cpu) < 0) {
2114 int thiscpu = max + disabled_cpus;
2115
2116 pr_warning("APIC: Package limit reached. Processor %d/0x%x ignored.\n",
2117 thiscpu, apicid);
2118
2119 disabled_cpus++;
2120 return -ENOSPC;
2121 }
2122
2123 num_processors++;
2124
2125 /*
2126 * Validate version
2127 */
2128 if (version == 0x0) {
2129 pr_warning("BIOS bug: APIC version is 0 for CPU %d/0x%x, fixing up to 0x10\n",
2130 cpu, apicid);
2131 version = 0x10;
2132 }
2133 apic_version[apicid] = version;
2134
2135 if (version != apic_version[boot_cpu_physical_apicid]) {
2136 pr_warning("BIOS bug: APIC version mismatch, boot CPU: %x, CPU %d: version %x\n",
2137 apic_version[boot_cpu_physical_apicid], cpu, version);
2138 }
2139
2140 physid_set(apicid, phys_cpu_present_map);
2141 if (apicid > max_physical_apicid)
2142 max_physical_apicid = apicid;
2143
2144 #if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
2145 early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
2146 early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
2147 #endif
2148 #ifdef CONFIG_X86_32
2149 early_per_cpu(x86_cpu_to_logical_apicid, cpu) =
2150 apic->x86_32_early_logical_apicid(cpu);
2151 #endif
2152 set_cpu_possible(cpu, true);
2153 set_cpu_present(cpu, true);
2154
2155 return cpu;
2156 }
2157
2158 int hard_smp_processor_id(void)
2159 {
2160 return read_apic_id();
2161 }
2162
2163 void default_init_apic_ldr(void)
2164 {
2165 unsigned long val;
2166
2167 apic_write(APIC_DFR, APIC_DFR_VALUE);
2168 val = apic_read(APIC_LDR) & ~APIC_LDR_MASK;
2169 val |= SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
2170 apic_write(APIC_LDR, val);
2171 }
2172
2173 int default_cpu_mask_to_apicid_and(const struct cpumask *cpumask,
2174 const struct cpumask *andmask,
2175 unsigned int *apicid)
2176 {
2177 unsigned int cpu;
2178
2179 for_each_cpu_and(cpu, cpumask, andmask) {
2180 if (cpumask_test_cpu(cpu, cpu_online_mask))
2181 break;
2182 }
2183
2184 if (likely(cpu < nr_cpu_ids)) {
2185 *apicid = per_cpu(x86_cpu_to_apicid, cpu);
2186 return 0;
2187 }
2188
2189 return -EINVAL;
2190 }
2191
2192 /*
2193 * Override the generic EOI implementation with an optimized version.
2194 * Only called during early boot when only one CPU is active and with
2195 * interrupts disabled, so we know this does not race with actual APIC driver
2196 * use.
2197 */
2198 void __init apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v))
2199 {
2200 struct apic **drv;
2201
2202 for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) {
2203 /* Should happen once for each apic */
2204 WARN_ON((*drv)->eoi_write == eoi_write);
2205 (*drv)->eoi_write = eoi_write;
2206 }
2207 }
2208
2209 static void __init apic_bsp_up_setup(void)
2210 {
2211 #ifdef CONFIG_X86_64
2212 apic_write(APIC_ID, SET_APIC_ID(boot_cpu_physical_apicid));
2213 #else
2214 /*
2215 * Hack: In case of kdump, after a crash, kernel might be booting
2216 * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid
2217 * might be zero if read from MP tables. Get it from LAPIC.
2218 */
2219 # ifdef CONFIG_CRASH_DUMP
2220 boot_cpu_physical_apicid = read_apic_id();
2221 # endif
2222 #endif
2223 physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
2224 }
2225
2226 /**
2227 * apic_bsp_setup - Setup function for local apic and io-apic
2228 * @upmode: Force UP mode (for APIC_init_uniprocessor)
2229 *
2230 * Returns:
2231 * apic_id of BSP APIC
2232 */
2233 int __init apic_bsp_setup(bool upmode)
2234 {
2235 int id;
2236
2237 connect_bsp_APIC();
2238 if (upmode)
2239 apic_bsp_up_setup();
2240 setup_local_APIC();
2241
2242 if (x2apic_mode)
2243 id = apic_read(APIC_LDR);
2244 else
2245 id = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR));
2246
2247 enable_IO_APIC();
2248 end_local_APIC_setup();
2249 irq_remap_enable_fault_handling();
2250 setup_IO_APIC();
2251 /* Setup local timer */
2252 x86_init.timers.setup_percpu_clockev();
2253 return id;
2254 }
2255
2256 /*
2257 * This initializes the IO-APIC and APIC hardware if this is
2258 * a UP kernel.
2259 */
2260 int __init APIC_init_uniprocessor(void)
2261 {
2262 if (disable_apic) {
2263 pr_info("Apic disabled\n");
2264 return -1;
2265 }
2266 #ifdef CONFIG_X86_64
2267 if (!boot_cpu_has(X86_FEATURE_APIC)) {
2268 disable_apic = 1;
2269 pr_info("Apic disabled by BIOS\n");
2270 return -1;
2271 }
2272 #else
2273 if (!smp_found_config && !boot_cpu_has(X86_FEATURE_APIC))
2274 return -1;
2275
2276 /*
2277 * Complain if the BIOS pretends there is one.
2278 */
2279 if (!boot_cpu_has(X86_FEATURE_APIC) &&
2280 APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
2281 pr_err("BIOS bug, local APIC 0x%x not detected!...\n",
2282 boot_cpu_physical_apicid);
2283 return -1;
2284 }
2285 #endif
2286
2287 if (!smp_found_config)
2288 disable_ioapic_support();
2289
2290 default_setup_apic_routing();
2291 apic_bsp_setup(true);
2292 return 0;
2293 }
2294
2295 #ifdef CONFIG_UP_LATE_INIT
2296 void __init up_late_init(void)
2297 {
2298 APIC_init_uniprocessor();
2299 }
2300 #endif
2301
2302 /*
2303 * Power management
2304 */
2305 #ifdef CONFIG_PM
2306
2307 static struct {
2308 /*
2309 * 'active' is true if the local APIC was enabled by us and
2310 * not the BIOS; this signifies that we are also responsible
2311 * for disabling it before entering apm/acpi suspend
2312 */
2313 int active;
2314 /* r/w apic fields */
2315 unsigned int apic_id;
2316 unsigned int apic_taskpri;
2317 unsigned int apic_ldr;
2318 unsigned int apic_dfr;
2319 unsigned int apic_spiv;
2320 unsigned int apic_lvtt;
2321 unsigned int apic_lvtpc;
2322 unsigned int apic_lvt0;
2323 unsigned int apic_lvt1;
2324 unsigned int apic_lvterr;
2325 unsigned int apic_tmict;
2326 unsigned int apic_tdcr;
2327 unsigned int apic_thmr;
2328 unsigned int apic_cmci;
2329 } apic_pm_state;
2330
2331 static int lapic_suspend(void)
2332 {
2333 unsigned long flags;
2334 int maxlvt;
2335
2336 if (!apic_pm_state.active)
2337 return 0;
2338
2339 maxlvt = lapic_get_maxlvt();
2340
2341 apic_pm_state.apic_id = apic_read(APIC_ID);
2342 apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
2343 apic_pm_state.apic_ldr = apic_read(APIC_LDR);
2344 apic_pm_state.apic_dfr = apic_read(APIC_DFR);
2345 apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
2346 apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
2347 if (maxlvt >= 4)
2348 apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
2349 apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
2350 apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
2351 apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
2352 apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
2353 apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
2354 #ifdef CONFIG_X86_THERMAL_VECTOR
2355 if (maxlvt >= 5)
2356 apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
2357 #endif
2358 #ifdef CONFIG_X86_MCE_INTEL
2359 if (maxlvt >= 6)
2360 apic_pm_state.apic_cmci = apic_read(APIC_LVTCMCI);
2361 #endif
2362
2363 local_irq_save(flags);
2364 disable_local_APIC();
2365
2366 irq_remapping_disable();
2367
2368 local_irq_restore(flags);
2369 return 0;
2370 }
2371
2372 static void lapic_resume(void)
2373 {
2374 unsigned int l, h;
2375 unsigned long flags;
2376 int maxlvt;
2377
2378 if (!apic_pm_state.active)
2379 return;
2380
2381 local_irq_save(flags);
2382
2383 /*
2384 * IO-APIC and PIC have their own resume routines.
2385 * We just mask them here to make sure the interrupt
2386 * subsystem is completely quiet while we enable x2apic
2387 * and interrupt-remapping.
2388 */
2389 mask_ioapic_entries();
2390 legacy_pic->mask_all();
2391
2392 if (x2apic_mode) {
2393 __x2apic_enable();
2394 } else {
2395 /*
2396 * Make sure the APICBASE points to the right address
2397 *
2398 * FIXME! This will be wrong if we ever support suspend on
2399 * SMP! We'll need to do this as part of the CPU restore!
2400 */
2401 if (boot_cpu_data.x86 >= 6) {
2402 rdmsr(MSR_IA32_APICBASE, l, h);
2403 l &= ~MSR_IA32_APICBASE_BASE;
2404 l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
2405 wrmsr(MSR_IA32_APICBASE, l, h);
2406 }
2407 }
2408
2409 maxlvt = lapic_get_maxlvt();
2410 apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
2411 apic_write(APIC_ID, apic_pm_state.apic_id);
2412 apic_write(APIC_DFR, apic_pm_state.apic_dfr);
2413 apic_write(APIC_LDR, apic_pm_state.apic_ldr);
2414 apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri);
2415 apic_write(APIC_SPIV, apic_pm_state.apic_spiv);
2416 apic_write(APIC_LVT0, apic_pm_state.apic_lvt0);
2417 apic_write(APIC_LVT1, apic_pm_state.apic_lvt1);
2418 #ifdef CONFIG_X86_THERMAL_VECTOR
2419 if (maxlvt >= 5)
2420 apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr);
2421 #endif
2422 #ifdef CONFIG_X86_MCE_INTEL
2423 if (maxlvt >= 6)
2424 apic_write(APIC_LVTCMCI, apic_pm_state.apic_cmci);
2425 #endif
2426 if (maxlvt >= 4)
2427 apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc);
2428 apic_write(APIC_LVTT, apic_pm_state.apic_lvtt);
2429 apic_write(APIC_TDCR, apic_pm_state.apic_tdcr);
2430 apic_write(APIC_TMICT, apic_pm_state.apic_tmict);
2431 apic_write(APIC_ESR, 0);
2432 apic_read(APIC_ESR);
2433 apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr);
2434 apic_write(APIC_ESR, 0);
2435 apic_read(APIC_ESR);
2436
2437 irq_remapping_reenable(x2apic_mode);
2438
2439 local_irq_restore(flags);
2440 }
2441
2442 /*
2443 * This device has no shutdown method - fully functioning local APICs
2444 * are needed on every CPU up until machine_halt/restart/poweroff.
2445 */
2446
2447 static struct syscore_ops lapic_syscore_ops = {
2448 .resume = lapic_resume,
2449 .suspend = lapic_suspend,
2450 };
2451
2452 static void apic_pm_activate(void)
2453 {
2454 apic_pm_state.active = 1;
2455 }
2456
2457 static int __init init_lapic_sysfs(void)
2458 {
2459 /* XXX: remove suspend/resume procs if !apic_pm_state.active? */
2460 if (boot_cpu_has(X86_FEATURE_APIC))
2461 register_syscore_ops(&lapic_syscore_ops);
2462
2463 return 0;
2464 }
2465
2466 /* local apic needs to resume before other devices access its registers. */
2467 core_initcall(init_lapic_sysfs);
2468
2469 #else /* CONFIG_PM */
2470
2471 static void apic_pm_activate(void) { }
2472
2473 #endif /* CONFIG_PM */
2474
2475 #ifdef CONFIG_X86_64
2476
2477 static int multi_checked;
2478 static int multi;
2479
2480 static int set_multi(const struct dmi_system_id *d)
2481 {
2482 if (multi)
2483 return 0;
2484 pr_info("APIC: %s detected, Multi Chassis\n", d->ident);
2485 multi = 1;
2486 return 0;
2487 }
2488
2489 static const struct dmi_system_id multi_dmi_table[] = {
2490 {
2491 .callback = set_multi,
2492 .ident = "IBM System Summit2",
2493 .matches = {
2494 DMI_MATCH(DMI_SYS_VENDOR, "IBM"),
2495 DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"),
2496 },
2497 },
2498 {}
2499 };
2500
2501 static void dmi_check_multi(void)
2502 {
2503 if (multi_checked)
2504 return;
2505
2506 dmi_check_system(multi_dmi_table);
2507 multi_checked = 1;
2508 }
2509
2510 /*
2511 * apic_is_clustered_box() -- Check if we can expect good TSC
2512 *
2513 * Thus far, the major user of this is IBM's Summit2 series:
2514 * Clustered boxes may have unsynced TSC problems if they are
2515 * multi-chassis.
2516 * Use DMI to check them
2517 */
2518 int apic_is_clustered_box(void)
2519 {
2520 dmi_check_multi();
2521 return multi;
2522 }
2523 #endif
2524
2525 /*
2526 * APIC command line parameters
2527 */
2528 static int __init setup_disableapic(char *arg)
2529 {
2530 disable_apic = 1;
2531 setup_clear_cpu_cap(X86_FEATURE_APIC);
2532 return 0;
2533 }
2534 early_param("disableapic", setup_disableapic);
2535
2536 /* same as disableapic, for compatibility */
2537 static int __init setup_nolapic(char *arg)
2538 {
2539 return setup_disableapic(arg);
2540 }
2541 early_param("nolapic", setup_nolapic);
2542
2543 static int __init parse_lapic_timer_c2_ok(char *arg)
2544 {
2545 local_apic_timer_c2_ok = 1;
2546 return 0;
2547 }
2548 early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok);
2549
2550 static int __init parse_disable_apic_timer(char *arg)
2551 {
2552 disable_apic_timer = 1;
2553 return 0;
2554 }
2555 early_param("noapictimer", parse_disable_apic_timer);
2556
2557 static int __init parse_nolapic_timer(char *arg)
2558 {
2559 disable_apic_timer = 1;
2560 return 0;
2561 }
2562 early_param("nolapic_timer", parse_nolapic_timer);
2563
2564 static int __init apic_set_verbosity(char *arg)
2565 {
2566 if (!arg) {
2567 #ifdef CONFIG_X86_64
2568 skip_ioapic_setup = 0;
2569 return 0;
2570 #endif
2571 return -EINVAL;
2572 }
2573
2574 if (strcmp("debug", arg) == 0)
2575 apic_verbosity = APIC_DEBUG;
2576 else if (strcmp("verbose", arg) == 0)
2577 apic_verbosity = APIC_VERBOSE;
2578 else {
2579 pr_warning("APIC Verbosity level %s not recognised"
2580 " use apic=verbose or apic=debug\n", arg);
2581 return -EINVAL;
2582 }
2583
2584 return 0;
2585 }
2586 early_param("apic", apic_set_verbosity);
2587
2588 static int __init lapic_insert_resource(void)
2589 {
2590 if (!apic_phys)
2591 return -1;
2592
2593 /* Put local APIC into the resource map. */
2594 lapic_resource.start = apic_phys;
2595 lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;
2596 insert_resource(&iomem_resource, &lapic_resource);
2597
2598 return 0;
2599 }
2600
2601 /*
2602 * need call insert after e820_reserve_resources()
2603 * that is using request_resource
2604 */
2605 late_initcall(lapic_insert_resource);
2606
2607 static int __init apic_set_disabled_cpu_apicid(char *arg)
2608 {
2609 if (!arg || !get_option(&arg, &disabled_cpu_apicid))
2610 return -EINVAL;
2611
2612 return 0;
2613 }
2614 early_param("disable_cpu_apicid", apic_set_disabled_cpu_apicid);
2615
2616 static int __init apic_set_extnmi(char *arg)
2617 {
2618 if (!arg)
2619 return -EINVAL;
2620
2621 if (!strncmp("all", arg, 3))
2622 apic_extnmi = APIC_EXTNMI_ALL;
2623 else if (!strncmp("none", arg, 4))
2624 apic_extnmi = APIC_EXTNMI_NONE;
2625 else if (!strncmp("bsp", arg, 3))
2626 apic_extnmi = APIC_EXTNMI_BSP;
2627 else {
2628 pr_warn("Unknown external NMI delivery mode `%s' ignored\n", arg);
2629 return -EINVAL;
2630 }
2631
2632 return 0;
2633 }
2634 early_param("apic_extnmi", apic_set_extnmi);
Linux kernel - Deliverables
This page took 0.086388 seconds and 4 git commands to generate.