2 * Written by: Patricia Gaughen, IBM Corporation
4 * Copyright (C) 2002, IBM Corp.
5 * Copyright (C) 2009, Red Hat, Inc., Ingo Molnar
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
14 * This program is distributed in the hope that it will be useful, but
15 * WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
17 * NON INFRINGEMENT. See the GNU General Public License for more
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
24 * Send feedback to <gone@us.ibm.com>
26 #include <linux/nodemask.h>
27 #include <linux/topology.h>
28 #include <linux/bootmem.h>
29 #include <linux/threads.h>
30 #include <linux/cpumask.h>
31 #include <linux/kernel.h>
32 #include <linux/mmzone.h>
33 #include <linux/module.h>
34 #include <linux/string.h>
35 #include <linux/init.h>
36 #include <linux/numa.h>
37 #include <linux/smp.h>
41 #include <asm/processor.h>
42 #include <asm/fixmap.h>
43 #include <asm/mpspec.h>
44 #include <asm/numaq.h>
45 #include <asm/setup.h>
50 #define MB_TO_PAGES(addr) ((addr) << (20 - PAGE_SHIFT))
55 * Have to match translation table entries to main table entries by counter
56 * hence the mpc_record variable .... can't see a less disgusting way of
60 unsigned char mpc_type
;
61 unsigned char trans_len
;
62 unsigned char trans_type
;
63 unsigned char trans_quad
;
64 unsigned char trans_global
;
65 unsigned char trans_local
;
66 unsigned short trans_reserved
;
69 static int mpc_record
;
71 static struct mpc_trans
*translation_table
[MAX_MPC_ENTRY
];
73 int mp_bus_id_to_node
[MAX_MP_BUSSES
];
74 int mp_bus_id_to_local
[MAX_MP_BUSSES
];
75 int quad_local_to_mp_bus_id
[NR_CPUS
/4][4];
78 static inline void numaq_register_node(int node
, struct sys_cfg_data
*scd
)
80 struct eachquadmem
*eq
= scd
->eq
+ node
;
82 node_set_online(node
);
84 /* Convert to pages */
85 node_start_pfn
[node
] =
86 MB_TO_PAGES(eq
->hi_shrd_mem_start
- eq
->priv_mem_size
);
89 MB_TO_PAGES(eq
->hi_shrd_mem_start
+ eq
->hi_shrd_mem_size
);
91 e820_register_active_regions(node
, node_start_pfn
[node
],
94 memory_present(node
, node_start_pfn
[node
], node_end_pfn
[node
]);
96 node_remap_size
[node
] = node_memmap_size_bytes(node
,
102 * Function: smp_dump_qct()
104 * Description: gets memory layout from the quad config table. This
105 * function also updates node_online_map with the nodes (quads) present.
107 static void __init
smp_dump_qct(void)
109 struct sys_cfg_data
*scd
;
112 scd
= (void *)__va(SYS_CFG_DATA_PRIV_ADDR
);
114 nodes_clear(node_online_map
);
115 for_each_node(node
) {
116 if (scd
->quads_present31_0
& (1 << node
))
117 numaq_register_node(node
, scd
);
121 void __cpuinit
numaq_tsc_disable(void)
126 if (num_online_nodes() > 1) {
127 printk(KERN_DEBUG
"NUMAQ: disabling TSC\n");
128 setup_clear_cpu_cap(X86_FEATURE_TSC
);
132 static int __init
numaq_pre_time_init(void)
138 static inline int generate_logical_apicid(int quad
, int phys_apicid
)
140 return (quad
<< 4) + (phys_apicid
? phys_apicid
<< 1 : 1);
143 /* x86_quirks member */
144 static int mpc_apic_id(struct mpc_cpu
*m
)
146 int quad
= translation_table
[mpc_record
]->trans_quad
;
147 int logical_apicid
= generate_logical_apicid(quad
, m
->apicid
);
150 "Processor #%d %u:%u APIC version %d (quad %d, apic %d)\n",
151 m
->apicid
, (m
->cpufeature
& CPU_FAMILY_MASK
) >> 8,
152 (m
->cpufeature
& CPU_MODEL_MASK
) >> 4,
153 m
->apicver
, quad
, logical_apicid
);
155 return logical_apicid
;
158 /* x86_quirks member */
159 static void mpc_oem_bus_info(struct mpc_bus
*m
, char *name
)
161 int quad
= translation_table
[mpc_record
]->trans_quad
;
162 int local
= translation_table
[mpc_record
]->trans_local
;
164 mp_bus_id_to_node
[m
->busid
] = quad
;
165 mp_bus_id_to_local
[m
->busid
] = local
;
167 printk(KERN_INFO
"Bus #%d is %s (node %d)\n", m
->busid
, name
, quad
);
170 /* x86_quirks member */
171 static void mpc_oem_pci_bus(struct mpc_bus
*m
)
173 int quad
= translation_table
[mpc_record
]->trans_quad
;
174 int local
= translation_table
[mpc_record
]->trans_local
;
176 quad_local_to_mp_bus_id
[quad
][local
] = m
->busid
;
180 * Called from mpparse code.
182 * mode = 1: one mpc entry scanned
184 static void numaq_mpc_record(unsigned int mode
)
192 static void __init
MP_translation_info(struct mpc_trans
*m
)
195 "Translation: record %d, type %d, quad %d, global %d, local %d\n",
196 mpc_record
, m
->trans_type
, m
->trans_quad
, m
->trans_global
,
199 if (mpc_record
>= MAX_MPC_ENTRY
)
200 printk(KERN_ERR
"MAX_MPC_ENTRY exceeded!\n");
202 translation_table
[mpc_record
] = m
; /* stash this for later */
204 if (m
->trans_quad
< MAX_NUMNODES
&& !node_online(m
->trans_quad
))
205 node_set_online(m
->trans_quad
);
208 static int __init
mpf_checksum(unsigned char *mp
, int len
)
219 * Read/parse the MPC oem tables
222 smp_read_mpc_oem(struct mpc_oemtable
*oemtable
, unsigned short oemsize
)
224 int count
= sizeof(*oemtable
); /* the header size */
225 unsigned char *oemptr
= ((unsigned char *)oemtable
) + count
;
229 "Found an OEM MPC table at %8p - parsing it ... \n", oemtable
);
231 if (memcmp(oemtable
->signature
, MPC_OEM_SIGNATURE
, 4)) {
233 "SMP mpc oemtable: bad signature [%c%c%c%c]!\n",
234 oemtable
->signature
[0], oemtable
->signature
[1],
235 oemtable
->signature
[2], oemtable
->signature
[3]);
239 if (mpf_checksum((unsigned char *)oemtable
, oemtable
->length
)) {
240 printk(KERN_WARNING
"SMP oem mptable: checksum error!\n");
244 while (count
< oemtable
->length
) {
248 struct mpc_trans
*m
= (void *)oemptr
;
250 MP_translation_info(m
);
251 oemptr
+= sizeof(*m
);
258 "Unrecognised OEM table entry type! - %d\n",
265 static struct x86_quirks numaq_x86_quirks __initdata
= {
266 .arch_pre_time_init
= numaq_pre_time_init
,
267 .arch_time_init
= NULL
,
268 .arch_pre_intr_init
= NULL
,
269 .arch_intr_init
= NULL
,
270 .arch_trap_init
= NULL
,
271 .mach_get_smp_config
= NULL
,
272 .mach_find_smp_config
= NULL
,
273 .mpc_apic_id
= mpc_apic_id
,
274 .mpc_oem_bus_info
= mpc_oem_bus_info
,
275 .mpc_oem_pci_bus
= mpc_oem_pci_bus
,
276 .smp_read_mpc_oem
= smp_read_mpc_oem
,
279 static __init
void early_check_numaq(void)
282 * Find possible boot-time SMP configuration:
284 early_find_smp_config();
287 * get boot-time SMP configuration:
289 if (smp_found_config
)
290 early_get_smp_config();
293 x86_quirks
= &numaq_x86_quirks
;
294 x86_init
.mpparse
.mpc_record
= numaq_mpc_record
;
295 x86_init
.mpparse
.setup_ioapic_ids
= x86_init_noop
;
299 int __init
get_memcfg_numaq(void)
309 #define NUMAQ_APIC_DFR_VALUE (APIC_DFR_CLUSTER)
311 static inline unsigned int numaq_get_apic_id(unsigned long x
)
313 return (x
>> 24) & 0x0F;
316 static inline void numaq_send_IPI_mask(const struct cpumask
*mask
, int vector
)
318 default_send_IPI_mask_sequence_logical(mask
, vector
);
321 static inline void numaq_send_IPI_allbutself(int vector
)
323 default_send_IPI_mask_allbutself_logical(cpu_online_mask
, vector
);
326 static inline void numaq_send_IPI_all(int vector
)
328 numaq_send_IPI_mask(cpu_online_mask
, vector
);
331 #define NUMAQ_TRAMPOLINE_PHYS_LOW (0x8)
332 #define NUMAQ_TRAMPOLINE_PHYS_HIGH (0xa)
335 * Because we use NMIs rather than the INIT-STARTUP sequence to
336 * bootstrap the CPUs, the APIC may be in a weird state. Kick it:
338 static inline void numaq_smp_callin_clear_local_apic(void)
343 static inline const struct cpumask
*numaq_target_cpus(void)
348 static inline unsigned long
349 numaq_check_apicid_used(physid_mask_t bitmap
, int apicid
)
351 return physid_isset(apicid
, bitmap
);
354 static inline unsigned long numaq_check_apicid_present(int bit
)
356 return physid_isset(bit
, phys_cpu_present_map
);
359 static inline int numaq_apic_id_registered(void)
364 static inline void numaq_init_apic_ldr(void)
366 /* Already done in NUMA-Q firmware */
369 static inline void numaq_setup_apic_routing(void)
372 "Enabling APIC mode: NUMA-Q. Using %d I/O APICs\n",
377 * Skip adding the timer int on secondary nodes, which causes
378 * a small but painful rift in the time-space continuum.
380 static inline int numaq_multi_timer_check(int apic
, int irq
)
382 return apic
!= 0 && irq
== 0;
385 static inline physid_mask_t
numaq_ioapic_phys_id_map(physid_mask_t phys_map
)
387 /* We don't have a good way to do this yet - hack */
388 return physids_promote(0xFUL
);
391 static inline int numaq_cpu_to_logical_apicid(int cpu
)
393 if (cpu
>= nr_cpu_ids
)
395 return cpu_2_logical_apicid
[cpu
];
399 * Supporting over 60 cpus on NUMA-Q requires a locality-dependent
400 * cpu to APIC ID relation to properly interact with the intelligent
401 * mode of the cluster controller.
403 static inline int numaq_cpu_present_to_apicid(int mps_cpu
)
406 return ((mps_cpu
>> 2) << 4) | (1 << (mps_cpu
& 0x3));
411 static inline int numaq_apicid_to_node(int logical_apicid
)
413 return logical_apicid
>> 4;
416 static inline physid_mask_t
numaq_apicid_to_cpu_present(int logical_apicid
)
418 int node
= numaq_apicid_to_node(logical_apicid
);
419 int cpu
= __ffs(logical_apicid
& 0xf);
421 return physid_mask_of_physid(cpu
+ 4*node
);
424 /* Where the IO area was mapped on multiquad, always 0 otherwise */
427 static inline int numaq_check_phys_apicid_present(int boot_cpu_physical_apicid
)
433 * We use physical apicids here, not logical, so just return the default
434 * physical broadcast to stop people from breaking us
436 static unsigned int numaq_cpu_mask_to_apicid(const struct cpumask
*cpumask
)
441 static inline unsigned int
442 numaq_cpu_mask_to_apicid_and(const struct cpumask
*cpumask
,
443 const struct cpumask
*andmask
)
448 /* No NUMA-Q box has a HT CPU, but it can't hurt to use the default code. */
449 static inline int numaq_phys_pkg_id(int cpuid_apic
, int index_msb
)
451 return cpuid_apic
>> index_msb
;
455 numaq_mps_oem_check(struct mpc_table
*mpc
, char *oem
, char *productid
)
457 if (strncmp(oem
, "IBM NUMA", 8))
458 printk(KERN_ERR
"Warning! Not a NUMA-Q system!\n");
465 static int probe_numaq(void)
467 /* already know from get_memcfg_numaq() */
471 static void numaq_vector_allocation_domain(int cpu
, struct cpumask
*retmask
)
473 /* Careful. Some cpus do not strictly honor the set of cpus
474 * specified in the interrupt destination when using lowest
475 * priority interrupt delivery mode.
477 * In particular there was a hyperthreading cpu observed to
478 * deliver interrupts to the wrong hyperthread when only one
479 * hyperthread was specified in the interrupt desitination.
481 cpumask_clear(retmask
);
482 cpumask_bits(retmask
)[0] = APIC_ALL_CPUS
;
485 static void numaq_setup_portio_remap(void)
487 int num_quads
= num_online_nodes();
493 "Remapping cross-quad port I/O for %d quads\n", num_quads
);
495 xquad_portio
= ioremap(XQUAD_PORTIO_BASE
, num_quads
*XQUAD_PORTIO_QUAD
);
498 "xquad_portio vaddr 0x%08lx, len %08lx\n",
499 (u_long
) xquad_portio
, (u_long
) num_quads
*XQUAD_PORTIO_QUAD
);
502 /* Use __refdata to keep false positive warning calm. */
503 struct apic __refdata apic_numaq
= {
506 .probe
= probe_numaq
,
507 .acpi_madt_oem_check
= NULL
,
508 .apic_id_registered
= numaq_apic_id_registered
,
510 .irq_delivery_mode
= dest_LowestPrio
,
511 /* physical delivery on LOCAL quad: */
514 .target_cpus
= numaq_target_cpus
,
516 .dest_logical
= APIC_DEST_LOGICAL
,
517 .check_apicid_used
= numaq_check_apicid_used
,
518 .check_apicid_present
= numaq_check_apicid_present
,
520 .vector_allocation_domain
= numaq_vector_allocation_domain
,
521 .init_apic_ldr
= numaq_init_apic_ldr
,
523 .ioapic_phys_id_map
= numaq_ioapic_phys_id_map
,
524 .setup_apic_routing
= numaq_setup_apic_routing
,
525 .multi_timer_check
= numaq_multi_timer_check
,
526 .apicid_to_node
= numaq_apicid_to_node
,
527 .cpu_to_logical_apicid
= numaq_cpu_to_logical_apicid
,
528 .cpu_present_to_apicid
= numaq_cpu_present_to_apicid
,
529 .apicid_to_cpu_present
= numaq_apicid_to_cpu_present
,
530 .setup_portio_remap
= numaq_setup_portio_remap
,
531 .check_phys_apicid_present
= numaq_check_phys_apicid_present
,
532 .enable_apic_mode
= NULL
,
533 .phys_pkg_id
= numaq_phys_pkg_id
,
534 .mps_oem_check
= numaq_mps_oem_check
,
536 .get_apic_id
= numaq_get_apic_id
,
538 .apic_id_mask
= 0x0F << 24,
540 .cpu_mask_to_apicid
= numaq_cpu_mask_to_apicid
,
541 .cpu_mask_to_apicid_and
= numaq_cpu_mask_to_apicid_and
,
543 .send_IPI_mask
= numaq_send_IPI_mask
,
544 .send_IPI_mask_allbutself
= NULL
,
545 .send_IPI_allbutself
= numaq_send_IPI_allbutself
,
546 .send_IPI_all
= numaq_send_IPI_all
,
547 .send_IPI_self
= default_send_IPI_self
,
549 .wakeup_secondary_cpu
= wakeup_secondary_cpu_via_nmi
,
550 .trampoline_phys_low
= NUMAQ_TRAMPOLINE_PHYS_LOW
,
551 .trampoline_phys_high
= NUMAQ_TRAMPOLINE_PHYS_HIGH
,
553 /* We don't do anything here because we use NMI's to boot instead */
554 .wait_for_init_deassert
= NULL
,
556 .smp_callin_clear_local_apic
= numaq_smp_callin_clear_local_apic
,
557 .inquire_remote_apic
= NULL
,
559 .read
= native_apic_mem_read
,
560 .write
= native_apic_mem_write
,
561 .icr_read
= native_apic_icr_read
,
562 .icr_write
= native_apic_icr_write
,
563 .wait_icr_idle
= native_apic_wait_icr_idle
,
564 .safe_wait_icr_idle
= native_safe_apic_wait_icr_idle
,