2 * NUMA support, based on the x86 implementation.
4 * Copyright (C) 2015 Cavium Inc.
5 * Author: Ganapatrao Kulkarni <gkulkarni@cavium.com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 #include <linux/acpi.h>
21 #include <linux/bootmem.h>
22 #include <linux/memblock.h>
23 #include <linux/module.h>
28 struct pglist_data
*node_data
[MAX_NUMNODES
] __read_mostly
;
29 EXPORT_SYMBOL(node_data
);
30 nodemask_t numa_nodes_parsed __initdata
;
31 static int cpu_to_node_map
[NR_CPUS
] = { [0 ... NR_CPUS
-1] = NUMA_NO_NODE
};
33 static int numa_distance_cnt
;
34 static u8
*numa_distance
;
37 static __init
int numa_parse_early_param(char *opt
)
41 if (!strncmp(opt
, "off", 3)) {
42 pr_info("%s\n", "NUMA turned off");
47 early_param("numa", numa_parse_early_param
);
49 cpumask_var_t node_to_cpumask_map
[MAX_NUMNODES
];
50 EXPORT_SYMBOL(node_to_cpumask_map
);
52 #ifdef CONFIG_DEBUG_PER_CPU_MAPS
55 * Returns a pointer to the bitmask of CPUs on Node 'node'.
57 const struct cpumask
*cpumask_of_node(int node
)
59 if (WARN_ON(node
>= nr_node_ids
))
62 if (WARN_ON(node_to_cpumask_map
[node
] == NULL
))
63 return cpu_online_mask
;
65 return node_to_cpumask_map
[node
];
67 EXPORT_SYMBOL(cpumask_of_node
);
71 static void map_cpu_to_node(unsigned int cpu
, int nid
)
73 set_cpu_numa_node(cpu
, nid
);
75 cpumask_set_cpu(cpu
, node_to_cpumask_map
[nid
]);
78 void numa_clear_node(unsigned int cpu
)
80 int nid
= cpu_to_node(cpu
);
83 cpumask_clear_cpu(cpu
, node_to_cpumask_map
[nid
]);
84 set_cpu_numa_node(cpu
, NUMA_NO_NODE
);
88 * Allocate node_to_cpumask_map based on number of available nodes
89 * Requires node_possible_map to be valid.
91 * Note: cpumask_of_node() is not valid until after this is done.
92 * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
94 static void __init
setup_node_to_cpumask_map(void)
99 /* setup nr_node_ids if not done yet */
100 if (nr_node_ids
== MAX_NUMNODES
)
103 /* allocate and clear the mapping */
104 for (node
= 0; node
< nr_node_ids
; node
++) {
105 alloc_bootmem_cpumask_var(&node_to_cpumask_map
[node
]);
106 cpumask_clear(node_to_cpumask_map
[node
]);
109 for_each_possible_cpu(cpu
)
110 set_cpu_numa_node(cpu
, NUMA_NO_NODE
);
112 /* cpumask_of_node() will now work */
113 pr_debug("NUMA: Node to cpumask map for %d nodes\n", nr_node_ids
);
117 * Set the cpu to node and mem mapping
119 void numa_store_cpu_info(unsigned int cpu
)
121 map_cpu_to_node(cpu
, numa_off
? 0 : cpu_to_node_map
[cpu
]);
124 void __init
early_map_cpu_to_node(unsigned int cpu
, int nid
)
126 /* fallback to node 0 */
127 if (nid
< 0 || nid
>= MAX_NUMNODES
)
130 cpu_to_node_map
[cpu
] = nid
;
134 * numa_add_memblk - Set node id to memblk
135 * @nid: NUMA node ID of the new memblk
136 * @start: Start address of the new memblk
137 * @end: End address of the new memblk
140 * 0 on success, -errno on failure.
142 int __init
numa_add_memblk(int nid
, u64 start
, u64 end
)
146 ret
= memblock_set_node(start
, (end
- start
), &memblock
.memory
, nid
);
148 pr_err("NUMA: memblock [0x%llx - 0x%llx] failed to add on node %d\n",
149 start
, (end
- 1), nid
);
153 node_set(nid
, numa_nodes_parsed
);
154 pr_info("NUMA: Adding memblock [0x%llx - 0x%llx] on node %d\n",
155 start
, (end
- 1), nid
);
160 * Initialize NODE_DATA for a node on the local memory
162 static void __init
setup_node_data(int nid
, u64 start_pfn
, u64 end_pfn
)
164 const size_t nd_size
= roundup(sizeof(pg_data_t
), SMP_CACHE_BYTES
);
169 pr_info("NUMA: Initmem setup node %d [mem %#010Lx-%#010Lx]\n",
170 nid
, start_pfn
<< PAGE_SHIFT
,
171 (end_pfn
<< PAGE_SHIFT
) - 1);
173 nd_pa
= memblock_alloc_try_nid(nd_size
, SMP_CACHE_BYTES
, nid
);
176 /* report and initialize */
177 pr_info("NUMA: NODE_DATA [mem %#010Lx-%#010Lx]\n",
178 nd_pa
, nd_pa
+ nd_size
- 1);
179 tnid
= early_pfn_to_nid(nd_pa
>> PAGE_SHIFT
);
181 pr_info("NUMA: NODE_DATA(%d) on node %d\n", nid
, tnid
);
184 memset(NODE_DATA(nid
), 0, sizeof(pg_data_t
));
185 NODE_DATA(nid
)->node_id
= nid
;
186 NODE_DATA(nid
)->node_start_pfn
= start_pfn
;
187 NODE_DATA(nid
)->node_spanned_pages
= end_pfn
- start_pfn
;
193 * The current table is freed.
195 void __init
numa_free_distance(void)
202 size
= numa_distance_cnt
* numa_distance_cnt
*
203 sizeof(numa_distance
[0]);
205 memblock_free(__pa(numa_distance
), size
);
206 numa_distance_cnt
= 0;
207 numa_distance
= NULL
;
212 * Create a new NUMA distance table.
215 static int __init
numa_alloc_distance(void)
221 size
= nr_node_ids
* nr_node_ids
* sizeof(numa_distance
[0]);
222 phys
= memblock_find_in_range(0, PFN_PHYS(max_pfn
),
227 memblock_reserve(phys
, size
);
229 numa_distance
= __va(phys
);
230 numa_distance_cnt
= nr_node_ids
;
232 /* fill with the default distances */
233 for (i
= 0; i
< numa_distance_cnt
; i
++)
234 for (j
= 0; j
< numa_distance_cnt
; j
++)
235 numa_distance
[i
* numa_distance_cnt
+ j
] = i
== j
?
236 LOCAL_DISTANCE
: REMOTE_DISTANCE
;
238 pr_debug("NUMA: Initialized distance table, cnt=%d\n",
245 * numa_set_distance - Set inter node NUMA distance from node to node.
246 * @from: the 'from' node to set distance
247 * @to: the 'to' node to set distance
248 * @distance: NUMA distance
250 * Set the distance from node @from to @to to @distance.
251 * If distance table doesn't exist, a warning is printed.
253 * If @from or @to is higher than the highest known node or lower than zero
254 * or @distance doesn't make sense, the call is ignored.
257 void __init
numa_set_distance(int from
, int to
, int distance
)
259 if (!numa_distance
) {
260 pr_warn_once("NUMA: Warning: distance table not allocated yet\n");
264 if (from
>= numa_distance_cnt
|| to
>= numa_distance_cnt
||
265 from
< 0 || to
< 0) {
266 pr_warn_once("NUMA: Warning: node ids are out of bound, from=%d to=%d distance=%d\n",
271 if ((u8
)distance
!= distance
||
272 (from
== to
&& distance
!= LOCAL_DISTANCE
)) {
273 pr_warn_once("NUMA: Warning: invalid distance parameter, from=%d to=%d distance=%d\n",
278 numa_distance
[from
* numa_distance_cnt
+ to
] = distance
;
282 * Return NUMA distance @from to @to
284 int __node_distance(int from
, int to
)
286 if (from
>= numa_distance_cnt
|| to
>= numa_distance_cnt
)
287 return from
== to
? LOCAL_DISTANCE
: REMOTE_DISTANCE
;
288 return numa_distance
[from
* numa_distance_cnt
+ to
];
290 EXPORT_SYMBOL(__node_distance
);
292 static int __init
numa_register_nodes(void)
295 struct memblock_region
*mblk
;
297 /* Check that valid nid is set to memblks */
298 for_each_memblock(memory
, mblk
)
299 if (mblk
->nid
== NUMA_NO_NODE
|| mblk
->nid
>= MAX_NUMNODES
) {
300 pr_warn("NUMA: Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n",
301 mblk
->nid
, mblk
->base
,
302 mblk
->base
+ mblk
->size
- 1);
306 /* Finally register nodes. */
307 for_each_node_mask(nid
, numa_nodes_parsed
) {
308 unsigned long start_pfn
, end_pfn
;
310 get_pfn_range_for_nid(nid
, &start_pfn
, &end_pfn
);
311 setup_node_data(nid
, start_pfn
, end_pfn
);
312 node_set_online(nid
);
315 /* Setup online nodes to actual nodes*/
316 node_possible_map
= numa_nodes_parsed
;
321 static int __init
numa_init(int (*init_func
)(void))
325 nodes_clear(numa_nodes_parsed
);
326 nodes_clear(node_possible_map
);
327 nodes_clear(node_online_map
);
328 numa_free_distance();
330 ret
= numa_alloc_distance();
338 if (nodes_empty(numa_nodes_parsed
))
341 ret
= numa_register_nodes();
345 setup_node_to_cpumask_map();
347 /* init boot processor */
348 cpu_to_node_map
[0] = 0;
349 map_cpu_to_node(0, 0);
355 * dummy_numa_init - Fallback dummy NUMA init
357 * Used if there's no underlying NUMA architecture, NUMA initialization
358 * fails, or NUMA is disabled on the command line.
360 * Must online at least one node (node 0) and add memory blocks that cover all
361 * allowed memory. It is unlikely that this function fails.
363 static int __init
dummy_numa_init(void)
366 struct memblock_region
*mblk
;
369 pr_info("NUMA disabled\n"); /* Forced off on command line. */
371 pr_info("No NUMA configuration found\n");
372 pr_info("NUMA: Faking a node at [mem %#018Lx-%#018Lx]\n",
373 0LLU, PFN_PHYS(max_pfn
) - 1);
375 for_each_memblock(memory
, mblk
) {
376 ret
= numa_add_memblk(0, mblk
->base
, mblk
->base
+ mblk
->size
);
380 pr_err("NUMA init failed\n");
389 * arm64_numa_init - Initialize NUMA
391 * Try each configured NUMA initialization method until one succeeds. The
392 * last fallback is dummy single node config encomapssing whole memory.
394 void __init
arm64_numa_init(void)
397 if (!acpi_disabled
&& !numa_init(arm64_acpi_numa_init
))
399 if (acpi_disabled
&& !numa_init(of_numa_init
))
403 numa_init(dummy_numa_init
);