Commit | Line | Data |
---|---|---|
f14f75b8 JS |
1 | /* |
2 | * Copyright (C) 2001-2005 Silicon Graphics, Inc. All rights reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify it | |
5 | * under the terms of version 2 of the GNU General Public License | |
6 | * as published by the Free Software Foundation. | |
7 | * | |
8 | * A simple uncached page allocator using the generic allocator. This | |
9 | * allocator first utilizes the spare (spill) pages found in the EFI | |
10 | * memmap and will then start converting cached pages to uncached ones | |
11 | * at a granule at a time. Node awareness is implemented by having a | |
12 | * pool of pages per node. | |
13 | */ | |
14 | ||
15 | #include <linux/types.h> | |
16 | #include <linux/kernel.h> | |
17 | #include <linux/module.h> | |
18 | #include <linux/init.h> | |
19 | #include <linux/errno.h> | |
20 | #include <linux/string.h> | |
21 | #include <linux/slab.h> | |
22 | #include <linux/efi.h> | |
23 | #include <linux/genalloc.h> | |
24 | #include <asm/page.h> | |
25 | #include <asm/pal.h> | |
26 | #include <asm/system.h> | |
27 | #include <asm/pgtable.h> | |
28 | #include <asm/atomic.h> | |
29 | #include <asm/tlbflush.h> | |
30 | #include <asm/sn/arch.h> | |
31 | ||
32 | #define DEBUG 0 | |
33 | ||
34 | #if DEBUG | |
35 | #define dprintk printk | |
36 | #else | |
37 | #define dprintk(x...) do { } while (0) | |
38 | #endif | |
39 | ||
40 | void __init efi_memmap_walk_uc (efi_freemem_callback_t callback); | |
41 | ||
42 | #define MAX_UNCACHED_GRANULES 5 | |
43 | static int allocated_granules; | |
44 | ||
45 | struct gen_pool *uncached_pool[MAX_NUMNODES]; | |
46 | ||
47 | ||
48 | static void uncached_ipi_visibility(void *data) | |
49 | { | |
50 | int status; | |
51 | ||
52 | status = ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL); | |
53 | if ((status != PAL_VISIBILITY_OK) && | |
54 | (status != PAL_VISIBILITY_OK_REMOTE_NEEDED)) | |
55 | printk(KERN_DEBUG "pal_prefetch_visibility() returns %i on " | |
3bd7f017 | 56 | "CPU %i\n", status, raw_smp_processor_id()); |
f14f75b8 JS |
57 | } |
58 | ||
59 | ||
60 | static void uncached_ipi_mc_drain(void *data) | |
61 | { | |
62 | int status; | |
63 | status = ia64_pal_mc_drain(); | |
64 | if (status) | |
65 | printk(KERN_WARNING "ia64_pal_mc_drain() failed with %i on " | |
3bd7f017 | 66 | "CPU %i\n", status, raw_smp_processor_id()); |
f14f75b8 JS |
67 | } |
68 | ||
69 | ||
70 | static unsigned long | |
71 | uncached_get_new_chunk(struct gen_pool *poolp) | |
72 | { | |
73 | struct page *page; | |
74 | void *tmp; | |
75 | int status, i; | |
76 | unsigned long addr, node; | |
77 | ||
78 | if (allocated_granules >= MAX_UNCACHED_GRANULES) | |
79 | return 0; | |
80 | ||
81 | node = poolp->private; | |
82 | page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, | |
83 | IA64_GRANULE_SHIFT-PAGE_SHIFT); | |
84 | ||
85 | dprintk(KERN_INFO "get_new_chunk page %p, addr %lx\n", | |
86 | page, (unsigned long)(page-vmem_map) << PAGE_SHIFT); | |
87 | ||
88 | /* | |
89 | * Do magic if no mem on local node! XXX | |
90 | */ | |
91 | if (!page) | |
92 | return 0; | |
93 | tmp = page_address(page); | |
94 | ||
95 | /* | |
96 | * There's a small race here where it's possible for someone to | |
97 | * access the page through /dev/mem halfway through the conversion | |
98 | * to uncached - not sure it's really worth bothering about | |
99 | */ | |
100 | for (i = 0; i < (IA64_GRANULE_SIZE / PAGE_SIZE); i++) | |
101 | SetPageUncached(&page[i]); | |
102 | ||
103 | flush_tlb_kernel_range(tmp, tmp + IA64_GRANULE_SIZE); | |
104 | ||
105 | status = ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL); | |
106 | ||
107 | dprintk(KERN_INFO "pal_prefetch_visibility() returns %i on cpu %i\n", | |
3bd7f017 | 108 | status, raw_smp_processor_id()); |
f14f75b8 JS |
109 | |
110 | if (!status) { | |
111 | status = smp_call_function(uncached_ipi_visibility, NULL, 0, 1); | |
112 | if (status) | |
113 | printk(KERN_WARNING "smp_call_function failed for " | |
114 | "uncached_ipi_visibility! (%i)\n", status); | |
115 | } | |
116 | ||
117 | if (ia64_platform_is("sn2")) | |
118 | sn_flush_all_caches((unsigned long)tmp, IA64_GRANULE_SIZE); | |
119 | else | |
120 | flush_icache_range((unsigned long)tmp, | |
121 | (unsigned long)tmp+IA64_GRANULE_SIZE); | |
122 | ||
123 | ia64_pal_mc_drain(); | |
124 | status = smp_call_function(uncached_ipi_mc_drain, NULL, 0, 1); | |
125 | if (status) | |
126 | printk(KERN_WARNING "smp_call_function failed for " | |
127 | "uncached_ipi_mc_drain! (%i)\n", status); | |
128 | ||
129 | addr = (unsigned long)tmp - PAGE_OFFSET + __IA64_UNCACHED_OFFSET; | |
130 | ||
131 | allocated_granules++; | |
132 | return addr; | |
133 | } | |
134 | ||
135 | ||
136 | /* | |
137 | * uncached_alloc_page | |
138 | * | |
139 | * Allocate 1 uncached page. Allocates on the requested node. If no | |
140 | * uncached pages are available on the requested node, roundrobin starting | |
141 | * with higher nodes. | |
142 | */ | |
143 | unsigned long | |
144 | uncached_alloc_page(int nid) | |
145 | { | |
146 | unsigned long maddr; | |
147 | ||
148 | maddr = gen_pool_alloc(uncached_pool[nid], PAGE_SIZE); | |
149 | ||
150 | dprintk(KERN_DEBUG "uncached_alloc_page returns %lx on node %i\n", | |
151 | maddr, nid); | |
152 | ||
153 | /* | |
154 | * If no memory is availble on our local node, try the | |
155 | * remaining nodes in the system. | |
156 | */ | |
157 | if (!maddr) { | |
158 | int i; | |
159 | ||
160 | for (i = MAX_NUMNODES - 1; i >= 0; i--) { | |
161 | if (i == nid || !node_online(i)) | |
162 | continue; | |
163 | maddr = gen_pool_alloc(uncached_pool[i], PAGE_SIZE); | |
164 | dprintk(KERN_DEBUG "uncached_alloc_page alternate search " | |
165 | "returns %lx on node %i\n", maddr, i); | |
166 | if (maddr) { | |
167 | break; | |
168 | } | |
169 | } | |
170 | } | |
171 | ||
172 | return maddr; | |
173 | } | |
174 | EXPORT_SYMBOL(uncached_alloc_page); | |
175 | ||
176 | ||
177 | /* | |
178 | * uncached_free_page | |
179 | * | |
180 | * Free a single uncached page. | |
181 | */ | |
182 | void | |
183 | uncached_free_page(unsigned long maddr) | |
184 | { | |
185 | int node; | |
186 | ||
a994018a | 187 | node = paddr_to_nid(maddr - __IA64_UNCACHED_OFFSET); |
f14f75b8 JS |
188 | |
189 | dprintk(KERN_DEBUG "uncached_free_page(%lx) on node %i\n", maddr, node); | |
190 | ||
191 | if ((maddr & (0XFUL << 60)) != __IA64_UNCACHED_OFFSET) | |
192 | panic("uncached_free_page invalid address %lx\n", maddr); | |
193 | ||
194 | gen_pool_free(uncached_pool[node], maddr, PAGE_SIZE); | |
195 | } | |
196 | EXPORT_SYMBOL(uncached_free_page); | |
197 | ||
198 | ||
199 | /* | |
200 | * uncached_build_memmap, | |
201 | * | |
202 | * Called at boot time to build a map of pages that can be used for | |
203 | * memory special operations. | |
204 | */ | |
205 | static int __init | |
206 | uncached_build_memmap(unsigned long start, unsigned long end, void *arg) | |
207 | { | |
d8c97d5f | 208 | long length = end - start; |
f14f75b8 JS |
209 | int node; |
210 | ||
f14f75b8 JS |
211 | dprintk(KERN_ERR "uncached_build_memmap(%lx %lx)\n", start, end); |
212 | ||
386d1d50 | 213 | touch_softlockup_watchdog(); |
d8c97d5f | 214 | memset((char *)start, 0, length); |
f14f75b8 | 215 | |
d8c97d5f | 216 | node = paddr_to_nid(start - __IA64_UNCACHED_OFFSET); |
f14f75b8 | 217 | |
d8c97d5f TL |
218 | for (; start < end ; start += PAGE_SIZE) { |
219 | dprintk(KERN_INFO "sticking %lx into the pool!\n", start); | |
220 | gen_pool_free(uncached_pool[node], start, PAGE_SIZE); | |
f14f75b8 JS |
221 | } |
222 | ||
223 | return 0; | |
224 | } | |
225 | ||
226 | ||
227 | static int __init uncached_init(void) { | |
228 | int i; | |
229 | ||
230 | for (i = 0; i < MAX_NUMNODES; i++) { | |
231 | if (!node_online(i)) | |
232 | continue; | |
233 | uncached_pool[i] = gen_pool_create(0, IA64_GRANULE_SHIFT, | |
234 | &uncached_get_new_chunk, i); | |
235 | } | |
236 | ||
237 | efi_memmap_walk_uc(uncached_build_memmap); | |
238 | ||
239 | return 0; | |
240 | } | |
241 | ||
242 | __initcall(uncached_init); |