Commit | Line | Data |
---|---|---|
4a907dec | 1 | /* irq.c: UltraSparc IRQ handling/init/registry. |
1da177e4 | 2 | * |
227c3311 | 3 | * Copyright (C) 1997, 2007, 2008 David S. Miller (davem@davemloft.net) |
1da177e4 LT |
4 | * Copyright (C) 1998 Eddie C. Dost (ecd@skynet.be) |
5 | * Copyright (C) 1998 Jakub Jelinek (jj@ultra.linux.cz) | |
6 | */ | |
7 | ||
1da177e4 | 8 | #include <linux/sched.h> |
9843099f | 9 | #include <linux/linkage.h> |
1da177e4 LT |
10 | #include <linux/ptrace.h> |
11 | #include <linux/errno.h> | |
12 | #include <linux/kernel_stat.h> | |
13 | #include <linux/signal.h> | |
14 | #include <linux/mm.h> | |
15 | #include <linux/interrupt.h> | |
16 | #include <linux/slab.h> | |
17 | #include <linux/random.h> | |
18 | #include <linux/init.h> | |
19 | #include <linux/delay.h> | |
20 | #include <linux/proc_fs.h> | |
21 | #include <linux/seq_file.h> | |
9960e9e8 | 22 | #include <linux/ftrace.h> |
e18e2a00 | 23 | #include <linux/irq.h> |
2e2dc1d7 | 24 | #include <linux/kmemleak.h> |
1da177e4 LT |
25 | |
26 | #include <asm/ptrace.h> | |
27 | #include <asm/processor.h> | |
60063497 | 28 | #include <linux/atomic.h> |
1da177e4 | 29 | #include <asm/irq.h> |
2e457ef6 | 30 | #include <asm/io.h> |
1da177e4 LT |
31 | #include <asm/iommu.h> |
32 | #include <asm/upa.h> | |
33 | #include <asm/oplib.h> | |
25c7581b | 34 | #include <asm/prom.h> |
1da177e4 LT |
35 | #include <asm/timer.h> |
36 | #include <asm/smp.h> | |
37 | #include <asm/starfire.h> | |
38 | #include <asm/uaccess.h> | |
39 | #include <asm/cache.h> | |
40 | #include <asm/cpudata.h> | |
63b61452 | 41 | #include <asm/auxio.h> |
92704a1c | 42 | #include <asm/head.h> |
4a907dec | 43 | #include <asm/hypervisor.h> |
42d5f99b | 44 | #include <asm/cacheflush.h> |
1da177e4 | 45 | |
d91aa123 | 46 | #include "entry.h" |
280ff974 | 47 | #include "cpumap.h" |
ec687886 | 48 | #include "kstack.h" |
e18e2a00 | 49 | |
10397e40 | 50 | struct ino_bucket *ivector_table; |
eb2d8d60 | 51 | unsigned long ivector_table_pa; |
1da177e4 | 52 | |
42d5f99b DM |
53 | /* On several sun4u processors, it is illegal to mix bypass and |
54 | * non-bypass accesses. Therefore we access all INO buckets | |
55 | * using bypass accesses only. | |
56 | */ | |
57 | static unsigned long bucket_get_chain_pa(unsigned long bucket_pa) | |
58 | { | |
59 | unsigned long ret; | |
60 | ||
61 | __asm__ __volatile__("ldxa [%1] %2, %0" | |
62 | : "=&r" (ret) | |
63 | : "r" (bucket_pa + | |
64 | offsetof(struct ino_bucket, | |
65 | __irq_chain_pa)), | |
66 | "i" (ASI_PHYS_USE_EC)); | |
67 | ||
68 | return ret; | |
69 | } | |
70 | ||
71 | static void bucket_clear_chain_pa(unsigned long bucket_pa) | |
72 | { | |
73 | __asm__ __volatile__("stxa %%g0, [%0] %1" | |
74 | : /* no outputs */ | |
75 | : "r" (bucket_pa + | |
76 | offsetof(struct ino_bucket, | |
77 | __irq_chain_pa)), | |
78 | "i" (ASI_PHYS_USE_EC)); | |
79 | } | |
80 | ||
fe41493f | 81 | static unsigned int bucket_get_irq(unsigned long bucket_pa) |
42d5f99b DM |
82 | { |
83 | unsigned int ret; | |
84 | ||
85 | __asm__ __volatile__("lduwa [%1] %2, %0" | |
86 | : "=&r" (ret) | |
87 | : "r" (bucket_pa + | |
88 | offsetof(struct ino_bucket, | |
fe41493f | 89 | __irq)), |
42d5f99b DM |
90 | "i" (ASI_PHYS_USE_EC)); |
91 | ||
92 | return ret; | |
93 | } | |
94 | ||
fe41493f | 95 | static void bucket_set_irq(unsigned long bucket_pa, unsigned int irq) |
42d5f99b DM |
96 | { |
97 | __asm__ __volatile__("stwa %0, [%1] %2" | |
98 | : /* no outputs */ | |
fe41493f | 99 | : "r" (irq), |
42d5f99b DM |
100 | "r" (bucket_pa + |
101 | offsetof(struct ino_bucket, | |
fe41493f | 102 | __irq)), |
42d5f99b DM |
103 | "i" (ASI_PHYS_USE_EC)); |
104 | } | |
105 | ||
eb2d8d60 | 106 | #define irq_work_pa(__cpu) &(trap_block[(__cpu)].irq_worklist_pa) |
1da177e4 | 107 | |
ee6a9333 | 108 | static unsigned long hvirq_major __initdata; |
109 | static int __init early_hvirq_major(char *p) | |
110 | { | |
111 | int rc = kstrtoul(p, 10, &hvirq_major); | |
112 | ||
113 | return rc; | |
114 | } | |
115 | early_param("hvirq", early_hvirq_major); | |
116 | ||
117 | static int hv_irq_version; | |
118 | ||
119 | /* Major version 2.0 of HV_GRP_INTR added support for the VIRQ cookie | |
120 | * based interfaces, but: | |
121 | * | |
122 | * 1) Several OSs, Solaris and Linux included, use them even when only | |
123 | * negotiating version 1.0 (or failing to negotiate at all). So the | |
124 | * hypervisor has a workaround that provides the VIRQ interfaces even | |
125 | * when only verion 1.0 of the API is in use. | |
126 | * | |
127 | * 2) Second, and more importantly, with major version 2.0 these VIRQ | |
128 | * interfaces only were actually hooked up for LDC interrupts, even | |
129 | * though the Hypervisor specification clearly stated: | |
130 | * | |
131 | * The new interrupt API functions will be available to a guest | |
132 | * when it negotiates version 2.0 in the interrupt API group 0x2. When | |
133 | * a guest negotiates version 2.0, all interrupt sources will only | |
134 | * support using the cookie interface, and any attempt to use the | |
135 | * version 1.0 interrupt APIs numbered 0xa0 to 0xa6 will result in the | |
136 | * ENOTSUPPORTED error being returned. | |
137 | * | |
138 | * with an emphasis on "all interrupt sources". | |
139 | * | |
140 | * To correct this, major version 3.0 was created which does actually | |
141 | * support VIRQs for all interrupt sources (not just LDC devices). So | |
142 | * if we want to move completely over the cookie based VIRQs we must | |
143 | * negotiate major version 3.0 or later of HV_GRP_INTR. | |
144 | */ | |
145 | static bool sun4v_cookie_only_virqs(void) | |
146 | { | |
147 | if (hv_irq_version >= 3) | |
148 | return true; | |
149 | return false; | |
150 | } | |
8047e247 | 151 | |
ee6a9333 | 152 | static void __init irq_init_hv(void) |
8047e247 | 153 | { |
ee6a9333 | 154 | unsigned long hv_error, major, minor = 0; |
155 | ||
156 | if (tlb_type != hypervisor) | |
157 | return; | |
8047e247 | 158 | |
ee6a9333 | 159 | if (hvirq_major) |
160 | major = hvirq_major; | |
161 | else | |
162 | major = 3; | |
8047e247 | 163 | |
ee6a9333 | 164 | hv_error = sun4v_hvapi_register(HV_GRP_INTR, major, &minor); |
165 | if (!hv_error) | |
166 | hv_irq_version = major; | |
167 | else | |
168 | hv_irq_version = 1; | |
759f89e0 | 169 | |
ee6a9333 | 170 | pr_info("SUN4V: Using IRQ API major %d, cookie only virqs %s\n", |
171 | hv_irq_version, | |
172 | sun4v_cookie_only_virqs() ? "enabled" : "disabled"); | |
173 | } | |
174 | ||
175 | /* This function is for the timer interrupt.*/ | |
176 | int __init arch_probe_nr_irqs(void) | |
177 | { | |
178 | return 1; | |
179 | } | |
180 | ||
181 | #define DEFAULT_NUM_IVECS (0xfffU) | |
182 | static unsigned int nr_ivec = DEFAULT_NUM_IVECS; | |
183 | #define NUM_IVECS (nr_ivec) | |
184 | ||
185 | static unsigned int __init size_nr_ivec(void) | |
186 | { | |
187 | if (tlb_type == hypervisor) { | |
188 | switch (sun4v_chip_type) { | |
189 | /* Athena's devhandle|devino is large.*/ | |
190 | case SUN4V_CHIP_SPARC64X: | |
191 | nr_ivec = 0xffff; | |
35a17eb6 | 192 | break; |
ee6a9333 | 193 | } |
35a17eb6 | 194 | } |
ee6a9333 | 195 | return nr_ivec; |
196 | } | |
197 | ||
198 | struct irq_handler_data { | |
199 | union { | |
200 | struct { | |
201 | unsigned int dev_handle; | |
202 | unsigned int dev_ino; | |
203 | }; | |
204 | unsigned long sysino; | |
205 | }; | |
206 | struct ino_bucket bucket; | |
207 | unsigned long iclr; | |
208 | unsigned long imap; | |
209 | }; | |
210 | ||
211 | static inline unsigned int irq_data_to_handle(struct irq_data *data) | |
212 | { | |
6a4a5b34 | 213 | struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data); |
ee6a9333 | 214 | |
215 | return ihd->dev_handle; | |
216 | } | |
217 | ||
218 | static inline unsigned int irq_data_to_ino(struct irq_data *data) | |
219 | { | |
6a4a5b34 | 220 | struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data); |
8047e247 | 221 | |
ee6a9333 | 222 | return ihd->dev_ino; |
223 | } | |
224 | ||
225 | static inline unsigned long irq_data_to_sysino(struct irq_data *data) | |
226 | { | |
6a4a5b34 | 227 | struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data); |
8047e247 | 228 | |
ee6a9333 | 229 | return ihd->sysino; |
8047e247 DM |
230 | } |
231 | ||
fe41493f | 232 | void irq_free(unsigned int irq) |
8047e247 | 233 | { |
ee6a9333 | 234 | void *data = irq_get_handler_data(irq); |
8047e247 | 235 | |
ee6a9333 | 236 | kfree(data); |
237 | irq_set_handler_data(irq, NULL); | |
238 | irq_free_descs(irq, 1); | |
239 | } | |
35a17eb6 | 240 | |
ee6a9333 | 241 | unsigned int irq_alloc(unsigned int dev_handle, unsigned int dev_ino) |
242 | { | |
243 | int irq; | |
759f89e0 | 244 | |
06ee6d57 | 245 | irq = __irq_alloc_descs(-1, 1, 1, numa_node_id(), NULL, NULL); |
ee6a9333 | 246 | if (irq <= 0) |
247 | goto out; | |
35a17eb6 | 248 | |
ee6a9333 | 249 | return irq; |
250 | out: | |
251 | return 0; | |
252 | } | |
253 | ||
254 | static unsigned int cookie_exists(u32 devhandle, unsigned int devino) | |
255 | { | |
256 | unsigned long hv_err, cookie; | |
257 | struct ino_bucket *bucket; | |
258 | unsigned int irq = 0U; | |
259 | ||
260 | hv_err = sun4v_vintr_get_cookie(devhandle, devino, &cookie); | |
261 | if (hv_err) { | |
262 | pr_err("HV get cookie failed hv_err = %ld\n", hv_err); | |
263 | goto out; | |
264 | } | |
265 | ||
266 | if (cookie & ((1UL << 63UL))) { | |
267 | cookie = ~cookie; | |
268 | bucket = (struct ino_bucket *) __va(cookie); | |
269 | irq = bucket->__irq; | |
270 | } | |
271 | out: | |
272 | return irq; | |
273 | } | |
274 | ||
275 | static unsigned int sysino_exists(u32 devhandle, unsigned int devino) | |
276 | { | |
277 | unsigned long sysino = sun4v_devino_to_sysino(devhandle, devino); | |
278 | struct ino_bucket *bucket; | |
279 | unsigned int irq; | |
280 | ||
281 | bucket = &ivector_table[sysino]; | |
282 | irq = bucket_get_irq(__pa(bucket)); | |
283 | ||
284 | return irq; | |
285 | } | |
286 | ||
287 | void ack_bad_irq(unsigned int irq) | |
288 | { | |
289 | pr_crit("BAD IRQ ack %d\n", irq); | |
290 | } | |
291 | ||
292 | void irq_install_pre_handler(int irq, | |
293 | void (*func)(unsigned int, void *, void *), | |
294 | void *arg1, void *arg2) | |
295 | { | |
296 | pr_warn("IRQ pre handler NOT supported.\n"); | |
8047e247 | 297 | } |
8047e247 | 298 | |
1da177e4 | 299 | /* |
e18e2a00 | 300 | * /proc/interrupts printing: |
1da177e4 | 301 | */ |
fa680c7c | 302 | int arch_show_interrupts(struct seq_file *p, int prec) |
1da177e4 | 303 | { |
fa680c7c | 304 | int j; |
e18e2a00 | 305 | |
fa680c7c TG |
306 | seq_printf(p, "NMI: "); |
307 | for_each_online_cpu(j) | |
308 | seq_printf(p, "%10u ", cpu_data(j).__nmi_count); | |
309 | seq_printf(p, " Non-maskable interrupts\n"); | |
1da177e4 LT |
310 | return 0; |
311 | } | |
312 | ||
ebd8c56c DM |
313 | static unsigned int sun4u_compute_tid(unsigned long imap, unsigned long cpuid) |
314 | { | |
315 | unsigned int tid; | |
316 | ||
317 | if (this_is_starfire) { | |
318 | tid = starfire_translate(imap, cpuid); | |
319 | tid <<= IMAP_TID_SHIFT; | |
320 | tid &= IMAP_TID_UPA; | |
321 | } else { | |
322 | if (tlb_type == cheetah || tlb_type == cheetah_plus) { | |
323 | unsigned long ver; | |
324 | ||
325 | __asm__ ("rdpr %%ver, %0" : "=r" (ver)); | |
326 | if ((ver >> 32UL) == __JALAPENO_ID || | |
327 | (ver >> 32UL) == __SERRANO_ID) { | |
328 | tid = cpuid << IMAP_TID_SHIFT; | |
329 | tid &= IMAP_TID_JBUS; | |
330 | } else { | |
331 | unsigned int a = cpuid & 0x1f; | |
332 | unsigned int n = (cpuid >> 5) & 0x1f; | |
333 | ||
334 | tid = ((a << IMAP_AID_SHIFT) | | |
335 | (n << IMAP_NID_SHIFT)); | |
336 | tid &= (IMAP_AID_SAFARI | | |
a419aef8 | 337 | IMAP_NID_SAFARI); |
ebd8c56c DM |
338 | } |
339 | } else { | |
340 | tid = cpuid << IMAP_TID_SHIFT; | |
341 | tid &= IMAP_TID_UPA; | |
342 | } | |
343 | } | |
344 | ||
345 | return tid; | |
346 | } | |
347 | ||
e18e2a00 | 348 | #ifdef CONFIG_SMP |
fe41493f | 349 | static int irq_choose_cpu(unsigned int irq, const struct cpumask *affinity) |
088dd1f8 | 350 | { |
e65e49d0 | 351 | cpumask_t mask; |
e18e2a00 | 352 | int cpuid; |
088dd1f8 | 353 | |
1091ce62 | 354 | cpumask_copy(&mask, affinity); |
fb1fece5 | 355 | if (cpumask_equal(&mask, cpu_online_mask)) { |
fe41493f | 356 | cpuid = map_to_cpu(irq); |
e18e2a00 DM |
357 | } else { |
358 | cpumask_t tmp; | |
088dd1f8 | 359 | |
fb1fece5 KM |
360 | cpumask_and(&tmp, cpu_online_mask, &mask); |
361 | cpuid = cpumask_empty(&tmp) ? map_to_cpu(irq) : cpumask_first(&tmp); | |
1da177e4 | 362 | } |
088dd1f8 | 363 | |
e18e2a00 DM |
364 | return cpuid; |
365 | } | |
366 | #else | |
fe41493f | 367 | #define irq_choose_cpu(irq, affinity) \ |
6abce771 | 368 | real_hard_smp_processor_id() |
e18e2a00 | 369 | #endif |
1da177e4 | 370 | |
4832b992 | 371 | static void sun4u_irq_enable(struct irq_data *data) |
e3999574 | 372 | { |
6a4a5b34 | 373 | struct irq_handler_data *handler_data; |
e3999574 | 374 | |
6a4a5b34 | 375 | handler_data = irq_data_get_irq_handler_data(data); |
cae78728 | 376 | if (likely(handler_data)) { |
861fe906 | 377 | unsigned long cpuid, imap, val; |
e18e2a00 | 378 | unsigned int tid; |
e3999574 | 379 | |
d7185a98 JL |
380 | cpuid = irq_choose_cpu(data->irq, |
381 | irq_data_get_affinity_mask(data)); | |
cae78728 | 382 | imap = handler_data->imap; |
e3999574 | 383 | |
e18e2a00 | 384 | tid = sun4u_compute_tid(imap, cpuid); |
e3999574 | 385 | |
861fe906 DM |
386 | val = upa_readq(imap); |
387 | val &= ~(IMAP_TID_UPA | IMAP_TID_JBUS | | |
388 | IMAP_AID_SAFARI | IMAP_NID_SAFARI); | |
389 | val |= tid | IMAP_VALID; | |
390 | upa_writeq(val, imap); | |
cae78728 | 391 | upa_writeq(ICLR_IDLE, handler_data->iclr); |
e3999574 | 392 | } |
e3999574 DM |
393 | } |
394 | ||
4832b992 SR |
395 | static int sun4u_set_affinity(struct irq_data *data, |
396 | const struct cpumask *mask, bool force) | |
b53bcb67 | 397 | { |
6a4a5b34 | 398 | struct irq_handler_data *handler_data; |
1091ce62 | 399 | |
6a4a5b34 | 400 | handler_data = irq_data_get_irq_handler_data(data); |
cae78728 | 401 | if (likely(handler_data)) { |
1091ce62 DM |
402 | unsigned long cpuid, imap, val; |
403 | unsigned int tid; | |
404 | ||
4832b992 | 405 | cpuid = irq_choose_cpu(data->irq, mask); |
cae78728 | 406 | imap = handler_data->imap; |
1091ce62 DM |
407 | |
408 | tid = sun4u_compute_tid(imap, cpuid); | |
409 | ||
410 | val = upa_readq(imap); | |
411 | val &= ~(IMAP_TID_UPA | IMAP_TID_JBUS | | |
412 | IMAP_AID_SAFARI | IMAP_NID_SAFARI); | |
413 | val |= tid | IMAP_VALID; | |
414 | upa_writeq(val, imap); | |
cae78728 | 415 | upa_writeq(ICLR_IDLE, handler_data->iclr); |
1091ce62 | 416 | } |
d5dedd45 YL |
417 | |
418 | return 0; | |
b53bcb67 DM |
419 | } |
420 | ||
d0cac39e DM |
421 | /* Don't do anything. The desc->status check for IRQ_DISABLED in |
422 | * handler_irq() will skip the handler call and that will leave the | |
423 | * interrupt in the sent state. The next ->enable() call will hit the | |
424 | * ICLR register to reset the state machine. | |
425 | * | |
426 | * This scheme is necessary, instead of clearing the Valid bit in the | |
427 | * IMAP register, to handle the case of IMAP registers being shared by | |
428 | * multiple INOs (and thus ICLR registers). Since we use a different | |
429 | * virtual IRQ for each shared IMAP instance, the generic code thinks | |
430 | * there is only one user so it prematurely calls ->disable() on | |
431 | * free_irq(). | |
432 | * | |
433 | * We have to provide an explicit ->disable() method instead of using | |
434 | * NULL to get the default. The reason is that if the generic code | |
435 | * sees that, it also hooks up a default ->shutdown method which | |
436 | * invokes ->mask() which we do not want. See irq_chip_set_defaults(). | |
437 | */ | |
4832b992 | 438 | static void sun4u_irq_disable(struct irq_data *data) |
1da177e4 | 439 | { |
088dd1f8 DM |
440 | } |
441 | ||
4832b992 | 442 | static void sun4u_irq_eoi(struct irq_data *data) |
088dd1f8 | 443 | { |
6a4a5b34 | 444 | struct irq_handler_data *handler_data; |
088dd1f8 | 445 | |
6a4a5b34 | 446 | handler_data = irq_data_get_irq_handler_data(data); |
cae78728 SR |
447 | if (likely(handler_data)) |
448 | upa_writeq(ICLR_IDLE, handler_data->iclr); | |
088dd1f8 DM |
449 | } |
450 | ||
4832b992 | 451 | static void sun4v_irq_enable(struct irq_data *data) |
088dd1f8 | 452 | { |
d7185a98 JL |
453 | unsigned long cpuid = irq_choose_cpu(data->irq, |
454 | irq_data_get_affinity_mask(data)); | |
ee6a9333 | 455 | unsigned int ino = irq_data_to_sysino(data); |
77182300 DM |
456 | int err; |
457 | ||
458 | err = sun4v_intr_settarget(ino, cpuid); | |
459 | if (err != HV_EOK) | |
460 | printk(KERN_ERR "sun4v_intr_settarget(%x,%lu): " | |
461 | "err(%d)\n", ino, cpuid, err); | |
462 | err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE); | |
463 | if (err != HV_EOK) | |
464 | printk(KERN_ERR "sun4v_intr_setstate(%x): " | |
465 | "err(%d)\n", ino, err); | |
466 | err = sun4v_intr_setenabled(ino, HV_INTR_ENABLED); | |
467 | if (err != HV_EOK) | |
468 | printk(KERN_ERR "sun4v_intr_setenabled(%x): err(%d)\n", | |
469 | ino, err); | |
088dd1f8 DM |
470 | } |
471 | ||
4832b992 SR |
472 | static int sun4v_set_affinity(struct irq_data *data, |
473 | const struct cpumask *mask, bool force) | |
b53bcb67 | 474 | { |
4832b992 | 475 | unsigned long cpuid = irq_choose_cpu(data->irq, mask); |
ee6a9333 | 476 | unsigned int ino = irq_data_to_sysino(data); |
77182300 DM |
477 | int err; |
478 | ||
479 | err = sun4v_intr_settarget(ino, cpuid); | |
480 | if (err != HV_EOK) | |
481 | printk(KERN_ERR "sun4v_intr_settarget(%x,%lu): " | |
482 | "err(%d)\n", ino, cpuid, err); | |
d5dedd45 YL |
483 | |
484 | return 0; | |
b53bcb67 DM |
485 | } |
486 | ||
4832b992 | 487 | static void sun4v_irq_disable(struct irq_data *data) |
1da177e4 | 488 | { |
ee6a9333 | 489 | unsigned int ino = irq_data_to_sysino(data); |
77182300 | 490 | int err; |
1da177e4 | 491 | |
77182300 DM |
492 | err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED); |
493 | if (err != HV_EOK) | |
494 | printk(KERN_ERR "sun4v_intr_setenabled(%x): " | |
495 | "err(%d)\n", ino, err); | |
e18e2a00 | 496 | } |
1da177e4 | 497 | |
4832b992 | 498 | static void sun4v_irq_eoi(struct irq_data *data) |
e18e2a00 | 499 | { |
ee6a9333 | 500 | unsigned int ino = irq_data_to_sysino(data); |
77182300 | 501 | int err; |
5a606b72 | 502 | |
77182300 DM |
503 | err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE); |
504 | if (err != HV_EOK) | |
505 | printk(KERN_ERR "sun4v_intr_setstate(%x): " | |
506 | "err(%d)\n", ino, err); | |
1da177e4 LT |
507 | } |
508 | ||
4832b992 | 509 | static void sun4v_virq_enable(struct irq_data *data) |
4a907dec | 510 | { |
ee6a9333 | 511 | unsigned long dev_handle = irq_data_to_handle(data); |
512 | unsigned long dev_ino = irq_data_to_ino(data); | |
513 | unsigned long cpuid; | |
77182300 DM |
514 | int err; |
515 | ||
d7185a98 | 516 | cpuid = irq_choose_cpu(data->irq, irq_data_get_affinity_mask(data)); |
77182300 | 517 | |
77182300 DM |
518 | err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid); |
519 | if (err != HV_EOK) | |
520 | printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): " | |
521 | "err(%d)\n", | |
522 | dev_handle, dev_ino, cpuid, err); | |
523 | err = sun4v_vintr_set_state(dev_handle, dev_ino, | |
524 | HV_INTR_STATE_IDLE); | |
525 | if (err != HV_EOK) | |
526 | printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx," | |
527 | "HV_INTR_STATE_IDLE): err(%d)\n", | |
528 | dev_handle, dev_ino, err); | |
529 | err = sun4v_vintr_set_valid(dev_handle, dev_ino, | |
530 | HV_INTR_ENABLED); | |
531 | if (err != HV_EOK) | |
532 | printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx," | |
533 | "HV_INTR_ENABLED): err(%d)\n", | |
534 | dev_handle, dev_ino, err); | |
4a907dec DM |
535 | } |
536 | ||
4832b992 SR |
537 | static int sun4v_virt_set_affinity(struct irq_data *data, |
538 | const struct cpumask *mask, bool force) | |
b53bcb67 | 539 | { |
ee6a9333 | 540 | unsigned long dev_handle = irq_data_to_handle(data); |
541 | unsigned long dev_ino = irq_data_to_ino(data); | |
542 | unsigned long cpuid; | |
77182300 | 543 | int err; |
b53bcb67 | 544 | |
4832b992 | 545 | cpuid = irq_choose_cpu(data->irq, mask); |
b53bcb67 | 546 | |
77182300 DM |
547 | err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid); |
548 | if (err != HV_EOK) | |
549 | printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): " | |
550 | "err(%d)\n", | |
551 | dev_handle, dev_ino, cpuid, err); | |
d5dedd45 YL |
552 | |
553 | return 0; | |
b53bcb67 DM |
554 | } |
555 | ||
4832b992 | 556 | static void sun4v_virq_disable(struct irq_data *data) |
4a907dec | 557 | { |
ee6a9333 | 558 | unsigned long dev_handle = irq_data_to_handle(data); |
559 | unsigned long dev_ino = irq_data_to_ino(data); | |
77182300 DM |
560 | int err; |
561 | ||
77182300 DM |
562 | |
563 | err = sun4v_vintr_set_valid(dev_handle, dev_ino, | |
564 | HV_INTR_DISABLED); | |
565 | if (err != HV_EOK) | |
566 | printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx," | |
567 | "HV_INTR_DISABLED): err(%d)\n", | |
568 | dev_handle, dev_ino, err); | |
4a907dec DM |
569 | } |
570 | ||
4832b992 | 571 | static void sun4v_virq_eoi(struct irq_data *data) |
4a907dec | 572 | { |
ee6a9333 | 573 | unsigned long dev_handle = irq_data_to_handle(data); |
574 | unsigned long dev_ino = irq_data_to_ino(data); | |
77182300 | 575 | int err; |
5a606b72 | 576 | |
77182300 DM |
577 | err = sun4v_vintr_set_state(dev_handle, dev_ino, |
578 | HV_INTR_STATE_IDLE); | |
579 | if (err != HV_EOK) | |
580 | printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx," | |
581 | "HV_INTR_STATE_IDLE): err(%d)\n", | |
582 | dev_handle, dev_ino, err); | |
4a907dec DM |
583 | } |
584 | ||
729e7d7e | 585 | static struct irq_chip sun4u_irq = { |
4832b992 SR |
586 | .name = "sun4u", |
587 | .irq_enable = sun4u_irq_enable, | |
588 | .irq_disable = sun4u_irq_disable, | |
589 | .irq_eoi = sun4u_irq_eoi, | |
590 | .irq_set_affinity = sun4u_set_affinity, | |
fcd8d4f4 | 591 | .flags = IRQCHIP_EOI_IF_HANDLED, |
e18e2a00 | 592 | }; |
088dd1f8 | 593 | |
729e7d7e | 594 | static struct irq_chip sun4v_irq = { |
4832b992 SR |
595 | .name = "sun4v", |
596 | .irq_enable = sun4v_irq_enable, | |
597 | .irq_disable = sun4v_irq_disable, | |
598 | .irq_eoi = sun4v_irq_eoi, | |
599 | .irq_set_affinity = sun4v_set_affinity, | |
fcd8d4f4 | 600 | .flags = IRQCHIP_EOI_IF_HANDLED, |
e18e2a00 | 601 | }; |
1da177e4 | 602 | |
4a907dec | 603 | static struct irq_chip sun4v_virq = { |
4832b992 SR |
604 | .name = "vsun4v", |
605 | .irq_enable = sun4v_virq_enable, | |
606 | .irq_disable = sun4v_virq_disable, | |
607 | .irq_eoi = sun4v_virq_eoi, | |
608 | .irq_set_affinity = sun4v_virt_set_affinity, | |
fcd8d4f4 | 609 | .flags = IRQCHIP_EOI_IF_HANDLED, |
4a907dec DM |
610 | }; |
611 | ||
e18e2a00 DM |
612 | unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap) |
613 | { | |
cae78728 | 614 | struct irq_handler_data *handler_data; |
ee6a9333 | 615 | struct ino_bucket *bucket; |
fe41493f | 616 | unsigned int irq; |
e18e2a00 | 617 | int ino; |
1da177e4 | 618 | |
e18e2a00 | 619 | BUG_ON(tlb_type == hypervisor); |
088dd1f8 | 620 | |
861fe906 | 621 | ino = (upa_readq(imap) & (IMAP_IGN | IMAP_INO)) + inofixup; |
e18e2a00 | 622 | bucket = &ivector_table[ino]; |
fe41493f SR |
623 | irq = bucket_get_irq(__pa(bucket)); |
624 | if (!irq) { | |
625 | irq = irq_alloc(0, ino); | |
626 | bucket_set_irq(__pa(bucket), irq); | |
394d441b TG |
627 | irq_set_chip_and_handler_name(irq, &sun4u_irq, |
628 | handle_fasteoi_irq, "IVEC"); | |
fd0504c3 | 629 | } |
1da177e4 | 630 | |
394d441b | 631 | handler_data = irq_get_handler_data(irq); |
cae78728 | 632 | if (unlikely(handler_data)) |
e18e2a00 | 633 | goto out; |
fd0504c3 | 634 | |
cae78728 SR |
635 | handler_data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); |
636 | if (unlikely(!handler_data)) { | |
e18e2a00 DM |
637 | prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n"); |
638 | prom_halt(); | |
1da177e4 | 639 | } |
394d441b | 640 | irq_set_handler_data(irq, handler_data); |
1da177e4 | 641 | |
cae78728 SR |
642 | handler_data->imap = imap; |
643 | handler_data->iclr = iclr; | |
1da177e4 | 644 | |
e18e2a00 | 645 | out: |
fe41493f | 646 | return irq; |
e18e2a00 | 647 | } |
1da177e4 | 648 | |
ee6a9333 | 649 | static unsigned int sun4v_build_common(u32 devhandle, unsigned int devino, |
650 | void (*handler_data_init)(struct irq_handler_data *data, | |
651 | u32 devhandle, unsigned int devino), | |
652 | struct irq_chip *chip) | |
1da177e4 | 653 | { |
ee6a9333 | 654 | struct irq_handler_data *data; |
fe41493f | 655 | unsigned int irq; |
8047e247 | 656 | |
ee6a9333 | 657 | irq = irq_alloc(devhandle, devino); |
658 | if (!irq) | |
659 | goto out; | |
1da177e4 | 660 | |
ee6a9333 | 661 | data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); |
662 | if (unlikely(!data)) { | |
663 | pr_err("IRQ handler data allocation failed.\n"); | |
664 | irq_free(irq); | |
665 | irq = 0; | |
666 | goto out; | |
1da177e4 | 667 | } |
1da177e4 | 668 | |
ee6a9333 | 669 | irq_set_handler_data(irq, data); |
670 | handler_data_init(data, devhandle, devino); | |
671 | irq_set_chip_and_handler_name(irq, chip, handle_fasteoi_irq, "IVEC"); | |
672 | data->imap = ~0UL; | |
673 | data->iclr = ~0UL; | |
674 | out: | |
675 | return irq; | |
676 | } | |
1da177e4 | 677 | |
ee6a9333 | 678 | static unsigned long cookie_assign(unsigned int irq, u32 devhandle, |
679 | unsigned int devino) | |
680 | { | |
681 | struct irq_handler_data *ihd = irq_get_handler_data(irq); | |
682 | unsigned long hv_error, cookie; | |
1da177e4 | 683 | |
ee6a9333 | 684 | /* handler_irq needs to find the irq. cookie is seen signed in |
685 | * sun4v_dev_mondo and treated as a non ivector_table delivery. | |
e18e2a00 | 686 | */ |
ee6a9333 | 687 | ihd->bucket.__irq = irq; |
688 | cookie = ~__pa(&ihd->bucket); | |
1da177e4 | 689 | |
ee6a9333 | 690 | hv_error = sun4v_vintr_set_cookie(devhandle, devino, cookie); |
691 | if (hv_error) | |
692 | pr_err("HV vintr set cookie failed = %ld\n", hv_error); | |
693 | ||
694 | return hv_error; | |
e18e2a00 | 695 | } |
1da177e4 | 696 | |
ee6a9333 | 697 | static void cookie_handler_data(struct irq_handler_data *data, |
698 | u32 devhandle, unsigned int devino) | |
4a907dec | 699 | { |
ee6a9333 | 700 | data->dev_handle = devhandle; |
701 | data->dev_ino = devino; | |
702 | } | |
4a907dec | 703 | |
ee6a9333 | 704 | static unsigned int cookie_build_irq(u32 devhandle, unsigned int devino, |
705 | struct irq_chip *chip) | |
706 | { | |
707 | unsigned long hv_error; | |
708 | unsigned int irq; | |
709 | ||
710 | irq = sun4v_build_common(devhandle, devino, cookie_handler_data, chip); | |
711 | ||
712 | hv_error = cookie_assign(irq, devhandle, devino); | |
713 | if (hv_error) { | |
714 | irq_free(irq); | |
715 | irq = 0; | |
716 | } | |
717 | ||
718 | return irq; | |
4a907dec DM |
719 | } |
720 | ||
ee6a9333 | 721 | static unsigned int sun4v_build_cookie(u32 devhandle, unsigned int devino) |
4a907dec | 722 | { |
fe41493f | 723 | unsigned int irq; |
b80e6998 | 724 | |
ee6a9333 | 725 | irq = cookie_exists(devhandle, devino); |
726 | if (irq) | |
727 | goto out; | |
25ad403f | 728 | |
ee6a9333 | 729 | irq = cookie_build_irq(devhandle, devino, &sun4v_virq); |
25ad403f | 730 | |
ee6a9333 | 731 | out: |
732 | return irq; | |
733 | } | |
b80e6998 | 734 | |
ee6a9333 | 735 | static void sysino_set_bucket(unsigned int irq) |
736 | { | |
737 | struct irq_handler_data *ihd = irq_get_handler_data(irq); | |
738 | struct ino_bucket *bucket; | |
739 | unsigned long sysino; | |
740 | ||
741 | sysino = sun4v_devino_to_sysino(ihd->dev_handle, ihd->dev_ino); | |
742 | BUG_ON(sysino >= nr_ivec); | |
743 | bucket = &ivector_table[sysino]; | |
fe41493f | 744 | bucket_set_irq(__pa(bucket), irq); |
ee6a9333 | 745 | } |
8d57d3ad | 746 | |
ee6a9333 | 747 | static void sysino_handler_data(struct irq_handler_data *data, |
748 | u32 devhandle, unsigned int devino) | |
749 | { | |
750 | unsigned long sysino; | |
4a907dec | 751 | |
ee6a9333 | 752 | sysino = sun4v_devino_to_sysino(devhandle, devino); |
753 | data->sysino = sysino; | |
754 | } | |
4a907dec | 755 | |
ee6a9333 | 756 | static unsigned int sysino_build_irq(u32 devhandle, unsigned int devino, |
757 | struct irq_chip *chip) | |
758 | { | |
759 | unsigned int irq; | |
4a907dec | 760 | |
ee6a9333 | 761 | irq = sun4v_build_common(devhandle, devino, sysino_handler_data, chip); |
762 | if (!irq) | |
763 | goto out; | |
b80e6998 | 764 | |
ee6a9333 | 765 | sysino_set_bucket(irq); |
766 | out: | |
767 | return irq; | |
768 | } | |
4a907dec | 769 | |
ee6a9333 | 770 | static int sun4v_build_sysino(u32 devhandle, unsigned int devino) |
771 | { | |
772 | int irq; | |
773 | ||
774 | irq = sysino_exists(devhandle, devino); | |
775 | if (irq) | |
776 | goto out; | |
777 | ||
778 | irq = sysino_build_irq(devhandle, devino, &sun4v_irq); | |
779 | out: | |
fe41493f | 780 | return irq; |
4a907dec DM |
781 | } |
782 | ||
ee6a9333 | 783 | unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino) |
e18e2a00 | 784 | { |
ee6a9333 | 785 | unsigned int irq; |
ab66a50e | 786 | |
ee6a9333 | 787 | if (sun4v_cookie_only_virqs()) |
788 | irq = sun4v_build_cookie(devhandle, devino); | |
789 | else | |
790 | irq = sun4v_build_sysino(devhandle, devino); | |
6a76267f | 791 | |
ee6a9333 | 792 | return irq; |
793 | } | |
794 | ||
795 | unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino) | |
796 | { | |
797 | int irq; | |
798 | ||
799 | irq = cookie_build_irq(devhandle, devino, &sun4v_virq); | |
800 | if (!irq) | |
801 | goto out; | |
802 | ||
803 | /* This is borrowed from the original function. | |
804 | */ | |
805 | irq_set_status_flags(irq, IRQ_NOAUTOEN); | |
806 | ||
807 | out: | |
808 | return irq; | |
1da177e4 LT |
809 | } |
810 | ||
4f70f7a9 DM |
811 | void *hardirq_stack[NR_CPUS]; |
812 | void *softirq_stack[NR_CPUS]; | |
813 | ||
d4d1ec48 | 814 | void __irq_entry handler_irq(int pil, struct pt_regs *regs) |
1da177e4 | 815 | { |
eb2d8d60 | 816 | unsigned long pstate, bucket_pa; |
6d24c8dc | 817 | struct pt_regs *old_regs; |
4f70f7a9 | 818 | void *orig_sp; |
1da177e4 | 819 | |
d4d1ec48 | 820 | clear_softint(1 << pil); |
1da177e4 | 821 | |
6d24c8dc | 822 | old_regs = set_irq_regs(regs); |
1da177e4 | 823 | irq_enter(); |
1da177e4 | 824 | |
a650d383 DM |
825 | /* Grab an atomic snapshot of the pending IVECs. */ |
826 | __asm__ __volatile__("rdpr %%pstate, %0\n\t" | |
827 | "wrpr %0, %3, %%pstate\n\t" | |
828 | "ldx [%2], %1\n\t" | |
829 | "stx %%g0, [%2]\n\t" | |
830 | "wrpr %0, 0x0, %%pstate\n\t" | |
eb2d8d60 DM |
831 | : "=&r" (pstate), "=&r" (bucket_pa) |
832 | : "r" (irq_work_pa(smp_processor_id())), | |
a650d383 DM |
833 | "i" (PSTATE_IE) |
834 | : "memory"); | |
835 | ||
4f70f7a9 DM |
836 | orig_sp = set_hardirq_stack(); |
837 | ||
eb2d8d60 DM |
838 | while (bucket_pa) { |
839 | unsigned long next_pa; | |
fe41493f | 840 | unsigned int irq; |
1da177e4 | 841 | |
42d5f99b | 842 | next_pa = bucket_get_chain_pa(bucket_pa); |
fe41493f | 843 | irq = bucket_get_irq(bucket_pa); |
42d5f99b | 844 | bucket_clear_chain_pa(bucket_pa); |
fd0504c3 | 845 | |
fcd8d4f4 | 846 | generic_handle_irq(irq); |
eb2d8d60 DM |
847 | |
848 | bucket_pa = next_pa; | |
1da177e4 | 849 | } |
e18e2a00 | 850 | |
4f70f7a9 DM |
851 | restore_hardirq_stack(orig_sp); |
852 | ||
1da177e4 | 853 | irq_exit(); |
6d24c8dc | 854 | set_irq_regs(old_regs); |
1da177e4 LT |
855 | } |
856 | ||
7d65f4a6 | 857 | void do_softirq_own_stack(void) |
4f70f7a9 | 858 | { |
7d65f4a6 | 859 | void *orig_sp, *sp = softirq_stack[smp_processor_id()]; |
4f70f7a9 | 860 | |
7d65f4a6 | 861 | sp += THREAD_SIZE - 192 - STACK_BIAS; |
4f70f7a9 | 862 | |
7d65f4a6 FW |
863 | __asm__ __volatile__("mov %%sp, %0\n\t" |
864 | "mov %1, %%sp" | |
865 | : "=&r" (orig_sp) | |
866 | : "r" (sp)); | |
867 | __do_softirq(); | |
868 | __asm__ __volatile__("mov %0, %%sp" | |
869 | : : "r" (orig_sp)); | |
4f70f7a9 DM |
870 | } |
871 | ||
e0204409 DM |
872 | #ifdef CONFIG_HOTPLUG_CPU |
873 | void fixup_irqs(void) | |
874 | { | |
875 | unsigned int irq; | |
876 | ||
877 | for (irq = 0; irq < NR_IRQS; irq++) { | |
16741ea0 | 878 | struct irq_desc *desc = irq_to_desc(irq); |
ee6a9333 | 879 | struct irq_data *data; |
e0204409 DM |
880 | unsigned long flags; |
881 | ||
ee6a9333 | 882 | if (!desc) |
883 | continue; | |
884 | data = irq_desc_get_irq_data(desc); | |
16741ea0 TG |
885 | raw_spin_lock_irqsave(&desc->lock, flags); |
886 | if (desc->action && !irqd_is_per_cpu(data)) { | |
4832b992 SR |
887 | if (data->chip->irq_set_affinity) |
888 | data->chip->irq_set_affinity(data, | |
d7185a98 JL |
889 | irq_data_get_affinity_mask(data), |
890 | false); | |
e0204409 | 891 | } |
16741ea0 | 892 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
e0204409 | 893 | } |
2eb2f779 DM |
894 | |
895 | tick_ops->disable_irq(); | |
e0204409 DM |
896 | } |
897 | #endif | |
898 | ||
cdd5186f DM |
899 | struct sun5_timer { |
900 | u64 count0; | |
901 | u64 limit0; | |
902 | u64 count1; | |
903 | u64 limit1; | |
904 | }; | |
1da177e4 | 905 | |
cdd5186f | 906 | static struct sun5_timer *prom_timers; |
1da177e4 LT |
907 | static u64 prom_limit0, prom_limit1; |
908 | ||
909 | static void map_prom_timers(void) | |
910 | { | |
25c7581b | 911 | struct device_node *dp; |
6a23acf3 | 912 | const unsigned int *addr; |
1da177e4 LT |
913 | |
914 | /* PROM timer node hangs out in the top level of device siblings... */ | |
25c7581b DM |
915 | dp = of_find_node_by_path("/"); |
916 | dp = dp->child; | |
917 | while (dp) { | |
918 | if (!strcmp(dp->name, "counter-timer")) | |
919 | break; | |
920 | dp = dp->sibling; | |
921 | } | |
1da177e4 LT |
922 | |
923 | /* Assume if node is not present, PROM uses different tick mechanism | |
924 | * which we should not care about. | |
925 | */ | |
25c7581b | 926 | if (!dp) { |
1da177e4 LT |
927 | prom_timers = (struct sun5_timer *) 0; |
928 | return; | |
929 | } | |
930 | ||
931 | /* If PROM is really using this, it must be mapped by him. */ | |
25c7581b DM |
932 | addr = of_get_property(dp, "address", NULL); |
933 | if (!addr) { | |
1da177e4 LT |
934 | prom_printf("PROM does not have timer mapped, trying to continue.\n"); |
935 | prom_timers = (struct sun5_timer *) 0; | |
936 | return; | |
937 | } | |
938 | prom_timers = (struct sun5_timer *) ((unsigned long)addr[0]); | |
939 | } | |
940 | ||
941 | static void kill_prom_timer(void) | |
942 | { | |
943 | if (!prom_timers) | |
944 | return; | |
945 | ||
946 | /* Save them away for later. */ | |
947 | prom_limit0 = prom_timers->limit0; | |
948 | prom_limit1 = prom_timers->limit1; | |
949 | ||
ee906c9e | 950 | /* Just as in sun4c PROM uses timer which ticks at IRQ 14. |
1da177e4 LT |
951 | * We turn both off here just to be paranoid. |
952 | */ | |
953 | prom_timers->limit0 = 0; | |
954 | prom_timers->limit1 = 0; | |
955 | ||
956 | /* Wheee, eat the interrupt packet too... */ | |
957 | __asm__ __volatile__( | |
958 | " mov 0x40, %%g2\n" | |
959 | " ldxa [%%g0] %0, %%g1\n" | |
960 | " ldxa [%%g2] %1, %%g1\n" | |
961 | " stxa %%g0, [%%g0] %0\n" | |
962 | " membar #Sync\n" | |
963 | : /* no outputs */ | |
964 | : "i" (ASI_INTR_RECEIVE), "i" (ASI_INTR_R) | |
965 | : "g1", "g2"); | |
966 | } | |
967 | ||
9843099f | 968 | void notrace init_irqwork_curcpu(void) |
1da177e4 | 969 | { |
1da177e4 LT |
970 | int cpu = hard_smp_processor_id(); |
971 | ||
eb2d8d60 | 972 | trap_block[cpu].irq_worklist_pa = 0UL; |
1da177e4 LT |
973 | } |
974 | ||
5cbc3073 DM |
975 | /* Please be very careful with register_one_mondo() and |
976 | * sun4v_register_mondo_queues(). | |
977 | * | |
978 | * On SMP this gets invoked from the CPU trampoline before | |
979 | * the cpu has fully taken over the trap table from OBP, | |
980 | * and it's kernel stack + %g6 thread register state is | |
981 | * not fully cooked yet. | |
982 | * | |
983 | * Therefore you cannot make any OBP calls, not even prom_printf, | |
984 | * from these two routines. | |
985 | */ | |
2066aadd PG |
986 | static void notrace register_one_mondo(unsigned long paddr, unsigned long type, |
987 | unsigned long qmask) | |
ac29c11d | 988 | { |
5cbc3073 | 989 | unsigned long num_entries = (qmask + 1) / 64; |
94f8762d DM |
990 | unsigned long status; |
991 | ||
992 | status = sun4v_cpu_qconf(type, paddr, num_entries); | |
993 | if (status != HV_EOK) { | |
994 | prom_printf("SUN4V: sun4v_cpu_qconf(%lu:%lx:%lu) failed, " | |
995 | "err %lu\n", type, paddr, num_entries, status); | |
ac29c11d DM |
996 | prom_halt(); |
997 | } | |
998 | } | |
999 | ||
2066aadd | 1000 | void notrace sun4v_register_mondo_queues(int this_cpu) |
5b0c0572 | 1001 | { |
b5a37e96 DM |
1002 | struct trap_per_cpu *tb = &trap_block[this_cpu]; |
1003 | ||
5cbc3073 DM |
1004 | register_one_mondo(tb->cpu_mondo_pa, HV_CPU_QUEUE_CPU_MONDO, |
1005 | tb->cpu_mondo_qmask); | |
1006 | register_one_mondo(tb->dev_mondo_pa, HV_CPU_QUEUE_DEVICE_MONDO, | |
1007 | tb->dev_mondo_qmask); | |
1008 | register_one_mondo(tb->resum_mondo_pa, HV_CPU_QUEUE_RES_ERROR, | |
1009 | tb->resum_qmask); | |
1010 | register_one_mondo(tb->nonresum_mondo_pa, HV_CPU_QUEUE_NONRES_ERROR, | |
1011 | tb->nonresum_qmask); | |
b5a37e96 DM |
1012 | } |
1013 | ||
14a2ff6e DM |
1014 | /* Each queue region must be a power of 2 multiple of 64 bytes in |
1015 | * size. The base real address must be aligned to the size of the | |
1016 | * region. Thus, an 8KB queue must be 8KB aligned, for example. | |
1017 | */ | |
1018 | static void __init alloc_one_queue(unsigned long *pa_ptr, unsigned long qmask) | |
b5a37e96 | 1019 | { |
5cbc3073 | 1020 | unsigned long size = PAGE_ALIGN(qmask + 1); |
14a2ff6e DM |
1021 | unsigned long order = get_order(size); |
1022 | unsigned long p; | |
5b0c0572 | 1023 | |
14a2ff6e | 1024 | p = __get_free_pages(GFP_KERNEL, order); |
5cbc3073 | 1025 | if (!p) { |
14a2ff6e | 1026 | prom_printf("SUN4V: Error, cannot allocate queue.\n"); |
5b0c0572 DM |
1027 | prom_halt(); |
1028 | } | |
1029 | ||
5cbc3073 | 1030 | *pa_ptr = __pa(p); |
5b0c0572 DM |
1031 | } |
1032 | ||
b434e719 | 1033 | static void __init init_cpu_send_mondo_info(struct trap_per_cpu *tb) |
1d2f1f90 DM |
1034 | { |
1035 | #ifdef CONFIG_SMP | |
14a2ff6e | 1036 | unsigned long page; |
1d2f1f90 DM |
1037 | |
1038 | BUILD_BUG_ON((NR_CPUS * sizeof(u16)) > (PAGE_SIZE - 64)); | |
1039 | ||
14a2ff6e | 1040 | page = get_zeroed_page(GFP_KERNEL); |
1d2f1f90 DM |
1041 | if (!page) { |
1042 | prom_printf("SUN4V: Error, cannot allocate cpu mondo page.\n"); | |
1043 | prom_halt(); | |
1044 | } | |
1045 | ||
1046 | tb->cpu_mondo_block_pa = __pa(page); | |
1047 | tb->cpu_list_pa = __pa(page + 64); | |
1048 | #endif | |
1049 | } | |
1050 | ||
b434e719 DM |
1051 | /* Allocate mondo and error queues for all possible cpus. */ |
1052 | static void __init sun4v_init_mondo_queues(void) | |
ac29c11d | 1053 | { |
b434e719 | 1054 | int cpu; |
ac29c11d | 1055 | |
b434e719 DM |
1056 | for_each_possible_cpu(cpu) { |
1057 | struct trap_per_cpu *tb = &trap_block[cpu]; | |
1d2f1f90 | 1058 | |
14a2ff6e DM |
1059 | alloc_one_queue(&tb->cpu_mondo_pa, tb->cpu_mondo_qmask); |
1060 | alloc_one_queue(&tb->dev_mondo_pa, tb->dev_mondo_qmask); | |
1061 | alloc_one_queue(&tb->resum_mondo_pa, tb->resum_qmask); | |
1062 | alloc_one_queue(&tb->resum_kernel_buf_pa, tb->resum_qmask); | |
1063 | alloc_one_queue(&tb->nonresum_mondo_pa, tb->nonresum_qmask); | |
1064 | alloc_one_queue(&tb->nonresum_kernel_buf_pa, | |
1065 | tb->nonresum_qmask); | |
43f58923 DM |
1066 | } |
1067 | } | |
1068 | ||
1069 | static void __init init_send_mondo_info(void) | |
1070 | { | |
1071 | int cpu; | |
1072 | ||
1073 | for_each_possible_cpu(cpu) { | |
1074 | struct trap_per_cpu *tb = &trap_block[cpu]; | |
1d2f1f90 | 1075 | |
b434e719 | 1076 | init_cpu_send_mondo_info(tb); |
72aff53f | 1077 | } |
ac29c11d DM |
1078 | } |
1079 | ||
e18e2a00 DM |
1080 | static struct irqaction timer_irq_action = { |
1081 | .name = "timer", | |
1082 | }; | |
1083 | ||
ee6a9333 | 1084 | static void __init irq_ivector_init(void) |
1da177e4 | 1085 | { |
ee6a9333 | 1086 | unsigned long size, order; |
1087 | unsigned int ivecs; | |
10397e40 | 1088 | |
ee6a9333 | 1089 | /* If we are doing cookie only VIRQs then we do not need the ivector |
1090 | * table to process interrupts. | |
1091 | */ | |
1092 | if (sun4v_cookie_only_virqs()) | |
1093 | return; | |
1da177e4 | 1094 | |
ee6a9333 | 1095 | ivecs = size_nr_ivec(); |
1096 | size = sizeof(struct ino_bucket) * ivecs; | |
1097 | order = get_order(size); | |
1098 | ivector_table = (struct ino_bucket *) | |
1099 | __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); | |
10397e40 DM |
1100 | if (!ivector_table) { |
1101 | prom_printf("Fatal error, cannot allocate ivector_table\n"); | |
1102 | prom_halt(); | |
1103 | } | |
42d5f99b DM |
1104 | __flush_dcache_range((unsigned long) ivector_table, |
1105 | ((unsigned long) ivector_table) + size); | |
10397e40 DM |
1106 | |
1107 | ivector_table_pa = __pa(ivector_table); | |
ee6a9333 | 1108 | } |
1109 | ||
1110 | /* Only invoked on boot processor.*/ | |
1111 | void __init init_IRQ(void) | |
1112 | { | |
1113 | irq_init_hv(); | |
1114 | irq_ivector_init(); | |
1115 | map_prom_timers(); | |
1116 | kill_prom_timer(); | |
eb2d8d60 | 1117 | |
ac29c11d | 1118 | if (tlb_type == hypervisor) |
b434e719 | 1119 | sun4v_init_mondo_queues(); |
ac29c11d | 1120 | |
43f58923 DM |
1121 | init_send_mondo_info(); |
1122 | ||
1123 | if (tlb_type == hypervisor) { | |
1124 | /* Load up the boot cpu's entries. */ | |
1125 | sun4v_register_mondo_queues(hard_smp_processor_id()); | |
1126 | } | |
1127 | ||
1da177e4 LT |
1128 | /* We need to clear any IRQ's pending in the soft interrupt |
1129 | * registers, a spurious one could be left around from the | |
1130 | * PROM timer which we just disabled. | |
1131 | */ | |
1132 | clear_softint(get_softint()); | |
1133 | ||
1134 | /* Now that ivector table is initialized, it is safe | |
1135 | * to receive IRQ vector traps. We will normally take | |
1136 | * one or two right now, in case some device PROM used | |
1137 | * to boot us wants to speak to us. We just ignore them. | |
1138 | */ | |
1139 | __asm__ __volatile__("rdpr %%pstate, %%g1\n\t" | |
1140 | "or %%g1, %0, %%g1\n\t" | |
1141 | "wrpr %%g1, 0x0, %%pstate" | |
1142 | : /* No outputs */ | |
1143 | : "i" (PSTATE_IE) | |
1144 | : "g1"); | |
1da177e4 | 1145 | |
16741ea0 | 1146 | irq_to_desc(0)->action = &timer_irq_action; |
1da177e4 | 1147 | } |