Commit | Line | Data |
---|---|---|
8531a35e KK |
1 | /* |
2 | * This program is free software; you can redistribute it and/or | |
3 | * modify it under the terms of the GNU General Public License | |
4 | * as published by the Free Software Foundation; either version 2 | |
5 | * of the License, or (at your option) any later version. | |
6 | * | |
7 | * This program is distributed in the hope that it will be useful, | |
8 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
9 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
10 | * GNU General Public License for more details. | |
11 | * | |
12 | * You should have received a copy of the GNU General Public License | |
13 | * along with this program; if not, write to the Free Software | |
14 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
15 | * | |
16 | * Copyright (C) 2004 Mips Technologies, Inc | |
17 | * Copyright (C) 2008 Kevin D. Kissell | |
18 | */ | |
41c594ab | 19 | |
ea580401 | 20 | #include <linux/clockchips.h> |
41c594ab RB |
21 | #include <linux/kernel.h> |
22 | #include <linux/sched.h> | |
631330f5 | 23 | #include <linux/smp.h> |
41c594ab RB |
24 | #include <linux/cpumask.h> |
25 | #include <linux/interrupt.h> | |
ae036b79 | 26 | #include <linux/kernel_stat.h> |
ec43c014 | 27 | #include <linux/module.h> |
8f99a162 | 28 | #include <linux/ftrace.h> |
5a0e3ad6 | 29 | #include <linux/slab.h> |
41c594ab RB |
30 | |
31 | #include <asm/cpu.h> | |
32 | #include <asm/processor.h> | |
60063497 | 33 | #include <linux/atomic.h> |
41c594ab RB |
34 | #include <asm/system.h> |
35 | #include <asm/hardirq.h> | |
36 | #include <asm/hazards.h> | |
3b1d4ed5 | 37 | #include <asm/irq.h> |
41c594ab | 38 | #include <asm/mmu_context.h> |
41c594ab RB |
39 | #include <asm/mipsregs.h> |
40 | #include <asm/cacheflush.h> | |
41 | #include <asm/time.h> | |
42 | #include <asm/addrspace.h> | |
43 | #include <asm/smtc.h> | |
41c594ab RB |
44 | #include <asm/smtc_proc.h> |
45 | ||
46 | /* | |
1146fe30 RB |
47 | * SMTC Kernel needs to manipulate low-level CPU interrupt mask |
48 | * in do_IRQ. These are passed in setup_irq_smtc() and stored | |
49 | * in this table. | |
41c594ab | 50 | */ |
1146fe30 | 51 | unsigned long irq_hwmask[NR_IRQS]; |
41c594ab | 52 | |
41c594ab RB |
53 | #define LOCK_MT_PRA() \ |
54 | local_irq_save(flags); \ | |
55 | mtflags = dmt() | |
56 | ||
57 | #define UNLOCK_MT_PRA() \ | |
58 | emt(mtflags); \ | |
59 | local_irq_restore(flags) | |
60 | ||
61 | #define LOCK_CORE_PRA() \ | |
62 | local_irq_save(flags); \ | |
63 | mtflags = dvpe() | |
64 | ||
65 | #define UNLOCK_CORE_PRA() \ | |
66 | evpe(mtflags); \ | |
67 | local_irq_restore(flags) | |
68 | ||
69 | /* | |
70 | * Data structures purely associated with SMTC parallelism | |
71 | */ | |
72 | ||
73 | ||
74 | /* | |
75 | * Table for tracking ASIDs whose lifetime is prolonged. | |
76 | */ | |
77 | ||
78 | asiduse smtc_live_asid[MAX_SMTC_TLBS][MAX_SMTC_ASIDS]; | |
79 | ||
41c594ab | 80 | /* |
603e82ed | 81 | * Number of InterProcessor Interrupt (IPI) message buffers to allocate |
41c594ab RB |
82 | */ |
83 | ||
84 | #define IPIBUF_PER_CPU 4 | |
85 | ||
d2bb01b0 | 86 | struct smtc_ipi_q IPIQ[NR_CPUS]; |
5868756d | 87 | static struct smtc_ipi_q freeIPIq; |
41c594ab RB |
88 | |
89 | ||
90 | /* Forward declarations */ | |
91 | ||
937a8015 | 92 | void ipi_decode(struct smtc_ipi *); |
5868756d | 93 | static void post_direct_ipi(int cpu, struct smtc_ipi *pipi); |
20bb25d1 | 94 | static void setup_cross_vpe_interrupts(unsigned int nvpe); |
41c594ab RB |
95 | void init_smtc_stats(void); |
96 | ||
97 | /* Global SMTC Status */ | |
98 | ||
982f6ffe | 99 | unsigned int smtc_status; |
41c594ab RB |
100 | |
101 | /* Boot command line configuration overrides */ | |
102 | ||
be5f1f21 | 103 | static int vpe0limit; |
982f6ffe RB |
104 | static int ipibuffers; |
105 | static int nostlb; | |
106 | static int asidmask; | |
41c594ab RB |
107 | unsigned long smtc_asid_mask = 0xff; |
108 | ||
be5f1f21 KK |
109 | static int __init vpe0tcs(char *str) |
110 | { | |
111 | get_option(&str, &vpe0limit); | |
112 | ||
113 | return 1; | |
114 | } | |
115 | ||
41c594ab RB |
116 | static int __init ipibufs(char *str) |
117 | { | |
118 | get_option(&str, &ipibuffers); | |
119 | return 1; | |
120 | } | |
121 | ||
122 | static int __init stlb_disable(char *s) | |
123 | { | |
124 | nostlb = 1; | |
125 | return 1; | |
126 | } | |
127 | ||
128 | static int __init asidmask_set(char *str) | |
129 | { | |
130 | get_option(&str, &asidmask); | |
4bf42d42 | 131 | switch (asidmask) { |
41c594ab RB |
132 | case 0x1: |
133 | case 0x3: | |
134 | case 0x7: | |
135 | case 0xf: | |
136 | case 0x1f: | |
137 | case 0x3f: | |
138 | case 0x7f: | |
139 | case 0xff: | |
140 | smtc_asid_mask = (unsigned long)asidmask; | |
141 | break; | |
142 | default: | |
143 | printk("ILLEGAL ASID mask 0x%x from command line\n", asidmask); | |
144 | } | |
145 | return 1; | |
146 | } | |
147 | ||
be5f1f21 | 148 | __setup("vpe0tcs=", vpe0tcs); |
41c594ab RB |
149 | __setup("ipibufs=", ipibufs); |
150 | __setup("nostlb", stlb_disable); | |
151 | __setup("asidmask=", asidmask_set); | |
152 | ||
c68644d3 | 153 | #ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG |
41c594ab | 154 | |
982f6ffe | 155 | static int hang_trig; |
41c594ab RB |
156 | |
157 | static int __init hangtrig_enable(char *s) | |
158 | { | |
159 | hang_trig = 1; | |
160 | return 1; | |
161 | } | |
162 | ||
163 | ||
164 | __setup("hangtrig", hangtrig_enable); | |
165 | ||
166 | #define DEFAULT_BLOCKED_IPI_LIMIT 32 | |
167 | ||
168 | static int timerq_limit = DEFAULT_BLOCKED_IPI_LIMIT; | |
169 | ||
170 | static int __init tintq(char *str) | |
171 | { | |
172 | get_option(&str, &timerq_limit); | |
173 | return 1; | |
174 | } | |
175 | ||
176 | __setup("tintq=", tintq); | |
177 | ||
97aef63c | 178 | static int imstuckcount[2][8]; |
41c594ab | 179 | /* vpemask represents IM/IE bits of per-VPE Status registers, low-to-high */ |
97aef63c | 180 | static int vpemask[2][8] = { |
20bb25d1 RB |
181 | {0, 0, 1, 0, 0, 0, 0, 1}, |
182 | {0, 0, 0, 0, 0, 0, 0, 1} | |
183 | }; | |
41c594ab | 184 | int tcnoprog[NR_CPUS]; |
52553664 | 185 | static atomic_t idle_hook_initialized = ATOMIC_INIT(0); |
41c594ab RB |
186 | static int clock_hang_reported[NR_CPUS]; |
187 | ||
c68644d3 | 188 | #endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */ |
41c594ab | 189 | |
41c594ab RB |
190 | /* |
191 | * Configure shared TLB - VPC configuration bit must be set by caller | |
192 | */ | |
193 | ||
5868756d | 194 | static void smtc_configure_tlb(void) |
41c594ab | 195 | { |
21a151d8 | 196 | int i, tlbsiz, vpes; |
41c594ab RB |
197 | unsigned long mvpconf0; |
198 | unsigned long config1val; | |
199 | ||
200 | /* Set up ASID preservation table */ | |
201 | for (vpes=0; vpes<MAX_SMTC_TLBS; vpes++) { | |
202 | for(i = 0; i < MAX_SMTC_ASIDS; i++) { | |
203 | smtc_live_asid[vpes][i] = 0; | |
204 | } | |
205 | } | |
206 | mvpconf0 = read_c0_mvpconf0(); | |
207 | ||
208 | if ((vpes = ((mvpconf0 & MVPCONF0_PVPE) | |
209 | >> MVPCONF0_PVPE_SHIFT) + 1) > 1) { | |
210 | /* If we have multiple VPEs, try to share the TLB */ | |
211 | if ((mvpconf0 & MVPCONF0_TLBS) && !nostlb) { | |
212 | /* | |
213 | * If TLB sizing is programmable, shared TLB | |
214 | * size is the total available complement. | |
215 | * Otherwise, we have to take the sum of all | |
216 | * static VPE TLB entries. | |
217 | */ | |
218 | if ((tlbsiz = ((mvpconf0 & MVPCONF0_PTLBE) | |
219 | >> MVPCONF0_PTLBE_SHIFT)) == 0) { | |
220 | /* | |
221 | * If there's more than one VPE, there had better | |
222 | * be more than one TC, because we need one to bind | |
223 | * to each VPE in turn to be able to read | |
224 | * its configuration state! | |
225 | */ | |
226 | settc(1); | |
227 | /* Stop the TC from doing anything foolish */ | |
228 | write_tc_c0_tchalt(TCHALT_H); | |
229 | mips_ihb(); | |
230 | /* No need to un-Halt - that happens later anyway */ | |
231 | for (i=0; i < vpes; i++) { | |
232 | write_tc_c0_tcbind(i); | |
233 | /* | |
234 | * To be 100% sure we're really getting the right | |
235 | * information, we exit the configuration state | |
236 | * and do an IHB after each rebinding. | |
237 | */ | |
238 | write_c0_mvpcontrol( | |
239 | read_c0_mvpcontrol() & ~ MVPCONTROL_VPC ); | |
240 | mips_ihb(); | |
241 | /* | |
242 | * Only count if the MMU Type indicated is TLB | |
243 | */ | |
4bf42d42 | 244 | if (((read_vpe_c0_config() & MIPS_CONF_MT) >> 7) == 1) { |
41c594ab RB |
245 | config1val = read_vpe_c0_config1(); |
246 | tlbsiz += ((config1val >> 25) & 0x3f) + 1; | |
247 | } | |
248 | ||
249 | /* Put core back in configuration state */ | |
250 | write_c0_mvpcontrol( | |
251 | read_c0_mvpcontrol() | MVPCONTROL_VPC ); | |
252 | mips_ihb(); | |
253 | } | |
254 | } | |
255 | write_c0_mvpcontrol(read_c0_mvpcontrol() | MVPCONTROL_STLB); | |
c80697b3 | 256 | ehb(); |
41c594ab RB |
257 | |
258 | /* | |
259 | * Setup kernel data structures to use software total, | |
260 | * rather than read the per-VPE Config1 value. The values | |
261 | * for "CPU 0" gets copied to all the other CPUs as part | |
262 | * of their initialization in smtc_cpu_setup(). | |
263 | */ | |
264 | ||
a0b62180 RB |
265 | /* MIPS32 limits TLB indices to 64 */ |
266 | if (tlbsiz > 64) | |
267 | tlbsiz = 64; | |
268 | cpu_data[0].tlbsize = current_cpu_data.tlbsize = tlbsiz; | |
41c594ab | 269 | smtc_status |= SMTC_TLB_SHARED; |
a0b62180 | 270 | local_flush_tlb_all(); |
41c594ab RB |
271 | |
272 | printk("TLB of %d entry pairs shared by %d VPEs\n", | |
273 | tlbsiz, vpes); | |
274 | } else { | |
275 | printk("WARNING: TLB Not Sharable on SMTC Boot!\n"); | |
276 | } | |
277 | } | |
278 | } | |
279 | ||
280 | ||
281 | /* | |
282 | * Incrementally build the CPU map out of constituent MIPS MT cores, | |
283 | * using the specified available VPEs and TCs. Plaform code needs | |
284 | * to ensure that each MIPS MT core invokes this routine on reset, | |
285 | * one at a time(!). | |
286 | * | |
287 | * This version of the build_cpu_map and prepare_cpus routines assumes | |
288 | * that *all* TCs of a MIPS MT core will be used for Linux, and that | |
289 | * they will be spread across *all* available VPEs (to minimise the | |
290 | * loss of efficiency due to exception service serialization). | |
291 | * An improved version would pick up configuration information and | |
292 | * possibly leave some TCs/VPEs as "slave" processors. | |
293 | * | |
294 | * Use c0_MVPConf0 to find out how many TCs are available, setting up | |
98a79d6a | 295 | * cpu_possible_map and the logical/physical mappings. |
41c594ab RB |
296 | */ |
297 | ||
8531a35e | 298 | int __init smtc_build_cpu_map(int start_cpu_slot) |
41c594ab RB |
299 | { |
300 | int i, ntcs; | |
301 | ||
302 | /* | |
303 | * The CPU map isn't actually used for anything at this point, | |
304 | * so it's not clear what else we should do apart from set | |
305 | * everything up so that "logical" = "physical". | |
306 | */ | |
307 | ntcs = ((read_c0_mvpconf0() & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; | |
308 | for (i=start_cpu_slot; i<NR_CPUS && i<ntcs; i++) { | |
4037ac6e | 309 | set_cpu_possible(i, true); |
41c594ab RB |
310 | __cpu_number_map[i] = i; |
311 | __cpu_logical_map[i] = i; | |
312 | } | |
ea580401 | 313 | #ifdef CONFIG_MIPS_MT_FPAFF |
41c594ab RB |
314 | /* Initialize map of CPUs with FPUs */ |
315 | cpus_clear(mt_fpu_cpumask); | |
ea580401 | 316 | #endif |
41c594ab RB |
317 | |
318 | /* One of those TC's is the one booting, and not a secondary... */ | |
319 | printk("%i available secondary CPU TC(s)\n", i - 1); | |
320 | ||
321 | return i; | |
322 | } | |
323 | ||
324 | /* | |
325 | * Common setup before any secondaries are started | |
326 | * Make sure all CPU's are in a sensible state before we boot any of the | |
327 | * secondaries. | |
328 | * | |
329 | * For MIPS MT "SMTC" operation, we set up all TCs, spread as evenly | |
330 | * as possible across the available VPEs. | |
331 | */ | |
332 | ||
333 | static void smtc_tc_setup(int vpe, int tc, int cpu) | |
334 | { | |
335 | settc(tc); | |
336 | write_tc_c0_tchalt(TCHALT_H); | |
337 | mips_ihb(); | |
338 | write_tc_c0_tcstatus((read_tc_c0_tcstatus() | |
339 | & ~(TCSTATUS_TKSU | TCSTATUS_DA | TCSTATUS_IXMT)) | |
340 | | TCSTATUS_A); | |
8531a35e KK |
341 | /* |
342 | * TCContext gets an offset from the base of the IPIQ array | |
343 | * to be used in low-level code to detect the presence of | |
344 | * an active IPI queue | |
345 | */ | |
346 | write_tc_c0_tccontext((sizeof(struct smtc_ipi_q) * cpu) << 16); | |
41c594ab RB |
347 | /* Bind tc to vpe */ |
348 | write_tc_c0_tcbind(vpe); | |
349 | /* In general, all TCs should have the same cpu_data indications */ | |
350 | memcpy(&cpu_data[cpu], &cpu_data[0], sizeof(struct cpuinfo_mips)); | |
351 | /* For 34Kf, start with TC/CPU 0 as sole owner of single FPU context */ | |
39b8d525 RB |
352 | if (cpu_data[0].cputype == CPU_34K || |
353 | cpu_data[0].cputype == CPU_1004K) | |
41c594ab RB |
354 | cpu_data[cpu].options &= ~MIPS_CPU_FPU; |
355 | cpu_data[cpu].vpe_id = vpe; | |
356 | cpu_data[cpu].tc_id = tc; | |
8531a35e KK |
357 | /* Multi-core SMTC hasn't been tested, but be prepared */ |
358 | cpu_data[cpu].core = (read_vpe_c0_ebase() >> 1) & 0xff; | |
41c594ab RB |
359 | } |
360 | ||
8531a35e KK |
361 | /* |
362 | * Tweak to get Count registes in as close a sync as possible. | |
363 | * Value seems good for 34K-class cores. | |
364 | */ | |
365 | ||
366 | #define CP0_SKEW 8 | |
41c594ab | 367 | |
8531a35e | 368 | void smtc_prepare_cpus(int cpus) |
41c594ab | 369 | { |
be5f1f21 | 370 | int i, vpe, tc, ntc, nvpe, tcpervpe[NR_CPUS], slop, cpu; |
41c594ab RB |
371 | unsigned long flags; |
372 | unsigned long val; | |
373 | int nipi; | |
374 | struct smtc_ipi *pipi; | |
375 | ||
376 | /* disable interrupts so we can disable MT */ | |
377 | local_irq_save(flags); | |
378 | /* disable MT so we can configure */ | |
379 | dvpe(); | |
380 | dmt(); | |
381 | ||
34af946a | 382 | spin_lock_init(&freeIPIq.lock); |
41c594ab RB |
383 | |
384 | /* | |
385 | * We probably don't have as many VPEs as we do SMP "CPUs", | |
386 | * but it's possible - and in any case we'll never use more! | |
387 | */ | |
388 | for (i=0; i<NR_CPUS; i++) { | |
389 | IPIQ[i].head = IPIQ[i].tail = NULL; | |
34af946a | 390 | spin_lock_init(&IPIQ[i].lock); |
41c594ab | 391 | IPIQ[i].depth = 0; |
2e41f91d | 392 | IPIQ[i].resched_flag = 0; /* No reschedules queued initially */ |
41c594ab RB |
393 | } |
394 | ||
395 | /* cpu_data index starts at zero */ | |
396 | cpu = 0; | |
397 | cpu_data[cpu].vpe_id = 0; | |
398 | cpu_data[cpu].tc_id = 0; | |
8531a35e | 399 | cpu_data[cpu].core = (read_c0_ebase() >> 1) & 0xff; |
41c594ab RB |
400 | cpu++; |
401 | ||
402 | /* Report on boot-time options */ | |
49a89efb | 403 | mips_mt_set_cpuoptions(); |
41c594ab RB |
404 | if (vpelimit > 0) |
405 | printk("Limit of %d VPEs set\n", vpelimit); | |
406 | if (tclimit > 0) | |
407 | printk("Limit of %d TCs set\n", tclimit); | |
408 | if (nostlb) { | |
409 | printk("Shared TLB Use Inhibited - UNSAFE for Multi-VPE Operation\n"); | |
410 | } | |
411 | if (asidmask) | |
412 | printk("ASID mask value override to 0x%x\n", asidmask); | |
413 | ||
414 | /* Temporary */ | |
c68644d3 | 415 | #ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG |
41c594ab RB |
416 | if (hang_trig) |
417 | printk("Logic Analyser Trigger on suspected TC hang\n"); | |
c68644d3 | 418 | #endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */ |
41c594ab RB |
419 | |
420 | /* Put MVPE's into 'configuration state' */ | |
421 | write_c0_mvpcontrol( read_c0_mvpcontrol() | MVPCONTROL_VPC ); | |
422 | ||
423 | val = read_c0_mvpconf0(); | |
424 | nvpe = ((val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1; | |
425 | if (vpelimit > 0 && nvpe > vpelimit) | |
426 | nvpe = vpelimit; | |
427 | ntc = ((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; | |
428 | if (ntc > NR_CPUS) | |
429 | ntc = NR_CPUS; | |
430 | if (tclimit > 0 && ntc > tclimit) | |
431 | ntc = tclimit; | |
be5f1f21 KK |
432 | slop = ntc % nvpe; |
433 | for (i = 0; i < nvpe; i++) { | |
434 | tcpervpe[i] = ntc / nvpe; | |
435 | if (slop) { | |
436 | if((slop - i) > 0) tcpervpe[i]++; | |
437 | } | |
438 | } | |
439 | /* Handle command line override for VPE0 */ | |
440 | if (vpe0limit > ntc) vpe0limit = ntc; | |
441 | if (vpe0limit > 0) { | |
442 | int slopslop; | |
443 | if (vpe0limit < tcpervpe[0]) { | |
444 | /* Reducing TC count - distribute to others */ | |
445 | slop = tcpervpe[0] - vpe0limit; | |
446 | slopslop = slop % (nvpe - 1); | |
447 | tcpervpe[0] = vpe0limit; | |
448 | for (i = 1; i < nvpe; i++) { | |
449 | tcpervpe[i] += slop / (nvpe - 1); | |
450 | if(slopslop && ((slopslop - (i - 1) > 0))) | |
451 | tcpervpe[i]++; | |
452 | } | |
453 | } else if (vpe0limit > tcpervpe[0]) { | |
454 | /* Increasing TC count - steal from others */ | |
455 | slop = vpe0limit - tcpervpe[0]; | |
456 | slopslop = slop % (nvpe - 1); | |
457 | tcpervpe[0] = vpe0limit; | |
458 | for (i = 1; i < nvpe; i++) { | |
459 | tcpervpe[i] -= slop / (nvpe - 1); | |
460 | if(slopslop && ((slopslop - (i - 1) > 0))) | |
461 | tcpervpe[i]--; | |
462 | } | |
463 | } | |
464 | } | |
41c594ab RB |
465 | |
466 | /* Set up shared TLB */ | |
467 | smtc_configure_tlb(); | |
468 | ||
469 | for (tc = 0, vpe = 0 ; (vpe < nvpe) && (tc < ntc) ; vpe++) { | |
d8e5f9fe KM |
470 | if (tcpervpe[vpe] == 0) |
471 | continue; | |
41c594ab RB |
472 | if (vpe != 0) |
473 | printk(", "); | |
474 | printk("VPE %d: TC", vpe); | |
be5f1f21 | 475 | for (i = 0; i < tcpervpe[vpe]; i++) { |
41c594ab RB |
476 | /* |
477 | * TC 0 is bound to VPE 0 at reset, | |
478 | * and is presumably executing this | |
479 | * code. Leave it alone! | |
480 | */ | |
481 | if (tc != 0) { | |
21a151d8 | 482 | smtc_tc_setup(vpe, tc, cpu); |
41c594ab RB |
483 | cpu++; |
484 | } | |
485 | printk(" %d", tc); | |
486 | tc++; | |
487 | } | |
41c594ab | 488 | if (vpe != 0) { |
d8e5f9fe KM |
489 | /* |
490 | * Allow this VPE to control others. | |
491 | */ | |
492 | write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | | |
493 | VPECONF0_MVP); | |
494 | ||
41c594ab RB |
495 | /* |
496 | * Clear any stale software interrupts from VPE's Cause | |
497 | */ | |
498 | write_vpe_c0_cause(0); | |
499 | ||
500 | /* | |
501 | * Clear ERL/EXL of VPEs other than 0 | |
502 | * and set restricted interrupt enable/mask. | |
503 | */ | |
504 | write_vpe_c0_status((read_vpe_c0_status() | |
505 | & ~(ST0_BEV | ST0_ERL | ST0_EXL | ST0_IM)) | |
506 | | (STATUSF_IP0 | STATUSF_IP1 | STATUSF_IP7 | |
507 | | ST0_IE)); | |
508 | /* | |
509 | * set config to be the same as vpe0, | |
510 | * particularly kseg0 coherency alg | |
511 | */ | |
512 | write_vpe_c0_config(read_c0_config()); | |
513 | /* Clear any pending timer interrupt */ | |
514 | write_vpe_c0_compare(0); | |
515 | /* Propagate Config7 */ | |
516 | write_vpe_c0_config7(read_c0_config7()); | |
8531a35e KK |
517 | write_vpe_c0_count(read_c0_count() + CP0_SKEW); |
518 | ehb(); | |
41c594ab RB |
519 | } |
520 | /* enable multi-threading within VPE */ | |
521 | write_vpe_c0_vpecontrol(read_vpe_c0_vpecontrol() | VPECONTROL_TE); | |
522 | /* enable the VPE */ | |
523 | write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_VPA); | |
524 | } | |
525 | ||
526 | /* | |
527 | * Pull any physically present but unused TCs out of circulation. | |
528 | */ | |
529 | while (tc < (((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1)) { | |
4037ac6e RR |
530 | set_cpu_possible(tc, false); |
531 | set_cpu_present(tc, false); | |
41c594ab RB |
532 | tc++; |
533 | } | |
534 | ||
535 | /* release config state */ | |
536 | write_c0_mvpcontrol( read_c0_mvpcontrol() & ~ MVPCONTROL_VPC ); | |
537 | ||
538 | printk("\n"); | |
539 | ||
540 | /* Set up coprocessor affinity CPU mask(s) */ | |
541 | ||
ea580401 | 542 | #ifdef CONFIG_MIPS_MT_FPAFF |
41c594ab | 543 | for (tc = 0; tc < ntc; tc++) { |
4bf42d42 | 544 | if (cpu_data[tc].options & MIPS_CPU_FPU) |
41c594ab RB |
545 | cpu_set(tc, mt_fpu_cpumask); |
546 | } | |
ea580401 | 547 | #endif |
41c594ab RB |
548 | |
549 | /* set up ipi interrupts... */ | |
550 | ||
551 | /* If we have multiple VPEs running, set up the cross-VPE interrupt */ | |
552 | ||
20bb25d1 | 553 | setup_cross_vpe_interrupts(nvpe); |
41c594ab RB |
554 | |
555 | /* Set up queue of free IPI "messages". */ | |
556 | nipi = NR_CPUS * IPIBUF_PER_CPU; | |
557 | if (ipibuffers > 0) | |
558 | nipi = ipibuffers; | |
559 | ||
560 | pipi = kmalloc(nipi *sizeof(struct smtc_ipi), GFP_KERNEL); | |
561 | if (pipi == NULL) | |
562 | panic("kmalloc of IPI message buffers failed\n"); | |
563 | else | |
564 | printk("IPI buffer pool of %d buffers\n", nipi); | |
565 | for (i = 0; i < nipi; i++) { | |
566 | smtc_ipi_nq(&freeIPIq, pipi); | |
567 | pipi++; | |
568 | } | |
569 | ||
570 | /* Arm multithreading and enable other VPEs - but all TCs are Halted */ | |
571 | emt(EMT_ENABLE); | |
572 | evpe(EVPE_ENABLE); | |
573 | local_irq_restore(flags); | |
574 | /* Initialize SMTC /proc statistics/diagnostics */ | |
575 | init_smtc_stats(); | |
576 | } | |
577 | ||
578 | ||
579 | /* | |
580 | * Setup the PC, SP, and GP of a secondary processor and start it | |
581 | * running! | |
582 | * smp_bootstrap is the place to resume from | |
583 | * __KSTK_TOS(idle) is apparently the stack pointer | |
584 | * (unsigned long)idle->thread_info the gp | |
585 | * | |
586 | */ | |
e119d49a | 587 | void __cpuinit smtc_boot_secondary(int cpu, struct task_struct *idle) |
41c594ab RB |
588 | { |
589 | extern u32 kernelsp[NR_CPUS]; | |
b7e4226e | 590 | unsigned long flags; |
41c594ab RB |
591 | int mtflags; |
592 | ||
593 | LOCK_MT_PRA(); | |
594 | if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { | |
595 | dvpe(); | |
596 | } | |
597 | settc(cpu_data[cpu].tc_id); | |
598 | ||
599 | /* pc */ | |
600 | write_tc_c0_tcrestart((unsigned long)&smp_bootstrap); | |
601 | ||
602 | /* stack pointer */ | |
603 | kernelsp[cpu] = __KSTK_TOS(idle); | |
604 | write_tc_gpr_sp(__KSTK_TOS(idle)); | |
605 | ||
606 | /* global pointer */ | |
c9f4f06d | 607 | write_tc_gpr_gp((unsigned long)task_thread_info(idle)); |
41c594ab RB |
608 | |
609 | smtc_status |= SMTC_MTC_ACTIVE; | |
610 | write_tc_c0_tchalt(0); | |
611 | if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { | |
612 | evpe(EVPE_ENABLE); | |
613 | } | |
614 | UNLOCK_MT_PRA(); | |
615 | } | |
616 | ||
617 | void smtc_init_secondary(void) | |
618 | { | |
41c594ab RB |
619 | local_irq_enable(); |
620 | } | |
621 | ||
622 | void smtc_smp_finish(void) | |
623 | { | |
8531a35e KK |
624 | int cpu = smp_processor_id(); |
625 | ||
626 | /* | |
627 | * Lowest-numbered CPU per VPE starts a clock tick. | |
628 | * Like per_cpu_trap_init() hack, this assumes that | |
629 | * SMTC init code assigns TCs consdecutively and | |
630 | * in ascending order across available VPEs. | |
631 | */ | |
632 | if (cpu > 0 && (cpu_data[cpu].vpe_id != cpu_data[cpu - 1].vpe_id)) | |
633 | write_c0_compare(read_c0_count() + mips_hpt_frequency/HZ); | |
634 | ||
41c594ab RB |
635 | printk("TC %d going on-line as CPU %d\n", |
636 | cpu_data[smp_processor_id()].tc_id, smp_processor_id()); | |
637 | } | |
638 | ||
639 | void smtc_cpus_done(void) | |
640 | { | |
641 | } | |
642 | ||
643 | /* | |
644 | * Support for SMTC-optimized driver IRQ registration | |
645 | */ | |
646 | ||
647 | /* | |
648 | * SMTC Kernel needs to manipulate low-level CPU interrupt mask | |
649 | * in do_IRQ. These are passed in setup_irq_smtc() and stored | |
650 | * in this table. | |
651 | */ | |
652 | ||
653 | int setup_irq_smtc(unsigned int irq, struct irqaction * new, | |
654 | unsigned long hwmask) | |
655 | { | |
ef36fc3c | 656 | #ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG |
20bb25d1 RB |
657 | unsigned int vpe = current_cpu_data.vpe_id; |
658 | ||
3b1d4ed5 | 659 | vpemask[vpe][irq - MIPS_CPU_IRQ_BASE] = 1; |
20bb25d1 | 660 | #endif |
ef36fc3c | 661 | irq_hwmask[irq] = hwmask; |
41c594ab RB |
662 | |
663 | return setup_irq(irq, new); | |
664 | } | |
665 | ||
f571eff0 KK |
666 | #ifdef CONFIG_MIPS_MT_SMTC_IRQAFF |
667 | /* | |
668 | * Support for IRQ affinity to TCs | |
669 | */ | |
670 | ||
671 | void smtc_set_irq_affinity(unsigned int irq, cpumask_t affinity) | |
672 | { | |
673 | /* | |
674 | * If a "fast path" cache of quickly decodable affinity state | |
675 | * is maintained, this is where it gets done, on a call up | |
676 | * from the platform affinity code. | |
677 | */ | |
678 | } | |
679 | ||
930cd54b | 680 | void smtc_forward_irq(struct irq_data *d) |
f571eff0 | 681 | { |
930cd54b | 682 | unsigned int irq = d->irq; |
f571eff0 KK |
683 | int target; |
684 | ||
685 | /* | |
686 | * OK wise guy, now figure out how to get the IRQ | |
687 | * to be serviced on an authorized "CPU". | |
688 | * | |
689 | * Ideally, to handle the situation where an IRQ has multiple | |
690 | * eligible CPUS, we would maintain state per IRQ that would | |
691 | * allow a fair distribution of service requests. Since the | |
692 | * expected use model is any-or-only-one, for simplicity | |
693 | * and efficiency, we just pick the easiest one to find. | |
694 | */ | |
695 | ||
2a2b2212 | 696 | target = cpumask_first(d->affinity); |
f571eff0 KK |
697 | |
698 | /* | |
699 | * We depend on the platform code to have correctly processed | |
700 | * IRQ affinity change requests to ensure that the IRQ affinity | |
701 | * mask has been purged of bits corresponding to nonexistent and | |
702 | * offline "CPUs", and to TCs bound to VPEs other than the VPE | |
703 | * connected to the physical interrupt input for the interrupt | |
704 | * in question. Otherwise we have a nasty problem with interrupt | |
705 | * mask management. This is best handled in non-performance-critical | |
706 | * platform IRQ affinity setting code, to minimize interrupt-time | |
707 | * checks. | |
708 | */ | |
709 | ||
710 | /* If no one is eligible, service locally */ | |
930cd54b | 711 | if (target >= NR_CPUS) |
f571eff0 | 712 | do_IRQ_no_affinity(irq); |
930cd54b TG |
713 | else |
714 | smtc_send_ipi(target, IRQ_AFFINITY_IPI, irq); | |
f571eff0 KK |
715 | } |
716 | ||
717 | #endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */ | |
718 | ||
41c594ab RB |
719 | /* |
720 | * IPI model for SMTC is tricky, because interrupts aren't TC-specific. | |
721 | * Within a VPE one TC can interrupt another by different approaches. | |
722 | * The easiest to get right would probably be to make all TCs except | |
723 | * the target IXMT and set a software interrupt, but an IXMT-based | |
724 | * scheme requires that a handler must run before a new IPI could | |
725 | * be sent, which would break the "broadcast" loops in MIPS MT. | |
726 | * A more gonzo approach within a VPE is to halt the TC, extract | |
727 | * its Restart, Status, and a couple of GPRs, and program the Restart | |
728 | * address to emulate an interrupt. | |
729 | * | |
730 | * Within a VPE, one can be confident that the target TC isn't in | |
731 | * a critical EXL state when halted, since the write to the Halt | |
732 | * register could not have issued on the writing thread if the | |
733 | * halting thread had EXL set. So k0 and k1 of the target TC | |
734 | * can be used by the injection code. Across VPEs, one can't | |
735 | * be certain that the target TC isn't in a critical exception | |
736 | * state. So we try a two-step process of sending a software | |
737 | * interrupt to the target VPE, which either handles the event | |
738 | * itself (if it was the target) or injects the event within | |
739 | * the VPE. | |
740 | */ | |
741 | ||
5868756d | 742 | static void smtc_ipi_qdump(void) |
41c594ab RB |
743 | { |
744 | int i; | |
2e41f91d | 745 | struct smtc_ipi *temp; |
41c594ab RB |
746 | |
747 | for (i = 0; i < NR_CPUS ;i++) { | |
2e41f91d | 748 | pr_info("IPIQ[%d]: head = 0x%x, tail = 0x%x, depth = %d\n", |
41c594ab RB |
749 | i, (unsigned)IPIQ[i].head, (unsigned)IPIQ[i].tail, |
750 | IPIQ[i].depth); | |
2e41f91d JP |
751 | temp = IPIQ[i].head; |
752 | ||
753 | while (temp != IPIQ[i].tail) { | |
754 | pr_debug("%d %d %d: ", temp->type, temp->dest, | |
755 | (int)temp->arg); | |
756 | #ifdef SMTC_IPI_DEBUG | |
757 | pr_debug("%u %lu\n", temp->sender, temp->stamp); | |
758 | #else | |
759 | pr_debug("\n"); | |
760 | #endif | |
761 | temp = temp->flink; | |
762 | } | |
41c594ab RB |
763 | } |
764 | } | |
765 | ||
766 | /* | |
767 | * The standard atomic.h primitives don't quite do what we want | |
768 | * here: We need an atomic add-and-return-previous-value (which | |
769 | * could be done with atomic_add_return and a decrement) and an | |
770 | * atomic set/zero-and-return-previous-value (which can't really | |
771 | * be done with the atomic.h primitives). And since this is | |
772 | * MIPS MT, we can assume that we have LL/SC. | |
773 | */ | |
ea580401 | 774 | static inline int atomic_postincrement(atomic_t *v) |
41c594ab RB |
775 | { |
776 | unsigned long result; | |
777 | ||
778 | unsigned long temp; | |
779 | ||
780 | __asm__ __volatile__( | |
781 | "1: ll %0, %2 \n" | |
782 | " addu %1, %0, 1 \n" | |
783 | " sc %1, %2 \n" | |
784 | " beqz %1, 1b \n" | |
d87d0c93 | 785 | __WEAK_LLSC_MB |
ea580401 RB |
786 | : "=&r" (result), "=&r" (temp), "=m" (v->counter) |
787 | : "m" (v->counter) | |
41c594ab RB |
788 | : "memory"); |
789 | ||
790 | return result; | |
791 | } | |
792 | ||
41c594ab RB |
793 | void smtc_send_ipi(int cpu, int type, unsigned int action) |
794 | { | |
795 | int tcstatus; | |
796 | struct smtc_ipi *pipi; | |
b7e4226e | 797 | unsigned long flags; |
41c594ab | 798 | int mtflags; |
8531a35e KK |
799 | unsigned long tcrestart; |
800 | extern void r4k_wait_irqoff(void), __pastwait(void); | |
2e41f91d JP |
801 | int set_resched_flag = (type == LINUX_SMP_IPI && |
802 | action == SMP_RESCHEDULE_YOURSELF); | |
41c594ab RB |
803 | |
804 | if (cpu == smp_processor_id()) { | |
805 | printk("Cannot Send IPI to self!\n"); | |
806 | return; | |
807 | } | |
2e41f91d JP |
808 | if (set_resched_flag && IPIQ[cpu].resched_flag != 0) |
809 | return; /* There is a reschedule queued already */ | |
810 | ||
41c594ab RB |
811 | /* Set up a descriptor, to be delivered either promptly or queued */ |
812 | pipi = smtc_ipi_dq(&freeIPIq); | |
813 | if (pipi == NULL) { | |
814 | bust_spinlocks(1); | |
815 | mips_mt_regdump(dvpe()); | |
816 | panic("IPI Msg. Buffers Depleted\n"); | |
817 | } | |
818 | pipi->type = type; | |
819 | pipi->arg = (void *)action; | |
820 | pipi->dest = cpu; | |
821 | if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { | |
603e82ed | 822 | /* If not on same VPE, enqueue and send cross-VPE interrupt */ |
2e41f91d | 823 | IPIQ[cpu].resched_flag |= set_resched_flag; |
41c594ab RB |
824 | smtc_ipi_nq(&IPIQ[cpu], pipi); |
825 | LOCK_CORE_PRA(); | |
826 | settc(cpu_data[cpu].tc_id); | |
827 | write_vpe_c0_cause(read_vpe_c0_cause() | C_SW1); | |
828 | UNLOCK_CORE_PRA(); | |
829 | } else { | |
830 | /* | |
831 | * Not sufficient to do a LOCK_MT_PRA (dmt) here, | |
832 | * since ASID shootdown on the other VPE may | |
833 | * collide with this operation. | |
834 | */ | |
835 | LOCK_CORE_PRA(); | |
836 | settc(cpu_data[cpu].tc_id); | |
837 | /* Halt the targeted TC */ | |
838 | write_tc_c0_tchalt(TCHALT_H); | |
839 | mips_ihb(); | |
840 | ||
841 | /* | |
842 | * Inspect TCStatus - if IXMT is set, we have to queue | |
843 | * a message. Otherwise, we set up the "interrupt" | |
844 | * of the other TC | |
845 | */ | |
846 | tcstatus = read_tc_c0_tcstatus(); | |
847 | ||
848 | if ((tcstatus & TCSTATUS_IXMT) != 0) { | |
849 | /* | |
8531a35e KK |
850 | * If we're in the the irq-off version of the wait |
851 | * loop, we need to force exit from the wait and | |
852 | * do a direct post of the IPI. | |
853 | */ | |
854 | if (cpu_wait == r4k_wait_irqoff) { | |
855 | tcrestart = read_tc_c0_tcrestart(); | |
856 | if (tcrestart >= (unsigned long)r4k_wait_irqoff | |
857 | && tcrestart < (unsigned long)__pastwait) { | |
858 | write_tc_c0_tcrestart(__pastwait); | |
859 | tcstatus &= ~TCSTATUS_IXMT; | |
860 | write_tc_c0_tcstatus(tcstatus); | |
861 | goto postdirect; | |
862 | } | |
863 | } | |
864 | /* | |
865 | * Otherwise we queue the message for the target TC | |
866 | * to pick up when he does a local_irq_restore() | |
41c594ab RB |
867 | */ |
868 | write_tc_c0_tchalt(0); | |
869 | UNLOCK_CORE_PRA(); | |
2e41f91d | 870 | IPIQ[cpu].resched_flag |= set_resched_flag; |
41c594ab RB |
871 | smtc_ipi_nq(&IPIQ[cpu], pipi); |
872 | } else { | |
8531a35e | 873 | postdirect: |
41c594ab RB |
874 | post_direct_ipi(cpu, pipi); |
875 | write_tc_c0_tchalt(0); | |
876 | UNLOCK_CORE_PRA(); | |
877 | } | |
878 | } | |
879 | } | |
880 | ||
881 | /* | |
882 | * Send IPI message to Halted TC, TargTC/TargVPE already having been set | |
883 | */ | |
5868756d | 884 | static void post_direct_ipi(int cpu, struct smtc_ipi *pipi) |
41c594ab RB |
885 | { |
886 | struct pt_regs *kstack; | |
887 | unsigned long tcstatus; | |
888 | unsigned long tcrestart; | |
889 | extern u32 kernelsp[NR_CPUS]; | |
890 | extern void __smtc_ipi_vector(void); | |
ea580401 | 891 | //printk("%s: on %d for %d\n", __func__, smp_processor_id(), cpu); |
41c594ab RB |
892 | |
893 | /* Extract Status, EPC from halted TC */ | |
894 | tcstatus = read_tc_c0_tcstatus(); | |
895 | tcrestart = read_tc_c0_tcrestart(); | |
896 | /* If TCRestart indicates a WAIT instruction, advance the PC */ | |
897 | if ((tcrestart & 0x80000000) | |
898 | && ((*(unsigned int *)tcrestart & 0xfe00003f) == 0x42000020)) { | |
899 | tcrestart += 4; | |
900 | } | |
901 | /* | |
902 | * Save on TC's future kernel stack | |
903 | * | |
904 | * CU bit of Status is indicator that TC was | |
905 | * already running on a kernel stack... | |
906 | */ | |
4bf42d42 | 907 | if (tcstatus & ST0_CU0) { |
41c594ab RB |
908 | /* Note that this "- 1" is pointer arithmetic */ |
909 | kstack = ((struct pt_regs *)read_tc_gpr_sp()) - 1; | |
910 | } else { | |
911 | kstack = ((struct pt_regs *)kernelsp[cpu]) - 1; | |
912 | } | |
913 | ||
914 | kstack->cp0_epc = (long)tcrestart; | |
915 | /* Save TCStatus */ | |
916 | kstack->cp0_tcstatus = tcstatus; | |
917 | /* Pass token of operation to be performed kernel stack pad area */ | |
918 | kstack->pad0[4] = (unsigned long)pipi; | |
919 | /* Pass address of function to be called likewise */ | |
920 | kstack->pad0[5] = (unsigned long)&ipi_decode; | |
921 | /* Set interrupt exempt and kernel mode */ | |
922 | tcstatus |= TCSTATUS_IXMT; | |
923 | tcstatus &= ~TCSTATUS_TKSU; | |
924 | write_tc_c0_tcstatus(tcstatus); | |
925 | ehb(); | |
926 | /* Set TC Restart address to be SMTC IPI vector */ | |
927 | write_tc_c0_tcrestart(__smtc_ipi_vector); | |
928 | } | |
929 | ||
937a8015 | 930 | static void ipi_resched_interrupt(void) |
41c594ab | 931 | { |
184748cc | 932 | scheduler_ipi(); |
41c594ab RB |
933 | } |
934 | ||
937a8015 | 935 | static void ipi_call_interrupt(void) |
41c594ab RB |
936 | { |
937 | /* Invoke generic function invocation code in smp.c */ | |
938 | smp_call_function_interrupt(); | |
939 | } | |
940 | ||
8531a35e | 941 | DECLARE_PER_CPU(struct clock_event_device, mips_clockevent_device); |
ea580401 | 942 | |
8f99a162 | 943 | static void __irq_entry smtc_clock_tick_interrupt(void) |
41c594ab | 944 | { |
ea580401 RB |
945 | unsigned int cpu = smp_processor_id(); |
946 | struct clock_event_device *cd; | |
8f99a162 WZ |
947 | int irq = MIPS_CPU_IRQ_BASE + 1; |
948 | ||
949 | irq_enter(); | |
950 | kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq)); | |
951 | cd = &per_cpu(mips_clockevent_device, cpu); | |
952 | cd->event_handler(cd); | |
953 | irq_exit(); | |
954 | } | |
955 | ||
956 | void ipi_decode(struct smtc_ipi *pipi) | |
957 | { | |
41c594ab RB |
958 | void *arg_copy = pipi->arg; |
959 | int type_copy = pipi->type; | |
d2287f5e | 960 | |
41c594ab | 961 | smtc_ipi_nq(&freeIPIq, pipi); |
dbc1d911 | 962 | |
41c594ab | 963 | switch (type_copy) { |
4bf42d42 | 964 | case SMTC_CLOCK_TICK: |
8f99a162 | 965 | smtc_clock_tick_interrupt(); |
4bf42d42 | 966 | break; |
ea580401 | 967 | |
4bf42d42 RB |
968 | case LINUX_SMP_IPI: |
969 | switch ((int)arg_copy) { | |
970 | case SMP_RESCHEDULE_YOURSELF: | |
937a8015 | 971 | ipi_resched_interrupt(); |
41c594ab | 972 | break; |
4bf42d42 | 973 | case SMP_CALL_FUNCTION: |
937a8015 | 974 | ipi_call_interrupt(); |
41c594ab RB |
975 | break; |
976 | default: | |
fa90c872 | 977 | printk("Impossible SMTC IPI Argument %p\n", arg_copy); |
41c594ab | 978 | break; |
4bf42d42 RB |
979 | } |
980 | break; | |
f571eff0 KK |
981 | #ifdef CONFIG_MIPS_MT_SMTC_IRQAFF |
982 | case IRQ_AFFINITY_IPI: | |
983 | /* | |
984 | * Accept a "forwarded" interrupt that was initially | |
985 | * taken by a TC who doesn't have affinity for the IRQ. | |
986 | */ | |
987 | do_IRQ_no_affinity((int)arg_copy); | |
988 | break; | |
989 | #endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */ | |
4bf42d42 RB |
990 | default: |
991 | printk("Impossible SMTC IPI Type 0x%x\n", type_copy); | |
992 | break; | |
41c594ab RB |
993 | } |
994 | } | |
995 | ||
8531a35e KK |
996 | /* |
997 | * Similar to smtc_ipi_replay(), but invoked from context restore, | |
998 | * so it reuses the current exception frame rather than set up a | |
999 | * new one with self_ipi. | |
1000 | */ | |
1001 | ||
937a8015 | 1002 | void deferred_smtc_ipi(void) |
41c594ab | 1003 | { |
8531a35e | 1004 | int cpu = smp_processor_id(); |
41c594ab RB |
1005 | |
1006 | /* | |
1007 | * Test is not atomic, but much faster than a dequeue, | |
1008 | * and the vast majority of invocations will have a null queue. | |
8531a35e KK |
1009 | * If irq_disabled when this was called, then any IPIs queued |
1010 | * after we test last will be taken on the next irq_enable/restore. | |
1011 | * If interrupts were enabled, then any IPIs added after the | |
1012 | * last test will be taken directly. | |
41c594ab | 1013 | */ |
8531a35e KK |
1014 | |
1015 | while (IPIQ[cpu].head != NULL) { | |
1016 | struct smtc_ipi_q *q = &IPIQ[cpu]; | |
1017 | struct smtc_ipi *pipi; | |
1018 | unsigned long flags; | |
1019 | ||
1020 | /* | |
1021 | * It may be possible we'll come in with interrupts | |
1022 | * already enabled. | |
1023 | */ | |
1024 | local_irq_save(flags); | |
8531a35e KK |
1025 | spin_lock(&q->lock); |
1026 | pipi = __smtc_ipi_dq(q); | |
1027 | spin_unlock(&q->lock); | |
2e41f91d JP |
1028 | if (pipi != NULL) { |
1029 | if (pipi->type == LINUX_SMP_IPI && | |
1030 | (int)pipi->arg == SMP_RESCHEDULE_YOURSELF) | |
1031 | IPIQ[cpu].resched_flag = 0; | |
937a8015 | 1032 | ipi_decode(pipi); |
2e41f91d | 1033 | } |
8531a35e KK |
1034 | /* |
1035 | * The use of the __raw_local restore isn't | |
1036 | * as obviously necessary here as in smtc_ipi_replay(), | |
1037 | * but it's more efficient, given that we're already | |
1038 | * running down the IPI queue. | |
1039 | */ | |
df9ee292 | 1040 | __arch_local_irq_restore(flags); |
41c594ab RB |
1041 | } |
1042 | } | |
1043 | ||
41c594ab RB |
1044 | /* |
1045 | * Cross-VPE interrupts in the SMTC prototype use "software interrupts" | |
1046 | * set via cross-VPE MTTR manipulation of the Cause register. It would be | |
1047 | * in some regards preferable to have external logic for "doorbell" hardware | |
1048 | * interrupts. | |
1049 | */ | |
1050 | ||
97dcb82d | 1051 | static int cpu_ipi_irq = MIPS_CPU_IRQ_BASE + MIPS_CPU_IPI_IRQ; |
41c594ab | 1052 | |
937a8015 | 1053 | static irqreturn_t ipi_interrupt(int irq, void *dev_idm) |
41c594ab RB |
1054 | { |
1055 | int my_vpe = cpu_data[smp_processor_id()].vpe_id; | |
1056 | int my_tc = cpu_data[smp_processor_id()].tc_id; | |
1057 | int cpu; | |
1058 | struct smtc_ipi *pipi; | |
1059 | unsigned long tcstatus; | |
1060 | int sent; | |
b7e4226e | 1061 | unsigned long flags; |
41c594ab RB |
1062 | unsigned int mtflags; |
1063 | unsigned int vpflags; | |
1064 | ||
1065 | /* | |
1066 | * So long as cross-VPE interrupts are done via | |
1067 | * MFTR/MTTR read-modify-writes of Cause, we need | |
1068 | * to stop other VPEs whenever the local VPE does | |
1069 | * anything similar. | |
1070 | */ | |
1071 | local_irq_save(flags); | |
1072 | vpflags = dvpe(); | |
1073 | clear_c0_cause(0x100 << MIPS_CPU_IPI_IRQ); | |
1074 | set_c0_status(0x100 << MIPS_CPU_IPI_IRQ); | |
1075 | irq_enable_hazard(); | |
1076 | evpe(vpflags); | |
1077 | local_irq_restore(flags); | |
1078 | ||
1079 | /* | |
1080 | * Cross-VPE Interrupt handler: Try to directly deliver IPIs | |
1081 | * queued for TCs on this VPE other than the current one. | |
1082 | * Return-from-interrupt should cause us to drain the queue | |
1083 | * for the current TC, so we ought not to have to do it explicitly here. | |
1084 | */ | |
1085 | ||
1086 | for_each_online_cpu(cpu) { | |
1087 | if (cpu_data[cpu].vpe_id != my_vpe) | |
1088 | continue; | |
1089 | ||
1090 | pipi = smtc_ipi_dq(&IPIQ[cpu]); | |
1091 | if (pipi != NULL) { | |
1092 | if (cpu_data[cpu].tc_id != my_tc) { | |
1093 | sent = 0; | |
1094 | LOCK_MT_PRA(); | |
1095 | settc(cpu_data[cpu].tc_id); | |
1096 | write_tc_c0_tchalt(TCHALT_H); | |
1097 | mips_ihb(); | |
1098 | tcstatus = read_tc_c0_tcstatus(); | |
1099 | if ((tcstatus & TCSTATUS_IXMT) == 0) { | |
1100 | post_direct_ipi(cpu, pipi); | |
1101 | sent = 1; | |
1102 | } | |
1103 | write_tc_c0_tchalt(0); | |
1104 | UNLOCK_MT_PRA(); | |
1105 | if (!sent) { | |
1106 | smtc_ipi_req(&IPIQ[cpu], pipi); | |
1107 | } | |
1108 | } else { | |
1109 | /* | |
1110 | * ipi_decode() should be called | |
1111 | * with interrupts off | |
1112 | */ | |
1113 | local_irq_save(flags); | |
2e41f91d JP |
1114 | if (pipi->type == LINUX_SMP_IPI && |
1115 | (int)pipi->arg == SMP_RESCHEDULE_YOURSELF) | |
1116 | IPIQ[cpu].resched_flag = 0; | |
937a8015 | 1117 | ipi_decode(pipi); |
41c594ab RB |
1118 | local_irq_restore(flags); |
1119 | } | |
1120 | } | |
1121 | } | |
1122 | ||
1123 | return IRQ_HANDLED; | |
1124 | } | |
1125 | ||
937a8015 | 1126 | static void ipi_irq_dispatch(void) |
41c594ab | 1127 | { |
937a8015 | 1128 | do_IRQ(cpu_ipi_irq); |
41c594ab RB |
1129 | } |
1130 | ||
033890b0 RB |
1131 | static struct irqaction irq_ipi = { |
1132 | .handler = ipi_interrupt, | |
b2651583 JL |
1133 | .flags = IRQF_DISABLED | IRQF_PERCPU, |
1134 | .name = "SMTC_IPI" | |
033890b0 | 1135 | }; |
41c594ab | 1136 | |
20bb25d1 | 1137 | static void setup_cross_vpe_interrupts(unsigned int nvpe) |
41c594ab | 1138 | { |
20bb25d1 RB |
1139 | if (nvpe < 1) |
1140 | return; | |
1141 | ||
41c594ab | 1142 | if (!cpu_has_vint) |
603e82ed | 1143 | panic("SMTC Kernel requires Vectored Interrupt support"); |
41c594ab RB |
1144 | |
1145 | set_vi_handler(MIPS_CPU_IPI_IRQ, ipi_irq_dispatch); | |
1146 | ||
41c594ab RB |
1147 | setup_irq_smtc(cpu_ipi_irq, &irq_ipi, (0x100 << MIPS_CPU_IPI_IRQ)); |
1148 | ||
e4ec7989 | 1149 | irq_set_handler(cpu_ipi_irq, handle_percpu_irq); |
41c594ab RB |
1150 | } |
1151 | ||
1152 | /* | |
1153 | * SMTC-specific hacks invoked from elsewhere in the kernel. | |
1154 | */ | |
1155 | ||
8531a35e KK |
1156 | /* |
1157 | * smtc_ipi_replay is called from raw_local_irq_restore | |
1158 | */ | |
1159 | ||
1160 | void smtc_ipi_replay(void) | |
ac8be955 | 1161 | { |
8a1e97ee RB |
1162 | unsigned int cpu = smp_processor_id(); |
1163 | ||
ac8be955 RB |
1164 | /* |
1165 | * To the extent that we've ever turned interrupts off, | |
1166 | * we may have accumulated deferred IPIs. This is subtle. | |
ac8be955 RB |
1167 | * we should be OK: If we pick up something and dispatch |
1168 | * it here, that's great. If we see nothing, but concurrent | |
1169 | * with this operation, another TC sends us an IPI, IXMT | |
1170 | * is clear, and we'll handle it as a real pseudo-interrupt | |
8531a35e KK |
1171 | * and not a pseudo-pseudo interrupt. The important thing |
1172 | * is to do the last check for queued message *after* the | |
1173 | * re-enabling of interrupts. | |
ac8be955 | 1174 | */ |
8531a35e KK |
1175 | while (IPIQ[cpu].head != NULL) { |
1176 | struct smtc_ipi_q *q = &IPIQ[cpu]; | |
1177 | struct smtc_ipi *pipi; | |
1178 | unsigned long flags; | |
ac8be955 | 1179 | |
8531a35e KK |
1180 | /* |
1181 | * It's just possible we'll come in with interrupts | |
1182 | * already enabled. | |
1183 | */ | |
1184 | local_irq_save(flags); | |
1185 | ||
1186 | spin_lock(&q->lock); | |
1187 | pipi = __smtc_ipi_dq(q); | |
1188 | spin_unlock(&q->lock); | |
1189 | /* | |
1190 | ** But use a raw restore here to avoid recursion. | |
1191 | */ | |
df9ee292 | 1192 | __arch_local_irq_restore(flags); |
8531a35e KK |
1193 | |
1194 | if (pipi) { | |
ac8be955 | 1195 | self_ipi(pipi); |
8a1e97ee | 1196 | smtc_cpu_stats[cpu].selfipis++; |
ac8be955 RB |
1197 | } |
1198 | } | |
1199 | } | |
1200 | ||
ec43c014 RB |
1201 | EXPORT_SYMBOL(smtc_ipi_replay); |
1202 | ||
41c594ab RB |
1203 | void smtc_idle_loop_hook(void) |
1204 | { | |
c68644d3 | 1205 | #ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG |
41c594ab RB |
1206 | int im; |
1207 | int flags; | |
1208 | int mtflags; | |
1209 | int bit; | |
1210 | int vpe; | |
1211 | int tc; | |
1212 | int hook_ntcs; | |
1213 | /* | |
1214 | * printk within DMT-protected regions can deadlock, | |
1215 | * so buffer diagnostic messages for later output. | |
1216 | */ | |
1217 | char *pdb_msg; | |
1218 | char id_ho_db_msg[768]; /* worst-case use should be less than 700 */ | |
1219 | ||
1220 | if (atomic_read(&idle_hook_initialized) == 0) { /* fast test */ | |
1221 | if (atomic_add_return(1, &idle_hook_initialized) == 1) { | |
1222 | int mvpconf0; | |
1223 | /* Tedious stuff to just do once */ | |
1224 | mvpconf0 = read_c0_mvpconf0(); | |
1225 | hook_ntcs = ((mvpconf0 & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; | |
1226 | if (hook_ntcs > NR_CPUS) | |
1227 | hook_ntcs = NR_CPUS; | |
1228 | for (tc = 0; tc < hook_ntcs; tc++) { | |
1229 | tcnoprog[tc] = 0; | |
1230 | clock_hang_reported[tc] = 0; | |
1231 | } | |
1232 | for (vpe = 0; vpe < 2; vpe++) | |
1233 | for (im = 0; im < 8; im++) | |
1234 | imstuckcount[vpe][im] = 0; | |
1235 | printk("Idle loop test hook initialized for %d TCs\n", hook_ntcs); | |
1236 | atomic_set(&idle_hook_initialized, 1000); | |
1237 | } else { | |
1238 | /* Someone else is initializing in parallel - let 'em finish */ | |
1239 | while (atomic_read(&idle_hook_initialized) < 1000) | |
1240 | ; | |
1241 | } | |
1242 | } | |
1243 | ||
1244 | /* Have we stupidly left IXMT set somewhere? */ | |
1245 | if (read_c0_tcstatus() & 0x400) { | |
1246 | write_c0_tcstatus(read_c0_tcstatus() & ~0x400); | |
1247 | ehb(); | |
1248 | printk("Dangling IXMT in cpu_idle()\n"); | |
1249 | } | |
1250 | ||
1251 | /* Have we stupidly left an IM bit turned off? */ | |
1252 | #define IM_LIMIT 2000 | |
1253 | local_irq_save(flags); | |
1254 | mtflags = dmt(); | |
1255 | pdb_msg = &id_ho_db_msg[0]; | |
1256 | im = read_c0_status(); | |
8f8771a0 | 1257 | vpe = current_cpu_data.vpe_id; |
41c594ab RB |
1258 | for (bit = 0; bit < 8; bit++) { |
1259 | /* | |
1260 | * In current prototype, I/O interrupts | |
1261 | * are masked for VPE > 0 | |
1262 | */ | |
1263 | if (vpemask[vpe][bit]) { | |
1264 | if (!(im & (0x100 << bit))) | |
1265 | imstuckcount[vpe][bit]++; | |
1266 | else | |
1267 | imstuckcount[vpe][bit] = 0; | |
1268 | if (imstuckcount[vpe][bit] > IM_LIMIT) { | |
1269 | set_c0_status(0x100 << bit); | |
1270 | ehb(); | |
1271 | imstuckcount[vpe][bit] = 0; | |
1272 | pdb_msg += sprintf(pdb_msg, | |
1273 | "Dangling IM %d fixed for VPE %d\n", bit, | |
1274 | vpe); | |
1275 | } | |
1276 | } | |
1277 | } | |
1278 | ||
41c594ab RB |
1279 | emt(mtflags); |
1280 | local_irq_restore(flags); | |
1281 | if (pdb_msg != &id_ho_db_msg[0]) | |
1282 | printk("CPU%d: %s", smp_processor_id(), id_ho_db_msg); | |
c68644d3 | 1283 | #endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */ |
ac8be955 | 1284 | |
8531a35e | 1285 | smtc_ipi_replay(); |
41c594ab RB |
1286 | } |
1287 | ||
1288 | void smtc_soft_dump(void) | |
1289 | { | |
1290 | int i; | |
1291 | ||
1292 | printk("Counter Interrupts taken per CPU (TC)\n"); | |
1293 | for (i=0; i < NR_CPUS; i++) { | |
1294 | printk("%d: %ld\n", i, smtc_cpu_stats[i].timerints); | |
1295 | } | |
1296 | printk("Self-IPI invocations:\n"); | |
1297 | for (i=0; i < NR_CPUS; i++) { | |
1298 | printk("%d: %ld\n", i, smtc_cpu_stats[i].selfipis); | |
1299 | } | |
1300 | smtc_ipi_qdump(); | |
41c594ab RB |
1301 | printk("%d Recoveries of \"stolen\" FPU\n", |
1302 | atomic_read(&smtc_fpu_recoveries)); | |
1303 | } | |
1304 | ||
1305 | ||
1306 | /* | |
1307 | * TLB management routines special to SMTC | |
1308 | */ | |
1309 | ||
1310 | void smtc_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu) | |
1311 | { | |
1312 | unsigned long flags, mtflags, tcstat, prevhalt, asid; | |
1313 | int tlb, i; | |
1314 | ||
1315 | /* | |
1316 | * It would be nice to be able to use a spinlock here, | |
1317 | * but this is invoked from within TLB flush routines | |
1318 | * that protect themselves with DVPE, so if a lock is | |
e0daad44 | 1319 | * held by another TC, it'll never be freed. |
41c594ab RB |
1320 | * |
1321 | * DVPE/DMT must not be done with interrupts enabled, | |
1322 | * so even so most callers will already have disabled | |
1323 | * them, let's be really careful... | |
1324 | */ | |
1325 | ||
1326 | local_irq_save(flags); | |
1327 | if (smtc_status & SMTC_TLB_SHARED) { | |
1328 | mtflags = dvpe(); | |
1329 | tlb = 0; | |
1330 | } else { | |
1331 | mtflags = dmt(); | |
1332 | tlb = cpu_data[cpu].vpe_id; | |
1333 | } | |
1334 | asid = asid_cache(cpu); | |
1335 | ||
1336 | do { | |
1337 | if (!((asid += ASID_INC) & ASID_MASK) ) { | |
1338 | if (cpu_has_vtag_icache) | |
1339 | flush_icache_all(); | |
af901ca1 | 1340 | /* Traverse all online CPUs (hack requires contiguous range) */ |
b5eb5511 | 1341 | for_each_online_cpu(i) { |
41c594ab RB |
1342 | /* |
1343 | * We don't need to worry about our own CPU, nor those of | |
1344 | * CPUs who don't share our TLB. | |
1345 | */ | |
1346 | if ((i != smp_processor_id()) && | |
1347 | ((smtc_status & SMTC_TLB_SHARED) || | |
1348 | (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))) { | |
1349 | settc(cpu_data[i].tc_id); | |
1350 | prevhalt = read_tc_c0_tchalt() & TCHALT_H; | |
1351 | if (!prevhalt) { | |
1352 | write_tc_c0_tchalt(TCHALT_H); | |
1353 | mips_ihb(); | |
1354 | } | |
1355 | tcstat = read_tc_c0_tcstatus(); | |
1356 | smtc_live_asid[tlb][(tcstat & ASID_MASK)] |= (asiduse)(0x1 << i); | |
1357 | if (!prevhalt) | |
1358 | write_tc_c0_tchalt(0); | |
1359 | } | |
1360 | } | |
1361 | if (!asid) /* fix version if needed */ | |
1362 | asid = ASID_FIRST_VERSION; | |
1363 | local_flush_tlb_all(); /* start new asid cycle */ | |
1364 | } | |
1365 | } while (smtc_live_asid[tlb][(asid & ASID_MASK)]); | |
1366 | ||
1367 | /* | |
1368 | * SMTC shares the TLB within VPEs and possibly across all VPEs. | |
1369 | */ | |
b5eb5511 | 1370 | for_each_online_cpu(i) { |
41c594ab RB |
1371 | if ((smtc_status & SMTC_TLB_SHARED) || |
1372 | (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id)) | |
1373 | cpu_context(i, mm) = asid_cache(i) = asid; | |
1374 | } | |
1375 | ||
1376 | if (smtc_status & SMTC_TLB_SHARED) | |
1377 | evpe(mtflags); | |
1378 | else | |
1379 | emt(mtflags); | |
1380 | local_irq_restore(flags); | |
1381 | } | |
1382 | ||
1383 | /* | |
1384 | * Invoked from macros defined in mmu_context.h | |
1385 | * which must already have disabled interrupts | |
1386 | * and done a DVPE or DMT as appropriate. | |
1387 | */ | |
1388 | ||
1389 | void smtc_flush_tlb_asid(unsigned long asid) | |
1390 | { | |
1391 | int entry; | |
1392 | unsigned long ehi; | |
1393 | ||
1394 | entry = read_c0_wired(); | |
1395 | ||
1396 | /* Traverse all non-wired entries */ | |
1397 | while (entry < current_cpu_data.tlbsize) { | |
1398 | write_c0_index(entry); | |
1399 | ehb(); | |
1400 | tlb_read(); | |
1401 | ehb(); | |
1402 | ehi = read_c0_entryhi(); | |
4bf42d42 | 1403 | if ((ehi & ASID_MASK) == asid) { |
41c594ab RB |
1404 | /* |
1405 | * Invalidate only entries with specified ASID, | |
1406 | * makiing sure all entries differ. | |
1407 | */ | |
1408 | write_c0_entryhi(CKSEG0 + (entry << (PAGE_SHIFT + 1))); | |
1409 | write_c0_entrylo0(0); | |
1410 | write_c0_entrylo1(0); | |
1411 | mtc0_tlbw_hazard(); | |
1412 | tlb_write_indexed(); | |
1413 | } | |
1414 | entry++; | |
1415 | } | |
1416 | write_c0_index(PARKED_INDEX); | |
1417 | tlbw_use_hazard(); | |
1418 | } | |
1419 | ||
1420 | /* | |
1421 | * Support for single-threading cache flush operations. | |
1422 | */ | |
1423 | ||
5868756d | 1424 | static int halt_state_save[NR_CPUS]; |
41c594ab RB |
1425 | |
1426 | /* | |
1427 | * To really, really be sure that nothing is being done | |
1428 | * by other TCs, halt them all. This code assumes that | |
1429 | * a DVPE has already been done, so while their Halted | |
1430 | * state is theoretically architecturally unstable, in | |
1431 | * practice, it's not going to change while we're looking | |
1432 | * at it. | |
1433 | */ | |
1434 | ||
1435 | void smtc_cflush_lockdown(void) | |
1436 | { | |
1437 | int cpu; | |
1438 | ||
1439 | for_each_online_cpu(cpu) { | |
1440 | if (cpu != smp_processor_id()) { | |
1441 | settc(cpu_data[cpu].tc_id); | |
1442 | halt_state_save[cpu] = read_tc_c0_tchalt(); | |
1443 | write_tc_c0_tchalt(TCHALT_H); | |
1444 | } | |
1445 | } | |
1446 | mips_ihb(); | |
1447 | } | |
1448 | ||
1449 | /* It would be cheating to change the cpu_online states during a flush! */ | |
1450 | ||
1451 | void smtc_cflush_release(void) | |
1452 | { | |
1453 | int cpu; | |
1454 | ||
1455 | /* | |
1456 | * Start with a hazard barrier to ensure | |
1457 | * that all CACHE ops have played through. | |
1458 | */ | |
1459 | mips_ihb(); | |
1460 | ||
1461 | for_each_online_cpu(cpu) { | |
1462 | if (cpu != smp_processor_id()) { | |
1463 | settc(cpu_data[cpu].tc_id); | |
1464 | write_tc_c0_tchalt(halt_state_save[cpu]); | |
1465 | } | |
1466 | } | |
1467 | mips_ihb(); | |
1468 | } |