Commit | Line | Data |
---|---|---|
41c594ab RB |
1 | /* Copyright (C) 2004 Mips Technologies, Inc */ |
2 | ||
3 | #include <linux/kernel.h> | |
4 | #include <linux/sched.h> | |
5 | #include <linux/cpumask.h> | |
6 | #include <linux/interrupt.h> | |
7 | ||
8 | #include <asm/cpu.h> | |
9 | #include <asm/processor.h> | |
10 | #include <asm/atomic.h> | |
11 | #include <asm/system.h> | |
12 | #include <asm/hardirq.h> | |
13 | #include <asm/hazards.h> | |
14 | #include <asm/mmu_context.h> | |
15 | #include <asm/smp.h> | |
16 | #include <asm/mipsregs.h> | |
17 | #include <asm/cacheflush.h> | |
18 | #include <asm/time.h> | |
19 | #include <asm/addrspace.h> | |
20 | #include <asm/smtc.h> | |
21 | #include <asm/smtc_ipi.h> | |
22 | #include <asm/smtc_proc.h> | |
23 | ||
24 | /* | |
25 | * This file should be built into the kernel only if CONFIG_MIPS_MT_SMTC is set. | |
26 | */ | |
27 | ||
28 | /* | |
29 | * MIPSCPU_INT_BASE is identically defined in both | |
30 | * asm-mips/mips-boards/maltaint.h and asm-mips/mips-boards/simint.h, | |
31 | * but as yet there's no properly organized include structure that | |
32 | * will ensure that the right *int.h file will be included for a | |
33 | * given platform build. | |
34 | */ | |
35 | ||
36 | #define MIPSCPU_INT_BASE 16 | |
37 | ||
38 | #define MIPS_CPU_IPI_IRQ 1 | |
39 | ||
40 | #define LOCK_MT_PRA() \ | |
41 | local_irq_save(flags); \ | |
42 | mtflags = dmt() | |
43 | ||
44 | #define UNLOCK_MT_PRA() \ | |
45 | emt(mtflags); \ | |
46 | local_irq_restore(flags) | |
47 | ||
48 | #define LOCK_CORE_PRA() \ | |
49 | local_irq_save(flags); \ | |
50 | mtflags = dvpe() | |
51 | ||
52 | #define UNLOCK_CORE_PRA() \ | |
53 | evpe(mtflags); \ | |
54 | local_irq_restore(flags) | |
55 | ||
56 | /* | |
57 | * Data structures purely associated with SMTC parallelism | |
58 | */ | |
59 | ||
60 | ||
61 | /* | |
62 | * Table for tracking ASIDs whose lifetime is prolonged. | |
63 | */ | |
64 | ||
65 | asiduse smtc_live_asid[MAX_SMTC_TLBS][MAX_SMTC_ASIDS]; | |
66 | ||
67 | /* | |
68 | * Clock interrupt "latch" buffers, per "CPU" | |
69 | */ | |
70 | ||
71 | unsigned int ipi_timer_latch[NR_CPUS]; | |
72 | ||
73 | /* | |
74 | * Number of InterProcessor Interupt (IPI) message buffers to allocate | |
75 | */ | |
76 | ||
77 | #define IPIBUF_PER_CPU 4 | |
78 | ||
79 | struct smtc_ipi_q IPIQ[NR_CPUS]; | |
80 | struct smtc_ipi_q freeIPIq; | |
81 | ||
82 | ||
83 | /* Forward declarations */ | |
84 | ||
85 | void ipi_decode(struct pt_regs *, struct smtc_ipi *); | |
86 | void post_direct_ipi(int cpu, struct smtc_ipi *pipi); | |
87 | void setup_cross_vpe_interrupts(void); | |
88 | void init_smtc_stats(void); | |
89 | ||
90 | /* Global SMTC Status */ | |
91 | ||
92 | unsigned int smtc_status = 0; | |
93 | ||
94 | /* Boot command line configuration overrides */ | |
95 | ||
96 | static int vpelimit = 0; | |
97 | static int tclimit = 0; | |
98 | static int ipibuffers = 0; | |
99 | static int nostlb = 0; | |
100 | static int asidmask = 0; | |
101 | unsigned long smtc_asid_mask = 0xff; | |
102 | ||
103 | static int __init maxvpes(char *str) | |
104 | { | |
105 | get_option(&str, &vpelimit); | |
106 | return 1; | |
107 | } | |
108 | ||
109 | static int __init maxtcs(char *str) | |
110 | { | |
111 | get_option(&str, &tclimit); | |
112 | return 1; | |
113 | } | |
114 | ||
115 | static int __init ipibufs(char *str) | |
116 | { | |
117 | get_option(&str, &ipibuffers); | |
118 | return 1; | |
119 | } | |
120 | ||
121 | static int __init stlb_disable(char *s) | |
122 | { | |
123 | nostlb = 1; | |
124 | return 1; | |
125 | } | |
126 | ||
127 | static int __init asidmask_set(char *str) | |
128 | { | |
129 | get_option(&str, &asidmask); | |
130 | switch(asidmask) { | |
131 | case 0x1: | |
132 | case 0x3: | |
133 | case 0x7: | |
134 | case 0xf: | |
135 | case 0x1f: | |
136 | case 0x3f: | |
137 | case 0x7f: | |
138 | case 0xff: | |
139 | smtc_asid_mask = (unsigned long)asidmask; | |
140 | break; | |
141 | default: | |
142 | printk("ILLEGAL ASID mask 0x%x from command line\n", asidmask); | |
143 | } | |
144 | return 1; | |
145 | } | |
146 | ||
147 | __setup("maxvpes=", maxvpes); | |
148 | __setup("maxtcs=", maxtcs); | |
149 | __setup("ipibufs=", ipibufs); | |
150 | __setup("nostlb", stlb_disable); | |
151 | __setup("asidmask=", asidmask_set); | |
152 | ||
153 | /* Enable additional debug checks before going into CPU idle loop */ | |
154 | #define SMTC_IDLE_HOOK_DEBUG | |
155 | ||
156 | #ifdef SMTC_IDLE_HOOK_DEBUG | |
157 | ||
158 | static int hang_trig = 0; | |
159 | ||
160 | static int __init hangtrig_enable(char *s) | |
161 | { | |
162 | hang_trig = 1; | |
163 | return 1; | |
164 | } | |
165 | ||
166 | ||
167 | __setup("hangtrig", hangtrig_enable); | |
168 | ||
169 | #define DEFAULT_BLOCKED_IPI_LIMIT 32 | |
170 | ||
171 | static int timerq_limit = DEFAULT_BLOCKED_IPI_LIMIT; | |
172 | ||
173 | static int __init tintq(char *str) | |
174 | { | |
175 | get_option(&str, &timerq_limit); | |
176 | return 1; | |
177 | } | |
178 | ||
179 | __setup("tintq=", tintq); | |
180 | ||
181 | int imstuckcount[2][8]; | |
182 | /* vpemask represents IM/IE bits of per-VPE Status registers, low-to-high */ | |
183 | int vpemask[2][8] = {{0,1,1,0,0,0,0,1},{0,1,0,0,0,0,0,1}}; | |
184 | int tcnoprog[NR_CPUS]; | |
185 | static atomic_t idle_hook_initialized = {0}; | |
186 | static int clock_hang_reported[NR_CPUS]; | |
187 | ||
188 | #endif /* SMTC_IDLE_HOOK_DEBUG */ | |
189 | ||
190 | /* Initialize shared TLB - the should probably migrate to smtc_setup_cpus() */ | |
191 | ||
192 | void __init sanitize_tlb_entries(void) | |
193 | { | |
194 | printk("Deprecated sanitize_tlb_entries() invoked\n"); | |
195 | } | |
196 | ||
197 | ||
198 | /* | |
199 | * Configure shared TLB - VPC configuration bit must be set by caller | |
200 | */ | |
201 | ||
202 | void smtc_configure_tlb(void) | |
203 | { | |
204 | int i,tlbsiz,vpes; | |
205 | unsigned long mvpconf0; | |
206 | unsigned long config1val; | |
207 | ||
208 | /* Set up ASID preservation table */ | |
209 | for (vpes=0; vpes<MAX_SMTC_TLBS; vpes++) { | |
210 | for(i = 0; i < MAX_SMTC_ASIDS; i++) { | |
211 | smtc_live_asid[vpes][i] = 0; | |
212 | } | |
213 | } | |
214 | mvpconf0 = read_c0_mvpconf0(); | |
215 | ||
216 | if ((vpes = ((mvpconf0 & MVPCONF0_PVPE) | |
217 | >> MVPCONF0_PVPE_SHIFT) + 1) > 1) { | |
218 | /* If we have multiple VPEs, try to share the TLB */ | |
219 | if ((mvpconf0 & MVPCONF0_TLBS) && !nostlb) { | |
220 | /* | |
221 | * If TLB sizing is programmable, shared TLB | |
222 | * size is the total available complement. | |
223 | * Otherwise, we have to take the sum of all | |
224 | * static VPE TLB entries. | |
225 | */ | |
226 | if ((tlbsiz = ((mvpconf0 & MVPCONF0_PTLBE) | |
227 | >> MVPCONF0_PTLBE_SHIFT)) == 0) { | |
228 | /* | |
229 | * If there's more than one VPE, there had better | |
230 | * be more than one TC, because we need one to bind | |
231 | * to each VPE in turn to be able to read | |
232 | * its configuration state! | |
233 | */ | |
234 | settc(1); | |
235 | /* Stop the TC from doing anything foolish */ | |
236 | write_tc_c0_tchalt(TCHALT_H); | |
237 | mips_ihb(); | |
238 | /* No need to un-Halt - that happens later anyway */ | |
239 | for (i=0; i < vpes; i++) { | |
240 | write_tc_c0_tcbind(i); | |
241 | /* | |
242 | * To be 100% sure we're really getting the right | |
243 | * information, we exit the configuration state | |
244 | * and do an IHB after each rebinding. | |
245 | */ | |
246 | write_c0_mvpcontrol( | |
247 | read_c0_mvpcontrol() & ~ MVPCONTROL_VPC ); | |
248 | mips_ihb(); | |
249 | /* | |
250 | * Only count if the MMU Type indicated is TLB | |
251 | */ | |
252 | if(((read_vpe_c0_config() & MIPS_CONF_MT) >> 7) == 1) { | |
253 | config1val = read_vpe_c0_config1(); | |
254 | tlbsiz += ((config1val >> 25) & 0x3f) + 1; | |
255 | } | |
256 | ||
257 | /* Put core back in configuration state */ | |
258 | write_c0_mvpcontrol( | |
259 | read_c0_mvpcontrol() | MVPCONTROL_VPC ); | |
260 | mips_ihb(); | |
261 | } | |
262 | } | |
263 | write_c0_mvpcontrol(read_c0_mvpcontrol() | MVPCONTROL_STLB); | |
264 | ||
265 | /* | |
266 | * Setup kernel data structures to use software total, | |
267 | * rather than read the per-VPE Config1 value. The values | |
268 | * for "CPU 0" gets copied to all the other CPUs as part | |
269 | * of their initialization in smtc_cpu_setup(). | |
270 | */ | |
271 | ||
272 | tlbsiz = tlbsiz & 0x3f; /* MIPS32 limits TLB indices to 64 */ | |
273 | cpu_data[0].tlbsize = tlbsiz; | |
274 | smtc_status |= SMTC_TLB_SHARED; | |
275 | ||
276 | printk("TLB of %d entry pairs shared by %d VPEs\n", | |
277 | tlbsiz, vpes); | |
278 | } else { | |
279 | printk("WARNING: TLB Not Sharable on SMTC Boot!\n"); | |
280 | } | |
281 | } | |
282 | } | |
283 | ||
284 | ||
285 | /* | |
286 | * Incrementally build the CPU map out of constituent MIPS MT cores, | |
287 | * using the specified available VPEs and TCs. Plaform code needs | |
288 | * to ensure that each MIPS MT core invokes this routine on reset, | |
289 | * one at a time(!). | |
290 | * | |
291 | * This version of the build_cpu_map and prepare_cpus routines assumes | |
292 | * that *all* TCs of a MIPS MT core will be used for Linux, and that | |
293 | * they will be spread across *all* available VPEs (to minimise the | |
294 | * loss of efficiency due to exception service serialization). | |
295 | * An improved version would pick up configuration information and | |
296 | * possibly leave some TCs/VPEs as "slave" processors. | |
297 | * | |
298 | * Use c0_MVPConf0 to find out how many TCs are available, setting up | |
299 | * phys_cpu_present_map and the logical/physical mappings. | |
300 | */ | |
301 | ||
302 | int __init mipsmt_build_cpu_map(int start_cpu_slot) | |
303 | { | |
304 | int i, ntcs; | |
305 | ||
306 | /* | |
307 | * The CPU map isn't actually used for anything at this point, | |
308 | * so it's not clear what else we should do apart from set | |
309 | * everything up so that "logical" = "physical". | |
310 | */ | |
311 | ntcs = ((read_c0_mvpconf0() & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; | |
312 | for (i=start_cpu_slot; i<NR_CPUS && i<ntcs; i++) { | |
313 | cpu_set(i, phys_cpu_present_map); | |
314 | __cpu_number_map[i] = i; | |
315 | __cpu_logical_map[i] = i; | |
316 | } | |
317 | /* Initialize map of CPUs with FPUs */ | |
318 | cpus_clear(mt_fpu_cpumask); | |
319 | ||
320 | /* One of those TC's is the one booting, and not a secondary... */ | |
321 | printk("%i available secondary CPU TC(s)\n", i - 1); | |
322 | ||
323 | return i; | |
324 | } | |
325 | ||
326 | /* | |
327 | * Common setup before any secondaries are started | |
328 | * Make sure all CPU's are in a sensible state before we boot any of the | |
329 | * secondaries. | |
330 | * | |
331 | * For MIPS MT "SMTC" operation, we set up all TCs, spread as evenly | |
332 | * as possible across the available VPEs. | |
333 | */ | |
334 | ||
335 | static void smtc_tc_setup(int vpe, int tc, int cpu) | |
336 | { | |
337 | settc(tc); | |
338 | write_tc_c0_tchalt(TCHALT_H); | |
339 | mips_ihb(); | |
340 | write_tc_c0_tcstatus((read_tc_c0_tcstatus() | |
341 | & ~(TCSTATUS_TKSU | TCSTATUS_DA | TCSTATUS_IXMT)) | |
342 | | TCSTATUS_A); | |
343 | write_tc_c0_tccontext(0); | |
344 | /* Bind tc to vpe */ | |
345 | write_tc_c0_tcbind(vpe); | |
346 | /* In general, all TCs should have the same cpu_data indications */ | |
347 | memcpy(&cpu_data[cpu], &cpu_data[0], sizeof(struct cpuinfo_mips)); | |
348 | /* For 34Kf, start with TC/CPU 0 as sole owner of single FPU context */ | |
349 | if (cpu_data[0].cputype == CPU_34K) | |
350 | cpu_data[cpu].options &= ~MIPS_CPU_FPU; | |
351 | cpu_data[cpu].vpe_id = vpe; | |
352 | cpu_data[cpu].tc_id = tc; | |
353 | } | |
354 | ||
355 | ||
356 | void mipsmt_prepare_cpus(void) | |
357 | { | |
358 | int i, vpe, tc, ntc, nvpe, tcpervpe, slop, cpu; | |
359 | unsigned long flags; | |
360 | unsigned long val; | |
361 | int nipi; | |
362 | struct smtc_ipi *pipi; | |
363 | ||
364 | /* disable interrupts so we can disable MT */ | |
365 | local_irq_save(flags); | |
366 | /* disable MT so we can configure */ | |
367 | dvpe(); | |
368 | dmt(); | |
369 | ||
34af946a | 370 | spin_lock_init(&freeIPIq.lock); |
41c594ab RB |
371 | |
372 | /* | |
373 | * We probably don't have as many VPEs as we do SMP "CPUs", | |
374 | * but it's possible - and in any case we'll never use more! | |
375 | */ | |
376 | for (i=0; i<NR_CPUS; i++) { | |
377 | IPIQ[i].head = IPIQ[i].tail = NULL; | |
34af946a | 378 | spin_lock_init(&IPIQ[i].lock); |
41c594ab RB |
379 | IPIQ[i].depth = 0; |
380 | ipi_timer_latch[i] = 0; | |
381 | } | |
382 | ||
383 | /* cpu_data index starts at zero */ | |
384 | cpu = 0; | |
385 | cpu_data[cpu].vpe_id = 0; | |
386 | cpu_data[cpu].tc_id = 0; | |
387 | cpu++; | |
388 | ||
389 | /* Report on boot-time options */ | |
390 | mips_mt_set_cpuoptions (); | |
391 | if (vpelimit > 0) | |
392 | printk("Limit of %d VPEs set\n", vpelimit); | |
393 | if (tclimit > 0) | |
394 | printk("Limit of %d TCs set\n", tclimit); | |
395 | if (nostlb) { | |
396 | printk("Shared TLB Use Inhibited - UNSAFE for Multi-VPE Operation\n"); | |
397 | } | |
398 | if (asidmask) | |
399 | printk("ASID mask value override to 0x%x\n", asidmask); | |
400 | ||
401 | /* Temporary */ | |
402 | #ifdef SMTC_IDLE_HOOK_DEBUG | |
403 | if (hang_trig) | |
404 | printk("Logic Analyser Trigger on suspected TC hang\n"); | |
405 | #endif /* SMTC_IDLE_HOOK_DEBUG */ | |
406 | ||
407 | /* Put MVPE's into 'configuration state' */ | |
408 | write_c0_mvpcontrol( read_c0_mvpcontrol() | MVPCONTROL_VPC ); | |
409 | ||
410 | val = read_c0_mvpconf0(); | |
411 | nvpe = ((val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1; | |
412 | if (vpelimit > 0 && nvpe > vpelimit) | |
413 | nvpe = vpelimit; | |
414 | ntc = ((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; | |
415 | if (ntc > NR_CPUS) | |
416 | ntc = NR_CPUS; | |
417 | if (tclimit > 0 && ntc > tclimit) | |
418 | ntc = tclimit; | |
419 | tcpervpe = ntc / nvpe; | |
420 | slop = ntc % nvpe; /* Residual TCs, < NVPE */ | |
421 | ||
422 | /* Set up shared TLB */ | |
423 | smtc_configure_tlb(); | |
424 | ||
425 | for (tc = 0, vpe = 0 ; (vpe < nvpe) && (tc < ntc) ; vpe++) { | |
426 | /* | |
427 | * Set the MVP bits. | |
428 | */ | |
429 | settc(tc); | |
430 | write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_MVP); | |
431 | if (vpe != 0) | |
432 | printk(", "); | |
433 | printk("VPE %d: TC", vpe); | |
434 | for (i = 0; i < tcpervpe; i++) { | |
435 | /* | |
436 | * TC 0 is bound to VPE 0 at reset, | |
437 | * and is presumably executing this | |
438 | * code. Leave it alone! | |
439 | */ | |
440 | if (tc != 0) { | |
441 | smtc_tc_setup(vpe,tc, cpu); | |
442 | cpu++; | |
443 | } | |
444 | printk(" %d", tc); | |
445 | tc++; | |
446 | } | |
447 | if (slop) { | |
448 | if (tc != 0) { | |
449 | smtc_tc_setup(vpe,tc, cpu); | |
450 | cpu++; | |
451 | } | |
452 | printk(" %d", tc); | |
453 | tc++; | |
454 | slop--; | |
455 | } | |
456 | if (vpe != 0) { | |
457 | /* | |
458 | * Clear any stale software interrupts from VPE's Cause | |
459 | */ | |
460 | write_vpe_c0_cause(0); | |
461 | ||
462 | /* | |
463 | * Clear ERL/EXL of VPEs other than 0 | |
464 | * and set restricted interrupt enable/mask. | |
465 | */ | |
466 | write_vpe_c0_status((read_vpe_c0_status() | |
467 | & ~(ST0_BEV | ST0_ERL | ST0_EXL | ST0_IM)) | |
468 | | (STATUSF_IP0 | STATUSF_IP1 | STATUSF_IP7 | |
469 | | ST0_IE)); | |
470 | /* | |
471 | * set config to be the same as vpe0, | |
472 | * particularly kseg0 coherency alg | |
473 | */ | |
474 | write_vpe_c0_config(read_c0_config()); | |
475 | /* Clear any pending timer interrupt */ | |
476 | write_vpe_c0_compare(0); | |
477 | /* Propagate Config7 */ | |
478 | write_vpe_c0_config7(read_c0_config7()); | |
479 | } | |
480 | /* enable multi-threading within VPE */ | |
481 | write_vpe_c0_vpecontrol(read_vpe_c0_vpecontrol() | VPECONTROL_TE); | |
482 | /* enable the VPE */ | |
483 | write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_VPA); | |
484 | } | |
485 | ||
486 | /* | |
487 | * Pull any physically present but unused TCs out of circulation. | |
488 | */ | |
489 | while (tc < (((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1)) { | |
490 | cpu_clear(tc, phys_cpu_present_map); | |
491 | cpu_clear(tc, cpu_present_map); | |
492 | tc++; | |
493 | } | |
494 | ||
495 | /* release config state */ | |
496 | write_c0_mvpcontrol( read_c0_mvpcontrol() & ~ MVPCONTROL_VPC ); | |
497 | ||
498 | printk("\n"); | |
499 | ||
500 | /* Set up coprocessor affinity CPU mask(s) */ | |
501 | ||
502 | for (tc = 0; tc < ntc; tc++) { | |
503 | if(cpu_data[tc].options & MIPS_CPU_FPU) | |
504 | cpu_set(tc, mt_fpu_cpumask); | |
505 | } | |
506 | ||
507 | /* set up ipi interrupts... */ | |
508 | ||
509 | /* If we have multiple VPEs running, set up the cross-VPE interrupt */ | |
510 | ||
511 | if (nvpe > 1) | |
512 | setup_cross_vpe_interrupts(); | |
513 | ||
514 | /* Set up queue of free IPI "messages". */ | |
515 | nipi = NR_CPUS * IPIBUF_PER_CPU; | |
516 | if (ipibuffers > 0) | |
517 | nipi = ipibuffers; | |
518 | ||
519 | pipi = kmalloc(nipi *sizeof(struct smtc_ipi), GFP_KERNEL); | |
520 | if (pipi == NULL) | |
521 | panic("kmalloc of IPI message buffers failed\n"); | |
522 | else | |
523 | printk("IPI buffer pool of %d buffers\n", nipi); | |
524 | for (i = 0; i < nipi; i++) { | |
525 | smtc_ipi_nq(&freeIPIq, pipi); | |
526 | pipi++; | |
527 | } | |
528 | ||
529 | /* Arm multithreading and enable other VPEs - but all TCs are Halted */ | |
530 | emt(EMT_ENABLE); | |
531 | evpe(EVPE_ENABLE); | |
532 | local_irq_restore(flags); | |
533 | /* Initialize SMTC /proc statistics/diagnostics */ | |
534 | init_smtc_stats(); | |
535 | } | |
536 | ||
537 | ||
538 | /* | |
539 | * Setup the PC, SP, and GP of a secondary processor and start it | |
540 | * running! | |
541 | * smp_bootstrap is the place to resume from | |
542 | * __KSTK_TOS(idle) is apparently the stack pointer | |
543 | * (unsigned long)idle->thread_info the gp | |
544 | * | |
545 | */ | |
546 | void smtc_boot_secondary(int cpu, struct task_struct *idle) | |
547 | { | |
548 | extern u32 kernelsp[NR_CPUS]; | |
549 | long flags; | |
550 | int mtflags; | |
551 | ||
552 | LOCK_MT_PRA(); | |
553 | if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { | |
554 | dvpe(); | |
555 | } | |
556 | settc(cpu_data[cpu].tc_id); | |
557 | ||
558 | /* pc */ | |
559 | write_tc_c0_tcrestart((unsigned long)&smp_bootstrap); | |
560 | ||
561 | /* stack pointer */ | |
562 | kernelsp[cpu] = __KSTK_TOS(idle); | |
563 | write_tc_gpr_sp(__KSTK_TOS(idle)); | |
564 | ||
565 | /* global pointer */ | |
566 | write_tc_gpr_gp((unsigned long)idle->thread_info); | |
567 | ||
568 | smtc_status |= SMTC_MTC_ACTIVE; | |
569 | write_tc_c0_tchalt(0); | |
570 | if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { | |
571 | evpe(EVPE_ENABLE); | |
572 | } | |
573 | UNLOCK_MT_PRA(); | |
574 | } | |
575 | ||
576 | void smtc_init_secondary(void) | |
577 | { | |
578 | /* | |
579 | * Start timer on secondary VPEs if necessary. | |
580 | * mips_timer_setup should already have been invoked by init/main | |
581 | * on "boot" TC. Like per_cpu_trap_init() hack, this assumes that | |
582 | * SMTC init code assigns TCs consdecutively and in ascending order | |
583 | * to across available VPEs. | |
584 | */ | |
585 | if(((read_c0_tcbind() & TCBIND_CURTC) != 0) | |
586 | && ((read_c0_tcbind() & TCBIND_CURVPE) | |
587 | != cpu_data[smp_processor_id() - 1].vpe_id)){ | |
588 | write_c0_compare (read_c0_count() + mips_hpt_frequency/HZ); | |
589 | } | |
590 | ||
591 | local_irq_enable(); | |
592 | } | |
593 | ||
594 | void smtc_smp_finish(void) | |
595 | { | |
596 | printk("TC %d going on-line as CPU %d\n", | |
597 | cpu_data[smp_processor_id()].tc_id, smp_processor_id()); | |
598 | } | |
599 | ||
600 | void smtc_cpus_done(void) | |
601 | { | |
602 | } | |
603 | ||
604 | /* | |
605 | * Support for SMTC-optimized driver IRQ registration | |
606 | */ | |
607 | ||
608 | /* | |
609 | * SMTC Kernel needs to manipulate low-level CPU interrupt mask | |
610 | * in do_IRQ. These are passed in setup_irq_smtc() and stored | |
611 | * in this table. | |
612 | */ | |
613 | ||
614 | int setup_irq_smtc(unsigned int irq, struct irqaction * new, | |
615 | unsigned long hwmask) | |
616 | { | |
617 | irq_hwmask[irq] = hwmask; | |
618 | ||
619 | return setup_irq(irq, new); | |
620 | } | |
621 | ||
622 | /* | |
623 | * IPI model for SMTC is tricky, because interrupts aren't TC-specific. | |
624 | * Within a VPE one TC can interrupt another by different approaches. | |
625 | * The easiest to get right would probably be to make all TCs except | |
626 | * the target IXMT and set a software interrupt, but an IXMT-based | |
627 | * scheme requires that a handler must run before a new IPI could | |
628 | * be sent, which would break the "broadcast" loops in MIPS MT. | |
629 | * A more gonzo approach within a VPE is to halt the TC, extract | |
630 | * its Restart, Status, and a couple of GPRs, and program the Restart | |
631 | * address to emulate an interrupt. | |
632 | * | |
633 | * Within a VPE, one can be confident that the target TC isn't in | |
634 | * a critical EXL state when halted, since the write to the Halt | |
635 | * register could not have issued on the writing thread if the | |
636 | * halting thread had EXL set. So k0 and k1 of the target TC | |
637 | * can be used by the injection code. Across VPEs, one can't | |
638 | * be certain that the target TC isn't in a critical exception | |
639 | * state. So we try a two-step process of sending a software | |
640 | * interrupt to the target VPE, which either handles the event | |
641 | * itself (if it was the target) or injects the event within | |
642 | * the VPE. | |
643 | */ | |
644 | ||
645 | void smtc_ipi_qdump(void) | |
646 | { | |
647 | int i; | |
648 | ||
649 | for (i = 0; i < NR_CPUS ;i++) { | |
650 | printk("IPIQ[%d]: head = 0x%x, tail = 0x%x, depth = %d\n", | |
651 | i, (unsigned)IPIQ[i].head, (unsigned)IPIQ[i].tail, | |
652 | IPIQ[i].depth); | |
653 | } | |
654 | } | |
655 | ||
656 | /* | |
657 | * The standard atomic.h primitives don't quite do what we want | |
658 | * here: We need an atomic add-and-return-previous-value (which | |
659 | * could be done with atomic_add_return and a decrement) and an | |
660 | * atomic set/zero-and-return-previous-value (which can't really | |
661 | * be done with the atomic.h primitives). And since this is | |
662 | * MIPS MT, we can assume that we have LL/SC. | |
663 | */ | |
664 | static __inline__ int atomic_postincrement(unsigned int *pv) | |
665 | { | |
666 | unsigned long result; | |
667 | ||
668 | unsigned long temp; | |
669 | ||
670 | __asm__ __volatile__( | |
671 | "1: ll %0, %2 \n" | |
672 | " addu %1, %0, 1 \n" | |
673 | " sc %1, %2 \n" | |
674 | " beqz %1, 1b \n" | |
675 | " sync \n" | |
676 | : "=&r" (result), "=&r" (temp), "=m" (*pv) | |
677 | : "m" (*pv) | |
678 | : "memory"); | |
679 | ||
680 | return result; | |
681 | } | |
682 | ||
683 | /* No longer used in IPI dispatch, but retained for future recycling */ | |
684 | ||
685 | static __inline__ int atomic_postclear(unsigned int *pv) | |
686 | { | |
687 | unsigned long result; | |
688 | ||
689 | unsigned long temp; | |
690 | ||
691 | __asm__ __volatile__( | |
692 | "1: ll %0, %2 \n" | |
693 | " or %1, $0, $0 \n" | |
694 | " sc %1, %2 \n" | |
695 | " beqz %1, 1b \n" | |
696 | " sync \n" | |
697 | : "=&r" (result), "=&r" (temp), "=m" (*pv) | |
698 | : "m" (*pv) | |
699 | : "memory"); | |
700 | ||
701 | return result; | |
702 | } | |
703 | ||
704 | ||
705 | void smtc_send_ipi(int cpu, int type, unsigned int action) | |
706 | { | |
707 | int tcstatus; | |
708 | struct smtc_ipi *pipi; | |
709 | long flags; | |
710 | int mtflags; | |
711 | ||
712 | if (cpu == smp_processor_id()) { | |
713 | printk("Cannot Send IPI to self!\n"); | |
714 | return; | |
715 | } | |
716 | /* Set up a descriptor, to be delivered either promptly or queued */ | |
717 | pipi = smtc_ipi_dq(&freeIPIq); | |
718 | if (pipi == NULL) { | |
719 | bust_spinlocks(1); | |
720 | mips_mt_regdump(dvpe()); | |
721 | panic("IPI Msg. Buffers Depleted\n"); | |
722 | } | |
723 | pipi->type = type; | |
724 | pipi->arg = (void *)action; | |
725 | pipi->dest = cpu; | |
726 | if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { | |
727 | /* If not on same VPE, enqueue and send cross-VPE interupt */ | |
728 | smtc_ipi_nq(&IPIQ[cpu], pipi); | |
729 | LOCK_CORE_PRA(); | |
730 | settc(cpu_data[cpu].tc_id); | |
731 | write_vpe_c0_cause(read_vpe_c0_cause() | C_SW1); | |
732 | UNLOCK_CORE_PRA(); | |
733 | } else { | |
734 | /* | |
735 | * Not sufficient to do a LOCK_MT_PRA (dmt) here, | |
736 | * since ASID shootdown on the other VPE may | |
737 | * collide with this operation. | |
738 | */ | |
739 | LOCK_CORE_PRA(); | |
740 | settc(cpu_data[cpu].tc_id); | |
741 | /* Halt the targeted TC */ | |
742 | write_tc_c0_tchalt(TCHALT_H); | |
743 | mips_ihb(); | |
744 | ||
745 | /* | |
746 | * Inspect TCStatus - if IXMT is set, we have to queue | |
747 | * a message. Otherwise, we set up the "interrupt" | |
748 | * of the other TC | |
749 | */ | |
750 | tcstatus = read_tc_c0_tcstatus(); | |
751 | ||
752 | if ((tcstatus & TCSTATUS_IXMT) != 0) { | |
753 | /* | |
754 | * Spin-waiting here can deadlock, | |
755 | * so we queue the message for the target TC. | |
756 | */ | |
757 | write_tc_c0_tchalt(0); | |
758 | UNLOCK_CORE_PRA(); | |
759 | /* Try to reduce redundant timer interrupt messages */ | |
760 | if(type == SMTC_CLOCK_TICK) { | |
761 | if(atomic_postincrement(&ipi_timer_latch[cpu])!=0) { | |
762 | smtc_ipi_nq(&freeIPIq, pipi); | |
763 | return; | |
764 | } | |
765 | } | |
766 | smtc_ipi_nq(&IPIQ[cpu], pipi); | |
767 | } else { | |
768 | post_direct_ipi(cpu, pipi); | |
769 | write_tc_c0_tchalt(0); | |
770 | UNLOCK_CORE_PRA(); | |
771 | } | |
772 | } | |
773 | } | |
774 | ||
775 | /* | |
776 | * Send IPI message to Halted TC, TargTC/TargVPE already having been set | |
777 | */ | |
778 | void post_direct_ipi(int cpu, struct smtc_ipi *pipi) | |
779 | { | |
780 | struct pt_regs *kstack; | |
781 | unsigned long tcstatus; | |
782 | unsigned long tcrestart; | |
783 | extern u32 kernelsp[NR_CPUS]; | |
784 | extern void __smtc_ipi_vector(void); | |
785 | ||
786 | /* Extract Status, EPC from halted TC */ | |
787 | tcstatus = read_tc_c0_tcstatus(); | |
788 | tcrestart = read_tc_c0_tcrestart(); | |
789 | /* If TCRestart indicates a WAIT instruction, advance the PC */ | |
790 | if ((tcrestart & 0x80000000) | |
791 | && ((*(unsigned int *)tcrestart & 0xfe00003f) == 0x42000020)) { | |
792 | tcrestart += 4; | |
793 | } | |
794 | /* | |
795 | * Save on TC's future kernel stack | |
796 | * | |
797 | * CU bit of Status is indicator that TC was | |
798 | * already running on a kernel stack... | |
799 | */ | |
800 | if(tcstatus & ST0_CU0) { | |
801 | /* Note that this "- 1" is pointer arithmetic */ | |
802 | kstack = ((struct pt_regs *)read_tc_gpr_sp()) - 1; | |
803 | } else { | |
804 | kstack = ((struct pt_regs *)kernelsp[cpu]) - 1; | |
805 | } | |
806 | ||
807 | kstack->cp0_epc = (long)tcrestart; | |
808 | /* Save TCStatus */ | |
809 | kstack->cp0_tcstatus = tcstatus; | |
810 | /* Pass token of operation to be performed kernel stack pad area */ | |
811 | kstack->pad0[4] = (unsigned long)pipi; | |
812 | /* Pass address of function to be called likewise */ | |
813 | kstack->pad0[5] = (unsigned long)&ipi_decode; | |
814 | /* Set interrupt exempt and kernel mode */ | |
815 | tcstatus |= TCSTATUS_IXMT; | |
816 | tcstatus &= ~TCSTATUS_TKSU; | |
817 | write_tc_c0_tcstatus(tcstatus); | |
818 | ehb(); | |
819 | /* Set TC Restart address to be SMTC IPI vector */ | |
820 | write_tc_c0_tcrestart(__smtc_ipi_vector); | |
821 | } | |
822 | ||
823 | void ipi_resched_interrupt(struct pt_regs *regs) | |
824 | { | |
825 | /* Return from interrupt should be enough to cause scheduler check */ | |
826 | } | |
827 | ||
828 | ||
829 | void ipi_call_interrupt(struct pt_regs *regs) | |
830 | { | |
831 | /* Invoke generic function invocation code in smp.c */ | |
832 | smp_call_function_interrupt(); | |
833 | } | |
834 | ||
835 | void ipi_decode(struct pt_regs *regs, struct smtc_ipi *pipi) | |
836 | { | |
837 | void *arg_copy = pipi->arg; | |
838 | int type_copy = pipi->type; | |
839 | int dest_copy = pipi->dest; | |
840 | ||
841 | smtc_ipi_nq(&freeIPIq, pipi); | |
842 | switch (type_copy) { | |
843 | case SMTC_CLOCK_TICK: | |
844 | /* Invoke Clock "Interrupt" */ | |
845 | ipi_timer_latch[dest_copy] = 0; | |
846 | #ifdef SMTC_IDLE_HOOK_DEBUG | |
847 | clock_hang_reported[dest_copy] = 0; | |
848 | #endif /* SMTC_IDLE_HOOK_DEBUG */ | |
849 | local_timer_interrupt(0, NULL, regs); | |
850 | break; | |
851 | case LINUX_SMP_IPI: | |
852 | switch ((int)arg_copy) { | |
853 | case SMP_RESCHEDULE_YOURSELF: | |
854 | ipi_resched_interrupt(regs); | |
855 | break; | |
856 | case SMP_CALL_FUNCTION: | |
857 | ipi_call_interrupt(regs); | |
858 | break; | |
859 | default: | |
860 | printk("Impossible SMTC IPI Argument 0x%x\n", | |
861 | (int)arg_copy); | |
862 | break; | |
863 | } | |
864 | break; | |
865 | default: | |
866 | printk("Impossible SMTC IPI Type 0x%x\n", type_copy); | |
867 | break; | |
868 | } | |
869 | } | |
870 | ||
871 | void deferred_smtc_ipi(struct pt_regs *regs) | |
872 | { | |
873 | struct smtc_ipi *pipi; | |
874 | unsigned long flags; | |
875 | /* DEBUG */ | |
876 | int q = smp_processor_id(); | |
877 | ||
878 | /* | |
879 | * Test is not atomic, but much faster than a dequeue, | |
880 | * and the vast majority of invocations will have a null queue. | |
881 | */ | |
882 | if(IPIQ[q].head != NULL) { | |
883 | while((pipi = smtc_ipi_dq(&IPIQ[q])) != NULL) { | |
884 | /* ipi_decode() should be called with interrupts off */ | |
885 | local_irq_save(flags); | |
886 | ipi_decode(regs, pipi); | |
887 | local_irq_restore(flags); | |
888 | } | |
889 | } | |
890 | } | |
891 | ||
892 | /* | |
893 | * Send clock tick to all TCs except the one executing the funtion | |
894 | */ | |
895 | ||
896 | void smtc_timer_broadcast(int vpe) | |
897 | { | |
898 | int cpu; | |
899 | int myTC = cpu_data[smp_processor_id()].tc_id; | |
900 | int myVPE = cpu_data[smp_processor_id()].vpe_id; | |
901 | ||
902 | smtc_cpu_stats[smp_processor_id()].timerints++; | |
903 | ||
904 | for_each_online_cpu(cpu) { | |
905 | if (cpu_data[cpu].vpe_id == myVPE && | |
906 | cpu_data[cpu].tc_id != myTC) | |
907 | smtc_send_ipi(cpu, SMTC_CLOCK_TICK, 0); | |
908 | } | |
909 | } | |
910 | ||
911 | /* | |
912 | * Cross-VPE interrupts in the SMTC prototype use "software interrupts" | |
913 | * set via cross-VPE MTTR manipulation of the Cause register. It would be | |
914 | * in some regards preferable to have external logic for "doorbell" hardware | |
915 | * interrupts. | |
916 | */ | |
917 | ||
918 | static int cpu_ipi_irq = MIPSCPU_INT_BASE + MIPS_CPU_IPI_IRQ; | |
919 | ||
920 | static irqreturn_t ipi_interrupt(int irq, void *dev_idm, struct pt_regs *regs) | |
921 | { | |
922 | int my_vpe = cpu_data[smp_processor_id()].vpe_id; | |
923 | int my_tc = cpu_data[smp_processor_id()].tc_id; | |
924 | int cpu; | |
925 | struct smtc_ipi *pipi; | |
926 | unsigned long tcstatus; | |
927 | int sent; | |
928 | long flags; | |
929 | unsigned int mtflags; | |
930 | unsigned int vpflags; | |
931 | ||
932 | /* | |
933 | * So long as cross-VPE interrupts are done via | |
934 | * MFTR/MTTR read-modify-writes of Cause, we need | |
935 | * to stop other VPEs whenever the local VPE does | |
936 | * anything similar. | |
937 | */ | |
938 | local_irq_save(flags); | |
939 | vpflags = dvpe(); | |
940 | clear_c0_cause(0x100 << MIPS_CPU_IPI_IRQ); | |
941 | set_c0_status(0x100 << MIPS_CPU_IPI_IRQ); | |
942 | irq_enable_hazard(); | |
943 | evpe(vpflags); | |
944 | local_irq_restore(flags); | |
945 | ||
946 | /* | |
947 | * Cross-VPE Interrupt handler: Try to directly deliver IPIs | |
948 | * queued for TCs on this VPE other than the current one. | |
949 | * Return-from-interrupt should cause us to drain the queue | |
950 | * for the current TC, so we ought not to have to do it explicitly here. | |
951 | */ | |
952 | ||
953 | for_each_online_cpu(cpu) { | |
954 | if (cpu_data[cpu].vpe_id != my_vpe) | |
955 | continue; | |
956 | ||
957 | pipi = smtc_ipi_dq(&IPIQ[cpu]); | |
958 | if (pipi != NULL) { | |
959 | if (cpu_data[cpu].tc_id != my_tc) { | |
960 | sent = 0; | |
961 | LOCK_MT_PRA(); | |
962 | settc(cpu_data[cpu].tc_id); | |
963 | write_tc_c0_tchalt(TCHALT_H); | |
964 | mips_ihb(); | |
965 | tcstatus = read_tc_c0_tcstatus(); | |
966 | if ((tcstatus & TCSTATUS_IXMT) == 0) { | |
967 | post_direct_ipi(cpu, pipi); | |
968 | sent = 1; | |
969 | } | |
970 | write_tc_c0_tchalt(0); | |
971 | UNLOCK_MT_PRA(); | |
972 | if (!sent) { | |
973 | smtc_ipi_req(&IPIQ[cpu], pipi); | |
974 | } | |
975 | } else { | |
976 | /* | |
977 | * ipi_decode() should be called | |
978 | * with interrupts off | |
979 | */ | |
980 | local_irq_save(flags); | |
981 | ipi_decode(regs, pipi); | |
982 | local_irq_restore(flags); | |
983 | } | |
984 | } | |
985 | } | |
986 | ||
987 | return IRQ_HANDLED; | |
988 | } | |
989 | ||
990 | static void ipi_irq_dispatch(struct pt_regs *regs) | |
991 | { | |
992 | do_IRQ(cpu_ipi_irq, regs); | |
993 | } | |
994 | ||
995 | static struct irqaction irq_ipi; | |
996 | ||
997 | void setup_cross_vpe_interrupts(void) | |
998 | { | |
999 | if (!cpu_has_vint) | |
1000 | panic("SMTC Kernel requires Vectored Interupt support"); | |
1001 | ||
1002 | set_vi_handler(MIPS_CPU_IPI_IRQ, ipi_irq_dispatch); | |
1003 | ||
1004 | irq_ipi.handler = ipi_interrupt; | |
1005 | irq_ipi.flags = SA_INTERRUPT; | |
1006 | irq_ipi.name = "SMTC_IPI"; | |
1007 | ||
1008 | setup_irq_smtc(cpu_ipi_irq, &irq_ipi, (0x100 << MIPS_CPU_IPI_IRQ)); | |
1009 | ||
1010 | irq_desc[cpu_ipi_irq].status |= IRQ_PER_CPU; | |
1011 | } | |
1012 | ||
1013 | /* | |
1014 | * SMTC-specific hacks invoked from elsewhere in the kernel. | |
1015 | */ | |
1016 | ||
1017 | void smtc_idle_loop_hook(void) | |
1018 | { | |
1019 | #ifdef SMTC_IDLE_HOOK_DEBUG | |
1020 | int im; | |
1021 | int flags; | |
1022 | int mtflags; | |
1023 | int bit; | |
1024 | int vpe; | |
1025 | int tc; | |
1026 | int hook_ntcs; | |
1027 | /* | |
1028 | * printk within DMT-protected regions can deadlock, | |
1029 | * so buffer diagnostic messages for later output. | |
1030 | */ | |
1031 | char *pdb_msg; | |
1032 | char id_ho_db_msg[768]; /* worst-case use should be less than 700 */ | |
1033 | ||
1034 | if (atomic_read(&idle_hook_initialized) == 0) { /* fast test */ | |
1035 | if (atomic_add_return(1, &idle_hook_initialized) == 1) { | |
1036 | int mvpconf0; | |
1037 | /* Tedious stuff to just do once */ | |
1038 | mvpconf0 = read_c0_mvpconf0(); | |
1039 | hook_ntcs = ((mvpconf0 & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; | |
1040 | if (hook_ntcs > NR_CPUS) | |
1041 | hook_ntcs = NR_CPUS; | |
1042 | for (tc = 0; tc < hook_ntcs; tc++) { | |
1043 | tcnoprog[tc] = 0; | |
1044 | clock_hang_reported[tc] = 0; | |
1045 | } | |
1046 | for (vpe = 0; vpe < 2; vpe++) | |
1047 | for (im = 0; im < 8; im++) | |
1048 | imstuckcount[vpe][im] = 0; | |
1049 | printk("Idle loop test hook initialized for %d TCs\n", hook_ntcs); | |
1050 | atomic_set(&idle_hook_initialized, 1000); | |
1051 | } else { | |
1052 | /* Someone else is initializing in parallel - let 'em finish */ | |
1053 | while (atomic_read(&idle_hook_initialized) < 1000) | |
1054 | ; | |
1055 | } | |
1056 | } | |
1057 | ||
1058 | /* Have we stupidly left IXMT set somewhere? */ | |
1059 | if (read_c0_tcstatus() & 0x400) { | |
1060 | write_c0_tcstatus(read_c0_tcstatus() & ~0x400); | |
1061 | ehb(); | |
1062 | printk("Dangling IXMT in cpu_idle()\n"); | |
1063 | } | |
1064 | ||
1065 | /* Have we stupidly left an IM bit turned off? */ | |
1066 | #define IM_LIMIT 2000 | |
1067 | local_irq_save(flags); | |
1068 | mtflags = dmt(); | |
1069 | pdb_msg = &id_ho_db_msg[0]; | |
1070 | im = read_c0_status(); | |
1071 | vpe = cpu_data[smp_processor_id()].vpe_id; | |
1072 | for (bit = 0; bit < 8; bit++) { | |
1073 | /* | |
1074 | * In current prototype, I/O interrupts | |
1075 | * are masked for VPE > 0 | |
1076 | */ | |
1077 | if (vpemask[vpe][bit]) { | |
1078 | if (!(im & (0x100 << bit))) | |
1079 | imstuckcount[vpe][bit]++; | |
1080 | else | |
1081 | imstuckcount[vpe][bit] = 0; | |
1082 | if (imstuckcount[vpe][bit] > IM_LIMIT) { | |
1083 | set_c0_status(0x100 << bit); | |
1084 | ehb(); | |
1085 | imstuckcount[vpe][bit] = 0; | |
1086 | pdb_msg += sprintf(pdb_msg, | |
1087 | "Dangling IM %d fixed for VPE %d\n", bit, | |
1088 | vpe); | |
1089 | } | |
1090 | } | |
1091 | } | |
1092 | ||
1093 | /* | |
1094 | * Now that we limit outstanding timer IPIs, check for hung TC | |
1095 | */ | |
1096 | for (tc = 0; tc < NR_CPUS; tc++) { | |
1097 | /* Don't check ourself - we'll dequeue IPIs just below */ | |
1098 | if ((tc != smp_processor_id()) && | |
1099 | ipi_timer_latch[tc] > timerq_limit) { | |
1100 | if (clock_hang_reported[tc] == 0) { | |
1101 | pdb_msg += sprintf(pdb_msg, | |
1102 | "TC %d looks hung with timer latch at %d\n", | |
1103 | tc, ipi_timer_latch[tc]); | |
1104 | clock_hang_reported[tc]++; | |
1105 | } | |
1106 | } | |
1107 | } | |
1108 | emt(mtflags); | |
1109 | local_irq_restore(flags); | |
1110 | if (pdb_msg != &id_ho_db_msg[0]) | |
1111 | printk("CPU%d: %s", smp_processor_id(), id_ho_db_msg); | |
1112 | #endif /* SMTC_IDLE_HOOK_DEBUG */ | |
1113 | /* | |
1114 | * To the extent that we've ever turned interrupts off, | |
1115 | * we may have accumulated deferred IPIs. This is subtle. | |
1116 | * If we use the smtc_ipi_qdepth() macro, we'll get an | |
1117 | * exact number - but we'll also disable interrupts | |
1118 | * and create a window of failure where a new IPI gets | |
1119 | * queued after we test the depth but before we re-enable | |
1120 | * interrupts. So long as IXMT never gets set, however, | |
1121 | * we should be OK: If we pick up something and dispatch | |
1122 | * it here, that's great. If we see nothing, but concurrent | |
1123 | * with this operation, another TC sends us an IPI, IXMT | |
1124 | * is clear, and we'll handle it as a real pseudo-interrupt | |
1125 | * and not a pseudo-pseudo interrupt. | |
1126 | */ | |
1127 | if (IPIQ[smp_processor_id()].depth > 0) { | |
1128 | struct smtc_ipi *pipi; | |
1129 | extern void self_ipi(struct smtc_ipi *); | |
1130 | ||
1131 | if ((pipi = smtc_ipi_dq(&IPIQ[smp_processor_id()])) != NULL) { | |
1132 | self_ipi(pipi); | |
1133 | smtc_cpu_stats[smp_processor_id()].selfipis++; | |
1134 | } | |
1135 | } | |
1136 | } | |
1137 | ||
1138 | void smtc_soft_dump(void) | |
1139 | { | |
1140 | int i; | |
1141 | ||
1142 | printk("Counter Interrupts taken per CPU (TC)\n"); | |
1143 | for (i=0; i < NR_CPUS; i++) { | |
1144 | printk("%d: %ld\n", i, smtc_cpu_stats[i].timerints); | |
1145 | } | |
1146 | printk("Self-IPI invocations:\n"); | |
1147 | for (i=0; i < NR_CPUS; i++) { | |
1148 | printk("%d: %ld\n", i, smtc_cpu_stats[i].selfipis); | |
1149 | } | |
1150 | smtc_ipi_qdump(); | |
1151 | printk("Timer IPI Backlogs:\n"); | |
1152 | for (i=0; i < NR_CPUS; i++) { | |
1153 | printk("%d: %d\n", i, ipi_timer_latch[i]); | |
1154 | } | |
1155 | printk("%d Recoveries of \"stolen\" FPU\n", | |
1156 | atomic_read(&smtc_fpu_recoveries)); | |
1157 | } | |
1158 | ||
1159 | ||
1160 | /* | |
1161 | * TLB management routines special to SMTC | |
1162 | */ | |
1163 | ||
1164 | void smtc_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu) | |
1165 | { | |
1166 | unsigned long flags, mtflags, tcstat, prevhalt, asid; | |
1167 | int tlb, i; | |
1168 | ||
1169 | /* | |
1170 | * It would be nice to be able to use a spinlock here, | |
1171 | * but this is invoked from within TLB flush routines | |
1172 | * that protect themselves with DVPE, so if a lock is | |
1173 | * held by another TC, it'll never be freed. | |
1174 | * | |
1175 | * DVPE/DMT must not be done with interrupts enabled, | |
1176 | * so even so most callers will already have disabled | |
1177 | * them, let's be really careful... | |
1178 | */ | |
1179 | ||
1180 | local_irq_save(flags); | |
1181 | if (smtc_status & SMTC_TLB_SHARED) { | |
1182 | mtflags = dvpe(); | |
1183 | tlb = 0; | |
1184 | } else { | |
1185 | mtflags = dmt(); | |
1186 | tlb = cpu_data[cpu].vpe_id; | |
1187 | } | |
1188 | asid = asid_cache(cpu); | |
1189 | ||
1190 | do { | |
1191 | if (!((asid += ASID_INC) & ASID_MASK) ) { | |
1192 | if (cpu_has_vtag_icache) | |
1193 | flush_icache_all(); | |
1194 | /* Traverse all online CPUs (hack requires contigous range) */ | |
1195 | for (i = 0; i < num_online_cpus(); i++) { | |
1196 | /* | |
1197 | * We don't need to worry about our own CPU, nor those of | |
1198 | * CPUs who don't share our TLB. | |
1199 | */ | |
1200 | if ((i != smp_processor_id()) && | |
1201 | ((smtc_status & SMTC_TLB_SHARED) || | |
1202 | (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))) { | |
1203 | settc(cpu_data[i].tc_id); | |
1204 | prevhalt = read_tc_c0_tchalt() & TCHALT_H; | |
1205 | if (!prevhalt) { | |
1206 | write_tc_c0_tchalt(TCHALT_H); | |
1207 | mips_ihb(); | |
1208 | } | |
1209 | tcstat = read_tc_c0_tcstatus(); | |
1210 | smtc_live_asid[tlb][(tcstat & ASID_MASK)] |= (asiduse)(0x1 << i); | |
1211 | if (!prevhalt) | |
1212 | write_tc_c0_tchalt(0); | |
1213 | } | |
1214 | } | |
1215 | if (!asid) /* fix version if needed */ | |
1216 | asid = ASID_FIRST_VERSION; | |
1217 | local_flush_tlb_all(); /* start new asid cycle */ | |
1218 | } | |
1219 | } while (smtc_live_asid[tlb][(asid & ASID_MASK)]); | |
1220 | ||
1221 | /* | |
1222 | * SMTC shares the TLB within VPEs and possibly across all VPEs. | |
1223 | */ | |
1224 | for (i = 0; i < num_online_cpus(); i++) { | |
1225 | if ((smtc_status & SMTC_TLB_SHARED) || | |
1226 | (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id)) | |
1227 | cpu_context(i, mm) = asid_cache(i) = asid; | |
1228 | } | |
1229 | ||
1230 | if (smtc_status & SMTC_TLB_SHARED) | |
1231 | evpe(mtflags); | |
1232 | else | |
1233 | emt(mtflags); | |
1234 | local_irq_restore(flags); | |
1235 | } | |
1236 | ||
1237 | /* | |
1238 | * Invoked from macros defined in mmu_context.h | |
1239 | * which must already have disabled interrupts | |
1240 | * and done a DVPE or DMT as appropriate. | |
1241 | */ | |
1242 | ||
1243 | void smtc_flush_tlb_asid(unsigned long asid) | |
1244 | { | |
1245 | int entry; | |
1246 | unsigned long ehi; | |
1247 | ||
1248 | entry = read_c0_wired(); | |
1249 | ||
1250 | /* Traverse all non-wired entries */ | |
1251 | while (entry < current_cpu_data.tlbsize) { | |
1252 | write_c0_index(entry); | |
1253 | ehb(); | |
1254 | tlb_read(); | |
1255 | ehb(); | |
1256 | ehi = read_c0_entryhi(); | |
1257 | if((ehi & ASID_MASK) == asid) { | |
1258 | /* | |
1259 | * Invalidate only entries with specified ASID, | |
1260 | * makiing sure all entries differ. | |
1261 | */ | |
1262 | write_c0_entryhi(CKSEG0 + (entry << (PAGE_SHIFT + 1))); | |
1263 | write_c0_entrylo0(0); | |
1264 | write_c0_entrylo1(0); | |
1265 | mtc0_tlbw_hazard(); | |
1266 | tlb_write_indexed(); | |
1267 | } | |
1268 | entry++; | |
1269 | } | |
1270 | write_c0_index(PARKED_INDEX); | |
1271 | tlbw_use_hazard(); | |
1272 | } | |
1273 | ||
1274 | /* | |
1275 | * Support for single-threading cache flush operations. | |
1276 | */ | |
1277 | ||
1278 | int halt_state_save[NR_CPUS]; | |
1279 | ||
1280 | /* | |
1281 | * To really, really be sure that nothing is being done | |
1282 | * by other TCs, halt them all. This code assumes that | |
1283 | * a DVPE has already been done, so while their Halted | |
1284 | * state is theoretically architecturally unstable, in | |
1285 | * practice, it's not going to change while we're looking | |
1286 | * at it. | |
1287 | */ | |
1288 | ||
1289 | void smtc_cflush_lockdown(void) | |
1290 | { | |
1291 | int cpu; | |
1292 | ||
1293 | for_each_online_cpu(cpu) { | |
1294 | if (cpu != smp_processor_id()) { | |
1295 | settc(cpu_data[cpu].tc_id); | |
1296 | halt_state_save[cpu] = read_tc_c0_tchalt(); | |
1297 | write_tc_c0_tchalt(TCHALT_H); | |
1298 | } | |
1299 | } | |
1300 | mips_ihb(); | |
1301 | } | |
1302 | ||
1303 | /* It would be cheating to change the cpu_online states during a flush! */ | |
1304 | ||
1305 | void smtc_cflush_release(void) | |
1306 | { | |
1307 | int cpu; | |
1308 | ||
1309 | /* | |
1310 | * Start with a hazard barrier to ensure | |
1311 | * that all CACHE ops have played through. | |
1312 | */ | |
1313 | mips_ihb(); | |
1314 | ||
1315 | for_each_online_cpu(cpu) { | |
1316 | if (cpu != smp_processor_id()) { | |
1317 | settc(cpu_data[cpu].tc_id); | |
1318 | write_tc_c0_tchalt(halt_state_save[cpu]); | |
1319 | } | |
1320 | } | |
1321 | mips_ihb(); | |
1322 | } |