Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef __ALPHA_SYSTEM_H |
2 | #define __ALPHA_SYSTEM_H | |
3 | ||
1da177e4 LT |
4 | #include <asm/pal.h> |
5 | #include <asm/page.h> | |
0db9ae4a | 6 | #include <asm/barrier.h> |
1da177e4 LT |
7 | |
8 | /* | |
9 | * System defines.. Note that this is included both from .c and .S | |
10 | * files, so it does only defines, not any C code. | |
11 | */ | |
12 | ||
13 | /* | |
14 | * We leave one page for the initial stack page, and one page for | |
15 | * the initial process structure. Also, the console eats 3 MB for | |
16 | * the initial bootloader (one of which we can reclaim later). | |
17 | */ | |
18 | #define BOOT_PCB 0x20000000 | |
19 | #define BOOT_ADDR 0x20000000 | |
20 | /* Remove when official MILO sources have ELF support: */ | |
21 | #define BOOT_SIZE (16*1024) | |
22 | ||
23 | #ifdef CONFIG_ALPHA_LEGACY_START_ADDRESS | |
24 | #define KERNEL_START_PHYS 0x300000 /* Old bootloaders hardcoded this. */ | |
25 | #else | |
26 | #define KERNEL_START_PHYS 0x1000000 /* required: Wildfire/Titan/Marvel */ | |
27 | #endif | |
28 | ||
29 | #define KERNEL_START (PAGE_OFFSET+KERNEL_START_PHYS) | |
30 | #define SWAPPER_PGD KERNEL_START | |
31 | #define INIT_STACK (PAGE_OFFSET+KERNEL_START_PHYS+0x02000) | |
32 | #define EMPTY_PGT (PAGE_OFFSET+KERNEL_START_PHYS+0x04000) | |
33 | #define EMPTY_PGE (PAGE_OFFSET+KERNEL_START_PHYS+0x08000) | |
34 | #define ZERO_PGE (PAGE_OFFSET+KERNEL_START_PHYS+0x0A000) | |
35 | ||
36 | #define START_ADDR (PAGE_OFFSET+KERNEL_START_PHYS+0x10000) | |
37 | ||
38 | /* | |
39 | * This is setup by the secondary bootstrap loader. Because | |
40 | * the zero page is zeroed out as soon as the vm system is | |
41 | * initialized, we need to copy things out into a more permanent | |
42 | * place. | |
43 | */ | |
44 | #define PARAM ZERO_PGE | |
45 | #define COMMAND_LINE ((char*)(PARAM + 0x0000)) | |
46 | #define INITRD_START (*(unsigned long *) (PARAM+0x100)) | |
47 | #define INITRD_SIZE (*(unsigned long *) (PARAM+0x108)) | |
48 | ||
49 | #ifndef __ASSEMBLY__ | |
50 | #include <linux/kernel.h> | |
4f9a58d7 | 51 | #define AT_VECTOR_SIZE_ARCH 4 /* entries in ARCH_DLINFO */ |
1da177e4 LT |
52 | |
53 | /* | |
54 | * This is the logout header that should be common to all platforms | |
55 | * (assuming they are running OSF/1 PALcode, I guess). | |
56 | */ | |
57 | struct el_common { | |
58 | unsigned int size; /* size in bytes of logout area */ | |
59 | unsigned int sbz1 : 30; /* should be zero */ | |
60 | unsigned int err2 : 1; /* second error */ | |
61 | unsigned int retry : 1; /* retry flag */ | |
62 | unsigned int proc_offset; /* processor-specific offset */ | |
63 | unsigned int sys_offset; /* system-specific offset */ | |
64 | unsigned int code; /* machine check code */ | |
65 | unsigned int frame_rev; /* frame revision */ | |
66 | }; | |
67 | ||
68 | /* Machine Check Frame for uncorrectable errors (Large format) | |
69 | * --- This is used to log uncorrectable errors such as | |
70 | * double bit ECC errors. | |
71 | * --- These errors are detected by both processor and systems. | |
72 | */ | |
73 | struct el_common_EV5_uncorrectable_mcheck { | |
74 | unsigned long shadow[8]; /* Shadow reg. 8-14, 25 */ | |
75 | unsigned long paltemp[24]; /* PAL TEMP REGS. */ | |
76 | unsigned long exc_addr; /* Address of excepting instruction*/ | |
77 | unsigned long exc_sum; /* Summary of arithmetic traps. */ | |
78 | unsigned long exc_mask; /* Exception mask (from exc_sum). */ | |
79 | unsigned long pal_base; /* Base address for PALcode. */ | |
80 | unsigned long isr; /* Interrupt Status Reg. */ | |
81 | unsigned long icsr; /* CURRENT SETUP OF EV5 IBOX */ | |
82 | unsigned long ic_perr_stat; /* I-CACHE Reg. <11> set Data parity | |
83 | <12> set TAG parity*/ | |
84 | unsigned long dc_perr_stat; /* D-CACHE error Reg. Bits set to 1: | |
85 | <2> Data error in bank 0 | |
86 | <3> Data error in bank 1 | |
87 | <4> Tag error in bank 0 | |
88 | <5> Tag error in bank 1 */ | |
89 | unsigned long va; /* Effective VA of fault or miss. */ | |
90 | unsigned long mm_stat; /* Holds the reason for D-stream | |
91 | fault or D-cache parity errors */ | |
92 | unsigned long sc_addr; /* Address that was being accessed | |
93 | when EV5 detected Secondary cache | |
94 | failure. */ | |
95 | unsigned long sc_stat; /* Helps determine if the error was | |
96 | TAG/Data parity(Secondary Cache)*/ | |
97 | unsigned long bc_tag_addr; /* Contents of EV5 BC_TAG_ADDR */ | |
98 | unsigned long ei_addr; /* Physical address of any transfer | |
99 | that is logged in EV5 EI_STAT */ | |
100 | unsigned long fill_syndrome; /* For correcting ECC errors. */ | |
101 | unsigned long ei_stat; /* Helps identify reason of any | |
102 | processor uncorrectable error | |
103 | at its external interface. */ | |
104 | unsigned long ld_lock; /* Contents of EV5 LD_LOCK register*/ | |
105 | }; | |
106 | ||
107 | struct el_common_EV6_mcheck { | |
108 | unsigned int FrameSize; /* Bytes, including this field */ | |
109 | unsigned int FrameFlags; /* <31> = Retry, <30> = Second Error */ | |
110 | unsigned int CpuOffset; /* Offset to CPU-specific info */ | |
111 | unsigned int SystemOffset; /* Offset to system-specific info */ | |
112 | unsigned int MCHK_Code; | |
113 | unsigned int MCHK_Frame_Rev; | |
114 | unsigned long I_STAT; /* EV6 Internal Processor Registers */ | |
115 | unsigned long DC_STAT; /* (See the 21264 Spec) */ | |
116 | unsigned long C_ADDR; | |
117 | unsigned long DC1_SYNDROME; | |
118 | unsigned long DC0_SYNDROME; | |
119 | unsigned long C_STAT; | |
120 | unsigned long C_STS; | |
121 | unsigned long MM_STAT; | |
122 | unsigned long EXC_ADDR; | |
123 | unsigned long IER_CM; | |
124 | unsigned long ISUM; | |
125 | unsigned long RESERVED0; | |
126 | unsigned long PAL_BASE; | |
127 | unsigned long I_CTL; | |
128 | unsigned long PCTX; | |
129 | }; | |
130 | ||
131 | extern void halt(void) __attribute__((noreturn)); | |
132 | #define __halt() __asm__ __volatile__ ("call_pal %0 #halt" : : "i" (PAL_halt)) | |
133 | ||
37bfbaf9 AV |
134 | #define switch_to(P,N,L) \ |
135 | do { \ | |
136 | (L) = alpha_switch_to(virt_to_phys(&task_thread_info(N)->pcb), (P)); \ | |
137 | check_mmu_context(); \ | |
1da177e4 LT |
138 | } while (0) |
139 | ||
140 | struct task_struct; | |
141 | extern struct task_struct *alpha_switch_to(unsigned long, struct task_struct*); | |
142 | ||
1da177e4 LT |
143 | #define imb() \ |
144 | __asm__ __volatile__ ("call_pal %0 #imb" : : "i" (PAL_imb) : "memory") | |
145 | ||
146 | #define draina() \ | |
147 | __asm__ __volatile__ ("call_pal %0 #draina" : : "i" (PAL_draina) : "memory") | |
148 | ||
149 | enum implver_enum { | |
150 | IMPLVER_EV4, | |
151 | IMPLVER_EV5, | |
152 | IMPLVER_EV6 | |
153 | }; | |
154 | ||
155 | #ifdef CONFIG_ALPHA_GENERIC | |
156 | #define implver() \ | |
157 | ({ unsigned long __implver; \ | |
158 | __asm__ ("implver %0" : "=r"(__implver)); \ | |
159 | (enum implver_enum) __implver; }) | |
160 | #else | |
161 | /* Try to eliminate some dead code. */ | |
162 | #ifdef CONFIG_ALPHA_EV4 | |
163 | #define implver() IMPLVER_EV4 | |
164 | #endif | |
165 | #ifdef CONFIG_ALPHA_EV5 | |
166 | #define implver() IMPLVER_EV5 | |
167 | #endif | |
168 | #if defined(CONFIG_ALPHA_EV6) | |
169 | #define implver() IMPLVER_EV6 | |
170 | #endif | |
171 | #endif | |
172 | ||
173 | enum amask_enum { | |
174 | AMASK_BWX = (1UL << 0), | |
175 | AMASK_FIX = (1UL << 1), | |
176 | AMASK_CIX = (1UL << 2), | |
177 | AMASK_MAX = (1UL << 8), | |
178 | AMASK_PRECISE_TRAP = (1UL << 9), | |
179 | }; | |
180 | ||
181 | #define amask(mask) \ | |
182 | ({ unsigned long __amask, __input = (mask); \ | |
183 | __asm__ ("amask %1,%0" : "=r"(__amask) : "rI"(__input)); \ | |
184 | __amask; }) | |
185 | ||
186 | #define __CALL_PAL_R0(NAME, TYPE) \ | |
187 | static inline TYPE NAME(void) \ | |
188 | { \ | |
189 | register TYPE __r0 __asm__("$0"); \ | |
190 | __asm__ __volatile__( \ | |
191 | "call_pal %1 # " #NAME \ | |
192 | :"=r" (__r0) \ | |
193 | :"i" (PAL_ ## NAME) \ | |
194 | :"$1", "$16", "$22", "$23", "$24", "$25"); \ | |
195 | return __r0; \ | |
196 | } | |
197 | ||
198 | #define __CALL_PAL_W1(NAME, TYPE0) \ | |
199 | static inline void NAME(TYPE0 arg0) \ | |
200 | { \ | |
201 | register TYPE0 __r16 __asm__("$16") = arg0; \ | |
202 | __asm__ __volatile__( \ | |
203 | "call_pal %1 # "#NAME \ | |
204 | : "=r"(__r16) \ | |
205 | : "i"(PAL_ ## NAME), "0"(__r16) \ | |
206 | : "$1", "$22", "$23", "$24", "$25"); \ | |
207 | } | |
208 | ||
209 | #define __CALL_PAL_W2(NAME, TYPE0, TYPE1) \ | |
210 | static inline void NAME(TYPE0 arg0, TYPE1 arg1) \ | |
211 | { \ | |
212 | register TYPE0 __r16 __asm__("$16") = arg0; \ | |
213 | register TYPE1 __r17 __asm__("$17") = arg1; \ | |
214 | __asm__ __volatile__( \ | |
215 | "call_pal %2 # "#NAME \ | |
216 | : "=r"(__r16), "=r"(__r17) \ | |
217 | : "i"(PAL_ ## NAME), "0"(__r16), "1"(__r17) \ | |
218 | : "$1", "$22", "$23", "$24", "$25"); \ | |
219 | } | |
220 | ||
221 | #define __CALL_PAL_RW1(NAME, RTYPE, TYPE0) \ | |
222 | static inline RTYPE NAME(TYPE0 arg0) \ | |
223 | { \ | |
224 | register RTYPE __r0 __asm__("$0"); \ | |
225 | register TYPE0 __r16 __asm__("$16") = arg0; \ | |
226 | __asm__ __volatile__( \ | |
227 | "call_pal %2 # "#NAME \ | |
228 | : "=r"(__r16), "=r"(__r0) \ | |
229 | : "i"(PAL_ ## NAME), "0"(__r16) \ | |
230 | : "$1", "$22", "$23", "$24", "$25"); \ | |
231 | return __r0; \ | |
232 | } | |
233 | ||
234 | #define __CALL_PAL_RW2(NAME, RTYPE, TYPE0, TYPE1) \ | |
235 | static inline RTYPE NAME(TYPE0 arg0, TYPE1 arg1) \ | |
236 | { \ | |
237 | register RTYPE __r0 __asm__("$0"); \ | |
238 | register TYPE0 __r16 __asm__("$16") = arg0; \ | |
239 | register TYPE1 __r17 __asm__("$17") = arg1; \ | |
240 | __asm__ __volatile__( \ | |
241 | "call_pal %3 # "#NAME \ | |
242 | : "=r"(__r16), "=r"(__r17), "=r"(__r0) \ | |
243 | : "i"(PAL_ ## NAME), "0"(__r16), "1"(__r17) \ | |
244 | : "$1", "$22", "$23", "$24", "$25"); \ | |
245 | return __r0; \ | |
246 | } | |
247 | ||
248 | __CALL_PAL_W1(cflush, unsigned long); | |
249 | __CALL_PAL_R0(rdmces, unsigned long); | |
250 | __CALL_PAL_R0(rdps, unsigned long); | |
251 | __CALL_PAL_R0(rdusp, unsigned long); | |
252 | __CALL_PAL_RW1(swpipl, unsigned long, unsigned long); | |
253 | __CALL_PAL_R0(whami, unsigned long); | |
254 | __CALL_PAL_W2(wrent, void*, unsigned long); | |
255 | __CALL_PAL_W1(wripir, unsigned long); | |
256 | __CALL_PAL_W1(wrkgp, unsigned long); | |
257 | __CALL_PAL_W1(wrmces, unsigned long); | |
258 | __CALL_PAL_RW2(wrperfmon, unsigned long, unsigned long, unsigned long); | |
259 | __CALL_PAL_W1(wrusp, unsigned long); | |
260 | __CALL_PAL_W1(wrvptptr, unsigned long); | |
261 | ||
262 | #define IPL_MIN 0 | |
263 | #define IPL_SW0 1 | |
264 | #define IPL_SW1 2 | |
265 | #define IPL_DEV0 3 | |
266 | #define IPL_DEV1 4 | |
267 | #define IPL_TIMER 5 | |
268 | #define IPL_PERF 6 | |
269 | #define IPL_POWERFAIL 6 | |
270 | #define IPL_MCHECK 7 | |
271 | #define IPL_MAX 7 | |
272 | ||
273 | #ifdef CONFIG_ALPHA_BROKEN_IRQ_MASK | |
274 | #undef IPL_MIN | |
275 | #define IPL_MIN __min_ipl | |
276 | extern int __min_ipl; | |
277 | #endif | |
278 | ||
279 | #define getipl() (rdps() & 7) | |
280 | #define setipl(ipl) ((void) swpipl(ipl)) | |
281 | ||
282 | #define local_irq_disable() do { setipl(IPL_MAX); barrier(); } while(0) | |
283 | #define local_irq_enable() do { barrier(); setipl(IPL_MIN); } while(0) | |
284 | #define local_save_flags(flags) ((flags) = rdps()) | |
285 | #define local_irq_save(flags) do { (flags) = swpipl(IPL_MAX); barrier(); } while(0) | |
286 | #define local_irq_restore(flags) do { barrier(); setipl(flags); barrier(); } while(0) | |
287 | ||
288 | #define irqs_disabled() (getipl() == IPL_MAX) | |
289 | ||
290 | /* | |
291 | * TB routines.. | |
292 | */ | |
293 | #define __tbi(nr,arg,arg1...) \ | |
294 | ({ \ | |
295 | register unsigned long __r16 __asm__("$16") = (nr); \ | |
296 | register unsigned long __r17 __asm__("$17"); arg; \ | |
297 | __asm__ __volatile__( \ | |
298 | "call_pal %3 #__tbi" \ | |
299 | :"=r" (__r16),"=r" (__r17) \ | |
300 | :"0" (__r16),"i" (PAL_tbi) ,##arg1 \ | |
301 | :"$0", "$1", "$22", "$23", "$24", "$25"); \ | |
302 | }) | |
303 | ||
304 | #define tbi(x,y) __tbi(x,__r17=(y),"1" (__r17)) | |
305 | #define tbisi(x) __tbi(1,__r17=(x),"1" (__r17)) | |
306 | #define tbisd(x) __tbi(2,__r17=(x),"1" (__r17)) | |
307 | #define tbis(x) __tbi(3,__r17=(x),"1" (__r17)) | |
308 | #define tbiap() __tbi(-1, /* no second argument */) | |
309 | #define tbia() __tbi(-2, /* no second argument */) | |
310 | ||
311 | /* | |
312 | * Atomic exchange. | |
313 | * Since it can be used to implement critical sections | |
314 | * it must clobber "memory" (also for interrupts in UP). | |
315 | */ | |
316 | ||
317 | static inline unsigned long | |
318 | __xchg_u8(volatile char *m, unsigned long val) | |
319 | { | |
320 | unsigned long ret, tmp, addr64; | |
321 | ||
322 | __asm__ __volatile__( | |
323 | " andnot %4,7,%3\n" | |
324 | " insbl %1,%4,%1\n" | |
325 | "1: ldq_l %2,0(%3)\n" | |
326 | " extbl %2,%4,%0\n" | |
327 | " mskbl %2,%4,%2\n" | |
328 | " or %1,%2,%2\n" | |
329 | " stq_c %2,0(%3)\n" | |
330 | " beq %2,2f\n" | |
331 | #ifdef CONFIG_SMP | |
332 | " mb\n" | |
333 | #endif | |
334 | ".subsection 2\n" | |
335 | "2: br 1b\n" | |
336 | ".previous" | |
337 | : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) | |
338 | : "r" ((long)m), "1" (val) : "memory"); | |
339 | ||
340 | return ret; | |
341 | } | |
342 | ||
343 | static inline unsigned long | |
344 | __xchg_u16(volatile short *m, unsigned long val) | |
345 | { | |
346 | unsigned long ret, tmp, addr64; | |
347 | ||
348 | __asm__ __volatile__( | |
349 | " andnot %4,7,%3\n" | |
350 | " inswl %1,%4,%1\n" | |
351 | "1: ldq_l %2,0(%3)\n" | |
352 | " extwl %2,%4,%0\n" | |
353 | " mskwl %2,%4,%2\n" | |
354 | " or %1,%2,%2\n" | |
355 | " stq_c %2,0(%3)\n" | |
356 | " beq %2,2f\n" | |
357 | #ifdef CONFIG_SMP | |
358 | " mb\n" | |
359 | #endif | |
360 | ".subsection 2\n" | |
361 | "2: br 1b\n" | |
362 | ".previous" | |
363 | : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) | |
364 | : "r" ((long)m), "1" (val) : "memory"); | |
365 | ||
366 | return ret; | |
367 | } | |
368 | ||
369 | static inline unsigned long | |
370 | __xchg_u32(volatile int *m, unsigned long val) | |
371 | { | |
372 | unsigned long dummy; | |
373 | ||
374 | __asm__ __volatile__( | |
375 | "1: ldl_l %0,%4\n" | |
376 | " bis $31,%3,%1\n" | |
377 | " stl_c %1,%2\n" | |
378 | " beq %1,2f\n" | |
379 | #ifdef CONFIG_SMP | |
380 | " mb\n" | |
381 | #endif | |
382 | ".subsection 2\n" | |
383 | "2: br 1b\n" | |
384 | ".previous" | |
385 | : "=&r" (val), "=&r" (dummy), "=m" (*m) | |
386 | : "rI" (val), "m" (*m) : "memory"); | |
387 | ||
388 | return val; | |
389 | } | |
390 | ||
391 | static inline unsigned long | |
392 | __xchg_u64(volatile long *m, unsigned long val) | |
393 | { | |
394 | unsigned long dummy; | |
395 | ||
396 | __asm__ __volatile__( | |
397 | "1: ldq_l %0,%4\n" | |
398 | " bis $31,%3,%1\n" | |
399 | " stq_c %1,%2\n" | |
400 | " beq %1,2f\n" | |
401 | #ifdef CONFIG_SMP | |
402 | " mb\n" | |
403 | #endif | |
404 | ".subsection 2\n" | |
405 | "2: br 1b\n" | |
406 | ".previous" | |
407 | : "=&r" (val), "=&r" (dummy), "=m" (*m) | |
408 | : "rI" (val), "m" (*m) : "memory"); | |
409 | ||
410 | return val; | |
411 | } | |
412 | ||
413 | /* This function doesn't exist, so you'll get a linker error | |
414 | if something tries to do an invalid xchg(). */ | |
415 | extern void __xchg_called_with_bad_pointer(void); | |
416 | ||
79fb7bdc AV |
417 | #define __xchg(ptr, x, size) \ |
418 | ({ \ | |
419 | unsigned long __xchg__res; \ | |
420 | volatile void *__xchg__ptr = (ptr); \ | |
421 | switch (size) { \ | |
422 | case 1: __xchg__res = __xchg_u8(__xchg__ptr, x); break; \ | |
423 | case 2: __xchg__res = __xchg_u16(__xchg__ptr, x); break; \ | |
424 | case 4: __xchg__res = __xchg_u32(__xchg__ptr, x); break; \ | |
425 | case 8: __xchg__res = __xchg_u64(__xchg__ptr, x); break; \ | |
426 | default: __xchg_called_with_bad_pointer(); __xchg__res = x; \ | |
427 | } \ | |
428 | __xchg__res; \ | |
429 | }) | |
1da177e4 LT |
430 | |
431 | #define xchg(ptr,x) \ | |
432 | ({ \ | |
433 | __typeof__(*(ptr)) _x_ = (x); \ | |
434 | (__typeof__(*(ptr))) __xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \ | |
435 | }) | |
436 | ||
f43f7b46 MD |
437 | static inline unsigned long |
438 | __xchg_u8_local(volatile char *m, unsigned long val) | |
439 | { | |
440 | unsigned long ret, tmp, addr64; | |
441 | ||
442 | __asm__ __volatile__( | |
443 | " andnot %4,7,%3\n" | |
444 | " insbl %1,%4,%1\n" | |
445 | "1: ldq_l %2,0(%3)\n" | |
446 | " extbl %2,%4,%0\n" | |
447 | " mskbl %2,%4,%2\n" | |
448 | " or %1,%2,%2\n" | |
449 | " stq_c %2,0(%3)\n" | |
450 | " beq %2,2f\n" | |
451 | ".subsection 2\n" | |
452 | "2: br 1b\n" | |
453 | ".previous" | |
454 | : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) | |
455 | : "r" ((long)m), "1" (val) : "memory"); | |
456 | ||
457 | return ret; | |
458 | } | |
459 | ||
460 | static inline unsigned long | |
461 | __xchg_u16_local(volatile short *m, unsigned long val) | |
462 | { | |
463 | unsigned long ret, tmp, addr64; | |
464 | ||
465 | __asm__ __volatile__( | |
466 | " andnot %4,7,%3\n" | |
467 | " inswl %1,%4,%1\n" | |
468 | "1: ldq_l %2,0(%3)\n" | |
469 | " extwl %2,%4,%0\n" | |
470 | " mskwl %2,%4,%2\n" | |
471 | " or %1,%2,%2\n" | |
472 | " stq_c %2,0(%3)\n" | |
473 | " beq %2,2f\n" | |
474 | ".subsection 2\n" | |
475 | "2: br 1b\n" | |
476 | ".previous" | |
477 | : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) | |
478 | : "r" ((long)m), "1" (val) : "memory"); | |
479 | ||
480 | return ret; | |
481 | } | |
482 | ||
483 | static inline unsigned long | |
484 | __xchg_u32_local(volatile int *m, unsigned long val) | |
485 | { | |
486 | unsigned long dummy; | |
487 | ||
488 | __asm__ __volatile__( | |
489 | "1: ldl_l %0,%4\n" | |
490 | " bis $31,%3,%1\n" | |
491 | " stl_c %1,%2\n" | |
492 | " beq %1,2f\n" | |
493 | ".subsection 2\n" | |
494 | "2: br 1b\n" | |
495 | ".previous" | |
496 | : "=&r" (val), "=&r" (dummy), "=m" (*m) | |
497 | : "rI" (val), "m" (*m) : "memory"); | |
498 | ||
499 | return val; | |
500 | } | |
501 | ||
502 | static inline unsigned long | |
503 | __xchg_u64_local(volatile long *m, unsigned long val) | |
504 | { | |
505 | unsigned long dummy; | |
506 | ||
507 | __asm__ __volatile__( | |
508 | "1: ldq_l %0,%4\n" | |
509 | " bis $31,%3,%1\n" | |
510 | " stq_c %1,%2\n" | |
511 | " beq %1,2f\n" | |
512 | ".subsection 2\n" | |
513 | "2: br 1b\n" | |
514 | ".previous" | |
515 | : "=&r" (val), "=&r" (dummy), "=m" (*m) | |
516 | : "rI" (val), "m" (*m) : "memory"); | |
517 | ||
518 | return val; | |
519 | } | |
520 | ||
521 | #define __xchg_local(ptr, x, size) \ | |
522 | ({ \ | |
523 | unsigned long __xchg__res; \ | |
524 | volatile void *__xchg__ptr = (ptr); \ | |
525 | switch (size) { \ | |
526 | case 1: __xchg__res = __xchg_u8_local(__xchg__ptr, x); break; \ | |
527 | case 2: __xchg__res = __xchg_u16_local(__xchg__ptr, x); break; \ | |
528 | case 4: __xchg__res = __xchg_u32_local(__xchg__ptr, x); break; \ | |
529 | case 8: __xchg__res = __xchg_u64_local(__xchg__ptr, x); break; \ | |
530 | default: __xchg_called_with_bad_pointer(); __xchg__res = x; \ | |
531 | } \ | |
532 | __xchg__res; \ | |
533 | }) | |
534 | ||
535 | #define xchg_local(ptr,x) \ | |
536 | ({ \ | |
537 | __typeof__(*(ptr)) _x_ = (x); \ | |
538 | (__typeof__(*(ptr))) __xchg_local((ptr), (unsigned long)_x_, \ | |
539 | sizeof(*(ptr))); \ | |
540 | }) | |
541 | ||
1da177e4 LT |
542 | /* |
543 | * Atomic compare and exchange. Compare OLD with MEM, if identical, | |
544 | * store NEW in MEM. Return the initial value in MEM. Success is | |
545 | * indicated by comparing RETURN with OLD. | |
546 | * | |
547 | * The memory barrier should be placed in SMP only when we actually | |
548 | * make the change. If we don't change anything (so if the returned | |
549 | * prev is equal to old) then we aren't acquiring anything new and | |
550 | * we don't need any memory barrier as far I can tell. | |
551 | */ | |
552 | ||
553 | #define __HAVE_ARCH_CMPXCHG 1 | |
554 | ||
555 | static inline unsigned long | |
556 | __cmpxchg_u8(volatile char *m, long old, long new) | |
557 | { | |
558 | unsigned long prev, tmp, cmp, addr64; | |
559 | ||
560 | __asm__ __volatile__( | |
561 | " andnot %5,7,%4\n" | |
562 | " insbl %1,%5,%1\n" | |
563 | "1: ldq_l %2,0(%4)\n" | |
564 | " extbl %2,%5,%0\n" | |
565 | " cmpeq %0,%6,%3\n" | |
566 | " beq %3,2f\n" | |
567 | " mskbl %2,%5,%2\n" | |
568 | " or %1,%2,%2\n" | |
569 | " stq_c %2,0(%4)\n" | |
570 | " beq %2,3f\n" | |
571 | #ifdef CONFIG_SMP | |
572 | " mb\n" | |
573 | #endif | |
574 | "2:\n" | |
575 | ".subsection 2\n" | |
576 | "3: br 1b\n" | |
577 | ".previous" | |
578 | : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) | |
579 | : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); | |
580 | ||
581 | return prev; | |
582 | } | |
583 | ||
584 | static inline unsigned long | |
585 | __cmpxchg_u16(volatile short *m, long old, long new) | |
586 | { | |
587 | unsigned long prev, tmp, cmp, addr64; | |
588 | ||
589 | __asm__ __volatile__( | |
590 | " andnot %5,7,%4\n" | |
591 | " inswl %1,%5,%1\n" | |
592 | "1: ldq_l %2,0(%4)\n" | |
593 | " extwl %2,%5,%0\n" | |
594 | " cmpeq %0,%6,%3\n" | |
595 | " beq %3,2f\n" | |
596 | " mskwl %2,%5,%2\n" | |
597 | " or %1,%2,%2\n" | |
598 | " stq_c %2,0(%4)\n" | |
599 | " beq %2,3f\n" | |
600 | #ifdef CONFIG_SMP | |
601 | " mb\n" | |
602 | #endif | |
603 | "2:\n" | |
604 | ".subsection 2\n" | |
605 | "3: br 1b\n" | |
606 | ".previous" | |
607 | : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) | |
608 | : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); | |
609 | ||
610 | return prev; | |
611 | } | |
612 | ||
613 | static inline unsigned long | |
614 | __cmpxchg_u32(volatile int *m, int old, int new) | |
615 | { | |
616 | unsigned long prev, cmp; | |
617 | ||
618 | __asm__ __volatile__( | |
619 | "1: ldl_l %0,%5\n" | |
620 | " cmpeq %0,%3,%1\n" | |
621 | " beq %1,2f\n" | |
622 | " mov %4,%1\n" | |
623 | " stl_c %1,%2\n" | |
624 | " beq %1,3f\n" | |
625 | #ifdef CONFIG_SMP | |
626 | " mb\n" | |
627 | #endif | |
628 | "2:\n" | |
629 | ".subsection 2\n" | |
630 | "3: br 1b\n" | |
631 | ".previous" | |
632 | : "=&r"(prev), "=&r"(cmp), "=m"(*m) | |
633 | : "r"((long) old), "r"(new), "m"(*m) : "memory"); | |
634 | ||
635 | return prev; | |
636 | } | |
637 | ||
638 | static inline unsigned long | |
639 | __cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new) | |
640 | { | |
641 | unsigned long prev, cmp; | |
642 | ||
643 | __asm__ __volatile__( | |
644 | "1: ldq_l %0,%5\n" | |
645 | " cmpeq %0,%3,%1\n" | |
646 | " beq %1,2f\n" | |
647 | " mov %4,%1\n" | |
648 | " stq_c %1,%2\n" | |
649 | " beq %1,3f\n" | |
650 | #ifdef CONFIG_SMP | |
651 | " mb\n" | |
652 | #endif | |
653 | "2:\n" | |
654 | ".subsection 2\n" | |
655 | "3: br 1b\n" | |
656 | ".previous" | |
657 | : "=&r"(prev), "=&r"(cmp), "=m"(*m) | |
658 | : "r"((long) old), "r"(new), "m"(*m) : "memory"); | |
659 | ||
660 | return prev; | |
661 | } | |
662 | ||
663 | /* This function doesn't exist, so you'll get a linker error | |
664 | if something tries to do an invalid cmpxchg(). */ | |
665 | extern void __cmpxchg_called_with_bad_pointer(void); | |
666 | ||
c00a76ae | 667 | static __always_inline unsigned long |
1da177e4 LT |
668 | __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) |
669 | { | |
670 | switch (size) { | |
671 | case 1: | |
672 | return __cmpxchg_u8(ptr, old, new); | |
673 | case 2: | |
674 | return __cmpxchg_u16(ptr, old, new); | |
675 | case 4: | |
676 | return __cmpxchg_u32(ptr, old, new); | |
677 | case 8: | |
678 | return __cmpxchg_u64(ptr, old, new); | |
679 | } | |
680 | __cmpxchg_called_with_bad_pointer(); | |
681 | return old; | |
682 | } | |
683 | ||
a259b242 | 684 | #define cmpxchg(ptr, o, n) \ |
1da177e4 LT |
685 | ({ \ |
686 | __typeof__(*(ptr)) _o_ = (o); \ | |
687 | __typeof__(*(ptr)) _n_ = (n); \ | |
688 | (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ | |
689 | (unsigned long)_n_, sizeof(*(ptr))); \ | |
690 | }) | |
a259b242 MD |
691 | #define cmpxchg64(ptr, o, n) \ |
692 | ({ \ | |
693 | BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ | |
694 | cmpxchg((ptr), (o), (n)); \ | |
695 | }) | |
1da177e4 | 696 | |
f43f7b46 MD |
697 | static inline unsigned long |
698 | __cmpxchg_u8_local(volatile char *m, long old, long new) | |
699 | { | |
700 | unsigned long prev, tmp, cmp, addr64; | |
701 | ||
702 | __asm__ __volatile__( | |
703 | " andnot %5,7,%4\n" | |
704 | " insbl %1,%5,%1\n" | |
705 | "1: ldq_l %2,0(%4)\n" | |
706 | " extbl %2,%5,%0\n" | |
707 | " cmpeq %0,%6,%3\n" | |
708 | " beq %3,2f\n" | |
709 | " mskbl %2,%5,%2\n" | |
710 | " or %1,%2,%2\n" | |
711 | " stq_c %2,0(%4)\n" | |
712 | " beq %2,3f\n" | |
713 | "2:\n" | |
714 | ".subsection 2\n" | |
715 | "3: br 1b\n" | |
716 | ".previous" | |
717 | : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) | |
718 | : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); | |
719 | ||
720 | return prev; | |
721 | } | |
722 | ||
723 | static inline unsigned long | |
724 | __cmpxchg_u16_local(volatile short *m, long old, long new) | |
725 | { | |
726 | unsigned long prev, tmp, cmp, addr64; | |
727 | ||
728 | __asm__ __volatile__( | |
729 | " andnot %5,7,%4\n" | |
730 | " inswl %1,%5,%1\n" | |
731 | "1: ldq_l %2,0(%4)\n" | |
732 | " extwl %2,%5,%0\n" | |
733 | " cmpeq %0,%6,%3\n" | |
734 | " beq %3,2f\n" | |
735 | " mskwl %2,%5,%2\n" | |
736 | " or %1,%2,%2\n" | |
737 | " stq_c %2,0(%4)\n" | |
738 | " beq %2,3f\n" | |
739 | "2:\n" | |
740 | ".subsection 2\n" | |
741 | "3: br 1b\n" | |
742 | ".previous" | |
743 | : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) | |
744 | : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); | |
745 | ||
746 | return prev; | |
747 | } | |
748 | ||
749 | static inline unsigned long | |
750 | __cmpxchg_u32_local(volatile int *m, int old, int new) | |
751 | { | |
752 | unsigned long prev, cmp; | |
753 | ||
754 | __asm__ __volatile__( | |
755 | "1: ldl_l %0,%5\n" | |
756 | " cmpeq %0,%3,%1\n" | |
757 | " beq %1,2f\n" | |
758 | " mov %4,%1\n" | |
759 | " stl_c %1,%2\n" | |
760 | " beq %1,3f\n" | |
761 | "2:\n" | |
762 | ".subsection 2\n" | |
763 | "3: br 1b\n" | |
764 | ".previous" | |
765 | : "=&r"(prev), "=&r"(cmp), "=m"(*m) | |
766 | : "r"((long) old), "r"(new), "m"(*m) : "memory"); | |
767 | ||
768 | return prev; | |
769 | } | |
770 | ||
771 | static inline unsigned long | |
772 | __cmpxchg_u64_local(volatile long *m, unsigned long old, unsigned long new) | |
773 | { | |
774 | unsigned long prev, cmp; | |
775 | ||
776 | __asm__ __volatile__( | |
777 | "1: ldq_l %0,%5\n" | |
778 | " cmpeq %0,%3,%1\n" | |
779 | " beq %1,2f\n" | |
780 | " mov %4,%1\n" | |
781 | " stq_c %1,%2\n" | |
782 | " beq %1,3f\n" | |
783 | "2:\n" | |
784 | ".subsection 2\n" | |
785 | "3: br 1b\n" | |
786 | ".previous" | |
787 | : "=&r"(prev), "=&r"(cmp), "=m"(*m) | |
788 | : "r"((long) old), "r"(new), "m"(*m) : "memory"); | |
789 | ||
790 | return prev; | |
791 | } | |
792 | ||
793 | static __always_inline unsigned long | |
794 | __cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new, | |
795 | int size) | |
796 | { | |
797 | switch (size) { | |
798 | case 1: | |
799 | return __cmpxchg_u8_local(ptr, old, new); | |
800 | case 2: | |
801 | return __cmpxchg_u16_local(ptr, old, new); | |
802 | case 4: | |
803 | return __cmpxchg_u32_local(ptr, old, new); | |
804 | case 8: | |
805 | return __cmpxchg_u64_local(ptr, old, new); | |
806 | } | |
807 | __cmpxchg_called_with_bad_pointer(); | |
808 | return old; | |
809 | } | |
810 | ||
a259b242 | 811 | #define cmpxchg_local(ptr, o, n) \ |
f43f7b46 MD |
812 | ({ \ |
813 | __typeof__(*(ptr)) _o_ = (o); \ | |
814 | __typeof__(*(ptr)) _n_ = (n); \ | |
815 | (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \ | |
816 | (unsigned long)_n_, sizeof(*(ptr))); \ | |
817 | }) | |
a259b242 MD |
818 | #define cmpxchg64_local(ptr, o, n) \ |
819 | ({ \ | |
820 | BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ | |
821 | cmpxchg_local((ptr), (o), (n)); \ | |
822 | }) | |
823 | ||
f43f7b46 | 824 | |
1da177e4 LT |
825 | #endif /* __ASSEMBLY__ */ |
826 | ||
827 | #define arch_align_stack(x) (x) | |
828 | ||
829 | #endif |