Commit | Line | Data |
---|---|---|
a7386694 JB |
1 | /* |
2 | * x86 semaphore implementation. | |
3 | * | |
4 | * (C) Copyright 1999 Linus Torvalds | |
5 | * | |
6 | * Portions Copyright 1999 Red Hat, Inc. | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU General Public License | |
10 | * as published by the Free Software Foundation; either version | |
11 | * 2 of the License, or (at your option) any later version. | |
12 | * | |
13 | * rw semaphores implemented November 1999 by Benjamin LaHaise <bcrl@kvack.org> | |
14 | */ | |
15 | ||
16 | #include <linux/linkage.h> | |
17 | #include <asm/alternative-asm.h> | |
18 | #include <asm/dwarf2.h> | |
19 | ||
20 | #define __ASM_HALF_REG(reg) __ASM_SEL(reg, e##reg) | |
21 | #define __ASM_HALF_SIZE(inst) __ASM_SEL(inst##w, inst##l) | |
22 | ||
23 | #ifdef CONFIG_X86_32 | |
24 | ||
25 | /* | |
26 | * The semaphore operations have a special calling sequence that | |
27 | * allow us to do a simpler in-line version of them. These routines | |
28 | * need to convert that sequence back into the C sequence when | |
29 | * there is contention on the semaphore. | |
30 | * | |
31 | * %eax contains the semaphore pointer on entry. Save the C-clobbered | |
32 | * registers (%eax, %edx and %ecx) except %eax whish is either a return | |
33 | * value or just clobbered.. | |
34 | */ | |
35 | ||
36 | #define save_common_regs \ | |
37 | pushl_cfi %ecx; CFI_REL_OFFSET ecx, 0 | |
38 | ||
39 | #define restore_common_regs \ | |
40 | popl_cfi %ecx; CFI_RESTORE ecx | |
41 | ||
42 | /* Avoid uglifying the argument copying x86-64 needs to do. */ | |
43 | .macro movq src, dst | |
44 | .endm | |
45 | ||
46 | #else | |
47 | ||
bafaecd1 LT |
48 | /* |
49 | * x86-64 rwsem wrappers | |
50 | * | |
51 | * This interfaces the inline asm code to the slow-path | |
52 | * C routines. We need to save the call-clobbered regs | |
53 | * that the asm does not mark as clobbered, and move the | |
54 | * argument from %rax to %rdi. | |
55 | * | |
56 | * NOTE! We don't need to save %rax, because the functions | |
57 | * will always return the semaphore pointer in %rax (which | |
58 | * is also the input argument to these helpers) | |
59 | * | |
60 | * The following can clobber %rdx because the asm clobbers it: | |
61 | * call_rwsem_down_write_failed | |
62 | * call_rwsem_wake | |
63 | * but %rdi, %rsi, %rcx, %r8-r11 always need saving. | |
64 | */ | |
65 | ||
bafaecd1 | 66 | #define save_common_regs \ |
39f2205e JB |
67 | pushq_cfi %rdi; CFI_REL_OFFSET rdi, 0; \ |
68 | pushq_cfi %rsi; CFI_REL_OFFSET rsi, 0; \ | |
69 | pushq_cfi %rcx; CFI_REL_OFFSET rcx, 0; \ | |
70 | pushq_cfi %r8; CFI_REL_OFFSET r8, 0; \ | |
71 | pushq_cfi %r9; CFI_REL_OFFSET r9, 0; \ | |
72 | pushq_cfi %r10; CFI_REL_OFFSET r10, 0; \ | |
73 | pushq_cfi %r11; CFI_REL_OFFSET r11, 0 | |
bafaecd1 LT |
74 | |
75 | #define restore_common_regs \ | |
39f2205e JB |
76 | popq_cfi %r11; CFI_RESTORE r11; \ |
77 | popq_cfi %r10; CFI_RESTORE r10; \ | |
78 | popq_cfi %r9; CFI_RESTORE r9; \ | |
79 | popq_cfi %r8; CFI_RESTORE r8; \ | |
80 | popq_cfi %rcx; CFI_RESTORE rcx; \ | |
81 | popq_cfi %rsi; CFI_RESTORE rsi; \ | |
82 | popq_cfi %rdi; CFI_RESTORE rdi | |
bafaecd1 | 83 | |
a7386694 JB |
84 | #endif |
85 | ||
bafaecd1 LT |
86 | /* Fix up special calling conventions */ |
87 | ENTRY(call_rwsem_down_read_failed) | |
39f2205e | 88 | CFI_STARTPROC |
bafaecd1 | 89 | save_common_regs |
a7386694 JB |
90 | __ASM_SIZE(push,_cfi) %__ASM_REG(dx) |
91 | CFI_REL_OFFSET __ASM_REG(dx), 0 | |
bafaecd1 LT |
92 | movq %rax,%rdi |
93 | call rwsem_down_read_failed | |
a7386694 JB |
94 | __ASM_SIZE(pop,_cfi) %__ASM_REG(dx) |
95 | CFI_RESTORE __ASM_REG(dx) | |
bafaecd1 LT |
96 | restore_common_regs |
97 | ret | |
39f2205e JB |
98 | CFI_ENDPROC |
99 | ENDPROC(call_rwsem_down_read_failed) | |
bafaecd1 LT |
100 | |
101 | ENTRY(call_rwsem_down_write_failed) | |
39f2205e | 102 | CFI_STARTPROC |
bafaecd1 LT |
103 | save_common_regs |
104 | movq %rax,%rdi | |
105 | call rwsem_down_write_failed | |
106 | restore_common_regs | |
107 | ret | |
39f2205e JB |
108 | CFI_ENDPROC |
109 | ENDPROC(call_rwsem_down_write_failed) | |
bafaecd1 LT |
110 | |
111 | ENTRY(call_rwsem_wake) | |
39f2205e | 112 | CFI_STARTPROC |
a7386694 JB |
113 | /* do nothing if still outstanding active readers */ |
114 | __ASM_HALF_SIZE(dec) %__ASM_HALF_REG(dx) | |
bafaecd1 LT |
115 | jnz 1f |
116 | save_common_regs | |
117 | movq %rax,%rdi | |
118 | call rwsem_wake | |
119 | restore_common_regs | |
120 | 1: ret | |
39f2205e JB |
121 | CFI_ENDPROC |
122 | ENDPROC(call_rwsem_wake) | |
bafaecd1 | 123 | |
bafaecd1 | 124 | ENTRY(call_rwsem_downgrade_wake) |
39f2205e | 125 | CFI_STARTPROC |
bafaecd1 | 126 | save_common_regs |
a7386694 JB |
127 | __ASM_SIZE(push,_cfi) %__ASM_REG(dx) |
128 | CFI_REL_OFFSET __ASM_REG(dx), 0 | |
bafaecd1 LT |
129 | movq %rax,%rdi |
130 | call rwsem_downgrade_wake | |
a7386694 JB |
131 | __ASM_SIZE(pop,_cfi) %__ASM_REG(dx) |
132 | CFI_RESTORE __ASM_REG(dx) | |
bafaecd1 LT |
133 | restore_common_regs |
134 | ret | |
39f2205e JB |
135 | CFI_ENDPROC |
136 | ENDPROC(call_rwsem_downgrade_wake) |