Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef __ARCH_S390_PERCPU__ |
2 | #define __ARCH_S390_PERCPU__ | |
3 | ||
4c2241fd HC |
4 | #include <linux/preempt.h> |
5 | #include <asm/cmpxchg.h> | |
6 | ||
1da177e4 LT |
7 | /* |
8 | * s390 uses its own implementation for per cpu data, the offset of | |
9 | * the cpu local data area is cached in the cpu's lowcore memory. | |
1da177e4 | 10 | */ |
9a0ef292 | 11 | #define __my_cpu_offset S390_lowcore.percpu_offset |
1da177e4 | 12 | |
0702fbf5 HC |
13 | #ifdef CONFIG_64BIT |
14 | ||
9a0ef292 TH |
15 | /* |
16 | * For 64 bit module code, the module may be more than 4G above the | |
17 | * per cpu area, use weak definitions to force the compiler to | |
18 | * generate external references. | |
19 | */ | |
0702fbf5 | 20 | #if defined(CONFIG_SMP) && defined(MODULE) |
9a0ef292 | 21 | #define ARCH_NEEDS_WEAK_PER_CPU |
1da177e4 LT |
22 | #endif |
23 | ||
f84cd97e HC |
24 | /* |
25 | * We use a compare-and-swap loop since that uses less cpu cycles than | |
26 | * disabling and enabling interrupts like the generic variant would do. | |
27 | */ | |
28 | #define arch_this_cpu_to_op_simple(pcp, val, op) \ | |
ba6f5c2a | 29 | ({ \ |
4c2241fd HC |
30 | typedef typeof(pcp) pcp_op_T__; \ |
31 | pcp_op_T__ old__, new__, prev__; \ | |
32 | pcp_op_T__ *ptr__; \ | |
33 | preempt_disable(); \ | |
34 | ptr__ = __this_cpu_ptr(&(pcp)); \ | |
35 | prev__ = *ptr__; \ | |
36 | do { \ | |
37 | old__ = prev__; \ | |
38 | new__ = old__ op (val); \ | |
0702fbf5 | 39 | prev__ = cmpxchg(ptr__, old__, new__); \ |
4c2241fd HC |
40 | } while (prev__ != old__); \ |
41 | preempt_enable(); \ | |
ba6f5c2a HC |
42 | new__; \ |
43 | }) | |
4c2241fd | 44 | |
f84cd97e HC |
45 | #define this_cpu_add_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) |
46 | #define this_cpu_add_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) | |
47 | #define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) | |
48 | #define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) | |
49 | #define this_cpu_and_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) | |
50 | #define this_cpu_and_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) | |
51 | #define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) | |
52 | #define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) | |
f84cd97e HC |
53 | |
54 | #ifndef CONFIG_HAVE_MARCH_Z196_FEATURES | |
55 | ||
56 | #define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) | |
57 | #define this_cpu_add_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) | |
58 | #define this_cpu_add_return_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) | |
59 | #define this_cpu_add_return_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) | |
60 | #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) | |
61 | #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) | |
62 | #define this_cpu_or_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) | |
63 | #define this_cpu_or_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) | |
f84cd97e HC |
64 | |
65 | #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ | |
66 | ||
67 | #define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ | |
68 | { \ | |
69 | typedef typeof(pcp) pcp_op_T__; \ | |
70 | pcp_op_T__ val__ = (val); \ | |
71 | pcp_op_T__ old__, *ptr__; \ | |
72 | preempt_disable(); \ | |
73 | ptr__ = __this_cpu_ptr(&(pcp)); \ | |
74 | if (__builtin_constant_p(val__) && \ | |
75 | ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ | |
76 | asm volatile( \ | |
77 | op2 " %[ptr__],%[val__]\n" \ | |
78 | : [ptr__] "+Q" (*ptr__) \ | |
79 | : [val__] "i" ((szcast)val__) \ | |
80 | : "cc"); \ | |
81 | } else { \ | |
82 | asm volatile( \ | |
83 | op1 " %[old__],%[val__],%[ptr__]\n" \ | |
84 | : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ | |
85 | : [val__] "d" (val__) \ | |
86 | : "cc"); \ | |
87 | } \ | |
88 | preempt_enable(); \ | |
89 | } | |
90 | ||
91 | #define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int) | |
92 | #define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long) | |
93 | ||
94 | #define arch_this_cpu_add_return(pcp, val, op) \ | |
95 | ({ \ | |
96 | typedef typeof(pcp) pcp_op_T__; \ | |
97 | pcp_op_T__ val__ = (val); \ | |
98 | pcp_op_T__ old__, *ptr__; \ | |
99 | preempt_disable(); \ | |
100 | ptr__ = __this_cpu_ptr(&(pcp)); \ | |
101 | asm volatile( \ | |
102 | op " %[old__],%[val__],%[ptr__]\n" \ | |
103 | : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ | |
104 | : [val__] "d" (val__) \ | |
105 | : "cc"); \ | |
106 | preempt_enable(); \ | |
107 | old__ + val__; \ | |
108 | }) | |
109 | ||
110 | #define this_cpu_add_return_4(pcp, val) arch_this_cpu_add_return(pcp, val, "laa") | |
111 | #define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag") | |
112 | ||
113 | #define arch_this_cpu_to_op(pcp, val, op) \ | |
114 | { \ | |
115 | typedef typeof(pcp) pcp_op_T__; \ | |
116 | pcp_op_T__ val__ = (val); \ | |
117 | pcp_op_T__ old__, *ptr__; \ | |
118 | preempt_disable(); \ | |
119 | ptr__ = __this_cpu_ptr(&(pcp)); \ | |
120 | asm volatile( \ | |
121 | op " %[old__],%[val__],%[ptr__]\n" \ | |
122 | : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ | |
123 | : [val__] "d" (val__) \ | |
124 | : "cc"); \ | |
125 | preempt_enable(); \ | |
126 | } | |
127 | ||
128 | #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lan") | |
129 | #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op(pcp, val, "lang") | |
130 | #define this_cpu_or_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lao") | |
131 | #define this_cpu_or_8(pcp, val) arch_this_cpu_to_op(pcp, val, "laog") | |
f84cd97e HC |
132 | |
133 | #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ | |
4c2241fd | 134 | |
b1d6b40c | 135 | #define arch_this_cpu_cmpxchg(pcp, oval, nval) \ |
4c2241fd HC |
136 | ({ \ |
137 | typedef typeof(pcp) pcp_op_T__; \ | |
138 | pcp_op_T__ ret__; \ | |
139 | pcp_op_T__ *ptr__; \ | |
140 | preempt_disable(); \ | |
141 | ptr__ = __this_cpu_ptr(&(pcp)); \ | |
0702fbf5 | 142 | ret__ = cmpxchg(ptr__, oval, nval); \ |
4c2241fd HC |
143 | preempt_enable(); \ |
144 | ret__; \ | |
145 | }) | |
146 | ||
933393f5 CL |
147 | #define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) |
148 | #define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) | |
149 | #define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) | |
150 | #define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) | |
4c2241fd | 151 | |
28634a07 HC |
152 | #define arch_this_cpu_xchg(pcp, nval) \ |
153 | ({ \ | |
154 | typeof(pcp) *ptr__; \ | |
155 | typeof(pcp) ret__; \ | |
156 | preempt_disable(); \ | |
157 | ptr__ = __this_cpu_ptr(&(pcp)); \ | |
158 | ret__ = xchg(ptr__, nval); \ | |
159 | preempt_enable(); \ | |
160 | ret__; \ | |
161 | }) | |
162 | ||
163 | #define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval) | |
164 | #define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval) | |
165 | #define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval) | |
28634a07 | 166 | #define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval) |
28634a07 | 167 | |
b1d6b40c HC |
168 | #define arch_this_cpu_cmpxchg_double(pcp1, pcp2, o1, o2, n1, n2) \ |
169 | ({ \ | |
170 | typeof(pcp1) o1__ = (o1), n1__ = (n1); \ | |
171 | typeof(pcp2) o2__ = (o2), n2__ = (n2); \ | |
172 | typeof(pcp1) *p1__; \ | |
173 | typeof(pcp2) *p2__; \ | |
174 | int ret__; \ | |
175 | preempt_disable(); \ | |
176 | p1__ = __this_cpu_ptr(&(pcp1)); \ | |
177 | p2__ = __this_cpu_ptr(&(pcp2)); \ | |
178 | ret__ = __cmpxchg_double(p1__, p2__, o1__, o2__, n1__, n2__); \ | |
179 | preempt_enable(); \ | |
180 | ret__; \ | |
181 | }) | |
182 | ||
183 | #define this_cpu_cmpxchg_double_4 arch_this_cpu_cmpxchg_double | |
b1d6b40c | 184 | #define this_cpu_cmpxchg_double_8 arch_this_cpu_cmpxchg_double |
0702fbf5 HC |
185 | |
186 | #endif /* CONFIG_64BIT */ | |
b1d6b40c | 187 | |
f0343474 | 188 | #include <asm-generic/percpu.h> |
1da177e4 | 189 | |
1da177e4 | 190 | #endif /* __ARCH_S390_PERCPU__ */ |