Commit | Line | Data |
---|---|---|
94bd2708 DN |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
a374c57b | 6 | * Copyright (c) 2008-2009 Silicon Graphics, Inc. All Rights Reserved. |
94bd2708 DN |
7 | */ |
8 | ||
9 | /* | |
10 | * Cross Partition Communication (XPC) sn2-based functions. | |
11 | * | |
12 | * Architecture specific implementation of common functions. | |
13 | * | |
14 | */ | |
15 | ||
e17d416b | 16 | #include <linux/delay.h> |
94bd2708 | 17 | #include <asm/uncached.h> |
261f3b49 | 18 | #include <asm/sn/mspec.h> |
94bd2708 DN |
19 | #include <asm/sn/sn_sal.h> |
20 | #include "xpc.h" | |
21 | ||
ee6665e3 DN |
22 | /* |
23 | * Define the number of u64s required to represent all the C-brick nasids | |
24 | * as a bitmap. The cross-partition kernel modules deal only with | |
25 | * C-brick nasids, thus the need for bitmaps which don't account for | |
26 | * odd-numbered (non C-brick) nasids. | |
27 | */ | |
28 | #define XPC_MAX_PHYSNODES_SN2 (MAX_NUMALINK_NODES / 2) | |
29 | #define XP_NASID_MASK_BYTES_SN2 ((XPC_MAX_PHYSNODES_SN2 + 7) / 8) | |
30 | #define XP_NASID_MASK_WORDS_SN2 ((XPC_MAX_PHYSNODES_SN2 + 63) / 64) | |
31 | ||
32 | /* | |
33 | * Memory for XPC's amo variables is allocated by the MSPEC driver. These | |
34 | * pages are located in the lowest granule. The lowest granule uses 4k pages | |
35 | * for cached references and an alternate TLB handler to never provide a | |
36 | * cacheable mapping for the entire region. This will prevent speculative | |
37 | * reading of cached copies of our lines from being issued which will cause | |
38 | * a PI FSB Protocol error to be generated by the SHUB. For XPC, we need 64 | |
39 | * amo variables (based on XP_MAX_NPARTITIONS_SN2) to identify the senders of | |
40 | * NOTIFY IRQs, 128 amo variables (based on XP_NASID_MASK_WORDS_SN2) to identify | |
41 | * the senders of ACTIVATE IRQs, 1 amo variable to identify which remote | |
42 | * partitions (i.e., XPCs) consider themselves currently engaged with the | |
43 | * local XPC and 1 amo variable to request partition deactivation. | |
44 | */ | |
45 | #define XPC_NOTIFY_IRQ_AMOS_SN2 0 | |
46 | #define XPC_ACTIVATE_IRQ_AMOS_SN2 (XPC_NOTIFY_IRQ_AMOS_SN2 + \ | |
47 | XP_MAX_NPARTITIONS_SN2) | |
48 | #define XPC_ENGAGED_PARTITIONS_AMO_SN2 (XPC_ACTIVATE_IRQ_AMOS_SN2 + \ | |
49 | XP_NASID_MASK_WORDS_SN2) | |
50 | #define XPC_DEACTIVATE_REQUEST_AMO_SN2 (XPC_ENGAGED_PARTITIONS_AMO_SN2 + 1) | |
51 | ||
52 | /* | |
53 | * Buffer used to store a local copy of portions of a remote partition's | |
54 | * reserved page (either its header and part_nasids mask, or its vars). | |
55 | */ | |
ee6665e3 | 56 | static void *xpc_remote_copy_buffer_base_sn2; |
5b8669df | 57 | static char *xpc_remote_copy_buffer_sn2; |
ee6665e3 | 58 | |
8e85c23e DN |
59 | static struct xpc_vars_sn2 *xpc_vars_sn2; |
60 | static struct xpc_vars_part_sn2 *xpc_vars_part_sn2; | |
94bd2708 | 61 | |
5b8669df DN |
62 | static int |
63 | xpc_setup_partitions_sn_sn2(void) | |
64 | { | |
65 | /* nothing needs to be done */ | |
66 | return 0; | |
67 | } | |
68 | ||
6f2584f4 JS |
69 | static void |
70 | xpc_teardown_partitions_sn_sn2(void) | |
71 | { | |
72 | /* nothing needs to be done */ | |
73 | } | |
74 | ||
6e41017a | 75 | /* SH_IPI_ACCESS shub register value on startup */ |
8e85c23e DN |
76 | static u64 xpc_sh1_IPI_access_sn2; |
77 | static u64 xpc_sh2_IPI_access0_sn2; | |
78 | static u64 xpc_sh2_IPI_access1_sn2; | |
79 | static u64 xpc_sh2_IPI_access2_sn2; | |
80 | static u64 xpc_sh2_IPI_access3_sn2; | |
6e41017a DN |
81 | |
82 | /* | |
83 | * Change protections to allow IPI operations. | |
84 | */ | |
85 | static void | |
86 | xpc_allow_IPI_ops_sn2(void) | |
87 | { | |
88 | int node; | |
89 | int nasid; | |
90 | ||
ea57f80c | 91 | /* !!! The following should get moved into SAL. */ |
6e41017a | 92 | if (is_shub2()) { |
8e85c23e | 93 | xpc_sh2_IPI_access0_sn2 = |
6e41017a | 94 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS0)); |
8e85c23e | 95 | xpc_sh2_IPI_access1_sn2 = |
6e41017a | 96 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS1)); |
8e85c23e | 97 | xpc_sh2_IPI_access2_sn2 = |
6e41017a | 98 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS2)); |
8e85c23e | 99 | xpc_sh2_IPI_access3_sn2 = |
6e41017a DN |
100 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS3)); |
101 | ||
102 | for_each_online_node(node) { | |
103 | nasid = cnodeid_to_nasid(node); | |
104 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0), | |
105 | -1UL); | |
106 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1), | |
107 | -1UL); | |
108 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2), | |
109 | -1UL); | |
110 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3), | |
111 | -1UL); | |
112 | } | |
113 | } else { | |
8e85c23e | 114 | xpc_sh1_IPI_access_sn2 = |
6e41017a DN |
115 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH1_IPI_ACCESS)); |
116 | ||
117 | for_each_online_node(node) { | |
118 | nasid = cnodeid_to_nasid(node); | |
119 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS), | |
120 | -1UL); | |
121 | } | |
122 | } | |
123 | } | |
124 | ||
125 | /* | |
126 | * Restrict protections to disallow IPI operations. | |
127 | */ | |
128 | static void | |
129 | xpc_disallow_IPI_ops_sn2(void) | |
130 | { | |
131 | int node; | |
132 | int nasid; | |
133 | ||
ea57f80c | 134 | /* !!! The following should get moved into SAL. */ |
6e41017a DN |
135 | if (is_shub2()) { |
136 | for_each_online_node(node) { | |
137 | nasid = cnodeid_to_nasid(node); | |
138 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0), | |
8e85c23e | 139 | xpc_sh2_IPI_access0_sn2); |
6e41017a | 140 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1), |
8e85c23e | 141 | xpc_sh2_IPI_access1_sn2); |
6e41017a | 142 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2), |
8e85c23e | 143 | xpc_sh2_IPI_access2_sn2); |
6e41017a | 144 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3), |
8e85c23e | 145 | xpc_sh2_IPI_access3_sn2); |
6e41017a DN |
146 | } |
147 | } else { | |
148 | for_each_online_node(node) { | |
149 | nasid = cnodeid_to_nasid(node); | |
150 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS), | |
8e85c23e | 151 | xpc_sh1_IPI_access_sn2); |
6e41017a DN |
152 | } |
153 | } | |
154 | } | |
155 | ||
33ba3c77 | 156 | /* |
7fb5e59d DN |
157 | * The following set of functions are used for the sending and receiving of |
158 | * IRQs (also known as IPIs). There are two flavors of IRQs, one that is | |
159 | * associated with partition activity (SGI_XPC_ACTIVATE) and the other that | |
160 | * is associated with channel activity (SGI_XPC_NOTIFY). | |
33ba3c77 DN |
161 | */ |
162 | ||
163 | static u64 | |
c39838ce | 164 | xpc_receive_IRQ_amo_sn2(struct amo *amo) |
33ba3c77 DN |
165 | { |
166 | return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_CLEAR); | |
167 | } | |
168 | ||
169 | static enum xp_retval | |
c39838ce DN |
170 | xpc_send_IRQ_sn2(struct amo *amo, u64 flag, int nasid, int phys_cpuid, |
171 | int vector) | |
33ba3c77 DN |
172 | { |
173 | int ret = 0; | |
174 | unsigned long irq_flags; | |
175 | ||
176 | local_irq_save(irq_flags); | |
177 | ||
178 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, flag); | |
179 | sn_send_IPI_phys(nasid, phys_cpuid, vector, 0); | |
180 | ||
181 | /* | |
182 | * We must always use the nofault function regardless of whether we | |
183 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
184 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 185 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
186 | */ |
187 | ret = xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->variable), | |
188 | xp_nofault_PIOR_target)); | |
189 | ||
190 | local_irq_restore(irq_flags); | |
191 | ||
261f3b49 | 192 | return (ret == 0) ? xpSuccess : xpPioReadError; |
33ba3c77 DN |
193 | } |
194 | ||
c39838ce | 195 | static struct amo * |
7fb5e59d | 196 | xpc_init_IRQ_amo_sn2(int index) |
33ba3c77 | 197 | { |
8e85c23e | 198 | struct amo *amo = xpc_vars_sn2->amos_page + index; |
33ba3c77 | 199 | |
c39838ce | 200 | (void)xpc_receive_IRQ_amo_sn2(amo); /* clear amo variable */ |
33ba3c77 DN |
201 | return amo; |
202 | } | |
203 | ||
204 | /* | |
7fb5e59d | 205 | * Functions associated with SGI_XPC_ACTIVATE IRQ. |
33ba3c77 DN |
206 | */ |
207 | ||
6e41017a DN |
208 | /* |
209 | * Notify the heartbeat check thread that an activate IRQ has been received. | |
210 | */ | |
211 | static irqreturn_t | |
212 | xpc_handle_activate_IRQ_sn2(int irq, void *dev_id) | |
213 | { | |
5b8669df DN |
214 | unsigned long irq_flags; |
215 | ||
216 | spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); | |
217 | xpc_activate_IRQ_rcvd++; | |
218 | spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); | |
219 | ||
6e41017a DN |
220 | wake_up_interruptible(&xpc_activate_IRQ_wq); |
221 | return IRQ_HANDLED; | |
222 | } | |
223 | ||
33ba3c77 | 224 | /* |
c39838ce | 225 | * Flag the appropriate amo variable and send an IRQ to the specified node. |
33ba3c77 DN |
226 | */ |
227 | static void | |
a812dcc3 DN |
228 | xpc_send_activate_IRQ_sn2(unsigned long amos_page_pa, int from_nasid, |
229 | int to_nasid, int to_phys_cpuid) | |
33ba3c77 | 230 | { |
c39838ce | 231 | struct amo *amos = (struct amo *)__va(amos_page_pa + |
ee6665e3 | 232 | (XPC_ACTIVATE_IRQ_AMOS_SN2 * |
c39838ce | 233 | sizeof(struct amo))); |
33ba3c77 | 234 | |
04de7418 DN |
235 | (void)xpc_send_IRQ_sn2(&amos[BIT_WORD(from_nasid / 2)], |
236 | BIT_MASK(from_nasid / 2), to_nasid, | |
33ba3c77 DN |
237 | to_phys_cpuid, SGI_XPC_ACTIVATE); |
238 | } | |
239 | ||
240 | static void | |
7fb5e59d | 241 | xpc_send_local_activate_IRQ_sn2(int from_nasid) |
33ba3c77 | 242 | { |
5b8669df | 243 | unsigned long irq_flags; |
8e85c23e | 244 | struct amo *amos = (struct amo *)__va(xpc_vars_sn2->amos_page_pa + |
ee6665e3 | 245 | (XPC_ACTIVATE_IRQ_AMOS_SN2 * |
c39838ce | 246 | sizeof(struct amo))); |
33ba3c77 DN |
247 | |
248 | /* fake the sending and receipt of an activate IRQ from remote nasid */ | |
04de7418 DN |
249 | FETCHOP_STORE_OP(TO_AMO((u64)&amos[BIT_WORD(from_nasid / 2)].variable), |
250 | FETCHOP_OR, BIT_MASK(from_nasid / 2)); | |
251 | ||
5b8669df DN |
252 | spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); |
253 | xpc_activate_IRQ_rcvd++; | |
254 | spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); | |
255 | ||
6e41017a | 256 | wake_up_interruptible(&xpc_activate_IRQ_wq); |
33ba3c77 DN |
257 | } |
258 | ||
a47d5dac | 259 | /* |
7fb5e59d | 260 | * Functions associated with SGI_XPC_NOTIFY IRQ. |
a47d5dac | 261 | */ |
33ba3c77 | 262 | |
a47d5dac | 263 | /* |
7fb5e59d | 264 | * Check to see if any chctl flags were sent from the specified partition. |
a47d5dac | 265 | */ |
33ba3c77 | 266 | static void |
7fb5e59d | 267 | xpc_check_for_sent_chctl_flags_sn2(struct xpc_partition *part) |
33ba3c77 | 268 | { |
7fb5e59d | 269 | union xpc_channel_ctl_flags chctl; |
a47d5dac | 270 | unsigned long irq_flags; |
33ba3c77 | 271 | |
7fb5e59d DN |
272 | chctl.all_flags = xpc_receive_IRQ_amo_sn2(part->sn.sn2. |
273 | local_chctl_amo_va); | |
274 | if (chctl.all_flags == 0) | |
a47d5dac DN |
275 | return; |
276 | ||
7fb5e59d DN |
277 | spin_lock_irqsave(&part->chctl_lock, irq_flags); |
278 | part->chctl.all_flags |= chctl.all_flags; | |
279 | spin_unlock_irqrestore(&part->chctl_lock, irq_flags); | |
a47d5dac | 280 | |
7fb5e59d DN |
281 | dev_dbg(xpc_chan, "received notify IRQ from partid=%d, chctl.all_flags=" |
282 | "0x%lx\n", XPC_PARTID(part), chctl.all_flags); | |
a47d5dac DN |
283 | |
284 | xpc_wakeup_channel_mgr(part); | |
33ba3c77 DN |
285 | } |
286 | ||
a47d5dac DN |
287 | /* |
288 | * Handle the receipt of a SGI_XPC_NOTIFY IRQ by seeing whether the specified | |
289 | * partition actually sent it. Since SGI_XPC_NOTIFY IRQs may be shared by more | |
c39838ce | 290 | * than one partition, we use an amo structure per partition to indicate |
7fb5e59d | 291 | * whether a partition has sent an IRQ or not. If it has, then wake up the |
a47d5dac DN |
292 | * associated kthread to handle it. |
293 | * | |
7fb5e59d | 294 | * All SGI_XPC_NOTIFY IRQs received by XPC are the result of IRQs sent by XPC |
a47d5dac DN |
295 | * running on other partitions. |
296 | * | |
297 | * Noteworthy Arguments: | |
298 | * | |
299 | * irq - Interrupt ReQuest number. NOT USED. | |
300 | * | |
7fb5e59d | 301 | * dev_id - partid of IRQ's potential sender. |
a47d5dac DN |
302 | */ |
303 | static irqreturn_t | |
304 | xpc_handle_notify_IRQ_sn2(int irq, void *dev_id) | |
33ba3c77 | 305 | { |
a47d5dac DN |
306 | short partid = (short)(u64)dev_id; |
307 | struct xpc_partition *part = &xpc_partitions[partid]; | |
308 | ||
261f3b49 | 309 | DBUG_ON(partid < 0 || partid >= XP_MAX_NPARTITIONS_SN2); |
a47d5dac DN |
310 | |
311 | if (xpc_part_ref(part)) { | |
7fb5e59d | 312 | xpc_check_for_sent_chctl_flags_sn2(part); |
a47d5dac DN |
313 | |
314 | xpc_part_deref(part); | |
315 | } | |
316 | return IRQ_HANDLED; | |
33ba3c77 DN |
317 | } |
318 | ||
319 | /* | |
7fb5e59d DN |
320 | * Check to see if xpc_handle_notify_IRQ_sn2() dropped any IRQs on the floor |
321 | * because the write to their associated amo variable completed after the IRQ | |
a47d5dac | 322 | * was received. |
33ba3c77 | 323 | */ |
a47d5dac | 324 | static void |
7fb5e59d | 325 | xpc_check_for_dropped_notify_IRQ_sn2(struct xpc_partition *part) |
a47d5dac DN |
326 | { |
327 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; | |
328 | ||
329 | if (xpc_part_ref(part)) { | |
7fb5e59d | 330 | xpc_check_for_sent_chctl_flags_sn2(part); |
a47d5dac DN |
331 | |
332 | part_sn2->dropped_notify_IRQ_timer.expires = jiffies + | |
7fb5e59d | 333 | XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL; |
a47d5dac DN |
334 | add_timer(&part_sn2->dropped_notify_IRQ_timer); |
335 | xpc_part_deref(part); | |
336 | } | |
337 | } | |
33ba3c77 DN |
338 | |
339 | /* | |
7fb5e59d | 340 | * Send a notify IRQ to the remote partition that is associated with the |
33ba3c77 DN |
341 | * specified channel. |
342 | */ | |
343 | static void | |
7fb5e59d DN |
344 | xpc_send_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag, |
345 | char *chctl_flag_string, unsigned long *irq_flags) | |
33ba3c77 DN |
346 | { |
347 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
a47d5dac | 348 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
7fb5e59d | 349 | union xpc_channel_ctl_flags chctl = { 0 }; |
33ba3c77 DN |
350 | enum xp_retval ret; |
351 | ||
83469b55 | 352 | if (likely(part->act_state != XPC_P_AS_DEACTIVATING)) { |
7fb5e59d DN |
353 | chctl.flags[ch->number] = chctl_flag; |
354 | ret = xpc_send_IRQ_sn2(part_sn2->remote_chctl_amo_va, | |
355 | chctl.all_flags, | |
356 | part_sn2->notify_IRQ_nasid, | |
357 | part_sn2->notify_IRQ_phys_cpuid, | |
33ba3c77 DN |
358 | SGI_XPC_NOTIFY); |
359 | dev_dbg(xpc_chan, "%s sent to partid=%d, channel=%d, ret=%d\n", | |
7fb5e59d | 360 | chctl_flag_string, ch->partid, ch->number, ret); |
33ba3c77 DN |
361 | if (unlikely(ret != xpSuccess)) { |
362 | if (irq_flags != NULL) | |
363 | spin_unlock_irqrestore(&ch->lock, *irq_flags); | |
364 | XPC_DEACTIVATE_PARTITION(part, ret); | |
365 | if (irq_flags != NULL) | |
366 | spin_lock_irqsave(&ch->lock, *irq_flags); | |
367 | } | |
368 | } | |
369 | } | |
370 | ||
7fb5e59d DN |
371 | #define XPC_SEND_NOTIFY_IRQ_SN2(_ch, _ipi_f, _irq_f) \ |
372 | xpc_send_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f, _irq_f) | |
33ba3c77 DN |
373 | |
374 | /* | |
375 | * Make it look like the remote partition, which is associated with the | |
7fb5e59d DN |
376 | * specified channel, sent us a notify IRQ. This faked IRQ will be handled |
377 | * by xpc_check_for_dropped_notify_IRQ_sn2(). | |
33ba3c77 DN |
378 | */ |
379 | static void | |
7fb5e59d DN |
380 | xpc_send_local_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag, |
381 | char *chctl_flag_string) | |
33ba3c77 DN |
382 | { |
383 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
7fb5e59d | 384 | union xpc_channel_ctl_flags chctl = { 0 }; |
33ba3c77 | 385 | |
7fb5e59d DN |
386 | chctl.flags[ch->number] = chctl_flag; |
387 | FETCHOP_STORE_OP(TO_AMO((u64)&part->sn.sn2.local_chctl_amo_va-> | |
388 | variable), FETCHOP_OR, chctl.all_flags); | |
33ba3c77 | 389 | dev_dbg(xpc_chan, "%s sent local from partid=%d, channel=%d\n", |
7fb5e59d | 390 | chctl_flag_string, ch->partid, ch->number); |
33ba3c77 DN |
391 | } |
392 | ||
7fb5e59d DN |
393 | #define XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(_ch, _ipi_f) \ |
394 | xpc_send_local_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f) | |
33ba3c77 DN |
395 | |
396 | static void | |
7fb5e59d DN |
397 | xpc_send_chctl_closerequest_sn2(struct xpc_channel *ch, |
398 | unsigned long *irq_flags) | |
33ba3c77 | 399 | { |
5b8669df | 400 | struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args; |
33ba3c77 DN |
401 | |
402 | args->reason = ch->reason; | |
7fb5e59d | 403 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREQUEST, irq_flags); |
33ba3c77 DN |
404 | } |
405 | ||
406 | static void | |
7fb5e59d | 407 | xpc_send_chctl_closereply_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 | 408 | { |
7fb5e59d | 409 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREPLY, irq_flags); |
33ba3c77 DN |
410 | } |
411 | ||
412 | static void | |
7fb5e59d | 413 | xpc_send_chctl_openrequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 | 414 | { |
5b8669df | 415 | struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args; |
33ba3c77 | 416 | |
bd3e64c1 | 417 | args->entry_size = ch->entry_size; |
33ba3c77 | 418 | args->local_nentries = ch->local_nentries; |
7fb5e59d | 419 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREQUEST, irq_flags); |
33ba3c77 DN |
420 | } |
421 | ||
422 | static void | |
7fb5e59d | 423 | xpc_send_chctl_openreply_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 | 424 | { |
5b8669df | 425 | struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args; |
33ba3c77 DN |
426 | |
427 | args->remote_nentries = ch->remote_nentries; | |
428 | args->local_nentries = ch->local_nentries; | |
5b8669df | 429 | args->local_msgqueue_pa = xp_pa(ch->sn.sn2.local_msgqueue); |
7fb5e59d | 430 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREPLY, irq_flags); |
33ba3c77 DN |
431 | } |
432 | ||
433 | static void | |
7fb5e59d | 434 | xpc_send_chctl_msgrequest_sn2(struct xpc_channel *ch) |
33ba3c77 | 435 | { |
7fb5e59d | 436 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST, NULL); |
33ba3c77 DN |
437 | } |
438 | ||
439 | static void | |
7fb5e59d | 440 | xpc_send_chctl_local_msgrequest_sn2(struct xpc_channel *ch) |
33ba3c77 | 441 | { |
7fb5e59d | 442 | XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST); |
33ba3c77 DN |
443 | } |
444 | ||
6f2584f4 | 445 | static enum xp_retval |
5b8669df DN |
446 | xpc_save_remote_msgqueue_pa_sn2(struct xpc_channel *ch, |
447 | unsigned long msgqueue_pa) | |
448 | { | |
449 | ch->sn.sn2.remote_msgqueue_pa = msgqueue_pa; | |
6f2584f4 | 450 | return xpSuccess; |
5b8669df DN |
451 | } |
452 | ||
33ba3c77 DN |
453 | /* |
454 | * This next set of functions are used to keep track of when a partition is | |
455 | * potentially engaged in accessing memory belonging to another partition. | |
456 | */ | |
457 | ||
458 | static void | |
a47d5dac | 459 | xpc_indicate_partition_engaged_sn2(struct xpc_partition *part) |
33ba3c77 DN |
460 | { |
461 | unsigned long irq_flags; | |
c39838ce | 462 | struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa + |
ee6665e3 | 463 | (XPC_ENGAGED_PARTITIONS_AMO_SN2 * |
c39838ce | 464 | sizeof(struct amo))); |
33ba3c77 DN |
465 | |
466 | local_irq_save(irq_flags); | |
467 | ||
c39838ce | 468 | /* set bit corresponding to our partid in remote partition's amo */ |
33ba3c77 | 469 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, |
04de7418 DN |
470 | BIT(sn_partition_id)); |
471 | ||
33ba3c77 DN |
472 | /* |
473 | * We must always use the nofault function regardless of whether we | |
474 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
475 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 476 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
477 | */ |
478 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
479 | variable), | |
480 | xp_nofault_PIOR_target)); | |
481 | ||
482 | local_irq_restore(irq_flags); | |
483 | } | |
484 | ||
485 | static void | |
a47d5dac | 486 | xpc_indicate_partition_disengaged_sn2(struct xpc_partition *part) |
33ba3c77 | 487 | { |
a47d5dac | 488 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
33ba3c77 | 489 | unsigned long irq_flags; |
c39838ce | 490 | struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa + |
ee6665e3 | 491 | (XPC_ENGAGED_PARTITIONS_AMO_SN2 * |
c39838ce | 492 | sizeof(struct amo))); |
33ba3c77 DN |
493 | |
494 | local_irq_save(irq_flags); | |
495 | ||
c39838ce | 496 | /* clear bit corresponding to our partid in remote partition's amo */ |
33ba3c77 | 497 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, |
04de7418 DN |
498 | ~BIT(sn_partition_id)); |
499 | ||
33ba3c77 DN |
500 | /* |
501 | * We must always use the nofault function regardless of whether we | |
502 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
503 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 504 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
505 | */ |
506 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
507 | variable), | |
508 | xp_nofault_PIOR_target)); | |
509 | ||
510 | local_irq_restore(irq_flags); | |
33ba3c77 | 511 | |
33ba3c77 | 512 | /* |
a47d5dac | 513 | * Send activate IRQ to get other side to see that we've cleared our |
c39838ce | 514 | * bit in their engaged partitions amo. |
33ba3c77 | 515 | */ |
7fb5e59d | 516 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
517 | cnodeid_to_nasid(0), |
518 | part_sn2->activate_IRQ_nasid, | |
519 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 DN |
520 | } |
521 | ||
5b8669df DN |
522 | static void |
523 | xpc_assume_partition_disengaged_sn2(short partid) | |
524 | { | |
525 | struct amo *amo = xpc_vars_sn2->amos_page + | |
526 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
527 | ||
528 | /* clear bit(s) based on partid mask in our partition's amo */ | |
529 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, | |
530 | ~BIT(partid)); | |
531 | } | |
532 | ||
a47d5dac DN |
533 | static int |
534 | xpc_partition_engaged_sn2(short partid) | |
33ba3c77 | 535 | { |
8e85c23e DN |
536 | struct amo *amo = xpc_vars_sn2->amos_page + |
537 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
33ba3c77 | 538 | |
c39838ce | 539 | /* our partition's amo variable ANDed with partid mask */ |
33ba3c77 | 540 | return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) & |
04de7418 | 541 | BIT(partid)) != 0; |
33ba3c77 DN |
542 | } |
543 | ||
a47d5dac DN |
544 | static int |
545 | xpc_any_partition_engaged_sn2(void) | |
33ba3c77 | 546 | { |
8e85c23e DN |
547 | struct amo *amo = xpc_vars_sn2->amos_page + |
548 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
33ba3c77 | 549 | |
c39838ce | 550 | /* our partition's amo variable */ |
a47d5dac | 551 | return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) != 0; |
33ba3c77 DN |
552 | } |
553 | ||
6e41017a DN |
554 | /* original protection values for each node */ |
555 | static u64 xpc_prot_vec_sn2[MAX_NUMNODES]; | |
556 | ||
557 | /* | |
c39838ce | 558 | * Change protections to allow amo operations on non-Shub 1.1 systems. |
6e41017a DN |
559 | */ |
560 | static enum xp_retval | |
c39838ce | 561 | xpc_allow_amo_ops_sn2(struct amo *amos_page) |
6e41017a | 562 | { |
6c1c325d | 563 | enum xp_retval ret = xpSuccess; |
6e41017a DN |
564 | |
565 | /* | |
566 | * On SHUB 1.1, we cannot call sn_change_memprotect() since the BIST | |
567 | * collides with memory operations. On those systems we call | |
c39838ce | 568 | * xpc_allow_amo_ops_shub_wars_1_1_sn2() instead. |
6e41017a | 569 | */ |
6c1c325d DN |
570 | if (!enable_shub_wars_1_1()) |
571 | ret = xp_expand_memprotect(ia64_tpa((u64)amos_page), PAGE_SIZE); | |
572 | ||
573 | return ret; | |
6e41017a DN |
574 | } |
575 | ||
576 | /* | |
c39838ce | 577 | * Change protections to allow amo operations on Shub 1.1 systems. |
6e41017a DN |
578 | */ |
579 | static void | |
c39838ce | 580 | xpc_allow_amo_ops_shub_wars_1_1_sn2(void) |
6e41017a DN |
581 | { |
582 | int node; | |
583 | int nasid; | |
584 | ||
585 | if (!enable_shub_wars_1_1()) | |
586 | return; | |
587 | ||
588 | for_each_online_node(node) { | |
589 | nasid = cnodeid_to_nasid(node); | |
590 | /* save current protection values */ | |
591 | xpc_prot_vec_sn2[node] = | |
592 | (u64)HUB_L((u64 *)GLOBAL_MMR_ADDR(nasid, | |
593 | SH1_MD_DQLP_MMR_DIR_PRIVEC0)); | |
594 | /* open up everything */ | |
595 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, | |
596 | SH1_MD_DQLP_MMR_DIR_PRIVEC0), | |
597 | -1UL); | |
598 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, | |
599 | SH1_MD_DQRP_MMR_DIR_PRIVEC0), | |
600 | -1UL); | |
601 | } | |
602 | } | |
603 | ||
261f3b49 | 604 | static enum xp_retval |
a812dcc3 | 605 | xpc_get_partition_rsvd_page_pa_sn2(void *buf, u64 *cookie, unsigned long *rp_pa, |
261f3b49 DN |
606 | size_t *len) |
607 | { | |
608 | s64 status; | |
609 | enum xp_retval ret; | |
610 | ||
a812dcc3 | 611 | status = sn_partition_reserved_page_pa((u64)buf, cookie, rp_pa, len); |
261f3b49 DN |
612 | if (status == SALRET_OK) |
613 | ret = xpSuccess; | |
614 | else if (status == SALRET_MORE_PASSES) | |
615 | ret = xpNeedMoreInfo; | |
616 | else | |
617 | ret = xpSalError; | |
618 | ||
619 | return ret; | |
620 | } | |
621 | ||
622 | ||
5b8669df DN |
623 | static int |
624 | xpc_setup_rsvd_page_sn_sn2(struct xpc_rsvd_page *rp) | |
94bd2708 | 625 | { |
c39838ce | 626 | struct amo *amos_page; |
94bd2708 DN |
627 | int i; |
628 | int ret; | |
629 | ||
8e85c23e | 630 | xpc_vars_sn2 = XPC_RP_VARS(rp); |
94bd2708 | 631 | |
a374c57b | 632 | rp->sn.sn2.vars_pa = xp_pa(xpc_vars_sn2); |
94bd2708 | 633 | |
e17d416b | 634 | /* vars_part array follows immediately after vars */ |
8e85c23e DN |
635 | xpc_vars_part_sn2 = (struct xpc_vars_part_sn2 *)((u8 *)XPC_RP_VARS(rp) + |
636 | XPC_RP_VARS_SIZE); | |
e17d416b | 637 | |
94bd2708 | 638 | /* |
8e85c23e DN |
639 | * Before clearing xpc_vars_sn2, see if a page of amos had been |
640 | * previously allocated. If not we'll need to allocate one and set | |
641 | * permissions so that cross-partition amos are allowed. | |
94bd2708 | 642 | * |
c39838ce | 643 | * The allocated amo page needs MCA reporting to remain disabled after |
94bd2708 | 644 | * XPC has unloaded. To make this work, we keep a copy of the pointer |
8e85c23e | 645 | * to this page (i.e., amos_page) in the struct xpc_vars_sn2 structure, |
94bd2708 | 646 | * which is pointed to by the reserved page, and re-use that saved copy |
c39838ce | 647 | * on subsequent loads of XPC. This amo page is never freed, and its |
94bd2708 DN |
648 | * memory protections are never restricted. |
649 | */ | |
8e85c23e | 650 | amos_page = xpc_vars_sn2->amos_page; |
94bd2708 | 651 | if (amos_page == NULL) { |
c39838ce | 652 | amos_page = (struct amo *)TO_AMO(uncached_alloc_page(0, 1)); |
94bd2708 | 653 | if (amos_page == NULL) { |
c39838ce | 654 | dev_err(xpc_part, "can't allocate page of amos\n"); |
5b8669df | 655 | return -ENOMEM; |
94bd2708 DN |
656 | } |
657 | ||
658 | /* | |
c39838ce DN |
659 | * Open up amo-R/W to cpu. This is done on Shub 1.1 systems |
660 | * when xpc_allow_amo_ops_shub_wars_1_1_sn2() is called. | |
94bd2708 | 661 | */ |
c39838ce | 662 | ret = xpc_allow_amo_ops_sn2(amos_page); |
6e41017a | 663 | if (ret != xpSuccess) { |
c39838ce | 664 | dev_err(xpc_part, "can't allow amo operations\n"); |
6e41017a DN |
665 | uncached_free_page(__IA64_UNCACHED_OFFSET | |
666 | TO_PHYS((u64)amos_page), 1); | |
5b8669df | 667 | return -EPERM; |
94bd2708 DN |
668 | } |
669 | } | |
670 | ||
8e85c23e DN |
671 | /* clear xpc_vars_sn2 */ |
672 | memset(xpc_vars_sn2, 0, sizeof(struct xpc_vars_sn2)); | |
94bd2708 | 673 | |
8e85c23e DN |
674 | xpc_vars_sn2->version = XPC_V_VERSION; |
675 | xpc_vars_sn2->activate_IRQ_nasid = cpuid_to_nasid(0); | |
676 | xpc_vars_sn2->activate_IRQ_phys_cpuid = cpu_physical_id(0); | |
a812dcc3 | 677 | xpc_vars_sn2->vars_part_pa = xp_pa(xpc_vars_part_sn2); |
8e85c23e DN |
678 | xpc_vars_sn2->amos_page_pa = ia64_tpa((u64)amos_page); |
679 | xpc_vars_sn2->amos_page = amos_page; /* save for next load of XPC */ | |
94bd2708 | 680 | |
8e85c23e DN |
681 | /* clear xpc_vars_part_sn2 */ |
682 | memset((u64 *)xpc_vars_part_sn2, 0, sizeof(struct xpc_vars_part_sn2) * | |
261f3b49 | 683 | XP_MAX_NPARTITIONS_SN2); |
94bd2708 | 684 | |
c39838ce | 685 | /* initialize the activate IRQ related amo variables */ |
04de7418 | 686 | for (i = 0; i < xpc_nasid_mask_nlongs; i++) |
ee6665e3 | 687 | (void)xpc_init_IRQ_amo_sn2(XPC_ACTIVATE_IRQ_AMOS_SN2 + i); |
94bd2708 | 688 | |
c39838ce | 689 | /* initialize the engaged remote partitions related amo variables */ |
ee6665e3 DN |
690 | (void)xpc_init_IRQ_amo_sn2(XPC_ENGAGED_PARTITIONS_AMO_SN2); |
691 | (void)xpc_init_IRQ_amo_sn2(XPC_DEACTIVATE_REQUEST_AMO_SN2); | |
33ba3c77 | 692 | |
5b8669df | 693 | return 0; |
33ba3c77 DN |
694 | } |
695 | ||
a374c57b RH |
696 | static int |
697 | xpc_hb_allowed_sn2(short partid, void *heartbeating_to_mask) | |
698 | { | |
699 | return test_bit(partid, heartbeating_to_mask); | |
700 | } | |
701 | ||
702 | static void | |
703 | xpc_allow_hb_sn2(short partid) | |
704 | { | |
705 | DBUG_ON(xpc_vars_sn2 == NULL); | |
706 | set_bit(partid, xpc_vars_sn2->heartbeating_to_mask); | |
707 | } | |
708 | ||
709 | static void | |
710 | xpc_disallow_hb_sn2(short partid) | |
711 | { | |
712 | DBUG_ON(xpc_vars_sn2 == NULL); | |
713 | clear_bit(partid, xpc_vars_sn2->heartbeating_to_mask); | |
714 | } | |
715 | ||
716 | static void | |
717 | xpc_disallow_all_hbs_sn2(void) | |
718 | { | |
719 | DBUG_ON(xpc_vars_sn2 == NULL); | |
720 | bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, xp_max_npartitions); | |
721 | } | |
722 | ||
33ba3c77 DN |
723 | static void |
724 | xpc_increment_heartbeat_sn2(void) | |
725 | { | |
8e85c23e | 726 | xpc_vars_sn2->heartbeat++; |
33ba3c77 DN |
727 | } |
728 | ||
729 | static void | |
730 | xpc_offline_heartbeat_sn2(void) | |
731 | { | |
732 | xpc_increment_heartbeat_sn2(); | |
8e85c23e | 733 | xpc_vars_sn2->heartbeat_offline = 1; |
33ba3c77 DN |
734 | } |
735 | ||
736 | static void | |
737 | xpc_online_heartbeat_sn2(void) | |
738 | { | |
739 | xpc_increment_heartbeat_sn2(); | |
8e85c23e | 740 | xpc_vars_sn2->heartbeat_offline = 0; |
33ba3c77 DN |
741 | } |
742 | ||
743 | static void | |
744 | xpc_heartbeat_init_sn2(void) | |
745 | { | |
8e85c23e | 746 | DBUG_ON(xpc_vars_sn2 == NULL); |
33ba3c77 | 747 | |
8e85c23e | 748 | bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, XP_MAX_NPARTITIONS_SN2); |
33ba3c77 DN |
749 | xpc_online_heartbeat_sn2(); |
750 | } | |
751 | ||
752 | static void | |
753 | xpc_heartbeat_exit_sn2(void) | |
754 | { | |
755 | xpc_offline_heartbeat_sn2(); | |
756 | } | |
757 | ||
61deb86e DN |
758 | static enum xp_retval |
759 | xpc_get_remote_heartbeat_sn2(struct xpc_partition *part) | |
33ba3c77 DN |
760 | { |
761 | struct xpc_vars_sn2 *remote_vars; | |
33ba3c77 DN |
762 | enum xp_retval ret; |
763 | ||
ee6665e3 | 764 | remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2; |
33ba3c77 | 765 | |
61deb86e DN |
766 | /* pull the remote vars structure that contains the heartbeat */ |
767 | ret = xp_remote_memcpy(xp_pa(remote_vars), | |
768 | part->sn.sn2.remote_vars_pa, | |
769 | XPC_RP_VARS_SIZE); | |
770 | if (ret != xpSuccess) | |
771 | return ret; | |
33ba3c77 | 772 | |
61deb86e DN |
773 | dev_dbg(xpc_part, "partid=%d, heartbeat=%ld, last_heartbeat=%ld, " |
774 | "heartbeat_offline=%ld, HB_mask[0]=0x%lx\n", XPC_PARTID(part), | |
775 | remote_vars->heartbeat, part->last_heartbeat, | |
776 | remote_vars->heartbeat_offline, | |
777 | remote_vars->heartbeating_to_mask[0]); | |
778 | ||
779 | if ((remote_vars->heartbeat == part->last_heartbeat && | |
a374c57b RH |
780 | !remote_vars->heartbeat_offline) || |
781 | !xpc_hb_allowed_sn2(sn_partition_id, | |
782 | remote_vars->heartbeating_to_mask)) { | |
61deb86e DN |
783 | ret = xpNoHeartbeat; |
784 | } else { | |
33ba3c77 DN |
785 | part->last_heartbeat = remote_vars->heartbeat; |
786 | } | |
61deb86e DN |
787 | |
788 | return ret; | |
33ba3c77 DN |
789 | } |
790 | ||
791 | /* | |
792 | * Get a copy of the remote partition's XPC variables from the reserved page. | |
793 | * | |
794 | * remote_vars points to a buffer that is cacheline aligned for BTE copies and | |
795 | * assumed to be of size XPC_RP_VARS_SIZE. | |
796 | */ | |
797 | static enum xp_retval | |
a812dcc3 DN |
798 | xpc_get_remote_vars_sn2(unsigned long remote_vars_pa, |
799 | struct xpc_vars_sn2 *remote_vars) | |
33ba3c77 DN |
800 | { |
801 | enum xp_retval ret; | |
802 | ||
803 | if (remote_vars_pa == 0) | |
804 | return xpVarsNotSet; | |
805 | ||
806 | /* pull over the cross partition variables */ | |
a812dcc3 | 807 | ret = xp_remote_memcpy(xp_pa(remote_vars), remote_vars_pa, |
33ba3c77 DN |
808 | XPC_RP_VARS_SIZE); |
809 | if (ret != xpSuccess) | |
810 | return ret; | |
811 | ||
812 | if (XPC_VERSION_MAJOR(remote_vars->version) != | |
813 | XPC_VERSION_MAJOR(XPC_V_VERSION)) { | |
814 | return xpBadVersion; | |
815 | } | |
94bd2708 DN |
816 | |
817 | return xpSuccess; | |
818 | } | |
819 | ||
33ba3c77 | 820 | static void |
a47d5dac | 821 | xpc_request_partition_activation_sn2(struct xpc_rsvd_page *remote_rp, |
a812dcc3 | 822 | unsigned long remote_rp_pa, int nasid) |
33ba3c77 | 823 | { |
7fb5e59d | 824 | xpc_send_local_activate_IRQ_sn2(nasid); |
a47d5dac DN |
825 | } |
826 | ||
827 | static void | |
828 | xpc_request_partition_reactivation_sn2(struct xpc_partition *part) | |
829 | { | |
7fb5e59d | 830 | xpc_send_local_activate_IRQ_sn2(part->sn.sn2.activate_IRQ_nasid); |
a47d5dac DN |
831 | } |
832 | ||
833 | static void | |
834 | xpc_request_partition_deactivation_sn2(struct xpc_partition *part) | |
835 | { | |
836 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; | |
837 | unsigned long irq_flags; | |
c39838ce | 838 | struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa + |
ee6665e3 | 839 | (XPC_DEACTIVATE_REQUEST_AMO_SN2 * |
c39838ce | 840 | sizeof(struct amo))); |
a47d5dac DN |
841 | |
842 | local_irq_save(irq_flags); | |
843 | ||
c39838ce | 844 | /* set bit corresponding to our partid in remote partition's amo */ |
a47d5dac | 845 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, |
04de7418 DN |
846 | BIT(sn_partition_id)); |
847 | ||
a47d5dac DN |
848 | /* |
849 | * We must always use the nofault function regardless of whether we | |
850 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
851 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 852 | * keep sending IRQs and amos to it until the heartbeat times out. |
a47d5dac DN |
853 | */ |
854 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
855 | variable), | |
856 | xp_nofault_PIOR_target)); | |
857 | ||
858 | local_irq_restore(irq_flags); | |
859 | ||
860 | /* | |
861 | * Send activate IRQ to get other side to see that we've set our | |
c39838ce | 862 | * bit in their deactivate request amo. |
a47d5dac | 863 | */ |
7fb5e59d | 864 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
865 | cnodeid_to_nasid(0), |
866 | part_sn2->activate_IRQ_nasid, | |
867 | part_sn2->activate_IRQ_phys_cpuid); | |
868 | } | |
869 | ||
870 | static void | |
871 | xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition *part) | |
872 | { | |
873 | unsigned long irq_flags; | |
c39838ce | 874 | struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa + |
ee6665e3 | 875 | (XPC_DEACTIVATE_REQUEST_AMO_SN2 * |
c39838ce | 876 | sizeof(struct amo))); |
a47d5dac DN |
877 | |
878 | local_irq_save(irq_flags); | |
879 | ||
c39838ce | 880 | /* clear bit corresponding to our partid in remote partition's amo */ |
a47d5dac | 881 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, |
04de7418 DN |
882 | ~BIT(sn_partition_id)); |
883 | ||
a47d5dac DN |
884 | /* |
885 | * We must always use the nofault function regardless of whether we | |
886 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
887 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 888 | * keep sending IRQs and amos to it until the heartbeat times out. |
a47d5dac DN |
889 | */ |
890 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
891 | variable), | |
892 | xp_nofault_PIOR_target)); | |
893 | ||
894 | local_irq_restore(irq_flags); | |
895 | } | |
896 | ||
897 | static int | |
898 | xpc_partition_deactivation_requested_sn2(short partid) | |
899 | { | |
8e85c23e DN |
900 | struct amo *amo = xpc_vars_sn2->amos_page + |
901 | XPC_DEACTIVATE_REQUEST_AMO_SN2; | |
a47d5dac | 902 | |
c39838ce | 903 | /* our partition's amo variable ANDed with partid mask */ |
a47d5dac | 904 | return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) & |
04de7418 | 905 | BIT(partid)) != 0; |
33ba3c77 DN |
906 | } |
907 | ||
908 | /* | |
909 | * Update the remote partition's info. | |
910 | */ | |
911 | static void | |
912 | xpc_update_partition_info_sn2(struct xpc_partition *part, u8 remote_rp_version, | |
81fe7883 | 913 | unsigned long *remote_rp_ts_jiffies, |
a812dcc3 DN |
914 | unsigned long remote_rp_pa, |
915 | unsigned long remote_vars_pa, | |
33ba3c77 DN |
916 | struct xpc_vars_sn2 *remote_vars) |
917 | { | |
a47d5dac DN |
918 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
919 | ||
33ba3c77 DN |
920 | part->remote_rp_version = remote_rp_version; |
921 | dev_dbg(xpc_part, " remote_rp_version = 0x%016x\n", | |
922 | part->remote_rp_version); | |
923 | ||
81fe7883 DN |
924 | part->remote_rp_ts_jiffies = *remote_rp_ts_jiffies; |
925 | dev_dbg(xpc_part, " remote_rp_ts_jiffies = 0x%016lx\n", | |
926 | part->remote_rp_ts_jiffies); | |
33ba3c77 DN |
927 | |
928 | part->remote_rp_pa = remote_rp_pa; | |
929 | dev_dbg(xpc_part, " remote_rp_pa = 0x%016lx\n", part->remote_rp_pa); | |
930 | ||
a47d5dac | 931 | part_sn2->remote_vars_pa = remote_vars_pa; |
33ba3c77 | 932 | dev_dbg(xpc_part, " remote_vars_pa = 0x%016lx\n", |
a47d5dac | 933 | part_sn2->remote_vars_pa); |
33ba3c77 | 934 | |
158bc69e | 935 | part->last_heartbeat = remote_vars->heartbeat - 1; |
33ba3c77 DN |
936 | dev_dbg(xpc_part, " last_heartbeat = 0x%016lx\n", |
937 | part->last_heartbeat); | |
938 | ||
a47d5dac | 939 | part_sn2->remote_vars_part_pa = remote_vars->vars_part_pa; |
33ba3c77 | 940 | dev_dbg(xpc_part, " remote_vars_part_pa = 0x%016lx\n", |
a47d5dac | 941 | part_sn2->remote_vars_part_pa); |
33ba3c77 | 942 | |
a47d5dac DN |
943 | part_sn2->activate_IRQ_nasid = remote_vars->activate_IRQ_nasid; |
944 | dev_dbg(xpc_part, " activate_IRQ_nasid = 0x%x\n", | |
945 | part_sn2->activate_IRQ_nasid); | |
33ba3c77 | 946 | |
a47d5dac DN |
947 | part_sn2->activate_IRQ_phys_cpuid = |
948 | remote_vars->activate_IRQ_phys_cpuid; | |
949 | dev_dbg(xpc_part, " activate_IRQ_phys_cpuid = 0x%x\n", | |
950 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 | 951 | |
a47d5dac | 952 | part_sn2->remote_amos_page_pa = remote_vars->amos_page_pa; |
33ba3c77 | 953 | dev_dbg(xpc_part, " remote_amos_page_pa = 0x%lx\n", |
a47d5dac | 954 | part_sn2->remote_amos_page_pa); |
33ba3c77 | 955 | |
a47d5dac | 956 | part_sn2->remote_vars_version = remote_vars->version; |
33ba3c77 | 957 | dev_dbg(xpc_part, " remote_vars_version = 0x%x\n", |
a47d5dac | 958 | part_sn2->remote_vars_version); |
33ba3c77 DN |
959 | } |
960 | ||
961 | /* | |
7fb5e59d DN |
962 | * Prior code has determined the nasid which generated a activate IRQ. |
963 | * Inspect that nasid to determine if its partition needs to be activated | |
964 | * or deactivated. | |
33ba3c77 | 965 | * |
7fb5e59d | 966 | * A partition is considered "awaiting activation" if our partition |
33ba3c77 DN |
967 | * flags indicate it is not active and it has a heartbeat. A |
968 | * partition is considered "awaiting deactivation" if our partition | |
969 | * flags indicate it is active but it has no heartbeat or it is not | |
970 | * sending its heartbeat to us. | |
971 | * | |
972 | * To determine the heartbeat, the remote nasid must have a properly | |
973 | * initialized reserved page. | |
974 | */ | |
975 | static void | |
6e41017a | 976 | xpc_identify_activate_IRQ_req_sn2(int nasid) |
33ba3c77 DN |
977 | { |
978 | struct xpc_rsvd_page *remote_rp; | |
979 | struct xpc_vars_sn2 *remote_vars; | |
a812dcc3 DN |
980 | unsigned long remote_rp_pa; |
981 | unsigned long remote_vars_pa; | |
33ba3c77 DN |
982 | int remote_rp_version; |
983 | int reactivate = 0; | |
81fe7883 | 984 | unsigned long remote_rp_ts_jiffies = 0; |
33ba3c77 DN |
985 | short partid; |
986 | struct xpc_partition *part; | |
a47d5dac | 987 | struct xpc_partition_sn2 *part_sn2; |
33ba3c77 DN |
988 | enum xp_retval ret; |
989 | ||
990 | /* pull over the reserved page structure */ | |
991 | ||
ee6665e3 | 992 | remote_rp = (struct xpc_rsvd_page *)xpc_remote_copy_buffer_sn2; |
33ba3c77 DN |
993 | |
994 | ret = xpc_get_remote_rp(nasid, NULL, remote_rp, &remote_rp_pa); | |
995 | if (ret != xpSuccess) { | |
996 | dev_warn(xpc_part, "unable to get reserved page from nasid %d, " | |
997 | "which sent interrupt, reason=%d\n", nasid, ret); | |
998 | return; | |
999 | } | |
1000 | ||
a374c57b | 1001 | remote_vars_pa = remote_rp->sn.sn2.vars_pa; |
33ba3c77 | 1002 | remote_rp_version = remote_rp->version; |
81fe7883 | 1003 | remote_rp_ts_jiffies = remote_rp->ts_jiffies; |
33ba3c77 DN |
1004 | |
1005 | partid = remote_rp->SAL_partid; | |
1006 | part = &xpc_partitions[partid]; | |
a47d5dac | 1007 | part_sn2 = &part->sn.sn2; |
33ba3c77 DN |
1008 | |
1009 | /* pull over the cross partition variables */ | |
1010 | ||
ee6665e3 | 1011 | remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2; |
33ba3c77 DN |
1012 | |
1013 | ret = xpc_get_remote_vars_sn2(remote_vars_pa, remote_vars); | |
1014 | if (ret != xpSuccess) { | |
33ba3c77 DN |
1015 | dev_warn(xpc_part, "unable to get XPC variables from nasid %d, " |
1016 | "which sent interrupt, reason=%d\n", nasid, ret); | |
1017 | ||
1018 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1019 | return; | |
1020 | } | |
1021 | ||
6e41017a | 1022 | part->activate_IRQ_rcvd++; |
33ba3c77 DN |
1023 | |
1024 | dev_dbg(xpc_part, "partid for nasid %d is %d; IRQs = %d; HB = " | |
6e41017a | 1025 | "%ld:0x%lx\n", (int)nasid, (int)partid, part->activate_IRQ_rcvd, |
33ba3c77 DN |
1026 | remote_vars->heartbeat, remote_vars->heartbeating_to_mask[0]); |
1027 | ||
1028 | if (xpc_partition_disengaged(part) && | |
83469b55 | 1029 | part->act_state == XPC_P_AS_INACTIVE) { |
33ba3c77 DN |
1030 | |
1031 | xpc_update_partition_info_sn2(part, remote_rp_version, | |
81fe7883 DN |
1032 | &remote_rp_ts_jiffies, |
1033 | remote_rp_pa, remote_vars_pa, | |
1034 | remote_vars); | |
33ba3c77 | 1035 | |
a47d5dac DN |
1036 | if (xpc_partition_deactivation_requested_sn2(partid)) { |
1037 | /* | |
1038 | * Other side is waiting on us to deactivate even though | |
1039 | * we already have. | |
1040 | */ | |
1041 | return; | |
33ba3c77 DN |
1042 | } |
1043 | ||
1044 | xpc_activate_partition(part); | |
1045 | return; | |
1046 | } | |
1047 | ||
1048 | DBUG_ON(part->remote_rp_version == 0); | |
a47d5dac | 1049 | DBUG_ON(part_sn2->remote_vars_version == 0); |
33ba3c77 | 1050 | |
81fe7883 | 1051 | if (remote_rp_ts_jiffies != part->remote_rp_ts_jiffies) { |
33ba3c77 | 1052 | |
a47d5dac | 1053 | /* the other side rebooted */ |
33ba3c77 | 1054 | |
a47d5dac DN |
1055 | DBUG_ON(xpc_partition_engaged_sn2(partid)); |
1056 | DBUG_ON(xpc_partition_deactivation_requested_sn2(partid)); | |
33ba3c77 DN |
1057 | |
1058 | xpc_update_partition_info_sn2(part, remote_rp_version, | |
81fe7883 DN |
1059 | &remote_rp_ts_jiffies, |
1060 | remote_rp_pa, remote_vars_pa, | |
1061 | remote_vars); | |
33ba3c77 | 1062 | reactivate = 1; |
33ba3c77 DN |
1063 | } |
1064 | ||
a47d5dac | 1065 | if (part->disengage_timeout > 0 && !xpc_partition_disengaged(part)) { |
33ba3c77 DN |
1066 | /* still waiting on other side to disengage from us */ |
1067 | return; | |
1068 | } | |
1069 | ||
a47d5dac | 1070 | if (reactivate) |
33ba3c77 | 1071 | XPC_DEACTIVATE_PARTITION(part, xpReactivating); |
a47d5dac | 1072 | else if (xpc_partition_deactivation_requested_sn2(partid)) |
33ba3c77 | 1073 | XPC_DEACTIVATE_PARTITION(part, xpOtherGoingDown); |
33ba3c77 DN |
1074 | } |
1075 | ||
1076 | /* | |
c39838ce | 1077 | * Loop through the activation amo variables and process any bits |
33ba3c77 DN |
1078 | * which are set. Each bit indicates a nasid sending a partition |
1079 | * activation or deactivation request. | |
1080 | * | |
1081 | * Return #of IRQs detected. | |
1082 | */ | |
1083 | int | |
6e41017a | 1084 | xpc_identify_activate_IRQ_sender_sn2(void) |
33ba3c77 | 1085 | { |
04de7418 DN |
1086 | int l; |
1087 | int b; | |
1088 | unsigned long nasid_mask_long; | |
33ba3c77 DN |
1089 | u64 nasid; /* remote nasid */ |
1090 | int n_IRQs_detected = 0; | |
c39838ce | 1091 | struct amo *act_amos; |
33ba3c77 | 1092 | |
8e85c23e | 1093 | act_amos = xpc_vars_sn2->amos_page + XPC_ACTIVATE_IRQ_AMOS_SN2; |
33ba3c77 | 1094 | |
04de7418 DN |
1095 | /* scan through activate amo variables looking for non-zero entries */ |
1096 | for (l = 0; l < xpc_nasid_mask_nlongs; l++) { | |
33ba3c77 DN |
1097 | |
1098 | if (xpc_exiting) | |
1099 | break; | |
1100 | ||
04de7418 DN |
1101 | nasid_mask_long = xpc_receive_IRQ_amo_sn2(&act_amos[l]); |
1102 | ||
1103 | b = find_first_bit(&nasid_mask_long, BITS_PER_LONG); | |
1104 | if (b >= BITS_PER_LONG) { | |
1105 | /* no IRQs from nasids in this amo variable */ | |
33ba3c77 DN |
1106 | continue; |
1107 | } | |
1108 | ||
04de7418 DN |
1109 | dev_dbg(xpc_part, "amo[%d] gave back 0x%lx\n", l, |
1110 | nasid_mask_long); | |
33ba3c77 DN |
1111 | |
1112 | /* | |
1113 | * If this nasid has been added to the machine since | |
1114 | * our partition was reset, this will retain the | |
1115 | * remote nasid in our reserved pages machine mask. | |
1116 | * This is used in the event of module reload. | |
1117 | */ | |
04de7418 | 1118 | xpc_mach_nasids[l] |= nasid_mask_long; |
33ba3c77 DN |
1119 | |
1120 | /* locate the nasid(s) which sent interrupts */ | |
1121 | ||
04de7418 DN |
1122 | do { |
1123 | n_IRQs_detected++; | |
1124 | nasid = (l * BITS_PER_LONG + b) * 2; | |
1125 | dev_dbg(xpc_part, "interrupt from nasid %ld\n", nasid); | |
1126 | xpc_identify_activate_IRQ_req_sn2(nasid); | |
1127 | ||
1128 | b = find_next_bit(&nasid_mask_long, BITS_PER_LONG, | |
1129 | b + 1); | |
1130 | } while (b < BITS_PER_LONG); | |
33ba3c77 DN |
1131 | } |
1132 | return n_IRQs_detected; | |
1133 | } | |
1134 | ||
1135 | static void | |
5b8669df | 1136 | xpc_process_activate_IRQ_rcvd_sn2(void) |
33ba3c77 | 1137 | { |
5b8669df DN |
1138 | unsigned long irq_flags; |
1139 | int n_IRQs_expected; | |
33ba3c77 DN |
1140 | int n_IRQs_detected; |
1141 | ||
5b8669df DN |
1142 | spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); |
1143 | n_IRQs_expected = xpc_activate_IRQ_rcvd; | |
1144 | xpc_activate_IRQ_rcvd = 0; | |
1145 | spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); | |
1146 | ||
6e41017a | 1147 | n_IRQs_detected = xpc_identify_activate_IRQ_sender_sn2(); |
33ba3c77 | 1148 | if (n_IRQs_detected < n_IRQs_expected) { |
c39838ce | 1149 | /* retry once to help avoid missing amo */ |
6e41017a | 1150 | (void)xpc_identify_activate_IRQ_sender_sn2(); |
33ba3c77 DN |
1151 | } |
1152 | } | |
1153 | ||
185c3a1b | 1154 | /* |
5b8669df | 1155 | * Setup the channel structures that are sn2 specific. |
e17d416b DN |
1156 | */ |
1157 | static enum xp_retval | |
5b8669df | 1158 | xpc_setup_ch_structures_sn_sn2(struct xpc_partition *part) |
e17d416b | 1159 | { |
a47d5dac | 1160 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
5b8669df | 1161 | struct xpc_channel_sn2 *ch_sn2; |
e17d416b DN |
1162 | enum xp_retval retval; |
1163 | int ret; | |
1164 | int cpuid; | |
1165 | int ch_number; | |
e17d416b DN |
1166 | struct timer_list *timer; |
1167 | short partid = XPC_PARTID(part); | |
1168 | ||
e17d416b DN |
1169 | /* allocate all the required GET/PUT values */ |
1170 | ||
185c3a1b | 1171 | part_sn2->local_GPs = |
5b8669df DN |
1172 | xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL, |
1173 | &part_sn2->local_GPs_base); | |
a47d5dac | 1174 | if (part_sn2->local_GPs == NULL) { |
e17d416b DN |
1175 | dev_err(xpc_chan, "can't get memory for local get/put " |
1176 | "values\n"); | |
5b8669df | 1177 | return xpNoMemory; |
e17d416b DN |
1178 | } |
1179 | ||
185c3a1b | 1180 | part_sn2->remote_GPs = |
5b8669df DN |
1181 | xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL, |
1182 | &part_sn2->remote_GPs_base); | |
a47d5dac | 1183 | if (part_sn2->remote_GPs == NULL) { |
e17d416b DN |
1184 | dev_err(xpc_chan, "can't get memory for remote get/put " |
1185 | "values\n"); | |
1186 | retval = xpNoMemory; | |
5b8669df | 1187 | goto out_1; |
e17d416b DN |
1188 | } |
1189 | ||
a47d5dac | 1190 | part_sn2->remote_GPs_pa = 0; |
e17d416b DN |
1191 | |
1192 | /* allocate all the required open and close args */ | |
1193 | ||
5b8669df DN |
1194 | part_sn2->local_openclose_args = |
1195 | xpc_kzalloc_cacheline_aligned(XPC_OPENCLOSE_ARGS_SIZE, | |
1196 | GFP_KERNEL, &part_sn2-> | |
1197 | local_openclose_args_base); | |
1198 | if (part_sn2->local_openclose_args == NULL) { | |
e17d416b DN |
1199 | dev_err(xpc_chan, "can't get memory for local connect args\n"); |
1200 | retval = xpNoMemory; | |
5b8669df | 1201 | goto out_2; |
e17d416b DN |
1202 | } |
1203 | ||
a47d5dac | 1204 | part_sn2->remote_openclose_args_pa = 0; |
e17d416b | 1205 | |
7fb5e59d | 1206 | part_sn2->local_chctl_amo_va = xpc_init_IRQ_amo_sn2(partid); |
e17d416b | 1207 | |
7fb5e59d DN |
1208 | part_sn2->notify_IRQ_nasid = 0; |
1209 | part_sn2->notify_IRQ_phys_cpuid = 0; | |
1210 | part_sn2->remote_chctl_amo_va = NULL; | |
e17d416b | 1211 | |
7fb5e59d | 1212 | sprintf(part_sn2->notify_IRQ_owner, "xpc%02d", partid); |
a47d5dac | 1213 | ret = request_irq(SGI_XPC_NOTIFY, xpc_handle_notify_IRQ_sn2, |
7fb5e59d | 1214 | IRQF_SHARED, part_sn2->notify_IRQ_owner, |
a47d5dac | 1215 | (void *)(u64)partid); |
e17d416b DN |
1216 | if (ret != 0) { |
1217 | dev_err(xpc_chan, "can't register NOTIFY IRQ handler, " | |
1218 | "errno=%d\n", -ret); | |
1219 | retval = xpLackOfResources; | |
5b8669df | 1220 | goto out_3; |
e17d416b DN |
1221 | } |
1222 | ||
7fb5e59d | 1223 | /* Setup a timer to check for dropped notify IRQs */ |
a47d5dac | 1224 | timer = &part_sn2->dropped_notify_IRQ_timer; |
e17d416b | 1225 | init_timer(timer); |
a47d5dac | 1226 | timer->function = |
7fb5e59d | 1227 | (void (*)(unsigned long))xpc_check_for_dropped_notify_IRQ_sn2; |
e17d416b | 1228 | timer->data = (unsigned long)part; |
7fb5e59d | 1229 | timer->expires = jiffies + XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL; |
e17d416b DN |
1230 | add_timer(timer); |
1231 | ||
e17d416b | 1232 | for (ch_number = 0; ch_number < part->nchannels; ch_number++) { |
5b8669df | 1233 | ch_sn2 = &part->channels[ch_number].sn.sn2; |
e17d416b | 1234 | |
5b8669df DN |
1235 | ch_sn2->local_GP = &part_sn2->local_GPs[ch_number]; |
1236 | ch_sn2->local_openclose_args = | |
1237 | &part_sn2->local_openclose_args[ch_number]; | |
e17d416b | 1238 | |
5b8669df | 1239 | mutex_init(&ch_sn2->msg_to_pull_mutex); |
e17d416b DN |
1240 | } |
1241 | ||
e17d416b DN |
1242 | /* |
1243 | * Setup the per partition specific variables required by the | |
1244 | * remote partition to establish channel connections with us. | |
1245 | * | |
1246 | * The setting of the magic # indicates that these per partition | |
1247 | * specific variables are ready to be used. | |
1248 | */ | |
a812dcc3 | 1249 | xpc_vars_part_sn2[partid].GPs_pa = xp_pa(part_sn2->local_GPs); |
8e85c23e | 1250 | xpc_vars_part_sn2[partid].openclose_args_pa = |
5b8669df | 1251 | xp_pa(part_sn2->local_openclose_args); |
8e85c23e | 1252 | xpc_vars_part_sn2[partid].chctl_amo_pa = |
a812dcc3 | 1253 | xp_pa(part_sn2->local_chctl_amo_va); |
e17d416b | 1254 | cpuid = raw_smp_processor_id(); /* any CPU in this partition will do */ |
8e85c23e DN |
1255 | xpc_vars_part_sn2[partid].notify_IRQ_nasid = cpuid_to_nasid(cpuid); |
1256 | xpc_vars_part_sn2[partid].notify_IRQ_phys_cpuid = | |
1257 | cpu_physical_id(cpuid); | |
1258 | xpc_vars_part_sn2[partid].nchannels = part->nchannels; | |
5b8669df | 1259 | xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC1_SN2; |
e17d416b DN |
1260 | |
1261 | return xpSuccess; | |
1262 | ||
5b8669df | 1263 | /* setup of ch structures failed */ |
e17d416b | 1264 | out_3: |
5b8669df DN |
1265 | kfree(part_sn2->local_openclose_args_base); |
1266 | part_sn2->local_openclose_args = NULL; | |
1267 | out_2: | |
a47d5dac DN |
1268 | kfree(part_sn2->remote_GPs_base); |
1269 | part_sn2->remote_GPs = NULL; | |
5b8669df | 1270 | out_1: |
a47d5dac DN |
1271 | kfree(part_sn2->local_GPs_base); |
1272 | part_sn2->local_GPs = NULL; | |
e17d416b DN |
1273 | return retval; |
1274 | } | |
1275 | ||
1276 | /* | |
5b8669df | 1277 | * Teardown the channel structures that are sn2 specific. |
e17d416b DN |
1278 | */ |
1279 | static void | |
5b8669df | 1280 | xpc_teardown_ch_structures_sn_sn2(struct xpc_partition *part) |
e17d416b | 1281 | { |
a47d5dac | 1282 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1283 | short partid = XPC_PARTID(part); |
1284 | ||
1285 | /* | |
5b8669df DN |
1286 | * Indicate that the variables specific to the remote partition are no |
1287 | * longer available for its use. | |
e17d416b | 1288 | */ |
8e85c23e | 1289 | xpc_vars_part_sn2[partid].magic = 0; |
e17d416b | 1290 | |
e17d416b | 1291 | /* in case we've still got outstanding timers registered... */ |
a47d5dac | 1292 | del_timer_sync(&part_sn2->dropped_notify_IRQ_timer); |
5b8669df | 1293 | free_irq(SGI_XPC_NOTIFY, (void *)(u64)partid); |
e17d416b | 1294 | |
5b8669df DN |
1295 | kfree(part_sn2->local_openclose_args_base); |
1296 | part_sn2->local_openclose_args = NULL; | |
a47d5dac DN |
1297 | kfree(part_sn2->remote_GPs_base); |
1298 | part_sn2->remote_GPs = NULL; | |
1299 | kfree(part_sn2->local_GPs_base); | |
1300 | part_sn2->local_GPs = NULL; | |
7fb5e59d | 1301 | part_sn2->local_chctl_amo_va = NULL; |
e17d416b DN |
1302 | } |
1303 | ||
1304 | /* | |
1305 | * Create a wrapper that hides the underlying mechanism for pulling a cacheline | |
1306 | * (or multiple cachelines) from a remote partition. | |
1307 | * | |
a812dcc3 | 1308 | * src_pa must be a cacheline aligned physical address on the remote partition. |
e17d416b DN |
1309 | * dst must be a cacheline aligned virtual address on this partition. |
1310 | * cnt must be cacheline sized | |
1311 | */ | |
ea57f80c | 1312 | /* ??? Replace this function by call to xp_remote_memcpy() or bte_copy()? */ |
e17d416b DN |
1313 | static enum xp_retval |
1314 | xpc_pull_remote_cachelines_sn2(struct xpc_partition *part, void *dst, | |
a812dcc3 | 1315 | const unsigned long src_pa, size_t cnt) |
e17d416b DN |
1316 | { |
1317 | enum xp_retval ret; | |
1318 | ||
a812dcc3 DN |
1319 | DBUG_ON(src_pa != L1_CACHE_ALIGN(src_pa)); |
1320 | DBUG_ON((unsigned long)dst != L1_CACHE_ALIGN((unsigned long)dst)); | |
e17d416b DN |
1321 | DBUG_ON(cnt != L1_CACHE_ALIGN(cnt)); |
1322 | ||
83469b55 | 1323 | if (part->act_state == XPC_P_AS_DEACTIVATING) |
e17d416b DN |
1324 | return part->reason; |
1325 | ||
a812dcc3 | 1326 | ret = xp_remote_memcpy(xp_pa(dst), src_pa, cnt); |
e17d416b DN |
1327 | if (ret != xpSuccess) { |
1328 | dev_dbg(xpc_chan, "xp_remote_memcpy() from partition %d failed," | |
1329 | " ret=%d\n", XPC_PARTID(part), ret); | |
1330 | } | |
1331 | return ret; | |
1332 | } | |
1333 | ||
1334 | /* | |
1335 | * Pull the remote per partition specific variables from the specified | |
1336 | * partition. | |
1337 | */ | |
1338 | static enum xp_retval | |
1339 | xpc_pull_remote_vars_part_sn2(struct xpc_partition *part) | |
1340 | { | |
a47d5dac | 1341 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1342 | u8 buffer[L1_CACHE_BYTES * 2]; |
1343 | struct xpc_vars_part_sn2 *pulled_entry_cacheline = | |
1344 | (struct xpc_vars_part_sn2 *)L1_CACHE_ALIGN((u64)buffer); | |
1345 | struct xpc_vars_part_sn2 *pulled_entry; | |
a812dcc3 DN |
1346 | unsigned long remote_entry_cacheline_pa; |
1347 | unsigned long remote_entry_pa; | |
e17d416b DN |
1348 | short partid = XPC_PARTID(part); |
1349 | enum xp_retval ret; | |
1350 | ||
1351 | /* pull the cacheline that contains the variables we're interested in */ | |
1352 | ||
a47d5dac DN |
1353 | DBUG_ON(part_sn2->remote_vars_part_pa != |
1354 | L1_CACHE_ALIGN(part_sn2->remote_vars_part_pa)); | |
e17d416b DN |
1355 | DBUG_ON(sizeof(struct xpc_vars_part_sn2) != L1_CACHE_BYTES / 2); |
1356 | ||
a47d5dac | 1357 | remote_entry_pa = part_sn2->remote_vars_part_pa + |
e17d416b DN |
1358 | sn_partition_id * sizeof(struct xpc_vars_part_sn2); |
1359 | ||
1360 | remote_entry_cacheline_pa = (remote_entry_pa & ~(L1_CACHE_BYTES - 1)); | |
1361 | ||
1362 | pulled_entry = (struct xpc_vars_part_sn2 *)((u64)pulled_entry_cacheline | |
1363 | + (remote_entry_pa & | |
1364 | (L1_CACHE_BYTES - 1))); | |
1365 | ||
1366 | ret = xpc_pull_remote_cachelines_sn2(part, pulled_entry_cacheline, | |
a812dcc3 | 1367 | remote_entry_cacheline_pa, |
e17d416b DN |
1368 | L1_CACHE_BYTES); |
1369 | if (ret != xpSuccess) { | |
1370 | dev_dbg(xpc_chan, "failed to pull XPC vars_part from " | |
1371 | "partition %d, ret=%d\n", partid, ret); | |
1372 | return ret; | |
1373 | } | |
1374 | ||
1375 | /* see if they've been set up yet */ | |
1376 | ||
5b8669df DN |
1377 | if (pulled_entry->magic != XPC_VP_MAGIC1_SN2 && |
1378 | pulled_entry->magic != XPC_VP_MAGIC2_SN2) { | |
e17d416b DN |
1379 | |
1380 | if (pulled_entry->magic != 0) { | |
1381 | dev_dbg(xpc_chan, "partition %d's XPC vars_part for " | |
1382 | "partition %d has bad magic value (=0x%lx)\n", | |
1383 | partid, sn_partition_id, pulled_entry->magic); | |
1384 | return xpBadMagic; | |
1385 | } | |
1386 | ||
1387 | /* they've not been initialized yet */ | |
1388 | return xpRetry; | |
1389 | } | |
1390 | ||
5b8669df | 1391 | if (xpc_vars_part_sn2[partid].magic == XPC_VP_MAGIC1_SN2) { |
e17d416b DN |
1392 | |
1393 | /* validate the variables */ | |
1394 | ||
1395 | if (pulled_entry->GPs_pa == 0 || | |
1396 | pulled_entry->openclose_args_pa == 0 || | |
7fb5e59d | 1397 | pulled_entry->chctl_amo_pa == 0) { |
e17d416b DN |
1398 | |
1399 | dev_err(xpc_chan, "partition %d's XPC vars_part for " | |
1400 | "partition %d are not valid\n", partid, | |
1401 | sn_partition_id); | |
1402 | return xpInvalidAddress; | |
1403 | } | |
1404 | ||
1405 | /* the variables we imported look to be valid */ | |
1406 | ||
a47d5dac DN |
1407 | part_sn2->remote_GPs_pa = pulled_entry->GPs_pa; |
1408 | part_sn2->remote_openclose_args_pa = | |
e17d416b | 1409 | pulled_entry->openclose_args_pa; |
7fb5e59d | 1410 | part_sn2->remote_chctl_amo_va = |
c39838ce | 1411 | (struct amo *)__va(pulled_entry->chctl_amo_pa); |
7fb5e59d DN |
1412 | part_sn2->notify_IRQ_nasid = pulled_entry->notify_IRQ_nasid; |
1413 | part_sn2->notify_IRQ_phys_cpuid = | |
1414 | pulled_entry->notify_IRQ_phys_cpuid; | |
e17d416b DN |
1415 | |
1416 | if (part->nchannels > pulled_entry->nchannels) | |
1417 | part->nchannels = pulled_entry->nchannels; | |
1418 | ||
1419 | /* let the other side know that we've pulled their variables */ | |
1420 | ||
5b8669df | 1421 | xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC2_SN2; |
e17d416b DN |
1422 | } |
1423 | ||
5b8669df | 1424 | if (pulled_entry->magic == XPC_VP_MAGIC1_SN2) |
e17d416b DN |
1425 | return xpRetry; |
1426 | ||
1427 | return xpSuccess; | |
1428 | } | |
1429 | ||
1430 | /* | |
1431 | * Establish first contact with the remote partititon. This involves pulling | |
1432 | * the XPC per partition variables from the remote partition and waiting for | |
1433 | * the remote partition to pull ours. | |
1434 | */ | |
1435 | static enum xp_retval | |
1436 | xpc_make_first_contact_sn2(struct xpc_partition *part) | |
1437 | { | |
a47d5dac | 1438 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1439 | enum xp_retval ret; |
1440 | ||
33ba3c77 | 1441 | /* |
c39838ce | 1442 | * Register the remote partition's amos with SAL so it can handle |
33ba3c77 DN |
1443 | * and cleanup errors within that address range should the remote |
1444 | * partition go down. We don't unregister this range because it is | |
1445 | * difficult to tell when outstanding writes to the remote partition | |
1446 | * are finished and thus when it is safe to unregister. This should | |
1447 | * not result in wasted space in the SAL xp_addr_region table because | |
1448 | * we should get the same page for remote_amos_page_pa after module | |
1449 | * reloads and system reboots. | |
1450 | */ | |
a47d5dac | 1451 | if (sn_register_xp_addr_region(part_sn2->remote_amos_page_pa, |
33ba3c77 DN |
1452 | PAGE_SIZE, 1) < 0) { |
1453 | dev_warn(xpc_part, "xpc_activating(%d) failed to register " | |
1454 | "xp_addr region\n", XPC_PARTID(part)); | |
1455 | ||
1456 | ret = xpPhysAddrRegFailed; | |
1457 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1458 | return ret; | |
1459 | } | |
1460 | ||
a47d5dac DN |
1461 | /* |
1462 | * Send activate IRQ to get other side to activate if they've not | |
1463 | * already begun to do so. | |
1464 | */ | |
7fb5e59d | 1465 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
1466 | cnodeid_to_nasid(0), |
1467 | part_sn2->activate_IRQ_nasid, | |
1468 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 | 1469 | |
e17d416b DN |
1470 | while ((ret = xpc_pull_remote_vars_part_sn2(part)) != xpSuccess) { |
1471 | if (ret != xpRetry) { | |
1472 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1473 | return ret; | |
1474 | } | |
1475 | ||
1476 | dev_dbg(xpc_part, "waiting to make first contact with " | |
1477 | "partition %d\n", XPC_PARTID(part)); | |
1478 | ||
1479 | /* wait a 1/4 of a second or so */ | |
1480 | (void)msleep_interruptible(250); | |
1481 | ||
83469b55 | 1482 | if (part->act_state == XPC_P_AS_DEACTIVATING) |
e17d416b DN |
1483 | return part->reason; |
1484 | } | |
1485 | ||
1486 | return xpSuccess; | |
1487 | } | |
1488 | ||
1489 | /* | |
7fb5e59d | 1490 | * Get the chctl flags and pull the openclose args and/or remote GPs as needed. |
e17d416b DN |
1491 | */ |
1492 | static u64 | |
7fb5e59d | 1493 | xpc_get_chctl_all_flags_sn2(struct xpc_partition *part) |
e17d416b | 1494 | { |
a47d5dac | 1495 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b | 1496 | unsigned long irq_flags; |
7fb5e59d | 1497 | union xpc_channel_ctl_flags chctl; |
e17d416b DN |
1498 | enum xp_retval ret; |
1499 | ||
1500 | /* | |
7fb5e59d | 1501 | * See if there are any chctl flags to be handled. |
e17d416b DN |
1502 | */ |
1503 | ||
7fb5e59d DN |
1504 | spin_lock_irqsave(&part->chctl_lock, irq_flags); |
1505 | chctl = part->chctl; | |
1506 | if (chctl.all_flags != 0) | |
1507 | part->chctl.all_flags = 0; | |
e17d416b | 1508 | |
7fb5e59d | 1509 | spin_unlock_irqrestore(&part->chctl_lock, irq_flags); |
e17d416b | 1510 | |
7fb5e59d | 1511 | if (xpc_any_openclose_chctl_flags_set(&chctl)) { |
a47d5dac DN |
1512 | ret = xpc_pull_remote_cachelines_sn2(part, part-> |
1513 | remote_openclose_args, | |
a812dcc3 | 1514 | part_sn2-> |
e17d416b DN |
1515 | remote_openclose_args_pa, |
1516 | XPC_OPENCLOSE_ARGS_SIZE); | |
1517 | if (ret != xpSuccess) { | |
1518 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1519 | ||
1520 | dev_dbg(xpc_chan, "failed to pull openclose args from " | |
1521 | "partition %d, ret=%d\n", XPC_PARTID(part), | |
1522 | ret); | |
1523 | ||
7fb5e59d DN |
1524 | /* don't bother processing chctl flags anymore */ |
1525 | chctl.all_flags = 0; | |
e17d416b DN |
1526 | } |
1527 | } | |
1528 | ||
7fb5e59d | 1529 | if (xpc_any_msg_chctl_flags_set(&chctl)) { |
a47d5dac | 1530 | ret = xpc_pull_remote_cachelines_sn2(part, part_sn2->remote_GPs, |
a812dcc3 | 1531 | part_sn2->remote_GPs_pa, |
e17d416b DN |
1532 | XPC_GP_SIZE); |
1533 | if (ret != xpSuccess) { | |
1534 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1535 | ||
1536 | dev_dbg(xpc_chan, "failed to pull GPs from partition " | |
1537 | "%d, ret=%d\n", XPC_PARTID(part), ret); | |
1538 | ||
7fb5e59d DN |
1539 | /* don't bother processing chctl flags anymore */ |
1540 | chctl.all_flags = 0; | |
e17d416b DN |
1541 | } |
1542 | } | |
1543 | ||
7fb5e59d | 1544 | return chctl.all_flags; |
e17d416b DN |
1545 | } |
1546 | ||
185c3a1b DN |
1547 | /* |
1548 | * Allocate the local message queue and the notify queue. | |
1549 | */ | |
1550 | static enum xp_retval | |
1551 | xpc_allocate_local_msgqueue_sn2(struct xpc_channel *ch) | |
1552 | { | |
5b8669df | 1553 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
185c3a1b DN |
1554 | unsigned long irq_flags; |
1555 | int nentries; | |
1556 | size_t nbytes; | |
1557 | ||
1558 | for (nentries = ch->local_nentries; nentries > 0; nentries--) { | |
1559 | ||
bd3e64c1 | 1560 | nbytes = nentries * ch->entry_size; |
5b8669df DN |
1561 | ch_sn2->local_msgqueue = |
1562 | xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL, | |
1563 | &ch_sn2->local_msgqueue_base); | |
1564 | if (ch_sn2->local_msgqueue == NULL) | |
185c3a1b DN |
1565 | continue; |
1566 | ||
bd3e64c1 | 1567 | nbytes = nentries * sizeof(struct xpc_notify_sn2); |
5b8669df DN |
1568 | ch_sn2->notify_queue = kzalloc(nbytes, GFP_KERNEL); |
1569 | if (ch_sn2->notify_queue == NULL) { | |
1570 | kfree(ch_sn2->local_msgqueue_base); | |
1571 | ch_sn2->local_msgqueue = NULL; | |
185c3a1b DN |
1572 | continue; |
1573 | } | |
1574 | ||
1575 | spin_lock_irqsave(&ch->lock, irq_flags); | |
1576 | if (nentries < ch->local_nentries) { | |
1577 | dev_dbg(xpc_chan, "nentries=%d local_nentries=%d, " | |
1578 | "partid=%d, channel=%d\n", nentries, | |
1579 | ch->local_nentries, ch->partid, ch->number); | |
1580 | ||
1581 | ch->local_nentries = nentries; | |
1582 | } | |
1583 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
1584 | return xpSuccess; | |
1585 | } | |
1586 | ||
1587 | dev_dbg(xpc_chan, "can't get memory for local message queue and notify " | |
1588 | "queue, partid=%d, channel=%d\n", ch->partid, ch->number); | |
1589 | return xpNoMemory; | |
1590 | } | |
1591 | ||
1592 | /* | |
1593 | * Allocate the cached remote message queue. | |
1594 | */ | |
1595 | static enum xp_retval | |
1596 | xpc_allocate_remote_msgqueue_sn2(struct xpc_channel *ch) | |
1597 | { | |
5b8669df | 1598 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
185c3a1b DN |
1599 | unsigned long irq_flags; |
1600 | int nentries; | |
1601 | size_t nbytes; | |
1602 | ||
1603 | DBUG_ON(ch->remote_nentries <= 0); | |
1604 | ||
1605 | for (nentries = ch->remote_nentries; nentries > 0; nentries--) { | |
1606 | ||
bd3e64c1 | 1607 | nbytes = nentries * ch->entry_size; |
5b8669df DN |
1608 | ch_sn2->remote_msgqueue = |
1609 | xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL, &ch_sn2-> | |
1610 | remote_msgqueue_base); | |
1611 | if (ch_sn2->remote_msgqueue == NULL) | |
185c3a1b DN |
1612 | continue; |
1613 | ||
1614 | spin_lock_irqsave(&ch->lock, irq_flags); | |
1615 | if (nentries < ch->remote_nentries) { | |
1616 | dev_dbg(xpc_chan, "nentries=%d remote_nentries=%d, " | |
1617 | "partid=%d, channel=%d\n", nentries, | |
1618 | ch->remote_nentries, ch->partid, ch->number); | |
1619 | ||
1620 | ch->remote_nentries = nentries; | |
1621 | } | |
1622 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
1623 | return xpSuccess; | |
1624 | } | |
1625 | ||
1626 | dev_dbg(xpc_chan, "can't get memory for cached remote message queue, " | |
1627 | "partid=%d, channel=%d\n", ch->partid, ch->number); | |
1628 | return xpNoMemory; | |
1629 | } | |
1630 | ||
1631 | /* | |
1632 | * Allocate message queues and other stuff associated with a channel. | |
1633 | * | |
1634 | * Note: Assumes all of the channel sizes are filled in. | |
1635 | */ | |
1636 | static enum xp_retval | |
5b8669df | 1637 | xpc_setup_msg_structures_sn2(struct xpc_channel *ch) |
185c3a1b | 1638 | { |
5b8669df | 1639 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
185c3a1b DN |
1640 | enum xp_retval ret; |
1641 | ||
1642 | DBUG_ON(ch->flags & XPC_C_SETUP); | |
1643 | ||
1644 | ret = xpc_allocate_local_msgqueue_sn2(ch); | |
1645 | if (ret == xpSuccess) { | |
1646 | ||
1647 | ret = xpc_allocate_remote_msgqueue_sn2(ch); | |
1648 | if (ret != xpSuccess) { | |
5b8669df DN |
1649 | kfree(ch_sn2->local_msgqueue_base); |
1650 | ch_sn2->local_msgqueue = NULL; | |
1651 | kfree(ch_sn2->notify_queue); | |
1652 | ch_sn2->notify_queue = NULL; | |
185c3a1b DN |
1653 | } |
1654 | } | |
1655 | return ret; | |
1656 | } | |
1657 | ||
1658 | /* | |
1659 | * Free up message queues and other stuff that were allocated for the specified | |
1660 | * channel. | |
185c3a1b DN |
1661 | */ |
1662 | static void | |
5b8669df | 1663 | xpc_teardown_msg_structures_sn2(struct xpc_channel *ch) |
185c3a1b DN |
1664 | { |
1665 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
1666 | ||
1667 | DBUG_ON(!spin_is_locked(&ch->lock)); | |
185c3a1b | 1668 | |
5b8669df | 1669 | ch_sn2->remote_msgqueue_pa = 0; |
185c3a1b DN |
1670 | |
1671 | ch_sn2->local_GP->get = 0; | |
1672 | ch_sn2->local_GP->put = 0; | |
1673 | ch_sn2->remote_GP.get = 0; | |
1674 | ch_sn2->remote_GP.put = 0; | |
1675 | ch_sn2->w_local_GP.get = 0; | |
1676 | ch_sn2->w_local_GP.put = 0; | |
1677 | ch_sn2->w_remote_GP.get = 0; | |
1678 | ch_sn2->w_remote_GP.put = 0; | |
1679 | ch_sn2->next_msg_to_pull = 0; | |
1680 | ||
1681 | if (ch->flags & XPC_C_SETUP) { | |
1682 | dev_dbg(xpc_chan, "ch->flags=0x%x, partid=%d, channel=%d\n", | |
1683 | ch->flags, ch->partid, ch->number); | |
1684 | ||
5b8669df DN |
1685 | kfree(ch_sn2->local_msgqueue_base); |
1686 | ch_sn2->local_msgqueue = NULL; | |
1687 | kfree(ch_sn2->remote_msgqueue_base); | |
1688 | ch_sn2->remote_msgqueue = NULL; | |
1689 | kfree(ch_sn2->notify_queue); | |
1690 | ch_sn2->notify_queue = NULL; | |
185c3a1b DN |
1691 | } |
1692 | } | |
1693 | ||
a47d5dac DN |
1694 | /* |
1695 | * Notify those who wanted to be notified upon delivery of their message. | |
1696 | */ | |
1697 | static void | |
1698 | xpc_notify_senders_sn2(struct xpc_channel *ch, enum xp_retval reason, s64 put) | |
1699 | { | |
bd3e64c1 | 1700 | struct xpc_notify_sn2 *notify; |
a47d5dac DN |
1701 | u8 notify_type; |
1702 | s64 get = ch->sn.sn2.w_remote_GP.get - 1; | |
1703 | ||
1704 | while (++get < put && atomic_read(&ch->n_to_notify) > 0) { | |
1705 | ||
5b8669df | 1706 | notify = &ch->sn.sn2.notify_queue[get % ch->local_nentries]; |
a47d5dac DN |
1707 | |
1708 | /* | |
1709 | * See if the notify entry indicates it was associated with | |
1710 | * a message who's sender wants to be notified. It is possible | |
1711 | * that it is, but someone else is doing or has done the | |
1712 | * notification. | |
1713 | */ | |
1714 | notify_type = notify->type; | |
1715 | if (notify_type == 0 || | |
1716 | cmpxchg(¬ify->type, notify_type, 0) != notify_type) { | |
1717 | continue; | |
1718 | } | |
1719 | ||
1720 | DBUG_ON(notify_type != XPC_N_CALL); | |
1721 | ||
1722 | atomic_dec(&ch->n_to_notify); | |
1723 | ||
1724 | if (notify->func != NULL) { | |
bd3e64c1 DN |
1725 | dev_dbg(xpc_chan, "notify->func() called, notify=0x%p " |
1726 | "msg_number=%ld partid=%d channel=%d\n", | |
a47d5dac DN |
1727 | (void *)notify, get, ch->partid, ch->number); |
1728 | ||
1729 | notify->func(reason, ch->partid, ch->number, | |
1730 | notify->key); | |
1731 | ||
bd3e64c1 DN |
1732 | dev_dbg(xpc_chan, "notify->func() returned, notify=0x%p" |
1733 | " msg_number=%ld partid=%d channel=%d\n", | |
1734 | (void *)notify, get, ch->partid, ch->number); | |
a47d5dac DN |
1735 | } |
1736 | } | |
1737 | } | |
1738 | ||
1739 | static void | |
1740 | xpc_notify_senders_of_disconnect_sn2(struct xpc_channel *ch) | |
1741 | { | |
1742 | xpc_notify_senders_sn2(ch, ch->reason, ch->sn.sn2.w_local_GP.put); | |
1743 | } | |
1744 | ||
1745 | /* | |
1746 | * Clear some of the msg flags in the local message queue. | |
1747 | */ | |
1748 | static inline void | |
1749 | xpc_clear_local_msgqueue_flags_sn2(struct xpc_channel *ch) | |
1750 | { | |
1751 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
bd3e64c1 | 1752 | struct xpc_msg_sn2 *msg; |
a47d5dac DN |
1753 | s64 get; |
1754 | ||
1755 | get = ch_sn2->w_remote_GP.get; | |
1756 | do { | |
bd3e64c1 DN |
1757 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue + |
1758 | (get % ch->local_nentries) * | |
1759 | ch->entry_size); | |
252523ef | 1760 | DBUG_ON(!(msg->flags & XPC_M_SN2_READY)); |
a47d5dac DN |
1761 | msg->flags = 0; |
1762 | } while (++get < ch_sn2->remote_GP.get); | |
1763 | } | |
1764 | ||
1765 | /* | |
1766 | * Clear some of the msg flags in the remote message queue. | |
1767 | */ | |
1768 | static inline void | |
1769 | xpc_clear_remote_msgqueue_flags_sn2(struct xpc_channel *ch) | |
1770 | { | |
1771 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
bd3e64c1 | 1772 | struct xpc_msg_sn2 *msg; |
6e873ec7 | 1773 | s64 put, remote_nentries = ch->remote_nentries; |
a47d5dac | 1774 | |
252523ef | 1775 | /* flags are zeroed when the buffer is allocated */ |
6e873ec7 | 1776 | if (ch_sn2->remote_GP.put < remote_nentries) |
252523ef RH |
1777 | return; |
1778 | ||
6e873ec7 | 1779 | put = max(ch_sn2->w_remote_GP.put, remote_nentries); |
a47d5dac | 1780 | do { |
bd3e64c1 | 1781 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + |
6e873ec7 | 1782 | (put % remote_nentries) * |
bd3e64c1 | 1783 | ch->entry_size); |
252523ef RH |
1784 | DBUG_ON(!(msg->flags & XPC_M_SN2_READY)); |
1785 | DBUG_ON(!(msg->flags & XPC_M_SN2_DONE)); | |
6e873ec7 | 1786 | DBUG_ON(msg->number != put - remote_nentries); |
a47d5dac DN |
1787 | msg->flags = 0; |
1788 | } while (++put < ch_sn2->remote_GP.put); | |
1789 | } | |
1790 | ||
bd3e64c1 DN |
1791 | static int |
1792 | xpc_n_of_deliverable_payloads_sn2(struct xpc_channel *ch) | |
1793 | { | |
1794 | return ch->sn.sn2.w_remote_GP.put - ch->sn.sn2.w_local_GP.get; | |
1795 | } | |
1796 | ||
a47d5dac | 1797 | static void |
7fb5e59d | 1798 | xpc_process_msg_chctl_flags_sn2(struct xpc_partition *part, int ch_number) |
a47d5dac DN |
1799 | { |
1800 | struct xpc_channel *ch = &part->channels[ch_number]; | |
1801 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
bd3e64c1 | 1802 | int npayloads_sent; |
a47d5dac DN |
1803 | |
1804 | ch_sn2->remote_GP = part->sn.sn2.remote_GPs[ch_number]; | |
1805 | ||
1806 | /* See what, if anything, has changed for each connected channel */ | |
1807 | ||
1808 | xpc_msgqueue_ref(ch); | |
1809 | ||
1810 | if (ch_sn2->w_remote_GP.get == ch_sn2->remote_GP.get && | |
1811 | ch_sn2->w_remote_GP.put == ch_sn2->remote_GP.put) { | |
1812 | /* nothing changed since GPs were last pulled */ | |
1813 | xpc_msgqueue_deref(ch); | |
1814 | return; | |
1815 | } | |
1816 | ||
1817 | if (!(ch->flags & XPC_C_CONNECTED)) { | |
1818 | xpc_msgqueue_deref(ch); | |
1819 | return; | |
1820 | } | |
1821 | ||
1822 | /* | |
1823 | * First check to see if messages recently sent by us have been | |
1824 | * received by the other side. (The remote GET value will have | |
1825 | * changed since we last looked at it.) | |
1826 | */ | |
1827 | ||
1828 | if (ch_sn2->w_remote_GP.get != ch_sn2->remote_GP.get) { | |
1829 | ||
1830 | /* | |
1831 | * We need to notify any senders that want to be notified | |
1832 | * that their sent messages have been received by their | |
1833 | * intended recipients. We need to do this before updating | |
1834 | * w_remote_GP.get so that we don't allocate the same message | |
1835 | * queue entries prematurely (see xpc_allocate_msg()). | |
1836 | */ | |
1837 | if (atomic_read(&ch->n_to_notify) > 0) { | |
1838 | /* | |
1839 | * Notify senders that messages sent have been | |
1840 | * received and delivered by the other side. | |
1841 | */ | |
1842 | xpc_notify_senders_sn2(ch, xpMsgDelivered, | |
1843 | ch_sn2->remote_GP.get); | |
1844 | } | |
1845 | ||
1846 | /* | |
1847 | * Clear msg->flags in previously sent messages, so that | |
1848 | * they're ready for xpc_allocate_msg(). | |
1849 | */ | |
1850 | xpc_clear_local_msgqueue_flags_sn2(ch); | |
1851 | ||
1852 | ch_sn2->w_remote_GP.get = ch_sn2->remote_GP.get; | |
1853 | ||
1854 | dev_dbg(xpc_chan, "w_remote_GP.get changed to %ld, partid=%d, " | |
1855 | "channel=%d\n", ch_sn2->w_remote_GP.get, ch->partid, | |
1856 | ch->number); | |
1857 | ||
1858 | /* | |
1859 | * If anyone was waiting for message queue entries to become | |
1860 | * available, wake them up. | |
1861 | */ | |
1862 | if (atomic_read(&ch->n_on_msg_allocate_wq) > 0) | |
1863 | wake_up(&ch->msg_allocate_wq); | |
1864 | } | |
1865 | ||
1866 | /* | |
1867 | * Now check for newly sent messages by the other side. (The remote | |
1868 | * PUT value will have changed since we last looked at it.) | |
1869 | */ | |
1870 | ||
1871 | if (ch_sn2->w_remote_GP.put != ch_sn2->remote_GP.put) { | |
1872 | /* | |
1873 | * Clear msg->flags in previously received messages, so that | |
bd3e64c1 | 1874 | * they're ready for xpc_get_deliverable_payload_sn2(). |
a47d5dac DN |
1875 | */ |
1876 | xpc_clear_remote_msgqueue_flags_sn2(ch); | |
1877 | ||
69b3bb65 | 1878 | smp_wmb(); /* ensure flags have been cleared before bte_copy */ |
a47d5dac DN |
1879 | ch_sn2->w_remote_GP.put = ch_sn2->remote_GP.put; |
1880 | ||
1881 | dev_dbg(xpc_chan, "w_remote_GP.put changed to %ld, partid=%d, " | |
1882 | "channel=%d\n", ch_sn2->w_remote_GP.put, ch->partid, | |
1883 | ch->number); | |
1884 | ||
bd3e64c1 DN |
1885 | npayloads_sent = xpc_n_of_deliverable_payloads_sn2(ch); |
1886 | if (npayloads_sent > 0) { | |
a47d5dac DN |
1887 | dev_dbg(xpc_chan, "msgs waiting to be copied and " |
1888 | "delivered=%d, partid=%d, channel=%d\n", | |
bd3e64c1 | 1889 | npayloads_sent, ch->partid, ch->number); |
a47d5dac DN |
1890 | |
1891 | if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) | |
bd3e64c1 | 1892 | xpc_activate_kthreads(ch, npayloads_sent); |
a47d5dac DN |
1893 | } |
1894 | } | |
1895 | ||
1896 | xpc_msgqueue_deref(ch); | |
1897 | } | |
1898 | ||
bd3e64c1 | 1899 | static struct xpc_msg_sn2 * |
e17d416b DN |
1900 | xpc_pull_remote_msg_sn2(struct xpc_channel *ch, s64 get) |
1901 | { | |
1902 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
a47d5dac | 1903 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
a812dcc3 | 1904 | unsigned long remote_msg_pa; |
bd3e64c1 | 1905 | struct xpc_msg_sn2 *msg; |
a812dcc3 DN |
1906 | u32 msg_index; |
1907 | u32 nmsgs; | |
e17d416b DN |
1908 | u64 msg_offset; |
1909 | enum xp_retval ret; | |
1910 | ||
a47d5dac | 1911 | if (mutex_lock_interruptible(&ch_sn2->msg_to_pull_mutex) != 0) { |
e17d416b DN |
1912 | /* we were interrupted by a signal */ |
1913 | return NULL; | |
1914 | } | |
1915 | ||
a47d5dac | 1916 | while (get >= ch_sn2->next_msg_to_pull) { |
e17d416b DN |
1917 | |
1918 | /* pull as many messages as are ready and able to be pulled */ | |
1919 | ||
a47d5dac | 1920 | msg_index = ch_sn2->next_msg_to_pull % ch->remote_nentries; |
e17d416b | 1921 | |
a47d5dac DN |
1922 | DBUG_ON(ch_sn2->next_msg_to_pull >= ch_sn2->w_remote_GP.put); |
1923 | nmsgs = ch_sn2->w_remote_GP.put - ch_sn2->next_msg_to_pull; | |
e17d416b DN |
1924 | if (msg_index + nmsgs > ch->remote_nentries) { |
1925 | /* ignore the ones that wrap the msg queue for now */ | |
1926 | nmsgs = ch->remote_nentries - msg_index; | |
1927 | } | |
1928 | ||
bd3e64c1 DN |
1929 | msg_offset = msg_index * ch->entry_size; |
1930 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + | |
5b8669df DN |
1931 | msg_offset); |
1932 | remote_msg_pa = ch_sn2->remote_msgqueue_pa + msg_offset; | |
e17d416b | 1933 | |
a812dcc3 | 1934 | ret = xpc_pull_remote_cachelines_sn2(part, msg, remote_msg_pa, |
bd3e64c1 | 1935 | nmsgs * ch->entry_size); |
e17d416b DN |
1936 | if (ret != xpSuccess) { |
1937 | ||
1938 | dev_dbg(xpc_chan, "failed to pull %d msgs starting with" | |
1939 | " msg %ld from partition %d, channel=%d, " | |
a47d5dac | 1940 | "ret=%d\n", nmsgs, ch_sn2->next_msg_to_pull, |
e17d416b DN |
1941 | ch->partid, ch->number, ret); |
1942 | ||
1943 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1944 | ||
a47d5dac | 1945 | mutex_unlock(&ch_sn2->msg_to_pull_mutex); |
e17d416b DN |
1946 | return NULL; |
1947 | } | |
1948 | ||
a47d5dac | 1949 | ch_sn2->next_msg_to_pull += nmsgs; |
e17d416b DN |
1950 | } |
1951 | ||
a47d5dac | 1952 | mutex_unlock(&ch_sn2->msg_to_pull_mutex); |
e17d416b DN |
1953 | |
1954 | /* return the message we were looking for */ | |
bd3e64c1 DN |
1955 | msg_offset = (get % ch->remote_nentries) * ch->entry_size; |
1956 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + msg_offset); | |
e17d416b DN |
1957 | |
1958 | return msg; | |
1959 | } | |
1960 | ||
1961 | /* | |
bd3e64c1 | 1962 | * Get the next deliverable message's payload. |
e17d416b | 1963 | */ |
bd3e64c1 DN |
1964 | static void * |
1965 | xpc_get_deliverable_payload_sn2(struct xpc_channel *ch) | |
e17d416b | 1966 | { |
a47d5dac | 1967 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 DN |
1968 | struct xpc_msg_sn2 *msg; |
1969 | void *payload = NULL; | |
e17d416b DN |
1970 | s64 get; |
1971 | ||
1972 | do { | |
1973 | if (ch->flags & XPC_C_DISCONNECTING) | |
1974 | break; | |
1975 | ||
a47d5dac | 1976 | get = ch_sn2->w_local_GP.get; |
69b3bb65 | 1977 | smp_rmb(); /* guarantee that .get loads before .put */ |
a47d5dac | 1978 | if (get == ch_sn2->w_remote_GP.put) |
e17d416b DN |
1979 | break; |
1980 | ||
1981 | /* There are messages waiting to be pulled and delivered. | |
1982 | * We need to try to secure one for ourselves. We'll do this | |
1983 | * by trying to increment w_local_GP.get and hope that no one | |
1984 | * else beats us to it. If they do, we'll we'll simply have | |
1985 | * to try again for the next one. | |
1986 | */ | |
1987 | ||
a47d5dac | 1988 | if (cmpxchg(&ch_sn2->w_local_GP.get, get, get + 1) == get) { |
e17d416b DN |
1989 | /* we got the entry referenced by get */ |
1990 | ||
1991 | dev_dbg(xpc_chan, "w_local_GP.get changed to %ld, " | |
1992 | "partid=%d, channel=%d\n", get + 1, | |
1993 | ch->partid, ch->number); | |
1994 | ||
1995 | /* pull the message from the remote partition */ | |
1996 | ||
1997 | msg = xpc_pull_remote_msg_sn2(ch, get); | |
1998 | ||
17e21616 RH |
1999 | if (msg != NULL) { |
2000 | DBUG_ON(msg->number != get); | |
2001 | DBUG_ON(msg->flags & XPC_M_SN2_DONE); | |
2002 | DBUG_ON(!(msg->flags & XPC_M_SN2_READY)); | |
e17d416b | 2003 | |
17e21616 RH |
2004 | payload = &msg->payload; |
2005 | } | |
e17d416b DN |
2006 | break; |
2007 | } | |
2008 | ||
2009 | } while (1); | |
2010 | ||
bd3e64c1 | 2011 | return payload; |
e17d416b DN |
2012 | } |
2013 | ||
33ba3c77 DN |
2014 | /* |
2015 | * Now we actually send the messages that are ready to be sent by advancing | |
7fb5e59d DN |
2016 | * the local message queue's Put value and then send a chctl msgrequest to the |
2017 | * recipient partition. | |
33ba3c77 DN |
2018 | */ |
2019 | static void | |
2020 | xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put) | |
2021 | { | |
a47d5dac | 2022 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 | 2023 | struct xpc_msg_sn2 *msg; |
33ba3c77 | 2024 | s64 put = initial_put + 1; |
7fb5e59d | 2025 | int send_msgrequest = 0; |
33ba3c77 DN |
2026 | |
2027 | while (1) { | |
2028 | ||
2029 | while (1) { | |
a47d5dac | 2030 | if (put == ch_sn2->w_local_GP.put) |
33ba3c77 DN |
2031 | break; |
2032 | ||
bd3e64c1 DN |
2033 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2-> |
2034 | local_msgqueue + (put % | |
2035 | ch->local_nentries) * | |
2036 | ch->entry_size); | |
33ba3c77 | 2037 | |
bd3e64c1 | 2038 | if (!(msg->flags & XPC_M_SN2_READY)) |
33ba3c77 DN |
2039 | break; |
2040 | ||
2041 | put++; | |
2042 | } | |
2043 | ||
2044 | if (put == initial_put) { | |
2045 | /* nothing's changed */ | |
2046 | break; | |
2047 | } | |
2048 | ||
a47d5dac | 2049 | if (cmpxchg_rel(&ch_sn2->local_GP->put, initial_put, put) != |
33ba3c77 DN |
2050 | initial_put) { |
2051 | /* someone else beat us to it */ | |
a47d5dac | 2052 | DBUG_ON(ch_sn2->local_GP->put < initial_put); |
33ba3c77 DN |
2053 | break; |
2054 | } | |
2055 | ||
2056 | /* we just set the new value of local_GP->put */ | |
2057 | ||
2058 | dev_dbg(xpc_chan, "local_GP->put changed to %ld, partid=%d, " | |
2059 | "channel=%d\n", put, ch->partid, ch->number); | |
2060 | ||
7fb5e59d | 2061 | send_msgrequest = 1; |
33ba3c77 DN |
2062 | |
2063 | /* | |
2064 | * We need to ensure that the message referenced by | |
bd3e64c1 | 2065 | * local_GP->put is not XPC_M_SN2_READY or that local_GP->put |
33ba3c77 DN |
2066 | * equals w_local_GP.put, so we'll go have a look. |
2067 | */ | |
2068 | initial_put = put; | |
2069 | } | |
2070 | ||
7fb5e59d DN |
2071 | if (send_msgrequest) |
2072 | xpc_send_chctl_msgrequest_sn2(ch); | |
33ba3c77 DN |
2073 | } |
2074 | ||
2075 | /* | |
2076 | * Allocate an entry for a message from the message queue associated with the | |
2077 | * specified channel. | |
2078 | */ | |
2079 | static enum xp_retval | |
2080 | xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags, | |
bd3e64c1 | 2081 | struct xpc_msg_sn2 **address_of_msg) |
33ba3c77 | 2082 | { |
a47d5dac | 2083 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 | 2084 | struct xpc_msg_sn2 *msg; |
33ba3c77 DN |
2085 | enum xp_retval ret; |
2086 | s64 put; | |
2087 | ||
33ba3c77 DN |
2088 | /* |
2089 | * Get the next available message entry from the local message queue. | |
2090 | * If none are available, we'll make sure that we grab the latest | |
2091 | * GP values. | |
2092 | */ | |
2093 | ret = xpTimeout; | |
2094 | ||
2095 | while (1) { | |
2096 | ||
a47d5dac | 2097 | put = ch_sn2->w_local_GP.put; |
69b3bb65 | 2098 | smp_rmb(); /* guarantee that .put loads before .get */ |
a47d5dac | 2099 | if (put - ch_sn2->w_remote_GP.get < ch->local_nentries) { |
33ba3c77 DN |
2100 | |
2101 | /* There are available message entries. We need to try | |
2102 | * to secure one for ourselves. We'll do this by trying | |
2103 | * to increment w_local_GP.put as long as someone else | |
2104 | * doesn't beat us to it. If they do, we'll have to | |
2105 | * try again. | |
2106 | */ | |
a47d5dac DN |
2107 | if (cmpxchg(&ch_sn2->w_local_GP.put, put, put + 1) == |
2108 | put) { | |
33ba3c77 DN |
2109 | /* we got the entry referenced by put */ |
2110 | break; | |
2111 | } | |
2112 | continue; /* try again */ | |
2113 | } | |
2114 | ||
2115 | /* | |
2116 | * There aren't any available msg entries at this time. | |
2117 | * | |
2118 | * In waiting for a message entry to become available, | |
7fb5e59d DN |
2119 | * we set a timeout in case the other side is not sending |
2120 | * completion interrupts. This lets us fake a notify IRQ | |
2121 | * that will cause the notify IRQ handler to fetch the latest | |
2122 | * GP values as if an interrupt was sent by the other side. | |
33ba3c77 DN |
2123 | */ |
2124 | if (ret == xpTimeout) | |
7fb5e59d | 2125 | xpc_send_chctl_local_msgrequest_sn2(ch); |
33ba3c77 | 2126 | |
97bf1aa1 | 2127 | if (flags & XPC_NOWAIT) |
33ba3c77 | 2128 | return xpNoWait; |
33ba3c77 DN |
2129 | |
2130 | ret = xpc_allocate_msg_wait(ch); | |
97bf1aa1 | 2131 | if (ret != xpInterrupted && ret != xpTimeout) |
33ba3c77 | 2132 | return ret; |
33ba3c77 DN |
2133 | } |
2134 | ||
2135 | /* get the message's address and initialize it */ | |
bd3e64c1 DN |
2136 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue + |
2137 | (put % ch->local_nentries) * | |
2138 | ch->entry_size); | |
33ba3c77 DN |
2139 | |
2140 | DBUG_ON(msg->flags != 0); | |
2141 | msg->number = put; | |
2142 | ||
2143 | dev_dbg(xpc_chan, "w_local_GP.put changed to %ld; msg=0x%p, " | |
2144 | "msg_number=%ld, partid=%d, channel=%d\n", put + 1, | |
2145 | (void *)msg, msg->number, ch->partid, ch->number); | |
2146 | ||
2147 | *address_of_msg = msg; | |
33ba3c77 DN |
2148 | return xpSuccess; |
2149 | } | |
2150 | ||
2151 | /* | |
2152 | * Common code that does the actual sending of the message by advancing the | |
7fb5e59d DN |
2153 | * local message queue's Put value and sends a chctl msgrequest to the |
2154 | * partition the message is being sent to. | |
33ba3c77 DN |
2155 | */ |
2156 | static enum xp_retval | |
bd3e64c1 DN |
2157 | xpc_send_payload_sn2(struct xpc_channel *ch, u32 flags, void *payload, |
2158 | u16 payload_size, u8 notify_type, xpc_notify_func func, | |
2159 | void *key) | |
33ba3c77 DN |
2160 | { |
2161 | enum xp_retval ret = xpSuccess; | |
5b8669df | 2162 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 DN |
2163 | struct xpc_msg_sn2 *msg = msg; |
2164 | struct xpc_notify_sn2 *notify = notify; | |
97bf1aa1 DN |
2165 | s64 msg_number; |
2166 | s64 put; | |
33ba3c77 DN |
2167 | |
2168 | DBUG_ON(notify_type == XPC_N_CALL && func == NULL); | |
97bf1aa1 | 2169 | |
bd3e64c1 | 2170 | if (XPC_MSG_SIZE(payload_size) > ch->entry_size) |
97bf1aa1 DN |
2171 | return xpPayloadTooBig; |
2172 | ||
2173 | xpc_msgqueue_ref(ch); | |
33ba3c77 DN |
2174 | |
2175 | if (ch->flags & XPC_C_DISCONNECTING) { | |
97bf1aa1 DN |
2176 | ret = ch->reason; |
2177 | goto out_1; | |
2178 | } | |
2179 | if (!(ch->flags & XPC_C_CONNECTED)) { | |
2180 | ret = xpNotConnected; | |
2181 | goto out_1; | |
33ba3c77 DN |
2182 | } |
2183 | ||
97bf1aa1 DN |
2184 | ret = xpc_allocate_msg_sn2(ch, flags, &msg); |
2185 | if (ret != xpSuccess) | |
2186 | goto out_1; | |
2187 | ||
2188 | msg_number = msg->number; | |
2189 | ||
33ba3c77 DN |
2190 | if (notify_type != 0) { |
2191 | /* | |
2192 | * Tell the remote side to send an ACK interrupt when the | |
2193 | * message has been delivered. | |
2194 | */ | |
bd3e64c1 | 2195 | msg->flags |= XPC_M_SN2_INTERRUPT; |
33ba3c77 DN |
2196 | |
2197 | atomic_inc(&ch->n_to_notify); | |
2198 | ||
5b8669df | 2199 | notify = &ch_sn2->notify_queue[msg_number % ch->local_nentries]; |
33ba3c77 DN |
2200 | notify->func = func; |
2201 | notify->key = key; | |
2202 | notify->type = notify_type; | |
2203 | ||
ea57f80c | 2204 | /* ??? Is a mb() needed here? */ |
33ba3c77 DN |
2205 | |
2206 | if (ch->flags & XPC_C_DISCONNECTING) { | |
2207 | /* | |
2208 | * An error occurred between our last error check and | |
2209 | * this one. We will try to clear the type field from | |
2210 | * the notify entry. If we succeed then | |
2211 | * xpc_disconnect_channel() didn't already process | |
2212 | * the notify entry. | |
2213 | */ | |
2214 | if (cmpxchg(¬ify->type, notify_type, 0) == | |
2215 | notify_type) { | |
2216 | atomic_dec(&ch->n_to_notify); | |
2217 | ret = ch->reason; | |
2218 | } | |
97bf1aa1 | 2219 | goto out_1; |
33ba3c77 DN |
2220 | } |
2221 | } | |
2222 | ||
97bf1aa1 DN |
2223 | memcpy(&msg->payload, payload, payload_size); |
2224 | ||
bd3e64c1 | 2225 | msg->flags |= XPC_M_SN2_READY; |
33ba3c77 DN |
2226 | |
2227 | /* | |
2228 | * The preceding store of msg->flags must occur before the following | |
a47d5dac | 2229 | * load of local_GP->put. |
33ba3c77 | 2230 | */ |
69b3bb65 | 2231 | smp_mb(); |
33ba3c77 DN |
2232 | |
2233 | /* see if the message is next in line to be sent, if so send it */ | |
2234 | ||
5b8669df | 2235 | put = ch_sn2->local_GP->put; |
33ba3c77 DN |
2236 | if (put == msg_number) |
2237 | xpc_send_msgs_sn2(ch, put); | |
2238 | ||
97bf1aa1 | 2239 | out_1: |
33ba3c77 DN |
2240 | xpc_msgqueue_deref(ch); |
2241 | return ret; | |
2242 | } | |
2243 | ||
2244 | /* | |
2245 | * Now we actually acknowledge the messages that have been delivered and ack'd | |
2246 | * by advancing the cached remote message queue's Get value and if requested | |
7fb5e59d | 2247 | * send a chctl msgrequest to the message sender's partition. |
bd3e64c1 DN |
2248 | * |
2249 | * If a message has XPC_M_SN2_INTERRUPT set, send an interrupt to the partition | |
2250 | * that sent the message. | |
33ba3c77 DN |
2251 | */ |
2252 | static void | |
2253 | xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags) | |
2254 | { | |
a47d5dac | 2255 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 | 2256 | struct xpc_msg_sn2 *msg; |
33ba3c77 | 2257 | s64 get = initial_get + 1; |
7fb5e59d | 2258 | int send_msgrequest = 0; |
33ba3c77 DN |
2259 | |
2260 | while (1) { | |
2261 | ||
2262 | while (1) { | |
a47d5dac | 2263 | if (get == ch_sn2->w_local_GP.get) |
33ba3c77 DN |
2264 | break; |
2265 | ||
bd3e64c1 DN |
2266 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2-> |
2267 | remote_msgqueue + (get % | |
2268 | ch->remote_nentries) * | |
2269 | ch->entry_size); | |
33ba3c77 | 2270 | |
bd3e64c1 | 2271 | if (!(msg->flags & XPC_M_SN2_DONE)) |
33ba3c77 DN |
2272 | break; |
2273 | ||
2274 | msg_flags |= msg->flags; | |
2275 | get++; | |
2276 | } | |
2277 | ||
2278 | if (get == initial_get) { | |
2279 | /* nothing's changed */ | |
2280 | break; | |
2281 | } | |
2282 | ||
a47d5dac | 2283 | if (cmpxchg_rel(&ch_sn2->local_GP->get, initial_get, get) != |
33ba3c77 DN |
2284 | initial_get) { |
2285 | /* someone else beat us to it */ | |
a47d5dac | 2286 | DBUG_ON(ch_sn2->local_GP->get <= initial_get); |
33ba3c77 DN |
2287 | break; |
2288 | } | |
2289 | ||
2290 | /* we just set the new value of local_GP->get */ | |
2291 | ||
2292 | dev_dbg(xpc_chan, "local_GP->get changed to %ld, partid=%d, " | |
2293 | "channel=%d\n", get, ch->partid, ch->number); | |
2294 | ||
bd3e64c1 | 2295 | send_msgrequest = (msg_flags & XPC_M_SN2_INTERRUPT); |
33ba3c77 DN |
2296 | |
2297 | /* | |
2298 | * We need to ensure that the message referenced by | |
bd3e64c1 | 2299 | * local_GP->get is not XPC_M_SN2_DONE or that local_GP->get |
33ba3c77 DN |
2300 | * equals w_local_GP.get, so we'll go have a look. |
2301 | */ | |
2302 | initial_get = get; | |
2303 | } | |
2304 | ||
7fb5e59d DN |
2305 | if (send_msgrequest) |
2306 | xpc_send_chctl_msgrequest_sn2(ch); | |
33ba3c77 DN |
2307 | } |
2308 | ||
2309 | static void | |
bd3e64c1 | 2310 | xpc_received_payload_sn2(struct xpc_channel *ch, void *payload) |
33ba3c77 | 2311 | { |
bd3e64c1 DN |
2312 | struct xpc_msg_sn2 *msg; |
2313 | s64 msg_number; | |
33ba3c77 | 2314 | s64 get; |
bd3e64c1 DN |
2315 | |
2316 | msg = container_of(payload, struct xpc_msg_sn2, payload); | |
2317 | msg_number = msg->number; | |
33ba3c77 DN |
2318 | |
2319 | dev_dbg(xpc_chan, "msg=0x%p, msg_number=%ld, partid=%d, channel=%d\n", | |
2320 | (void *)msg, msg_number, ch->partid, ch->number); | |
2321 | ||
252523ef | 2322 | DBUG_ON((((u64)msg - (u64)ch->sn.sn2.remote_msgqueue) / ch->entry_size) != |
33ba3c77 | 2323 | msg_number % ch->remote_nentries); |
252523ef | 2324 | DBUG_ON(!(msg->flags & XPC_M_SN2_READY)); |
bd3e64c1 | 2325 | DBUG_ON(msg->flags & XPC_M_SN2_DONE); |
33ba3c77 | 2326 | |
bd3e64c1 | 2327 | msg->flags |= XPC_M_SN2_DONE; |
33ba3c77 DN |
2328 | |
2329 | /* | |
2330 | * The preceding store of msg->flags must occur before the following | |
a47d5dac | 2331 | * load of local_GP->get. |
33ba3c77 | 2332 | */ |
69b3bb65 | 2333 | smp_mb(); |
33ba3c77 DN |
2334 | |
2335 | /* | |
2336 | * See if this message is next in line to be acknowledged as having | |
2337 | * been delivered. | |
2338 | */ | |
a47d5dac | 2339 | get = ch->sn.sn2.local_GP->get; |
33ba3c77 DN |
2340 | if (get == msg_number) |
2341 | xpc_acknowledge_msgs_sn2(ch, get, msg->flags); | |
2342 | } | |
2343 | ||
6e41017a | 2344 | int |
94bd2708 DN |
2345 | xpc_init_sn2(void) |
2346 | { | |
6e41017a | 2347 | int ret; |
ee6665e3 | 2348 | size_t buf_size; |
6e41017a | 2349 | |
5b8669df | 2350 | xpc_setup_partitions_sn = xpc_setup_partitions_sn_sn2; |
6f2584f4 | 2351 | xpc_teardown_partitions_sn = xpc_teardown_partitions_sn_sn2; |
261f3b49 | 2352 | xpc_get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_sn2; |
5b8669df | 2353 | xpc_setup_rsvd_page_sn = xpc_setup_rsvd_page_sn_sn2; |
a374c57b RH |
2354 | |
2355 | xpc_allow_hb = xpc_allow_hb_sn2; | |
2356 | xpc_disallow_hb = xpc_disallow_hb_sn2; | |
2357 | xpc_disallow_all_hbs = xpc_disallow_all_hbs_sn2; | |
33ba3c77 DN |
2358 | xpc_increment_heartbeat = xpc_increment_heartbeat_sn2; |
2359 | xpc_offline_heartbeat = xpc_offline_heartbeat_sn2; | |
2360 | xpc_online_heartbeat = xpc_online_heartbeat_sn2; | |
2361 | xpc_heartbeat_init = xpc_heartbeat_init_sn2; | |
2362 | xpc_heartbeat_exit = xpc_heartbeat_exit_sn2; | |
61deb86e | 2363 | xpc_get_remote_heartbeat = xpc_get_remote_heartbeat_sn2; |
33ba3c77 | 2364 | |
a47d5dac DN |
2365 | xpc_request_partition_activation = xpc_request_partition_activation_sn2; |
2366 | xpc_request_partition_reactivation = | |
2367 | xpc_request_partition_reactivation_sn2; | |
2368 | xpc_request_partition_deactivation = | |
2369 | xpc_request_partition_deactivation_sn2; | |
2370 | xpc_cancel_partition_deactivation_request = | |
2371 | xpc_cancel_partition_deactivation_request_sn2; | |
2372 | ||
6e41017a | 2373 | xpc_process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_sn2; |
5b8669df DN |
2374 | xpc_setup_ch_structures_sn = xpc_setup_ch_structures_sn_sn2; |
2375 | xpc_teardown_ch_structures_sn = xpc_teardown_ch_structures_sn_sn2; | |
e17d416b | 2376 | xpc_make_first_contact = xpc_make_first_contact_sn2; |
5b8669df | 2377 | |
7fb5e59d | 2378 | xpc_get_chctl_all_flags = xpc_get_chctl_all_flags_sn2; |
5b8669df DN |
2379 | xpc_send_chctl_closerequest = xpc_send_chctl_closerequest_sn2; |
2380 | xpc_send_chctl_closereply = xpc_send_chctl_closereply_sn2; | |
2381 | xpc_send_chctl_openrequest = xpc_send_chctl_openrequest_sn2; | |
2382 | xpc_send_chctl_openreply = xpc_send_chctl_openreply_sn2; | |
2383 | ||
2384 | xpc_save_remote_msgqueue_pa = xpc_save_remote_msgqueue_pa_sn2; | |
2385 | ||
2386 | xpc_setup_msg_structures = xpc_setup_msg_structures_sn2; | |
2387 | xpc_teardown_msg_structures = xpc_teardown_msg_structures_sn2; | |
2388 | ||
a47d5dac | 2389 | xpc_notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_sn2; |
7fb5e59d | 2390 | xpc_process_msg_chctl_flags = xpc_process_msg_chctl_flags_sn2; |
bd3e64c1 DN |
2391 | xpc_n_of_deliverable_payloads = xpc_n_of_deliverable_payloads_sn2; |
2392 | xpc_get_deliverable_payload = xpc_get_deliverable_payload_sn2; | |
33ba3c77 | 2393 | |
a47d5dac | 2394 | xpc_indicate_partition_engaged = xpc_indicate_partition_engaged_sn2; |
a47d5dac DN |
2395 | xpc_indicate_partition_disengaged = |
2396 | xpc_indicate_partition_disengaged_sn2; | |
5b8669df DN |
2397 | xpc_partition_engaged = xpc_partition_engaged_sn2; |
2398 | xpc_any_partition_engaged = xpc_any_partition_engaged_sn2; | |
a47d5dac DN |
2399 | xpc_assume_partition_disengaged = xpc_assume_partition_disengaged_sn2; |
2400 | ||
bd3e64c1 DN |
2401 | xpc_send_payload = xpc_send_payload_sn2; |
2402 | xpc_received_payload = xpc_received_payload_sn2; | |
2403 | ||
2404 | if (offsetof(struct xpc_msg_sn2, payload) > XPC_MSG_HDR_MAX_SIZE) { | |
2405 | dev_err(xpc_part, "header portion of struct xpc_msg_sn2 is " | |
2406 | "larger than %d\n", XPC_MSG_HDR_MAX_SIZE); | |
2407 | return -E2BIG; | |
2408 | } | |
6e41017a | 2409 | |
ee6665e3 DN |
2410 | buf_size = max(XPC_RP_VARS_SIZE, |
2411 | XPC_RP_HEADER_SIZE + XP_NASID_MASK_BYTES_SN2); | |
2412 | xpc_remote_copy_buffer_sn2 = xpc_kmalloc_cacheline_aligned(buf_size, | |
2413 | GFP_KERNEL, | |
2414 | &xpc_remote_copy_buffer_base_sn2); | |
2415 | if (xpc_remote_copy_buffer_sn2 == NULL) { | |
2416 | dev_err(xpc_part, "can't get memory for remote copy buffer\n"); | |
2417 | return -ENOMEM; | |
2418 | } | |
2419 | ||
c39838ce | 2420 | /* open up protections for IPI and [potentially] amo operations */ |
6e41017a | 2421 | xpc_allow_IPI_ops_sn2(); |
c39838ce | 2422 | xpc_allow_amo_ops_shub_wars_1_1_sn2(); |
6e41017a DN |
2423 | |
2424 | /* | |
2425 | * This is safe to do before the xpc_hb_checker thread has started | |
2426 | * because the handler releases a wait queue. If an interrupt is | |
2427 | * received before the thread is waiting, it will not go to sleep, | |
2428 | * but rather immediately process the interrupt. | |
2429 | */ | |
2430 | ret = request_irq(SGI_XPC_ACTIVATE, xpc_handle_activate_IRQ_sn2, 0, | |
2431 | "xpc hb", NULL); | |
2432 | if (ret != 0) { | |
2433 | dev_err(xpc_part, "can't register ACTIVATE IRQ handler, " | |
2434 | "errno=%d\n", -ret); | |
2435 | xpc_disallow_IPI_ops_sn2(); | |
ee6665e3 | 2436 | kfree(xpc_remote_copy_buffer_base_sn2); |
6e41017a DN |
2437 | } |
2438 | return ret; | |
94bd2708 DN |
2439 | } |
2440 | ||
2441 | void | |
2442 | xpc_exit_sn2(void) | |
2443 | { | |
6e41017a DN |
2444 | free_irq(SGI_XPC_ACTIVATE, NULL); |
2445 | xpc_disallow_IPI_ops_sn2(); | |
ee6665e3 | 2446 | kfree(xpc_remote_copy_buffer_base_sn2); |
94bd2708 | 2447 | } |