Commit | Line | Data |
---|---|---|
77241056 MM |
1 | /* |
2 | * | |
3 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
4 | * redistributing this file, you may do so under either license. | |
5 | * | |
6 | * GPL LICENSE SUMMARY | |
7 | * | |
8 | * Copyright(c) 2015 Intel Corporation. | |
9 | * | |
10 | * This program is free software; you can redistribute it and/or modify | |
11 | * it under the terms of version 2 of the GNU General Public License as | |
12 | * published by the Free Software Foundation. | |
13 | * | |
14 | * This program is distributed in the hope that it will be useful, but | |
15 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
17 | * General Public License for more details. | |
18 | * | |
19 | * BSD LICENSE | |
20 | * | |
21 | * Copyright(c) 2015 Intel Corporation. | |
22 | * | |
23 | * Redistribution and use in source and binary forms, with or without | |
24 | * modification, are permitted provided that the following conditions | |
25 | * are met: | |
26 | * | |
27 | * - Redistributions of source code must retain the above copyright | |
28 | * notice, this list of conditions and the following disclaimer. | |
29 | * - Redistributions in binary form must reproduce the above copyright | |
30 | * notice, this list of conditions and the following disclaimer in | |
31 | * the documentation and/or other materials provided with the | |
32 | * distribution. | |
33 | * - Neither the name of Intel Corporation nor the names of its | |
34 | * contributors may be used to endorse or promote products derived | |
35 | * from this software without specific prior written permission. | |
36 | * | |
37 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
38 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
39 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
40 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
41 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
42 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
43 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
44 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
45 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
46 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
47 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
48 | * | |
49 | */ | |
50 | ||
51 | #include <linux/err.h> | |
52 | #include <linux/vmalloc.h> | |
53 | #include <linux/hash.h> | |
54 | #include <linux/module.h> | |
55 | #include <linux/random.h> | |
56 | #include <linux/seq_file.h> | |
ec4274f1 DD |
57 | #include <rdma/rdma_vt.h> |
58 | #include <rdma/rdmavt_qp.h> | |
77241056 MM |
59 | |
60 | #include "hfi.h" | |
61 | #include "qp.h" | |
62 | #include "trace.h" | |
45842abb | 63 | #include "verbs_txreq.h" |
77241056 | 64 | |
a2c2d608 | 65 | unsigned int hfi1_qp_table_size = 256; |
77241056 MM |
66 | module_param_named(qp_table_size, hfi1_qp_table_size, uint, S_IRUGO); |
67 | MODULE_PARM_DESC(qp_table_size, "QP table size"); | |
68 | ||
895420dd | 69 | static void flush_tx_list(struct rvt_qp *qp); |
77241056 MM |
70 | static int iowait_sleep( |
71 | struct sdma_engine *sde, | |
72 | struct iowait *wait, | |
73 | struct sdma_txreq *stx, | |
74 | unsigned seq); | |
75 | static void iowait_wakeup(struct iowait *wait, int reason); | |
a545f530 | 76 | static void iowait_sdma_drained(struct iowait *wait); |
91702b4a | 77 | static void qp_pio_drain(struct rvt_qp *qp); |
77241056 | 78 | |
1c4b7d97 DD |
79 | static inline unsigned mk_qpn(struct rvt_qpn_table *qpt, |
80 | struct rvt_qpn_map *map, unsigned off) | |
77241056 | 81 | { |
1c4b7d97 | 82 | return (map - qpt->map) * RVT_BITS_PER_PAGE + off; |
77241056 MM |
83 | } |
84 | ||
85 | /* | |
86 | * Convert the AETH credit code into the number of credits. | |
87 | */ | |
88 | static const u16 credit_table[31] = { | |
89 | 0, /* 0 */ | |
90 | 1, /* 1 */ | |
91 | 2, /* 2 */ | |
92 | 3, /* 3 */ | |
93 | 4, /* 4 */ | |
94 | 6, /* 5 */ | |
95 | 8, /* 6 */ | |
96 | 12, /* 7 */ | |
97 | 16, /* 8 */ | |
98 | 24, /* 9 */ | |
99 | 32, /* A */ | |
100 | 48, /* B */ | |
101 | 64, /* C */ | |
102 | 96, /* D */ | |
103 | 128, /* E */ | |
104 | 192, /* F */ | |
105 | 256, /* 10 */ | |
106 | 384, /* 11 */ | |
107 | 512, /* 12 */ | |
108 | 768, /* 13 */ | |
109 | 1024, /* 14 */ | |
110 | 1536, /* 15 */ | |
111 | 2048, /* 16 */ | |
112 | 3072, /* 17 */ | |
113 | 4096, /* 18 */ | |
114 | 6144, /* 19 */ | |
115 | 8192, /* 1A */ | |
116 | 12288, /* 1B */ | |
117 | 16384, /* 1C */ | |
118 | 24576, /* 1D */ | |
119 | 32768 /* 1E */ | |
120 | }; | |
121 | ||
895420dd | 122 | static void flush_tx_list(struct rvt_qp *qp) |
77241056 | 123 | { |
4c6829c5 DD |
124 | struct hfi1_qp_priv *priv = qp->priv; |
125 | ||
126 | while (!list_empty(&priv->s_iowait.tx_head)) { | |
77241056 MM |
127 | struct sdma_txreq *tx; |
128 | ||
129 | tx = list_first_entry( | |
4c6829c5 | 130 | &priv->s_iowait.tx_head, |
77241056 MM |
131 | struct sdma_txreq, |
132 | list); | |
133 | list_del_init(&tx->list); | |
134 | hfi1_put_txreq( | |
135 | container_of(tx, struct verbs_txreq, txreq)); | |
136 | } | |
137 | } | |
138 | ||
895420dd | 139 | static void flush_iowait(struct rvt_qp *qp) |
77241056 | 140 | { |
4c6829c5 | 141 | struct hfi1_qp_priv *priv = qp->priv; |
77241056 MM |
142 | struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); |
143 | unsigned long flags; | |
144 | ||
145 | write_seqlock_irqsave(&dev->iowait_lock, flags); | |
4c6829c5 DD |
146 | if (!list_empty(&priv->s_iowait.list)) { |
147 | list_del_init(&priv->s_iowait.list); | |
77241056 MM |
148 | if (atomic_dec_and_test(&qp->refcount)) |
149 | wake_up(&qp->wait); | |
150 | } | |
151 | write_sequnlock_irqrestore(&dev->iowait_lock, flags); | |
152 | } | |
153 | ||
154 | static inline int opa_mtu_enum_to_int(int mtu) | |
155 | { | |
156 | switch (mtu) { | |
157 | case OPA_MTU_8192: return 8192; | |
158 | case OPA_MTU_10240: return 10240; | |
159 | default: return -1; | |
160 | } | |
161 | } | |
162 | ||
163 | /** | |
164 | * This function is what we would push to the core layer if we wanted to be a | |
165 | * "first class citizen". Instead we hide this here and rely on Verbs ULPs | |
166 | * to blindly pass the MTU enum value from the PathRecord to us. | |
167 | * | |
168 | * The actual flag used to determine "8k MTU" will change and is currently | |
169 | * unknown. | |
170 | */ | |
171 | static inline int verbs_mtu_enum_to_int(struct ib_device *dev, enum ib_mtu mtu) | |
172 | { | |
173 | int val = opa_mtu_enum_to_int((int)mtu); | |
174 | ||
175 | if (val > 0) | |
176 | return val; | |
177 | return ib_mtu_enum_to_int(mtu); | |
178 | } | |
179 | ||
ec4274f1 DD |
180 | int hfi1_check_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, |
181 | int attr_mask, struct ib_udata *udata) | |
77241056 | 182 | { |
ec4274f1 | 183 | struct ib_qp *ibqp = &qp->ibqp; |
77241056 | 184 | struct hfi1_ibdev *dev = to_idev(ibqp->device); |
d7b8ba51 | 185 | struct hfi1_devdata *dd = dd_from_dev(dev); |
ec4274f1 | 186 | u8 sc; |
77241056 MM |
187 | |
188 | if (attr_mask & IB_QP_AV) { | |
d7b8ba51 | 189 | sc = ah_to_sc(ibqp->device, &attr->ah_attr); |
31e7af1c IW |
190 | if (sc == 0xf) |
191 | return -EINVAL; | |
192 | ||
d7b8ba51 MM |
193 | if (!qp_to_sdma_engine(qp, sc) && |
194 | dd->flags & HFI1_HAS_SEND_DMA) | |
ec4274f1 | 195 | return -EINVAL; |
721d0427 JJ |
196 | |
197 | if (!qp_to_send_context(qp, sc)) | |
198 | return -EINVAL; | |
77241056 MM |
199 | } |
200 | ||
201 | if (attr_mask & IB_QP_ALT_PATH) { | |
d7b8ba51 | 202 | sc = ah_to_sc(ibqp->device, &attr->alt_ah_attr); |
31e7af1c IW |
203 | if (sc == 0xf) |
204 | return -EINVAL; | |
205 | ||
d7b8ba51 MM |
206 | if (!qp_to_sdma_engine(qp, sc) && |
207 | dd->flags & HFI1_HAS_SEND_DMA) | |
ec4274f1 | 208 | return -EINVAL; |
721d0427 JJ |
209 | |
210 | if (!qp_to_send_context(qp, sc)) | |
211 | return -EINVAL; | |
77241056 MM |
212 | } |
213 | ||
ec4274f1 DD |
214 | return 0; |
215 | } | |
77241056 | 216 | |
ec4274f1 DD |
217 | void hfi1_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, |
218 | int attr_mask, struct ib_udata *udata) | |
219 | { | |
220 | struct ib_qp *ibqp = &qp->ibqp; | |
221 | struct hfi1_qp_priv *priv = qp->priv; | |
77241056 MM |
222 | |
223 | if (attr_mask & IB_QP_AV) { | |
4c6829c5 DD |
224 | priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr); |
225 | priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc); | |
77241056 MM |
226 | } |
227 | ||
ec4274f1 DD |
228 | if (attr_mask & IB_QP_PATH_MIG_STATE && |
229 | attr->path_mig_state == IB_MIG_MIGRATED && | |
230 | qp->s_mig_state == IB_MIG_ARMED) { | |
231 | qp->s_flags |= RVT_S_AHG_CLEAR; | |
232 | priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr); | |
233 | priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc); | |
77241056 | 234 | } |
77241056 MM |
235 | } |
236 | ||
46a80d62 MM |
237 | /** |
238 | * hfi1_check_send_wqe - validate wqe | |
239 | * @qp - The qp | |
240 | * @wqe - The built wqe | |
241 | * | |
242 | * validate wqe. This is called | |
243 | * prior to inserting the wqe into | |
244 | * the ring but after the wqe has been | |
245 | * setup. | |
246 | * | |
247 | * Returns 0 on success, -EINVAL on failure | |
248 | * | |
249 | */ | |
250 | int hfi1_check_send_wqe(struct rvt_qp *qp, | |
251 | struct rvt_swqe *wqe) | |
31e7af1c IW |
252 | { |
253 | struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); | |
46a80d62 | 254 | struct rvt_ah *ah; |
31e7af1c | 255 | |
46a80d62 MM |
256 | switch (qp->ibqp.qp_type) { |
257 | case IB_QPT_RC: | |
258 | case IB_QPT_UC: | |
259 | if (wqe->length > 0x80000000U) | |
260 | return -EINVAL; | |
261 | break; | |
262 | case IB_QPT_SMI: | |
263 | ah = ibah_to_rvtah(wqe->ud_wr.ah); | |
264 | if (wqe->length > (1 << ah->log_pmtu)) | |
265 | return -EINVAL; | |
266 | break; | |
267 | case IB_QPT_GSI: | |
268 | case IB_QPT_UD: | |
269 | ah = ibah_to_rvtah(wqe->ud_wr.ah); | |
270 | if (wqe->length > (1 << ah->log_pmtu)) | |
271 | return -EINVAL; | |
272 | if (ibp->sl_to_sc[ah->attr.sl] == 0xf) | |
273 | return -EINVAL; | |
274 | default: | |
275 | break; | |
31e7af1c | 276 | } |
91702b4a | 277 | return wqe->length <= piothreshold; |
31e7af1c IW |
278 | } |
279 | ||
77241056 MM |
280 | /** |
281 | * hfi1_compute_aeth - compute the AETH (syndrome + MSN) | |
282 | * @qp: the queue pair to compute the AETH for | |
283 | * | |
284 | * Returns the AETH. | |
285 | */ | |
895420dd | 286 | __be32 hfi1_compute_aeth(struct rvt_qp *qp) |
77241056 MM |
287 | { |
288 | u32 aeth = qp->r_msn & HFI1_MSN_MASK; | |
289 | ||
290 | if (qp->ibqp.srq) { | |
291 | /* | |
292 | * Shared receive queues don't generate credits. | |
293 | * Set the credit field to the invalid value. | |
294 | */ | |
295 | aeth |= HFI1_AETH_CREDIT_INVAL << HFI1_AETH_CREDIT_SHIFT; | |
296 | } else { | |
297 | u32 min, max, x; | |
298 | u32 credits; | |
895420dd | 299 | struct rvt_rwq *wq = qp->r_rq.wq; |
77241056 MM |
300 | u32 head; |
301 | u32 tail; | |
302 | ||
303 | /* sanity check pointers before trusting them */ | |
304 | head = wq->head; | |
305 | if (head >= qp->r_rq.size) | |
306 | head = 0; | |
307 | tail = wq->tail; | |
308 | if (tail >= qp->r_rq.size) | |
309 | tail = 0; | |
310 | /* | |
311 | * Compute the number of credits available (RWQEs). | |
312 | * There is a small chance that the pair of reads are | |
313 | * not atomic, which is OK, since the fuzziness is | |
314 | * resolved as further ACKs go out. | |
315 | */ | |
316 | credits = head - tail; | |
317 | if ((int)credits < 0) | |
318 | credits += qp->r_rq.size; | |
319 | /* | |
320 | * Binary search the credit table to find the code to | |
321 | * use. | |
322 | */ | |
323 | min = 0; | |
324 | max = 31; | |
325 | for (;;) { | |
326 | x = (min + max) / 2; | |
327 | if (credit_table[x] == credits) | |
328 | break; | |
e490974e | 329 | if (credit_table[x] > credits) { |
77241056 | 330 | max = x; |
e490974e | 331 | } else { |
edddfca0 | 332 | if (min == x) |
e490974e | 333 | break; |
edddfca0 | 334 | min = x; |
e490974e | 335 | } |
77241056 MM |
336 | } |
337 | aeth |= x << HFI1_AETH_CREDIT_SHIFT; | |
338 | } | |
339 | return cpu_to_be32(aeth); | |
340 | } | |
341 | ||
46a80d62 MM |
342 | /** |
343 | * _hfi1_schedule_send - schedule progress | |
344 | * @qp: the QP | |
345 | * | |
346 | * This schedules qp progress w/o regard to the s_flags. | |
347 | * | |
348 | * It is only used in the post send, which doesn't hold | |
349 | * the s_lock. | |
350 | */ | |
351 | void _hfi1_schedule_send(struct rvt_qp *qp) | |
352 | { | |
353 | struct hfi1_qp_priv *priv = qp->priv; | |
354 | struct hfi1_ibport *ibp = | |
355 | to_iport(qp->ibqp.device, qp->port_num); | |
356 | struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); | |
357 | struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); | |
358 | ||
359 | iowait_schedule(&priv->s_iowait, ppd->hfi1_wq, | |
360 | priv->s_sde ? | |
361 | priv->s_sde->cpu : | |
362 | cpumask_first(cpumask_of_node(dd->node))); | |
363 | } | |
364 | ||
14553ca1 MM |
365 | static void qp_pio_drain(struct rvt_qp *qp) |
366 | { | |
367 | struct hfi1_ibdev *dev; | |
368 | struct hfi1_qp_priv *priv = qp->priv; | |
369 | ||
370 | if (!priv->s_sendcontext) | |
371 | return; | |
372 | dev = to_idev(qp->ibqp.device); | |
373 | while (iowait_pio_pending(&priv->s_iowait)) { | |
374 | write_seqlock_irq(&dev->iowait_lock); | |
375 | hfi1_sc_wantpiobuf_intr(priv->s_sendcontext, 1); | |
376 | write_sequnlock_irq(&dev->iowait_lock); | |
377 | iowait_pio_drain(&priv->s_iowait); | |
378 | write_seqlock_irq(&dev->iowait_lock); | |
379 | hfi1_sc_wantpiobuf_intr(priv->s_sendcontext, 0); | |
380 | write_sequnlock_irq(&dev->iowait_lock); | |
381 | } | |
382 | } | |
383 | ||
46a80d62 MM |
384 | /** |
385 | * hfi1_schedule_send - schedule progress | |
386 | * @qp: the QP | |
387 | * | |
388 | * This schedules qp progress and caller should hold | |
389 | * the s_lock. | |
390 | */ | |
391 | void hfi1_schedule_send(struct rvt_qp *qp) | |
392 | { | |
393 | if (hfi1_send_ok(qp)) | |
394 | _hfi1_schedule_send(qp); | |
395 | } | |
396 | ||
77241056 MM |
397 | /** |
398 | * hfi1_get_credit - flush the send work queue of a QP | |
399 | * @qp: the qp who's send work queue to flush | |
400 | * @aeth: the Acknowledge Extended Transport Header | |
401 | * | |
402 | * The QP s_lock should be held. | |
403 | */ | |
895420dd | 404 | void hfi1_get_credit(struct rvt_qp *qp, u32 aeth) |
77241056 MM |
405 | { |
406 | u32 credit = (aeth >> HFI1_AETH_CREDIT_SHIFT) & HFI1_AETH_CREDIT_MASK; | |
407 | ||
408 | /* | |
409 | * If the credit is invalid, we can send | |
410 | * as many packets as we like. Otherwise, we have to | |
411 | * honor the credit field. | |
412 | */ | |
413 | if (credit == HFI1_AETH_CREDIT_INVAL) { | |
54d10c1e DD |
414 | if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) { |
415 | qp->s_flags |= RVT_S_UNLIMITED_CREDIT; | |
416 | if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) { | |
417 | qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT; | |
77241056 MM |
418 | hfi1_schedule_send(qp); |
419 | } | |
420 | } | |
54d10c1e | 421 | } else if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) { |
77241056 MM |
422 | /* Compute new LSN (i.e., MSN + credit) */ |
423 | credit = (aeth + credit_table[credit]) & HFI1_MSN_MASK; | |
424 | if (cmp_msn(credit, qp->s_lsn) > 0) { | |
425 | qp->s_lsn = credit; | |
54d10c1e DD |
426 | if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) { |
427 | qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT; | |
77241056 MM |
428 | hfi1_schedule_send(qp); |
429 | } | |
430 | } | |
431 | } | |
432 | } | |
433 | ||
895420dd | 434 | void hfi1_qp_wakeup(struct rvt_qp *qp, u32 flag) |
77241056 MM |
435 | { |
436 | unsigned long flags; | |
437 | ||
438 | spin_lock_irqsave(&qp->s_lock, flags); | |
439 | if (qp->s_flags & flag) { | |
440 | qp->s_flags &= ~flag; | |
441 | trace_hfi1_qpwakeup(qp, flag); | |
442 | hfi1_schedule_send(qp); | |
443 | } | |
444 | spin_unlock_irqrestore(&qp->s_lock, flags); | |
445 | /* Notify hfi1_destroy_qp() if it is waiting. */ | |
446 | if (atomic_dec_and_test(&qp->refcount)) | |
447 | wake_up(&qp->wait); | |
448 | } | |
449 | ||
450 | static int iowait_sleep( | |
451 | struct sdma_engine *sde, | |
452 | struct iowait *wait, | |
453 | struct sdma_txreq *stx, | |
454 | unsigned seq) | |
455 | { | |
456 | struct verbs_txreq *tx = container_of(stx, struct verbs_txreq, txreq); | |
895420dd | 457 | struct rvt_qp *qp; |
4c6829c5 | 458 | struct hfi1_qp_priv *priv; |
77241056 MM |
459 | unsigned long flags; |
460 | int ret = 0; | |
461 | struct hfi1_ibdev *dev; | |
462 | ||
463 | qp = tx->qp; | |
4c6829c5 | 464 | priv = qp->priv; |
77241056 MM |
465 | |
466 | spin_lock_irqsave(&qp->s_lock, flags); | |
83693bd1 | 467 | if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { |
77241056 MM |
468 | /* |
469 | * If we couldn't queue the DMA request, save the info | |
470 | * and try again later rather than destroying the | |
471 | * buffer and undoing the side effects of the copy. | |
472 | */ | |
473 | /* Make a common routine? */ | |
474 | dev = &sde->dd->verbs_dev; | |
475 | list_add_tail(&stx->list, &wait->tx_head); | |
476 | write_seqlock(&dev->iowait_lock); | |
477 | if (sdma_progress(sde, seq, stx)) | |
478 | goto eagain; | |
4c6829c5 | 479 | if (list_empty(&priv->s_iowait.list)) { |
77241056 MM |
480 | struct hfi1_ibport *ibp = |
481 | to_iport(qp->ibqp.device, qp->port_num); | |
482 | ||
4eb06882 | 483 | ibp->rvp.n_dmawait++; |
54d10c1e | 484 | qp->s_flags |= RVT_S_WAIT_DMA_DESC; |
4c6829c5 | 485 | list_add_tail(&priv->s_iowait.list, &sde->dmawait); |
54d10c1e | 486 | trace_hfi1_qpsleep(qp, RVT_S_WAIT_DMA_DESC); |
77241056 MM |
487 | atomic_inc(&qp->refcount); |
488 | } | |
489 | write_sequnlock(&dev->iowait_lock); | |
54d10c1e | 490 | qp->s_flags &= ~RVT_S_BUSY; |
77241056 MM |
491 | spin_unlock_irqrestore(&qp->s_lock, flags); |
492 | ret = -EBUSY; | |
493 | } else { | |
494 | spin_unlock_irqrestore(&qp->s_lock, flags); | |
495 | hfi1_put_txreq(tx); | |
496 | } | |
497 | return ret; | |
498 | eagain: | |
499 | write_sequnlock(&dev->iowait_lock); | |
500 | spin_unlock_irqrestore(&qp->s_lock, flags); | |
501 | list_del_init(&stx->list); | |
502 | return -EAGAIN; | |
503 | } | |
504 | ||
505 | static void iowait_wakeup(struct iowait *wait, int reason) | |
506 | { | |
895420dd | 507 | struct rvt_qp *qp = iowait_to_qp(wait); |
77241056 MM |
508 | |
509 | WARN_ON(reason != SDMA_AVAIL_REASON); | |
54d10c1e | 510 | hfi1_qp_wakeup(qp, RVT_S_WAIT_DMA_DESC); |
77241056 MM |
511 | } |
512 | ||
a545f530 MM |
513 | static void iowait_sdma_drained(struct iowait *wait) |
514 | { | |
515 | struct rvt_qp *qp = iowait_to_qp(wait); | |
516 | ||
517 | /* | |
518 | * This happens when the send engine notes | |
519 | * a QP in the error state and cannot | |
520 | * do the flush work until that QP's | |
521 | * sdma work has finished. | |
522 | */ | |
523 | if (qp->s_flags & RVT_S_WAIT_DMA) { | |
524 | qp->s_flags &= ~RVT_S_WAIT_DMA; | |
525 | hfi1_schedule_send(qp); | |
526 | } | |
527 | } | |
528 | ||
77241056 MM |
529 | /** |
530 | * | |
531 | * qp_to_sdma_engine - map a qp to a send engine | |
532 | * @qp: the QP | |
533 | * @sc5: the 5 bit sc | |
534 | * | |
535 | * Return: | |
536 | * A send engine for the qp or NULL for SMI type qp. | |
537 | */ | |
895420dd | 538 | struct sdma_engine *qp_to_sdma_engine(struct rvt_qp *qp, u8 sc5) |
77241056 MM |
539 | { |
540 | struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); | |
541 | struct sdma_engine *sde; | |
542 | ||
543 | if (!(dd->flags & HFI1_HAS_SEND_DMA)) | |
544 | return NULL; | |
545 | switch (qp->ibqp.qp_type) { | |
77241056 MM |
546 | case IB_QPT_SMI: |
547 | return NULL; | |
548 | default: | |
549 | break; | |
550 | } | |
551 | sde = sdma_select_engine_sc(dd, qp->ibqp.qp_num >> dd->qos_shift, sc5); | |
552 | return sde; | |
553 | } | |
554 | ||
35f6befc JJ |
555 | /* |
556 | * qp_to_send_context - map a qp to a send context | |
557 | * @qp: the QP | |
558 | * @sc5: the 5 bit sc | |
559 | * | |
560 | * Return: | |
561 | * A send context for the qp | |
562 | */ | |
563 | struct send_context *qp_to_send_context(struct rvt_qp *qp, u8 sc5) | |
564 | { | |
565 | struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); | |
566 | ||
567 | switch (qp->ibqp.qp_type) { | |
568 | case IB_QPT_SMI: | |
569 | /* SMA packets to VL15 */ | |
570 | return dd->vld[15].sc; | |
571 | default: | |
572 | break; | |
573 | } | |
574 | ||
575 | return pio_select_send_context_sc(dd, qp->ibqp.qp_num >> dd->qos_shift, | |
576 | sc5); | |
577 | } | |
578 | ||
77241056 MM |
579 | struct qp_iter { |
580 | struct hfi1_ibdev *dev; | |
895420dd | 581 | struct rvt_qp *qp; |
77241056 MM |
582 | int specials; |
583 | int n; | |
584 | }; | |
585 | ||
586 | struct qp_iter *qp_iter_init(struct hfi1_ibdev *dev) | |
587 | { | |
588 | struct qp_iter *iter; | |
589 | ||
590 | iter = kzalloc(sizeof(*iter), GFP_KERNEL); | |
591 | if (!iter) | |
592 | return NULL; | |
593 | ||
594 | iter->dev = dev; | |
ec3f2c12 | 595 | iter->specials = dev->rdi.ibdev.phys_port_cnt * 2; |
77241056 MM |
596 | if (qp_iter_next(iter)) { |
597 | kfree(iter); | |
598 | return NULL; | |
599 | } | |
600 | ||
601 | return iter; | |
602 | } | |
603 | ||
604 | int qp_iter_next(struct qp_iter *iter) | |
605 | { | |
606 | struct hfi1_ibdev *dev = iter->dev; | |
607 | int n = iter->n; | |
608 | int ret = 1; | |
895420dd DD |
609 | struct rvt_qp *pqp = iter->qp; |
610 | struct rvt_qp *qp; | |
77241056 MM |
611 | |
612 | /* | |
613 | * The approach is to consider the special qps | |
614 | * as an additional table entries before the | |
615 | * real hash table. Since the qp code sets | |
616 | * the qp->next hash link to NULL, this works just fine. | |
617 | * | |
618 | * iter->specials is 2 * # ports | |
619 | * | |
620 | * n = 0..iter->specials is the special qp indices | |
621 | * | |
1c4b7d97 | 622 | * n = iter->specials..dev->rdi.qp_dev->qp_table_size+iter->specials are |
77241056 MM |
623 | * the potential hash bucket entries |
624 | * | |
625 | */ | |
1c4b7d97 | 626 | for (; n < dev->rdi.qp_dev->qp_table_size + iter->specials; n++) { |
77241056 MM |
627 | if (pqp) { |
628 | qp = rcu_dereference(pqp->next); | |
629 | } else { | |
630 | if (n < iter->specials) { | |
631 | struct hfi1_pportdata *ppd; | |
632 | struct hfi1_ibport *ibp; | |
633 | int pidx; | |
634 | ||
ec3f2c12 | 635 | pidx = n % dev->rdi.ibdev.phys_port_cnt; |
77241056 MM |
636 | ppd = &dd_from_dev(dev)->pport[pidx]; |
637 | ibp = &ppd->ibport_data; | |
638 | ||
639 | if (!(n & 1)) | |
4eb06882 | 640 | qp = rcu_dereference(ibp->rvp.qp[0]); |
77241056 | 641 | else |
4eb06882 | 642 | qp = rcu_dereference(ibp->rvp.qp[1]); |
77241056 MM |
643 | } else { |
644 | qp = rcu_dereference( | |
1c4b7d97 | 645 | dev->rdi.qp_dev->qp_table[ |
77241056 MM |
646 | (n - iter->specials)]); |
647 | } | |
648 | } | |
649 | pqp = qp; | |
650 | if (qp) { | |
651 | iter->qp = qp; | |
652 | iter->n = n; | |
653 | return 0; | |
654 | } | |
655 | } | |
656 | return ret; | |
657 | } | |
658 | ||
659 | static const char * const qp_type_str[] = { | |
660 | "SMI", "GSI", "RC", "UC", "UD", | |
661 | }; | |
662 | ||
895420dd | 663 | static int qp_idle(struct rvt_qp *qp) |
77241056 MM |
664 | { |
665 | return | |
666 | qp->s_last == qp->s_acked && | |
667 | qp->s_acked == qp->s_cur && | |
668 | qp->s_cur == qp->s_tail && | |
669 | qp->s_tail == qp->s_head; | |
670 | } | |
671 | ||
672 | void qp_iter_print(struct seq_file *s, struct qp_iter *iter) | |
673 | { | |
895420dd DD |
674 | struct rvt_swqe *wqe; |
675 | struct rvt_qp *qp = iter->qp; | |
4c6829c5 | 676 | struct hfi1_qp_priv *priv = qp->priv; |
77241056 | 677 | struct sdma_engine *sde; |
721d0427 | 678 | struct send_context *send_context; |
77241056 | 679 | |
4c6829c5 | 680 | sde = qp_to_sdma_engine(qp, priv->s_sc); |
83693bd1 | 681 | wqe = rvt_get_swqe_ptr(qp, qp->s_last); |
721d0427 | 682 | send_context = qp_to_send_context(qp, priv->s_sc); |
77241056 | 683 | seq_printf(s, |
0358a440 | 684 | "N %d %s QP%x R %u %s %u %u %u f=%x %u %u %u %u %u %u PSN %x %x %x %x %x (%u %u %u %u %u %u %u) QP%x LID %x SL %u MTU %u %u %u %u SDE %p,%u SC %p,%u CQ %u %u\n", |
77241056 MM |
685 | iter->n, |
686 | qp_idle(qp) ? "I" : "B", | |
687 | qp->ibqp.qp_num, | |
688 | atomic_read(&qp->refcount), | |
689 | qp_type_str[qp->ibqp.qp_type], | |
690 | qp->state, | |
691 | wqe ? wqe->wr.opcode : 0, | |
692 | qp->s_hdrwords, | |
693 | qp->s_flags, | |
14553ca1 MM |
694 | iowait_sdma_pending(&priv->s_iowait), |
695 | iowait_pio_pending(&priv->s_iowait), | |
4c6829c5 | 696 | !list_empty(&priv->s_iowait.list), |
77241056 MM |
697 | qp->timeout, |
698 | wqe ? wqe->ssn : 0, | |
699 | qp->s_lsn, | |
700 | qp->s_last_psn, | |
701 | qp->s_psn, qp->s_next_psn, | |
702 | qp->s_sending_psn, qp->s_sending_hpsn, | |
703 | qp->s_last, qp->s_acked, qp->s_cur, | |
704 | qp->s_tail, qp->s_head, qp->s_size, | |
3585254d | 705 | qp->s_avail, |
77241056 MM |
706 | qp->remote_qpn, |
707 | qp->remote_ah_attr.dlid, | |
708 | qp->remote_ah_attr.sl, | |
709 | qp->pmtu, | |
20658661 | 710 | qp->s_retry, |
77241056 | 711 | qp->s_retry_cnt, |
77241056 MM |
712 | qp->s_rnr_retry_cnt, |
713 | sde, | |
721d0427 | 714 | sde ? sde->this_idx : 0, |
77e7639f | 715 | send_context, |
0358a440 VM |
716 | send_context ? send_context->sw_index : 0, |
717 | ibcq_to_rvtcq(qp->ibqp.send_cq)->queue->head, | |
718 | ibcq_to_rvtcq(qp->ibqp.send_cq)->queue->tail); | |
77241056 MM |
719 | } |
720 | ||
895420dd | 721 | void qp_comm_est(struct rvt_qp *qp) |
77241056 | 722 | { |
54d10c1e | 723 | qp->r_flags |= RVT_R_COMM_EST; |
77241056 MM |
724 | if (qp->ibqp.event_handler) { |
725 | struct ib_event ev; | |
726 | ||
727 | ev.device = qp->ibqp.device; | |
728 | ev.element.qp = &qp->ibqp; | |
729 | ev.event = IB_EVENT_COMM_EST; | |
730 | qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); | |
731 | } | |
732 | } | |
c2f3ffb0 | 733 | |
a2c2d608 DD |
734 | void *qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp, |
735 | gfp_t gfp) | |
736 | { | |
737 | struct hfi1_qp_priv *priv; | |
738 | ||
377f111e | 739 | priv = kzalloc_node(sizeof(*priv), gfp, rdi->dparms.node); |
a2c2d608 DD |
740 | if (!priv) |
741 | return ERR_PTR(-ENOMEM); | |
742 | ||
743 | priv->owner = qp; | |
744 | ||
377f111e | 745 | priv->s_hdr = kzalloc_node(sizeof(*priv->s_hdr), gfp, rdi->dparms.node); |
a2c2d608 DD |
746 | if (!priv->s_hdr) { |
747 | kfree(priv); | |
748 | return ERR_PTR(-ENOMEM); | |
749 | } | |
3c9d149b | 750 | setup_timer(&priv->s_rnr_timer, hfi1_rc_rnr_retry, (unsigned long)qp); |
08279d5c | 751 | qp->s_timer.function = hfi1_rc_timeout; |
a2c2d608 DD |
752 | return priv; |
753 | } | |
754 | ||
755 | void qp_priv_free(struct rvt_dev_info *rdi, struct rvt_qp *qp) | |
756 | { | |
757 | struct hfi1_qp_priv *priv = qp->priv; | |
758 | ||
759 | kfree(priv->s_hdr); | |
760 | kfree(priv); | |
761 | } | |
762 | ||
763 | unsigned free_all_qps(struct rvt_dev_info *rdi) | |
764 | { | |
765 | struct hfi1_ibdev *verbs_dev = container_of(rdi, | |
766 | struct hfi1_ibdev, | |
767 | rdi); | |
768 | struct hfi1_devdata *dd = container_of(verbs_dev, | |
769 | struct hfi1_devdata, | |
770 | verbs_dev); | |
771 | int n; | |
772 | unsigned qp_inuse = 0; | |
773 | ||
774 | for (n = 0; n < dd->num_pports; n++) { | |
775 | struct hfi1_ibport *ibp = &dd->pport[n].ibport_data; | |
776 | ||
a2c2d608 DD |
777 | rcu_read_lock(); |
778 | if (rcu_dereference(ibp->rvp.qp[0])) | |
779 | qp_inuse++; | |
780 | if (rcu_dereference(ibp->rvp.qp[1])) | |
781 | qp_inuse++; | |
782 | rcu_read_unlock(); | |
783 | } | |
784 | ||
785 | return qp_inuse; | |
786 | } | |
787 | ||
ec4274f1 DD |
788 | void flush_qp_waiters(struct rvt_qp *qp) |
789 | { | |
790 | flush_iowait(qp); | |
08279d5c | 791 | hfi1_stop_rc_timers(qp); |
ec4274f1 DD |
792 | } |
793 | ||
794 | void stop_send_queue(struct rvt_qp *qp) | |
795 | { | |
796 | struct hfi1_qp_priv *priv = qp->priv; | |
797 | ||
798 | cancel_work_sync(&priv->s_iowait.iowork); | |
08279d5c | 799 | hfi1_del_timers_sync(qp); |
ec4274f1 DD |
800 | } |
801 | ||
802 | void quiesce_qp(struct rvt_qp *qp) | |
803 | { | |
804 | struct hfi1_qp_priv *priv = qp->priv; | |
805 | ||
806 | iowait_sdma_drain(&priv->s_iowait); | |
14553ca1 | 807 | qp_pio_drain(qp); |
ec4274f1 DD |
808 | flush_tx_list(qp); |
809 | } | |
810 | ||
a2c2d608 DD |
811 | void notify_qp_reset(struct rvt_qp *qp) |
812 | { | |
813 | struct hfi1_qp_priv *priv = qp->priv; | |
814 | ||
815 | iowait_init( | |
816 | &priv->s_iowait, | |
817 | 1, | |
83693bd1 | 818 | _hfi1_do_send, |
a2c2d608 | 819 | iowait_sleep, |
a545f530 MM |
820 | iowait_wakeup, |
821 | iowait_sdma_drained); | |
a2c2d608 DD |
822 | priv->r_adefered = 0; |
823 | clear_ahg(qp); | |
824 | } | |
825 | ||
c2f3ffb0 MM |
826 | /* |
827 | * Switch to alternate path. | |
828 | * The QP s_lock should be held and interrupts disabled. | |
829 | */ | |
895420dd | 830 | void hfi1_migrate_qp(struct rvt_qp *qp) |
c2f3ffb0 | 831 | { |
4c6829c5 | 832 | struct hfi1_qp_priv *priv = qp->priv; |
c2f3ffb0 MM |
833 | struct ib_event ev; |
834 | ||
835 | qp->s_mig_state = IB_MIG_MIGRATED; | |
836 | qp->remote_ah_attr = qp->alt_ah_attr; | |
837 | qp->port_num = qp->alt_ah_attr.port_num; | |
838 | qp->s_pkey_index = qp->s_alt_pkey_index; | |
54d10c1e | 839 | qp->s_flags |= RVT_S_AHG_CLEAR; |
4c6829c5 DD |
840 | priv->s_sc = ah_to_sc(qp->ibqp.device, &qp->remote_ah_attr); |
841 | priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc); | |
c2f3ffb0 MM |
842 | |
843 | ev.device = qp->ibqp.device; | |
844 | ev.element.qp = &qp->ibqp; | |
845 | ev.event = IB_EVENT_PATH_MIG; | |
846 | qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); | |
847 | } | |
ec4274f1 DD |
848 | |
849 | int mtu_to_path_mtu(u32 mtu) | |
850 | { | |
851 | return mtu_to_enum(mtu, OPA_MTU_8192); | |
852 | } | |
853 | ||
854 | u32 mtu_from_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, u32 pmtu) | |
855 | { | |
856 | u32 mtu; | |
857 | struct hfi1_ibdev *verbs_dev = container_of(rdi, | |
858 | struct hfi1_ibdev, | |
859 | rdi); | |
860 | struct hfi1_devdata *dd = container_of(verbs_dev, | |
861 | struct hfi1_devdata, | |
862 | verbs_dev); | |
863 | struct hfi1_ibport *ibp; | |
864 | u8 sc, vl; | |
865 | ||
866 | ibp = &dd->pport[qp->port_num - 1].ibport_data; | |
867 | sc = ibp->sl_to_sc[qp->remote_ah_attr.sl]; | |
868 | vl = sc_to_vlt(dd, sc); | |
869 | ||
870 | mtu = verbs_mtu_enum_to_int(qp->ibqp.device, pmtu); | |
871 | if (vl < PER_VL_SEND_CONTEXTS) | |
872 | mtu = min_t(u32, mtu, dd->vld[vl].mtu); | |
873 | return mtu; | |
874 | } | |
875 | ||
876 | int get_pmtu_from_attr(struct rvt_dev_info *rdi, struct rvt_qp *qp, | |
877 | struct ib_qp_attr *attr) | |
878 | { | |
879 | int mtu, pidx = qp->port_num - 1; | |
880 | struct hfi1_ibdev *verbs_dev = container_of(rdi, | |
881 | struct hfi1_ibdev, | |
882 | rdi); | |
883 | struct hfi1_devdata *dd = container_of(verbs_dev, | |
884 | struct hfi1_devdata, | |
885 | verbs_dev); | |
886 | mtu = verbs_mtu_enum_to_int(qp->ibqp.device, attr->path_mtu); | |
887 | if (mtu == -1) | |
888 | return -1; /* values less than 0 are error */ | |
889 | ||
890 | if (mtu > dd->pport[pidx].ibmtu) | |
891 | return mtu_to_enum(dd->pport[pidx].ibmtu, IB_MTU_2048); | |
892 | else | |
893 | return attr->path_mtu; | |
894 | } | |
895 | ||
896 | void notify_error_qp(struct rvt_qp *qp) | |
897 | { | |
898 | struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); | |
899 | struct hfi1_qp_priv *priv = qp->priv; | |
900 | ||
901 | write_seqlock(&dev->iowait_lock); | |
902 | if (!list_empty(&priv->s_iowait.list) && !(qp->s_flags & RVT_S_BUSY)) { | |
903 | qp->s_flags &= ~RVT_S_ANY_WAIT_IO; | |
904 | list_del_init(&priv->s_iowait.list); | |
905 | if (atomic_dec_and_test(&qp->refcount)) | |
906 | wake_up(&qp->wait); | |
907 | } | |
908 | write_sequnlock(&dev->iowait_lock); | |
909 | ||
910 | if (!(qp->s_flags & RVT_S_BUSY)) { | |
911 | qp->s_hdrwords = 0; | |
912 | if (qp->s_rdma_mr) { | |
913 | rvt_put_mr(qp->s_rdma_mr); | |
914 | qp->s_rdma_mr = NULL; | |
915 | } | |
916 | flush_tx_list(qp); | |
917 | } | |
918 | } | |
919 | ||
0ec79e87 KW |
920 | /** |
921 | * hfi1_error_port_qps - put a port's RC/UC qps into error state | |
922 | * @ibp: the ibport. | |
923 | * @sl: the service level. | |
924 | * | |
925 | * This function places all RC/UC qps with a given service level into error | |
926 | * state. It is generally called to force upper lay apps to abandon stale qps | |
927 | * after an sl->sc mapping change. | |
928 | */ | |
929 | void hfi1_error_port_qps(struct hfi1_ibport *ibp, u8 sl) | |
930 | { | |
931 | struct rvt_qp *qp = NULL; | |
932 | struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); | |
933 | struct hfi1_ibdev *dev = &ppd->dd->verbs_dev; | |
934 | int n; | |
935 | int lastwqe; | |
936 | struct ib_event ev; | |
937 | ||
938 | rcu_read_lock(); | |
939 | ||
940 | /* Deal only with RC/UC qps that use the given SL. */ | |
941 | for (n = 0; n < dev->rdi.qp_dev->qp_table_size; n++) { | |
942 | for (qp = rcu_dereference(dev->rdi.qp_dev->qp_table[n]); qp; | |
943 | qp = rcu_dereference(qp->next)) { | |
944 | if (qp->port_num == ppd->port && | |
945 | (qp->ibqp.qp_type == IB_QPT_UC || | |
946 | qp->ibqp.qp_type == IB_QPT_RC) && | |
947 | qp->remote_ah_attr.sl == sl && | |
948 | (ib_rvt_state_ops[qp->state] & | |
949 | RVT_POST_SEND_OK)) { | |
950 | spin_lock_irq(&qp->r_lock); | |
951 | spin_lock(&qp->s_hlock); | |
952 | spin_lock(&qp->s_lock); | |
953 | lastwqe = rvt_error_qp(qp, | |
954 | IB_WC_WR_FLUSH_ERR); | |
955 | spin_unlock(&qp->s_lock); | |
956 | spin_unlock(&qp->s_hlock); | |
957 | spin_unlock_irq(&qp->r_lock); | |
958 | if (lastwqe) { | |
959 | ev.device = qp->ibqp.device; | |
960 | ev.element.qp = &qp->ibqp; | |
961 | ev.event = | |
962 | IB_EVENT_QP_LAST_WQE_REACHED; | |
963 | qp->ibqp.event_handler(&ev, | |
964 | qp->ibqp.qp_context); | |
965 | } | |
966 | } | |
967 | } | |
968 | } | |
969 | ||
970 | rcu_read_unlock(); | |
971 | } |