2 * Copyright (c) 2005 Voltaire Inc. All rights reserved.
3 * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
4 * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
5 * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #include <linux/completion.h>
38 #include <linux/in6.h>
39 #include <linux/mutex.h>
40 #include <linux/random.h>
41 #include <linux/idr.h>
42 #include <linux/inetdevice.h>
43 #include <linux/slab.h>
44 #include <linux/module.h>
45 #include <net/route.h>
49 #include <net/ip_fib.h>
50 #include <net/ip6_route.h>
52 #include <rdma/rdma_cm.h>
53 #include <rdma/rdma_cm_ib.h>
54 #include <rdma/rdma_netlink.h>
56 #include <rdma/ib_cache.h>
57 #include <rdma/ib_cm.h>
58 #include <rdma/ib_sa.h>
59 #include <rdma/iw_cm.h>
61 MODULE_AUTHOR("Sean Hefty");
62 MODULE_DESCRIPTION("Generic RDMA CM Agent");
63 MODULE_LICENSE("Dual BSD/GPL");
65 #define CMA_CM_RESPONSE_TIMEOUT 20
66 #define CMA_MAX_CM_RETRIES 15
67 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
68 #define CMA_IBOE_PACKET_LIFETIME 18
70 static const char * const cma_events
[] = {
71 [RDMA_CM_EVENT_ADDR_RESOLVED
] = "address resolved",
72 [RDMA_CM_EVENT_ADDR_ERROR
] = "address error",
73 [RDMA_CM_EVENT_ROUTE_RESOLVED
] = "route resolved ",
74 [RDMA_CM_EVENT_ROUTE_ERROR
] = "route error",
75 [RDMA_CM_EVENT_CONNECT_REQUEST
] = "connect request",
76 [RDMA_CM_EVENT_CONNECT_RESPONSE
] = "connect response",
77 [RDMA_CM_EVENT_CONNECT_ERROR
] = "connect error",
78 [RDMA_CM_EVENT_UNREACHABLE
] = "unreachable",
79 [RDMA_CM_EVENT_REJECTED
] = "rejected",
80 [RDMA_CM_EVENT_ESTABLISHED
] = "established",
81 [RDMA_CM_EVENT_DISCONNECTED
] = "disconnected",
82 [RDMA_CM_EVENT_DEVICE_REMOVAL
] = "device removal",
83 [RDMA_CM_EVENT_MULTICAST_JOIN
] = "multicast join",
84 [RDMA_CM_EVENT_MULTICAST_ERROR
] = "multicast error",
85 [RDMA_CM_EVENT_ADDR_CHANGE
] = "address change",
86 [RDMA_CM_EVENT_TIMEWAIT_EXIT
] = "timewait exit",
89 const char *rdma_event_msg(enum rdma_cm_event_type event
)
93 return (index
< ARRAY_SIZE(cma_events
) && cma_events
[index
]) ?
94 cma_events
[index
] : "unrecognized event";
96 EXPORT_SYMBOL(rdma_event_msg
);
98 static void cma_add_one(struct ib_device
*device
);
99 static void cma_remove_one(struct ib_device
*device
, void *client_data
);
101 static struct ib_client cma_client
= {
104 .remove
= cma_remove_one
107 static struct ib_sa_client sa_client
;
108 static struct rdma_addr_client addr_client
;
109 static LIST_HEAD(dev_list
);
110 static LIST_HEAD(listen_any_list
);
111 static DEFINE_MUTEX(lock
);
112 static struct workqueue_struct
*cma_wq
;
113 static DEFINE_IDR(tcp_ps
);
114 static DEFINE_IDR(udp_ps
);
115 static DEFINE_IDR(ipoib_ps
);
116 static DEFINE_IDR(ib_ps
);
118 static struct idr
*cma_idr(enum rdma_port_space ps
)
135 struct list_head list
;
136 struct ib_device
*device
;
137 struct completion comp
;
139 struct list_head id_list
;
142 struct rdma_bind_list
{
143 enum rdma_port_space ps
;
144 struct hlist_head owners
;
148 static int cma_ps_alloc(enum rdma_port_space ps
,
149 struct rdma_bind_list
*bind_list
, int snum
)
151 struct idr
*idr
= cma_idr(ps
);
153 return idr_alloc(idr
, bind_list
, snum
, snum
+ 1, GFP_KERNEL
);
156 static struct rdma_bind_list
*cma_ps_find(enum rdma_port_space ps
, int snum
)
158 struct idr
*idr
= cma_idr(ps
);
160 return idr_find(idr
, snum
);
163 static void cma_ps_remove(enum rdma_port_space ps
, int snum
)
165 struct idr
*idr
= cma_idr(ps
);
167 idr_remove(idr
, snum
);
175 * Device removal can occur at anytime, so we need extra handling to
176 * serialize notifying the user of device removal with other callbacks.
177 * We do this by disabling removal notification while a callback is in process,
178 * and reporting it after the callback completes.
180 struct rdma_id_private
{
181 struct rdma_cm_id id
;
183 struct rdma_bind_list
*bind_list
;
184 struct hlist_node node
;
185 struct list_head list
; /* listen_any_list or cma_device.list */
186 struct list_head listen_list
; /* per device listens */
187 struct cma_device
*cma_dev
;
188 struct list_head mc_list
;
191 enum rdma_cm_state state
;
193 struct mutex qp_mutex
;
195 struct completion comp
;
197 struct mutex handler_mutex
;
201 struct ib_sa_query
*query
;
219 struct cma_multicast
{
220 struct rdma_id_private
*id_priv
;
222 struct ib_sa_multicast
*ib
;
224 struct list_head list
;
226 struct sockaddr_storage addr
;
231 struct work_struct work
;
232 struct rdma_id_private
*id
;
233 enum rdma_cm_state old_state
;
234 enum rdma_cm_state new_state
;
235 struct rdma_cm_event event
;
238 struct cma_ndev_work
{
239 struct work_struct work
;
240 struct rdma_id_private
*id
;
241 struct rdma_cm_event event
;
244 struct iboe_mcast_work
{
245 struct work_struct work
;
246 struct rdma_id_private
*id
;
247 struct cma_multicast
*mc
;
260 u8 ip_version
; /* IP version: 7:4 */
262 union cma_ip_addr src_addr
;
263 union cma_ip_addr dst_addr
;
266 #define CMA_VERSION 0x00
268 struct cma_req_info
{
269 struct ib_device
*device
;
271 union ib_gid local_gid
;
277 static int cma_comp(struct rdma_id_private
*id_priv
, enum rdma_cm_state comp
)
282 spin_lock_irqsave(&id_priv
->lock
, flags
);
283 ret
= (id_priv
->state
== comp
);
284 spin_unlock_irqrestore(&id_priv
->lock
, flags
);
288 static int cma_comp_exch(struct rdma_id_private
*id_priv
,
289 enum rdma_cm_state comp
, enum rdma_cm_state exch
)
294 spin_lock_irqsave(&id_priv
->lock
, flags
);
295 if ((ret
= (id_priv
->state
== comp
)))
296 id_priv
->state
= exch
;
297 spin_unlock_irqrestore(&id_priv
->lock
, flags
);
301 static enum rdma_cm_state
cma_exch(struct rdma_id_private
*id_priv
,
302 enum rdma_cm_state exch
)
305 enum rdma_cm_state old
;
307 spin_lock_irqsave(&id_priv
->lock
, flags
);
308 old
= id_priv
->state
;
309 id_priv
->state
= exch
;
310 spin_unlock_irqrestore(&id_priv
->lock
, flags
);
314 static inline u8
cma_get_ip_ver(const struct cma_hdr
*hdr
)
316 return hdr
->ip_version
>> 4;
319 static inline void cma_set_ip_ver(struct cma_hdr
*hdr
, u8 ip_ver
)
321 hdr
->ip_version
= (ip_ver
<< 4) | (hdr
->ip_version
& 0xF);
324 static void cma_attach_to_dev(struct rdma_id_private
*id_priv
,
325 struct cma_device
*cma_dev
)
327 atomic_inc(&cma_dev
->refcount
);
328 id_priv
->cma_dev
= cma_dev
;
329 id_priv
->id
.device
= cma_dev
->device
;
330 id_priv
->id
.route
.addr
.dev_addr
.transport
=
331 rdma_node_get_transport(cma_dev
->device
->node_type
);
332 list_add_tail(&id_priv
->list
, &cma_dev
->id_list
);
335 static inline void cma_deref_dev(struct cma_device
*cma_dev
)
337 if (atomic_dec_and_test(&cma_dev
->refcount
))
338 complete(&cma_dev
->comp
);
341 static inline void release_mc(struct kref
*kref
)
343 struct cma_multicast
*mc
= container_of(kref
, struct cma_multicast
, mcref
);
345 kfree(mc
->multicast
.ib
);
349 static void cma_release_dev(struct rdma_id_private
*id_priv
)
352 list_del(&id_priv
->list
);
353 cma_deref_dev(id_priv
->cma_dev
);
354 id_priv
->cma_dev
= NULL
;
358 static inline struct sockaddr
*cma_src_addr(struct rdma_id_private
*id_priv
)
360 return (struct sockaddr
*) &id_priv
->id
.route
.addr
.src_addr
;
363 static inline struct sockaddr
*cma_dst_addr(struct rdma_id_private
*id_priv
)
365 return (struct sockaddr
*) &id_priv
->id
.route
.addr
.dst_addr
;
368 static inline unsigned short cma_family(struct rdma_id_private
*id_priv
)
370 return id_priv
->id
.route
.addr
.src_addr
.ss_family
;
373 static int cma_set_qkey(struct rdma_id_private
*id_priv
, u32 qkey
)
375 struct ib_sa_mcmember_rec rec
;
379 if (qkey
&& id_priv
->qkey
!= qkey
)
385 id_priv
->qkey
= qkey
;
389 switch (id_priv
->id
.ps
) {
392 id_priv
->qkey
= RDMA_UDP_QKEY
;
395 ib_addr_get_mgid(&id_priv
->id
.route
.addr
.dev_addr
, &rec
.mgid
);
396 ret
= ib_sa_get_mcmember_rec(id_priv
->id
.device
,
397 id_priv
->id
.port_num
, &rec
.mgid
,
400 id_priv
->qkey
= be32_to_cpu(rec
.qkey
);
408 static void cma_translate_ib(struct sockaddr_ib
*sib
, struct rdma_dev_addr
*dev_addr
)
410 dev_addr
->dev_type
= ARPHRD_INFINIBAND
;
411 rdma_addr_set_sgid(dev_addr
, (union ib_gid
*) &sib
->sib_addr
);
412 ib_addr_set_pkey(dev_addr
, ntohs(sib
->sib_pkey
));
415 static int cma_translate_addr(struct sockaddr
*addr
, struct rdma_dev_addr
*dev_addr
)
419 if (addr
->sa_family
!= AF_IB
) {
420 ret
= rdma_translate_ip(addr
, dev_addr
, NULL
);
422 cma_translate_ib((struct sockaddr_ib
*) addr
, dev_addr
);
429 static inline int cma_validate_port(struct ib_device
*device
, u8 port
,
430 union ib_gid
*gid
, int dev_type
)
435 if ((dev_type
== ARPHRD_INFINIBAND
) && !rdma_protocol_ib(device
, port
))
438 if ((dev_type
!= ARPHRD_INFINIBAND
) && rdma_protocol_ib(device
, port
))
441 ret
= ib_find_cached_gid(device
, gid
, NULL
, &found_port
, NULL
);
442 if (port
!= found_port
)
448 static int cma_acquire_dev(struct rdma_id_private
*id_priv
,
449 struct rdma_id_private
*listen_id_priv
)
451 struct rdma_dev_addr
*dev_addr
= &id_priv
->id
.route
.addr
.dev_addr
;
452 struct cma_device
*cma_dev
;
453 union ib_gid gid
, iboe_gid
, *gidp
;
457 if (dev_addr
->dev_type
!= ARPHRD_INFINIBAND
&&
458 id_priv
->id
.ps
== RDMA_PS_IPOIB
)
462 rdma_ip2gid((struct sockaddr
*)&id_priv
->id
.route
.addr
.src_addr
,
465 memcpy(&gid
, dev_addr
->src_dev_addr
+
466 rdma_addr_gid_offset(dev_addr
), sizeof gid
);
468 if (listen_id_priv
) {
469 cma_dev
= listen_id_priv
->cma_dev
;
470 port
= listen_id_priv
->id
.port_num
;
471 gidp
= rdma_protocol_roce(cma_dev
->device
, port
) ?
474 ret
= cma_validate_port(cma_dev
->device
, port
, gidp
,
477 id_priv
->id
.port_num
= port
;
482 list_for_each_entry(cma_dev
, &dev_list
, list
) {
483 for (port
= 1; port
<= cma_dev
->device
->phys_port_cnt
; ++port
) {
484 if (listen_id_priv
&&
485 listen_id_priv
->cma_dev
== cma_dev
&&
486 listen_id_priv
->id
.port_num
== port
)
489 gidp
= rdma_protocol_roce(cma_dev
->device
, port
) ?
492 ret
= cma_validate_port(cma_dev
->device
, port
, gidp
,
495 id_priv
->id
.port_num
= port
;
503 cma_attach_to_dev(id_priv
, cma_dev
);
510 * Select the source IB device and address to reach the destination IB address.
512 static int cma_resolve_ib_dev(struct rdma_id_private
*id_priv
)
514 struct cma_device
*cma_dev
, *cur_dev
;
515 struct sockaddr_ib
*addr
;
516 union ib_gid gid
, sgid
, *dgid
;
522 addr
= (struct sockaddr_ib
*) cma_dst_addr(id_priv
);
523 dgid
= (union ib_gid
*) &addr
->sib_addr
;
524 pkey
= ntohs(addr
->sib_pkey
);
526 list_for_each_entry(cur_dev
, &dev_list
, list
) {
527 for (p
= 1; p
<= cur_dev
->device
->phys_port_cnt
; ++p
) {
528 if (!rdma_cap_af_ib(cur_dev
->device
, p
))
531 if (ib_find_cached_pkey(cur_dev
->device
, p
, pkey
, &index
))
534 for (i
= 0; !ib_get_cached_gid(cur_dev
->device
, p
, i
,
537 if (!memcmp(&gid
, dgid
, sizeof(gid
))) {
540 id_priv
->id
.port_num
= p
;
544 if (!cma_dev
&& (gid
.global
.subnet_prefix
==
545 dgid
->global
.subnet_prefix
)) {
548 id_priv
->id
.port_num
= p
;
558 cma_attach_to_dev(id_priv
, cma_dev
);
559 addr
= (struct sockaddr_ib
*) cma_src_addr(id_priv
);
560 memcpy(&addr
->sib_addr
, &sgid
, sizeof sgid
);
561 cma_translate_ib(addr
, &id_priv
->id
.route
.addr
.dev_addr
);
565 static void cma_deref_id(struct rdma_id_private
*id_priv
)
567 if (atomic_dec_and_test(&id_priv
->refcount
))
568 complete(&id_priv
->comp
);
571 static int cma_disable_callback(struct rdma_id_private
*id_priv
,
572 enum rdma_cm_state state
)
574 mutex_lock(&id_priv
->handler_mutex
);
575 if (id_priv
->state
!= state
) {
576 mutex_unlock(&id_priv
->handler_mutex
);
582 struct rdma_cm_id
*rdma_create_id(rdma_cm_event_handler event_handler
,
583 void *context
, enum rdma_port_space ps
,
584 enum ib_qp_type qp_type
)
586 struct rdma_id_private
*id_priv
;
588 id_priv
= kzalloc(sizeof *id_priv
, GFP_KERNEL
);
590 return ERR_PTR(-ENOMEM
);
592 id_priv
->owner
= task_pid_nr(current
);
593 id_priv
->state
= RDMA_CM_IDLE
;
594 id_priv
->id
.context
= context
;
595 id_priv
->id
.event_handler
= event_handler
;
597 id_priv
->id
.qp_type
= qp_type
;
598 spin_lock_init(&id_priv
->lock
);
599 mutex_init(&id_priv
->qp_mutex
);
600 init_completion(&id_priv
->comp
);
601 atomic_set(&id_priv
->refcount
, 1);
602 mutex_init(&id_priv
->handler_mutex
);
603 INIT_LIST_HEAD(&id_priv
->listen_list
);
604 INIT_LIST_HEAD(&id_priv
->mc_list
);
605 get_random_bytes(&id_priv
->seq_num
, sizeof id_priv
->seq_num
);
609 EXPORT_SYMBOL(rdma_create_id
);
611 static int cma_init_ud_qp(struct rdma_id_private
*id_priv
, struct ib_qp
*qp
)
613 struct ib_qp_attr qp_attr
;
614 int qp_attr_mask
, ret
;
616 qp_attr
.qp_state
= IB_QPS_INIT
;
617 ret
= rdma_init_qp_attr(&id_priv
->id
, &qp_attr
, &qp_attr_mask
);
621 ret
= ib_modify_qp(qp
, &qp_attr
, qp_attr_mask
);
625 qp_attr
.qp_state
= IB_QPS_RTR
;
626 ret
= ib_modify_qp(qp
, &qp_attr
, IB_QP_STATE
);
630 qp_attr
.qp_state
= IB_QPS_RTS
;
632 ret
= ib_modify_qp(qp
, &qp_attr
, IB_QP_STATE
| IB_QP_SQ_PSN
);
637 static int cma_init_conn_qp(struct rdma_id_private
*id_priv
, struct ib_qp
*qp
)
639 struct ib_qp_attr qp_attr
;
640 int qp_attr_mask
, ret
;
642 qp_attr
.qp_state
= IB_QPS_INIT
;
643 ret
= rdma_init_qp_attr(&id_priv
->id
, &qp_attr
, &qp_attr_mask
);
647 return ib_modify_qp(qp
, &qp_attr
, qp_attr_mask
);
650 int rdma_create_qp(struct rdma_cm_id
*id
, struct ib_pd
*pd
,
651 struct ib_qp_init_attr
*qp_init_attr
)
653 struct rdma_id_private
*id_priv
;
657 id_priv
= container_of(id
, struct rdma_id_private
, id
);
658 if (id
->device
!= pd
->device
)
661 qp
= ib_create_qp(pd
, qp_init_attr
);
665 if (id
->qp_type
== IB_QPT_UD
)
666 ret
= cma_init_ud_qp(id_priv
, qp
);
668 ret
= cma_init_conn_qp(id_priv
, qp
);
673 id_priv
->qp_num
= qp
->qp_num
;
674 id_priv
->srq
= (qp
->srq
!= NULL
);
680 EXPORT_SYMBOL(rdma_create_qp
);
682 void rdma_destroy_qp(struct rdma_cm_id
*id
)
684 struct rdma_id_private
*id_priv
;
686 id_priv
= container_of(id
, struct rdma_id_private
, id
);
687 mutex_lock(&id_priv
->qp_mutex
);
688 ib_destroy_qp(id_priv
->id
.qp
);
689 id_priv
->id
.qp
= NULL
;
690 mutex_unlock(&id_priv
->qp_mutex
);
692 EXPORT_SYMBOL(rdma_destroy_qp
);
694 static int cma_modify_qp_rtr(struct rdma_id_private
*id_priv
,
695 struct rdma_conn_param
*conn_param
)
697 struct ib_qp_attr qp_attr
;
698 int qp_attr_mask
, ret
;
701 mutex_lock(&id_priv
->qp_mutex
);
702 if (!id_priv
->id
.qp
) {
707 /* Need to update QP attributes from default values. */
708 qp_attr
.qp_state
= IB_QPS_INIT
;
709 ret
= rdma_init_qp_attr(&id_priv
->id
, &qp_attr
, &qp_attr_mask
);
713 ret
= ib_modify_qp(id_priv
->id
.qp
, &qp_attr
, qp_attr_mask
);
717 qp_attr
.qp_state
= IB_QPS_RTR
;
718 ret
= rdma_init_qp_attr(&id_priv
->id
, &qp_attr
, &qp_attr_mask
);
722 ret
= ib_query_gid(id_priv
->id
.device
, id_priv
->id
.port_num
,
723 qp_attr
.ah_attr
.grh
.sgid_index
, &sgid
, NULL
);
727 BUG_ON(id_priv
->cma_dev
->device
!= id_priv
->id
.device
);
729 if (rdma_protocol_roce(id_priv
->id
.device
, id_priv
->id
.port_num
)) {
730 ret
= rdma_addr_find_smac_by_sgid(&sgid
, qp_attr
.smac
, NULL
);
736 qp_attr
.max_dest_rd_atomic
= conn_param
->responder_resources
;
737 ret
= ib_modify_qp(id_priv
->id
.qp
, &qp_attr
, qp_attr_mask
);
739 mutex_unlock(&id_priv
->qp_mutex
);
743 static int cma_modify_qp_rts(struct rdma_id_private
*id_priv
,
744 struct rdma_conn_param
*conn_param
)
746 struct ib_qp_attr qp_attr
;
747 int qp_attr_mask
, ret
;
749 mutex_lock(&id_priv
->qp_mutex
);
750 if (!id_priv
->id
.qp
) {
755 qp_attr
.qp_state
= IB_QPS_RTS
;
756 ret
= rdma_init_qp_attr(&id_priv
->id
, &qp_attr
, &qp_attr_mask
);
761 qp_attr
.max_rd_atomic
= conn_param
->initiator_depth
;
762 ret
= ib_modify_qp(id_priv
->id
.qp
, &qp_attr
, qp_attr_mask
);
764 mutex_unlock(&id_priv
->qp_mutex
);
768 static int cma_modify_qp_err(struct rdma_id_private
*id_priv
)
770 struct ib_qp_attr qp_attr
;
773 mutex_lock(&id_priv
->qp_mutex
);
774 if (!id_priv
->id
.qp
) {
779 qp_attr
.qp_state
= IB_QPS_ERR
;
780 ret
= ib_modify_qp(id_priv
->id
.qp
, &qp_attr
, IB_QP_STATE
);
782 mutex_unlock(&id_priv
->qp_mutex
);
786 static int cma_ib_init_qp_attr(struct rdma_id_private
*id_priv
,
787 struct ib_qp_attr
*qp_attr
, int *qp_attr_mask
)
789 struct rdma_dev_addr
*dev_addr
= &id_priv
->id
.route
.addr
.dev_addr
;
793 if (rdma_cap_eth_ah(id_priv
->id
.device
, id_priv
->id
.port_num
))
796 pkey
= ib_addr_get_pkey(dev_addr
);
798 ret
= ib_find_cached_pkey(id_priv
->id
.device
, id_priv
->id
.port_num
,
799 pkey
, &qp_attr
->pkey_index
);
803 qp_attr
->port_num
= id_priv
->id
.port_num
;
804 *qp_attr_mask
= IB_QP_STATE
| IB_QP_PKEY_INDEX
| IB_QP_PORT
;
806 if (id_priv
->id
.qp_type
== IB_QPT_UD
) {
807 ret
= cma_set_qkey(id_priv
, 0);
811 qp_attr
->qkey
= id_priv
->qkey
;
812 *qp_attr_mask
|= IB_QP_QKEY
;
814 qp_attr
->qp_access_flags
= 0;
815 *qp_attr_mask
|= IB_QP_ACCESS_FLAGS
;
820 int rdma_init_qp_attr(struct rdma_cm_id
*id
, struct ib_qp_attr
*qp_attr
,
823 struct rdma_id_private
*id_priv
;
826 id_priv
= container_of(id
, struct rdma_id_private
, id
);
827 if (rdma_cap_ib_cm(id
->device
, id
->port_num
)) {
828 if (!id_priv
->cm_id
.ib
|| (id_priv
->id
.qp_type
== IB_QPT_UD
))
829 ret
= cma_ib_init_qp_attr(id_priv
, qp_attr
, qp_attr_mask
);
831 ret
= ib_cm_init_qp_attr(id_priv
->cm_id
.ib
, qp_attr
,
834 if (qp_attr
->qp_state
== IB_QPS_RTR
)
835 qp_attr
->rq_psn
= id_priv
->seq_num
;
836 } else if (rdma_cap_iw_cm(id
->device
, id
->port_num
)) {
837 if (!id_priv
->cm_id
.iw
) {
838 qp_attr
->qp_access_flags
= 0;
839 *qp_attr_mask
= IB_QP_STATE
| IB_QP_ACCESS_FLAGS
;
841 ret
= iw_cm_init_qp_attr(id_priv
->cm_id
.iw
, qp_attr
,
848 EXPORT_SYMBOL(rdma_init_qp_attr
);
850 static inline int cma_zero_addr(struct sockaddr
*addr
)
852 switch (addr
->sa_family
) {
854 return ipv4_is_zeronet(((struct sockaddr_in
*)addr
)->sin_addr
.s_addr
);
856 return ipv6_addr_any(&((struct sockaddr_in6
*) addr
)->sin6_addr
);
858 return ib_addr_any(&((struct sockaddr_ib
*) addr
)->sib_addr
);
864 static inline int cma_loopback_addr(struct sockaddr
*addr
)
866 switch (addr
->sa_family
) {
868 return ipv4_is_loopback(((struct sockaddr_in
*) addr
)->sin_addr
.s_addr
);
870 return ipv6_addr_loopback(&((struct sockaddr_in6
*) addr
)->sin6_addr
);
872 return ib_addr_loopback(&((struct sockaddr_ib
*) addr
)->sib_addr
);
878 static inline int cma_any_addr(struct sockaddr
*addr
)
880 return cma_zero_addr(addr
) || cma_loopback_addr(addr
);
883 static int cma_addr_cmp(struct sockaddr
*src
, struct sockaddr
*dst
)
885 if (src
->sa_family
!= dst
->sa_family
)
888 switch (src
->sa_family
) {
890 return ((struct sockaddr_in
*) src
)->sin_addr
.s_addr
!=
891 ((struct sockaddr_in
*) dst
)->sin_addr
.s_addr
;
893 return ipv6_addr_cmp(&((struct sockaddr_in6
*) src
)->sin6_addr
,
894 &((struct sockaddr_in6
*) dst
)->sin6_addr
);
896 return ib_addr_cmp(&((struct sockaddr_ib
*) src
)->sib_addr
,
897 &((struct sockaddr_ib
*) dst
)->sib_addr
);
901 static __be16
cma_port(struct sockaddr
*addr
)
903 struct sockaddr_ib
*sib
;
905 switch (addr
->sa_family
) {
907 return ((struct sockaddr_in
*) addr
)->sin_port
;
909 return ((struct sockaddr_in6
*) addr
)->sin6_port
;
911 sib
= (struct sockaddr_ib
*) addr
;
912 return htons((u16
) (be64_to_cpu(sib
->sib_sid
) &
913 be64_to_cpu(sib
->sib_sid_mask
)));
919 static inline int cma_any_port(struct sockaddr
*addr
)
921 return !cma_port(addr
);
924 static void cma_save_ib_info(struct sockaddr
*src_addr
,
925 struct sockaddr
*dst_addr
,
926 struct rdma_cm_id
*listen_id
,
927 struct ib_sa_path_rec
*path
)
929 struct sockaddr_ib
*listen_ib
, *ib
;
931 listen_ib
= (struct sockaddr_ib
*) &listen_id
->route
.addr
.src_addr
;
933 ib
= (struct sockaddr_ib
*)src_addr
;
934 ib
->sib_family
= AF_IB
;
936 ib
->sib_pkey
= path
->pkey
;
937 ib
->sib_flowinfo
= path
->flow_label
;
938 memcpy(&ib
->sib_addr
, &path
->sgid
, 16);
939 ib
->sib_sid
= path
->service_id
;
940 ib
->sib_scope_id
= 0;
942 ib
->sib_pkey
= listen_ib
->sib_pkey
;
943 ib
->sib_flowinfo
= listen_ib
->sib_flowinfo
;
944 ib
->sib_addr
= listen_ib
->sib_addr
;
945 ib
->sib_sid
= listen_ib
->sib_sid
;
946 ib
->sib_scope_id
= listen_ib
->sib_scope_id
;
948 ib
->sib_sid_mask
= cpu_to_be64(0xffffffffffffffffULL
);
951 ib
= (struct sockaddr_ib
*)dst_addr
;
952 ib
->sib_family
= AF_IB
;
954 ib
->sib_pkey
= path
->pkey
;
955 ib
->sib_flowinfo
= path
->flow_label
;
956 memcpy(&ib
->sib_addr
, &path
->dgid
, 16);
961 static void cma_save_ip4_info(struct sockaddr
*src_addr
,
962 struct sockaddr
*dst_addr
,
966 struct sockaddr_in
*ip4
;
969 ip4
= (struct sockaddr_in
*)src_addr
;
970 ip4
->sin_family
= AF_INET
;
971 ip4
->sin_addr
.s_addr
= hdr
->dst_addr
.ip4
.addr
;
972 ip4
->sin_port
= local_port
;
976 ip4
= (struct sockaddr_in
*)dst_addr
;
977 ip4
->sin_family
= AF_INET
;
978 ip4
->sin_addr
.s_addr
= hdr
->src_addr
.ip4
.addr
;
979 ip4
->sin_port
= hdr
->port
;
983 static void cma_save_ip6_info(struct sockaddr
*src_addr
,
984 struct sockaddr
*dst_addr
,
988 struct sockaddr_in6
*ip6
;
991 ip6
= (struct sockaddr_in6
*)src_addr
;
992 ip6
->sin6_family
= AF_INET6
;
993 ip6
->sin6_addr
= hdr
->dst_addr
.ip6
;
994 ip6
->sin6_port
= local_port
;
998 ip6
= (struct sockaddr_in6
*)dst_addr
;
999 ip6
->sin6_family
= AF_INET6
;
1000 ip6
->sin6_addr
= hdr
->src_addr
.ip6
;
1001 ip6
->sin6_port
= hdr
->port
;
1005 static u16
cma_port_from_service_id(__be64 service_id
)
1007 return (u16
)be64_to_cpu(service_id
);
1010 static int cma_save_ip_info(struct sockaddr
*src_addr
,
1011 struct sockaddr
*dst_addr
,
1012 struct ib_cm_event
*ib_event
,
1015 struct cma_hdr
*hdr
;
1018 hdr
= ib_event
->private_data
;
1019 if (hdr
->cma_version
!= CMA_VERSION
)
1022 port
= htons(cma_port_from_service_id(service_id
));
1024 switch (cma_get_ip_ver(hdr
)) {
1026 cma_save_ip4_info(src_addr
, dst_addr
, hdr
, port
);
1029 cma_save_ip6_info(src_addr
, dst_addr
, hdr
, port
);
1032 return -EAFNOSUPPORT
;
1038 static int cma_save_net_info(struct sockaddr
*src_addr
,
1039 struct sockaddr
*dst_addr
,
1040 struct rdma_cm_id
*listen_id
,
1041 struct ib_cm_event
*ib_event
,
1042 sa_family_t sa_family
, __be64 service_id
)
1044 if (sa_family
== AF_IB
) {
1045 if (ib_event
->event
== IB_CM_REQ_RECEIVED
)
1046 cma_save_ib_info(src_addr
, dst_addr
, listen_id
,
1047 ib_event
->param
.req_rcvd
.primary_path
);
1048 else if (ib_event
->event
== IB_CM_SIDR_REQ_RECEIVED
)
1049 cma_save_ib_info(src_addr
, dst_addr
, listen_id
, NULL
);
1053 return cma_save_ip_info(src_addr
, dst_addr
, ib_event
, service_id
);
1056 static int cma_save_req_info(const struct ib_cm_event
*ib_event
,
1057 struct cma_req_info
*req
)
1059 const struct ib_cm_req_event_param
*req_param
=
1060 &ib_event
->param
.req_rcvd
;
1061 const struct ib_cm_sidr_req_event_param
*sidr_param
=
1062 &ib_event
->param
.sidr_req_rcvd
;
1064 switch (ib_event
->event
) {
1065 case IB_CM_REQ_RECEIVED
:
1066 req
->device
= req_param
->listen_id
->device
;
1067 req
->port
= req_param
->port
;
1068 memcpy(&req
->local_gid
, &req_param
->primary_path
->sgid
,
1069 sizeof(req
->local_gid
));
1070 req
->has_gid
= true;
1071 req
->service_id
= req_param
->primary_path
->service_id
;
1072 req
->pkey
= be16_to_cpu(req_param
->primary_path
->pkey
);
1074 case IB_CM_SIDR_REQ_RECEIVED
:
1075 req
->device
= sidr_param
->listen_id
->device
;
1076 req
->port
= sidr_param
->port
;
1077 req
->has_gid
= false;
1078 req
->service_id
= sidr_param
->service_id
;
1079 req
->pkey
= sidr_param
->pkey
;
1088 static bool validate_ipv4_net_dev(struct net_device
*net_dev
,
1089 const struct sockaddr_in
*dst_addr
,
1090 const struct sockaddr_in
*src_addr
)
1092 __be32 daddr
= dst_addr
->sin_addr
.s_addr
,
1093 saddr
= src_addr
->sin_addr
.s_addr
;
1094 struct fib_result res
;
1099 if (ipv4_is_multicast(saddr
) || ipv4_is_lbcast(saddr
) ||
1100 ipv4_is_lbcast(daddr
) || ipv4_is_zeronet(saddr
) ||
1101 ipv4_is_zeronet(daddr
) || ipv4_is_loopback(daddr
) ||
1102 ipv4_is_loopback(saddr
))
1105 memset(&fl4
, 0, sizeof(fl4
));
1106 fl4
.flowi4_iif
= net_dev
->ifindex
;
1111 err
= fib_lookup(dev_net(net_dev
), &fl4
, &res
, 0);
1115 ret
= FIB_RES_DEV(res
) == net_dev
;
1121 static bool validate_ipv6_net_dev(struct net_device
*net_dev
,
1122 const struct sockaddr_in6
*dst_addr
,
1123 const struct sockaddr_in6
*src_addr
)
1125 #if IS_ENABLED(CONFIG_IPV6)
1126 const int strict
= ipv6_addr_type(&dst_addr
->sin6_addr
) &
1127 IPV6_ADDR_LINKLOCAL
;
1128 struct rt6_info
*rt
= rt6_lookup(dev_net(net_dev
), &dst_addr
->sin6_addr
,
1129 &src_addr
->sin6_addr
, net_dev
->ifindex
,
1136 ret
= rt
->rt6i_idev
->dev
== net_dev
;
1145 static bool validate_net_dev(struct net_device
*net_dev
,
1146 const struct sockaddr
*daddr
,
1147 const struct sockaddr
*saddr
)
1149 const struct sockaddr_in
*daddr4
= (const struct sockaddr_in
*)daddr
;
1150 const struct sockaddr_in
*saddr4
= (const struct sockaddr_in
*)saddr
;
1151 const struct sockaddr_in6
*daddr6
= (const struct sockaddr_in6
*)daddr
;
1152 const struct sockaddr_in6
*saddr6
= (const struct sockaddr_in6
*)saddr
;
1154 switch (daddr
->sa_family
) {
1156 return saddr
->sa_family
== AF_INET
&&
1157 validate_ipv4_net_dev(net_dev
, daddr4
, saddr4
);
1160 return saddr
->sa_family
== AF_INET6
&&
1161 validate_ipv6_net_dev(net_dev
, daddr6
, saddr6
);
1168 static struct net_device
*cma_get_net_dev(struct ib_cm_event
*ib_event
,
1169 const struct cma_req_info
*req
)
1171 struct sockaddr_storage listen_addr_storage
, src_addr_storage
;
1172 struct sockaddr
*listen_addr
= (struct sockaddr
*)&listen_addr_storage
,
1173 *src_addr
= (struct sockaddr
*)&src_addr_storage
;
1174 struct net_device
*net_dev
;
1175 const union ib_gid
*gid
= req
->has_gid
? &req
->local_gid
: NULL
;
1178 err
= cma_save_ip_info(listen_addr
, src_addr
, ib_event
,
1181 return ERR_PTR(err
);
1183 net_dev
= ib_get_net_dev_by_params(req
->device
, req
->port
, req
->pkey
,
1186 return ERR_PTR(-ENODEV
);
1188 if (!validate_net_dev(net_dev
, listen_addr
, src_addr
)) {
1190 return ERR_PTR(-EHOSTUNREACH
);
1196 static enum rdma_port_space
rdma_ps_from_service_id(__be64 service_id
)
1198 return (be64_to_cpu(service_id
) >> 16) & 0xffff;
1201 static bool cma_match_private_data(struct rdma_id_private
*id_priv
,
1202 const struct cma_hdr
*hdr
)
1204 struct sockaddr
*addr
= cma_src_addr(id_priv
);
1206 struct in6_addr ip6_addr
;
1208 if (cma_any_addr(addr
) && !id_priv
->afonly
)
1211 switch (addr
->sa_family
) {
1213 ip4_addr
= ((struct sockaddr_in
*)addr
)->sin_addr
.s_addr
;
1214 if (cma_get_ip_ver(hdr
) != 4)
1216 if (!cma_any_addr(addr
) &&
1217 hdr
->dst_addr
.ip4
.addr
!= ip4_addr
)
1221 ip6_addr
= ((struct sockaddr_in6
*)addr
)->sin6_addr
;
1222 if (cma_get_ip_ver(hdr
) != 6)
1224 if (!cma_any_addr(addr
) &&
1225 memcmp(&hdr
->dst_addr
.ip6
, &ip6_addr
, sizeof(ip6_addr
)))
1237 static bool cma_protocol_roce_dev_port(struct ib_device
*device
, int port_num
)
1239 enum rdma_link_layer ll
= rdma_port_get_link_layer(device
, port_num
);
1240 enum rdma_transport_type transport
=
1241 rdma_node_get_transport(device
->node_type
);
1243 return ll
== IB_LINK_LAYER_ETHERNET
&& transport
== RDMA_TRANSPORT_IB
;
1246 static bool cma_protocol_roce(const struct rdma_cm_id
*id
)
1248 struct ib_device
*device
= id
->device
;
1249 const int port_num
= id
->port_num
?: rdma_start_port(device
);
1251 return cma_protocol_roce_dev_port(device
, port_num
);
1254 static bool cma_match_net_dev(const struct rdma_id_private
*id_priv
,
1255 const struct net_device
*net_dev
)
1257 const struct rdma_addr
*addr
= &id_priv
->id
.route
.addr
;
1260 /* This request is an AF_IB request or a RoCE request */
1261 return addr
->src_addr
.ss_family
== AF_IB
||
1262 cma_protocol_roce(&id_priv
->id
);
1264 return !addr
->dev_addr
.bound_dev_if
||
1265 (net_eq(dev_net(net_dev
), &init_net
) &&
1266 addr
->dev_addr
.bound_dev_if
== net_dev
->ifindex
);
1269 static struct rdma_id_private
*cma_find_listener(
1270 const struct rdma_bind_list
*bind_list
,
1271 const struct ib_cm_id
*cm_id
,
1272 const struct ib_cm_event
*ib_event
,
1273 const struct cma_req_info
*req
,
1274 const struct net_device
*net_dev
)
1276 struct rdma_id_private
*id_priv
, *id_priv_dev
;
1279 return ERR_PTR(-EINVAL
);
1281 hlist_for_each_entry(id_priv
, &bind_list
->owners
, node
) {
1282 if (cma_match_private_data(id_priv
, ib_event
->private_data
)) {
1283 if (id_priv
->id
.device
== cm_id
->device
&&
1284 cma_match_net_dev(id_priv
, net_dev
))
1286 list_for_each_entry(id_priv_dev
,
1287 &id_priv
->listen_list
,
1289 if (id_priv_dev
->id
.device
== cm_id
->device
&&
1290 cma_match_net_dev(id_priv_dev
, net_dev
))
1296 return ERR_PTR(-EINVAL
);
1299 static struct rdma_id_private
*cma_id_from_event(struct ib_cm_id
*cm_id
,
1300 struct ib_cm_event
*ib_event
,
1301 struct net_device
**net_dev
)
1303 struct cma_req_info req
;
1304 struct rdma_bind_list
*bind_list
;
1305 struct rdma_id_private
*id_priv
;
1308 err
= cma_save_req_info(ib_event
, &req
);
1310 return ERR_PTR(err
);
1312 *net_dev
= cma_get_net_dev(ib_event
, &req
);
1313 if (IS_ERR(*net_dev
)) {
1314 if (PTR_ERR(*net_dev
) == -EAFNOSUPPORT
) {
1315 /* Assuming the protocol is AF_IB */
1317 } else if (cma_protocol_roce_dev_port(req
.device
, req
.port
)) {
1318 /* TODO find the net dev matching the request parameters
1319 * through the RoCE GID table */
1322 return ERR_CAST(*net_dev
);
1326 bind_list
= cma_ps_find(rdma_ps_from_service_id(req
.service_id
),
1327 cma_port_from_service_id(req
.service_id
));
1328 id_priv
= cma_find_listener(bind_list
, cm_id
, ib_event
, &req
, *net_dev
);
1329 if (IS_ERR(id_priv
) && *net_dev
) {
1337 static inline int cma_user_data_offset(struct rdma_id_private
*id_priv
)
1339 return cma_family(id_priv
) == AF_IB
? 0 : sizeof(struct cma_hdr
);
1342 static void cma_cancel_route(struct rdma_id_private
*id_priv
)
1344 if (rdma_cap_ib_sa(id_priv
->id
.device
, id_priv
->id
.port_num
)) {
1346 ib_sa_cancel_query(id_priv
->query_id
, id_priv
->query
);
1350 static void cma_cancel_listens(struct rdma_id_private
*id_priv
)
1352 struct rdma_id_private
*dev_id_priv
;
1355 * Remove from listen_any_list to prevent added devices from spawning
1356 * additional listen requests.
1359 list_del(&id_priv
->list
);
1361 while (!list_empty(&id_priv
->listen_list
)) {
1362 dev_id_priv
= list_entry(id_priv
->listen_list
.next
,
1363 struct rdma_id_private
, listen_list
);
1364 /* sync with device removal to avoid duplicate destruction */
1365 list_del_init(&dev_id_priv
->list
);
1366 list_del(&dev_id_priv
->listen_list
);
1367 mutex_unlock(&lock
);
1369 rdma_destroy_id(&dev_id_priv
->id
);
1372 mutex_unlock(&lock
);
1375 static void cma_cancel_operation(struct rdma_id_private
*id_priv
,
1376 enum rdma_cm_state state
)
1379 case RDMA_CM_ADDR_QUERY
:
1380 rdma_addr_cancel(&id_priv
->id
.route
.addr
.dev_addr
);
1382 case RDMA_CM_ROUTE_QUERY
:
1383 cma_cancel_route(id_priv
);
1385 case RDMA_CM_LISTEN
:
1386 if (cma_any_addr(cma_src_addr(id_priv
)) && !id_priv
->cma_dev
)
1387 cma_cancel_listens(id_priv
);
1394 static void cma_release_port(struct rdma_id_private
*id_priv
)
1396 struct rdma_bind_list
*bind_list
= id_priv
->bind_list
;
1402 hlist_del(&id_priv
->node
);
1403 if (hlist_empty(&bind_list
->owners
)) {
1404 cma_ps_remove(bind_list
->ps
, bind_list
->port
);
1407 mutex_unlock(&lock
);
1410 static void cma_leave_mc_groups(struct rdma_id_private
*id_priv
)
1412 struct cma_multicast
*mc
;
1414 while (!list_empty(&id_priv
->mc_list
)) {
1415 mc
= container_of(id_priv
->mc_list
.next
,
1416 struct cma_multicast
, list
);
1417 list_del(&mc
->list
);
1418 if (rdma_cap_ib_mcast(id_priv
->cma_dev
->device
,
1419 id_priv
->id
.port_num
)) {
1420 ib_sa_free_multicast(mc
->multicast
.ib
);
1423 kref_put(&mc
->mcref
, release_mc
);
1427 void rdma_destroy_id(struct rdma_cm_id
*id
)
1429 struct rdma_id_private
*id_priv
;
1430 enum rdma_cm_state state
;
1432 id_priv
= container_of(id
, struct rdma_id_private
, id
);
1433 state
= cma_exch(id_priv
, RDMA_CM_DESTROYING
);
1434 cma_cancel_operation(id_priv
, state
);
1437 * Wait for any active callback to finish. New callbacks will find
1438 * the id_priv state set to destroying and abort.
1440 mutex_lock(&id_priv
->handler_mutex
);
1441 mutex_unlock(&id_priv
->handler_mutex
);
1443 if (id_priv
->cma_dev
) {
1444 if (rdma_cap_ib_cm(id_priv
->id
.device
, 1)) {
1445 if (id_priv
->cm_id
.ib
)
1446 ib_destroy_cm_id(id_priv
->cm_id
.ib
);
1447 } else if (rdma_cap_iw_cm(id_priv
->id
.device
, 1)) {
1448 if (id_priv
->cm_id
.iw
)
1449 iw_destroy_cm_id(id_priv
->cm_id
.iw
);
1451 cma_leave_mc_groups(id_priv
);
1452 cma_release_dev(id_priv
);
1455 cma_release_port(id_priv
);
1456 cma_deref_id(id_priv
);
1457 wait_for_completion(&id_priv
->comp
);
1459 if (id_priv
->internal_id
)
1460 cma_deref_id(id_priv
->id
.context
);
1462 kfree(id_priv
->id
.route
.path_rec
);
1465 EXPORT_SYMBOL(rdma_destroy_id
);
1467 static int cma_rep_recv(struct rdma_id_private
*id_priv
)
1471 ret
= cma_modify_qp_rtr(id_priv
, NULL
);
1475 ret
= cma_modify_qp_rts(id_priv
, NULL
);
1479 ret
= ib_send_cm_rtu(id_priv
->cm_id
.ib
, NULL
, 0);
1485 cma_modify_qp_err(id_priv
);
1486 ib_send_cm_rej(id_priv
->cm_id
.ib
, IB_CM_REJ_CONSUMER_DEFINED
,
1491 static void cma_set_rep_event_data(struct rdma_cm_event
*event
,
1492 struct ib_cm_rep_event_param
*rep_data
,
1495 event
->param
.conn
.private_data
= private_data
;
1496 event
->param
.conn
.private_data_len
= IB_CM_REP_PRIVATE_DATA_SIZE
;
1497 event
->param
.conn
.responder_resources
= rep_data
->responder_resources
;
1498 event
->param
.conn
.initiator_depth
= rep_data
->initiator_depth
;
1499 event
->param
.conn
.flow_control
= rep_data
->flow_control
;
1500 event
->param
.conn
.rnr_retry_count
= rep_data
->rnr_retry_count
;
1501 event
->param
.conn
.srq
= rep_data
->srq
;
1502 event
->param
.conn
.qp_num
= rep_data
->remote_qpn
;
1505 static int cma_ib_handler(struct ib_cm_id
*cm_id
, struct ib_cm_event
*ib_event
)
1507 struct rdma_id_private
*id_priv
= cm_id
->context
;
1508 struct rdma_cm_event event
;
1511 if ((ib_event
->event
!= IB_CM_TIMEWAIT_EXIT
&&
1512 cma_disable_callback(id_priv
, RDMA_CM_CONNECT
)) ||
1513 (ib_event
->event
== IB_CM_TIMEWAIT_EXIT
&&
1514 cma_disable_callback(id_priv
, RDMA_CM_DISCONNECT
)))
1517 memset(&event
, 0, sizeof event
);
1518 switch (ib_event
->event
) {
1519 case IB_CM_REQ_ERROR
:
1520 case IB_CM_REP_ERROR
:
1521 event
.event
= RDMA_CM_EVENT_UNREACHABLE
;
1522 event
.status
= -ETIMEDOUT
;
1524 case IB_CM_REP_RECEIVED
:
1525 if (id_priv
->id
.qp
) {
1526 event
.status
= cma_rep_recv(id_priv
);
1527 event
.event
= event
.status
? RDMA_CM_EVENT_CONNECT_ERROR
:
1528 RDMA_CM_EVENT_ESTABLISHED
;
1530 event
.event
= RDMA_CM_EVENT_CONNECT_RESPONSE
;
1532 cma_set_rep_event_data(&event
, &ib_event
->param
.rep_rcvd
,
1533 ib_event
->private_data
);
1535 case IB_CM_RTU_RECEIVED
:
1536 case IB_CM_USER_ESTABLISHED
:
1537 event
.event
= RDMA_CM_EVENT_ESTABLISHED
;
1539 case IB_CM_DREQ_ERROR
:
1540 event
.status
= -ETIMEDOUT
; /* fall through */
1541 case IB_CM_DREQ_RECEIVED
:
1542 case IB_CM_DREP_RECEIVED
:
1543 if (!cma_comp_exch(id_priv
, RDMA_CM_CONNECT
,
1544 RDMA_CM_DISCONNECT
))
1546 event
.event
= RDMA_CM_EVENT_DISCONNECTED
;
1548 case IB_CM_TIMEWAIT_EXIT
:
1549 event
.event
= RDMA_CM_EVENT_TIMEWAIT_EXIT
;
1551 case IB_CM_MRA_RECEIVED
:
1554 case IB_CM_REJ_RECEIVED
:
1555 cma_modify_qp_err(id_priv
);
1556 event
.status
= ib_event
->param
.rej_rcvd
.reason
;
1557 event
.event
= RDMA_CM_EVENT_REJECTED
;
1558 event
.param
.conn
.private_data
= ib_event
->private_data
;
1559 event
.param
.conn
.private_data_len
= IB_CM_REJ_PRIVATE_DATA_SIZE
;
1562 printk(KERN_ERR
"RDMA CMA: unexpected IB CM event: %d\n",
1567 ret
= id_priv
->id
.event_handler(&id_priv
->id
, &event
);
1569 /* Destroy the CM ID by returning a non-zero value. */
1570 id_priv
->cm_id
.ib
= NULL
;
1571 cma_exch(id_priv
, RDMA_CM_DESTROYING
);
1572 mutex_unlock(&id_priv
->handler_mutex
);
1573 rdma_destroy_id(&id_priv
->id
);
1577 mutex_unlock(&id_priv
->handler_mutex
);
1581 static struct rdma_id_private
*cma_new_conn_id(struct rdma_cm_id
*listen_id
,
1582 struct ib_cm_event
*ib_event
,
1583 struct net_device
*net_dev
)
1585 struct rdma_id_private
*id_priv
;
1586 struct rdma_cm_id
*id
;
1587 struct rdma_route
*rt
;
1588 const sa_family_t ss_family
= listen_id
->route
.addr
.src_addr
.ss_family
;
1589 const __be64 service_id
=
1590 ib_event
->param
.req_rcvd
.primary_path
->service_id
;
1593 id
= rdma_create_id(listen_id
->event_handler
, listen_id
->context
,
1594 listen_id
->ps
, ib_event
->param
.req_rcvd
.qp_type
);
1598 id_priv
= container_of(id
, struct rdma_id_private
, id
);
1599 if (cma_save_net_info((struct sockaddr
*)&id
->route
.addr
.src_addr
,
1600 (struct sockaddr
*)&id
->route
.addr
.dst_addr
,
1601 listen_id
, ib_event
, ss_family
, service_id
))
1605 rt
->num_paths
= ib_event
->param
.req_rcvd
.alternate_path
? 2 : 1;
1606 rt
->path_rec
= kmalloc(sizeof *rt
->path_rec
* rt
->num_paths
,
1611 rt
->path_rec
[0] = *ib_event
->param
.req_rcvd
.primary_path
;
1612 if (rt
->num_paths
== 2)
1613 rt
->path_rec
[1] = *ib_event
->param
.req_rcvd
.alternate_path
;
1616 ret
= rdma_copy_addr(&rt
->addr
.dev_addr
, net_dev
, NULL
);
1620 if (!cma_protocol_roce(listen_id
) &&
1621 cma_any_addr(cma_src_addr(id_priv
))) {
1622 rt
->addr
.dev_addr
.dev_type
= ARPHRD_INFINIBAND
;
1623 rdma_addr_set_sgid(&rt
->addr
.dev_addr
, &rt
->path_rec
[0].sgid
);
1624 ib_addr_set_pkey(&rt
->addr
.dev_addr
, be16_to_cpu(rt
->path_rec
[0].pkey
));
1625 } else if (!cma_any_addr(cma_src_addr(id_priv
))) {
1626 ret
= cma_translate_addr(cma_src_addr(id_priv
), &rt
->addr
.dev_addr
);
1631 rdma_addr_set_dgid(&rt
->addr
.dev_addr
, &rt
->path_rec
[0].dgid
);
1633 id_priv
->state
= RDMA_CM_CONNECT
;
1637 rdma_destroy_id(id
);
1641 static struct rdma_id_private
*cma_new_udp_id(struct rdma_cm_id
*listen_id
,
1642 struct ib_cm_event
*ib_event
,
1643 struct net_device
*net_dev
)
1645 struct rdma_id_private
*id_priv
;
1646 struct rdma_cm_id
*id
;
1647 const sa_family_t ss_family
= listen_id
->route
.addr
.src_addr
.ss_family
;
1650 id
= rdma_create_id(listen_id
->event_handler
, listen_id
->context
,
1651 listen_id
->ps
, IB_QPT_UD
);
1655 id_priv
= container_of(id
, struct rdma_id_private
, id
);
1656 if (cma_save_net_info((struct sockaddr
*)&id
->route
.addr
.src_addr
,
1657 (struct sockaddr
*)&id
->route
.addr
.dst_addr
,
1658 listen_id
, ib_event
, ss_family
,
1659 ib_event
->param
.sidr_req_rcvd
.service_id
))
1663 ret
= rdma_copy_addr(&id
->route
.addr
.dev_addr
, net_dev
, NULL
);
1667 if (!cma_any_addr(cma_src_addr(id_priv
))) {
1668 ret
= cma_translate_addr(cma_src_addr(id_priv
),
1669 &id
->route
.addr
.dev_addr
);
1675 id_priv
->state
= RDMA_CM_CONNECT
;
1678 rdma_destroy_id(id
);
1682 static void cma_set_req_event_data(struct rdma_cm_event
*event
,
1683 struct ib_cm_req_event_param
*req_data
,
1684 void *private_data
, int offset
)
1686 event
->param
.conn
.private_data
= private_data
+ offset
;
1687 event
->param
.conn
.private_data_len
= IB_CM_REQ_PRIVATE_DATA_SIZE
- offset
;
1688 event
->param
.conn
.responder_resources
= req_data
->responder_resources
;
1689 event
->param
.conn
.initiator_depth
= req_data
->initiator_depth
;
1690 event
->param
.conn
.flow_control
= req_data
->flow_control
;
1691 event
->param
.conn
.retry_count
= req_data
->retry_count
;
1692 event
->param
.conn
.rnr_retry_count
= req_data
->rnr_retry_count
;
1693 event
->param
.conn
.srq
= req_data
->srq
;
1694 event
->param
.conn
.qp_num
= req_data
->remote_qpn
;
1697 static int cma_check_req_qp_type(struct rdma_cm_id
*id
, struct ib_cm_event
*ib_event
)
1699 return (((ib_event
->event
== IB_CM_REQ_RECEIVED
) &&
1700 (ib_event
->param
.req_rcvd
.qp_type
== id
->qp_type
)) ||
1701 ((ib_event
->event
== IB_CM_SIDR_REQ_RECEIVED
) &&
1702 (id
->qp_type
== IB_QPT_UD
)) ||
1706 static int cma_req_handler(struct ib_cm_id
*cm_id
, struct ib_cm_event
*ib_event
)
1708 struct rdma_id_private
*listen_id
, *conn_id
;
1709 struct rdma_cm_event event
;
1710 struct net_device
*net_dev
;
1713 listen_id
= cma_id_from_event(cm_id
, ib_event
, &net_dev
);
1714 if (IS_ERR(listen_id
))
1715 return PTR_ERR(listen_id
);
1717 if (!cma_check_req_qp_type(&listen_id
->id
, ib_event
)) {
1722 if (cma_disable_callback(listen_id
, RDMA_CM_LISTEN
)) {
1723 ret
= -ECONNABORTED
;
1727 memset(&event
, 0, sizeof event
);
1728 offset
= cma_user_data_offset(listen_id
);
1729 event
.event
= RDMA_CM_EVENT_CONNECT_REQUEST
;
1730 if (ib_event
->event
== IB_CM_SIDR_REQ_RECEIVED
) {
1731 conn_id
= cma_new_udp_id(&listen_id
->id
, ib_event
, net_dev
);
1732 event
.param
.ud
.private_data
= ib_event
->private_data
+ offset
;
1733 event
.param
.ud
.private_data_len
=
1734 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE
- offset
;
1736 conn_id
= cma_new_conn_id(&listen_id
->id
, ib_event
, net_dev
);
1737 cma_set_req_event_data(&event
, &ib_event
->param
.req_rcvd
,
1738 ib_event
->private_data
, offset
);
1745 mutex_lock_nested(&conn_id
->handler_mutex
, SINGLE_DEPTH_NESTING
);
1746 ret
= cma_acquire_dev(conn_id
, listen_id
);
1750 conn_id
->cm_id
.ib
= cm_id
;
1751 cm_id
->context
= conn_id
;
1752 cm_id
->cm_handler
= cma_ib_handler
;
1755 * Protect against the user destroying conn_id from another thread
1756 * until we're done accessing it.
1758 atomic_inc(&conn_id
->refcount
);
1759 ret
= conn_id
->id
.event_handler(&conn_id
->id
, &event
);
1763 * Acquire mutex to prevent user executing rdma_destroy_id()
1764 * while we're accessing the cm_id.
1767 if (cma_comp(conn_id
, RDMA_CM_CONNECT
) &&
1768 (conn_id
->id
.qp_type
!= IB_QPT_UD
))
1769 ib_send_cm_mra(cm_id
, CMA_CM_MRA_SETTING
, NULL
, 0);
1770 mutex_unlock(&lock
);
1771 mutex_unlock(&conn_id
->handler_mutex
);
1772 mutex_unlock(&listen_id
->handler_mutex
);
1773 cma_deref_id(conn_id
);
1779 cma_deref_id(conn_id
);
1780 /* Destroy the CM ID by returning a non-zero value. */
1781 conn_id
->cm_id
.ib
= NULL
;
1783 cma_exch(conn_id
, RDMA_CM_DESTROYING
);
1784 mutex_unlock(&conn_id
->handler_mutex
);
1786 mutex_unlock(&listen_id
->handler_mutex
);
1788 rdma_destroy_id(&conn_id
->id
);
1797 __be64
rdma_get_service_id(struct rdma_cm_id
*id
, struct sockaddr
*addr
)
1799 if (addr
->sa_family
== AF_IB
)
1800 return ((struct sockaddr_ib
*) addr
)->sib_sid
;
1802 return cpu_to_be64(((u64
)id
->ps
<< 16) + be16_to_cpu(cma_port(addr
)));
1804 EXPORT_SYMBOL(rdma_get_service_id
);
1806 static int cma_iw_handler(struct iw_cm_id
*iw_id
, struct iw_cm_event
*iw_event
)
1808 struct rdma_id_private
*id_priv
= iw_id
->context
;
1809 struct rdma_cm_event event
;
1811 struct sockaddr
*laddr
= (struct sockaddr
*)&iw_event
->local_addr
;
1812 struct sockaddr
*raddr
= (struct sockaddr
*)&iw_event
->remote_addr
;
1814 if (cma_disable_callback(id_priv
, RDMA_CM_CONNECT
))
1817 memset(&event
, 0, sizeof event
);
1818 switch (iw_event
->event
) {
1819 case IW_CM_EVENT_CLOSE
:
1820 event
.event
= RDMA_CM_EVENT_DISCONNECTED
;
1822 case IW_CM_EVENT_CONNECT_REPLY
:
1823 memcpy(cma_src_addr(id_priv
), laddr
,
1824 rdma_addr_size(laddr
));
1825 memcpy(cma_dst_addr(id_priv
), raddr
,
1826 rdma_addr_size(raddr
));
1827 switch (iw_event
->status
) {
1829 event
.event
= RDMA_CM_EVENT_ESTABLISHED
;
1830 event
.param
.conn
.initiator_depth
= iw_event
->ird
;
1831 event
.param
.conn
.responder_resources
= iw_event
->ord
;
1835 event
.event
= RDMA_CM_EVENT_REJECTED
;
1838 event
.event
= RDMA_CM_EVENT_UNREACHABLE
;
1841 event
.event
= RDMA_CM_EVENT_CONNECT_ERROR
;
1845 case IW_CM_EVENT_ESTABLISHED
:
1846 event
.event
= RDMA_CM_EVENT_ESTABLISHED
;
1847 event
.param
.conn
.initiator_depth
= iw_event
->ird
;
1848 event
.param
.conn
.responder_resources
= iw_event
->ord
;
1854 event
.status
= iw_event
->status
;
1855 event
.param
.conn
.private_data
= iw_event
->private_data
;
1856 event
.param
.conn
.private_data_len
= iw_event
->private_data_len
;
1857 ret
= id_priv
->id
.event_handler(&id_priv
->id
, &event
);
1859 /* Destroy the CM ID by returning a non-zero value. */
1860 id_priv
->cm_id
.iw
= NULL
;
1861 cma_exch(id_priv
, RDMA_CM_DESTROYING
);
1862 mutex_unlock(&id_priv
->handler_mutex
);
1863 rdma_destroy_id(&id_priv
->id
);
1867 mutex_unlock(&id_priv
->handler_mutex
);
1871 static int iw_conn_req_handler(struct iw_cm_id
*cm_id
,
1872 struct iw_cm_event
*iw_event
)
1874 struct rdma_cm_id
*new_cm_id
;
1875 struct rdma_id_private
*listen_id
, *conn_id
;
1876 struct rdma_cm_event event
;
1878 struct ib_device_attr attr
;
1879 struct sockaddr
*laddr
= (struct sockaddr
*)&iw_event
->local_addr
;
1880 struct sockaddr
*raddr
= (struct sockaddr
*)&iw_event
->remote_addr
;
1882 listen_id
= cm_id
->context
;
1883 if (cma_disable_callback(listen_id
, RDMA_CM_LISTEN
))
1884 return -ECONNABORTED
;
1886 /* Create a new RDMA id for the new IW CM ID */
1887 new_cm_id
= rdma_create_id(listen_id
->id
.event_handler
,
1888 listen_id
->id
.context
,
1889 RDMA_PS_TCP
, IB_QPT_RC
);
1890 if (IS_ERR(new_cm_id
)) {
1894 conn_id
= container_of(new_cm_id
, struct rdma_id_private
, id
);
1895 mutex_lock_nested(&conn_id
->handler_mutex
, SINGLE_DEPTH_NESTING
);
1896 conn_id
->state
= RDMA_CM_CONNECT
;
1898 ret
= rdma_translate_ip(laddr
, &conn_id
->id
.route
.addr
.dev_addr
, NULL
);
1900 mutex_unlock(&conn_id
->handler_mutex
);
1901 rdma_destroy_id(new_cm_id
);
1905 ret
= cma_acquire_dev(conn_id
, listen_id
);
1907 mutex_unlock(&conn_id
->handler_mutex
);
1908 rdma_destroy_id(new_cm_id
);
1912 conn_id
->cm_id
.iw
= cm_id
;
1913 cm_id
->context
= conn_id
;
1914 cm_id
->cm_handler
= cma_iw_handler
;
1916 memcpy(cma_src_addr(conn_id
), laddr
, rdma_addr_size(laddr
));
1917 memcpy(cma_dst_addr(conn_id
), raddr
, rdma_addr_size(raddr
));
1919 ret
= ib_query_device(conn_id
->id
.device
, &attr
);
1921 mutex_unlock(&conn_id
->handler_mutex
);
1922 rdma_destroy_id(new_cm_id
);
1926 memset(&event
, 0, sizeof event
);
1927 event
.event
= RDMA_CM_EVENT_CONNECT_REQUEST
;
1928 event
.param
.conn
.private_data
= iw_event
->private_data
;
1929 event
.param
.conn
.private_data_len
= iw_event
->private_data_len
;
1930 event
.param
.conn
.initiator_depth
= iw_event
->ird
;
1931 event
.param
.conn
.responder_resources
= iw_event
->ord
;
1934 * Protect against the user destroying conn_id from another thread
1935 * until we're done accessing it.
1937 atomic_inc(&conn_id
->refcount
);
1938 ret
= conn_id
->id
.event_handler(&conn_id
->id
, &event
);
1940 /* User wants to destroy the CM ID */
1941 conn_id
->cm_id
.iw
= NULL
;
1942 cma_exch(conn_id
, RDMA_CM_DESTROYING
);
1943 mutex_unlock(&conn_id
->handler_mutex
);
1944 cma_deref_id(conn_id
);
1945 rdma_destroy_id(&conn_id
->id
);
1949 mutex_unlock(&conn_id
->handler_mutex
);
1950 cma_deref_id(conn_id
);
1953 mutex_unlock(&listen_id
->handler_mutex
);
1957 static int cma_ib_listen(struct rdma_id_private
*id_priv
)
1959 struct sockaddr
*addr
;
1960 struct ib_cm_id
*id
;
1963 addr
= cma_src_addr(id_priv
);
1964 svc_id
= rdma_get_service_id(&id_priv
->id
, addr
);
1965 id
= ib_cm_insert_listen(id_priv
->id
.device
, cma_req_handler
, svc_id
);
1968 id_priv
->cm_id
.ib
= id
;
1973 static int cma_iw_listen(struct rdma_id_private
*id_priv
, int backlog
)
1976 struct iw_cm_id
*id
;
1978 id
= iw_create_cm_id(id_priv
->id
.device
,
1979 iw_conn_req_handler
,
1984 id
->tos
= id_priv
->tos
;
1985 id_priv
->cm_id
.iw
= id
;
1987 memcpy(&id_priv
->cm_id
.iw
->local_addr
, cma_src_addr(id_priv
),
1988 rdma_addr_size(cma_src_addr(id_priv
)));
1990 ret
= iw_cm_listen(id_priv
->cm_id
.iw
, backlog
);
1993 iw_destroy_cm_id(id_priv
->cm_id
.iw
);
1994 id_priv
->cm_id
.iw
= NULL
;
2000 static int cma_listen_handler(struct rdma_cm_id
*id
,
2001 struct rdma_cm_event
*event
)
2003 struct rdma_id_private
*id_priv
= id
->context
;
2005 id
->context
= id_priv
->id
.context
;
2006 id
->event_handler
= id_priv
->id
.event_handler
;
2007 return id_priv
->id
.event_handler(id
, event
);
2010 static void cma_listen_on_dev(struct rdma_id_private
*id_priv
,
2011 struct cma_device
*cma_dev
)
2013 struct rdma_id_private
*dev_id_priv
;
2014 struct rdma_cm_id
*id
;
2017 if (cma_family(id_priv
) == AF_IB
&& !rdma_cap_ib_cm(cma_dev
->device
, 1))
2020 id
= rdma_create_id(cma_listen_handler
, id_priv
, id_priv
->id
.ps
,
2021 id_priv
->id
.qp_type
);
2025 dev_id_priv
= container_of(id
, struct rdma_id_private
, id
);
2027 dev_id_priv
->state
= RDMA_CM_ADDR_BOUND
;
2028 memcpy(cma_src_addr(dev_id_priv
), cma_src_addr(id_priv
),
2029 rdma_addr_size(cma_src_addr(id_priv
)));
2031 cma_attach_to_dev(dev_id_priv
, cma_dev
);
2032 list_add_tail(&dev_id_priv
->listen_list
, &id_priv
->listen_list
);
2033 atomic_inc(&id_priv
->refcount
);
2034 dev_id_priv
->internal_id
= 1;
2035 dev_id_priv
->afonly
= id_priv
->afonly
;
2037 ret
= rdma_listen(id
, id_priv
->backlog
);
2039 printk(KERN_WARNING
"RDMA CMA: cma_listen_on_dev, error %d, "
2040 "listening on device %s\n", ret
, cma_dev
->device
->name
);
2043 static void cma_listen_on_all(struct rdma_id_private
*id_priv
)
2045 struct cma_device
*cma_dev
;
2048 list_add_tail(&id_priv
->list
, &listen_any_list
);
2049 list_for_each_entry(cma_dev
, &dev_list
, list
)
2050 cma_listen_on_dev(id_priv
, cma_dev
);
2051 mutex_unlock(&lock
);
2054 void rdma_set_service_type(struct rdma_cm_id
*id
, int tos
)
2056 struct rdma_id_private
*id_priv
;
2058 id_priv
= container_of(id
, struct rdma_id_private
, id
);
2059 id_priv
->tos
= (u8
) tos
;
2061 EXPORT_SYMBOL(rdma_set_service_type
);
2063 static void cma_query_handler(int status
, struct ib_sa_path_rec
*path_rec
,
2066 struct cma_work
*work
= context
;
2067 struct rdma_route
*route
;
2069 route
= &work
->id
->id
.route
;
2072 route
->num_paths
= 1;
2073 *route
->path_rec
= *path_rec
;
2075 work
->old_state
= RDMA_CM_ROUTE_QUERY
;
2076 work
->new_state
= RDMA_CM_ADDR_RESOLVED
;
2077 work
->event
.event
= RDMA_CM_EVENT_ROUTE_ERROR
;
2078 work
->event
.status
= status
;
2081 queue_work(cma_wq
, &work
->work
);
2084 static int cma_query_ib_route(struct rdma_id_private
*id_priv
, int timeout_ms
,
2085 struct cma_work
*work
)
2087 struct rdma_dev_addr
*dev_addr
= &id_priv
->id
.route
.addr
.dev_addr
;
2088 struct ib_sa_path_rec path_rec
;
2089 ib_sa_comp_mask comp_mask
;
2090 struct sockaddr_in6
*sin6
;
2091 struct sockaddr_ib
*sib
;
2093 memset(&path_rec
, 0, sizeof path_rec
);
2094 rdma_addr_get_sgid(dev_addr
, &path_rec
.sgid
);
2095 rdma_addr_get_dgid(dev_addr
, &path_rec
.dgid
);
2096 path_rec
.pkey
= cpu_to_be16(ib_addr_get_pkey(dev_addr
));
2097 path_rec
.numb_path
= 1;
2098 path_rec
.reversible
= 1;
2099 path_rec
.service_id
= rdma_get_service_id(&id_priv
->id
, cma_dst_addr(id_priv
));
2101 comp_mask
= IB_SA_PATH_REC_DGID
| IB_SA_PATH_REC_SGID
|
2102 IB_SA_PATH_REC_PKEY
| IB_SA_PATH_REC_NUMB_PATH
|
2103 IB_SA_PATH_REC_REVERSIBLE
| IB_SA_PATH_REC_SERVICE_ID
;
2105 switch (cma_family(id_priv
)) {
2107 path_rec
.qos_class
= cpu_to_be16((u16
) id_priv
->tos
);
2108 comp_mask
|= IB_SA_PATH_REC_QOS_CLASS
;
2111 sin6
= (struct sockaddr_in6
*) cma_src_addr(id_priv
);
2112 path_rec
.traffic_class
= (u8
) (be32_to_cpu(sin6
->sin6_flowinfo
) >> 20);
2113 comp_mask
|= IB_SA_PATH_REC_TRAFFIC_CLASS
;
2116 sib
= (struct sockaddr_ib
*) cma_src_addr(id_priv
);
2117 path_rec
.traffic_class
= (u8
) (be32_to_cpu(sib
->sib_flowinfo
) >> 20);
2118 comp_mask
|= IB_SA_PATH_REC_TRAFFIC_CLASS
;
2122 id_priv
->query_id
= ib_sa_path_rec_get(&sa_client
, id_priv
->id
.device
,
2123 id_priv
->id
.port_num
, &path_rec
,
2124 comp_mask
, timeout_ms
,
2125 GFP_KERNEL
, cma_query_handler
,
2126 work
, &id_priv
->query
);
2128 return (id_priv
->query_id
< 0) ? id_priv
->query_id
: 0;
2131 static void cma_work_handler(struct work_struct
*_work
)
2133 struct cma_work
*work
= container_of(_work
, struct cma_work
, work
);
2134 struct rdma_id_private
*id_priv
= work
->id
;
2137 mutex_lock(&id_priv
->handler_mutex
);
2138 if (!cma_comp_exch(id_priv
, work
->old_state
, work
->new_state
))
2141 if (id_priv
->id
.event_handler(&id_priv
->id
, &work
->event
)) {
2142 cma_exch(id_priv
, RDMA_CM_DESTROYING
);
2146 mutex_unlock(&id_priv
->handler_mutex
);
2147 cma_deref_id(id_priv
);
2149 rdma_destroy_id(&id_priv
->id
);
2153 static void cma_ndev_work_handler(struct work_struct
*_work
)
2155 struct cma_ndev_work
*work
= container_of(_work
, struct cma_ndev_work
, work
);
2156 struct rdma_id_private
*id_priv
= work
->id
;
2159 mutex_lock(&id_priv
->handler_mutex
);
2160 if (id_priv
->state
== RDMA_CM_DESTROYING
||
2161 id_priv
->state
== RDMA_CM_DEVICE_REMOVAL
)
2164 if (id_priv
->id
.event_handler(&id_priv
->id
, &work
->event
)) {
2165 cma_exch(id_priv
, RDMA_CM_DESTROYING
);
2170 mutex_unlock(&id_priv
->handler_mutex
);
2171 cma_deref_id(id_priv
);
2173 rdma_destroy_id(&id_priv
->id
);
2177 static int cma_resolve_ib_route(struct rdma_id_private
*id_priv
, int timeout_ms
)
2179 struct rdma_route
*route
= &id_priv
->id
.route
;
2180 struct cma_work
*work
;
2183 work
= kzalloc(sizeof *work
, GFP_KERNEL
);
2188 INIT_WORK(&work
->work
, cma_work_handler
);
2189 work
->old_state
= RDMA_CM_ROUTE_QUERY
;
2190 work
->new_state
= RDMA_CM_ROUTE_RESOLVED
;
2191 work
->event
.event
= RDMA_CM_EVENT_ROUTE_RESOLVED
;
2193 route
->path_rec
= kmalloc(sizeof *route
->path_rec
, GFP_KERNEL
);
2194 if (!route
->path_rec
) {
2199 ret
= cma_query_ib_route(id_priv
, timeout_ms
, work
);
2205 kfree(route
->path_rec
);
2206 route
->path_rec
= NULL
;
2212 int rdma_set_ib_paths(struct rdma_cm_id
*id
,
2213 struct ib_sa_path_rec
*path_rec
, int num_paths
)
2215 struct rdma_id_private
*id_priv
;
2218 id_priv
= container_of(id
, struct rdma_id_private
, id
);
2219 if (!cma_comp_exch(id_priv
, RDMA_CM_ADDR_RESOLVED
,
2220 RDMA_CM_ROUTE_RESOLVED
))
2223 id
->route
.path_rec
= kmemdup(path_rec
, sizeof *path_rec
* num_paths
,
2225 if (!id
->route
.path_rec
) {
2230 id
->route
.num_paths
= num_paths
;
2233 cma_comp_exch(id_priv
, RDMA_CM_ROUTE_RESOLVED
, RDMA_CM_ADDR_RESOLVED
);
2236 EXPORT_SYMBOL(rdma_set_ib_paths
);
2238 static int cma_resolve_iw_route(struct rdma_id_private
*id_priv
, int timeout_ms
)
2240 struct cma_work
*work
;
2242 work
= kzalloc(sizeof *work
, GFP_KERNEL
);
2247 INIT_WORK(&work
->work
, cma_work_handler
);
2248 work
->old_state
= RDMA_CM_ROUTE_QUERY
;
2249 work
->new_state
= RDMA_CM_ROUTE_RESOLVED
;
2250 work
->event
.event
= RDMA_CM_EVENT_ROUTE_RESOLVED
;
2251 queue_work(cma_wq
, &work
->work
);
2255 static int iboe_tos_to_sl(struct net_device
*ndev
, int tos
)
2258 struct net_device
*dev
;
2260 prio
= rt_tos2priority(tos
);
2261 dev
= ndev
->priv_flags
& IFF_802_1Q_VLAN
?
2262 vlan_dev_real_dev(ndev
) : ndev
;
2265 return netdev_get_prio_tc_map(dev
, prio
);
2267 #if IS_ENABLED(CONFIG_VLAN_8021Q)
2268 if (ndev
->priv_flags
& IFF_802_1Q_VLAN
)
2269 return (vlan_dev_get_egress_qos_mask(ndev
, prio
) &
2270 VLAN_PRIO_MASK
) >> VLAN_PRIO_SHIFT
;
2275 static int cma_resolve_iboe_route(struct rdma_id_private
*id_priv
)
2277 struct rdma_route
*route
= &id_priv
->id
.route
;
2278 struct rdma_addr
*addr
= &route
->addr
;
2279 struct cma_work
*work
;
2281 struct net_device
*ndev
= NULL
;
2284 work
= kzalloc(sizeof *work
, GFP_KERNEL
);
2289 INIT_WORK(&work
->work
, cma_work_handler
);
2291 route
->path_rec
= kzalloc(sizeof *route
->path_rec
, GFP_KERNEL
);
2292 if (!route
->path_rec
) {
2297 route
->num_paths
= 1;
2299 if (addr
->dev_addr
.bound_dev_if
)
2300 ndev
= dev_get_by_index(&init_net
, addr
->dev_addr
.bound_dev_if
);
2306 route
->path_rec
->vlan_id
= rdma_vlan_dev_vlan_id(ndev
);
2307 memcpy(route
->path_rec
->dmac
, addr
->dev_addr
.dst_dev_addr
, ETH_ALEN
);
2308 memcpy(route
->path_rec
->smac
, ndev
->dev_addr
, ndev
->addr_len
);
2310 rdma_ip2gid((struct sockaddr
*)&id_priv
->id
.route
.addr
.src_addr
,
2311 &route
->path_rec
->sgid
);
2312 rdma_ip2gid((struct sockaddr
*)&id_priv
->id
.route
.addr
.dst_addr
,
2313 &route
->path_rec
->dgid
);
2315 route
->path_rec
->hop_limit
= 1;
2316 route
->path_rec
->reversible
= 1;
2317 route
->path_rec
->pkey
= cpu_to_be16(0xffff);
2318 route
->path_rec
->mtu_selector
= IB_SA_EQ
;
2319 route
->path_rec
->sl
= iboe_tos_to_sl(ndev
, id_priv
->tos
);
2320 route
->path_rec
->mtu
= iboe_get_mtu(ndev
->mtu
);
2321 route
->path_rec
->rate_selector
= IB_SA_EQ
;
2322 route
->path_rec
->rate
= iboe_get_rate(ndev
);
2324 route
->path_rec
->packet_life_time_selector
= IB_SA_EQ
;
2325 route
->path_rec
->packet_life_time
= CMA_IBOE_PACKET_LIFETIME
;
2326 if (!route
->path_rec
->mtu
) {
2331 work
->old_state
= RDMA_CM_ROUTE_QUERY
;
2332 work
->new_state
= RDMA_CM_ROUTE_RESOLVED
;
2333 work
->event
.event
= RDMA_CM_EVENT_ROUTE_RESOLVED
;
2334 work
->event
.status
= 0;
2336 queue_work(cma_wq
, &work
->work
);
2341 kfree(route
->path_rec
);
2342 route
->path_rec
= NULL
;
2348 int rdma_resolve_route(struct rdma_cm_id
*id
, int timeout_ms
)
2350 struct rdma_id_private
*id_priv
;
2353 id_priv
= container_of(id
, struct rdma_id_private
, id
);
2354 if (!cma_comp_exch(id_priv
, RDMA_CM_ADDR_RESOLVED
, RDMA_CM_ROUTE_QUERY
))
2357 atomic_inc(&id_priv
->refcount
);
2358 if (rdma_cap_ib_sa(id
->device
, id
->port_num
))
2359 ret
= cma_resolve_ib_route(id_priv
, timeout_ms
);
2360 else if (rdma_protocol_roce(id
->device
, id
->port_num
))
2361 ret
= cma_resolve_iboe_route(id_priv
);
2362 else if (rdma_protocol_iwarp(id
->device
, id
->port_num
))
2363 ret
= cma_resolve_iw_route(id_priv
, timeout_ms
);
2372 cma_comp_exch(id_priv
, RDMA_CM_ROUTE_QUERY
, RDMA_CM_ADDR_RESOLVED
);
2373 cma_deref_id(id_priv
);
2376 EXPORT_SYMBOL(rdma_resolve_route
);
2378 static void cma_set_loopback(struct sockaddr
*addr
)
2380 switch (addr
->sa_family
) {
2382 ((struct sockaddr_in
*) addr
)->sin_addr
.s_addr
= htonl(INADDR_LOOPBACK
);
2385 ipv6_addr_set(&((struct sockaddr_in6
*) addr
)->sin6_addr
,
2389 ib_addr_set(&((struct sockaddr_ib
*) addr
)->sib_addr
,
2395 static int cma_bind_loopback(struct rdma_id_private
*id_priv
)
2397 struct cma_device
*cma_dev
, *cur_dev
;
2398 struct ib_port_attr port_attr
;
2406 list_for_each_entry(cur_dev
, &dev_list
, list
) {
2407 if (cma_family(id_priv
) == AF_IB
&&
2408 !rdma_cap_ib_cm(cur_dev
->device
, 1))
2414 for (p
= 1; p
<= cur_dev
->device
->phys_port_cnt
; ++p
) {
2415 if (!ib_query_port(cur_dev
->device
, p
, &port_attr
) &&
2416 port_attr
.state
== IB_PORT_ACTIVE
) {
2431 ret
= ib_get_cached_gid(cma_dev
->device
, p
, 0, &gid
, NULL
);
2435 ret
= ib_get_cached_pkey(cma_dev
->device
, p
, 0, &pkey
);
2439 id_priv
->id
.route
.addr
.dev_addr
.dev_type
=
2440 (rdma_protocol_ib(cma_dev
->device
, p
)) ?
2441 ARPHRD_INFINIBAND
: ARPHRD_ETHER
;
2443 rdma_addr_set_sgid(&id_priv
->id
.route
.addr
.dev_addr
, &gid
);
2444 ib_addr_set_pkey(&id_priv
->id
.route
.addr
.dev_addr
, pkey
);
2445 id_priv
->id
.port_num
= p
;
2446 cma_attach_to_dev(id_priv
, cma_dev
);
2447 cma_set_loopback(cma_src_addr(id_priv
));
2449 mutex_unlock(&lock
);
2453 static void addr_handler(int status
, struct sockaddr
*src_addr
,
2454 struct rdma_dev_addr
*dev_addr
, void *context
)
2456 struct rdma_id_private
*id_priv
= context
;
2457 struct rdma_cm_event event
;
2459 memset(&event
, 0, sizeof event
);
2460 mutex_lock(&id_priv
->handler_mutex
);
2461 if (!cma_comp_exch(id_priv
, RDMA_CM_ADDR_QUERY
,
2462 RDMA_CM_ADDR_RESOLVED
))
2465 memcpy(cma_src_addr(id_priv
), src_addr
, rdma_addr_size(src_addr
));
2466 if (!status
&& !id_priv
->cma_dev
)
2467 status
= cma_acquire_dev(id_priv
, NULL
);
2470 if (!cma_comp_exch(id_priv
, RDMA_CM_ADDR_RESOLVED
,
2471 RDMA_CM_ADDR_BOUND
))
2473 event
.event
= RDMA_CM_EVENT_ADDR_ERROR
;
2474 event
.status
= status
;
2476 event
.event
= RDMA_CM_EVENT_ADDR_RESOLVED
;
2478 if (id_priv
->id
.event_handler(&id_priv
->id
, &event
)) {
2479 cma_exch(id_priv
, RDMA_CM_DESTROYING
);
2480 mutex_unlock(&id_priv
->handler_mutex
);
2481 cma_deref_id(id_priv
);
2482 rdma_destroy_id(&id_priv
->id
);
2486 mutex_unlock(&id_priv
->handler_mutex
);
2487 cma_deref_id(id_priv
);
2490 static int cma_resolve_loopback(struct rdma_id_private
*id_priv
)
2492 struct cma_work
*work
;
2496 work
= kzalloc(sizeof *work
, GFP_KERNEL
);
2500 if (!id_priv
->cma_dev
) {
2501 ret
= cma_bind_loopback(id_priv
);
2506 rdma_addr_get_sgid(&id_priv
->id
.route
.addr
.dev_addr
, &gid
);
2507 rdma_addr_set_dgid(&id_priv
->id
.route
.addr
.dev_addr
, &gid
);
2510 INIT_WORK(&work
->work
, cma_work_handler
);
2511 work
->old_state
= RDMA_CM_ADDR_QUERY
;
2512 work
->new_state
= RDMA_CM_ADDR_RESOLVED
;
2513 work
->event
.event
= RDMA_CM_EVENT_ADDR_RESOLVED
;
2514 queue_work(cma_wq
, &work
->work
);
2521 static int cma_resolve_ib_addr(struct rdma_id_private
*id_priv
)
2523 struct cma_work
*work
;
2526 work
= kzalloc(sizeof *work
, GFP_KERNEL
);
2530 if (!id_priv
->cma_dev
) {
2531 ret
= cma_resolve_ib_dev(id_priv
);
2536 rdma_addr_set_dgid(&id_priv
->id
.route
.addr
.dev_addr
, (union ib_gid
*)
2537 &(((struct sockaddr_ib
*) &id_priv
->id
.route
.addr
.dst_addr
)->sib_addr
));
2540 INIT_WORK(&work
->work
, cma_work_handler
);
2541 work
->old_state
= RDMA_CM_ADDR_QUERY
;
2542 work
->new_state
= RDMA_CM_ADDR_RESOLVED
;
2543 work
->event
.event
= RDMA_CM_EVENT_ADDR_RESOLVED
;
2544 queue_work(cma_wq
, &work
->work
);
2551 static int cma_bind_addr(struct rdma_cm_id
*id
, struct sockaddr
*src_addr
,
2552 struct sockaddr
*dst_addr
)
2554 if (!src_addr
|| !src_addr
->sa_family
) {
2555 src_addr
= (struct sockaddr
*) &id
->route
.addr
.src_addr
;
2556 src_addr
->sa_family
= dst_addr
->sa_family
;
2557 if (dst_addr
->sa_family
== AF_INET6
) {
2558 struct sockaddr_in6
*src_addr6
= (struct sockaddr_in6
*) src_addr
;
2559 struct sockaddr_in6
*dst_addr6
= (struct sockaddr_in6
*) dst_addr
;
2560 src_addr6
->sin6_scope_id
= dst_addr6
->sin6_scope_id
;
2561 if (ipv6_addr_type(&dst_addr6
->sin6_addr
) & IPV6_ADDR_LINKLOCAL
)
2562 id
->route
.addr
.dev_addr
.bound_dev_if
= dst_addr6
->sin6_scope_id
;
2563 } else if (dst_addr
->sa_family
== AF_IB
) {
2564 ((struct sockaddr_ib
*) src_addr
)->sib_pkey
=
2565 ((struct sockaddr_ib
*) dst_addr
)->sib_pkey
;
2568 return rdma_bind_addr(id
, src_addr
);
2571 int rdma_resolve_addr(struct rdma_cm_id
*id
, struct sockaddr
*src_addr
,
2572 struct sockaddr
*dst_addr
, int timeout_ms
)
2574 struct rdma_id_private
*id_priv
;
2577 id_priv
= container_of(id
, struct rdma_id_private
, id
);
2578 if (id_priv
->state
== RDMA_CM_IDLE
) {
2579 ret
= cma_bind_addr(id
, src_addr
, dst_addr
);
2584 if (cma_family(id_priv
) != dst_addr
->sa_family
)
2587 if (!cma_comp_exch(id_priv
, RDMA_CM_ADDR_BOUND
, RDMA_CM_ADDR_QUERY
))
2590 atomic_inc(&id_priv
->refcount
);
2591 memcpy(cma_dst_addr(id_priv
), dst_addr
, rdma_addr_size(dst_addr
));
2592 if (cma_any_addr(dst_addr
)) {
2593 ret
= cma_resolve_loopback(id_priv
);
2595 if (dst_addr
->sa_family
== AF_IB
) {
2596 ret
= cma_resolve_ib_addr(id_priv
);
2598 ret
= rdma_resolve_ip(&addr_client
, cma_src_addr(id_priv
),
2599 dst_addr
, &id
->route
.addr
.dev_addr
,
2600 timeout_ms
, addr_handler
, id_priv
);
2608 cma_comp_exch(id_priv
, RDMA_CM_ADDR_QUERY
, RDMA_CM_ADDR_BOUND
);
2609 cma_deref_id(id_priv
);
2612 EXPORT_SYMBOL(rdma_resolve_addr
);
2614 int rdma_set_reuseaddr(struct rdma_cm_id
*id
, int reuse
)
2616 struct rdma_id_private
*id_priv
;
2617 unsigned long flags
;
2620 id_priv
= container_of(id
, struct rdma_id_private
, id
);
2621 spin_lock_irqsave(&id_priv
->lock
, flags
);
2622 if (reuse
|| id_priv
->state
== RDMA_CM_IDLE
) {
2623 id_priv
->reuseaddr
= reuse
;
2628 spin_unlock_irqrestore(&id_priv
->lock
, flags
);
2631 EXPORT_SYMBOL(rdma_set_reuseaddr
);
2633 int rdma_set_afonly(struct rdma_cm_id
*id
, int afonly
)
2635 struct rdma_id_private
*id_priv
;
2636 unsigned long flags
;
2639 id_priv
= container_of(id
, struct rdma_id_private
, id
);
2640 spin_lock_irqsave(&id_priv
->lock
, flags
);
2641 if (id_priv
->state
== RDMA_CM_IDLE
|| id_priv
->state
== RDMA_CM_ADDR_BOUND
) {
2642 id_priv
->options
|= (1 << CMA_OPTION_AFONLY
);
2643 id_priv
->afonly
= afonly
;
2648 spin_unlock_irqrestore(&id_priv
->lock
, flags
);
2651 EXPORT_SYMBOL(rdma_set_afonly
);
2653 static void cma_bind_port(struct rdma_bind_list
*bind_list
,
2654 struct rdma_id_private
*id_priv
)
2656 struct sockaddr
*addr
;
2657 struct sockaddr_ib
*sib
;
2661 addr
= cma_src_addr(id_priv
);
2662 port
= htons(bind_list
->port
);
2664 switch (addr
->sa_family
) {
2666 ((struct sockaddr_in
*) addr
)->sin_port
= port
;
2669 ((struct sockaddr_in6
*) addr
)->sin6_port
= port
;
2672 sib
= (struct sockaddr_ib
*) addr
;
2673 sid
= be64_to_cpu(sib
->sib_sid
);
2674 mask
= be64_to_cpu(sib
->sib_sid_mask
);
2675 sib
->sib_sid
= cpu_to_be64((sid
& mask
) | (u64
) ntohs(port
));
2676 sib
->sib_sid_mask
= cpu_to_be64(~0ULL);
2679 id_priv
->bind_list
= bind_list
;
2680 hlist_add_head(&id_priv
->node
, &bind_list
->owners
);
2683 static int cma_alloc_port(enum rdma_port_space ps
,
2684 struct rdma_id_private
*id_priv
, unsigned short snum
)
2686 struct rdma_bind_list
*bind_list
;
2689 bind_list
= kzalloc(sizeof *bind_list
, GFP_KERNEL
);
2693 ret
= cma_ps_alloc(ps
, bind_list
, snum
);
2698 bind_list
->port
= (unsigned short)ret
;
2699 cma_bind_port(bind_list
, id_priv
);
2703 return ret
== -ENOSPC
? -EADDRNOTAVAIL
: ret
;
2706 static int cma_alloc_any_port(enum rdma_port_space ps
,
2707 struct rdma_id_private
*id_priv
)
2709 static unsigned int last_used_port
;
2710 int low
, high
, remaining
;
2713 inet_get_local_port_range(&init_net
, &low
, &high
);
2714 remaining
= (high
- low
) + 1;
2715 rover
= prandom_u32() % remaining
+ low
;
2717 if (last_used_port
!= rover
&&
2718 !cma_ps_find(ps
, (unsigned short)rover
)) {
2719 int ret
= cma_alloc_port(ps
, id_priv
, rover
);
2721 * Remember previously used port number in order to avoid
2722 * re-using same port immediately after it is closed.
2725 last_used_port
= rover
;
2726 if (ret
!= -EADDRNOTAVAIL
)
2731 if ((rover
< low
) || (rover
> high
))
2735 return -EADDRNOTAVAIL
;
2739 * Check that the requested port is available. This is called when trying to
2740 * bind to a specific port, or when trying to listen on a bound port. In
2741 * the latter case, the provided id_priv may already be on the bind_list, but
2742 * we still need to check that it's okay to start listening.
2744 static int cma_check_port(struct rdma_bind_list
*bind_list
,
2745 struct rdma_id_private
*id_priv
, uint8_t reuseaddr
)
2747 struct rdma_id_private
*cur_id
;
2748 struct sockaddr
*addr
, *cur_addr
;
2750 addr
= cma_src_addr(id_priv
);
2751 hlist_for_each_entry(cur_id
, &bind_list
->owners
, node
) {
2752 if (id_priv
== cur_id
)
2755 if ((cur_id
->state
!= RDMA_CM_LISTEN
) && reuseaddr
&&
2759 cur_addr
= cma_src_addr(cur_id
);
2760 if (id_priv
->afonly
&& cur_id
->afonly
&&
2761 (addr
->sa_family
!= cur_addr
->sa_family
))
2764 if (cma_any_addr(addr
) || cma_any_addr(cur_addr
))
2765 return -EADDRNOTAVAIL
;
2767 if (!cma_addr_cmp(addr
, cur_addr
))
2773 static int cma_use_port(enum rdma_port_space ps
,
2774 struct rdma_id_private
*id_priv
)
2776 struct rdma_bind_list
*bind_list
;
2777 unsigned short snum
;
2780 snum
= ntohs(cma_port(cma_src_addr(id_priv
)));
2781 if (snum
< PROT_SOCK
&& !capable(CAP_NET_BIND_SERVICE
))
2784 bind_list
= cma_ps_find(ps
, snum
);
2786 ret
= cma_alloc_port(ps
, id_priv
, snum
);
2788 ret
= cma_check_port(bind_list
, id_priv
, id_priv
->reuseaddr
);
2790 cma_bind_port(bind_list
, id_priv
);
2795 static int cma_bind_listen(struct rdma_id_private
*id_priv
)
2797 struct rdma_bind_list
*bind_list
= id_priv
->bind_list
;
2801 if (bind_list
->owners
.first
->next
)
2802 ret
= cma_check_port(bind_list
, id_priv
, 0);
2803 mutex_unlock(&lock
);
2807 static enum rdma_port_space
cma_select_inet_ps(
2808 struct rdma_id_private
*id_priv
)
2810 switch (id_priv
->id
.ps
) {
2815 return id_priv
->id
.ps
;
2822 static enum rdma_port_space
cma_select_ib_ps(struct rdma_id_private
*id_priv
)
2824 enum rdma_port_space ps
= 0;
2825 struct sockaddr_ib
*sib
;
2826 u64 sid_ps
, mask
, sid
;
2828 sib
= (struct sockaddr_ib
*) cma_src_addr(id_priv
);
2829 mask
= be64_to_cpu(sib
->sib_sid_mask
) & RDMA_IB_IP_PS_MASK
;
2830 sid
= be64_to_cpu(sib
->sib_sid
) & mask
;
2832 if ((id_priv
->id
.ps
== RDMA_PS_IB
) && (sid
== (RDMA_IB_IP_PS_IB
& mask
))) {
2833 sid_ps
= RDMA_IB_IP_PS_IB
;
2835 } else if (((id_priv
->id
.ps
== RDMA_PS_IB
) || (id_priv
->id
.ps
== RDMA_PS_TCP
)) &&
2836 (sid
== (RDMA_IB_IP_PS_TCP
& mask
))) {
2837 sid_ps
= RDMA_IB_IP_PS_TCP
;
2839 } else if (((id_priv
->id
.ps
== RDMA_PS_IB
) || (id_priv
->id
.ps
== RDMA_PS_UDP
)) &&
2840 (sid
== (RDMA_IB_IP_PS_UDP
& mask
))) {
2841 sid_ps
= RDMA_IB_IP_PS_UDP
;
2846 sib
->sib_sid
= cpu_to_be64(sid_ps
| ntohs(cma_port((struct sockaddr
*) sib
)));
2847 sib
->sib_sid_mask
= cpu_to_be64(RDMA_IB_IP_PS_MASK
|
2848 be64_to_cpu(sib
->sib_sid_mask
));
2853 static int cma_get_port(struct rdma_id_private
*id_priv
)
2855 enum rdma_port_space ps
;
2858 if (cma_family(id_priv
) != AF_IB
)
2859 ps
= cma_select_inet_ps(id_priv
);
2861 ps
= cma_select_ib_ps(id_priv
);
2863 return -EPROTONOSUPPORT
;
2866 if (cma_any_port(cma_src_addr(id_priv
)))
2867 ret
= cma_alloc_any_port(ps
, id_priv
);
2869 ret
= cma_use_port(ps
, id_priv
);
2870 mutex_unlock(&lock
);
2875 static int cma_check_linklocal(struct rdma_dev_addr
*dev_addr
,
2876 struct sockaddr
*addr
)
2878 #if IS_ENABLED(CONFIG_IPV6)
2879 struct sockaddr_in6
*sin6
;
2881 if (addr
->sa_family
!= AF_INET6
)
2884 sin6
= (struct sockaddr_in6
*) addr
;
2886 if (!(ipv6_addr_type(&sin6
->sin6_addr
) & IPV6_ADDR_LINKLOCAL
))
2889 if (!sin6
->sin6_scope_id
)
2892 dev_addr
->bound_dev_if
= sin6
->sin6_scope_id
;
2897 int rdma_listen(struct rdma_cm_id
*id
, int backlog
)
2899 struct rdma_id_private
*id_priv
;
2902 id_priv
= container_of(id
, struct rdma_id_private
, id
);
2903 if (id_priv
->state
== RDMA_CM_IDLE
) {
2904 id
->route
.addr
.src_addr
.ss_family
= AF_INET
;
2905 ret
= rdma_bind_addr(id
, cma_src_addr(id_priv
));
2910 if (!cma_comp_exch(id_priv
, RDMA_CM_ADDR_BOUND
, RDMA_CM_LISTEN
))
2913 if (id_priv
->reuseaddr
) {
2914 ret
= cma_bind_listen(id_priv
);
2919 id_priv
->backlog
= backlog
;
2921 if (rdma_cap_ib_cm(id
->device
, 1)) {
2922 ret
= cma_ib_listen(id_priv
);
2925 } else if (rdma_cap_iw_cm(id
->device
, 1)) {
2926 ret
= cma_iw_listen(id_priv
, backlog
);
2934 cma_listen_on_all(id_priv
);
2938 id_priv
->backlog
= 0;
2939 cma_comp_exch(id_priv
, RDMA_CM_LISTEN
, RDMA_CM_ADDR_BOUND
);
2942 EXPORT_SYMBOL(rdma_listen
);
2944 int rdma_bind_addr(struct rdma_cm_id
*id
, struct sockaddr
*addr
)
2946 struct rdma_id_private
*id_priv
;
2949 if (addr
->sa_family
!= AF_INET
&& addr
->sa_family
!= AF_INET6
&&
2950 addr
->sa_family
!= AF_IB
)
2951 return -EAFNOSUPPORT
;
2953 id_priv
= container_of(id
, struct rdma_id_private
, id
);
2954 if (!cma_comp_exch(id_priv
, RDMA_CM_IDLE
, RDMA_CM_ADDR_BOUND
))
2957 ret
= cma_check_linklocal(&id
->route
.addr
.dev_addr
, addr
);
2961 memcpy(cma_src_addr(id_priv
), addr
, rdma_addr_size(addr
));
2962 if (!cma_any_addr(addr
)) {
2963 ret
= cma_translate_addr(addr
, &id
->route
.addr
.dev_addr
);
2967 ret
= cma_acquire_dev(id_priv
, NULL
);
2972 if (!(id_priv
->options
& (1 << CMA_OPTION_AFONLY
))) {
2973 if (addr
->sa_family
== AF_INET
)
2974 id_priv
->afonly
= 1;
2975 #if IS_ENABLED(CONFIG_IPV6)
2976 else if (addr
->sa_family
== AF_INET6
)
2977 id_priv
->afonly
= init_net
.ipv6
.sysctl
.bindv6only
;
2980 ret
= cma_get_port(id_priv
);
2986 if (id_priv
->cma_dev
)
2987 cma_release_dev(id_priv
);
2989 cma_comp_exch(id_priv
, RDMA_CM_ADDR_BOUND
, RDMA_CM_IDLE
);
2992 EXPORT_SYMBOL(rdma_bind_addr
);
2994 static int cma_format_hdr(void *hdr
, struct rdma_id_private
*id_priv
)
2996 struct cma_hdr
*cma_hdr
;
2999 cma_hdr
->cma_version
= CMA_VERSION
;
3000 if (cma_family(id_priv
) == AF_INET
) {
3001 struct sockaddr_in
*src4
, *dst4
;
3003 src4
= (struct sockaddr_in
*) cma_src_addr(id_priv
);
3004 dst4
= (struct sockaddr_in
*) cma_dst_addr(id_priv
);
3006 cma_set_ip_ver(cma_hdr
, 4);
3007 cma_hdr
->src_addr
.ip4
.addr
= src4
->sin_addr
.s_addr
;
3008 cma_hdr
->dst_addr
.ip4
.addr
= dst4
->sin_addr
.s_addr
;
3009 cma_hdr
->port
= src4
->sin_port
;
3010 } else if (cma_family(id_priv
) == AF_INET6
) {
3011 struct sockaddr_in6
*src6
, *dst6
;
3013 src6
= (struct sockaddr_in6
*) cma_src_addr(id_priv
);
3014 dst6
= (struct sockaddr_in6
*) cma_dst_addr(id_priv
);
3016 cma_set_ip_ver(cma_hdr
, 6);
3017 cma_hdr
->src_addr
.ip6
= src6
->sin6_addr
;
3018 cma_hdr
->dst_addr
.ip6
= dst6
->sin6_addr
;
3019 cma_hdr
->port
= src6
->sin6_port
;
3024 static int cma_sidr_rep_handler(struct ib_cm_id
*cm_id
,
3025 struct ib_cm_event
*ib_event
)
3027 struct rdma_id_private
*id_priv
= cm_id
->context
;
3028 struct rdma_cm_event event
;
3029 struct ib_cm_sidr_rep_event_param
*rep
= &ib_event
->param
.sidr_rep_rcvd
;
3032 if (cma_disable_callback(id_priv
, RDMA_CM_CONNECT
))
3035 memset(&event
, 0, sizeof event
);
3036 switch (ib_event
->event
) {
3037 case IB_CM_SIDR_REQ_ERROR
:
3038 event
.event
= RDMA_CM_EVENT_UNREACHABLE
;
3039 event
.status
= -ETIMEDOUT
;
3041 case IB_CM_SIDR_REP_RECEIVED
:
3042 event
.param
.ud
.private_data
= ib_event
->private_data
;
3043 event
.param
.ud
.private_data_len
= IB_CM_SIDR_REP_PRIVATE_DATA_SIZE
;
3044 if (rep
->status
!= IB_SIDR_SUCCESS
) {
3045 event
.event
= RDMA_CM_EVENT_UNREACHABLE
;
3046 event
.status
= ib_event
->param
.sidr_rep_rcvd
.status
;
3049 ret
= cma_set_qkey(id_priv
, rep
->qkey
);
3051 event
.event
= RDMA_CM_EVENT_ADDR_ERROR
;
3055 ib_init_ah_from_path(id_priv
->id
.device
, id_priv
->id
.port_num
,
3056 id_priv
->id
.route
.path_rec
,
3057 &event
.param
.ud
.ah_attr
);
3058 event
.param
.ud
.qp_num
= rep
->qpn
;
3059 event
.param
.ud
.qkey
= rep
->qkey
;
3060 event
.event
= RDMA_CM_EVENT_ESTABLISHED
;
3064 printk(KERN_ERR
"RDMA CMA: unexpected IB CM event: %d\n",
3069 ret
= id_priv
->id
.event_handler(&id_priv
->id
, &event
);
3071 /* Destroy the CM ID by returning a non-zero value. */
3072 id_priv
->cm_id
.ib
= NULL
;
3073 cma_exch(id_priv
, RDMA_CM_DESTROYING
);
3074 mutex_unlock(&id_priv
->handler_mutex
);
3075 rdma_destroy_id(&id_priv
->id
);
3079 mutex_unlock(&id_priv
->handler_mutex
);
3083 static int cma_resolve_ib_udp(struct rdma_id_private
*id_priv
,
3084 struct rdma_conn_param
*conn_param
)
3086 struct ib_cm_sidr_req_param req
;
3087 struct ib_cm_id
*id
;
3091 memset(&req
, 0, sizeof req
);
3092 offset
= cma_user_data_offset(id_priv
);
3093 req
.private_data_len
= offset
+ conn_param
->private_data_len
;
3094 if (req
.private_data_len
< conn_param
->private_data_len
)
3097 if (req
.private_data_len
) {
3098 private_data
= kzalloc(req
.private_data_len
, GFP_ATOMIC
);
3102 private_data
= NULL
;
3105 if (conn_param
->private_data
&& conn_param
->private_data_len
)
3106 memcpy(private_data
+ offset
, conn_param
->private_data
,
3107 conn_param
->private_data_len
);
3110 ret
= cma_format_hdr(private_data
, id_priv
);
3113 req
.private_data
= private_data
;
3116 id
= ib_create_cm_id(id_priv
->id
.device
, cma_sidr_rep_handler
,
3122 id_priv
->cm_id
.ib
= id
;
3124 req
.path
= id_priv
->id
.route
.path_rec
;
3125 req
.service_id
= rdma_get_service_id(&id_priv
->id
, cma_dst_addr(id_priv
));
3126 req
.timeout_ms
= 1 << (CMA_CM_RESPONSE_TIMEOUT
- 8);
3127 req
.max_cm_retries
= CMA_MAX_CM_RETRIES
;
3129 ret
= ib_send_cm_sidr_req(id_priv
->cm_id
.ib
, &req
);
3131 ib_destroy_cm_id(id_priv
->cm_id
.ib
);
3132 id_priv
->cm_id
.ib
= NULL
;
3135 kfree(private_data
);
3139 static int cma_connect_ib(struct rdma_id_private
*id_priv
,
3140 struct rdma_conn_param
*conn_param
)
3142 struct ib_cm_req_param req
;
3143 struct rdma_route
*route
;
3145 struct ib_cm_id
*id
;
3148 memset(&req
, 0, sizeof req
);
3149 offset
= cma_user_data_offset(id_priv
);
3150 req
.private_data_len
= offset
+ conn_param
->private_data_len
;
3151 if (req
.private_data_len
< conn_param
->private_data_len
)
3154 if (req
.private_data_len
) {
3155 private_data
= kzalloc(req
.private_data_len
, GFP_ATOMIC
);
3159 private_data
= NULL
;
3162 if (conn_param
->private_data
&& conn_param
->private_data_len
)
3163 memcpy(private_data
+ offset
, conn_param
->private_data
,
3164 conn_param
->private_data_len
);
3166 id
= ib_create_cm_id(id_priv
->id
.device
, cma_ib_handler
, id_priv
);
3171 id_priv
->cm_id
.ib
= id
;
3173 route
= &id_priv
->id
.route
;
3175 ret
= cma_format_hdr(private_data
, id_priv
);
3178 req
.private_data
= private_data
;
3181 req
.primary_path
= &route
->path_rec
[0];
3182 if (route
->num_paths
== 2)
3183 req
.alternate_path
= &route
->path_rec
[1];
3185 req
.service_id
= rdma_get_service_id(&id_priv
->id
, cma_dst_addr(id_priv
));
3186 req
.qp_num
= id_priv
->qp_num
;
3187 req
.qp_type
= id_priv
->id
.qp_type
;
3188 req
.starting_psn
= id_priv
->seq_num
;
3189 req
.responder_resources
= conn_param
->responder_resources
;
3190 req
.initiator_depth
= conn_param
->initiator_depth
;
3191 req
.flow_control
= conn_param
->flow_control
;
3192 req
.retry_count
= min_t(u8
, 7, conn_param
->retry_count
);
3193 req
.rnr_retry_count
= min_t(u8
, 7, conn_param
->rnr_retry_count
);
3194 req
.remote_cm_response_timeout
= CMA_CM_RESPONSE_TIMEOUT
;
3195 req
.local_cm_response_timeout
= CMA_CM_RESPONSE_TIMEOUT
;
3196 req
.max_cm_retries
= CMA_MAX_CM_RETRIES
;
3197 req
.srq
= id_priv
->srq
? 1 : 0;
3199 ret
= ib_send_cm_req(id_priv
->cm_id
.ib
, &req
);
3201 if (ret
&& !IS_ERR(id
)) {
3202 ib_destroy_cm_id(id
);
3203 id_priv
->cm_id
.ib
= NULL
;
3206 kfree(private_data
);
3210 static int cma_connect_iw(struct rdma_id_private
*id_priv
,
3211 struct rdma_conn_param
*conn_param
)
3213 struct iw_cm_id
*cm_id
;
3215 struct iw_cm_conn_param iw_param
;
3217 cm_id
= iw_create_cm_id(id_priv
->id
.device
, cma_iw_handler
, id_priv
);
3219 return PTR_ERR(cm_id
);
3221 cm_id
->tos
= id_priv
->tos
;
3222 id_priv
->cm_id
.iw
= cm_id
;
3224 memcpy(&cm_id
->local_addr
, cma_src_addr(id_priv
),
3225 rdma_addr_size(cma_src_addr(id_priv
)));
3226 memcpy(&cm_id
->remote_addr
, cma_dst_addr(id_priv
),
3227 rdma_addr_size(cma_dst_addr(id_priv
)));
3229 ret
= cma_modify_qp_rtr(id_priv
, conn_param
);
3234 iw_param
.ord
= conn_param
->initiator_depth
;
3235 iw_param
.ird
= conn_param
->responder_resources
;
3236 iw_param
.private_data
= conn_param
->private_data
;
3237 iw_param
.private_data_len
= conn_param
->private_data_len
;
3238 iw_param
.qpn
= id_priv
->id
.qp
? id_priv
->qp_num
: conn_param
->qp_num
;
3240 memset(&iw_param
, 0, sizeof iw_param
);
3241 iw_param
.qpn
= id_priv
->qp_num
;
3243 ret
= iw_cm_connect(cm_id
, &iw_param
);
3246 iw_destroy_cm_id(cm_id
);
3247 id_priv
->cm_id
.iw
= NULL
;
3252 int rdma_connect(struct rdma_cm_id
*id
, struct rdma_conn_param
*conn_param
)
3254 struct rdma_id_private
*id_priv
;
3257 id_priv
= container_of(id
, struct rdma_id_private
, id
);
3258 if (!cma_comp_exch(id_priv
, RDMA_CM_ROUTE_RESOLVED
, RDMA_CM_CONNECT
))
3262 id_priv
->qp_num
= conn_param
->qp_num
;
3263 id_priv
->srq
= conn_param
->srq
;
3266 if (rdma_cap_ib_cm(id
->device
, id
->port_num
)) {
3267 if (id
->qp_type
== IB_QPT_UD
)
3268 ret
= cma_resolve_ib_udp(id_priv
, conn_param
);
3270 ret
= cma_connect_ib(id_priv
, conn_param
);
3271 } else if (rdma_cap_iw_cm(id
->device
, id
->port_num
))
3272 ret
= cma_connect_iw(id_priv
, conn_param
);
3280 cma_comp_exch(id_priv
, RDMA_CM_CONNECT
, RDMA_CM_ROUTE_RESOLVED
);
3283 EXPORT_SYMBOL(rdma_connect
);
3285 static int cma_accept_ib(struct rdma_id_private
*id_priv
,
3286 struct rdma_conn_param
*conn_param
)
3288 struct ib_cm_rep_param rep
;
3291 ret
= cma_modify_qp_rtr(id_priv
, conn_param
);
3295 ret
= cma_modify_qp_rts(id_priv
, conn_param
);
3299 memset(&rep
, 0, sizeof rep
);
3300 rep
.qp_num
= id_priv
->qp_num
;
3301 rep
.starting_psn
= id_priv
->seq_num
;
3302 rep
.private_data
= conn_param
->private_data
;
3303 rep
.private_data_len
= conn_param
->private_data_len
;
3304 rep
.responder_resources
= conn_param
->responder_resources
;
3305 rep
.initiator_depth
= conn_param
->initiator_depth
;
3306 rep
.failover_accepted
= 0;
3307 rep
.flow_control
= conn_param
->flow_control
;
3308 rep
.rnr_retry_count
= min_t(u8
, 7, conn_param
->rnr_retry_count
);
3309 rep
.srq
= id_priv
->srq
? 1 : 0;
3311 ret
= ib_send_cm_rep(id_priv
->cm_id
.ib
, &rep
);
3316 static int cma_accept_iw(struct rdma_id_private
*id_priv
,
3317 struct rdma_conn_param
*conn_param
)
3319 struct iw_cm_conn_param iw_param
;
3322 ret
= cma_modify_qp_rtr(id_priv
, conn_param
);
3326 iw_param
.ord
= conn_param
->initiator_depth
;
3327 iw_param
.ird
= conn_param
->responder_resources
;
3328 iw_param
.private_data
= conn_param
->private_data
;
3329 iw_param
.private_data_len
= conn_param
->private_data_len
;
3330 if (id_priv
->id
.qp
) {
3331 iw_param
.qpn
= id_priv
->qp_num
;
3333 iw_param
.qpn
= conn_param
->qp_num
;
3335 return iw_cm_accept(id_priv
->cm_id
.iw
, &iw_param
);
3338 static int cma_send_sidr_rep(struct rdma_id_private
*id_priv
,
3339 enum ib_cm_sidr_status status
, u32 qkey
,
3340 const void *private_data
, int private_data_len
)
3342 struct ib_cm_sidr_rep_param rep
;
3345 memset(&rep
, 0, sizeof rep
);
3346 rep
.status
= status
;
3347 if (status
== IB_SIDR_SUCCESS
) {
3348 ret
= cma_set_qkey(id_priv
, qkey
);
3351 rep
.qp_num
= id_priv
->qp_num
;
3352 rep
.qkey
= id_priv
->qkey
;
3354 rep
.private_data
= private_data
;
3355 rep
.private_data_len
= private_data_len
;
3357 return ib_send_cm_sidr_rep(id_priv
->cm_id
.ib
, &rep
);
3360 int rdma_accept(struct rdma_cm_id
*id
, struct rdma_conn_param
*conn_param
)
3362 struct rdma_id_private
*id_priv
;
3365 id_priv
= container_of(id
, struct rdma_id_private
, id
);
3367 id_priv
->owner
= task_pid_nr(current
);
3369 if (!cma_comp(id_priv
, RDMA_CM_CONNECT
))
3372 if (!id
->qp
&& conn_param
) {
3373 id_priv
->qp_num
= conn_param
->qp_num
;
3374 id_priv
->srq
= conn_param
->srq
;
3377 if (rdma_cap_ib_cm(id
->device
, id
->port_num
)) {
3378 if (id
->qp_type
== IB_QPT_UD
) {
3380 ret
= cma_send_sidr_rep(id_priv
, IB_SIDR_SUCCESS
,
3382 conn_param
->private_data
,
3383 conn_param
->private_data_len
);
3385 ret
= cma_send_sidr_rep(id_priv
, IB_SIDR_SUCCESS
,
3389 ret
= cma_accept_ib(id_priv
, conn_param
);
3391 ret
= cma_rep_recv(id_priv
);
3393 } else if (rdma_cap_iw_cm(id
->device
, id
->port_num
))
3394 ret
= cma_accept_iw(id_priv
, conn_param
);
3403 cma_modify_qp_err(id_priv
);
3404 rdma_reject(id
, NULL
, 0);
3407 EXPORT_SYMBOL(rdma_accept
);
3409 int rdma_notify(struct rdma_cm_id
*id
, enum ib_event_type event
)
3411 struct rdma_id_private
*id_priv
;
3414 id_priv
= container_of(id
, struct rdma_id_private
, id
);
3415 if (!id_priv
->cm_id
.ib
)
3418 switch (id
->device
->node_type
) {
3419 case RDMA_NODE_IB_CA
:
3420 ret
= ib_cm_notify(id_priv
->cm_id
.ib
, event
);
3428 EXPORT_SYMBOL(rdma_notify
);
3430 int rdma_reject(struct rdma_cm_id
*id
, const void *private_data
,
3431 u8 private_data_len
)
3433 struct rdma_id_private
*id_priv
;
3436 id_priv
= container_of(id
, struct rdma_id_private
, id
);
3437 if (!id_priv
->cm_id
.ib
)
3440 if (rdma_cap_ib_cm(id
->device
, id
->port_num
)) {
3441 if (id
->qp_type
== IB_QPT_UD
)
3442 ret
= cma_send_sidr_rep(id_priv
, IB_SIDR_REJECT
, 0,
3443 private_data
, private_data_len
);
3445 ret
= ib_send_cm_rej(id_priv
->cm_id
.ib
,
3446 IB_CM_REJ_CONSUMER_DEFINED
, NULL
,
3447 0, private_data
, private_data_len
);
3448 } else if (rdma_cap_iw_cm(id
->device
, id
->port_num
)) {
3449 ret
= iw_cm_reject(id_priv
->cm_id
.iw
,
3450 private_data
, private_data_len
);
3456 EXPORT_SYMBOL(rdma_reject
);
3458 int rdma_disconnect(struct rdma_cm_id
*id
)
3460 struct rdma_id_private
*id_priv
;
3463 id_priv
= container_of(id
, struct rdma_id_private
, id
);
3464 if (!id_priv
->cm_id
.ib
)
3467 if (rdma_cap_ib_cm(id
->device
, id
->port_num
)) {
3468 ret
= cma_modify_qp_err(id_priv
);
3471 /* Initiate or respond to a disconnect. */
3472 if (ib_send_cm_dreq(id_priv
->cm_id
.ib
, NULL
, 0))
3473 ib_send_cm_drep(id_priv
->cm_id
.ib
, NULL
, 0);
3474 } else if (rdma_cap_iw_cm(id
->device
, id
->port_num
)) {
3475 ret
= iw_cm_disconnect(id_priv
->cm_id
.iw
, 0);
3482 EXPORT_SYMBOL(rdma_disconnect
);
3484 static int cma_ib_mc_handler(int status
, struct ib_sa_multicast
*multicast
)
3486 struct rdma_id_private
*id_priv
;
3487 struct cma_multicast
*mc
= multicast
->context
;
3488 struct rdma_cm_event event
;
3491 id_priv
= mc
->id_priv
;
3492 if (cma_disable_callback(id_priv
, RDMA_CM_ADDR_BOUND
) &&
3493 cma_disable_callback(id_priv
, RDMA_CM_ADDR_RESOLVED
))
3497 status
= cma_set_qkey(id_priv
, be32_to_cpu(multicast
->rec
.qkey
));
3498 mutex_lock(&id_priv
->qp_mutex
);
3499 if (!status
&& id_priv
->id
.qp
)
3500 status
= ib_attach_mcast(id_priv
->id
.qp
, &multicast
->rec
.mgid
,
3501 be16_to_cpu(multicast
->rec
.mlid
));
3502 mutex_unlock(&id_priv
->qp_mutex
);
3504 memset(&event
, 0, sizeof event
);
3505 event
.status
= status
;
3506 event
.param
.ud
.private_data
= mc
->context
;
3508 event
.event
= RDMA_CM_EVENT_MULTICAST_JOIN
;
3509 ib_init_ah_from_mcmember(id_priv
->id
.device
,
3510 id_priv
->id
.port_num
, &multicast
->rec
,
3511 &event
.param
.ud
.ah_attr
);
3512 event
.param
.ud
.qp_num
= 0xFFFFFF;
3513 event
.param
.ud
.qkey
= be32_to_cpu(multicast
->rec
.qkey
);
3515 event
.event
= RDMA_CM_EVENT_MULTICAST_ERROR
;
3517 ret
= id_priv
->id
.event_handler(&id_priv
->id
, &event
);
3519 cma_exch(id_priv
, RDMA_CM_DESTROYING
);
3520 mutex_unlock(&id_priv
->handler_mutex
);
3521 rdma_destroy_id(&id_priv
->id
);
3525 mutex_unlock(&id_priv
->handler_mutex
);
3529 static void cma_set_mgid(struct rdma_id_private
*id_priv
,
3530 struct sockaddr
*addr
, union ib_gid
*mgid
)
3532 unsigned char mc_map
[MAX_ADDR_LEN
];
3533 struct rdma_dev_addr
*dev_addr
= &id_priv
->id
.route
.addr
.dev_addr
;
3534 struct sockaddr_in
*sin
= (struct sockaddr_in
*) addr
;
3535 struct sockaddr_in6
*sin6
= (struct sockaddr_in6
*) addr
;
3537 if (cma_any_addr(addr
)) {
3538 memset(mgid
, 0, sizeof *mgid
);
3539 } else if ((addr
->sa_family
== AF_INET6
) &&
3540 ((be32_to_cpu(sin6
->sin6_addr
.s6_addr32
[0]) & 0xFFF0FFFF) ==
3542 /* IPv6 address is an SA assigned MGID. */
3543 memcpy(mgid
, &sin6
->sin6_addr
, sizeof *mgid
);
3544 } else if (addr
->sa_family
== AF_IB
) {
3545 memcpy(mgid
, &((struct sockaddr_ib
*) addr
)->sib_addr
, sizeof *mgid
);
3546 } else if ((addr
->sa_family
== AF_INET6
)) {
3547 ipv6_ib_mc_map(&sin6
->sin6_addr
, dev_addr
->broadcast
, mc_map
);
3548 if (id_priv
->id
.ps
== RDMA_PS_UDP
)
3549 mc_map
[7] = 0x01; /* Use RDMA CM signature */
3550 *mgid
= *(union ib_gid
*) (mc_map
+ 4);
3552 ip_ib_mc_map(sin
->sin_addr
.s_addr
, dev_addr
->broadcast
, mc_map
);
3553 if (id_priv
->id
.ps
== RDMA_PS_UDP
)
3554 mc_map
[7] = 0x01; /* Use RDMA CM signature */
3555 *mgid
= *(union ib_gid
*) (mc_map
+ 4);
3559 static int cma_join_ib_multicast(struct rdma_id_private
*id_priv
,
3560 struct cma_multicast
*mc
)
3562 struct ib_sa_mcmember_rec rec
;
3563 struct rdma_dev_addr
*dev_addr
= &id_priv
->id
.route
.addr
.dev_addr
;
3564 ib_sa_comp_mask comp_mask
;
3567 ib_addr_get_mgid(dev_addr
, &rec
.mgid
);
3568 ret
= ib_sa_get_mcmember_rec(id_priv
->id
.device
, id_priv
->id
.port_num
,
3573 ret
= cma_set_qkey(id_priv
, 0);
3577 cma_set_mgid(id_priv
, (struct sockaddr
*) &mc
->addr
, &rec
.mgid
);
3578 rec
.qkey
= cpu_to_be32(id_priv
->qkey
);
3579 rdma_addr_get_sgid(dev_addr
, &rec
.port_gid
);
3580 rec
.pkey
= cpu_to_be16(ib_addr_get_pkey(dev_addr
));
3583 comp_mask
= IB_SA_MCMEMBER_REC_MGID
| IB_SA_MCMEMBER_REC_PORT_GID
|
3584 IB_SA_MCMEMBER_REC_PKEY
| IB_SA_MCMEMBER_REC_JOIN_STATE
|
3585 IB_SA_MCMEMBER_REC_QKEY
| IB_SA_MCMEMBER_REC_SL
|
3586 IB_SA_MCMEMBER_REC_FLOW_LABEL
|
3587 IB_SA_MCMEMBER_REC_TRAFFIC_CLASS
;
3589 if (id_priv
->id
.ps
== RDMA_PS_IPOIB
)
3590 comp_mask
|= IB_SA_MCMEMBER_REC_RATE
|
3591 IB_SA_MCMEMBER_REC_RATE_SELECTOR
|
3592 IB_SA_MCMEMBER_REC_MTU_SELECTOR
|
3593 IB_SA_MCMEMBER_REC_MTU
|
3594 IB_SA_MCMEMBER_REC_HOP_LIMIT
;
3596 mc
->multicast
.ib
= ib_sa_join_multicast(&sa_client
, id_priv
->id
.device
,
3597 id_priv
->id
.port_num
, &rec
,
3598 comp_mask
, GFP_KERNEL
,
3599 cma_ib_mc_handler
, mc
);
3600 return PTR_ERR_OR_ZERO(mc
->multicast
.ib
);
3603 static void iboe_mcast_work_handler(struct work_struct
*work
)
3605 struct iboe_mcast_work
*mw
= container_of(work
, struct iboe_mcast_work
, work
);
3606 struct cma_multicast
*mc
= mw
->mc
;
3607 struct ib_sa_multicast
*m
= mc
->multicast
.ib
;
3609 mc
->multicast
.ib
->context
= mc
;
3610 cma_ib_mc_handler(0, m
);
3611 kref_put(&mc
->mcref
, release_mc
);
3615 static void cma_iboe_set_mgid(struct sockaddr
*addr
, union ib_gid
*mgid
)
3617 struct sockaddr_in
*sin
= (struct sockaddr_in
*)addr
;
3618 struct sockaddr_in6
*sin6
= (struct sockaddr_in6
*)addr
;
3620 if (cma_any_addr(addr
)) {
3621 memset(mgid
, 0, sizeof *mgid
);
3622 } else if (addr
->sa_family
== AF_INET6
) {
3623 memcpy(mgid
, &sin6
->sin6_addr
, sizeof *mgid
);
3625 mgid
->raw
[0] = 0xff;
3626 mgid
->raw
[1] = 0x0e;
3635 mgid
->raw
[10] = 0xff;
3636 mgid
->raw
[11] = 0xff;
3637 *(__be32
*)(&mgid
->raw
[12]) = sin
->sin_addr
.s_addr
;
3641 static int cma_iboe_join_multicast(struct rdma_id_private
*id_priv
,
3642 struct cma_multicast
*mc
)
3644 struct iboe_mcast_work
*work
;
3645 struct rdma_dev_addr
*dev_addr
= &id_priv
->id
.route
.addr
.dev_addr
;
3647 struct sockaddr
*addr
= (struct sockaddr
*)&mc
->addr
;
3648 struct net_device
*ndev
= NULL
;
3650 if (cma_zero_addr((struct sockaddr
*)&mc
->addr
))
3653 work
= kzalloc(sizeof *work
, GFP_KERNEL
);
3657 mc
->multicast
.ib
= kzalloc(sizeof(struct ib_sa_multicast
), GFP_KERNEL
);
3658 if (!mc
->multicast
.ib
) {
3663 cma_iboe_set_mgid(addr
, &mc
->multicast
.ib
->rec
.mgid
);
3665 mc
->multicast
.ib
->rec
.pkey
= cpu_to_be16(0xffff);
3666 if (id_priv
->id
.ps
== RDMA_PS_UDP
)
3667 mc
->multicast
.ib
->rec
.qkey
= cpu_to_be32(RDMA_UDP_QKEY
);
3669 if (dev_addr
->bound_dev_if
)
3670 ndev
= dev_get_by_index(&init_net
, dev_addr
->bound_dev_if
);
3675 mc
->multicast
.ib
->rec
.rate
= iboe_get_rate(ndev
);
3676 mc
->multicast
.ib
->rec
.hop_limit
= 1;
3677 mc
->multicast
.ib
->rec
.mtu
= iboe_get_mtu(ndev
->mtu
);
3679 if (!mc
->multicast
.ib
->rec
.mtu
) {
3683 rdma_ip2gid((struct sockaddr
*)&id_priv
->id
.route
.addr
.src_addr
,
3684 &mc
->multicast
.ib
->rec
.port_gid
);
3687 INIT_WORK(&work
->work
, iboe_mcast_work_handler
);
3688 kref_get(&mc
->mcref
);
3689 queue_work(cma_wq
, &work
->work
);
3694 kfree(mc
->multicast
.ib
);
3700 int rdma_join_multicast(struct rdma_cm_id
*id
, struct sockaddr
*addr
,
3703 struct rdma_id_private
*id_priv
;
3704 struct cma_multicast
*mc
;
3707 id_priv
= container_of(id
, struct rdma_id_private
, id
);
3708 if (!cma_comp(id_priv
, RDMA_CM_ADDR_BOUND
) &&
3709 !cma_comp(id_priv
, RDMA_CM_ADDR_RESOLVED
))
3712 mc
= kmalloc(sizeof *mc
, GFP_KERNEL
);
3716 memcpy(&mc
->addr
, addr
, rdma_addr_size(addr
));
3717 mc
->context
= context
;
3718 mc
->id_priv
= id_priv
;
3720 spin_lock(&id_priv
->lock
);
3721 list_add(&mc
->list
, &id_priv
->mc_list
);
3722 spin_unlock(&id_priv
->lock
);
3724 if (rdma_protocol_roce(id
->device
, id
->port_num
)) {
3725 kref_init(&mc
->mcref
);
3726 ret
= cma_iboe_join_multicast(id_priv
, mc
);
3727 } else if (rdma_cap_ib_mcast(id
->device
, id
->port_num
))
3728 ret
= cma_join_ib_multicast(id_priv
, mc
);
3733 spin_lock_irq(&id_priv
->lock
);
3734 list_del(&mc
->list
);
3735 spin_unlock_irq(&id_priv
->lock
);
3740 EXPORT_SYMBOL(rdma_join_multicast
);
3742 void rdma_leave_multicast(struct rdma_cm_id
*id
, struct sockaddr
*addr
)
3744 struct rdma_id_private
*id_priv
;
3745 struct cma_multicast
*mc
;
3747 id_priv
= container_of(id
, struct rdma_id_private
, id
);
3748 spin_lock_irq(&id_priv
->lock
);
3749 list_for_each_entry(mc
, &id_priv
->mc_list
, list
) {
3750 if (!memcmp(&mc
->addr
, addr
, rdma_addr_size(addr
))) {
3751 list_del(&mc
->list
);
3752 spin_unlock_irq(&id_priv
->lock
);
3755 ib_detach_mcast(id
->qp
,
3756 &mc
->multicast
.ib
->rec
.mgid
,
3757 be16_to_cpu(mc
->multicast
.ib
->rec
.mlid
));
3759 BUG_ON(id_priv
->cma_dev
->device
!= id
->device
);
3761 if (rdma_cap_ib_mcast(id
->device
, id
->port_num
)) {
3762 ib_sa_free_multicast(mc
->multicast
.ib
);
3764 } else if (rdma_protocol_roce(id
->device
, id
->port_num
))
3765 kref_put(&mc
->mcref
, release_mc
);
3770 spin_unlock_irq(&id_priv
->lock
);
3772 EXPORT_SYMBOL(rdma_leave_multicast
);
3774 static int cma_netdev_change(struct net_device
*ndev
, struct rdma_id_private
*id_priv
)
3776 struct rdma_dev_addr
*dev_addr
;
3777 struct cma_ndev_work
*work
;
3779 dev_addr
= &id_priv
->id
.route
.addr
.dev_addr
;
3781 if ((dev_addr
->bound_dev_if
== ndev
->ifindex
) &&
3782 memcmp(dev_addr
->src_dev_addr
, ndev
->dev_addr
, ndev
->addr_len
)) {
3783 printk(KERN_INFO
"RDMA CM addr change for ndev %s used by id %p\n",
3784 ndev
->name
, &id_priv
->id
);
3785 work
= kzalloc(sizeof *work
, GFP_KERNEL
);
3789 INIT_WORK(&work
->work
, cma_ndev_work_handler
);
3791 work
->event
.event
= RDMA_CM_EVENT_ADDR_CHANGE
;
3792 atomic_inc(&id_priv
->refcount
);
3793 queue_work(cma_wq
, &work
->work
);
3799 static int cma_netdev_callback(struct notifier_block
*self
, unsigned long event
,
3802 struct net_device
*ndev
= netdev_notifier_info_to_dev(ptr
);
3803 struct cma_device
*cma_dev
;
3804 struct rdma_id_private
*id_priv
;
3805 int ret
= NOTIFY_DONE
;
3807 if (dev_net(ndev
) != &init_net
)
3810 if (event
!= NETDEV_BONDING_FAILOVER
)
3813 if (!(ndev
->flags
& IFF_MASTER
) || !(ndev
->priv_flags
& IFF_BONDING
))
3817 list_for_each_entry(cma_dev
, &dev_list
, list
)
3818 list_for_each_entry(id_priv
, &cma_dev
->id_list
, list
) {
3819 ret
= cma_netdev_change(ndev
, id_priv
);
3825 mutex_unlock(&lock
);
3829 static struct notifier_block cma_nb
= {
3830 .notifier_call
= cma_netdev_callback
3833 static void cma_add_one(struct ib_device
*device
)
3835 struct cma_device
*cma_dev
;
3836 struct rdma_id_private
*id_priv
;
3838 cma_dev
= kmalloc(sizeof *cma_dev
, GFP_KERNEL
);
3842 cma_dev
->device
= device
;
3844 init_completion(&cma_dev
->comp
);
3845 atomic_set(&cma_dev
->refcount
, 1);
3846 INIT_LIST_HEAD(&cma_dev
->id_list
);
3847 ib_set_client_data(device
, &cma_client
, cma_dev
);
3850 list_add_tail(&cma_dev
->list
, &dev_list
);
3851 list_for_each_entry(id_priv
, &listen_any_list
, list
)
3852 cma_listen_on_dev(id_priv
, cma_dev
);
3853 mutex_unlock(&lock
);
3856 static int cma_remove_id_dev(struct rdma_id_private
*id_priv
)
3858 struct rdma_cm_event event
;
3859 enum rdma_cm_state state
;
3862 /* Record that we want to remove the device */
3863 state
= cma_exch(id_priv
, RDMA_CM_DEVICE_REMOVAL
);
3864 if (state
== RDMA_CM_DESTROYING
)
3867 cma_cancel_operation(id_priv
, state
);
3868 mutex_lock(&id_priv
->handler_mutex
);
3870 /* Check for destruction from another callback. */
3871 if (!cma_comp(id_priv
, RDMA_CM_DEVICE_REMOVAL
))
3874 memset(&event
, 0, sizeof event
);
3875 event
.event
= RDMA_CM_EVENT_DEVICE_REMOVAL
;
3876 ret
= id_priv
->id
.event_handler(&id_priv
->id
, &event
);
3878 mutex_unlock(&id_priv
->handler_mutex
);
3882 static void cma_process_remove(struct cma_device
*cma_dev
)
3884 struct rdma_id_private
*id_priv
;
3888 while (!list_empty(&cma_dev
->id_list
)) {
3889 id_priv
= list_entry(cma_dev
->id_list
.next
,
3890 struct rdma_id_private
, list
);
3892 list_del(&id_priv
->listen_list
);
3893 list_del_init(&id_priv
->list
);
3894 atomic_inc(&id_priv
->refcount
);
3895 mutex_unlock(&lock
);
3897 ret
= id_priv
->internal_id
? 1 : cma_remove_id_dev(id_priv
);
3898 cma_deref_id(id_priv
);
3900 rdma_destroy_id(&id_priv
->id
);
3904 mutex_unlock(&lock
);
3906 cma_deref_dev(cma_dev
);
3907 wait_for_completion(&cma_dev
->comp
);
3910 static void cma_remove_one(struct ib_device
*device
, void *client_data
)
3912 struct cma_device
*cma_dev
= client_data
;
3918 list_del(&cma_dev
->list
);
3919 mutex_unlock(&lock
);
3921 cma_process_remove(cma_dev
);
3925 static int cma_get_id_stats(struct sk_buff
*skb
, struct netlink_callback
*cb
)
3927 struct nlmsghdr
*nlh
;
3928 struct rdma_cm_id_stats
*id_stats
;
3929 struct rdma_id_private
*id_priv
;
3930 struct rdma_cm_id
*id
= NULL
;
3931 struct cma_device
*cma_dev
;
3932 int i_dev
= 0, i_id
= 0;
3935 * We export all of the IDs as a sequence of messages. Each
3936 * ID gets its own netlink message.
3940 list_for_each_entry(cma_dev
, &dev_list
, list
) {
3941 if (i_dev
< cb
->args
[0]) {
3947 list_for_each_entry(id_priv
, &cma_dev
->id_list
, list
) {
3948 if (i_id
< cb
->args
[1]) {
3953 id_stats
= ibnl_put_msg(skb
, &nlh
, cb
->nlh
->nlmsg_seq
,
3954 sizeof *id_stats
, RDMA_NL_RDMA_CM
,
3955 RDMA_NL_RDMA_CM_ID_STATS
,
3960 memset(id_stats
, 0, sizeof *id_stats
);
3962 id_stats
->node_type
= id
->route
.addr
.dev_addr
.dev_type
;
3963 id_stats
->port_num
= id
->port_num
;
3964 id_stats
->bound_dev_if
=
3965 id
->route
.addr
.dev_addr
.bound_dev_if
;
3967 if (ibnl_put_attr(skb
, nlh
,
3968 rdma_addr_size(cma_src_addr(id_priv
)),
3969 cma_src_addr(id_priv
),
3970 RDMA_NL_RDMA_CM_ATTR_SRC_ADDR
))
3972 if (ibnl_put_attr(skb
, nlh
,
3973 rdma_addr_size(cma_src_addr(id_priv
)),
3974 cma_dst_addr(id_priv
),
3975 RDMA_NL_RDMA_CM_ATTR_DST_ADDR
))
3978 id_stats
->pid
= id_priv
->owner
;
3979 id_stats
->port_space
= id
->ps
;
3980 id_stats
->cm_state
= id_priv
->state
;
3981 id_stats
->qp_num
= id_priv
->qp_num
;
3982 id_stats
->qp_type
= id
->qp_type
;
3992 mutex_unlock(&lock
);
3993 cb
->args
[0] = i_dev
;
3999 static const struct ibnl_client_cbs cma_cb_table
[] = {
4000 [RDMA_NL_RDMA_CM_ID_STATS
] = { .dump
= cma_get_id_stats
,
4001 .module
= THIS_MODULE
},
4004 static int __init
cma_init(void)
4008 cma_wq
= create_singlethread_workqueue("rdma_cm");
4012 ib_sa_register_client(&sa_client
);
4013 rdma_addr_register_client(&addr_client
);
4014 register_netdevice_notifier(&cma_nb
);
4016 ret
= ib_register_client(&cma_client
);
4020 if (ibnl_add_client(RDMA_NL_RDMA_CM
, RDMA_NL_RDMA_CM_NUM_OPS
, cma_cb_table
))
4021 printk(KERN_WARNING
"RDMA CMA: failed to add netlink callback\n");
4026 unregister_netdevice_notifier(&cma_nb
);
4027 rdma_addr_unregister_client(&addr_client
);
4028 ib_sa_unregister_client(&sa_client
);
4029 destroy_workqueue(cma_wq
);
4033 static void __exit
cma_cleanup(void)
4035 ibnl_remove_client(RDMA_NL_RDMA_CM
);
4036 ib_unregister_client(&cma_client
);
4037 unregister_netdevice_notifier(&cma_nb
);
4038 rdma_addr_unregister_client(&addr_client
);
4039 ib_sa_unregister_client(&sa_client
);
4040 destroy_workqueue(cma_wq
);
4041 idr_destroy(&tcp_ps
);
4042 idr_destroy(&udp_ps
);
4043 idr_destroy(&ipoib_ps
);
4044 idr_destroy(&ib_ps
);
4047 module_init(cma_init
);
4048 module_exit(cma_cleanup
);