2 * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the BSD-type
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
14 * Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
17 * Redistributions in binary form must reproduce the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer in the documentation and/or other materials provided
20 * with the distribution.
22 * Neither the name of the Network Appliance, Inc. nor the names of
23 * its contributors may be used to endorse or promote products
24 * derived from this software without specific prior written
27 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
29 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
30 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
31 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
32 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
33 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
34 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
35 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
36 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
37 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43 * This file contains the top-level implementation of an RPC RDMA
46 * Naming convention: functions beginning with xprt_ are part of the
47 * transport switch. All others are RPC RDMA internal.
50 #include <linux/module.h>
51 #include <linux/slab.h>
52 #include <linux/seq_file.h>
53 #include <linux/sunrpc/addr.h>
55 #include "xprt_rdma.h"
57 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
58 # define RPCDBG_FACILITY RPCDBG_TRANS
65 static unsigned int xprt_rdma_slot_table_entries
= RPCRDMA_DEF_SLOT_TABLE
;
66 unsigned int xprt_rdma_max_inline_read
= RPCRDMA_DEF_INLINE
;
67 static unsigned int xprt_rdma_max_inline_write
= RPCRDMA_DEF_INLINE
;
68 static unsigned int xprt_rdma_inline_write_padding
;
69 static unsigned int xprt_rdma_memreg_strategy
= RPCRDMA_FRMR
;
70 int xprt_rdma_pad_optimize
= 1;
72 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
74 static unsigned int min_slot_table_size
= RPCRDMA_MIN_SLOT_TABLE
;
75 static unsigned int max_slot_table_size
= RPCRDMA_MAX_SLOT_TABLE
;
76 static unsigned int min_inline_size
= RPCRDMA_MIN_INLINE
;
77 static unsigned int max_inline_size
= RPCRDMA_MAX_INLINE
;
78 static unsigned int zero
;
79 static unsigned int max_padding
= PAGE_SIZE
;
80 static unsigned int min_memreg
= RPCRDMA_BOUNCEBUFFERS
;
81 static unsigned int max_memreg
= RPCRDMA_LAST
- 1;
83 static struct ctl_table_header
*sunrpc_table_header
;
85 static struct ctl_table xr_tunables_table
[] = {
87 .procname
= "rdma_slot_table_entries",
88 .data
= &xprt_rdma_slot_table_entries
,
89 .maxlen
= sizeof(unsigned int),
91 .proc_handler
= proc_dointvec_minmax
,
92 .extra1
= &min_slot_table_size
,
93 .extra2
= &max_slot_table_size
96 .procname
= "rdma_max_inline_read",
97 .data
= &xprt_rdma_max_inline_read
,
98 .maxlen
= sizeof(unsigned int),
100 .proc_handler
= proc_dointvec
,
101 .extra1
= &min_inline_size
,
102 .extra2
= &max_inline_size
,
105 .procname
= "rdma_max_inline_write",
106 .data
= &xprt_rdma_max_inline_write
,
107 .maxlen
= sizeof(unsigned int),
109 .proc_handler
= proc_dointvec
,
110 .extra1
= &min_inline_size
,
111 .extra2
= &max_inline_size
,
114 .procname
= "rdma_inline_write_padding",
115 .data
= &xprt_rdma_inline_write_padding
,
116 .maxlen
= sizeof(unsigned int),
118 .proc_handler
= proc_dointvec_minmax
,
120 .extra2
= &max_padding
,
123 .procname
= "rdma_memreg_strategy",
124 .data
= &xprt_rdma_memreg_strategy
,
125 .maxlen
= sizeof(unsigned int),
127 .proc_handler
= proc_dointvec_minmax
,
128 .extra1
= &min_memreg
,
129 .extra2
= &max_memreg
,
132 .procname
= "rdma_pad_optimize",
133 .data
= &xprt_rdma_pad_optimize
,
134 .maxlen
= sizeof(unsigned int),
136 .proc_handler
= proc_dointvec
,
141 static struct ctl_table sunrpc_table
[] = {
143 .procname
= "sunrpc",
145 .child
= xr_tunables_table
152 static struct rpc_xprt_ops xprt_rdma_procs
; /*forward reference */
155 xprt_rdma_format_addresses4(struct rpc_xprt
*xprt
, struct sockaddr
*sap
)
157 struct sockaddr_in
*sin
= (struct sockaddr_in
*)sap
;
160 snprintf(buf
, sizeof(buf
), "%08x", ntohl(sin
->sin_addr
.s_addr
));
161 xprt
->address_strings
[RPC_DISPLAY_HEX_ADDR
] = kstrdup(buf
, GFP_KERNEL
);
163 xprt
->address_strings
[RPC_DISPLAY_NETID
] = RPCBIND_NETID_RDMA
;
167 xprt_rdma_format_addresses6(struct rpc_xprt
*xprt
, struct sockaddr
*sap
)
169 struct sockaddr_in6
*sin6
= (struct sockaddr_in6
*)sap
;
172 snprintf(buf
, sizeof(buf
), "%pi6", &sin6
->sin6_addr
);
173 xprt
->address_strings
[RPC_DISPLAY_HEX_ADDR
] = kstrdup(buf
, GFP_KERNEL
);
175 xprt
->address_strings
[RPC_DISPLAY_NETID
] = RPCBIND_NETID_RDMA6
;
179 xprt_rdma_format_addresses(struct rpc_xprt
*xprt
, struct sockaddr
*sap
)
183 switch (sap
->sa_family
) {
185 xprt_rdma_format_addresses4(xprt
, sap
);
188 xprt_rdma_format_addresses6(xprt
, sap
);
191 pr_err("rpcrdma: Unrecognized address family\n");
195 (void)rpc_ntop(sap
, buf
, sizeof(buf
));
196 xprt
->address_strings
[RPC_DISPLAY_ADDR
] = kstrdup(buf
, GFP_KERNEL
);
198 snprintf(buf
, sizeof(buf
), "%u", rpc_get_port(sap
));
199 xprt
->address_strings
[RPC_DISPLAY_PORT
] = kstrdup(buf
, GFP_KERNEL
);
201 snprintf(buf
, sizeof(buf
), "%4hx", rpc_get_port(sap
));
202 xprt
->address_strings
[RPC_DISPLAY_HEX_PORT
] = kstrdup(buf
, GFP_KERNEL
);
204 xprt
->address_strings
[RPC_DISPLAY_PROTO
] = "rdma";
208 xprt_rdma_free_addresses(struct rpc_xprt
*xprt
)
212 for (i
= 0; i
< RPC_DISPLAY_MAX
; i
++)
214 case RPC_DISPLAY_PROTO
:
215 case RPC_DISPLAY_NETID
:
218 kfree(xprt
->address_strings
[i
]);
223 xprt_rdma_connect_worker(struct work_struct
*work
)
225 struct rpcrdma_xprt
*r_xprt
= container_of(work
, struct rpcrdma_xprt
,
226 rx_connect_worker
.work
);
227 struct rpc_xprt
*xprt
= &r_xprt
->rx_xprt
;
230 xprt_clear_connected(xprt
);
232 dprintk("RPC: %s: %sconnect\n", __func__
,
233 r_xprt
->rx_ep
.rep_connected
!= 0 ? "re" : "");
234 rc
= rpcrdma_ep_connect(&r_xprt
->rx_ep
, &r_xprt
->rx_ia
);
236 xprt_wake_pending_tasks(xprt
, rc
);
238 dprintk("RPC: %s: exit\n", __func__
);
239 xprt_clear_connecting(xprt
);
243 xprt_rdma_inject_disconnect(struct rpc_xprt
*xprt
)
245 struct rpcrdma_xprt
*r_xprt
= container_of(xprt
, struct rpcrdma_xprt
,
248 pr_info("rpcrdma: injecting transport disconnect on xprt=%p\n", xprt
);
249 rdma_disconnect(r_xprt
->rx_ia
.ri_id
);
256 * Free all memory associated with the object, including its own.
257 * NOTE: none of the *destroy methods free memory for their top-level
258 * objects, even though they may have allocated it (they do free
259 * private memory). It's up to the caller to handle it. In this
260 * case (RDMA transport), all structure memory is inlined with the
261 * struct rpcrdma_xprt.
264 xprt_rdma_destroy(struct rpc_xprt
*xprt
)
266 struct rpcrdma_xprt
*r_xprt
= rpcx_to_rdmax(xprt
);
268 dprintk("RPC: %s: called\n", __func__
);
270 cancel_delayed_work_sync(&r_xprt
->rx_connect_worker
);
272 xprt_clear_connected(xprt
);
274 rpcrdma_ep_destroy(&r_xprt
->rx_ep
, &r_xprt
->rx_ia
);
275 rpcrdma_buffer_destroy(&r_xprt
->rx_buf
);
276 rpcrdma_ia_close(&r_xprt
->rx_ia
);
278 xprt_rdma_free_addresses(xprt
);
282 dprintk("RPC: %s: returning\n", __func__
);
284 module_put(THIS_MODULE
);
287 static const struct rpc_timeout xprt_rdma_default_timeout
= {
288 .to_initval
= 60 * HZ
,
289 .to_maxval
= 60 * HZ
,
293 * xprt_setup_rdma - Set up transport to use RDMA
295 * @args: rpc transport arguments
297 static struct rpc_xprt
*
298 xprt_setup_rdma(struct xprt_create
*args
)
300 struct rpcrdma_create_data_internal cdata
;
301 struct rpc_xprt
*xprt
;
302 struct rpcrdma_xprt
*new_xprt
;
303 struct rpcrdma_ep
*new_ep
;
304 struct sockaddr
*sap
;
307 if (args
->addrlen
> sizeof(xprt
->addr
)) {
308 dprintk("RPC: %s: address too large\n", __func__
);
309 return ERR_PTR(-EBADF
);
312 xprt
= xprt_alloc(args
->net
, sizeof(struct rpcrdma_xprt
),
313 xprt_rdma_slot_table_entries
,
314 xprt_rdma_slot_table_entries
);
316 dprintk("RPC: %s: couldn't allocate rpcrdma_xprt\n",
318 return ERR_PTR(-ENOMEM
);
321 /* 60 second timeout, no retries */
322 xprt
->timeout
= &xprt_rdma_default_timeout
;
323 xprt
->bind_timeout
= RPCRDMA_BIND_TO
;
324 xprt
->reestablish_timeout
= RPCRDMA_INIT_REEST_TO
;
325 xprt
->idle_timeout
= RPCRDMA_IDLE_DISC_TO
;
327 xprt
->resvport
= 0; /* privileged port not needed */
328 xprt
->tsh_size
= 0; /* RPC-RDMA handles framing */
329 xprt
->ops
= &xprt_rdma_procs
;
332 * Set up RDMA-specific connect data.
335 sap
= (struct sockaddr
*)&cdata
.addr
;
336 memcpy(sap
, args
->dstaddr
, args
->addrlen
);
338 /* Ensure xprt->addr holds valid server TCP (not RDMA)
339 * address, for any side protocols which peek at it */
340 xprt
->prot
= IPPROTO_TCP
;
341 xprt
->addrlen
= args
->addrlen
;
342 memcpy(&xprt
->addr
, sap
, xprt
->addrlen
);
344 if (rpc_get_port(sap
))
345 xprt_set_bound(xprt
);
347 cdata
.max_requests
= xprt
->max_reqs
;
349 cdata
.rsize
= RPCRDMA_MAX_SEGS
* PAGE_SIZE
; /* RDMA write max */
350 cdata
.wsize
= RPCRDMA_MAX_SEGS
* PAGE_SIZE
; /* RDMA read max */
352 cdata
.inline_wsize
= xprt_rdma_max_inline_write
;
353 if (cdata
.inline_wsize
> cdata
.wsize
)
354 cdata
.inline_wsize
= cdata
.wsize
;
356 cdata
.inline_rsize
= xprt_rdma_max_inline_read
;
357 if (cdata
.inline_rsize
> cdata
.rsize
)
358 cdata
.inline_rsize
= cdata
.rsize
;
360 cdata
.padding
= xprt_rdma_inline_write_padding
;
363 * Create new transport instance, which includes initialized
369 new_xprt
= rpcx_to_rdmax(xprt
);
371 rc
= rpcrdma_ia_open(new_xprt
, sap
, xprt_rdma_memreg_strategy
);
376 * initialize and create ep
378 new_xprt
->rx_data
= cdata
;
379 new_ep
= &new_xprt
->rx_ep
;
380 new_ep
->rep_remote_addr
= cdata
.addr
;
382 rc
= rpcrdma_ep_create(&new_xprt
->rx_ep
,
383 &new_xprt
->rx_ia
, &new_xprt
->rx_data
);
388 * Allocate pre-registered send and receive buffers for headers and
389 * any inline data. Also specify any padding which will be provided
390 * from a preregistered zero buffer.
392 rc
= rpcrdma_buffer_create(new_xprt
);
397 * Register a callback for connection events. This is necessary because
398 * connection loss notification is async. We also catch connection loss
399 * when reaping receives.
401 INIT_DELAYED_WORK(&new_xprt
->rx_connect_worker
,
402 xprt_rdma_connect_worker
);
404 xprt_rdma_format_addresses(xprt
, sap
);
405 xprt
->max_payload
= new_xprt
->rx_ia
.ri_ops
->ro_maxpages(new_xprt
);
406 if (xprt
->max_payload
== 0)
408 xprt
->max_payload
<<= PAGE_SHIFT
;
409 dprintk("RPC: %s: transport data payload maximum: %zu bytes\n",
410 __func__
, xprt
->max_payload
);
412 if (!try_module_get(THIS_MODULE
))
415 dprintk("RPC: %s: %s:%s\n", __func__
,
416 xprt
->address_strings
[RPC_DISPLAY_ADDR
],
417 xprt
->address_strings
[RPC_DISPLAY_PORT
]);
421 xprt_rdma_free_addresses(xprt
);
424 rpcrdma_ep_destroy(new_ep
, &new_xprt
->rx_ia
);
426 rpcrdma_ia_close(&new_xprt
->rx_ia
);
433 * Close a connection, during shutdown or timeout/reconnect
436 xprt_rdma_close(struct rpc_xprt
*xprt
)
438 struct rpcrdma_xprt
*r_xprt
= rpcx_to_rdmax(xprt
);
440 dprintk("RPC: %s: closing\n", __func__
);
441 if (r_xprt
->rx_ep
.rep_connected
> 0)
442 xprt
->reestablish_timeout
= 0;
443 xprt_disconnect_done(xprt
);
444 rpcrdma_ep_disconnect(&r_xprt
->rx_ep
, &r_xprt
->rx_ia
);
448 xprt_rdma_set_port(struct rpc_xprt
*xprt
, u16 port
)
450 struct sockaddr_in
*sap
;
452 sap
= (struct sockaddr_in
*)&xprt
->addr
;
453 sap
->sin_port
= htons(port
);
454 sap
= (struct sockaddr_in
*)&rpcx_to_rdmad(xprt
).addr
;
455 sap
->sin_port
= htons(port
);
456 dprintk("RPC: %s: %u\n", __func__
, port
);
460 xprt_rdma_connect(struct rpc_xprt
*xprt
, struct rpc_task
*task
)
462 struct rpcrdma_xprt
*r_xprt
= rpcx_to_rdmax(xprt
);
464 if (r_xprt
->rx_ep
.rep_connected
!= 0) {
466 schedule_delayed_work(&r_xprt
->rx_connect_worker
,
467 xprt
->reestablish_timeout
);
468 xprt
->reestablish_timeout
<<= 1;
469 if (xprt
->reestablish_timeout
> RPCRDMA_MAX_REEST_TO
)
470 xprt
->reestablish_timeout
= RPCRDMA_MAX_REEST_TO
;
471 else if (xprt
->reestablish_timeout
< RPCRDMA_INIT_REEST_TO
)
472 xprt
->reestablish_timeout
= RPCRDMA_INIT_REEST_TO
;
474 schedule_delayed_work(&r_xprt
->rx_connect_worker
, 0);
475 if (!RPC_IS_ASYNC(task
))
476 flush_delayed_work(&r_xprt
->rx_connect_worker
);
481 * The RDMA allocate/free functions need the task structure as a place
482 * to hide the struct rpcrdma_req, which is necessary for the actual send/recv
485 * The RPC layer allocates both send and receive buffers in the same call
486 * (rq_send_buf and rq_rcv_buf are both part of a single contiguous buffer).
487 * We may register rq_rcv_buf when using reply chunks.
490 xprt_rdma_allocate(struct rpc_task
*task
, size_t size
)
492 struct rpc_xprt
*xprt
= task
->tk_rqstp
->rq_xprt
;
493 struct rpcrdma_xprt
*r_xprt
= rpcx_to_rdmax(xprt
);
494 struct rpcrdma_regbuf
*rb
;
495 struct rpcrdma_req
*req
;
499 req
= rpcrdma_buffer_get(&r_xprt
->rx_buf
);
503 flags
= RPCRDMA_DEF_GFP
;
504 if (RPC_IS_SWAPPER(task
))
505 flags
= __GFP_MEMALLOC
| GFP_NOWAIT
| __GFP_NOWARN
;
507 if (req
->rl_rdmabuf
== NULL
)
509 if (req
->rl_sendbuf
== NULL
)
511 if (size
> req
->rl_sendbuf
->rg_size
)
515 dprintk("RPC: %s: size %zd, request 0x%p\n", __func__
, size
, req
);
516 req
->rl_connect_cookie
= 0; /* our reserved value */
518 return req
->rl_sendbuf
->rg_base
;
521 min_size
= RPCRDMA_INLINE_WRITE_THRESHOLD(task
->tk_rqstp
);
522 rb
= rpcrdma_alloc_regbuf(&r_xprt
->rx_ia
, min_size
, flags
);
525 req
->rl_rdmabuf
= rb
;
528 /* XDR encoding and RPC/RDMA marshaling of this request has not
529 * yet occurred. Thus a lower bound is needed to prevent buffer
530 * overrun during marshaling.
532 * RPC/RDMA marshaling may choose to send payload bearing ops
533 * inline, if the result is smaller than the inline threshold.
534 * The value of the "size" argument accounts for header
535 * requirements but not for the payload in these cases.
537 * Likewise, allocate enough space to receive a reply up to the
538 * size of the inline threshold.
540 * It's unlikely that both the send header and the received
541 * reply will be large, but slush is provided here to allow
542 * flexibility when marshaling.
544 min_size
= RPCRDMA_INLINE_READ_THRESHOLD(task
->tk_rqstp
);
545 min_size
+= RPCRDMA_INLINE_WRITE_THRESHOLD(task
->tk_rqstp
);
549 rb
= rpcrdma_alloc_regbuf(&r_xprt
->rx_ia
, size
, flags
);
554 r_xprt
->rx_stats
.hardway_register_count
+= size
;
555 rpcrdma_free_regbuf(&r_xprt
->rx_ia
, req
->rl_sendbuf
);
556 req
->rl_sendbuf
= rb
;
560 rpcrdma_buffer_put(req
);
565 * This function returns all RDMA resources to the pool.
568 xprt_rdma_free(void *buffer
)
570 struct rpcrdma_req
*req
;
571 struct rpcrdma_xprt
*r_xprt
;
572 struct rpcrdma_regbuf
*rb
;
577 rb
= container_of(buffer
, struct rpcrdma_regbuf
, rg_base
[0]);
579 if (req
->rl_backchannel
)
582 r_xprt
= container_of(req
->rl_buffer
, struct rpcrdma_xprt
, rx_buf
);
584 dprintk("RPC: %s: called on 0x%p\n", __func__
, req
->rl_reply
);
586 r_xprt
->rx_ia
.ri_ops
->ro_unmap_safe(r_xprt
, req
,
587 !RPC_IS_ASYNC(req
->rl_task
));
589 rpcrdma_buffer_put(req
);
593 * xprt_rdma_send_request - marshal and send an RPC request
594 * @task: RPC task with an RPC message in rq_snd_buf
597 * 0: The request has been sent
598 * ENOTCONN: Caller needs to invoke connect logic then call again
599 * ENOBUFS: Call again later to send the request
600 * EIO: A permanent error occurred. The request was not sent,
601 * and don't try it again
603 * send_request invokes the meat of RPC RDMA. It must do the following:
605 * 1. Marshal the RPC request into an RPC RDMA request, which means
606 * putting a header in front of data, and creating IOVs for RDMA
607 * from those in the request.
608 * 2. In marshaling, detect opportunities for RDMA, and use them.
609 * 3. Post a recv message to set up asynch completion, then send
610 * the request (rpcrdma_ep_post).
611 * 4. No partial sends are possible in the RPC-RDMA protocol (as in UDP).
614 xprt_rdma_send_request(struct rpc_task
*task
)
616 struct rpc_rqst
*rqst
= task
->tk_rqstp
;
617 struct rpc_xprt
*xprt
= rqst
->rq_xprt
;
618 struct rpcrdma_req
*req
= rpcr_to_rdmar(rqst
);
619 struct rpcrdma_xprt
*r_xprt
= rpcx_to_rdmax(xprt
);
622 rc
= rpcrdma_marshal_req(rqst
);
626 if (req
->rl_reply
== NULL
) /* e.g. reconnection */
627 rpcrdma_recv_buffer_get(req
);
629 /* Must suppress retransmit to maintain credits */
630 if (req
->rl_connect_cookie
== xprt
->connect_cookie
)
631 goto drop_connection
;
632 req
->rl_connect_cookie
= xprt
->connect_cookie
;
634 if (rpcrdma_ep_post(&r_xprt
->rx_ia
, &r_xprt
->rx_ep
, req
))
635 goto drop_connection
;
637 rqst
->rq_xmit_bytes_sent
+= rqst
->rq_snd_buf
.len
;
638 rqst
->rq_bytes_sent
= 0;
642 dprintk("RPC: %s: rpcrdma_marshal_req failed, status %i\n",
645 r_xprt
->rx_stats
.failed_marshal_count
++;
649 xprt_disconnect_done(xprt
);
650 return -ENOTCONN
; /* implies disconnect */
653 void xprt_rdma_print_stats(struct rpc_xprt
*xprt
, struct seq_file
*seq
)
655 struct rpcrdma_xprt
*r_xprt
= rpcx_to_rdmax(xprt
);
658 if (xprt_connected(xprt
))
659 idle_time
= (long)(jiffies
- xprt
->last_used
) / HZ
;
661 seq_puts(seq
, "\txprt:\trdma ");
662 seq_printf(seq
, "%u %lu %lu %lu %ld %lu %lu %lu %llu %llu ",
663 0, /* need a local port? */
664 xprt
->stat
.bind_count
,
665 xprt
->stat
.connect_count
,
666 xprt
->stat
.connect_time
,
673 seq_printf(seq
, "%lu %lu %lu %llu %llu %llu %llu %lu %lu %lu %lu ",
674 r_xprt
->rx_stats
.read_chunk_count
,
675 r_xprt
->rx_stats
.write_chunk_count
,
676 r_xprt
->rx_stats
.reply_chunk_count
,
677 r_xprt
->rx_stats
.total_rdma_request
,
678 r_xprt
->rx_stats
.total_rdma_reply
,
679 r_xprt
->rx_stats
.pullup_copy_count
,
680 r_xprt
->rx_stats
.fixup_copy_count
,
681 r_xprt
->rx_stats
.hardway_register_count
,
682 r_xprt
->rx_stats
.failed_marshal_count
,
683 r_xprt
->rx_stats
.bad_reply_count
,
684 r_xprt
->rx_stats
.nomsg_call_count
);
685 seq_printf(seq
, "%lu %lu %lu\n",
686 r_xprt
->rx_stats
.mrs_recovered
,
687 r_xprt
->rx_stats
.mrs_orphaned
,
688 r_xprt
->rx_stats
.mrs_allocated
);
692 xprt_rdma_enable_swap(struct rpc_xprt
*xprt
)
698 xprt_rdma_disable_swap(struct rpc_xprt
*xprt
)
703 * Plumbing for rpc transport switch and kernel module
706 static struct rpc_xprt_ops xprt_rdma_procs
= {
707 .reserve_xprt
= xprt_reserve_xprt_cong
,
708 .release_xprt
= xprt_release_xprt_cong
, /* sunrpc/xprt.c */
709 .alloc_slot
= xprt_alloc_slot
,
710 .release_request
= xprt_release_rqst_cong
, /* ditto */
711 .set_retrans_timeout
= xprt_set_retrans_timeout_def
, /* ditto */
712 .rpcbind
= rpcb_getport_async
, /* sunrpc/rpcb_clnt.c */
713 .set_port
= xprt_rdma_set_port
,
714 .connect
= xprt_rdma_connect
,
715 .buf_alloc
= xprt_rdma_allocate
,
716 .buf_free
= xprt_rdma_free
,
717 .send_request
= xprt_rdma_send_request
,
718 .close
= xprt_rdma_close
,
719 .destroy
= xprt_rdma_destroy
,
720 .print_stats
= xprt_rdma_print_stats
,
721 .enable_swap
= xprt_rdma_enable_swap
,
722 .disable_swap
= xprt_rdma_disable_swap
,
723 .inject_disconnect
= xprt_rdma_inject_disconnect
,
724 #if defined(CONFIG_SUNRPC_BACKCHANNEL)
725 .bc_setup
= xprt_rdma_bc_setup
,
726 .bc_up
= xprt_rdma_bc_up
,
727 .bc_maxpayload
= xprt_rdma_bc_maxpayload
,
728 .bc_free_rqst
= xprt_rdma_bc_free_rqst
,
729 .bc_destroy
= xprt_rdma_bc_destroy
,
733 static struct xprt_class xprt_rdma
= {
734 .list
= LIST_HEAD_INIT(xprt_rdma
.list
),
736 .owner
= THIS_MODULE
,
737 .ident
= XPRT_TRANSPORT_RDMA
,
738 .setup
= xprt_setup_rdma
,
741 void xprt_rdma_cleanup(void)
745 dprintk("RPCRDMA Module Removed, deregister RPC RDMA transport\n");
746 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
747 if (sunrpc_table_header
) {
748 unregister_sysctl_table(sunrpc_table_header
);
749 sunrpc_table_header
= NULL
;
752 rc
= xprt_unregister_transport(&xprt_rdma
);
754 dprintk("RPC: %s: xprt_unregister returned %i\n",
757 rpcrdma_destroy_wq();
759 rc
= xprt_unregister_transport(&xprt_rdma_bc
);
761 dprintk("RPC: %s: xprt_unregister(bc) returned %i\n",
765 int xprt_rdma_init(void)
769 rc
= rpcrdma_alloc_wq();
773 rc
= xprt_register_transport(&xprt_rdma
);
775 rpcrdma_destroy_wq();
779 rc
= xprt_register_transport(&xprt_rdma_bc
);
781 xprt_unregister_transport(&xprt_rdma
);
782 rpcrdma_destroy_wq();
786 dprintk("RPCRDMA Module Init, register RPC RDMA transport\n");
788 dprintk("Defaults:\n");
789 dprintk("\tSlots %d\n"
790 "\tMaxInlineRead %d\n\tMaxInlineWrite %d\n",
791 xprt_rdma_slot_table_entries
,
792 xprt_rdma_max_inline_read
, xprt_rdma_max_inline_write
);
793 dprintk("\tPadding %d\n\tMemreg %d\n",
794 xprt_rdma_inline_write_padding
, xprt_rdma_memreg_strategy
);
796 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
797 if (!sunrpc_table_header
)
798 sunrpc_table_header
= register_sysctl_table(sunrpc_table
);