2 * Copyright (c) 2005-2006 Network Appliance, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the BSD-type
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
14 * Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
17 * Redistributions in binary form must reproduce the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer in the documentation and/or other materials provided
20 * with the distribution.
22 * Neither the name of the Network Appliance, Inc. nor the names of
23 * its contributors may be used to endorse or promote products
24 * derived from this software without specific prior written
27 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
29 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
30 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
31 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
32 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
33 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
34 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
35 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
36 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
37 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 * Author: Tom Tucker <tom@opengridcomputing.com>
44 #include <linux/sunrpc/xdr.h>
45 #include <linux/sunrpc/svcsock.h>
46 #include <linux/sunrpc/rpc_rdma.h>
47 #include <rdma/ib_verbs.h>
48 #include <rdma/rdma_cm.h>
51 /* RPC/RDMA parameters and stats */
52 extern unsigned int svcrdma_ord
;
53 extern unsigned int svcrdma_max_requests
;
54 extern unsigned int svcrdma_max_bc_requests
;
55 extern unsigned int svcrdma_max_req_size
;
57 extern atomic_t rdma_stat_recv
;
58 extern atomic_t rdma_stat_read
;
59 extern atomic_t rdma_stat_write
;
60 extern atomic_t rdma_stat_sq_starve
;
61 extern atomic_t rdma_stat_rq_starve
;
62 extern atomic_t rdma_stat_rq_poll
;
63 extern atomic_t rdma_stat_rq_prod
;
64 extern atomic_t rdma_stat_sq_poll
;
65 extern atomic_t rdma_stat_sq_prod
;
68 * Contexts are built when an RDMA request is created and are a
69 * record of the resources that can be recovered when the request
72 struct svc_rdma_op_ctxt
{
73 struct list_head free
;
74 struct svc_rdma_op_ctxt
*read_hdr
;
75 struct svc_rdma_fastreg_mr
*frmr
;
78 struct list_head dto_q
;
79 enum ib_wr_opcode wr_op
;
80 enum ib_wc_status wc_status
;
83 struct svcxprt_rdma
*xprt
;
85 enum dma_data_direction direction
;
87 struct ib_sge sge
[RPCSVC_MAXPAGES
];
88 struct page
*pages
[RPCSVC_MAXPAGES
];
92 * NFS_ requests are mapped on the client side by the chunk lists in
93 * the RPCRDMA header. During the fetching of the RPC from the client
94 * and the writing of the reply to the client, the memory in the
95 * client and the memory in the server must be mapped as contiguous
96 * vaddr/len for access by the hardware. These data strucures keep
99 * For an RDMA_WRITE, the 'sge' maps the RPC REPLY. For RDMA_READ, the
100 * 'sge' in the svc_rdma_req_map maps the server side RPC reply and the
101 * 'ch' field maps the read-list of the RPCRDMA header to the 'sge'
102 * mapping of the reply.
104 struct svc_rdma_chunk_sge
{
105 int start
; /* sge no for this chunk */
106 int count
; /* sge count for this chunk */
108 struct svc_rdma_fastreg_mr
{
110 struct scatterlist
*sg
;
112 unsigned long access_flags
;
113 enum dma_data_direction direction
;
114 struct list_head frmr_list
;
116 struct svc_rdma_req_map
{
117 struct list_head free
;
120 struct kvec sge
[RPCSVC_MAXPAGES
];
121 struct svc_rdma_chunk_sge ch
[RPCSVC_MAXPAGES
];
122 unsigned long lkey
[RPCSVC_MAXPAGES
];
125 #define RDMACTXT_F_LAST_CTXT 2
127 #define SVCRDMA_DEVCAP_FAST_REG 1 /* fast mr registration */
128 #define SVCRDMA_DEVCAP_READ_W_INV 2 /* read w/ invalidate */
130 struct svcxprt_rdma
{
131 struct svc_xprt sc_xprt
; /* SVC transport structure */
132 struct rdma_cm_id
*sc_cm_id
; /* RDMA connection id */
133 struct list_head sc_accept_q
; /* Conn. waiting accept */
134 int sc_ord
; /* RDMA read limit */
136 int sc_max_sge_rd
; /* max sge for read target */
138 atomic_t sc_sq_count
; /* Number of SQ WR on queue */
139 unsigned int sc_sq_depth
; /* Depth of SQ */
140 unsigned int sc_rq_depth
; /* Depth of RQ */
141 u32 sc_max_requests
; /* Forward credits */
142 u32 sc_max_bc_requests
;/* Backward credits */
143 int sc_max_req_size
; /* Size of each RQ WR buf */
147 atomic_t sc_dma_used
;
148 spinlock_t sc_ctxt_lock
;
149 struct list_head sc_ctxts
;
151 spinlock_t sc_map_lock
;
152 struct list_head sc_maps
;
154 struct list_head sc_rq_dto_q
;
155 spinlock_t sc_rq_dto_lock
;
157 struct ib_cq
*sc_rq_cq
;
158 struct ib_cq
*sc_sq_cq
;
159 int (*sc_reader
)(struct svcxprt_rdma
*,
161 struct svc_rdma_op_ctxt
*,
162 int *, u32
*, u32
, u32
, u64
, bool);
163 u32 sc_dev_caps
; /* distilled device caps */
164 unsigned int sc_frmr_pg_list_len
;
165 struct list_head sc_frmr_q
;
166 spinlock_t sc_frmr_q_lock
;
168 spinlock_t sc_lock
; /* transport lock */
170 wait_queue_head_t sc_send_wait
; /* SQ exhaustion waitlist */
171 unsigned long sc_flags
;
172 struct list_head sc_dto_q
; /* DTO tasklet I/O pending Q */
173 struct list_head sc_read_complete_q
;
174 struct work_struct sc_work
;
177 #define RDMAXPRT_RQ_PENDING 1
178 #define RDMAXPRT_SQ_PENDING 2
179 #define RDMAXPRT_CONN_PENDING 3
181 #define RPCRDMA_LISTEN_BACKLOG 10
182 /* The default ORD value is based on two outstanding full-size writes with a
183 * page size of 4k, or 32k * 2 ops / 4k = 16 outstanding RDMA_READ. */
184 #define RPCRDMA_ORD (64/4)
185 #define RPCRDMA_SQ_DEPTH_MULT 8
186 #define RPCRDMA_MAX_REQUESTS 32
187 #define RPCRDMA_MAX_REQ_SIZE 4096
189 /* Typical ULP usage of BC requests is NFSv4.1 backchannel. Our
190 * current NFSv4.1 implementation supports one backchannel slot.
192 #define RPCRDMA_MAX_BC_REQUESTS 2
194 #define RPCSVC_MAXPAYLOAD_RDMA RPCSVC_MAXPAYLOAD
196 /* svc_rdma_backchannel.c */
197 extern int svc_rdma_handle_bc_reply(struct rpc_xprt
*xprt
,
198 struct rpcrdma_msg
*rmsgp
,
199 struct xdr_buf
*rcvbuf
);
201 /* svc_rdma_marshal.c */
202 extern int svc_rdma_xdr_decode_req(struct rpcrdma_msg
**, struct svc_rqst
*);
203 extern int svc_rdma_xdr_encode_error(struct svcxprt_rdma
*,
204 struct rpcrdma_msg
*,
205 enum rpcrdma_errcode
, __be32
*);
206 extern void svc_rdma_xdr_encode_write_list(struct rpcrdma_msg
*, int);
207 extern void svc_rdma_xdr_encode_reply_array(struct rpcrdma_write_array
*, int);
208 extern void svc_rdma_xdr_encode_array_chunk(struct rpcrdma_write_array
*, int,
209 __be32
, __be64
, u32
);
210 extern void svc_rdma_xdr_encode_reply_header(struct svcxprt_rdma
*,
211 struct rpcrdma_msg
*,
212 struct rpcrdma_msg
*,
214 extern int svc_rdma_xdr_get_reply_hdr_len(struct rpcrdma_msg
*);
216 /* svc_rdma_recvfrom.c */
217 extern int svc_rdma_recvfrom(struct svc_rqst
*);
218 extern int rdma_read_chunk_lcl(struct svcxprt_rdma
*, struct svc_rqst
*,
219 struct svc_rdma_op_ctxt
*, int *, u32
*,
220 u32
, u32
, u64
, bool);
221 extern int rdma_read_chunk_frmr(struct svcxprt_rdma
*, struct svc_rqst
*,
222 struct svc_rdma_op_ctxt
*, int *, u32
*,
223 u32
, u32
, u64
, bool);
225 /* svc_rdma_sendto.c */
226 extern int svc_rdma_map_xdr(struct svcxprt_rdma
*, struct xdr_buf
*,
227 struct svc_rdma_req_map
*);
228 extern int svc_rdma_sendto(struct svc_rqst
*);
229 extern struct rpcrdma_read_chunk
*
230 svc_rdma_get_read_chunk(struct rpcrdma_msg
*);
232 /* svc_rdma_transport.c */
233 extern int svc_rdma_send(struct svcxprt_rdma
*, struct ib_send_wr
*);
234 extern void svc_rdma_send_error(struct svcxprt_rdma
*, struct rpcrdma_msg
*,
235 enum rpcrdma_errcode
);
236 extern int svc_rdma_post_recv(struct svcxprt_rdma
*, gfp_t
);
237 extern int svc_rdma_create_listen(struct svc_serv
*, int, struct sockaddr
*);
238 extern struct svc_rdma_op_ctxt
*svc_rdma_get_context(struct svcxprt_rdma
*);
239 extern void svc_rdma_put_context(struct svc_rdma_op_ctxt
*, int);
240 extern void svc_rdma_unmap_dma(struct svc_rdma_op_ctxt
*ctxt
);
241 extern struct svc_rdma_req_map
*svc_rdma_get_req_map(struct svcxprt_rdma
*);
242 extern void svc_rdma_put_req_map(struct svcxprt_rdma
*,
243 struct svc_rdma_req_map
*);
244 extern struct svc_rdma_fastreg_mr
*svc_rdma_get_frmr(struct svcxprt_rdma
*);
245 extern void svc_rdma_put_frmr(struct svcxprt_rdma
*,
246 struct svc_rdma_fastreg_mr
*);
247 extern void svc_sq_reap(struct svcxprt_rdma
*);
248 extern void svc_rq_reap(struct svcxprt_rdma
*);
249 extern void svc_rdma_prep_reply_hdr(struct svc_rqst
*);
251 extern struct svc_xprt_class svc_rdma_class
;
252 #ifdef CONFIG_SUNRPC_BACKCHANNEL
253 extern struct svc_xprt_class svc_rdma_bc_class
;
257 extern struct workqueue_struct
*svc_rdma_wq
;
258 extern int svc_rdma_init(void);
259 extern void svc_rdma_cleanup(void);