2 * Copyright (c) 2005-2006 Network Appliance, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the BSD-type
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
14 * Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
17 * Redistributions in binary form must reproduce the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer in the documentation and/or other materials provided
20 * with the distribution.
22 * Neither the name of the Network Appliance, Inc. nor the names of
23 * its contributors may be used to endorse or promote products
24 * derived from this software without specific prior written
27 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
29 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
30 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
31 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
32 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
33 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
34 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
35 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
36 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
37 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 * Author: Tom Tucker <tom@opengridcomputing.com>
44 #include <linux/sunrpc/xdr.h>
45 #include <linux/sunrpc/svcsock.h>
46 #include <linux/sunrpc/rpc_rdma.h>
47 #include <rdma/ib_verbs.h>
48 #include <rdma/rdma_cm.h>
51 /* RPC/RDMA parameters and stats */
52 extern unsigned int svcrdma_ord
;
53 extern unsigned int svcrdma_max_requests
;
54 extern unsigned int svcrdma_max_req_size
;
56 extern atomic_t rdma_stat_recv
;
57 extern atomic_t rdma_stat_read
;
58 extern atomic_t rdma_stat_write
;
59 extern atomic_t rdma_stat_sq_starve
;
60 extern atomic_t rdma_stat_rq_starve
;
61 extern atomic_t rdma_stat_rq_poll
;
62 extern atomic_t rdma_stat_rq_prod
;
63 extern atomic_t rdma_stat_sq_poll
;
64 extern atomic_t rdma_stat_sq_prod
;
67 * Contexts are built when an RDMA request is created and are a
68 * record of the resources that can be recovered when the request
71 struct svc_rdma_op_ctxt
{
72 struct list_head free
;
73 struct svc_rdma_op_ctxt
*read_hdr
;
74 struct svc_rdma_fastreg_mr
*frmr
;
77 struct list_head dto_q
;
78 enum ib_wr_opcode wr_op
;
79 enum ib_wc_status wc_status
;
82 struct svcxprt_rdma
*xprt
;
84 enum dma_data_direction direction
;
86 struct ib_sge sge
[RPCSVC_MAXPAGES
];
87 struct page
*pages
[RPCSVC_MAXPAGES
];
91 * NFS_ requests are mapped on the client side by the chunk lists in
92 * the RPCRDMA header. During the fetching of the RPC from the client
93 * and the writing of the reply to the client, the memory in the
94 * client and the memory in the server must be mapped as contiguous
95 * vaddr/len for access by the hardware. These data strucures keep
98 * For an RDMA_WRITE, the 'sge' maps the RPC REPLY. For RDMA_READ, the
99 * 'sge' in the svc_rdma_req_map maps the server side RPC reply and the
100 * 'ch' field maps the read-list of the RPCRDMA header to the 'sge'
101 * mapping of the reply.
103 struct svc_rdma_chunk_sge
{
104 int start
; /* sge no for this chunk */
105 int count
; /* sge count for this chunk */
107 struct svc_rdma_fastreg_mr
{
109 struct scatterlist
*sg
;
111 unsigned long access_flags
;
112 enum dma_data_direction direction
;
113 struct list_head frmr_list
;
115 struct svc_rdma_req_map
{
116 struct list_head free
;
119 struct kvec sge
[RPCSVC_MAXPAGES
];
120 struct svc_rdma_chunk_sge ch
[RPCSVC_MAXPAGES
];
121 unsigned long lkey
[RPCSVC_MAXPAGES
];
124 #define RDMACTXT_F_LAST_CTXT 2
126 #define SVCRDMA_DEVCAP_FAST_REG 1 /* fast mr registration */
127 #define SVCRDMA_DEVCAP_READ_W_INV 2 /* read w/ invalidate */
129 struct svcxprt_rdma
{
130 struct svc_xprt sc_xprt
; /* SVC transport structure */
131 struct rdma_cm_id
*sc_cm_id
; /* RDMA connection id */
132 struct list_head sc_accept_q
; /* Conn. waiting accept */
133 int sc_ord
; /* RDMA read limit */
135 int sc_max_sge_rd
; /* max sge for read target */
137 int sc_sq_depth
; /* Depth of SQ */
138 atomic_t sc_sq_count
; /* Number of SQ WR on queue */
140 int sc_max_requests
; /* Depth of RQ */
141 int sc_max_req_size
; /* Size of each RQ WR buf */
145 atomic_t sc_dma_used
;
146 spinlock_t sc_ctxt_lock
;
147 struct list_head sc_ctxts
;
149 spinlock_t sc_map_lock
;
150 struct list_head sc_maps
;
152 struct list_head sc_rq_dto_q
;
153 spinlock_t sc_rq_dto_lock
;
155 struct ib_cq
*sc_rq_cq
;
156 struct ib_cq
*sc_sq_cq
;
157 struct ib_mr
*sc_phys_mr
; /* MR for server memory */
158 int (*sc_reader
)(struct svcxprt_rdma
*,
160 struct svc_rdma_op_ctxt
*,
161 int *, u32
*, u32
, u32
, u64
, bool);
162 u32 sc_dev_caps
; /* distilled device caps */
163 u32 sc_dma_lkey
; /* local dma key */
164 unsigned int sc_frmr_pg_list_len
;
165 struct list_head sc_frmr_q
;
166 spinlock_t sc_frmr_q_lock
;
168 spinlock_t sc_lock
; /* transport lock */
170 wait_queue_head_t sc_send_wait
; /* SQ exhaustion waitlist */
171 unsigned long sc_flags
;
172 struct list_head sc_dto_q
; /* DTO tasklet I/O pending Q */
173 struct list_head sc_read_complete_q
;
174 struct work_struct sc_work
;
177 #define RDMAXPRT_RQ_PENDING 1
178 #define RDMAXPRT_SQ_PENDING 2
179 #define RDMAXPRT_CONN_PENDING 3
181 #define RPCRDMA_LISTEN_BACKLOG 10
182 /* The default ORD value is based on two outstanding full-size writes with a
183 * page size of 4k, or 32k * 2 ops / 4k = 16 outstanding RDMA_READ. */
184 #define RPCRDMA_ORD (64/4)
185 #define RPCRDMA_SQ_DEPTH_MULT 8
186 #define RPCRDMA_MAX_REQUESTS 32
187 #define RPCRDMA_MAX_REQ_SIZE 4096
189 #define RPCSVC_MAXPAYLOAD_RDMA RPCSVC_MAXPAYLOAD
191 /* svc_rdma_marshal.c */
192 extern int svc_rdma_xdr_decode_req(struct rpcrdma_msg
**, struct svc_rqst
*);
193 extern int svc_rdma_xdr_encode_error(struct svcxprt_rdma
*,
194 struct rpcrdma_msg
*,
195 enum rpcrdma_errcode
, __be32
*);
196 extern void svc_rdma_xdr_encode_write_list(struct rpcrdma_msg
*, int);
197 extern void svc_rdma_xdr_encode_reply_array(struct rpcrdma_write_array
*, int);
198 extern void svc_rdma_xdr_encode_array_chunk(struct rpcrdma_write_array
*, int,
199 __be32
, __be64
, u32
);
200 extern void svc_rdma_xdr_encode_reply_header(struct svcxprt_rdma
*,
201 struct rpcrdma_msg
*,
202 struct rpcrdma_msg
*,
204 extern int svc_rdma_xdr_get_reply_hdr_len(struct rpcrdma_msg
*);
206 /* svc_rdma_recvfrom.c */
207 extern int svc_rdma_recvfrom(struct svc_rqst
*);
208 extern int rdma_read_chunk_lcl(struct svcxprt_rdma
*, struct svc_rqst
*,
209 struct svc_rdma_op_ctxt
*, int *, u32
*,
210 u32
, u32
, u64
, bool);
211 extern int rdma_read_chunk_frmr(struct svcxprt_rdma
*, struct svc_rqst
*,
212 struct svc_rdma_op_ctxt
*, int *, u32
*,
213 u32
, u32
, u64
, bool);
215 /* svc_rdma_sendto.c */
216 extern int svc_rdma_map_xdr(struct svcxprt_rdma
*, struct xdr_buf
*,
217 struct svc_rdma_req_map
*);
218 extern int svc_rdma_sendto(struct svc_rqst
*);
219 extern struct rpcrdma_read_chunk
*
220 svc_rdma_get_read_chunk(struct rpcrdma_msg
*);
222 /* svc_rdma_transport.c */
223 extern int svc_rdma_send(struct svcxprt_rdma
*, struct ib_send_wr
*);
224 extern void svc_rdma_send_error(struct svcxprt_rdma
*, struct rpcrdma_msg
*,
225 enum rpcrdma_errcode
);
226 extern int svc_rdma_post_recv(struct svcxprt_rdma
*, gfp_t
);
227 extern int svc_rdma_create_listen(struct svc_serv
*, int, struct sockaddr
*);
228 extern struct svc_rdma_op_ctxt
*svc_rdma_get_context(struct svcxprt_rdma
*);
229 extern void svc_rdma_put_context(struct svc_rdma_op_ctxt
*, int);
230 extern void svc_rdma_unmap_dma(struct svc_rdma_op_ctxt
*ctxt
);
231 extern struct svc_rdma_req_map
*svc_rdma_get_req_map(struct svcxprt_rdma
*);
232 extern void svc_rdma_put_req_map(struct svcxprt_rdma
*,
233 struct svc_rdma_req_map
*);
234 extern struct svc_rdma_fastreg_mr
*svc_rdma_get_frmr(struct svcxprt_rdma
*);
235 extern void svc_rdma_put_frmr(struct svcxprt_rdma
*,
236 struct svc_rdma_fastreg_mr
*);
237 extern void svc_sq_reap(struct svcxprt_rdma
*);
238 extern void svc_rq_reap(struct svcxprt_rdma
*);
239 extern void svc_rdma_prep_reply_hdr(struct svc_rqst
*);
241 extern struct svc_xprt_class svc_rdma_class
;
242 #ifdef CONFIG_SUNRPC_BACKCHANNEL
243 extern struct svc_xprt_class svc_rdma_bc_class
;
247 extern struct workqueue_struct
*svc_rdma_wq
;
248 extern int svc_rdma_init(void);
249 extern void svc_rdma_cleanup(void);