Commit | Line | Data |
---|---|---|
c06b540a | 1 | /* |
0bf48289 | 2 | * Copyright (c) 2014 Open Grid Computing, Inc. All rights reserved. |
c06b540a TT |
3 | * Copyright (c) 2005-2006 Network Appliance, Inc. All rights reserved. |
4 | * | |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the BSD-type | |
9 | * license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or without | |
12 | * modification, are permitted provided that the following conditions | |
13 | * are met: | |
14 | * | |
15 | * Redistributions of source code must retain the above copyright | |
16 | * notice, this list of conditions and the following disclaimer. | |
17 | * | |
18 | * Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials provided | |
21 | * with the distribution. | |
22 | * | |
23 | * Neither the name of the Network Appliance, Inc. nor the names of | |
24 | * its contributors may be used to endorse or promote products | |
25 | * derived from this software without specific prior written | |
26 | * permission. | |
27 | * | |
28 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
29 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
30 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
31 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
32 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
33 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
34 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
35 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
36 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
37 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
38 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
39 | * | |
40 | * Author: Tom Tucker <tom@opengridcomputing.com> | |
41 | */ | |
42 | ||
43 | #include <linux/sunrpc/debug.h> | |
44 | #include <linux/sunrpc/rpc_rdma.h> | |
45 | #include <linux/spinlock.h> | |
46 | #include <asm/unaligned.h> | |
47 | #include <rdma/ib_verbs.h> | |
48 | #include <rdma/rdma_cm.h> | |
49 | #include <linux/sunrpc/svc_rdma.h> | |
50 | ||
51 | #define RPCDBG_FACILITY RPCDBG_SVCXPRT | |
52 | ||
cf570a93 CL |
53 | static u32 xdr_padsize(u32 len) |
54 | { | |
55 | return (len & 3) ? (4 - (len & 3)) : 0; | |
56 | } | |
57 | ||
ba986c96 CL |
58 | int svc_rdma_map_xdr(struct svcxprt_rdma *xprt, |
59 | struct xdr_buf *xdr, | |
f6763c29 CL |
60 | struct svc_rdma_req_map *vec, |
61 | bool write_chunk_present) | |
c06b540a | 62 | { |
c06b540a | 63 | int sge_no; |
c06b540a TT |
64 | u32 sge_bytes; |
65 | u32 page_bytes; | |
34d16e42 | 66 | u32 page_off; |
c06b540a TT |
67 | int page_no; |
68 | ||
3fe04ee9 CL |
69 | if (xdr->len != |
70 | (xdr->head[0].iov_len + xdr->page_len + xdr->tail[0].iov_len)) { | |
ba986c96 | 71 | pr_err("svcrdma: %s: XDR buffer length error\n", __func__); |
3fe04ee9 CL |
72 | return -EIO; |
73 | } | |
34d16e42 | 74 | |
c06b540a TT |
75 | /* Skip the first sge, this is for the RPCRDMA header */ |
76 | sge_no = 1; | |
77 | ||
78 | /* Head SGE */ | |
34d16e42 TT |
79 | vec->sge[sge_no].iov_base = xdr->head[0].iov_base; |
80 | vec->sge[sge_no].iov_len = xdr->head[0].iov_len; | |
c06b540a TT |
81 | sge_no++; |
82 | ||
83 | /* pages SGE */ | |
84 | page_no = 0; | |
85 | page_bytes = xdr->page_len; | |
86 | page_off = xdr->page_base; | |
34d16e42 TT |
87 | while (page_bytes) { |
88 | vec->sge[sge_no].iov_base = | |
89 | page_address(xdr->pages[page_no]) + page_off; | |
90 | sge_bytes = min_t(u32, page_bytes, (PAGE_SIZE - page_off)); | |
c06b540a | 91 | page_bytes -= sge_bytes; |
34d16e42 | 92 | vec->sge[sge_no].iov_len = sge_bytes; |
c06b540a TT |
93 | |
94 | sge_no++; | |
95 | page_no++; | |
96 | page_off = 0; /* reset for next time through loop */ | |
97 | } | |
98 | ||
99 | /* Tail SGE */ | |
34d16e42 | 100 | if (xdr->tail[0].iov_len) { |
f6763c29 CL |
101 | unsigned char *base = xdr->tail[0].iov_base; |
102 | size_t len = xdr->tail[0].iov_len; | |
103 | u32 xdr_pad = xdr_padsize(xdr->page_len); | |
104 | ||
105 | if (write_chunk_present && xdr_pad) { | |
106 | base += xdr_pad; | |
107 | len -= xdr_pad; | |
108 | } | |
109 | ||
110 | if (len) { | |
111 | vec->sge[sge_no].iov_base = base; | |
112 | vec->sge[sge_no].iov_len = len; | |
113 | sge_no++; | |
114 | } | |
c06b540a TT |
115 | } |
116 | ||
ba986c96 | 117 | dprintk("svcrdma: %s: sge_no %d page_no %d " |
2e3c230b | 118 | "page_base %u page_len %u head_len %zu tail_len %zu\n", |
ba986c96 | 119 | __func__, sge_no, page_no, xdr->page_base, xdr->page_len, |
b1e1e158 TT |
120 | xdr->head[0].iov_len, xdr->tail[0].iov_len); |
121 | ||
34d16e42 | 122 | vec->count = sge_no; |
afd566ea | 123 | return 0; |
c06b540a TT |
124 | } |
125 | ||
b432e6b3 TT |
126 | static dma_addr_t dma_map_xdr(struct svcxprt_rdma *xprt, |
127 | struct xdr_buf *xdr, | |
128 | u32 xdr_off, size_t len, int dir) | |
129 | { | |
130 | struct page *page; | |
131 | dma_addr_t dma_addr; | |
132 | if (xdr_off < xdr->head[0].iov_len) { | |
133 | /* This offset is in the head */ | |
134 | xdr_off += (unsigned long)xdr->head[0].iov_base & ~PAGE_MASK; | |
135 | page = virt_to_page(xdr->head[0].iov_base); | |
136 | } else { | |
137 | xdr_off -= xdr->head[0].iov_len; | |
138 | if (xdr_off < xdr->page_len) { | |
139 | /* This offset is in the page list */ | |
3cbe01a9 | 140 | xdr_off += xdr->page_base; |
b432e6b3 TT |
141 | page = xdr->pages[xdr_off >> PAGE_SHIFT]; |
142 | xdr_off &= ~PAGE_MASK; | |
143 | } else { | |
144 | /* This offset is in the tail */ | |
145 | xdr_off -= xdr->page_len; | |
146 | xdr_off += (unsigned long) | |
147 | xdr->tail[0].iov_base & ~PAGE_MASK; | |
148 | page = virt_to_page(xdr->tail[0].iov_base); | |
149 | } | |
150 | } | |
151 | dma_addr = ib_dma_map_page(xprt->sc_cm_id->device, page, xdr_off, | |
152 | min_t(size_t, PAGE_SIZE, len), dir); | |
153 | return dma_addr; | |
154 | } | |
155 | ||
10dc4512 CL |
156 | /* Returns the address of the first read chunk or <nul> if no read chunk |
157 | * is present | |
158 | */ | |
159 | struct rpcrdma_read_chunk * | |
160 | svc_rdma_get_read_chunk(struct rpcrdma_msg *rmsgp) | |
161 | { | |
162 | struct rpcrdma_read_chunk *ch = | |
163 | (struct rpcrdma_read_chunk *)&rmsgp->rm_body.rm_chunks[0]; | |
164 | ||
165 | if (ch->rc_discrim == xdr_zero) | |
166 | return NULL; | |
167 | return ch; | |
168 | } | |
169 | ||
170 | /* Returns the address of the first read write array element or <nul> | |
171 | * if no write array list is present | |
172 | */ | |
173 | static struct rpcrdma_write_array * | |
174 | svc_rdma_get_write_array(struct rpcrdma_msg *rmsgp) | |
175 | { | |
176 | if (rmsgp->rm_body.rm_chunks[0] != xdr_zero || | |
177 | rmsgp->rm_body.rm_chunks[1] == xdr_zero) | |
178 | return NULL; | |
179 | return (struct rpcrdma_write_array *)&rmsgp->rm_body.rm_chunks[1]; | |
180 | } | |
181 | ||
182 | /* Returns the address of the first reply array element or <nul> if no | |
183 | * reply array is present | |
184 | */ | |
185 | static struct rpcrdma_write_array * | |
08ae4e7f CL |
186 | svc_rdma_get_reply_array(struct rpcrdma_msg *rmsgp, |
187 | struct rpcrdma_write_array *wr_ary) | |
10dc4512 CL |
188 | { |
189 | struct rpcrdma_read_chunk *rch; | |
10dc4512 CL |
190 | struct rpcrdma_write_array *rp_ary; |
191 | ||
192 | /* XXX: Need to fix when reply chunk may occur with read list | |
193 | * and/or write list. | |
194 | */ | |
195 | if (rmsgp->rm_body.rm_chunks[0] != xdr_zero || | |
196 | rmsgp->rm_body.rm_chunks[1] != xdr_zero) | |
197 | return NULL; | |
198 | ||
199 | rch = svc_rdma_get_read_chunk(rmsgp); | |
200 | if (rch) { | |
201 | while (rch->rc_discrim != xdr_zero) | |
202 | rch++; | |
203 | ||
204 | /* The reply chunk follows an empty write array located | |
205 | * at 'rc_position' here. The reply array is at rc_target. | |
206 | */ | |
207 | rp_ary = (struct rpcrdma_write_array *)&rch->rc_target; | |
208 | goto found_it; | |
209 | } | |
210 | ||
10dc4512 CL |
211 | if (wr_ary) { |
212 | int chunk = be32_to_cpu(wr_ary->wc_nchunks); | |
213 | ||
214 | rp_ary = (struct rpcrdma_write_array *) | |
215 | &wr_ary->wc_array[chunk].wc_target.rs_length; | |
216 | goto found_it; | |
217 | } | |
218 | ||
219 | /* No read list, no write list */ | |
220 | rp_ary = (struct rpcrdma_write_array *)&rmsgp->rm_body.rm_chunks[2]; | |
221 | ||
222 | found_it: | |
223 | if (rp_ary->wc_discrim == xdr_zero) | |
224 | return NULL; | |
225 | return rp_ary; | |
226 | } | |
227 | ||
c06b540a TT |
228 | /* Assumptions: |
229 | * - The specified write_len can be represented in sc_max_sge * PAGE_SIZE | |
230 | */ | |
231 | static int send_write(struct svcxprt_rdma *xprt, struct svc_rqst *rqstp, | |
232 | u32 rmr, u64 to, | |
233 | u32 xdr_off, int write_len, | |
34d16e42 | 234 | struct svc_rdma_req_map *vec) |
c06b540a | 235 | { |
e622f2f4 | 236 | struct ib_rdma_wr write_wr; |
c06b540a TT |
237 | struct ib_sge *sge; |
238 | int xdr_sge_no; | |
239 | int sge_no; | |
240 | int sge_bytes; | |
241 | int sge_off; | |
242 | int bc; | |
243 | struct svc_rdma_op_ctxt *ctxt; | |
c06b540a | 244 | |
3fe04ee9 CL |
245 | if (vec->count > RPCSVC_MAXPAGES) { |
246 | pr_err("svcrdma: Too many pages (%lu)\n", vec->count); | |
247 | return -EIO; | |
248 | } | |
249 | ||
c06b540a | 250 | dprintk("svcrdma: RDMA_WRITE rmr=%x, to=%llx, xdr_off=%d, " |
34d16e42 | 251 | "write_len=%d, vec->sge=%p, vec->count=%lu\n", |
bb50c801 | 252 | rmr, (unsigned long long)to, xdr_off, |
34d16e42 | 253 | write_len, vec->sge, vec->count); |
c06b540a TT |
254 | |
255 | ctxt = svc_rdma_get_context(xprt); | |
34d16e42 TT |
256 | ctxt->direction = DMA_TO_DEVICE; |
257 | sge = ctxt->sge; | |
c06b540a TT |
258 | |
259 | /* Find the SGE associated with xdr_off */ | |
34d16e42 | 260 | for (bc = xdr_off, xdr_sge_no = 1; bc && xdr_sge_no < vec->count; |
c06b540a | 261 | xdr_sge_no++) { |
34d16e42 | 262 | if (vec->sge[xdr_sge_no].iov_len > bc) |
c06b540a | 263 | break; |
34d16e42 | 264 | bc -= vec->sge[xdr_sge_no].iov_len; |
c06b540a TT |
265 | } |
266 | ||
267 | sge_off = bc; | |
268 | bc = write_len; | |
269 | sge_no = 0; | |
270 | ||
271 | /* Copy the remaining SGE */ | |
afd566ea TT |
272 | while (bc != 0) { |
273 | sge_bytes = min_t(size_t, | |
274 | bc, vec->sge[xdr_sge_no].iov_len-sge_off); | |
c06b540a | 275 | sge[sge_no].length = sge_bytes; |
0bf48289 SW |
276 | sge[sge_no].addr = |
277 | dma_map_xdr(xprt, &rqstp->rq_res, xdr_off, | |
278 | sge_bytes, DMA_TO_DEVICE); | |
279 | xdr_off += sge_bytes; | |
280 | if (ib_dma_mapping_error(xprt->sc_cm_id->device, | |
281 | sge[sge_no].addr)) | |
282 | goto err; | |
283 | atomic_inc(&xprt->sc_dma_used); | |
5fe1043d | 284 | sge[sge_no].lkey = xprt->sc_pd->local_dma_lkey; |
afd566ea | 285 | ctxt->count++; |
c06b540a TT |
286 | sge_off = 0; |
287 | sge_no++; | |
288 | xdr_sge_no++; | |
3fe04ee9 CL |
289 | if (xdr_sge_no > vec->count) { |
290 | pr_err("svcrdma: Too many sges (%d)\n", xdr_sge_no); | |
291 | goto err; | |
292 | } | |
c06b540a | 293 | bc -= sge_bytes; |
25594290 SW |
294 | if (sge_no == xprt->sc_max_sge) |
295 | break; | |
c06b540a TT |
296 | } |
297 | ||
c06b540a TT |
298 | /* Prepare WRITE WR */ |
299 | memset(&write_wr, 0, sizeof write_wr); | |
be99bb11 CL |
300 | ctxt->cqe.done = svc_rdma_wc_write; |
301 | write_wr.wr.wr_cqe = &ctxt->cqe; | |
e622f2f4 CH |
302 | write_wr.wr.sg_list = &sge[0]; |
303 | write_wr.wr.num_sge = sge_no; | |
304 | write_wr.wr.opcode = IB_WR_RDMA_WRITE; | |
305 | write_wr.wr.send_flags = IB_SEND_SIGNALED; | |
306 | write_wr.rkey = rmr; | |
307 | write_wr.remote_addr = to; | |
c06b540a TT |
308 | |
309 | /* Post It */ | |
310 | atomic_inc(&rdma_stat_write); | |
e622f2f4 | 311 | if (svc_rdma_send(xprt, &write_wr.wr)) |
34d16e42 | 312 | goto err; |
25594290 | 313 | return write_len - bc; |
34d16e42 | 314 | err: |
4a84386f | 315 | svc_rdma_unmap_dma(ctxt); |
34d16e42 | 316 | svc_rdma_put_context(ctxt, 0); |
34d16e42 | 317 | return -EIO; |
c06b540a TT |
318 | } |
319 | ||
08ae4e7f | 320 | noinline |
c06b540a | 321 | static int send_write_chunks(struct svcxprt_rdma *xprt, |
08ae4e7f | 322 | struct rpcrdma_write_array *wr_ary, |
c06b540a TT |
323 | struct rpcrdma_msg *rdma_resp, |
324 | struct svc_rqst *rqstp, | |
34d16e42 | 325 | struct svc_rdma_req_map *vec) |
c06b540a | 326 | { |
cf570a93 | 327 | u32 xfer_len = rqstp->rq_res.page_len; |
c06b540a | 328 | int write_len; |
c06b540a TT |
329 | u32 xdr_off; |
330 | int chunk_off; | |
331 | int chunk_no; | |
70747c25 | 332 | int nchunks; |
c06b540a TT |
333 | struct rpcrdma_write_array *res_ary; |
334 | int ret; | |
335 | ||
c06b540a TT |
336 | res_ary = (struct rpcrdma_write_array *) |
337 | &rdma_resp->rm_body.rm_chunks[1]; | |
338 | ||
c06b540a | 339 | /* Write chunks start at the pagelist */ |
08ae4e7f | 340 | nchunks = be32_to_cpu(wr_ary->wc_nchunks); |
c06b540a | 341 | for (xdr_off = rqstp->rq_res.head[0].iov_len, chunk_no = 0; |
70747c25 | 342 | xfer_len && chunk_no < nchunks; |
c06b540a TT |
343 | chunk_no++) { |
344 | struct rpcrdma_segment *arg_ch; | |
345 | u64 rs_offset; | |
346 | ||
08ae4e7f | 347 | arg_ch = &wr_ary->wc_array[chunk_no].wc_target; |
70747c25 | 348 | write_len = min(xfer_len, be32_to_cpu(arg_ch->rs_length)); |
c06b540a TT |
349 | |
350 | /* Prepare the response chunk given the length actually | |
351 | * written */ | |
cec56c8f | 352 | xdr_decode_hyper((__be32 *)&arg_ch->rs_offset, &rs_offset); |
c06b540a | 353 | svc_rdma_xdr_encode_array_chunk(res_ary, chunk_no, |
cec56c8f TT |
354 | arg_ch->rs_handle, |
355 | arg_ch->rs_offset, | |
356 | write_len); | |
c06b540a TT |
357 | chunk_off = 0; |
358 | while (write_len) { | |
c06b540a | 359 | ret = send_write(xprt, rqstp, |
70747c25 | 360 | be32_to_cpu(arg_ch->rs_handle), |
c06b540a TT |
361 | rs_offset + chunk_off, |
362 | xdr_off, | |
25594290 | 363 | write_len, |
34d16e42 | 364 | vec); |
08ae4e7f CL |
365 | if (ret <= 0) |
366 | goto out_err; | |
25594290 SW |
367 | chunk_off += ret; |
368 | xdr_off += ret; | |
369 | xfer_len -= ret; | |
370 | write_len -= ret; | |
c06b540a TT |
371 | } |
372 | } | |
373 | /* Update the req with the number of chunks actually used */ | |
374 | svc_rdma_xdr_encode_write_list(rdma_resp, chunk_no); | |
375 | ||
cf570a93 | 376 | return rqstp->rq_res.page_len; |
08ae4e7f CL |
377 | |
378 | out_err: | |
379 | pr_err("svcrdma: failed to send write chunks, rc=%d\n", ret); | |
380 | return -EIO; | |
c06b540a TT |
381 | } |
382 | ||
08ae4e7f | 383 | noinline |
c06b540a | 384 | static int send_reply_chunks(struct svcxprt_rdma *xprt, |
08ae4e7f | 385 | struct rpcrdma_write_array *rp_ary, |
c06b540a TT |
386 | struct rpcrdma_msg *rdma_resp, |
387 | struct svc_rqst *rqstp, | |
34d16e42 | 388 | struct svc_rdma_req_map *vec) |
c06b540a TT |
389 | { |
390 | u32 xfer_len = rqstp->rq_res.len; | |
391 | int write_len; | |
c06b540a TT |
392 | u32 xdr_off; |
393 | int chunk_no; | |
394 | int chunk_off; | |
cec56c8f | 395 | int nchunks; |
c06b540a | 396 | struct rpcrdma_segment *ch; |
c06b540a TT |
397 | struct rpcrdma_write_array *res_ary; |
398 | int ret; | |
399 | ||
c06b540a TT |
400 | /* XXX: need to fix when reply lists occur with read-list and or |
401 | * write-list */ | |
402 | res_ary = (struct rpcrdma_write_array *) | |
403 | &rdma_resp->rm_body.rm_chunks[2]; | |
404 | ||
c06b540a | 405 | /* xdr offset starts at RPC message */ |
08ae4e7f | 406 | nchunks = be32_to_cpu(rp_ary->wc_nchunks); |
c06b540a | 407 | for (xdr_off = 0, chunk_no = 0; |
cec56c8f | 408 | xfer_len && chunk_no < nchunks; |
c06b540a TT |
409 | chunk_no++) { |
410 | u64 rs_offset; | |
08ae4e7f | 411 | ch = &rp_ary->wc_array[chunk_no].wc_target; |
70747c25 | 412 | write_len = min(xfer_len, be32_to_cpu(ch->rs_length)); |
c06b540a | 413 | |
c06b540a TT |
414 | /* Prepare the reply chunk given the length actually |
415 | * written */ | |
cec56c8f | 416 | xdr_decode_hyper((__be32 *)&ch->rs_offset, &rs_offset); |
c06b540a | 417 | svc_rdma_xdr_encode_array_chunk(res_ary, chunk_no, |
cec56c8f TT |
418 | ch->rs_handle, ch->rs_offset, |
419 | write_len); | |
c06b540a TT |
420 | chunk_off = 0; |
421 | while (write_len) { | |
c06b540a | 422 | ret = send_write(xprt, rqstp, |
70747c25 | 423 | be32_to_cpu(ch->rs_handle), |
c06b540a TT |
424 | rs_offset + chunk_off, |
425 | xdr_off, | |
25594290 | 426 | write_len, |
34d16e42 | 427 | vec); |
08ae4e7f CL |
428 | if (ret <= 0) |
429 | goto out_err; | |
25594290 SW |
430 | chunk_off += ret; |
431 | xdr_off += ret; | |
432 | xfer_len -= ret; | |
433 | write_len -= ret; | |
c06b540a TT |
434 | } |
435 | } | |
436 | /* Update the req with the number of chunks actually used */ | |
437 | svc_rdma_xdr_encode_reply_array(res_ary, chunk_no); | |
438 | ||
439 | return rqstp->rq_res.len; | |
08ae4e7f CL |
440 | |
441 | out_err: | |
442 | pr_err("svcrdma: failed to send reply chunks, rc=%d\n", ret); | |
443 | return -EIO; | |
c06b540a TT |
444 | } |
445 | ||
446 | /* This function prepares the portion of the RPCRDMA message to be | |
447 | * sent in the RDMA_SEND. This function is called after data sent via | |
448 | * RDMA has already been transmitted. There are three cases: | |
449 | * - The RPCRDMA header, RPC header, and payload are all sent in a | |
450 | * single RDMA_SEND. This is the "inline" case. | |
451 | * - The RPCRDMA header and some portion of the RPC header and data | |
452 | * are sent via this RDMA_SEND and another portion of the data is | |
453 | * sent via RDMA. | |
454 | * - The RPCRDMA header [NOMSG] is sent in this RDMA_SEND and the RPC | |
455 | * header and data are all transmitted via RDMA. | |
456 | * In all three cases, this function prepares the RPCRDMA header in | |
457 | * sge[0], the 'type' parameter indicates the type to place in the | |
458 | * RPCRDMA header, and the 'byte_count' field indicates how much of | |
b432e6b3 TT |
459 | * the XDR to include in this RDMA_SEND. NB: The offset of the payload |
460 | * to send is zero in the XDR. | |
c06b540a TT |
461 | */ |
462 | static int send_reply(struct svcxprt_rdma *rdma, | |
463 | struct svc_rqst *rqstp, | |
464 | struct page *page, | |
465 | struct rpcrdma_msg *rdma_resp, | |
466 | struct svc_rdma_op_ctxt *ctxt, | |
34d16e42 | 467 | struct svc_rdma_req_map *vec, |
c06b540a TT |
468 | int byte_count) |
469 | { | |
470 | struct ib_send_wr send_wr; | |
9d11b51c | 471 | u32 xdr_off; |
c06b540a TT |
472 | int sge_no; |
473 | int sge_bytes; | |
474 | int page_no; | |
afc59400 | 475 | int pages; |
c06b540a TT |
476 | int ret; |
477 | ||
bf36387a | 478 | ret = svc_rdma_repost_recv(rdma, GFP_KERNEL); |
0e7f011a | 479 | if (ret) { |
5ac461a6 TT |
480 | svc_rdma_put_context(ctxt, 0); |
481 | return -ENOTCONN; | |
0e7f011a TT |
482 | } |
483 | ||
c06b540a TT |
484 | /* Prepare the context */ |
485 | ctxt->pages[0] = page; | |
486 | ctxt->count = 1; | |
487 | ||
488 | /* Prepare the SGE for the RPCRDMA Header */ | |
5fe1043d | 489 | ctxt->sge[0].lkey = rdma->sc_pd->local_dma_lkey; |
98779be8 | 490 | ctxt->sge[0].length = svc_rdma_xdr_get_reply_hdr_len(rdma_resp); |
c06b540a | 491 | ctxt->sge[0].addr = |
b432e6b3 TT |
492 | ib_dma_map_page(rdma->sc_cm_id->device, page, 0, |
493 | ctxt->sge[0].length, DMA_TO_DEVICE); | |
afd566ea TT |
494 | if (ib_dma_mapping_error(rdma->sc_cm_id->device, ctxt->sge[0].addr)) |
495 | goto err; | |
496 | atomic_inc(&rdma->sc_dma_used); | |
497 | ||
c06b540a | 498 | ctxt->direction = DMA_TO_DEVICE; |
afd566ea | 499 | |
b432e6b3 | 500 | /* Map the payload indicated by 'byte_count' */ |
9d11b51c | 501 | xdr_off = 0; |
34d16e42 TT |
502 | for (sge_no = 1; byte_count && sge_no < vec->count; sge_no++) { |
503 | sge_bytes = min_t(size_t, vec->sge[sge_no].iov_len, byte_count); | |
c06b540a | 504 | byte_count -= sge_bytes; |
0bf48289 SW |
505 | ctxt->sge[sge_no].addr = |
506 | dma_map_xdr(rdma, &rqstp->rq_res, xdr_off, | |
507 | sge_bytes, DMA_TO_DEVICE); | |
508 | xdr_off += sge_bytes; | |
509 | if (ib_dma_mapping_error(rdma->sc_cm_id->device, | |
510 | ctxt->sge[sge_no].addr)) | |
511 | goto err; | |
512 | atomic_inc(&rdma->sc_dma_used); | |
5fe1043d | 513 | ctxt->sge[sge_no].lkey = rdma->sc_pd->local_dma_lkey; |
34d16e42 | 514 | ctxt->sge[sge_no].length = sge_bytes; |
c06b540a | 515 | } |
3fe04ee9 CL |
516 | if (byte_count != 0) { |
517 | pr_err("svcrdma: Could not map %d bytes\n", byte_count); | |
518 | goto err; | |
519 | } | |
c06b540a TT |
520 | |
521 | /* Save all respages in the ctxt and remove them from the | |
522 | * respages array. They are our pages until the I/O | |
523 | * completes. | |
524 | */ | |
afc59400 BF |
525 | pages = rqstp->rq_next_page - rqstp->rq_respages; |
526 | for (page_no = 0; page_no < pages; page_no++) { | |
c06b540a TT |
527 | ctxt->pages[page_no+1] = rqstp->rq_respages[page_no]; |
528 | ctxt->count++; | |
529 | rqstp->rq_respages[page_no] = NULL; | |
afd566ea TT |
530 | /* |
531 | * If there are more pages than SGE, terminate SGE | |
532 | * list so that svc_rdma_unmap_dma doesn't attempt to | |
533 | * unmap garbage. | |
534 | */ | |
34d16e42 TT |
535 | if (page_no+1 >= sge_no) |
536 | ctxt->sge[page_no+1].length = 0; | |
c06b540a | 537 | } |
7e4359e2 | 538 | rqstp->rq_next_page = rqstp->rq_respages + 1; |
0bf48289 | 539 | |
9d11b51c CL |
540 | /* The loop above bumps sc_dma_used for each sge. The |
541 | * xdr_buf.tail gets a separate sge, but resides in the | |
542 | * same page as xdr_buf.head. Don't count it twice. | |
543 | */ | |
544 | if (sge_no > ctxt->count) | |
545 | atomic_dec(&rdma->sc_dma_used); | |
546 | ||
3fe04ee9 CL |
547 | if (sge_no > rdma->sc_max_sge) { |
548 | pr_err("svcrdma: Too many sges (%d)\n", sge_no); | |
549 | goto err; | |
550 | } | |
c06b540a | 551 | memset(&send_wr, 0, sizeof send_wr); |
be99bb11 CL |
552 | ctxt->cqe.done = svc_rdma_wc_send; |
553 | send_wr.wr_cqe = &ctxt->cqe; | |
c06b540a TT |
554 | send_wr.sg_list = ctxt->sge; |
555 | send_wr.num_sge = sge_no; | |
556 | send_wr.opcode = IB_WR_SEND; | |
557 | send_wr.send_flags = IB_SEND_SIGNALED; | |
558 | ||
559 | ret = svc_rdma_send(rdma, &send_wr); | |
560 | if (ret) | |
afd566ea | 561 | goto err; |
c06b540a | 562 | |
afd566ea TT |
563 | return 0; |
564 | ||
565 | err: | |
21515e46 | 566 | svc_rdma_unmap_dma(ctxt); |
afd566ea | 567 | svc_rdma_put_context(ctxt, 1); |
3e1eeb98 | 568 | pr_err("svcrdma: failed to send reply, rc=%d\n", ret); |
afd566ea | 569 | return -EIO; |
c06b540a TT |
570 | } |
571 | ||
572 | void svc_rdma_prep_reply_hdr(struct svc_rqst *rqstp) | |
573 | { | |
574 | } | |
575 | ||
c06b540a TT |
576 | int svc_rdma_sendto(struct svc_rqst *rqstp) |
577 | { | |
578 | struct svc_xprt *xprt = rqstp->rq_xprt; | |
579 | struct svcxprt_rdma *rdma = | |
580 | container_of(xprt, struct svcxprt_rdma, sc_xprt); | |
581 | struct rpcrdma_msg *rdma_argp; | |
582 | struct rpcrdma_msg *rdma_resp; | |
08ae4e7f | 583 | struct rpcrdma_write_array *wr_ary, *rp_ary; |
c06b540a TT |
584 | enum rpcrdma_proc reply_type; |
585 | int ret; | |
586 | int inline_bytes; | |
c06b540a TT |
587 | struct page *res_page; |
588 | struct svc_rdma_op_ctxt *ctxt; | |
34d16e42 | 589 | struct svc_rdma_req_map *vec; |
c06b540a TT |
590 | |
591 | dprintk("svcrdma: sending response for rqstp=%p\n", rqstp); | |
592 | ||
e5523bd2 CL |
593 | /* Get the RDMA request header. The receive logic always |
594 | * places this at the start of page 0. | |
595 | */ | |
596 | rdma_argp = page_address(rqstp->rq_pages[0]); | |
08ae4e7f CL |
597 | wr_ary = svc_rdma_get_write_array(rdma_argp); |
598 | rp_ary = svc_rdma_get_reply_array(rdma_argp, wr_ary); | |
c06b540a | 599 | |
34d16e42 | 600 | /* Build an req vec for the XDR */ |
c06b540a TT |
601 | ctxt = svc_rdma_get_context(rdma); |
602 | ctxt->direction = DMA_TO_DEVICE; | |
2fe81b23 | 603 | vec = svc_rdma_get_req_map(rdma); |
f6763c29 | 604 | ret = svc_rdma_map_xdr(rdma, &rqstp->rq_res, vec, wr_ary != NULL); |
afd566ea TT |
605 | if (ret) |
606 | goto err0; | |
c06b540a TT |
607 | inline_bytes = rqstp->rq_res.len; |
608 | ||
609 | /* Create the RDMA response header */ | |
78da2b3c CL |
610 | ret = -ENOMEM; |
611 | res_page = alloc_page(GFP_KERNEL); | |
612 | if (!res_page) | |
613 | goto err0; | |
c06b540a | 614 | rdma_resp = page_address(res_page); |
08ae4e7f | 615 | if (rp_ary) |
c06b540a TT |
616 | reply_type = RDMA_NOMSG; |
617 | else | |
618 | reply_type = RDMA_MSG; | |
619 | svc_rdma_xdr_encode_reply_header(rdma, rdma_argp, | |
620 | rdma_resp, reply_type); | |
621 | ||
622 | /* Send any write-chunk data and build resp write-list */ | |
08ae4e7f CL |
623 | if (wr_ary) { |
624 | ret = send_write_chunks(rdma, wr_ary, rdma_resp, rqstp, vec); | |
625 | if (ret < 0) | |
626 | goto err1; | |
cf570a93 | 627 | inline_bytes -= ret + xdr_padsize(ret); |
c06b540a | 628 | } |
c06b540a TT |
629 | |
630 | /* Send any reply-list data and update resp reply-list */ | |
08ae4e7f CL |
631 | if (rp_ary) { |
632 | ret = send_reply_chunks(rdma, rp_ary, rdma_resp, rqstp, vec); | |
633 | if (ret < 0) | |
634 | goto err1; | |
635 | inline_bytes -= ret; | |
c06b540a | 636 | } |
c06b540a | 637 | |
34d16e42 | 638 | ret = send_reply(rdma, rqstp, res_page, rdma_resp, ctxt, vec, |
c06b540a | 639 | inline_bytes); |
3e1eeb98 CL |
640 | if (ret < 0) |
641 | goto err1; | |
642 | ||
2fe81b23 | 643 | svc_rdma_put_req_map(rdma, vec); |
c06b540a TT |
644 | dprintk("svcrdma: send_reply returns %d\n", ret); |
645 | return ret; | |
afd566ea TT |
646 | |
647 | err1: | |
648 | put_page(res_page); | |
649 | err0: | |
2fe81b23 | 650 | svc_rdma_put_req_map(rdma, vec); |
c06b540a | 651 | svc_rdma_put_context(ctxt, 0); |
3e1eeb98 CL |
652 | set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags); |
653 | return -ENOTCONN; | |
c06b540a | 654 | } |
a6081b82 CL |
655 | |
656 | void svc_rdma_send_error(struct svcxprt_rdma *xprt, struct rpcrdma_msg *rmsgp, | |
657 | int status) | |
658 | { | |
659 | struct ib_send_wr err_wr; | |
660 | struct page *p; | |
661 | struct svc_rdma_op_ctxt *ctxt; | |
662 | enum rpcrdma_errcode err; | |
663 | __be32 *va; | |
664 | int length; | |
665 | int ret; | |
666 | ||
667 | ret = svc_rdma_repost_recv(xprt, GFP_KERNEL); | |
668 | if (ret) | |
669 | return; | |
670 | ||
671 | p = alloc_page(GFP_KERNEL); | |
672 | if (!p) | |
673 | return; | |
674 | va = page_address(p); | |
675 | ||
676 | /* XDR encode an error reply */ | |
677 | err = ERR_CHUNK; | |
678 | if (status == -EPROTONOSUPPORT) | |
679 | err = ERR_VERS; | |
680 | length = svc_rdma_xdr_encode_error(xprt, rmsgp, err, va); | |
681 | ||
682 | ctxt = svc_rdma_get_context(xprt); | |
683 | ctxt->direction = DMA_TO_DEVICE; | |
684 | ctxt->count = 1; | |
685 | ctxt->pages[0] = p; | |
686 | ||
687 | /* Prepare SGE for local address */ | |
688 | ctxt->sge[0].lkey = xprt->sc_pd->local_dma_lkey; | |
689 | ctxt->sge[0].length = length; | |
690 | ctxt->sge[0].addr = ib_dma_map_page(xprt->sc_cm_id->device, | |
691 | p, 0, length, DMA_TO_DEVICE); | |
692 | if (ib_dma_mapping_error(xprt->sc_cm_id->device, ctxt->sge[0].addr)) { | |
693 | dprintk("svcrdma: Error mapping buffer for protocol error\n"); | |
694 | svc_rdma_put_context(ctxt, 1); | |
695 | return; | |
696 | } | |
697 | atomic_inc(&xprt->sc_dma_used); | |
698 | ||
699 | /* Prepare SEND WR */ | |
700 | memset(&err_wr, 0, sizeof(err_wr)); | |
be99bb11 CL |
701 | ctxt->cqe.done = svc_rdma_wc_send; |
702 | err_wr.wr_cqe = &ctxt->cqe; | |
a6081b82 CL |
703 | err_wr.sg_list = ctxt->sge; |
704 | err_wr.num_sge = 1; | |
705 | err_wr.opcode = IB_WR_SEND; | |
706 | err_wr.send_flags = IB_SEND_SIGNALED; | |
707 | ||
708 | /* Post It */ | |
709 | ret = svc_rdma_send(xprt, &err_wr); | |
710 | if (ret) { | |
711 | dprintk("svcrdma: Error %d posting send for protocol error\n", | |
712 | ret); | |
713 | svc_rdma_unmap_dma(ctxt); | |
714 | svc_rdma_put_context(ctxt, 1); | |
715 | } | |
716 | } |