2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * $Id: iser_initiator.c 6964 2006-05-07 11:11:43Z ogerlitz $
34 #include <linux/kernel.h>
35 #include <linux/slab.h>
38 #include <asm/scatterlist.h>
39 #include <linux/scatterlist.h>
40 #include <linux/kfifo.h>
41 #include <scsi/scsi_cmnd.h>
42 #include <scsi/scsi_host.h>
44 #include "iscsi_iser.h"
46 /* Constant PDU lengths calculations */
47 #define ISER_TOTAL_HEADERS_LEN (sizeof (struct iser_hdr) + \
48 sizeof (struct iscsi_hdr))
50 /* iser_dto_add_regd_buff - increments the reference count for *
51 * the registered buffer & adds it to the DTO object */
52 static void iser_dto_add_regd_buff(struct iser_dto
*dto
,
53 struct iser_regd_buf
*regd_buf
,
54 unsigned long use_offset
,
55 unsigned long use_size
)
59 atomic_inc(®d_buf
->ref_count
);
61 add_idx
= dto
->regd_vector_len
;
62 dto
->regd
[add_idx
] = regd_buf
;
63 dto
->used_sz
[add_idx
] = use_size
;
64 dto
->offset
[add_idx
] = use_offset
;
66 dto
->regd_vector_len
++;
69 /* Register user buffer memory and initialize passive rdma
70 * dto descriptor. Total data size is stored in
71 * iser_ctask->data[ISER_DIR_IN].data_len
73 static int iser_prepare_read_cmd(struct iscsi_cmd_task
*ctask
,
77 struct iscsi_iser_cmd_task
*iser_ctask
= ctask
->dd_data
;
78 struct iser_regd_buf
*regd_buf
;
80 struct iser_hdr
*hdr
= &iser_ctask
->desc
.iser_header
;
81 struct iser_data_buf
*buf_in
= &iser_ctask
->data
[ISER_DIR_IN
];
83 err
= iser_dma_map_task_data(iser_ctask
,
90 if (edtl
> iser_ctask
->data
[ISER_DIR_IN
].data_len
) {
91 iser_err("Total data length: %ld, less than EDTL: "
92 "%d, in READ cmd BHS itt: %d, conn: 0x%p\n",
93 iser_ctask
->data
[ISER_DIR_IN
].data_len
, edtl
,
94 ctask
->itt
, iser_ctask
->iser_conn
);
98 err
= iser_reg_rdma_mem(iser_ctask
,ISER_DIR_IN
);
100 iser_err("Failed to set up Data-IN RDMA\n");
103 regd_buf
= &iser_ctask
->rdma_regd
[ISER_DIR_IN
];
105 hdr
->flags
|= ISER_RSV
;
106 hdr
->read_stag
= cpu_to_be32(regd_buf
->reg
.rkey
);
107 hdr
->read_va
= cpu_to_be64(regd_buf
->reg
.va
);
109 iser_dbg("Cmd itt:%d READ tags RKEY:%#.4X VA:%#llX\n",
110 ctask
->itt
, regd_buf
->reg
.rkey
,
111 (unsigned long long)regd_buf
->reg
.va
);
116 /* Register user buffer memory and initialize passive rdma
117 * dto descriptor. Total data size is stored in
118 * ctask->data[ISER_DIR_OUT].data_len
121 iser_prepare_write_cmd(struct iscsi_cmd_task
*ctask
,
123 unsigned int unsol_sz
,
126 struct iscsi_iser_cmd_task
*iser_ctask
= ctask
->dd_data
;
127 struct iser_regd_buf
*regd_buf
;
129 struct iser_dto
*send_dto
= &iser_ctask
->desc
.dto
;
130 struct iser_hdr
*hdr
= &iser_ctask
->desc
.iser_header
;
131 struct iser_data_buf
*buf_out
= &iser_ctask
->data
[ISER_DIR_OUT
];
133 err
= iser_dma_map_task_data(iser_ctask
,
140 if (edtl
> iser_ctask
->data
[ISER_DIR_OUT
].data_len
) {
141 iser_err("Total data length: %ld, less than EDTL: %d, "
142 "in WRITE cmd BHS itt: %d, conn: 0x%p\n",
143 iser_ctask
->data
[ISER_DIR_OUT
].data_len
,
144 edtl
, ctask
->itt
, ctask
->conn
);
148 err
= iser_reg_rdma_mem(iser_ctask
,ISER_DIR_OUT
);
150 iser_err("Failed to register write cmd RDMA mem\n");
154 regd_buf
= &iser_ctask
->rdma_regd
[ISER_DIR_OUT
];
156 if (unsol_sz
< edtl
) {
157 hdr
->flags
|= ISER_WSV
;
158 hdr
->write_stag
= cpu_to_be32(regd_buf
->reg
.rkey
);
159 hdr
->write_va
= cpu_to_be64(regd_buf
->reg
.va
+ unsol_sz
);
161 iser_dbg("Cmd itt:%d, WRITE tags, RKEY:%#.4X "
162 "VA:%#llX + unsol:%d\n",
163 ctask
->itt
, regd_buf
->reg
.rkey
,
164 (unsigned long long)regd_buf
->reg
.va
, unsol_sz
);
168 iser_dbg("Cmd itt:%d, WRITE, adding imm.data sz: %d\n",
170 iser_dto_add_regd_buff(send_dto
,
180 * iser_post_receive_control - allocates, initializes and posts receive DTO.
182 static int iser_post_receive_control(struct iscsi_conn
*conn
)
184 struct iscsi_iser_conn
*iser_conn
= conn
->dd_data
;
185 struct iser_desc
*rx_desc
;
186 struct iser_regd_buf
*regd_hdr
;
187 struct iser_regd_buf
*regd_data
;
188 struct iser_dto
*recv_dto
= NULL
;
189 struct iser_device
*device
= iser_conn
->ib_conn
->device
;
190 int rx_data_size
, err
= 0;
192 rx_desc
= kmem_cache_alloc(ig
.desc_cache
, GFP_NOIO
);
193 if (rx_desc
== NULL
) {
194 iser_err("Failed to alloc desc for post recv\n");
197 rx_desc
->type
= ISCSI_RX
;
199 /* for the login sequence we must support rx of upto 8K; login is done
200 * after conn create/bind (connect) and conn stop/bind (reconnect),
201 * what's common for both schemes is that the connection is not started
203 if (conn
->c_stage
!= ISCSI_CONN_STARTED
)
204 rx_data_size
= DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH
;
205 else /* FIXME till user space sets conn->max_recv_dlength correctly */
208 rx_desc
->data
= kmalloc(rx_data_size
, GFP_NOIO
);
209 if (rx_desc
->data
== NULL
) {
210 iser_err("Failed to alloc data buf for post recv\n");
212 goto post_rx_kmalloc_failure
;
215 recv_dto
= &rx_desc
->dto
;
216 recv_dto
->ib_conn
= iser_conn
->ib_conn
;
217 recv_dto
->regd_vector_len
= 0;
219 regd_hdr
= &rx_desc
->hdr_regd_buf
;
220 memset(regd_hdr
, 0, sizeof(struct iser_regd_buf
));
221 regd_hdr
->device
= device
;
222 regd_hdr
->virt_addr
= rx_desc
; /* == &rx_desc->iser_header */
223 regd_hdr
->data_size
= ISER_TOTAL_HEADERS_LEN
;
225 iser_reg_single(device
, regd_hdr
, DMA_FROM_DEVICE
);
227 iser_dto_add_regd_buff(recv_dto
, regd_hdr
, 0, 0);
229 regd_data
= &rx_desc
->data_regd_buf
;
230 memset(regd_data
, 0, sizeof(struct iser_regd_buf
));
231 regd_data
->device
= device
;
232 regd_data
->virt_addr
= rx_desc
->data
;
233 regd_data
->data_size
= rx_data_size
;
235 iser_reg_single(device
, regd_data
, DMA_FROM_DEVICE
);
237 iser_dto_add_regd_buff(recv_dto
, regd_data
, 0, 0);
239 err
= iser_post_recv(rx_desc
);
243 /* iser_post_recv failed */
244 iser_dto_buffs_release(recv_dto
);
245 kfree(rx_desc
->data
);
246 post_rx_kmalloc_failure
:
247 kmem_cache_free(ig
.desc_cache
, rx_desc
);
251 /* creates a new tx descriptor and adds header regd buffer */
252 static void iser_create_send_desc(struct iscsi_iser_conn
*iser_conn
,
253 struct iser_desc
*tx_desc
)
255 struct iser_regd_buf
*regd_hdr
= &tx_desc
->hdr_regd_buf
;
256 struct iser_dto
*send_dto
= &tx_desc
->dto
;
258 memset(regd_hdr
, 0, sizeof(struct iser_regd_buf
));
259 regd_hdr
->device
= iser_conn
->ib_conn
->device
;
260 regd_hdr
->virt_addr
= tx_desc
; /* == &tx_desc->iser_header */
261 regd_hdr
->data_size
= ISER_TOTAL_HEADERS_LEN
;
263 send_dto
->ib_conn
= iser_conn
->ib_conn
;
264 send_dto
->notify_enable
= 1;
265 send_dto
->regd_vector_len
= 0;
267 memset(&tx_desc
->iser_header
, 0, sizeof(struct iser_hdr
));
268 tx_desc
->iser_header
.flags
= ISER_VER
;
270 iser_dto_add_regd_buff(send_dto
, regd_hdr
, 0, 0);
274 * iser_conn_set_full_featured_mode - (iSER API)
276 int iser_conn_set_full_featured_mode(struct iscsi_conn
*conn
)
278 struct iscsi_iser_conn
*iser_conn
= conn
->dd_data
;
281 /* no need to keep it in a var, we are after login so if this should
282 * be negotiated, by now the result should be available here */
283 int initial_post_recv_bufs_num
= ISER_MAX_RX_MISC_PDUS
;
285 iser_dbg("Initially post: %d\n", initial_post_recv_bufs_num
);
287 /* Check that there is no posted recv or send buffers left - */
288 /* they must be consumed during the login phase */
289 BUG_ON(atomic_read(&iser_conn
->ib_conn
->post_recv_buf_count
) != 0);
290 BUG_ON(atomic_read(&iser_conn
->ib_conn
->post_send_buf_count
) != 0);
292 /* Initial post receive buffers */
293 for (i
= 0; i
< initial_post_recv_bufs_num
; i
++) {
294 if (iser_post_receive_control(conn
) != 0) {
295 iser_err("Failed to post recv bufs at:%d conn:0x%p\n",
300 iser_dbg("Posted %d post recv bufs, conn:0x%p\n", i
, conn
);
305 iser_check_xmit(struct iscsi_conn
*conn
, void *task
)
308 struct iscsi_iser_conn
*iser_conn
= conn
->dd_data
;
310 write_lock_bh(conn
->recv_lock
);
311 if (atomic_read(&iser_conn
->ib_conn
->post_send_buf_count
) ==
312 ISER_QP_MAX_REQ_DTOS
) {
313 iser_dbg("%ld can't xmit task %p, suspending tx\n",jiffies
,task
);
314 set_bit(ISCSI_SUSPEND_BIT
, &conn
->suspend_tx
);
317 write_unlock_bh(conn
->recv_lock
);
323 * iser_send_command - send command PDU
325 int iser_send_command(struct iscsi_conn
*conn
,
326 struct iscsi_cmd_task
*ctask
)
328 struct iscsi_iser_conn
*iser_conn
= conn
->dd_data
;
329 struct iscsi_iser_cmd_task
*iser_ctask
= ctask
->dd_data
;
330 struct iser_dto
*send_dto
= NULL
;
333 struct iser_data_buf
*data_buf
;
335 struct iscsi_cmd
*hdr
= ctask
->hdr
;
336 struct scsi_cmnd
*sc
= ctask
->sc
;
338 if (!iser_conn_state_comp(iser_conn
->ib_conn
, ISER_CONN_UP
)) {
339 iser_err("Failed to send, conn: 0x%p is not up\n", iser_conn
->ib_conn
);
342 if (iser_check_xmit(conn
, ctask
))
345 edtl
= ntohl(hdr
->data_length
);
347 /* build the tx desc regd header and add it to the tx desc dto */
348 iser_ctask
->desc
.type
= ISCSI_TX_SCSI_COMMAND
;
349 send_dto
= &iser_ctask
->desc
.dto
;
350 send_dto
->ctask
= iser_ctask
;
351 iser_create_send_desc(iser_conn
, &iser_ctask
->desc
);
353 if (hdr
->flags
& ISCSI_FLAG_CMD_READ
)
354 data_buf
= &iser_ctask
->data
[ISER_DIR_IN
];
356 data_buf
= &iser_ctask
->data
[ISER_DIR_OUT
];
358 if (sc
->use_sg
) { /* using a scatter list */
359 data_buf
->buf
= sc
->request_buffer
;
360 data_buf
->size
= sc
->use_sg
;
361 } else if (sc
->request_bufflen
) {
362 /* using a single buffer - convert it into one entry SG */
363 sg_init_one(&data_buf
->sg_single
,
364 sc
->request_buffer
, sc
->request_bufflen
);
365 data_buf
->buf
= &data_buf
->sg_single
;
369 data_buf
->data_len
= sc
->request_bufflen
;
371 if (hdr
->flags
& ISCSI_FLAG_CMD_READ
) {
372 err
= iser_prepare_read_cmd(ctask
, edtl
);
374 goto send_command_error
;
376 if (hdr
->flags
& ISCSI_FLAG_CMD_WRITE
) {
377 err
= iser_prepare_write_cmd(ctask
,
383 goto send_command_error
;
386 iser_reg_single(iser_conn
->ib_conn
->device
,
387 send_dto
->regd
[0], DMA_TO_DEVICE
);
389 if (iser_post_receive_control(conn
) != 0) {
390 iser_err("post_recv failed!\n");
392 goto send_command_error
;
395 iser_ctask
->status
= ISER_TASK_STATUS_STARTED
;
397 err
= iser_post_send(&iser_ctask
->desc
);
402 iser_dto_buffs_release(send_dto
);
403 iser_err("conn %p failed ctask->itt %d err %d\n",conn
, ctask
->itt
, err
);
408 * iser_send_data_out - send data out PDU
410 int iser_send_data_out(struct iscsi_conn
*conn
,
411 struct iscsi_cmd_task
*ctask
,
412 struct iscsi_data
*hdr
)
414 struct iscsi_iser_conn
*iser_conn
= conn
->dd_data
;
415 struct iscsi_iser_cmd_task
*iser_ctask
= ctask
->dd_data
;
416 struct iser_desc
*tx_desc
= NULL
;
417 struct iser_dto
*send_dto
= NULL
;
418 unsigned long buf_offset
;
419 unsigned long data_seg_len
;
423 if (!iser_conn_state_comp(iser_conn
->ib_conn
, ISER_CONN_UP
)) {
424 iser_err("Failed to send, conn: 0x%p is not up\n", iser_conn
->ib_conn
);
428 if (iser_check_xmit(conn
, ctask
))
431 itt
= ntohl(hdr
->itt
);
432 data_seg_len
= ntoh24(hdr
->dlength
);
433 buf_offset
= ntohl(hdr
->offset
);
435 iser_dbg("%s itt %d dseg_len %d offset %d\n",
436 __func__
,(int)itt
,(int)data_seg_len
,(int)buf_offset
);
438 tx_desc
= kmem_cache_alloc(ig
.desc_cache
, GFP_NOIO
);
439 if (tx_desc
== NULL
) {
440 iser_err("Failed to alloc desc for post dataout\n");
444 tx_desc
->type
= ISCSI_TX_DATAOUT
;
445 memcpy(&tx_desc
->iscsi_header
, hdr
, sizeof(struct iscsi_hdr
));
447 /* build the tx desc regd header and add it to the tx desc dto */
448 send_dto
= &tx_desc
->dto
;
449 send_dto
->ctask
= iser_ctask
;
450 iser_create_send_desc(iser_conn
, tx_desc
);
452 iser_reg_single(iser_conn
->ib_conn
->device
,
453 send_dto
->regd
[0], DMA_TO_DEVICE
);
455 /* all data was registered for RDMA, we can use the lkey */
456 iser_dto_add_regd_buff(send_dto
,
457 &iser_ctask
->rdma_regd
[ISER_DIR_OUT
],
461 if (buf_offset
+ data_seg_len
> iser_ctask
->data
[ISER_DIR_OUT
].data_len
) {
462 iser_err("Offset:%ld & DSL:%ld in Data-Out "
463 "inconsistent with total len:%ld, itt:%d\n",
464 buf_offset
, data_seg_len
,
465 iser_ctask
->data
[ISER_DIR_OUT
].data_len
, itt
);
467 goto send_data_out_error
;
469 iser_dbg("data-out itt: %d, offset: %ld, sz: %ld\n",
470 itt
, buf_offset
, data_seg_len
);
473 err
= iser_post_send(tx_desc
);
478 iser_dto_buffs_release(send_dto
);
479 kmem_cache_free(ig
.desc_cache
, tx_desc
);
480 iser_err("conn %p failed err %d\n",conn
, err
);
484 int iser_send_control(struct iscsi_conn
*conn
,
485 struct iscsi_mgmt_task
*mtask
)
487 struct iscsi_iser_conn
*iser_conn
= conn
->dd_data
;
488 struct iser_desc
*mdesc
= mtask
->dd_data
;
489 struct iser_dto
*send_dto
= NULL
;
491 unsigned long data_seg_len
;
493 unsigned char opcode
;
494 struct iser_regd_buf
*regd_buf
;
495 struct iser_device
*device
;
497 if (!iser_conn_state_comp(iser_conn
->ib_conn
, ISER_CONN_UP
)) {
498 iser_err("Failed to send, conn: 0x%p is not up\n", iser_conn
->ib_conn
);
502 if (iser_check_xmit(conn
,mtask
))
505 /* build the tx desc regd header and add it to the tx desc dto */
506 mdesc
->type
= ISCSI_TX_CONTROL
;
507 send_dto
= &mdesc
->dto
;
508 send_dto
->ctask
= NULL
;
509 iser_create_send_desc(iser_conn
, mdesc
);
511 device
= iser_conn
->ib_conn
->device
;
513 iser_reg_single(device
, send_dto
->regd
[0], DMA_TO_DEVICE
);
515 itt
= ntohl(mtask
->hdr
->itt
);
516 opcode
= mtask
->hdr
->opcode
& ISCSI_OPCODE_MASK
;
517 data_seg_len
= ntoh24(mtask
->hdr
->dlength
);
519 if (data_seg_len
> 0) {
520 regd_buf
= &mdesc
->data_regd_buf
;
521 memset(regd_buf
, 0, sizeof(struct iser_regd_buf
));
522 regd_buf
->device
= device
;
523 regd_buf
->virt_addr
= mtask
->data
;
524 regd_buf
->data_size
= mtask
->data_count
;
525 iser_reg_single(device
, regd_buf
,
527 iser_dto_add_regd_buff(send_dto
, regd_buf
,
532 if (iser_post_receive_control(conn
) != 0) {
533 iser_err("post_rcv_buff failed!\n");
535 goto send_control_error
;
538 err
= iser_post_send(mdesc
);
543 iser_dto_buffs_release(send_dto
);
544 iser_err("conn %p failed err %d\n",conn
, err
);
549 * iser_rcv_dto_completion - recv DTO completion
551 void iser_rcv_completion(struct iser_desc
*rx_desc
,
552 unsigned long dto_xfer_len
)
554 struct iser_dto
*dto
= &rx_desc
->dto
;
555 struct iscsi_iser_conn
*conn
= dto
->ib_conn
->iser_conn
;
556 struct iscsi_session
*session
= conn
->iscsi_conn
->session
;
557 struct iscsi_cmd_task
*ctask
;
558 struct iscsi_iser_cmd_task
*iser_ctask
;
559 struct iscsi_hdr
*hdr
;
560 char *rx_data
= NULL
;
563 unsigned char opcode
;
565 hdr
= &rx_desc
->iscsi_header
;
567 iser_dbg("op 0x%x itt 0x%x\n", hdr
->opcode
,hdr
->itt
);
569 if (dto_xfer_len
> ISER_TOTAL_HEADERS_LEN
) { /* we have data */
570 rx_data_len
= dto_xfer_len
- ISER_TOTAL_HEADERS_LEN
;
571 rx_data
= dto
->regd
[1]->virt_addr
;
572 rx_data
+= dto
->offset
[1];
575 opcode
= hdr
->opcode
& ISCSI_OPCODE_MASK
;
577 if (opcode
== ISCSI_OP_SCSI_CMD_RSP
) {
578 itt
= hdr
->itt
& ISCSI_ITT_MASK
; /* mask out cid and age bits */
579 if (!(itt
< session
->cmds_max
))
580 iser_err("itt can't be matched to task!!!"
581 "conn %p opcode %d cmds_max %d itt %d\n",
582 conn
->iscsi_conn
,opcode
,session
->cmds_max
,itt
);
583 /* use the mapping given with the cmds array indexed by itt */
584 ctask
= (struct iscsi_cmd_task
*)session
->cmds
[itt
];
585 iser_ctask
= ctask
->dd_data
;
586 iser_dbg("itt %d ctask %p\n",itt
,ctask
);
587 iser_ctask
->status
= ISER_TASK_STATUS_COMPLETED
;
588 iser_ctask_rdma_finalize(iser_ctask
);
591 iser_dto_buffs_release(dto
);
593 iscsi_iser_recv(conn
->iscsi_conn
, hdr
, rx_data
, rx_data_len
);
595 kfree(rx_desc
->data
);
596 kmem_cache_free(ig
.desc_cache
, rx_desc
);
598 /* decrementing conn->post_recv_buf_count only --after-- freeing the *
599 * task eliminates the need to worry on tasks which are completed in *
600 * parallel to the execution of iser_conn_term. So the code that waits *
601 * for the posted rx bufs refcount to become zero handles everything */
602 atomic_dec(&conn
->ib_conn
->post_recv_buf_count
);
605 void iser_snd_completion(struct iser_desc
*tx_desc
)
607 struct iser_dto
*dto
= &tx_desc
->dto
;
608 struct iser_conn
*ib_conn
= dto
->ib_conn
;
609 struct iscsi_iser_conn
*iser_conn
= ib_conn
->iser_conn
;
610 struct iscsi_conn
*conn
= iser_conn
->iscsi_conn
;
611 struct iscsi_mgmt_task
*mtask
;
613 iser_dbg("Initiator, Data sent dto=0x%p\n", dto
);
615 iser_dto_buffs_release(dto
);
617 if (tx_desc
->type
== ISCSI_TX_DATAOUT
)
618 kmem_cache_free(ig
.desc_cache
, tx_desc
);
620 atomic_dec(&ib_conn
->post_send_buf_count
);
622 write_lock(conn
->recv_lock
);
623 if (conn
->suspend_tx
) {
624 iser_dbg("%ld resuming tx\n",jiffies
);
625 clear_bit(ISCSI_SUSPEND_BIT
, &conn
->suspend_tx
);
626 scsi_queue_work(conn
->session
->host
, &conn
->xmitwork
);
628 write_unlock(conn
->recv_lock
);
630 if (tx_desc
->type
== ISCSI_TX_CONTROL
) {
631 /* this arithmetic is legal by libiscsi dd_data allocation */
632 mtask
= (void *) ((long)(void *)tx_desc
-
633 sizeof(struct iscsi_mgmt_task
));
634 if (mtask
->hdr
->itt
== cpu_to_be32(ISCSI_RESERVED_TAG
)) {
635 struct iscsi_session
*session
= conn
->session
;
637 spin_lock(&conn
->session
->lock
);
638 list_del(&mtask
->running
);
639 __kfifo_put(session
->mgmtpool
.queue
, (void*)&mtask
,
641 spin_unlock(&session
->lock
);
646 void iser_ctask_rdma_init(struct iscsi_iser_cmd_task
*iser_ctask
)
649 iser_ctask
->status
= ISER_TASK_STATUS_INIT
;
651 iser_ctask
->dir
[ISER_DIR_IN
] = 0;
652 iser_ctask
->dir
[ISER_DIR_OUT
] = 0;
654 iser_ctask
->data
[ISER_DIR_IN
].data_len
= 0;
655 iser_ctask
->data
[ISER_DIR_OUT
].data_len
= 0;
657 memset(&iser_ctask
->rdma_regd
[ISER_DIR_IN
], 0,
658 sizeof(struct iser_regd_buf
));
659 memset(&iser_ctask
->rdma_regd
[ISER_DIR_OUT
], 0,
660 sizeof(struct iser_regd_buf
));
663 void iser_ctask_rdma_finalize(struct iscsi_iser_cmd_task
*iser_ctask
)
666 int is_rdma_aligned
= 1;
668 /* if we were reading, copy back to unaligned sglist,
669 * anyway dma_unmap and free the copy
671 if (iser_ctask
->data_copy
[ISER_DIR_IN
].copy_buf
!= NULL
) {
673 iser_finalize_rdma_unaligned_sg(iser_ctask
, ISER_DIR_IN
);
675 if (iser_ctask
->data_copy
[ISER_DIR_OUT
].copy_buf
!= NULL
) {
677 iser_finalize_rdma_unaligned_sg(iser_ctask
, ISER_DIR_OUT
);
680 if (iser_ctask
->dir
[ISER_DIR_IN
]) {
681 deferred
= iser_regd_buff_release
682 (&iser_ctask
->rdma_regd
[ISER_DIR_IN
]);
684 iser_err("References remain for BUF-IN rdma reg\n");
689 if (iser_ctask
->dir
[ISER_DIR_OUT
]) {
690 deferred
= iser_regd_buff_release
691 (&iser_ctask
->rdma_regd
[ISER_DIR_OUT
]);
693 iser_err("References remain for BUF-OUT rdma reg\n");
698 /* if the data was unaligned, it was already unmapped and then copied */
700 iser_dma_unmap_task_data(iser_ctask
);
703 void iser_dto_buffs_release(struct iser_dto
*dto
)
707 for (i
= 0; i
< dto
->regd_vector_len
; i
++)
708 iser_regd_buff_release(dto
->regd
[i
]);