projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcm_fc: Use transport_generic_free_cmd for ft_sess_put in ft_send_tm
[deliverable/linux.git] / drivers / target / target_core_transport.c
CommitLineData
c66ac9db
NB		 1/*******************************************************************************
2 * Filename: target_core_transport.c
3 *
4 * This file contains the Generic Target Engine Core.
5 *
6 * Copyright (c) 2002, 2003, 2004, 2005 PyX Technologies, Inc.
7 * Copyright (c) 2005, 2006, 2007 SBE, Inc.
8 * Copyright (c) 2007-2010 Rising Tide Systems
9 * Copyright (c) 2008-2010 Linux-iSCSI.org
10 *
11 * Nicholas A. Bellinger <nab@kernel.org>
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
26 *
27 ******************************************************************************/
28
c66ac9db
NB		 29#include <linux/net.h>
30#include <linux/delay.h>
31#include <linux/string.h>
32#include <linux/timer.h>
33#include <linux/slab.h>
34#include <linux/blkdev.h>
35#include <linux/spinlock.h>
c66ac9db
NB		 36#include <linux/kthread.h>
37#include <linux/in.h>
38#include <linux/cdrom.h>
827509e3 39#include <linux/module.h>
c66ac9db
NB		 40#include <asm/unaligned.h>
41#include <net/sock.h>
42#include <net/tcp.h>
43#include <scsi/scsi.h>
44#include <scsi/scsi_cmnd.h>
e66ecd50 45#include <scsi/scsi_tcq.h>
c66ac9db
NB		 46
47#include <target/target_core_base.h>
c4795fb2
CH		 48#include <target/target_core_backend.h>
49#include <target/target_core_fabric.h>
c66ac9db
NB		 50#include <target/target_core_configfs.h>
51
e26d99ae 52#include "target_core_internal.h"
c66ac9db 53#include "target_core_alua.h"
c66ac9db 54#include "target_core_pr.h"
c66ac9db
NB		 55#include "target_core_ua.h"
56
e3d6f909 57static int sub_api_initialized;
c66ac9db 58
35e0e757 59static struct workqueue_struct *target_completion_wq;
c66ac9db 60static struct kmem_cache *se_sess_cache;
c66ac9db 61struct kmem_cache *se_ua_cache;
c66ac9db
NB		 62struct kmem_cache *t10_pr_reg_cache;
63struct kmem_cache *t10_alua_lu_gp_cache;
64struct kmem_cache *t10_alua_lu_gp_mem_cache;
65struct kmem_cache *t10_alua_tg_pt_gp_cache;
66struct kmem_cache *t10_alua_tg_pt_gp_mem_cache;
67
c66ac9db 68static int transport_generic_write_pending(struct se_cmd *);
5951146d 69static int transport_processing_thread(void *param);
4d2300cc 70static int __transport_execute_tasks(struct se_device *dev, struct se_cmd *);
c66ac9db 71static void transport_complete_task_attr(struct se_cmd *cmd);
07bde79a 72static void transport_handle_queue_full(struct se_cmd *cmd,
e057f533 73 struct se_device *dev);
c66ac9db 74static void transport_free_dev_tasks(struct se_cmd *cmd);
05d1c7c0 75static int transport_generic_get_mem(struct se_cmd *cmd);
39c05f32 76static void transport_put_cmd(struct se_cmd *cmd);
3df8d40b 77static void transport_remove_cmd_from_queue(struct se_cmd *cmd);
c66ac9db 78static int transport_set_sense_codes(struct se_cmd *cmd, u8 asc, u8 ascq);
03e98c9e 79static void transport_generic_request_failure(struct se_cmd *);
35e0e757 80static void target_complete_ok_work(struct work_struct *work);
c66ac9db 81
e3d6f909 82int init_se_kmem_caches(void)
c66ac9db 83{
c66ac9db
NB		 84 se_sess_cache = kmem_cache_create("se_sess_cache",
85 sizeof(struct se_session), __alignof__(struct se_session),
86 0, NULL);
6708bb27
AG		 87 if (!se_sess_cache) {
88 pr_err("kmem_cache_create() for struct se_session"
c66ac9db 89 " failed\n");
c8e31f26 90 goto out;
c66ac9db
NB		 91 }
92 se_ua_cache = kmem_cache_create("se_ua_cache",
93 sizeof(struct se_ua), __alignof__(struct se_ua),
94 0, NULL);
6708bb27
AG		 95 if (!se_ua_cache) {
96 pr_err("kmem_cache_create() for struct se_ua failed\n");
35e0e757 97 goto out_free_sess_cache;
c66ac9db 98 }
c66ac9db
NB		 99 t10_pr_reg_cache = kmem_cache_create("t10_pr_reg_cache",
100 sizeof(struct t10_pr_registration),
101 __alignof__(struct t10_pr_registration), 0, NULL);
6708bb27
AG		 102 if (!t10_pr_reg_cache) {
103 pr_err("kmem_cache_create() for struct t10_pr_registration"
c66ac9db 104 " failed\n");
35e0e757 105 goto out_free_ua_cache;
c66ac9db
NB		 106 }
107 t10_alua_lu_gp_cache = kmem_cache_create("t10_alua_lu_gp_cache",
108 sizeof(struct t10_alua_lu_gp), __alignof__(struct t10_alua_lu_gp),
109 0, NULL);
6708bb27
AG		 110 if (!t10_alua_lu_gp_cache) {
111 pr_err("kmem_cache_create() for t10_alua_lu_gp_cache"
c66ac9db 112 " failed\n");
35e0e757 113 goto out_free_pr_reg_cache;
c66ac9db
NB		 114 }
115 t10_alua_lu_gp_mem_cache = kmem_cache_create("t10_alua_lu_gp_mem_cache",
116 sizeof(struct t10_alua_lu_gp_member),
117 __alignof__(struct t10_alua_lu_gp_member), 0, NULL);
6708bb27
AG		 118 if (!t10_alua_lu_gp_mem_cache) {
119 pr_err("kmem_cache_create() for t10_alua_lu_gp_mem_"
c66ac9db 120 "cache failed\n");
35e0e757 121 goto out_free_lu_gp_cache;
c66ac9db
NB		 122 }
123 t10_alua_tg_pt_gp_cache = kmem_cache_create("t10_alua_tg_pt_gp_cache",
124 sizeof(struct t10_alua_tg_pt_gp),
125 __alignof__(struct t10_alua_tg_pt_gp), 0, NULL);
6708bb27
AG		 126 if (!t10_alua_tg_pt_gp_cache) {
127 pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_"
c66ac9db 128 "cache failed\n");
35e0e757 129 goto out_free_lu_gp_mem_cache;
c66ac9db
NB		 130 }
131 t10_alua_tg_pt_gp_mem_cache = kmem_cache_create(
132 "t10_alua_tg_pt_gp_mem_cache",
133 sizeof(struct t10_alua_tg_pt_gp_member),
134 __alignof__(struct t10_alua_tg_pt_gp_member),
135 0, NULL);
6708bb27
AG		 136 if (!t10_alua_tg_pt_gp_mem_cache) {
137 pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_"
c66ac9db 138 "mem_t failed\n");
35e0e757 139 goto out_free_tg_pt_gp_cache;
c66ac9db
NB		 140 }
141
35e0e757
CH		 142 target_completion_wq = alloc_workqueue("target_completion",
143 WQ_MEM_RECLAIM, 0);
144 if (!target_completion_wq)
145 goto out_free_tg_pt_gp_mem_cache;
146
c66ac9db 147 return 0;
35e0e757
CH		 148
149out_free_tg_pt_gp_mem_cache:
150 kmem_cache_destroy(t10_alua_tg_pt_gp_mem_cache);
151out_free_tg_pt_gp_cache:
152 kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
153out_free_lu_gp_mem_cache:
154 kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
155out_free_lu_gp_cache:
156 kmem_cache_destroy(t10_alua_lu_gp_cache);
157out_free_pr_reg_cache:
158 kmem_cache_destroy(t10_pr_reg_cache);
159out_free_ua_cache:
160 kmem_cache_destroy(se_ua_cache);
161out_free_sess_cache:
162 kmem_cache_destroy(se_sess_cache);
c66ac9db 163out:
e3d6f909 164 return -ENOMEM;
c66ac9db
NB		 165}
166
e3d6f909 167void release_se_kmem_caches(void)
c66ac9db 168{
35e0e757 169 destroy_workqueue(target_completion_wq);
c66ac9db
NB		 170 kmem_cache_destroy(se_sess_cache);
171 kmem_cache_destroy(se_ua_cache);
c66ac9db
NB		 172 kmem_cache_destroy(t10_pr_reg_cache);
173 kmem_cache_destroy(t10_alua_lu_gp_cache);
174 kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
175 kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
176 kmem_cache_destroy(t10_alua_tg_pt_gp_mem_cache);
c66ac9db
NB		 177}
178
e3d6f909
AG		 179/* This code ensures unique mib indexes are handed out. */
180static DEFINE_SPINLOCK(scsi_mib_index_lock);
181static u32 scsi_mib_index[SCSI_INDEX_TYPE_MAX];
e89d15ee
NB		 182
183/*
184 * Allocate a new row index for the entry type specified
185 */
186u32 scsi_get_new_index(scsi_index_t type)
187{
188 u32 new_index;
189
e3d6f909 190 BUG_ON((type < 0) || (type >= SCSI_INDEX_TYPE_MAX));
e89d15ee 191
e3d6f909
AG		 192 spin_lock(&scsi_mib_index_lock);
193 new_index = ++scsi_mib_index[type];
194 spin_unlock(&scsi_mib_index_lock);
e89d15ee
NB		 195
196 return new_index;
197}
198
e26d99ae 199static void transport_init_queue_obj(struct se_queue_obj *qobj)
c66ac9db
NB		 200{
201 atomic_set(&qobj->queue_cnt, 0);
202 INIT_LIST_HEAD(&qobj->qobj_list);
203 init_waitqueue_head(&qobj->thread_wq);
204 spin_lock_init(&qobj->cmd_queue_lock);
205}
c66ac9db 206
dbc5623e 207void transport_subsystem_check_init(void)
c66ac9db
NB		 208{
209 int ret;
210
dbc5623e
NB		 211 if (sub_api_initialized)
212 return;
213
c66ac9db
NB		 214 ret = request_module("target_core_iblock");
215 if (ret != 0)
6708bb27 216 pr_err("Unable to load target_core_iblock\n");
c66ac9db
NB		 217
218 ret = request_module("target_core_file");
219 if (ret != 0)
6708bb27 220 pr_err("Unable to load target_core_file\n");
c66ac9db
NB		 221
222 ret = request_module("target_core_pscsi");
223 if (ret != 0)
6708bb27 224 pr_err("Unable to load target_core_pscsi\n");
c66ac9db
NB		 225
226 ret = request_module("target_core_stgt");
227 if (ret != 0)
6708bb27 228 pr_err("Unable to load target_core_stgt\n");
c66ac9db 229
e3d6f909 230 sub_api_initialized = 1;
dbc5623e 231 return;
c66ac9db
NB		 232}
233
234struct se_session *transport_init_session(void)
235{
236 struct se_session *se_sess;
237
238 se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL);
6708bb27
AG		 239 if (!se_sess) {
240 pr_err("Unable to allocate struct se_session from"
c66ac9db
NB		 241 " se_sess_cache\n");
242 return ERR_PTR(-ENOMEM);
243 }
244 INIT_LIST_HEAD(&se_sess->sess_list);
245 INIT_LIST_HEAD(&se_sess->sess_acl_list);
a17f091d
NB		 246 INIT_LIST_HEAD(&se_sess->sess_cmd_list);
247 INIT_LIST_HEAD(&se_sess->sess_wait_list);
248 spin_lock_init(&se_sess->sess_cmd_lock);
c66ac9db
NB		 249
250 return se_sess;
251}
252EXPORT_SYMBOL(transport_init_session);
253
254/*
255 * Called with spin_lock_bh(&struct se_portal_group->session_lock called.
256 */
257void __transport_register_session(
258 struct se_portal_group *se_tpg,
259 struct se_node_acl *se_nacl,
260 struct se_session *se_sess,
261 void *fabric_sess_ptr)
262{
263 unsigned char buf[PR_REG_ISID_LEN];
264
265 se_sess->se_tpg = se_tpg;
266 se_sess->fabric_sess_ptr = fabric_sess_ptr;
267 /*
268 * Used by struct se_node_acl's under ConfigFS to locate active se_session-t
269 *
270 * Only set for struct se_session's that will actually be moving I/O.
271 * eg: *NOT* discovery sessions.
272 */
273 if (se_nacl) {
274 /*
275 * If the fabric module supports an ISID based TransportID,
276 * save this value in binary from the fabric I_T Nexus now.
277 */
e3d6f909 278 if (se_tpg->se_tpg_tfo->sess_get_initiator_sid != NULL) {
c66ac9db 279 memset(&buf[0], 0, PR_REG_ISID_LEN);
e3d6f909 280 se_tpg->se_tpg_tfo->sess_get_initiator_sid(se_sess,
c66ac9db
NB		 281 &buf[0], PR_REG_ISID_LEN);
282 se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]);
283 }
284 spin_lock_irq(&se_nacl->nacl_sess_lock);
285 /*
286 * The se_nacl->nacl_sess pointer will be set to the
287 * last active I_T Nexus for each struct se_node_acl.
288 */
289 se_nacl->nacl_sess = se_sess;
290
291 list_add_tail(&se_sess->sess_acl_list,
292 &se_nacl->acl_sess_list);
293 spin_unlock_irq(&se_nacl->nacl_sess_lock);
294 }
295 list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list);
296
6708bb27 297 pr_debug("TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n",
e3d6f909 298 se_tpg->se_tpg_tfo->get_fabric_name(), se_sess->fabric_sess_ptr);
c66ac9db
NB		 299}
300EXPORT_SYMBOL(__transport_register_session);
301
302void transport_register_session(
303 struct se_portal_group *se_tpg,
304 struct se_node_acl *se_nacl,
305 struct se_session *se_sess,
306 void *fabric_sess_ptr)
307{
308 spin_lock_bh(&se_tpg->session_lock);
309 __transport_register_session(se_tpg, se_nacl, se_sess, fabric_sess_ptr);
310 spin_unlock_bh(&se_tpg->session_lock);
311}
312EXPORT_SYMBOL(transport_register_session);
313
314void transport_deregister_session_configfs(struct se_session *se_sess)
315{
316 struct se_node_acl *se_nacl;
23388864 317 unsigned long flags;
c66ac9db
NB		 318 /*
319 * Used by struct se_node_acl's under ConfigFS to locate active struct se_session
320 */
321 se_nacl = se_sess->se_node_acl;
6708bb27 322 if (se_nacl) {
23388864 323 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
c66ac9db
NB		 324 list_del(&se_sess->sess_acl_list);
325 /*
326 * If the session list is empty, then clear the pointer.
327 * Otherwise, set the struct se_session pointer from the tail
328 * element of the per struct se_node_acl active session list.
329 */
330 if (list_empty(&se_nacl->acl_sess_list))
331 se_nacl->nacl_sess = NULL;
332 else {
333 se_nacl->nacl_sess = container_of(
334 se_nacl->acl_sess_list.prev,
335 struct se_session, sess_acl_list);
336 }
23388864 337 spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
c66ac9db
NB		 338 }
339}
340EXPORT_SYMBOL(transport_deregister_session_configfs);
341
342void transport_free_session(struct se_session *se_sess)
343{
344 kmem_cache_free(se_sess_cache, se_sess);
345}
346EXPORT_SYMBOL(transport_free_session);
347
348void transport_deregister_session(struct se_session *se_sess)
349{
350 struct se_portal_group *se_tpg = se_sess->se_tpg;
351 struct se_node_acl *se_nacl;
e63a8e19 352 unsigned long flags;
c66ac9db 353
6708bb27 354 if (!se_tpg) {
c66ac9db
NB		 355 transport_free_session(se_sess);
356 return;
357 }
c66ac9db 358
e63a8e19 359 spin_lock_irqsave(&se_tpg->session_lock, flags);
c66ac9db
NB		 360 list_del(&se_sess->sess_list);
361 se_sess->se_tpg = NULL;
362 se_sess->fabric_sess_ptr = NULL;
e63a8e19 363 spin_unlock_irqrestore(&se_tpg->session_lock, flags);
c66ac9db
NB		 364
365 /*
366 * Determine if we need to do extra work for this initiator node's
367 * struct se_node_acl if it had been previously dynamically generated.
368 */
369 se_nacl = se_sess->se_node_acl;
6708bb27 370 if (se_nacl) {
e63a8e19 371 spin_lock_irqsave(&se_tpg->acl_node_lock, flags);
c66ac9db 372 if (se_nacl->dynamic_node_acl) {
6708bb27
AG		 373 if (!se_tpg->se_tpg_tfo->tpg_check_demo_mode_cache(
374 se_tpg)) {
c66ac9db
NB		 375 list_del(&se_nacl->acl_list);
376 se_tpg->num_node_acls--;
e63a8e19 377 spin_unlock_irqrestore(&se_tpg->acl_node_lock, flags);
c66ac9db
NB		 378
379 core_tpg_wait_for_nacl_pr_ref(se_nacl);
c66ac9db 380 core_free_device_list_for_node(se_nacl, se_tpg);
e3d6f909 381 se_tpg->se_tpg_tfo->tpg_release_fabric_acl(se_tpg,
c66ac9db 382 se_nacl);
e63a8e19 383 spin_lock_irqsave(&se_tpg->acl_node_lock, flags);
c66ac9db
NB		 384 }
385 }
e63a8e19 386 spin_unlock_irqrestore(&se_tpg->acl_node_lock, flags);
c66ac9db
NB		 387 }
388
389 transport_free_session(se_sess);
390
6708bb27 391 pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n",
e3d6f909 392 se_tpg->se_tpg_tfo->get_fabric_name());
c66ac9db
NB		 393}
394EXPORT_SYMBOL(transport_deregister_session);
395
396/*
a1d8b49a 397 * Called with cmd->t_state_lock held.
c66ac9db
NB		 398 */
399static void transport_all_task_dev_remove_state(struct se_cmd *cmd)
400{
42bf829e 401 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 402 struct se_task *task;
403 unsigned long flags;
404
42bf829e
CH		 405 if (!dev)
406 return;
c66ac9db 407
42bf829e 408 list_for_each_entry(task, &cmd->t_task_list, t_list) {
6c76bf95 409 if (task->task_flags & TF_ACTIVE)
c66ac9db
NB		 410 continue;
411
c66ac9db 412 spin_lock_irqsave(&dev->execute_task_lock, flags);
1880807a
CH		 413 if (task->t_state_active) {
414 pr_debug("Removed ITT: 0x%08x dev: %p task[%p]\n",
415 cmd->se_tfo->get_task_tag(cmd), dev, task);
c66ac9db 416
1880807a
CH		 417 list_del(&task->t_state_list);
418 atomic_dec(&cmd->t_task_cdbs_ex_left);
419 task->t_state_active = false;
420 }
421 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
c66ac9db 422 }
1880807a 423
c66ac9db
NB		 424}
425
426/* transport_cmd_check_stop():
427 *
7d680f3b 428 * 'transport_off = 1' determines if CMD_T_ACTIVE should be cleared.
c66ac9db
NB		 429 * 'transport_off = 2' determines if task_dev_state should be removed.
430 *
431 * A non-zero u8 t_state sets cmd->t_state.
432 * Returns 1 when command is stopped, else 0.
433 */
434static int transport_cmd_check_stop(
435 struct se_cmd *cmd,
436 int transport_off,
437 u8 t_state)
438{
439 unsigned long flags;
440
a1d8b49a 441 spin_lock_irqsave(&cmd->t_state_lock, flags);
c66ac9db
NB		 442 /*
443 * Determine if IOCTL context caller in requesting the stopping of this
444 * command for LUN shutdown purposes.
445 */
7d680f3b
CH		 446 if (cmd->transport_state & CMD_T_LUN_STOP) {
447 pr_debug("%s:%d CMD_T_LUN_STOP for ITT: 0x%08x\n",
448 __func__, __LINE__, cmd->se_tfo->get_task_tag(cmd));
c66ac9db 449
7d680f3b 450 cmd->transport_state &= ~CMD_T_ACTIVE;
c66ac9db
NB		 451 if (transport_off == 2)
452 transport_all_task_dev_remove_state(cmd);
a1d8b49a 453 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 454
a1d8b49a 455 complete(&cmd->transport_lun_stop_comp);
c66ac9db
NB		 456 return 1;
457 }
458 /*
459 * Determine if frontend context caller is requesting the stopping of
e3d6f909 460 * this command for frontend exceptions.
c66ac9db 461 */
7d680f3b
CH		 462 if (cmd->transport_state & CMD_T_STOP) {
463 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08x\n",
464 __func__, __LINE__,
e3d6f909 465 cmd->se_tfo->get_task_tag(cmd));
c66ac9db 466
c66ac9db
NB		 467 if (transport_off == 2)
468 transport_all_task_dev_remove_state(cmd);
469
470 /*
471 * Clear struct se_cmd->se_lun before the transport_off == 2 handoff
472 * to FE.
473 */
474 if (transport_off == 2)
475 cmd->se_lun = NULL;
a1d8b49a 476 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 477
a1d8b49a 478 complete(&cmd->t_transport_stop_comp);
c66ac9db
NB		 479 return 1;
480 }
481 if (transport_off) {
7d680f3b 482 cmd->transport_state &= ~CMD_T_ACTIVE;
c66ac9db
NB		 483 if (transport_off == 2) {
484 transport_all_task_dev_remove_state(cmd);
485 /*
486 * Clear struct se_cmd->se_lun before the transport_off == 2
487 * handoff to fabric module.
488 */
489 cmd->se_lun = NULL;
490 /*
491 * Some fabric modules like tcm_loop can release
25985edc 492 * their internally allocated I/O reference now and
c66ac9db 493 * struct se_cmd now.
88dd9e26
NB		 494 *
495 * Fabric modules are expected to return '1' here if the
496 * se_cmd being passed is released at this point,
497 * or zero if not being released.
c66ac9db 498 */
e3d6f909 499 if (cmd->se_tfo->check_stop_free != NULL) {
c66ac9db 500 spin_unlock_irqrestore(
a1d8b49a 501 &cmd->t_state_lock, flags);
c66ac9db 502
88dd9e26 503 return cmd->se_tfo->check_stop_free(cmd);
c66ac9db
NB		 504 }
505 }
a1d8b49a 506 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 507
508 return 0;
509 } else if (t_state)
510 cmd->t_state = t_state;
a1d8b49a 511 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 512
513 return 0;
514}
515
516static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd)
517{
518 return transport_cmd_check_stop(cmd, 2, 0);
519}
520
521static void transport_lun_remove_cmd(struct se_cmd *cmd)
522{
e3d6f909 523 struct se_lun *lun = cmd->se_lun;
c66ac9db
NB		 524 unsigned long flags;
525
526 if (!lun)
527 return;
528
a1d8b49a 529 spin_lock_irqsave(&cmd->t_state_lock, flags);
7d680f3b
CH		 530 if (cmd->transport_state & CMD_T_DEV_ACTIVE) {
531 cmd->transport_state &= ~CMD_T_DEV_ACTIVE;
532 transport_all_task_dev_remove_state(cmd);
c66ac9db 533 }
a1d8b49a 534 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 535
c66ac9db 536 spin_lock_irqsave(&lun->lun_cmd_lock, flags);
3d26fea0
CH		 537 if (!list_empty(&cmd->se_lun_node))
538 list_del_init(&cmd->se_lun_node);
c66ac9db
NB		 539 spin_unlock_irqrestore(&lun->lun_cmd_lock, flags);
540}
541
542void transport_cmd_finish_abort(struct se_cmd *cmd, int remove)
543{
c8e31f26 544 if (!(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
8dc52b54 545 transport_lun_remove_cmd(cmd);
c66ac9db
NB		 546
547 if (transport_cmd_check_stop_to_fabric(cmd))
548 return;
77039d1e 549 if (remove) {
3df8d40b 550 transport_remove_cmd_from_queue(cmd);
e6a2573f 551 transport_put_cmd(cmd);
77039d1e 552 }
c66ac9db
NB		 553}
554
f7a5cc0b
CH		 555static void transport_add_cmd_to_queue(struct se_cmd *cmd, int t_state,
556 bool at_head)
c66ac9db
NB		 557{
558 struct se_device *dev = cmd->se_dev;
e3d6f909 559 struct se_queue_obj *qobj = &dev->dev_queue_obj;
c66ac9db
NB		 560 unsigned long flags;
561
c66ac9db 562 if (t_state) {
a1d8b49a 563 spin_lock_irqsave(&cmd->t_state_lock, flags);
c66ac9db 564 cmd->t_state = t_state;
7d680f3b 565 cmd->transport_state |= CMD_T_ACTIVE;
a1d8b49a 566 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 567 }
568
569 spin_lock_irqsave(&qobj->cmd_queue_lock, flags);
79a7fef2
RD		 570
571 /* If the cmd is already on the list, remove it before we add it */
572 if (!list_empty(&cmd->se_queue_node))
573 list_del(&cmd->se_queue_node);
574 else
575 atomic_inc(&qobj->queue_cnt);
576
f7a5cc0b 577 if (at_head)
07bde79a 578 list_add(&cmd->se_queue_node, &qobj->qobj_list);
f7a5cc0b 579 else
07bde79a 580 list_add_tail(&cmd->se_queue_node, &qobj->qobj_list);
7d680f3b 581 cmd->transport_state |= CMD_T_QUEUED;
c66ac9db
NB		 582 spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
583
c66ac9db 584 wake_up_interruptible(&qobj->thread_wq);
c66ac9db
NB		 585}
586
5951146d
AG		 587static struct se_cmd *
588transport_get_cmd_from_queue(struct se_queue_obj *qobj)
c66ac9db 589{
5951146d 590 struct se_cmd *cmd;
c66ac9db
NB		 591 unsigned long flags;
592
593 spin_lock_irqsave(&qobj->cmd_queue_lock, flags);
594 if (list_empty(&qobj->qobj_list)) {
595 spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
596 return NULL;
597 }
5951146d 598 cmd = list_first_entry(&qobj->qobj_list, struct se_cmd, se_queue_node);
c66ac9db 599
7d680f3b 600 cmd->transport_state &= ~CMD_T_QUEUED;
79a7fef2 601 list_del_init(&cmd->se_queue_node);
c66ac9db
NB		 602 atomic_dec(&qobj->queue_cnt);
603 spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
604
5951146d 605 return cmd;
c66ac9db
NB		 606}
607
3df8d40b 608static void transport_remove_cmd_from_queue(struct se_cmd *cmd)
c66ac9db 609{
3df8d40b 610 struct se_queue_obj *qobj = &cmd->se_dev->dev_queue_obj;
c66ac9db
NB		 611 unsigned long flags;
612
613 spin_lock_irqsave(&qobj->cmd_queue_lock, flags);
7d680f3b 614 if (!(cmd->transport_state & CMD_T_QUEUED)) {
c66ac9db
NB		 615 spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
616 return;
617 }
7d680f3b 618 cmd->transport_state &= ~CMD_T_QUEUED;
79a7fef2
RD		 619 atomic_dec(&qobj->queue_cnt);
620 list_del_init(&cmd->se_queue_node);
c66ac9db 621 spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
c66ac9db
NB		 622}
623
624/*
625 * Completion function used by TCM subsystem plugins (such as FILEIO)
626 * for queueing up response from struct se_subsystem_api->do_task()
627 */
628void transport_complete_sync_cache(struct se_cmd *cmd, int good)
629{
a1d8b49a 630 struct se_task *task = list_entry(cmd->t_task_list.next,
c66ac9db
NB		 631 struct se_task, t_list);
632
633 if (good) {
634 cmd->scsi_status = SAM_STAT_GOOD;
635 task->task_scsi_status = GOOD;
636 } else {
637 task->task_scsi_status = SAM_STAT_CHECK_CONDITION;
03e98c9e
NB		 638 task->task_se_cmd->scsi_sense_reason =
639 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
640
c66ac9db
NB		 641 }
642
643 transport_complete_task(task, good);
644}
645EXPORT_SYMBOL(transport_complete_sync_cache);
646
35e0e757
CH		 647static void target_complete_failure_work(struct work_struct *work)
648{
649 struct se_cmd *cmd = container_of(work, struct se_cmd, work);
650
03e98c9e 651 transport_generic_request_failure(cmd);
35e0e757
CH		 652}
653
c66ac9db
NB		 654/* transport_complete_task():
655 *
656 * Called from interrupt and non interrupt context depending
657 * on the transport plugin.
658 */
659void transport_complete_task(struct se_task *task, int success)
660{
e3d6f909 661 struct se_cmd *cmd = task->task_se_cmd;
42bf829e 662 struct se_device *dev = cmd->se_dev;
c66ac9db 663 unsigned long flags;
c66ac9db 664
a1d8b49a 665 spin_lock_irqsave(&cmd->t_state_lock, flags);
6c76bf95 666 task->task_flags &= ~TF_ACTIVE;
c66ac9db
NB		 667
668 /*
669 * See if any sense data exists, if so set the TASK_SENSE flag.
670 * Also check for any other post completion work that needs to be
671 * done by the plugins.
672 */
673 if (dev && dev->transport->transport_complete) {
674 if (dev->transport->transport_complete(task) != 0) {
675 cmd->se_cmd_flags |= SCF_TRANSPORT_TASK_SENSE;
ef804a84 676 task->task_flags |= TF_HAS_SENSE;
c66ac9db
NB		 677 success = 1;
678 }
679 }
680
681 /*
682 * See if we are waiting for outstanding struct se_task
683 * to complete for an exception condition
684 */
6c76bf95 685 if (task->task_flags & TF_REQUEST_STOP) {
a1d8b49a 686 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 687 complete(&task->task_stop_comp);
688 return;
689 }
2235007c
CH		 690
691 if (!success)
7d680f3b 692 cmd->transport_state |= CMD_T_FAILED;
2235007c 693
c66ac9db
NB		 694 /*
695 * Decrement the outstanding t_task_cdbs_left count. The last
696 * struct se_task from struct se_cmd will complete itself into the
697 * device queue depending upon int success.
698 */
6708bb27 699 if (!atomic_dec_and_test(&cmd->t_task_cdbs_left)) {
a1d8b49a 700 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 701 return;
702 }
703
7d680f3b 704 if (cmd->transport_state & CMD_T_FAILED) {
41e16e98 705 cmd->scsi_sense_reason = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
35e0e757 706 INIT_WORK(&cmd->work, target_complete_failure_work);
c66ac9db 707 } else {
7d680f3b 708 cmd->transport_state |= CMD_T_COMPLETE;
35e0e757 709 INIT_WORK(&cmd->work, target_complete_ok_work);
c66ac9db 710 }
35e0e757 711
35e0e757 712 cmd->t_state = TRANSPORT_COMPLETE;
7d680f3b 713 cmd->transport_state |= CMD_T_ACTIVE;
a1d8b49a 714 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 715
35e0e757 716 queue_work(target_completion_wq, &cmd->work);
c66ac9db
NB		 717}
718EXPORT_SYMBOL(transport_complete_task);
719
720/*
721 * Called by transport_add_tasks_from_cmd() once a struct se_cmd's
722 * struct se_task list are ready to be added to the active execution list
723 * struct se_device
724
725 * Called with se_dev_t->execute_task_lock called.
726 */
727static inline int transport_add_task_check_sam_attr(
728 struct se_task *task,
729 struct se_task *task_prev,
730 struct se_device *dev)
731{
732 /*
733 * No SAM Task attribute emulation enabled, add to tail of
734 * execution queue
735 */
736 if (dev->dev_task_attr_type != SAM_TASK_ATTR_EMULATED) {
737 list_add_tail(&task->t_execute_list, &dev->execute_task_list);
738 return 0;
739 }
740 /*
741 * HEAD_OF_QUEUE attribute for received CDB, which means
742 * the first task that is associated with a struct se_cmd goes to
743 * head of the struct se_device->execute_task_list, and task_prev
744 * after that for each subsequent task
745 */
e66ecd50 746 if (task->task_se_cmd->sam_task_attr == MSG_HEAD_TAG) {
c66ac9db
NB		 747 list_add(&task->t_execute_list,
748 (task_prev != NULL) ?
749 &task_prev->t_execute_list :
750 &dev->execute_task_list);
751
6708bb27 752 pr_debug("Set HEAD_OF_QUEUE for task CDB: 0x%02x"
c66ac9db 753 " in execution queue\n",
6708bb27 754 task->task_se_cmd->t_task_cdb[0]);
c66ac9db
NB		 755 return 1;
756 }
757 /*
758 * For ORDERED, SIMPLE or UNTAGGED attribute tasks once they have been
759 * transitioned from Dermant -> Active state, and are added to the end
760 * of the struct se_device->execute_task_list
761 */
762 list_add_tail(&task->t_execute_list, &dev->execute_task_list);
763 return 0;
764}
765
766/* __transport_add_task_to_execute_queue():
767 *
768 * Called with se_dev_t->execute_task_lock called.
769 */
770static void __transport_add_task_to_execute_queue(
771 struct se_task *task,
772 struct se_task *task_prev,
773 struct se_device *dev)
774{
775 int head_of_queue;
776
777 head_of_queue = transport_add_task_check_sam_attr(task, task_prev, dev);
778 atomic_inc(&dev->execute_tasks);
779
1880807a 780 if (task->t_state_active)
c66ac9db
NB		 781 return;
782 /*
783 * Determine if this task needs to go to HEAD_OF_QUEUE for the
784 * state list as well. Running with SAM Task Attribute emulation
785 * will always return head_of_queue == 0 here
786 */
787 if (head_of_queue)
788 list_add(&task->t_state_list, (task_prev) ?
789 &task_prev->t_state_list :
790 &dev->state_task_list);
791 else
792 list_add_tail(&task->t_state_list, &dev->state_task_list);
793
1880807a 794 task->t_state_active = true;
c66ac9db 795
6708bb27 796 pr_debug("Added ITT: 0x%08x task[%p] to dev: %p\n",
e3d6f909 797 task->task_se_cmd->se_tfo->get_task_tag(task->task_se_cmd),
c66ac9db
NB		 798 task, dev);
799}
800
801static void transport_add_tasks_to_state_queue(struct se_cmd *cmd)
802{
42bf829e 803 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 804 struct se_task *task;
805 unsigned long flags;
806
a1d8b49a
AG		 807 spin_lock_irqsave(&cmd->t_state_lock, flags);
808 list_for_each_entry(task, &cmd->t_task_list, t_list) {
c66ac9db 809 spin_lock(&dev->execute_task_lock);
1880807a
CH		 810 if (!task->t_state_active) {
811 list_add_tail(&task->t_state_list,
812 &dev->state_task_list);
813 task->t_state_active = true;
814
815 pr_debug("Added ITT: 0x%08x task[%p] to dev: %p\n",
816 task->task_se_cmd->se_tfo->get_task_tag(
817 task->task_se_cmd), task, dev);
818 }
c66ac9db
NB		 819 spin_unlock(&dev->execute_task_lock);
820 }
a1d8b49a 821 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 822}
823
4d2300cc 824static void __transport_add_tasks_from_cmd(struct se_cmd *cmd)
c66ac9db 825{
5951146d 826 struct se_device *dev = cmd->se_dev;
c66ac9db 827 struct se_task *task, *task_prev = NULL;
c66ac9db 828
a1d8b49a 829 list_for_each_entry(task, &cmd->t_task_list, t_list) {
04629b7b 830 if (!list_empty(&task->t_execute_list))
c66ac9db
NB		 831 continue;
832 /*
833 * __transport_add_task_to_execute_queue() handles the
834 * SAM Task Attribute emulation if enabled
835 */
836 __transport_add_task_to_execute_queue(task, task_prev, dev);
c66ac9db
NB		 837 task_prev = task;
838 }
4d2300cc
NB		 839}
840
841static void transport_add_tasks_from_cmd(struct se_cmd *cmd)
842{
843 unsigned long flags;
844 struct se_device *dev = cmd->se_dev;
845
846 spin_lock_irqsave(&dev->execute_task_lock, flags);
847 __transport_add_tasks_from_cmd(cmd);
c66ac9db 848 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
c66ac9db
NB		 849}
850
04629b7b
CH		 851void __transport_remove_task_from_execute_queue(struct se_task *task,
852 struct se_device *dev)
853{
854 list_del_init(&task->t_execute_list);
855 atomic_dec(&dev->execute_tasks);
856}
857
e26d99ae 858static void transport_remove_task_from_execute_queue(
c66ac9db
NB		 859 struct se_task *task,
860 struct se_device *dev)
861{
862 unsigned long flags;
863
04629b7b 864 if (WARN_ON(list_empty(&task->t_execute_list)))
af57c3ac 865 return;
af57c3ac 866
c66ac9db 867 spin_lock_irqsave(&dev->execute_task_lock, flags);
04629b7b 868 __transport_remove_task_from_execute_queue(task, dev);
c66ac9db
NB		 869 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
870}
871
07bde79a 872/*
f147abb4 873 * Handle QUEUE_FULL / -EAGAIN and -ENOMEM status
07bde79a
NB		 874 */
875
876static void target_qf_do_work(struct work_struct *work)
877{
878 struct se_device *dev = container_of(work, struct se_device,
879 qf_work_queue);
bcac364a 880 LIST_HEAD(qf_cmd_list);
07bde79a
NB		 881 struct se_cmd *cmd, *cmd_tmp;
882
883 spin_lock_irq(&dev->qf_cmd_lock);
bcac364a
RD		 884 list_splice_init(&dev->qf_cmd_list, &qf_cmd_list);
885 spin_unlock_irq(&dev->qf_cmd_lock);
07bde79a 886
bcac364a 887 list_for_each_entry_safe(cmd, cmd_tmp, &qf_cmd_list, se_qf_node) {
07bde79a
NB		 888 list_del(&cmd->se_qf_node);
889 atomic_dec(&dev->dev_qf_count);
890 smp_mb__after_atomic_dec();
07bde79a 891
6708bb27 892 pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue"
07bde79a 893 " context: %s\n", cmd->se_tfo->get_fabric_name(), cmd,
e057f533 894 (cmd->t_state == TRANSPORT_COMPLETE_QF_OK) ? "COMPLETE_OK" :
07bde79a
NB		 895 (cmd->t_state == TRANSPORT_COMPLETE_QF_WP) ? "WRITE_PENDING"
896 : "UNKNOWN");
f7a5cc0b
CH		 897
898 transport_add_cmd_to_queue(cmd, cmd->t_state, true);
07bde79a 899 }
07bde79a
NB		 900}
901
c66ac9db
NB		 902unsigned char *transport_dump_cmd_direction(struct se_cmd *cmd)
903{
904 switch (cmd->data_direction) {
905 case DMA_NONE:
906 return "NONE";
907 case DMA_FROM_DEVICE:
908 return "READ";
909 case DMA_TO_DEVICE:
910 return "WRITE";
911 case DMA_BIDIRECTIONAL:
912 return "BIDI";
913 default:
914 break;
915 }
916
917 return "UNKNOWN";
918}
919
920void transport_dump_dev_state(
921 struct se_device *dev,
922 char *b,
923 int *bl)
924{
925 *bl += sprintf(b + *bl, "Status: ");
926 switch (dev->dev_status) {
927 case TRANSPORT_DEVICE_ACTIVATED:
928 *bl += sprintf(b + *bl, "ACTIVATED");
929 break;
930 case TRANSPORT_DEVICE_DEACTIVATED:
931 *bl += sprintf(b + *bl, "DEACTIVATED");
932 break;
933 case TRANSPORT_DEVICE_SHUTDOWN:
934 *bl += sprintf(b + *bl, "SHUTDOWN");
935 break;
936 case TRANSPORT_DEVICE_OFFLINE_ACTIVATED:
937 case TRANSPORT_DEVICE_OFFLINE_DEACTIVATED:
938 *bl += sprintf(b + *bl, "OFFLINE");
939 break;
940 default:
941 *bl += sprintf(b + *bl, "UNKNOWN=%d", dev->dev_status);
942 break;
943 }
944
65586d51
NB		 945 *bl += sprintf(b + *bl, " Execute/Max Queue Depth: %d/%d",
946 atomic_read(&dev->execute_tasks), dev->queue_depth);
c66ac9db 947 *bl += sprintf(b + *bl, " SectorSize: %u MaxSectors: %u\n",
e3d6f909 948 dev->se_sub_dev->se_dev_attrib.block_size, dev->se_sub_dev->se_dev_attrib.max_sectors);
c66ac9db
NB		 949 *bl += sprintf(b + *bl, " ");
950}
951
c66ac9db
NB		 952void transport_dump_vpd_proto_id(
953 struct t10_vpd *vpd,
954 unsigned char *p_buf,
955 int p_buf_len)
956{
957 unsigned char buf[VPD_TMP_BUF_SIZE];
958 int len;
959
960 memset(buf, 0, VPD_TMP_BUF_SIZE);
961 len = sprintf(buf, "T10 VPD Protocol Identifier: ");
962
963 switch (vpd->protocol_identifier) {
964 case 0x00:
965 sprintf(buf+len, "Fibre Channel\n");
966 break;
967 case 0x10:
968 sprintf(buf+len, "Parallel SCSI\n");
969 break;
970 case 0x20:
971 sprintf(buf+len, "SSA\n");
972 break;
973 case 0x30:
974 sprintf(buf+len, "IEEE 1394\n");
975 break;
976 case 0x40:
977 sprintf(buf+len, "SCSI Remote Direct Memory Access"
978 " Protocol\n");
979 break;
980 case 0x50:
981 sprintf(buf+len, "Internet SCSI (iSCSI)\n");
982 break;
983 case 0x60:
984 sprintf(buf+len, "SAS Serial SCSI Protocol\n");
985 break;
986 case 0x70:
987 sprintf(buf+len, "Automation/Drive Interface Transport"
988 " Protocol\n");
989 break;
990 case 0x80:
991 sprintf(buf+len, "AT Attachment Interface ATA/ATAPI\n");
992 break;
993 default:
994 sprintf(buf+len, "Unknown 0x%02x\n",
995 vpd->protocol_identifier);
996 break;
997 }
998
999 if (p_buf)
1000 strncpy(p_buf, buf, p_buf_len);
1001 else
6708bb27 1002 pr_debug("%s", buf);
c66ac9db
NB		 1003}
1004
1005void
1006transport_set_vpd_proto_id(struct t10_vpd *vpd, unsigned char *page_83)
1007{
1008 /*
1009 * Check if the Protocol Identifier Valid (PIV) bit is set..
1010 *
1011 * from spc3r23.pdf section 7.5.1
1012 */
1013 if (page_83[1] & 0x80) {
1014 vpd->protocol_identifier = (page_83[0] & 0xf0);
1015 vpd->protocol_identifier_set = 1;
1016 transport_dump_vpd_proto_id(vpd, NULL, 0);
1017 }
1018}
1019EXPORT_SYMBOL(transport_set_vpd_proto_id);
1020
1021int transport_dump_vpd_assoc(
1022 struct t10_vpd *vpd,
1023 unsigned char *p_buf,
1024 int p_buf_len)
1025{
1026 unsigned char buf[VPD_TMP_BUF_SIZE];
e3d6f909
AG		 1027 int ret = 0;
1028 int len;
c66ac9db
NB		 1029
1030 memset(buf, 0, VPD_TMP_BUF_SIZE);
1031 len = sprintf(buf, "T10 VPD Identifier Association: ");
1032
1033 switch (vpd->association) {
1034 case 0x00:
1035 sprintf(buf+len, "addressed logical unit\n");
1036 break;
1037 case 0x10:
1038 sprintf(buf+len, "target port\n");
1039 break;
1040 case 0x20:
1041 sprintf(buf+len, "SCSI target device\n");
1042 break;
1043 default:
1044 sprintf(buf+len, "Unknown 0x%02x\n", vpd->association);
e3d6f909 1045 ret = -EINVAL;
c66ac9db
NB		 1046 break;
1047 }
1048
1049 if (p_buf)
1050 strncpy(p_buf, buf, p_buf_len);
1051 else
6708bb27 1052 pr_debug("%s", buf);
c66ac9db
NB		 1053
1054 return ret;
1055}
1056
1057int transport_set_vpd_assoc(struct t10_vpd *vpd, unsigned char *page_83)
1058{
1059 /*
1060 * The VPD identification association..
1061 *
1062 * from spc3r23.pdf Section 7.6.3.1 Table 297
1063 */
1064 vpd->association = (page_83[1] & 0x30);
1065 return transport_dump_vpd_assoc(vpd, NULL, 0);
1066}
1067EXPORT_SYMBOL(transport_set_vpd_assoc);
1068
1069int transport_dump_vpd_ident_type(
1070 struct t10_vpd *vpd,
1071 unsigned char *p_buf,
1072 int p_buf_len)
1073{
1074 unsigned char buf[VPD_TMP_BUF_SIZE];
e3d6f909
AG		 1075 int ret = 0;
1076 int len;
c66ac9db
NB		 1077
1078 memset(buf, 0, VPD_TMP_BUF_SIZE);
1079 len = sprintf(buf, "T10 VPD Identifier Type: ");
1080
1081 switch (vpd->device_identifier_type) {
1082 case 0x00:
1083 sprintf(buf+len, "Vendor specific\n");
1084 break;
1085 case 0x01:
1086 sprintf(buf+len, "T10 Vendor ID based\n");
1087 break;
1088 case 0x02:
1089 sprintf(buf+len, "EUI-64 based\n");
1090 break;
1091 case 0x03:
1092 sprintf(buf+len, "NAA\n");
1093 break;
1094 case 0x04:
1095 sprintf(buf+len, "Relative target port identifier\n");
1096 break;
1097 case 0x08:
1098 sprintf(buf+len, "SCSI name string\n");
1099 break;
1100 default:
1101 sprintf(buf+len, "Unsupported: 0x%02x\n",
1102 vpd->device_identifier_type);
e3d6f909 1103 ret = -EINVAL;
c66ac9db
NB		 1104 break;
1105 }
1106
e3d6f909
AG		 1107 if (p_buf) {
1108 if (p_buf_len < strlen(buf)+1)
1109 return -EINVAL;
c66ac9db 1110 strncpy(p_buf, buf, p_buf_len);
e3d6f909 1111 } else {
6708bb27 1112 pr_debug("%s", buf);
e3d6f909 1113 }
c66ac9db
NB		 1114
1115 return ret;
1116}
1117
1118int transport_set_vpd_ident_type(struct t10_vpd *vpd, unsigned char *page_83)
1119{
1120 /*
1121 * The VPD identifier type..
1122 *
1123 * from spc3r23.pdf Section 7.6.3.1 Table 298
1124 */
1125 vpd->device_identifier_type = (page_83[1] & 0x0f);
1126 return transport_dump_vpd_ident_type(vpd, NULL, 0);
1127}
1128EXPORT_SYMBOL(transport_set_vpd_ident_type);
1129
1130int transport_dump_vpd_ident(
1131 struct t10_vpd *vpd,
1132 unsigned char *p_buf,
1133 int p_buf_len)
1134{
1135 unsigned char buf[VPD_TMP_BUF_SIZE];
1136 int ret = 0;
1137
1138 memset(buf, 0, VPD_TMP_BUF_SIZE);
1139
1140 switch (vpd->device_identifier_code_set) {
1141 case 0x01: /* Binary */
1142 sprintf(buf, "T10 VPD Binary Device Identifier: %s\n",
1143 &vpd->device_identifier[0]);
1144 break;
1145 case 0x02: /* ASCII */
1146 sprintf(buf, "T10 VPD ASCII Device Identifier: %s\n",
1147 &vpd->device_identifier[0]);
1148 break;
1149 case 0x03: /* UTF-8 */
1150 sprintf(buf, "T10 VPD UTF-8 Device Identifier: %s\n",
1151 &vpd->device_identifier[0]);
1152 break;
1153 default:
1154 sprintf(buf, "T10 VPD Device Identifier encoding unsupported:"
1155 " 0x%02x", vpd->device_identifier_code_set);
e3d6f909 1156 ret = -EINVAL;
c66ac9db
NB		 1157 break;
1158 }
1159
1160 if (p_buf)
1161 strncpy(p_buf, buf, p_buf_len);
1162 else
6708bb27 1163 pr_debug("%s", buf);
c66ac9db
NB		 1164
1165 return ret;
1166}
1167
1168int
1169transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83)
1170{
1171 static const char hex_str[] = "0123456789abcdef";
1172 int j = 0, i = 4; /* offset to start of the identifer */
1173
1174 /*
1175 * The VPD Code Set (encoding)
1176 *
1177 * from spc3r23.pdf Section 7.6.3.1 Table 296
1178 */
1179 vpd->device_identifier_code_set = (page_83[0] & 0x0f);
1180 switch (vpd->device_identifier_code_set) {
1181 case 0x01: /* Binary */
1182 vpd->device_identifier[j++] =
1183 hex_str[vpd->device_identifier_type];
1184 while (i < (4 + page_83[3])) {
1185 vpd->device_identifier[j++] =
1186 hex_str[(page_83[i] & 0xf0) >> 4];
1187 vpd->device_identifier[j++] =
1188 hex_str[page_83[i] & 0x0f];
1189 i++;
1190 }
1191 break;
1192 case 0x02: /* ASCII */
1193 case 0x03: /* UTF-8 */
1194 while (i < (4 + page_83[3]))
1195 vpd->device_identifier[j++] = page_83[i++];
1196 break;
1197 default:
1198 break;
1199 }
1200
1201 return transport_dump_vpd_ident(vpd, NULL, 0);
1202}
1203EXPORT_SYMBOL(transport_set_vpd_ident);
1204
1205static void core_setup_task_attr_emulation(struct se_device *dev)
1206{
1207 /*
1208 * If this device is from Target_Core_Mod/pSCSI, disable the
1209 * SAM Task Attribute emulation.
1210 *
1211 * This is currently not available in upsream Linux/SCSI Target
1212 * mode code, and is assumed to be disabled while using TCM/pSCSI.
1213 */
e3d6f909 1214 if (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV) {
c66ac9db
NB		 1215 dev->dev_task_attr_type = SAM_TASK_ATTR_PASSTHROUGH;
1216 return;
1217 }
1218
1219 dev->dev_task_attr_type = SAM_TASK_ATTR_EMULATED;
6708bb27 1220 pr_debug("%s: Using SAM_TASK_ATTR_EMULATED for SPC: 0x%02x"
e3d6f909
AG		 1221 " device\n", dev->transport->name,
1222 dev->transport->get_device_rev(dev));
c66ac9db
NB		 1223}
1224
1225static void scsi_dump_inquiry(struct se_device *dev)
1226{
e3d6f909 1227 struct t10_wwn *wwn = &dev->se_sub_dev->t10_wwn;
e59a41b6 1228 char buf[17];
c66ac9db
NB		 1229 int i, device_type;
1230 /*
1231 * Print Linux/SCSI style INQUIRY formatting to the kernel ring buffer
1232 */
c66ac9db
NB		 1233 for (i = 0; i < 8; i++)
1234 if (wwn->vendor[i] >= 0x20)
e59a41b6 1235 buf[i] = wwn->vendor[i];
c66ac9db 1236 else
e59a41b6
SAS		 1237 buf[i] = ' ';
1238 buf[i] = '\0';
1239 pr_debug(" Vendor: %s\n", buf);
c66ac9db 1240
c66ac9db
NB		 1241 for (i = 0; i < 16; i++)
1242 if (wwn->model[i] >= 0x20)
e59a41b6 1243 buf[i] = wwn->model[i];
c66ac9db 1244 else
e59a41b6
SAS		 1245 buf[i] = ' ';
1246 buf[i] = '\0';
1247 pr_debug(" Model: %s\n", buf);
c66ac9db 1248
c66ac9db
NB		 1249 for (i = 0; i < 4; i++)
1250 if (wwn->revision[i] >= 0x20)
e59a41b6 1251 buf[i] = wwn->revision[i];
c66ac9db 1252 else
e59a41b6
SAS		 1253 buf[i] = ' ';
1254 buf[i] = '\0';
1255 pr_debug(" Revision: %s\n", buf);
c66ac9db 1256
e3d6f909 1257 device_type = dev->transport->get_device_type(dev);
6708bb27
AG		 1258 pr_debug(" Type: %s ", scsi_device_type(device_type));
1259 pr_debug(" ANSI SCSI revision: %02x\n",
e3d6f909 1260 dev->transport->get_device_rev(dev));
c66ac9db
NB		 1261}
1262
1263struct se_device *transport_add_device_to_core_hba(
1264 struct se_hba *hba,
1265 struct se_subsystem_api *transport,
1266 struct se_subsystem_dev *se_dev,
1267 u32 device_flags,
1268 void *transport_dev,
1269 struct se_dev_limits *dev_limits,
1270 const char *inquiry_prod,
1271 const char *inquiry_rev)
1272{
12a18bdc 1273 int force_pt;
c66ac9db
NB		 1274 struct se_device *dev;
1275
1276 dev = kzalloc(sizeof(struct se_device), GFP_KERNEL);
6708bb27
AG		 1277 if (!dev) {
1278 pr_err("Unable to allocate memory for se_dev_t\n");
c66ac9db
NB		 1279 return NULL;
1280 }
c66ac9db 1281
e3d6f909 1282 transport_init_queue_obj(&dev->dev_queue_obj);
c66ac9db
NB		 1283 dev->dev_flags = device_flags;
1284 dev->dev_status |= TRANSPORT_DEVICE_DEACTIVATED;
5951146d 1285 dev->dev_ptr = transport_dev;
c66ac9db
NB		 1286 dev->se_hba = hba;
1287 dev->se_sub_dev = se_dev;
1288 dev->transport = transport;
c66ac9db
NB		 1289 INIT_LIST_HEAD(&dev->dev_list);
1290 INIT_LIST_HEAD(&dev->dev_sep_list);
1291 INIT_LIST_HEAD(&dev->dev_tmr_list);
1292 INIT_LIST_HEAD(&dev->execute_task_list);
1293 INIT_LIST_HEAD(&dev->delayed_cmd_list);
c66ac9db 1294 INIT_LIST_HEAD(&dev->state_task_list);
07bde79a 1295 INIT_LIST_HEAD(&dev->qf_cmd_list);
c66ac9db
NB		 1296 spin_lock_init(&dev->execute_task_lock);
1297 spin_lock_init(&dev->delayed_cmd_lock);
c66ac9db
NB		 1298 spin_lock_init(&dev->dev_reservation_lock);
1299 spin_lock_init(&dev->dev_status_lock);
c66ac9db
NB		 1300 spin_lock_init(&dev->se_port_lock);
1301 spin_lock_init(&dev->se_tmr_lock);
07bde79a 1302 spin_lock_init(&dev->qf_cmd_lock);
c66ac9db
NB		 1303 atomic_set(&dev->dev_ordered_id, 0);
1304
1305 se_dev_set_default_attribs(dev, dev_limits);
1306
1307 dev->dev_index = scsi_get_new_index(SCSI_DEVICE_INDEX);
1308 dev->creation_time = get_jiffies_64();
1309 spin_lock_init(&dev->stats_lock);
1310
1311 spin_lock(&hba->device_lock);
1312 list_add_tail(&dev->dev_list, &hba->hba_dev_list);
1313 hba->dev_count++;
1314 spin_unlock(&hba->device_lock);
1315 /*
1316 * Setup the SAM Task Attribute emulation for struct se_device
1317 */
1318 core_setup_task_attr_emulation(dev);
1319 /*
1320 * Force PR and ALUA passthrough emulation with internal object use.
1321 */
1322 force_pt = (hba->hba_flags & HBA_FLAGS_INTERNAL_USE);
1323 /*
1324 * Setup the Reservations infrastructure for struct se_device
1325 */
1326 core_setup_reservations(dev, force_pt);
1327 /*
1328 * Setup the Asymmetric Logical Unit Assignment for struct se_device
1329 */
1330 if (core_setup_alua(dev, force_pt) < 0)
1331 goto out;
1332
1333 /*
1334 * Startup the struct se_device processing thread
1335 */
1336 dev->process_thread = kthread_run(transport_processing_thread, dev,
e3d6f909 1337 "LIO_%s", dev->transport->name);
c66ac9db 1338 if (IS_ERR(dev->process_thread)) {
6708bb27 1339 pr_err("Unable to create kthread: LIO_%s\n",
e3d6f909 1340 dev->transport->name);
c66ac9db
NB		 1341 goto out;
1342 }
07bde79a
NB		 1343 /*
1344 * Setup work_queue for QUEUE_FULL
1345 */
1346 INIT_WORK(&dev->qf_work_queue, target_qf_do_work);
c66ac9db
NB		 1347 /*
1348 * Preload the initial INQUIRY const values if we are doing
1349 * anything virtual (IBLOCK, FILEIO, RAMDISK), but not for TCM/pSCSI
1350 * passthrough because this is being provided by the backend LLD.
1351 * This is required so that transport_get_inquiry() copies these
1352 * originals once back into DEV_T10_WWN(dev) for the virtual device
1353 * setup.
1354 */
e3d6f909 1355 if (dev->transport->transport_type != TRANSPORT_PLUGIN_PHBA_PDEV) {
f22c1196 1356 if (!inquiry_prod || !inquiry_rev) {
6708bb27 1357 pr_err("All non TCM/pSCSI plugins require"
c66ac9db
NB		 1358 " INQUIRY consts\n");
1359 goto out;
1360 }
1361
e3d6f909
AG		 1362 strncpy(&dev->se_sub_dev->t10_wwn.vendor[0], "LIO-ORG", 8);
1363 strncpy(&dev->se_sub_dev->t10_wwn.model[0], inquiry_prod, 16);
1364 strncpy(&dev->se_sub_dev->t10_wwn.revision[0], inquiry_rev, 4);
c66ac9db
NB		 1365 }
1366 scsi_dump_inquiry(dev);
1367
12a18bdc 1368 return dev;
c66ac9db 1369out:
c66ac9db
NB		 1370 kthread_stop(dev->process_thread);
1371
1372 spin_lock(&hba->device_lock);
1373 list_del(&dev->dev_list);
1374 hba->dev_count--;
1375 spin_unlock(&hba->device_lock);
1376
1377 se_release_vpd_for_dev(dev);
1378
c66ac9db
NB		 1379 kfree(dev);
1380
1381 return NULL;
1382}
1383EXPORT_SYMBOL(transport_add_device_to_core_hba);
1384
1385/* transport_generic_prepare_cdb():
1386 *
1387 * Since the Initiator sees iSCSI devices as LUNs, the SCSI CDB will
1388 * contain the iSCSI LUN in bits 7-5 of byte 1 as per SAM-2.
1389 * The point of this is since we are mapping iSCSI LUNs to
1390 * SCSI Target IDs having a non-zero LUN in the CDB will throw the
1391 * devices and HBAs for a loop.
1392 */
1393static inline void transport_generic_prepare_cdb(
1394 unsigned char *cdb)
1395{
1396 switch (cdb[0]) {
1397 case READ_10: /* SBC - RDProtect */
1398 case READ_12: /* SBC - RDProtect */
1399 case READ_16: /* SBC - RDProtect */
1400 case SEND_DIAGNOSTIC: /* SPC - SELF-TEST Code */
1401 case VERIFY: /* SBC - VRProtect */
1402 case VERIFY_16: /* SBC - VRProtect */
1403 case WRITE_VERIFY: /* SBC - VRProtect */
1404 case WRITE_VERIFY_12: /* SBC - VRProtect */
1405 break;
1406 default:
1407 cdb[1] &= 0x1f; /* clear logical unit number */
1408 break;
1409 }
1410}
1411
1412static struct se_task *
1413transport_generic_get_task(struct se_cmd *cmd,
1414 enum dma_data_direction data_direction)
1415{
1416 struct se_task *task;
5951146d 1417 struct se_device *dev = cmd->se_dev;
c66ac9db 1418
6708bb27 1419 task = dev->transport->alloc_task(cmd->t_task_cdb);
c66ac9db 1420 if (!task) {
6708bb27 1421 pr_err("Unable to allocate struct se_task\n");
c66ac9db
NB		 1422 return NULL;
1423 }
1424
1425 INIT_LIST_HEAD(&task->t_list);
1426 INIT_LIST_HEAD(&task->t_execute_list);
1427 INIT_LIST_HEAD(&task->t_state_list);
1428 init_completion(&task->task_stop_comp);
c66ac9db 1429 task->task_se_cmd = cmd;
c66ac9db
NB		 1430 task->task_data_direction = data_direction;
1431
c66ac9db
NB		 1432 return task;
1433}
1434
1435static int transport_generic_cmd_sequencer(struct se_cmd *, unsigned char *);
1436
c66ac9db
NB		 1437/*
1438 * Used by fabric modules containing a local struct se_cmd within their
1439 * fabric dependent per I/O descriptor.
1440 */
1441void transport_init_se_cmd(
1442 struct se_cmd *cmd,
1443 struct target_core_fabric_ops *tfo,
1444 struct se_session *se_sess,
1445 u32 data_length,
1446 int data_direction,
1447 int task_attr,
1448 unsigned char *sense_buffer)
1449{
5951146d
AG		 1450 INIT_LIST_HEAD(&cmd->se_lun_node);
1451 INIT_LIST_HEAD(&cmd->se_delayed_node);
07bde79a 1452 INIT_LIST_HEAD(&cmd->se_qf_node);
79a7fef2 1453 INIT_LIST_HEAD(&cmd->se_queue_node);
a17f091d 1454 INIT_LIST_HEAD(&cmd->se_cmd_list);
a1d8b49a
AG		 1455 INIT_LIST_HEAD(&cmd->t_task_list);
1456 init_completion(&cmd->transport_lun_fe_stop_comp);
1457 init_completion(&cmd->transport_lun_stop_comp);
1458 init_completion(&cmd->t_transport_stop_comp);
a17f091d 1459 init_completion(&cmd->cmd_wait_comp);
a1d8b49a 1460 spin_lock_init(&cmd->t_state_lock);
7d680f3b 1461 cmd->transport_state = CMD_T_DEV_ACTIVE;
c66ac9db
NB		 1462
1463 cmd->se_tfo = tfo;
1464 cmd->se_sess = se_sess;
1465 cmd->data_length = data_length;
1466 cmd->data_direction = data_direction;
1467 cmd->sam_task_attr = task_attr;
1468 cmd->sense_buffer = sense_buffer;
1469}
1470EXPORT_SYMBOL(transport_init_se_cmd);
1471
1472static int transport_check_alloc_task_attr(struct se_cmd *cmd)
1473{
1474 /*
1475 * Check if SAM Task Attribute emulation is enabled for this
1476 * struct se_device storage object
1477 */
5951146d 1478 if (cmd->se_dev->dev_task_attr_type != SAM_TASK_ATTR_EMULATED)
c66ac9db
NB		 1479 return 0;
1480
e66ecd50 1481 if (cmd->sam_task_attr == MSG_ACA_TAG) {
6708bb27 1482 pr_debug("SAM Task Attribute ACA"
c66ac9db 1483 " emulation is not supported\n");
e3d6f909 1484 return -EINVAL;
c66ac9db
NB		 1485 }
1486 /*
1487 * Used to determine when ORDERED commands should go from
1488 * Dormant to Active status.
1489 */
5951146d 1490 cmd->se_ordered_id = atomic_inc_return(&cmd->se_dev->dev_ordered_id);
c66ac9db 1491 smp_mb__after_atomic_inc();
6708bb27 1492 pr_debug("Allocated se_ordered_id: %u for Task Attr: 0x%02x on %s\n",
c66ac9db 1493 cmd->se_ordered_id, cmd->sam_task_attr,
6708bb27 1494 cmd->se_dev->transport->name);
c66ac9db
NB		 1495 return 0;
1496}
1497
c66ac9db
NB		 1498/* transport_generic_allocate_tasks():
1499 *
1500 * Called from fabric RX Thread.
1501 */
1502int transport_generic_allocate_tasks(
1503 struct se_cmd *cmd,
1504 unsigned char *cdb)
1505{
1506 int ret;
1507
1508 transport_generic_prepare_cdb(cdb);
c66ac9db
NB		 1509 /*
1510 * Ensure that the received CDB is less than the max (252 + 8) bytes
1511 * for VARIABLE_LENGTH_CMD
1512 */
1513 if (scsi_command_size(cdb) > SCSI_MAX_VARLEN_CDB_SIZE) {
6708bb27 1514 pr_err("Received SCSI CDB with command_size: %d that"
c66ac9db
NB		 1515 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
1516 scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE);
03e98c9e
NB		 1517 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1518 cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD;
e3d6f909 1519 return -EINVAL;
c66ac9db
NB		 1520 }
1521 /*
1522 * If the received CDB is larger than TCM_MAX_COMMAND_SIZE,
1523 * allocate the additional extended CDB buffer now.. Otherwise
1524 * setup the pointer from __t_task_cdb to t_task_cdb.
1525 */
a1d8b49a
AG		 1526 if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) {
1527 cmd->t_task_cdb = kzalloc(scsi_command_size(cdb),
c66ac9db 1528 GFP_KERNEL);
6708bb27
AG		 1529 if (!cmd->t_task_cdb) {
1530 pr_err("Unable to allocate cmd->t_task_cdb"
a1d8b49a 1531 " %u > sizeof(cmd->__t_task_cdb): %lu ops\n",
c66ac9db 1532 scsi_command_size(cdb),
a1d8b49a 1533 (unsigned long)sizeof(cmd->__t_task_cdb));
03e98c9e
NB		 1534 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1535 cmd->scsi_sense_reason =
1536 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
e3d6f909 1537 return -ENOMEM;
c66ac9db
NB		 1538 }
1539 } else
a1d8b49a 1540 cmd->t_task_cdb = &cmd->__t_task_cdb[0];
c66ac9db 1541 /*
a1d8b49a 1542 * Copy the original CDB into cmd->
c66ac9db 1543 */
a1d8b49a 1544 memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb));
c66ac9db
NB		 1545 /*
1546 * Setup the received CDB based on SCSI defined opcodes and
1547 * perform unit attention, persistent reservations and ALUA
a1d8b49a 1548 * checks for virtual device backends. The cmd->t_task_cdb
c66ac9db
NB		 1549 * pointer is expected to be setup before we reach this point.
1550 */
1551 ret = transport_generic_cmd_sequencer(cmd, cdb);
1552 if (ret < 0)
1553 return ret;
1554 /*
1555 * Check for SAM Task Attribute Emulation
1556 */
1557 if (transport_check_alloc_task_attr(cmd) < 0) {
1558 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1559 cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD;
5951146d 1560 return -EINVAL;
c66ac9db
NB		 1561 }
1562 spin_lock(&cmd->se_lun->lun_sep_lock);
1563 if (cmd->se_lun->lun_sep)
1564 cmd->se_lun->lun_sep->sep_stats.cmd_pdus++;
1565 spin_unlock(&cmd->se_lun->lun_sep_lock);
1566 return 0;
1567}
1568EXPORT_SYMBOL(transport_generic_allocate_tasks);
1569
695434e1
NB		 1570/*
1571 * Used by fabric module frontends to queue tasks directly.
1572 * Many only be used from process context only
1573 */
1574int transport_handle_cdb_direct(
1575 struct se_cmd *cmd)
1576{
dd8ae59d
NB		 1577 int ret;
1578
695434e1
NB		 1579 if (!cmd->se_lun) {
1580 dump_stack();
6708bb27 1581 pr_err("cmd->se_lun is NULL\n");
695434e1
NB		 1582 return -EINVAL;
1583 }
1584 if (in_interrupt()) {
1585 dump_stack();
6708bb27 1586 pr_err("transport_generic_handle_cdb cannot be called"
695434e1
NB		 1587 " from interrupt context\n");
1588 return -EINVAL;
1589 }
dd8ae59d 1590 /*
7d680f3b 1591 * Set TRANSPORT_NEW_CMD state and CMD_T_ACTIVE following
dd8ae59d
NB		 1592 * transport_generic_handle_cdb*() -> transport_add_cmd_to_queue()
1593 * in existing usage to ensure that outstanding descriptors are handled
d14921d6 1594 * correctly during shutdown via transport_wait_for_tasks()
dd8ae59d
NB		 1595 *
1596 * Also, we don't take cmd->t_state_lock here as we only expect
1597 * this to be called for initial descriptor submission.
1598 */
1599 cmd->t_state = TRANSPORT_NEW_CMD;
7d680f3b
CH		 1600 cmd->transport_state |= CMD_T_ACTIVE;
1601
dd8ae59d
NB		 1602 /*
1603 * transport_generic_new_cmd() is already handling QUEUE_FULL,
1604 * so follow TRANSPORT_NEW_CMD processing thread context usage
1605 * and call transport_generic_request_failure() if necessary..
1606 */
1607 ret = transport_generic_new_cmd(cmd);
03e98c9e
NB		 1608 if (ret < 0)
1609 transport_generic_request_failure(cmd);
1610
dd8ae59d 1611 return 0;
695434e1
NB		 1612}
1613EXPORT_SYMBOL(transport_handle_cdb_direct);
1614
a6360785
NB		 1615/**
1616 * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd
1617 *
1618 * @se_cmd: command descriptor to submit
1619 * @se_sess: associated se_sess for endpoint
1620 * @cdb: pointer to SCSI CDB
1621 * @sense: pointer to SCSI sense buffer
1622 * @unpacked_lun: unpacked LUN to reference for struct se_lun
1623 * @data_length: fabric expected data transfer length
1624 * @task_addr: SAM task attribute
1625 * @data_dir: DMA data direction
1626 * @flags: flags for command submission from target_sc_flags_tables
1627 *
1628 * This may only be called from process context, and also currently
1629 * assumes internal allocation of fabric payload buffer by target-core.
1630 **/
1edcdb49 1631void target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess,
a6360785
NB		 1632 unsigned char *cdb, unsigned char *sense, u32 unpacked_lun,
1633 u32 data_length, int task_attr, int data_dir, int flags)
1634{
1635 struct se_portal_group *se_tpg;
1636 int rc;
1637
1638 se_tpg = se_sess->se_tpg;
1639 BUG_ON(!se_tpg);
1640 BUG_ON(se_cmd->se_tfo || se_cmd->se_sess);
1641 BUG_ON(in_interrupt());
1642 /*
1643 * Initialize se_cmd for target operation. From this point
1644 * exceptions are handled by sending exception status via
1645 * target_core_fabric_ops->queue_status() callback
1646 */
1647 transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
1648 data_length, data_dir, task_attr, sense);
1649 /*
1650 * Obtain struct se_cmd->cmd_kref reference and add new cmd to
1651 * se_sess->sess_cmd_list. A second kref_get here is necessary
1652 * for fabrics using TARGET_SCF_ACK_KREF that expect a second
1653 * kref_put() to happen during fabric packet acknowledgement.
1654 */
1655 target_get_sess_cmd(se_sess, se_cmd, (flags & TARGET_SCF_ACK_KREF));
1656 /*
1657 * Signal bidirectional data payloads to target-core
1658 */
1659 if (flags & TARGET_SCF_BIDI_OP)
1660 se_cmd->se_cmd_flags |= SCF_BIDI;
1661 /*
1662 * Locate se_lun pointer and attach it to struct se_cmd
1663 */
735703ca
NB		 1664 if (transport_lookup_cmd_lun(se_cmd, unpacked_lun) < 0) {
1665 transport_send_check_condition_and_sense(se_cmd,
1666 se_cmd->scsi_sense_reason, 0);
1667 target_put_sess_cmd(se_sess, se_cmd);
1668 return;
1669 }
a6360785
NB		 1670 /*
1671 * Sanitize CDBs via transport_generic_cmd_sequencer() and
1672 * allocate the necessary tasks to complete the received CDB+data
1673 */
1674 rc = transport_generic_allocate_tasks(se_cmd, cdb);
735703ca
NB		 1675 if (rc != 0) {
1676 transport_generic_request_failure(se_cmd);
1677 return;
1678 }
a6360785
NB		 1679 /*
1680 * Dispatch se_cmd descriptor to se_lun->lun_se_dev backend
1681 * for immediate execution of READs, otherwise wait for
1682 * transport_generic_handle_data() to be called for WRITEs
1683 * when fabric has filled the incoming buffer.
1684 */
1685 transport_handle_cdb_direct(se_cmd);
1edcdb49 1686 return;
a6360785
NB		 1687}
1688EXPORT_SYMBOL(target_submit_cmd);
1689
c66ac9db
NB		 1690/*
1691 * Used by fabric module frontends defining a TFO->new_cmd_map() caller
1692 * to queue up a newly setup se_cmd w/ TRANSPORT_NEW_CMD_MAP in order to
1693 * complete setup in TCM process context w/ TFO->new_cmd_map().
1694 */
1695int transport_generic_handle_cdb_map(
1696 struct se_cmd *cmd)
1697{
e3d6f909 1698 if (!cmd->se_lun) {
c66ac9db 1699 dump_stack();
6708bb27 1700 pr_err("cmd->se_lun is NULL\n");
e3d6f909 1701 return -EINVAL;
c66ac9db
NB		 1702 }
1703
f7a5cc0b 1704 transport_add_cmd_to_queue(cmd, TRANSPORT_NEW_CMD_MAP, false);
c66ac9db
NB		 1705 return 0;
1706}
1707EXPORT_SYMBOL(transport_generic_handle_cdb_map);
1708
1709/* transport_generic_handle_data():
1710 *
1711 *
1712 */
1713int transport_generic_handle_data(
1714 struct se_cmd *cmd)
1715{
1716 /*
1717 * For the software fabric case, then we assume the nexus is being
1718 * failed/shutdown when signals are pending from the kthread context
1719 * caller, so we return a failure. For the HW target mode case running
1720 * in interrupt code, the signal_pending() check is skipped.
1721 */
1722 if (!in_interrupt() && signal_pending(current))
e3d6f909 1723 return -EPERM;
c66ac9db
NB		 1724 /*
1725 * If the received CDB has aleady been ABORTED by the generic
1726 * target engine, we now call transport_check_aborted_status()
1727 * to queue any delated TASK_ABORTED status for the received CDB to the
25985edc 1728 * fabric module as we are expecting no further incoming DATA OUT
c66ac9db
NB		 1729 * sequences at this point.
1730 */
1731 if (transport_check_aborted_status(cmd, 1) != 0)
1732 return 0;
1733
f7a5cc0b 1734 transport_add_cmd_to_queue(cmd, TRANSPORT_PROCESS_WRITE, false);
c66ac9db
NB		 1735 return 0;
1736}
1737EXPORT_SYMBOL(transport_generic_handle_data);
1738
1739/* transport_generic_handle_tmr():
1740 *
1741 *
1742 */
1743int transport_generic_handle_tmr(
1744 struct se_cmd *cmd)
1745{
f7a5cc0b 1746 transport_add_cmd_to_queue(cmd, TRANSPORT_PROCESS_TMR, false);
c66ac9db
NB		 1747 return 0;
1748}
1749EXPORT_SYMBOL(transport_generic_handle_tmr);
1750
cdbb70bb
CH		 1751/*
1752 * If the task is active, request it to be stopped and sleep until it
1753 * has completed.
1754 */
1755bool target_stop_task(struct se_task *task, unsigned long *flags)
1756{
1757 struct se_cmd *cmd = task->task_se_cmd;
1758 bool was_active = false;
1759
1760 if (task->task_flags & TF_ACTIVE) {
1761 task->task_flags |= TF_REQUEST_STOP;
1762 spin_unlock_irqrestore(&cmd->t_state_lock, *flags);
1763
1764 pr_debug("Task %p waiting to complete\n", task);
1765 wait_for_completion(&task->task_stop_comp);
1766 pr_debug("Task %p stopped successfully\n", task);
1767
1768 spin_lock_irqsave(&cmd->t_state_lock, *flags);
1769 atomic_dec(&cmd->t_task_cdbs_left);
1770 task->task_flags &= ~(TF_ACTIVE | TF_REQUEST_STOP);
1771 was_active = true;
1772 }
1773
cdbb70bb
CH		 1774 return was_active;
1775}
1776
c66ac9db
NB		 1777static int transport_stop_tasks_for_cmd(struct se_cmd *cmd)
1778{
1779 struct se_task *task, *task_tmp;
1780 unsigned long flags;
1781 int ret = 0;
1782
6708bb27 1783 pr_debug("ITT[0x%08x] - Stopping tasks\n",
e3d6f909 1784 cmd->se_tfo->get_task_tag(cmd));
c66ac9db
NB		 1785
1786 /*
1787 * No tasks remain in the execution queue
1788 */
a1d8b49a 1789 spin_lock_irqsave(&cmd->t_state_lock, flags);
c66ac9db 1790 list_for_each_entry_safe(task, task_tmp,
a1d8b49a 1791 &cmd->t_task_list, t_list) {
04629b7b 1792 pr_debug("Processing task %p\n", task);
c66ac9db
NB		 1793 /*
1794 * If the struct se_task has not been sent and is not active,
1795 * remove the struct se_task from the execution queue.
1796 */
6c76bf95 1797 if (!(task->task_flags & (TF_ACTIVE | TF_SENT))) {
a1d8b49a 1798 spin_unlock_irqrestore(&cmd->t_state_lock,
c66ac9db
NB		 1799 flags);
1800 transport_remove_task_from_execute_queue(task,
42bf829e 1801 cmd->se_dev);
c66ac9db 1802
04629b7b 1803 pr_debug("Task %p removed from execute queue\n", task);
a1d8b49a 1804 spin_lock_irqsave(&cmd->t_state_lock, flags);
c66ac9db
NB		 1805 continue;
1806 }
1807
cdbb70bb 1808 if (!target_stop_task(task, &flags)) {
04629b7b 1809 pr_debug("Task %p - did nothing\n", task);
c66ac9db
NB		 1810 ret++;
1811 }
c66ac9db 1812 }
a1d8b49a 1813 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 1814
1815 return ret;
1816}
1817
c66ac9db
NB		 1818/*
1819 * Handle SAM-esque emulation for generic transport request failures.
1820 */
03e98c9e 1821static void transport_generic_request_failure(struct se_cmd *cmd)
c66ac9db 1822{
07bde79a
NB		 1823 int ret = 0;
1824
6708bb27 1825 pr_debug("-----[ Storage Engine Exception for cmd: %p ITT: 0x%08x"
e3d6f909 1826 " CDB: 0x%02x\n", cmd, cmd->se_tfo->get_task_tag(cmd),
a1d8b49a 1827 cmd->t_task_cdb[0]);
03e98c9e 1828 pr_debug("-----[ i_state: %d t_state: %d scsi_sense_reason: %d\n",
e3d6f909 1829 cmd->se_tfo->get_cmd_state(cmd),
03e98c9e 1830 cmd->t_state, cmd->scsi_sense_reason);
6708bb27 1831 pr_debug("-----[ t_tasks: %d t_task_cdbs_left: %d"
c66ac9db 1832 " t_task_cdbs_sent: %d t_task_cdbs_ex_left: %d --"
7d680f3b
CH		 1833 " CMD_T_ACTIVE: %d CMD_T_STOP: %d CMD_T_SENT: %d\n",
1834 cmd->t_task_list_num,
a1d8b49a
AG		 1835 atomic_read(&cmd->t_task_cdbs_left),
1836 atomic_read(&cmd->t_task_cdbs_sent),
1837 atomic_read(&cmd->t_task_cdbs_ex_left),
7d680f3b
CH		 1838 (cmd->transport_state & CMD_T_ACTIVE) != 0,
1839 (cmd->transport_state & CMD_T_STOP) != 0,
1840 (cmd->transport_state & CMD_T_SENT) != 0);
c66ac9db 1841
c66ac9db
NB		 1842 /*
1843 * For SAM Task Attribute emulation for failed struct se_cmd
1844 */
1845 if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
1846 transport_complete_task_attr(cmd);
1847
03e98c9e
NB		 1848 switch (cmd->scsi_sense_reason) {
1849 case TCM_NON_EXISTENT_LUN:
1850 case TCM_UNSUPPORTED_SCSI_OPCODE:
1851 case TCM_INVALID_CDB_FIELD:
1852 case TCM_INVALID_PARAMETER_LIST:
1853 case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE:
1854 case TCM_UNKNOWN_MODE_PAGE:
1855 case TCM_WRITE_PROTECTED:
1856 case TCM_CHECK_CONDITION_ABORT_CMD:
1857 case TCM_CHECK_CONDITION_UNIT_ATTENTION:
1858 case TCM_CHECK_CONDITION_NOT_READY:
c66ac9db 1859 break;
03e98c9e 1860 case TCM_RESERVATION_CONFLICT:
c66ac9db
NB		 1861 /*
1862 * No SENSE Data payload for this case, set SCSI Status
1863 * and queue the response to $FABRIC_MOD.
1864 *
1865 * Uses linux/include/scsi/scsi.h SAM status codes defs
1866 */
1867 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
1868 /*
1869 * For UA Interlock Code 11b, a RESERVATION CONFLICT will
1870 * establish a UNIT ATTENTION with PREVIOUS RESERVATION
1871 * CONFLICT STATUS.
1872 *
1873 * See spc4r17, section 7.4.6 Control Mode Page, Table 349
1874 */
e3d6f909
AG		 1875 if (cmd->se_sess &&
1876 cmd->se_dev->se_sub_dev->se_dev_attrib.emulate_ua_intlck_ctrl == 2)
1877 core_scsi3_ua_allocate(cmd->se_sess->se_node_acl,
c66ac9db
NB		 1878 cmd->orig_fe_lun, 0x2C,
1879 ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS);
1880
07bde79a 1881 ret = cmd->se_tfo->queue_status(cmd);
f147abb4 1882 if (ret == -EAGAIN || ret == -ENOMEM)
07bde79a 1883 goto queue_full;
c66ac9db 1884 goto check_stop;
c66ac9db 1885 default:
6708bb27 1886 pr_err("Unknown transport error for CDB 0x%02x: %d\n",
03e98c9e 1887 cmd->t_task_cdb[0], cmd->scsi_sense_reason);
c66ac9db
NB		 1888 cmd->scsi_sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
1889 break;
1890 }
16ab8e60
NB		 1891 /*
1892 * If a fabric does not define a cmd->se_tfo->new_cmd_map caller,
1893 * make the call to transport_send_check_condition_and_sense()
1894 * directly. Otherwise expect the fabric to make the call to
1895 * transport_send_check_condition_and_sense() after handling
1896 * possible unsoliticied write data payloads.
1897 */
03e98c9e
NB		 1898 ret = transport_send_check_condition_and_sense(cmd,
1899 cmd->scsi_sense_reason, 0);
1900 if (ret == -EAGAIN || ret == -ENOMEM)
1901 goto queue_full;
07bde79a 1902
c66ac9db
NB		 1903check_stop:
1904 transport_lun_remove_cmd(cmd);
6708bb27 1905 if (!transport_cmd_check_stop_to_fabric(cmd))
c66ac9db 1906 ;
07bde79a
NB		 1907 return;
1908
1909queue_full:
e057f533
CH		 1910 cmd->t_state = TRANSPORT_COMPLETE_QF_OK;
1911 transport_handle_queue_full(cmd, cmd->se_dev);
c66ac9db
NB		 1912}
1913
c66ac9db
NB		 1914static inline u32 transport_lba_21(unsigned char *cdb)
1915{
1916 return ((cdb[1] & 0x1f) << 16) | (cdb[2] << 8) | cdb[3];
1917}
1918
1919static inline u32 transport_lba_32(unsigned char *cdb)
1920{
1921 return (cdb[2] << 24) | (cdb[3] << 16) | (cdb[4] << 8) | cdb[5];
1922}
1923
1924static inline unsigned long long transport_lba_64(unsigned char *cdb)
1925{
1926 unsigned int __v1, __v2;
1927
1928 __v1 = (cdb[2] << 24) | (cdb[3] << 16) | (cdb[4] << 8) | cdb[5];
1929 __v2 = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9];
1930
1931 return ((unsigned long long)__v2) | (unsigned long long)__v1 << 32;
1932}
1933
1934/*
1935 * For VARIABLE_LENGTH_CDB w/ 32 byte extended CDBs
1936 */
1937static inline unsigned long long transport_lba_64_ext(unsigned char *cdb)
1938{
1939 unsigned int __v1, __v2;
1940
1941 __v1 = (cdb[12] << 24) | (cdb[13] << 16) | (cdb[14] << 8) | cdb[15];
1942 __v2 = (cdb[16] << 24) | (cdb[17] << 16) | (cdb[18] << 8) | cdb[19];
1943
1944 return ((unsigned long long)__v2) | (unsigned long long)__v1 << 32;
1945}
1946
1947static void transport_set_supported_SAM_opcode(struct se_cmd *se_cmd)
1948{
1949 unsigned long flags;
1950
a1d8b49a 1951 spin_lock_irqsave(&se_cmd->t_state_lock, flags);
c66ac9db 1952 se_cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE;
a1d8b49a 1953 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags);
c66ac9db
NB		 1954}
1955
c66ac9db
NB		 1956/*
1957 * Called from Fabric Module context from transport_execute_tasks()
1958 *
1959 * The return of this function determins if the tasks from struct se_cmd
1960 * get added to the execution queue in transport_execute_tasks(),
1961 * or are added to the delayed or ordered lists here.
1962 */
1963static inline int transport_execute_task_attr(struct se_cmd *cmd)
1964{
5951146d 1965 if (cmd->se_dev->dev_task_attr_type != SAM_TASK_ATTR_EMULATED)
c66ac9db
NB		 1966 return 1;
1967 /*
25985edc 1968 * Check for the existence of HEAD_OF_QUEUE, and if true return 1
c66ac9db
NB		 1969 * to allow the passed struct se_cmd list of tasks to the front of the list.
1970 */
e66ecd50 1971 if (cmd->sam_task_attr == MSG_HEAD_TAG) {
6708bb27 1972 pr_debug("Added HEAD_OF_QUEUE for CDB:"
c66ac9db 1973 " 0x%02x, se_ordered_id: %u\n",
6708bb27 1974 cmd->t_task_cdb[0],
c66ac9db
NB		 1975 cmd->se_ordered_id);
1976 return 1;
e66ecd50 1977 } else if (cmd->sam_task_attr == MSG_ORDERED_TAG) {
5951146d 1978 atomic_inc(&cmd->se_dev->dev_ordered_sync);
c66ac9db
NB		 1979 smp_mb__after_atomic_inc();
1980
6708bb27 1981 pr_debug("Added ORDERED for CDB: 0x%02x to ordered"
c66ac9db 1982 " list, se_ordered_id: %u\n",
a1d8b49a 1983 cmd->t_task_cdb[0],
c66ac9db
NB		 1984 cmd->se_ordered_id);
1985 /*
1986 * Add ORDERED command to tail of execution queue if
1987 * no other older commands exist that need to be
1988 * completed first.
1989 */
6708bb27 1990 if (!atomic_read(&cmd->se_dev->simple_cmds))
c66ac9db
NB		 1991 return 1;
1992 } else {
1993 /*
1994 * For SIMPLE and UNTAGGED Task Attribute commands
1995 */
5951146d 1996 atomic_inc(&cmd->se_dev->simple_cmds);
c66ac9db
NB		 1997 smp_mb__after_atomic_inc();
1998 }
1999 /*
2000 * Otherwise if one or more outstanding ORDERED task attribute exist,
2001 * add the dormant task(s) built for the passed struct se_cmd to the
2002 * execution queue and become in Active state for this struct se_device.
2003 */
5951146d 2004 if (atomic_read(&cmd->se_dev->dev_ordered_sync) != 0) {
c66ac9db
NB		 2005 /*
2006 * Otherwise, add cmd w/ tasks to delayed cmd queue that
25985edc 2007 * will be drained upon completion of HEAD_OF_QUEUE task.
c66ac9db 2008 */
5951146d 2009 spin_lock(&cmd->se_dev->delayed_cmd_lock);
c66ac9db 2010 cmd->se_cmd_flags |= SCF_DELAYED_CMD_FROM_SAM_ATTR;
5951146d
AG		 2011 list_add_tail(&cmd->se_delayed_node,
2012 &cmd->se_dev->delayed_cmd_list);
2013 spin_unlock(&cmd->se_dev->delayed_cmd_lock);
c66ac9db 2014
6708bb27 2015 pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to"
c66ac9db 2016 " delayed CMD list, se_ordered_id: %u\n",
a1d8b49a 2017 cmd->t_task_cdb[0], cmd->sam_task_attr,
c66ac9db
NB		 2018 cmd->se_ordered_id);
2019 /*
2020 * Return zero to let transport_execute_tasks() know
2021 * not to add the delayed tasks to the execution list.
2022 */
2023 return 0;
2024 }
2025 /*
2026 * Otherwise, no ORDERED task attributes exist..
2027 */
2028 return 1;
2029}
2030
2031/*
2032 * Called from fabric module context in transport_generic_new_cmd() and
2033 * transport_generic_process_write()
2034 */
2035static int transport_execute_tasks(struct se_cmd *cmd)
2036{
2037 int add_tasks;
40be67f4 2038 struct se_device *se_dev = cmd->se_dev;
c66ac9db
NB		 2039 /*
2040 * Call transport_cmd_check_stop() to see if a fabric exception
25985edc 2041 * has occurred that prevents execution.
c66ac9db 2042 */
6708bb27 2043 if (!transport_cmd_check_stop(cmd, 0, TRANSPORT_PROCESSING)) {
c66ac9db
NB		 2044 /*
2045 * Check for SAM Task Attribute emulation and HEAD_OF_QUEUE
2046 * attribute for the tasks of the received struct se_cmd CDB
2047 */
2048 add_tasks = transport_execute_task_attr(cmd);
e3d6f909 2049 if (!add_tasks)
c66ac9db
NB		 2050 goto execute_tasks;
2051 /*
4d2300cc
NB		 2052 * __transport_execute_tasks() -> __transport_add_tasks_from_cmd()
2053 * adds associated se_tasks while holding dev->execute_task_lock
2054 * before I/O dispath to avoid a double spinlock access.
c66ac9db 2055 */
4d2300cc
NB		 2056 __transport_execute_tasks(se_dev, cmd);
2057 return 0;
c66ac9db 2058 }
4d2300cc 2059
c66ac9db 2060execute_tasks:
4d2300cc 2061 __transport_execute_tasks(se_dev, NULL);
c66ac9db
NB		 2062 return 0;
2063}
2064
2065/*
2066 * Called to check struct se_device tcq depth window, and once open pull struct se_task
2067 * from struct se_device->execute_task_list and
2068 *
2069 * Called from transport_processing_thread()
2070 */
4d2300cc 2071static int __transport_execute_tasks(struct se_device *dev, struct se_cmd *new_cmd)
c66ac9db
NB		 2072{
2073 int error;
2074 struct se_cmd *cmd = NULL;
e3d6f909 2075 struct se_task *task = NULL;
c66ac9db
NB		 2076 unsigned long flags;
2077
c66ac9db 2078check_depth:
e3d6f909 2079 spin_lock_irq(&dev->execute_task_lock);
4d2300cc
NB		 2080 if (new_cmd != NULL)
2081 __transport_add_tasks_from_cmd(new_cmd);
2082
e3d6f909
AG		 2083 if (list_empty(&dev->execute_task_list)) {
2084 spin_unlock_irq(&dev->execute_task_lock);
c66ac9db
NB		 2085 return 0;
2086 }
e3d6f909
AG		 2087 task = list_first_entry(&dev->execute_task_list,
2088 struct se_task, t_execute_list);
04629b7b 2089 __transport_remove_task_from_execute_queue(task, dev);
e3d6f909 2090 spin_unlock_irq(&dev->execute_task_lock);
c66ac9db 2091
e3d6f909 2092 cmd = task->task_se_cmd;
a1d8b49a 2093 spin_lock_irqsave(&cmd->t_state_lock, flags);
6c76bf95 2094 task->task_flags |= (TF_ACTIVE | TF_SENT);
a1d8b49a 2095 atomic_inc(&cmd->t_task_cdbs_sent);
c66ac9db 2096
a1d8b49a
AG		 2097 if (atomic_read(&cmd->t_task_cdbs_sent) ==
2098 cmd->t_task_list_num)
7d680f3b 2099 cmd->transport_state |= CMD_T_SENT;
c66ac9db 2100
a1d8b49a 2101 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 2102
5bda90c8
CH		 2103 if (cmd->execute_task)
2104 error = cmd->execute_task(task);
2105 else
2106 error = dev->transport->do_task(task);
d29a5b6a 2107 if (error != 0) {
d29a5b6a
CH		 2108 spin_lock_irqsave(&cmd->t_state_lock, flags);
2109 task->task_flags &= ~TF_ACTIVE;
7d680f3b 2110 cmd->transport_state &= ~CMD_T_SENT;
d29a5b6a 2111 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
7d680f3b 2112
d29a5b6a 2113 transport_stop_tasks_for_cmd(cmd);
03e98c9e 2114 transport_generic_request_failure(cmd);
c66ac9db
NB		 2115 }
2116
4d2300cc 2117 new_cmd = NULL;
c66ac9db
NB		 2118 goto check_depth;
2119
2120 return 0;
2121}
2122
c66ac9db
NB		 2123static inline u32 transport_get_sectors_6(
2124 unsigned char *cdb,
2125 struct se_cmd *cmd,
2126 int *ret)
2127{
5951146d 2128 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 2129
2130 /*
2131 * Assume TYPE_DISK for non struct se_device objects.
2132 * Use 8-bit sector value.
2133 */
2134 if (!dev)
2135 goto type_disk;
2136
2137 /*
2138 * Use 24-bit allocation length for TYPE_TAPE.
2139 */
e3d6f909 2140 if (dev->transport->get_device_type(dev) == TYPE_TAPE)
c66ac9db
NB		 2141 return (u32)(cdb[2] << 16) + (cdb[3] << 8) + cdb[4];
2142
2143 /*
2144 * Everything else assume TYPE_DISK Sector CDB location.
9b5cd7f3
RD		 2145 * Use 8-bit sector value. SBC-3 says:
2146 *
2147 * A TRANSFER LENGTH field set to zero specifies that 256
2148 * logical blocks shall be written. Any other value
2149 * specifies the number of logical blocks that shall be
2150 * written.
c66ac9db
NB		 2151 */
2152type_disk:
9b5cd7f3 2153 return cdb[4] ? : 256;
c66ac9db
NB		 2154}
2155
2156static inline u32 transport_get_sectors_10(
2157 unsigned char *cdb,
2158 struct se_cmd *cmd,
2159 int *ret)
2160{
5951146d 2161 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 2162
2163 /*
2164 * Assume TYPE_DISK for non struct se_device objects.
2165 * Use 16-bit sector value.
2166 */
2167 if (!dev)
2168 goto type_disk;
2169
2170 /*
2171 * XXX_10 is not defined in SSC, throw an exception
2172 */
e3d6f909
AG		 2173 if (dev->transport->get_device_type(dev) == TYPE_TAPE) {
2174 *ret = -EINVAL;
c66ac9db
NB		 2175 return 0;
2176 }
2177
2178 /*
2179 * Everything else assume TYPE_DISK Sector CDB location.
2180 * Use 16-bit sector value.
2181 */
2182type_disk:
2183 return (u32)(cdb[7] << 8) + cdb[8];
2184}
2185
2186static inline u32 transport_get_sectors_12(
2187 unsigned char *cdb,
2188 struct se_cmd *cmd,
2189 int *ret)
2190{
5951146d 2191 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 2192
2193 /*
2194 * Assume TYPE_DISK for non struct se_device objects.
2195 * Use 32-bit sector value.
2196 */
2197 if (!dev)
2198 goto type_disk;
2199
2200 /*
2201 * XXX_12 is not defined in SSC, throw an exception
2202 */
e3d6f909
AG		 2203 if (dev->transport->get_device_type(dev) == TYPE_TAPE) {
2204 *ret = -EINVAL;
c66ac9db
NB		 2205 return 0;
2206 }
2207
2208 /*
2209 * Everything else assume TYPE_DISK Sector CDB location.
2210 * Use 32-bit sector value.
2211 */
2212type_disk:
2213 return (u32)(cdb[6] << 24) + (cdb[7] << 16) + (cdb[8] << 8) + cdb[9];
2214}
2215
2216static inline u32 transport_get_sectors_16(
2217 unsigned char *cdb,
2218 struct se_cmd *cmd,
2219 int *ret)
2220{
5951146d 2221 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 2222
2223 /*
2224 * Assume TYPE_DISK for non struct se_device objects.
2225 * Use 32-bit sector value.
2226 */
2227 if (!dev)
2228 goto type_disk;
2229
2230 /*
2231 * Use 24-bit allocation length for TYPE_TAPE.
2232 */
e3d6f909 2233 if (dev->transport->get_device_type(dev) == TYPE_TAPE)
c66ac9db
NB		 2234 return (u32)(cdb[12] << 16) + (cdb[13] << 8) + cdb[14];
2235
2236type_disk:
2237 return (u32)(cdb[10] << 24) + (cdb[11] << 16) +
2238 (cdb[12] << 8) + cdb[13];
2239}
2240
2241/*
2242 * Used for VARIABLE_LENGTH_CDB WRITE_32 and READ_32 variants
2243 */
2244static inline u32 transport_get_sectors_32(
2245 unsigned char *cdb,
2246 struct se_cmd *cmd,
2247 int *ret)
2248{
2249 /*
2250 * Assume TYPE_DISK for non struct se_device objects.
2251 * Use 32-bit sector value.
2252 */
2253 return (u32)(cdb[28] << 24) + (cdb[29] << 16) +
2254 (cdb[30] << 8) + cdb[31];
2255
2256}
2257
2258static inline u32 transport_get_size(
2259 u32 sectors,
2260 unsigned char *cdb,
2261 struct se_cmd *cmd)
2262{
5951146d 2263 struct se_device *dev = cmd->se_dev;
c66ac9db 2264
e3d6f909 2265 if (dev->transport->get_device_type(dev) == TYPE_TAPE) {
c66ac9db 2266 if (cdb[1] & 1) { /* sectors */
e3d6f909 2267 return dev->se_sub_dev->se_dev_attrib.block_size * sectors;
c66ac9db
NB		 2268 } else /* bytes */
2269 return sectors;
2270 }
2271#if 0
6708bb27 2272 pr_debug("Returning block_size: %u, sectors: %u == %u for"
e3d6f909
AG		 2273 " %s object\n", dev->se_sub_dev->se_dev_attrib.block_size, sectors,
2274 dev->se_sub_dev->se_dev_attrib.block_size * sectors,
2275 dev->transport->name);
c66ac9db 2276#endif
e3d6f909 2277 return dev->se_sub_dev->se_dev_attrib.block_size * sectors;
c66ac9db
NB		 2278}
2279
c66ac9db
NB		 2280static void transport_xor_callback(struct se_cmd *cmd)
2281{
2282 unsigned char *buf, *addr;
ec98f782 2283 struct scatterlist *sg;
c66ac9db
NB		 2284 unsigned int offset;
2285 int i;
ec98f782 2286 int count;
c66ac9db
NB		 2287 /*
2288 * From sbc3r22.pdf section 5.48 XDWRITEREAD (10) command
2289 *
2290 * 1) read the specified logical block(s);
2291 * 2) transfer logical blocks from the data-out buffer;
2292 * 3) XOR the logical blocks transferred from the data-out buffer with
2293 * the logical blocks read, storing the resulting XOR data in a buffer;
2294 * 4) if the DISABLE WRITE bit is set to zero, then write the logical
2295 * blocks transferred from the data-out buffer; and
2296 * 5) transfer the resulting XOR data to the data-in buffer.
2297 */
2298 buf = kmalloc(cmd->data_length, GFP_KERNEL);
6708bb27
AG		 2299 if (!buf) {
2300 pr_err("Unable to allocate xor_callback buf\n");
c66ac9db
NB		 2301 return;
2302 }
2303 /*
ec98f782 2304 * Copy the scatterlist WRITE buffer located at cmd->t_data_sg
c66ac9db
NB		 2305 * into the locally allocated *buf
2306 */
ec98f782
AG		 2307 sg_copy_to_buffer(cmd->t_data_sg,
2308 cmd->t_data_nents,
2309 buf,
2310 cmd->data_length);
2311
c66ac9db
NB		 2312 /*
2313 * Now perform the XOR against the BIDI read memory located at
a1d8b49a 2314 * cmd->t_mem_bidi_list
c66ac9db
NB		 2315 */
2316
2317 offset = 0;
ec98f782
AG		 2318 for_each_sg(cmd->t_bidi_data_sg, sg, cmd->t_bidi_data_nents, count) {
2319 addr = kmap_atomic(sg_page(sg), KM_USER0);
2320 if (!addr)
c66ac9db
NB		 2321 goto out;
2322
ec98f782
AG		 2323 for (i = 0; i < sg->length; i++)
2324 *(addr + sg->offset + i) ^= *(buf + offset + i);
c66ac9db 2325
ec98f782 2326 offset += sg->length;
c66ac9db
NB		 2327 kunmap_atomic(addr, KM_USER0);
2328 }
ec98f782 2329
c66ac9db
NB		 2330out:
2331 kfree(buf);
2332}
2333
2334/*
2335 * Used to obtain Sense Data from underlying Linux/SCSI struct scsi_cmnd
2336 */
2337static int transport_get_sense_data(struct se_cmd *cmd)
2338{
2339 unsigned char *buffer = cmd->sense_buffer, *sense_buffer = NULL;
42bf829e 2340 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 2341 struct se_task *task = NULL, *task_tmp;
2342 unsigned long flags;
2343 u32 offset = 0;
2344
e3d6f909
AG		 2345 WARN_ON(!cmd->se_lun);
2346
42bf829e
CH		 2347 if (!dev)
2348 return 0;
2349
a1d8b49a 2350 spin_lock_irqsave(&cmd->t_state_lock, flags);
c66ac9db 2351 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
a1d8b49a 2352 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 2353 return 0;
2354 }
2355
2356 list_for_each_entry_safe(task, task_tmp,
a1d8b49a 2357 &cmd->t_task_list, t_list) {
ef804a84 2358 if (!(task->task_flags & TF_HAS_SENSE))
c66ac9db
NB		 2359 continue;
2360
e3d6f909 2361 if (!dev->transport->get_sense_buffer) {
6708bb27 2362 pr_err("dev->transport->get_sense_buffer"
c66ac9db
NB		 2363 " is NULL\n");
2364 continue;
2365 }
2366
e3d6f909 2367 sense_buffer = dev->transport->get_sense_buffer(task);
6708bb27 2368 if (!sense_buffer) {
04629b7b 2369 pr_err("ITT[0x%08x]_TASK[%p]: Unable to locate"
c66ac9db 2370 " sense buffer for task with sense\n",
04629b7b 2371 cmd->se_tfo->get_task_tag(cmd), task);
c66ac9db
NB		 2372 continue;
2373 }
a1d8b49a 2374 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 2375
e3d6f909 2376 offset = cmd->se_tfo->set_fabric_sense_len(cmd,
c66ac9db
NB		 2377 TRANSPORT_SENSE_BUFFER);
2378
5951146d 2379 memcpy(&buffer[offset], sense_buffer,
c66ac9db
NB		 2380 TRANSPORT_SENSE_BUFFER);
2381 cmd->scsi_status = task->task_scsi_status;
2382 /* Automatically padded */
2383 cmd->scsi_sense_length =
2384 (TRANSPORT_SENSE_BUFFER + offset);
2385
6708bb27 2386 pr_debug("HBA_[%u]_PLUG[%s]: Set SAM STATUS: 0x%02x"
c66ac9db 2387 " and sense\n",
e3d6f909 2388 dev->se_hba->hba_id, dev->transport->name,
c66ac9db
NB		 2389 cmd->scsi_status);
2390 return 0;
2391 }
a1d8b49a 2392 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 2393
2394 return -1;
2395}
2396
ec98f782
AG		 2397static inline long long transport_dev_end_lba(struct se_device *dev)
2398{
2399 return dev->transport->get_blocks(dev) + 1;
2400}
2401
2402static int transport_cmd_get_valid_sectors(struct se_cmd *cmd)
2403{
2404 struct se_device *dev = cmd->se_dev;
2405 u32 sectors;
2406
2407 if (dev->transport->get_device_type(dev) != TYPE_DISK)
2408 return 0;
2409
2410 sectors = (cmd->data_length / dev->se_sub_dev->se_dev_attrib.block_size);
2411
6708bb27
AG		 2412 if ((cmd->t_task_lba + sectors) > transport_dev_end_lba(dev)) {
2413 pr_err("LBA: %llu Sectors: %u exceeds"
ec98f782
AG		 2414 " transport_dev_end_lba(): %llu\n",
2415 cmd->t_task_lba, sectors,
2416 transport_dev_end_lba(dev));
7abbe7f3 2417 return -EINVAL;
ec98f782
AG		 2418 }
2419
7abbe7f3 2420 return 0;
ec98f782
AG		 2421}
2422
706d5860
NB		 2423static int target_check_write_same_discard(unsigned char *flags, struct se_device *dev)
2424{
2425 /*
2426 * Determine if the received WRITE_SAME is used to for direct
2427 * passthrough into Linux/SCSI with struct request via TCM/pSCSI
2428 * or we are signaling the use of internal WRITE_SAME + UNMAP=1
2429 * emulation for -> Linux/BLOCK disbard with TCM/IBLOCK code.
2430 */
2431 int passthrough = (dev->transport->transport_type ==
2432 TRANSPORT_PLUGIN_PHBA_PDEV);
2433
2434 if (!passthrough) {
2435 if ((flags[0] & 0x04) || (flags[0] & 0x02)) {
2436 pr_err("WRITE_SAME PBDATA and LBDATA"
2437 " bits not supported for Block Discard"
2438 " Emulation\n");
2439 return -ENOSYS;
2440 }
2441 /*
2442 * Currently for the emulated case we only accept
2443 * tpws with the UNMAP=1 bit set.
2444 */
2445 if (!(flags[0] & 0x08)) {
2446 pr_err("WRITE_SAME w/o UNMAP bit not"
2447 " supported for Block Discard Emulation\n");
2448 return -ENOSYS;
2449 }
2450 }
2451
2452 return 0;
2453}
2454
c66ac9db
NB		 2455/* transport_generic_cmd_sequencer():
2456 *
2457 * Generic Command Sequencer that should work for most DAS transport
2458 * drivers.
2459 *
2460 * Called from transport_generic_allocate_tasks() in the $FABRIC_MOD
2461 * RX Thread.
2462 *
2463 * FIXME: Need to support other SCSI OPCODES where as well.
2464 */
2465static int transport_generic_cmd_sequencer(
2466 struct se_cmd *cmd,
2467 unsigned char *cdb)
2468{
5951146d 2469 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 2470 struct se_subsystem_dev *su_dev = dev->se_sub_dev;
2471 int ret = 0, sector_ret = 0, passthrough;
2472 u32 sectors = 0, size = 0, pr_reg_type = 0;
2473 u16 service_action;
2474 u8 alua_ascq = 0;
2475 /*
2476 * Check for an existing UNIT ATTENTION condition
2477 */
2478 if (core_scsi3_ua_check(cmd, cdb) < 0) {
c66ac9db
NB		 2479 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
2480 cmd->scsi_sense_reason = TCM_CHECK_CONDITION_UNIT_ATTENTION;
5951146d 2481 return -EINVAL;
c66ac9db
NB		 2482 }
2483 /*
2484 * Check status of Asymmetric Logical Unit Assignment port
2485 */
e3d6f909 2486 ret = su_dev->t10_alua.alua_state_check(cmd, cdb, &alua_ascq);
c66ac9db 2487 if (ret != 0) {
c66ac9db 2488 /*
25985edc 2489 * Set SCSI additional sense code (ASC) to 'LUN Not Accessible';
c66ac9db
NB		 2490 * The ALUA additional sense code qualifier (ASCQ) is determined
2491 * by the ALUA primary or secondary access state..
2492 */
2493 if (ret > 0) {
2494#if 0
6708bb27 2495 pr_debug("[%s]: ALUA TG Port not available,"
c66ac9db 2496 " SenseKey: NOT_READY, ASC/ASCQ: 0x04/0x%02x\n",
e3d6f909 2497 cmd->se_tfo->get_fabric_name(), alua_ascq);
c66ac9db
NB		 2498#endif
2499 transport_set_sense_codes(cmd, 0x04, alua_ascq);
2500 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
2501 cmd->scsi_sense_reason = TCM_CHECK_CONDITION_NOT_READY;
5951146d 2502 return -EINVAL;
c66ac9db
NB		 2503 }
2504 goto out_invalid_cdb_field;
2505 }
2506 /*
2507 * Check status for SPC-3 Persistent Reservations
2508 */
e3d6f909
AG		 2509 if (su_dev->t10_pr.pr_ops.t10_reservation_check(cmd, &pr_reg_type) != 0) {
2510 if (su_dev->t10_pr.pr_ops.t10_seq_non_holder(
03e98c9e
NB		 2511 cmd, cdb, pr_reg_type) != 0) {
2512 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
2513 cmd->se_cmd_flags |= SCF_SCSI_RESERVATION_CONFLICT;
2514 cmd->scsi_sense_reason = TCM_RESERVATION_CONFLICT;
2515 return -EBUSY;
2516 }
c66ac9db
NB		 2517 /*
2518 * This means the CDB is allowed for the SCSI Initiator port
2519 * when said port is *NOT* holding the legacy SPC-2 or
2520 * SPC-3 Persistent Reservation.
2521 */
2522 }
2523
5bda90c8
CH		 2524 /*
2525 * If we operate in passthrough mode we skip most CDB emulation and
2526 * instead hand the commands down to the physical SCSI device.
2527 */
2528 passthrough =
2529 (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV);
2530
c66ac9db
NB		 2531 switch (cdb[0]) {
2532 case READ_6:
2533 sectors = transport_get_sectors_6(cdb, cmd, &sector_ret);
2534 if (sector_ret)
2535 goto out_unsupported_cdb;
2536 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2537 cmd->t_task_lba = transport_lba_21(cdb);
c66ac9db
NB		 2538 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2539 break;
2540 case READ_10:
2541 sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
2542 if (sector_ret)
2543 goto out_unsupported_cdb;
2544 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2545 cmd->t_task_lba = transport_lba_32(cdb);
c66ac9db
NB		 2546 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2547 break;
2548 case READ_12:
2549 sectors = transport_get_sectors_12(cdb, cmd, &sector_ret);
2550 if (sector_ret)
2551 goto out_unsupported_cdb;
2552 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2553 cmd->t_task_lba = transport_lba_32(cdb);
c66ac9db
NB		 2554 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2555 break;
2556 case READ_16:
2557 sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
2558 if (sector_ret)
2559 goto out_unsupported_cdb;
2560 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2561 cmd->t_task_lba = transport_lba_64(cdb);
c66ac9db
NB		 2562 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2563 break;
2564 case WRITE_6:
2565 sectors = transport_get_sectors_6(cdb, cmd, &sector_ret);
2566 if (sector_ret)
2567 goto out_unsupported_cdb;
2568 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2569 cmd->t_task_lba = transport_lba_21(cdb);
c66ac9db
NB		 2570 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2571 break;
2572 case WRITE_10:
2573 sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
2574 if (sector_ret)
2575 goto out_unsupported_cdb;
2576 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2577 cmd->t_task_lba = transport_lba_32(cdb);
2d3a4b51
CH		 2578 if (cdb[1] & 0x8)
2579 cmd->se_cmd_flags |= SCF_FUA;
c66ac9db
NB		 2580 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2581 break;
2582 case WRITE_12:
2583 sectors = transport_get_sectors_12(cdb, cmd, &sector_ret);
2584 if (sector_ret)
2585 goto out_unsupported_cdb;
2586 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2587 cmd->t_task_lba = transport_lba_32(cdb);
2d3a4b51
CH		 2588 if (cdb[1] & 0x8)
2589 cmd->se_cmd_flags |= SCF_FUA;
c66ac9db
NB		 2590 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2591 break;
2592 case WRITE_16:
2593 sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
2594 if (sector_ret)
2595 goto out_unsupported_cdb;
2596 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2597 cmd->t_task_lba = transport_lba_64(cdb);
2d3a4b51
CH		 2598 if (cdb[1] & 0x8)
2599 cmd->se_cmd_flags |= SCF_FUA;
c66ac9db
NB		 2600 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2601 break;
2602 case XDWRITEREAD_10:
2603 if ((cmd->data_direction != DMA_TO_DEVICE) ||
33c3fafc 2604 !(cmd->se_cmd_flags & SCF_BIDI))
c66ac9db
NB		 2605 goto out_invalid_cdb_field;
2606 sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
2607 if (sector_ret)
2608 goto out_unsupported_cdb;
2609 size = transport_get_size(sectors, cdb, cmd);
a1d8b49a 2610 cmd->t_task_lba = transport_lba_32(cdb);
c66ac9db 2611 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
7c1c6af3 2612
5bda90c8
CH		 2613 /*
2614 * Do now allow BIDI commands for passthrough mode.
2615 */
2616 if (passthrough)
7c1c6af3 2617 goto out_unsupported_cdb;
5bda90c8 2618
c66ac9db 2619 /*
35e0e757 2620 * Setup BIDI XOR callback to be run after I/O completion.
c66ac9db
NB		 2621 */
2622 cmd->transport_complete_callback = &transport_xor_callback;
2d3a4b51
CH		 2623 if (cdb[1] & 0x8)
2624 cmd->se_cmd_flags |= SCF_FUA;
c66ac9db
NB		 2625 break;
2626 case VARIABLE_LENGTH_CMD:
2627 service_action = get_unaligned_be16(&cdb[8]);
c66ac9db
NB		 2628 switch (service_action) {
2629 case XDWRITEREAD_32:
2630 sectors = transport_get_sectors_32(cdb, cmd, &sector_ret);
2631 if (sector_ret)
2632 goto out_unsupported_cdb;
2633 size = transport_get_size(sectors, cdb, cmd);
2634 /*
2635 * Use WRITE_32 and READ_32 opcodes for the emulated
2636 * XDWRITE_READ_32 logic.
2637 */
a1d8b49a 2638 cmd->t_task_lba = transport_lba_64_ext(cdb);
c66ac9db
NB		 2639 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2640
5bda90c8
CH		 2641 /*
2642 * Do now allow BIDI commands for passthrough mode.
2643 */
c66ac9db 2644 if (passthrough)
7c1c6af3 2645 goto out_unsupported_cdb;
5bda90c8 2646
c66ac9db 2647 /*
35e0e757
CH		 2648 * Setup BIDI XOR callback to be run during after I/O
2649 * completion.
c66ac9db
NB		 2650 */
2651 cmd->transport_complete_callback = &transport_xor_callback;
2d3a4b51
CH		 2652 if (cdb[1] & 0x8)
2653 cmd->se_cmd_flags |= SCF_FUA;
c66ac9db
NB		 2654 break;
2655 case WRITE_SAME_32:
2656 sectors = transport_get_sectors_32(cdb, cmd, &sector_ret);
2657 if (sector_ret)
2658 goto out_unsupported_cdb;
dd3a5ad8 2659
6708bb27 2660 if (sectors)
12850626 2661 size = transport_get_size(1, cdb, cmd);
6708bb27
AG		 2662 else {
2663 pr_err("WSNZ=1, WRITE_SAME w/sectors=0 not"
2664 " supported\n");
2665 goto out_invalid_cdb_field;
2666 }
dd3a5ad8 2667
a1d8b49a 2668 cmd->t_task_lba = get_unaligned_be64(&cdb[12]);
c66ac9db
NB		 2669 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2670
706d5860 2671 if (target_check_write_same_discard(&cdb[10], dev) < 0)
67236c44 2672 goto out_unsupported_cdb;
5bda90c8
CH		 2673 if (!passthrough)
2674 cmd->execute_task = target_emulate_write_same;
c66ac9db
NB		 2675 break;
2676 default:
6708bb27 2677 pr_err("VARIABLE_LENGTH_CMD service action"
c66ac9db
NB		 2678 " 0x%04x not supported\n", service_action);
2679 goto out_unsupported_cdb;
2680 }
2681 break;
e434f1f1 2682 case MAINTENANCE_IN:
e3d6f909 2683 if (dev->transport->get_device_type(dev) != TYPE_ROM) {
c66ac9db
NB		 2684 /* MAINTENANCE_IN from SCC-2 */
2685 /*
2686 * Check for emulated MI_REPORT_TARGET_PGS.
2687 */
e76a35d6
CH		 2688 if (cdb[1] == MI_REPORT_TARGET_PGS &&
2689 su_dev->t10_alua.alua_type == SPC3_ALUA_EMULATED) {
2690 cmd->execute_task =
2691 target_emulate_report_target_port_groups;
c66ac9db
NB		 2692 }
2693 size = (cdb[6] << 24) | (cdb[7] << 16) |
2694 (cdb[8] << 8) | cdb[9];
2695 } else {
2696 /* GPCMD_SEND_KEY from multi media commands */
2697 size = (cdb[8] << 8) + cdb[9];
2698 }
05d1c7c0 2699 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2700 break;
2701 case MODE_SELECT:
2702 size = cdb[4];
2703 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2704 break;
2705 case MODE_SELECT_10:
2706 size = (cdb[7] << 8) + cdb[8];
2707 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2708 break;
2709 case MODE_SENSE:
2710 size = cdb[4];
05d1c7c0 2711 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
5bda90c8
CH		 2712 if (!passthrough)
2713 cmd->execute_task = target_emulate_modesense;
c66ac9db
NB		 2714 break;
2715 case MODE_SENSE_10:
5bda90c8
CH		 2716 size = (cdb[7] << 8) + cdb[8];
2717 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2718 if (!passthrough)
2719 cmd->execute_task = target_emulate_modesense;
2720 break;
c66ac9db
NB		 2721 case GPCMD_READ_BUFFER_CAPACITY:
2722 case GPCMD_SEND_OPC:
2723 case LOG_SELECT:
2724 case LOG_SENSE:
2725 size = (cdb[7] << 8) + cdb[8];
05d1c7c0 2726 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2727 break;
2728 case READ_BLOCK_LIMITS:
2729 size = READ_BLOCK_LEN;
05d1c7c0 2730 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2731 break;
2732 case GPCMD_GET_CONFIGURATION:
2733 case GPCMD_READ_FORMAT_CAPACITIES:
2734 case GPCMD_READ_DISC_INFO:
2735 case GPCMD_READ_TRACK_RZONE_INFO:
2736 size = (cdb[7] << 8) + cdb[8];
2737 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2738 break;
2739 case PERSISTENT_RESERVE_IN:
617c0e06 2740 if (su_dev->t10_pr.res_type == SPC3_PERSISTENT_RESERVATIONS)
e76a35d6 2741 cmd->execute_task = target_scsi3_emulate_pr_in;
617c0e06
CH		 2742 size = (cdb[7] << 8) + cdb[8];
2743 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2744 break;
c66ac9db 2745 case PERSISTENT_RESERVE_OUT:
617c0e06 2746 if (su_dev->t10_pr.res_type == SPC3_PERSISTENT_RESERVATIONS)
e76a35d6 2747 cmd->execute_task = target_scsi3_emulate_pr_out;
c66ac9db 2748 size = (cdb[7] << 8) + cdb[8];
05d1c7c0 2749 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2750 break;
2751 case GPCMD_MECHANISM_STATUS:
2752 case GPCMD_READ_DVD_STRUCTURE:
2753 size = (cdb[8] << 8) + cdb[9];
2754 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2755 break;
2756 case READ_POSITION:
2757 size = READ_POSITION_LEN;
05d1c7c0 2758 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db 2759 break;
e434f1f1 2760 case MAINTENANCE_OUT:
e3d6f909 2761 if (dev->transport->get_device_type(dev) != TYPE_ROM) {
c66ac9db
NB		 2762 /* MAINTENANCE_OUT from SCC-2
2763 *
2764 * Check for emulated MO_SET_TARGET_PGS.
2765 */
e76a35d6
CH		 2766 if (cdb[1] == MO_SET_TARGET_PGS &&
2767 su_dev->t10_alua.alua_type == SPC3_ALUA_EMULATED) {
2768 cmd->execute_task =
2769 target_emulate_set_target_port_groups;
c66ac9db
NB		 2770 }
2771
2772 size = (cdb[6] << 24) | (cdb[7] << 16) |
2773 (cdb[8] << 8) | cdb[9];
2774 } else {
2775 /* GPCMD_REPORT_KEY from multi media commands */
2776 size = (cdb[8] << 8) + cdb[9];
2777 }
05d1c7c0 2778 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2779 break;
2780 case INQUIRY:
2781 size = (cdb[3] << 8) + cdb[4];
2782 /*
2783 * Do implict HEAD_OF_QUEUE processing for INQUIRY.
2784 * See spc4r17 section 5.3
2785 */
5951146d 2786 if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
e66ecd50 2787 cmd->sam_task_attr = MSG_HEAD_TAG;
05d1c7c0 2788 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
5bda90c8
CH		 2789 if (!passthrough)
2790 cmd->execute_task = target_emulate_inquiry;
c66ac9db
NB		 2791 break;
2792 case READ_BUFFER:
2793 size = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8];
05d1c7c0 2794 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2795 break;
2796 case READ_CAPACITY:
2797 size = READ_CAP_LEN;
05d1c7c0 2798 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
5bda90c8
CH		 2799 if (!passthrough)
2800 cmd->execute_task = target_emulate_readcapacity;
c66ac9db
NB		 2801 break;
2802 case READ_MEDIA_SERIAL_NUMBER:
2803 case SECURITY_PROTOCOL_IN:
2804 case SECURITY_PROTOCOL_OUT:
2805 size = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9];
05d1c7c0 2806 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2807 break;
2808 case SERVICE_ACTION_IN:
5bda90c8
CH		 2809 switch (cmd->t_task_cdb[1] & 0x1f) {
2810 case SAI_READ_CAPACITY_16:
2811 if (!passthrough)
2812 cmd->execute_task =
2813 target_emulate_readcapacity_16;
2814 break;
2815 default:
2816 if (passthrough)
2817 break;
2818
2819 pr_err("Unsupported SA: 0x%02x\n",
2820 cmd->t_task_cdb[1] & 0x1f);
2821 goto out_unsupported_cdb;
2822 }
2823 /*FALLTHROUGH*/
c66ac9db
NB		 2824 case ACCESS_CONTROL_IN:
2825 case ACCESS_CONTROL_OUT:
2826 case EXTENDED_COPY:
2827 case READ_ATTRIBUTE:
2828 case RECEIVE_COPY_RESULTS:
2829 case WRITE_ATTRIBUTE:
2830 size = (cdb[10] << 24) | (cdb[11] << 16) |
2831 (cdb[12] << 8) | cdb[13];
05d1c7c0 2832 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2833 break;
2834 case RECEIVE_DIAGNOSTIC:
2835 case SEND_DIAGNOSTIC:
2836 size = (cdb[3] << 8) | cdb[4];
05d1c7c0 2837 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2838 break;
2839/* #warning FIXME: Figure out correct GPCMD_READ_CD blocksize. */
2840#if 0
2841 case GPCMD_READ_CD:
2842 sectors = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8];
2843 size = (2336 * sectors);
05d1c7c0 2844 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2845 break;
2846#endif
2847 case READ_TOC:
2848 size = cdb[8];
05d1c7c0 2849 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2850 break;
2851 case REQUEST_SENSE:
2852 size = cdb[4];
05d1c7c0 2853 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
5bda90c8
CH		 2854 if (!passthrough)
2855 cmd->execute_task = target_emulate_request_sense;
c66ac9db
NB		 2856 break;
2857 case READ_ELEMENT_STATUS:
2858 size = 65536 * cdb[7] + 256 * cdb[8] + cdb[9];
05d1c7c0 2859 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2860 break;
2861 case WRITE_BUFFER:
2862 size = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8];
05d1c7c0 2863 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 2864 break;
2865 case RESERVE:
2866 case RESERVE_10:
2867 /*
2868 * The SPC-2 RESERVE does not contain a size in the SCSI CDB.
2869 * Assume the passthrough or $FABRIC_MOD will tell us about it.
2870 */
2871 if (cdb[0] == RESERVE_10)
2872 size = (cdb[7] << 8) | cdb[8];
2873 else
2874 size = cmd->data_length;
2875
2876 /*
2877 * Setup the legacy emulated handler for SPC-2 and
2878 * >= SPC-3 compatible reservation handling (CRH=1)
2879 * Otherwise, we assume the underlying SCSI logic is
2880 * is running in SPC_PASSTHROUGH, and wants reservations
2881 * emulation disabled.
2882 */
e76a35d6
CH		 2883 if (su_dev->t10_pr.res_type != SPC_PASSTHROUGH)
2884 cmd->execute_task = target_scsi2_reservation_reserve;
c66ac9db
NB		 2885 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
2886 break;
2887 case RELEASE:
2888 case RELEASE_10:
2889 /*
2890 * The SPC-2 RELEASE does not contain a size in the SCSI CDB.
2891 * Assume the passthrough or $FABRIC_MOD will tell us about it.
2892 */
2893 if (cdb[0] == RELEASE_10)
2894 size = (cdb[7] << 8) | cdb[8];
2895 else
2896 size = cmd->data_length;
2897
e76a35d6
CH		 2898 if (su_dev->t10_pr.res_type != SPC_PASSTHROUGH)
2899 cmd->execute_task = target_scsi2_reservation_release;
c66ac9db
NB		 2900 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
2901 break;
2902 case SYNCHRONIZE_CACHE:
8e94b8db 2903 case SYNCHRONIZE_CACHE_16:
c66ac9db
NB		 2904 /*
2905 * Extract LBA and range to be flushed for emulated SYNCHRONIZE_CACHE
2906 */
2907 if (cdb[0] == SYNCHRONIZE_CACHE) {
2908 sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
a1d8b49a 2909 cmd->t_task_lba = transport_lba_32(cdb);
c66ac9db
NB		 2910 } else {
2911 sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
a1d8b49a 2912 cmd->t_task_lba = transport_lba_64(cdb);
c66ac9db
NB		 2913 }
2914 if (sector_ret)
2915 goto out_unsupported_cdb;
2916
2917 size = transport_get_size(sectors, cdb, cmd);
2918 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
2919
5bda90c8 2920 if (passthrough)
c66ac9db 2921 break;
5bda90c8 2922
c66ac9db
NB		 2923 /*
2924 * Check to ensure that LBA + Range does not exceed past end of
7abbe7f3 2925 * device for IBLOCK and FILEIO ->do_sync_cache() backend calls
c66ac9db 2926 */
7abbe7f3
NB		 2927 if ((cmd->t_task_lba != 0) || (sectors != 0)) {
2928 if (transport_cmd_get_valid_sectors(cmd) < 0)
2929 goto out_invalid_cdb_field;
2930 }
5bda90c8 2931 cmd->execute_task = target_emulate_synchronize_cache;
c66ac9db
NB		 2932 break;
2933 case UNMAP:
2934 size = get_unaligned_be16(&cdb[7]);
05d1c7c0 2935 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
5bda90c8
CH		 2936 if (!passthrough)
2937 cmd->execute_task = target_emulate_unmap;
c66ac9db
NB		 2938 break;
2939 case WRITE_SAME_16:
2940 sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
2941 if (sector_ret)
2942 goto out_unsupported_cdb;
dd3a5ad8 2943
6708bb27 2944 if (sectors)
12850626 2945 size = transport_get_size(1, cdb, cmd);
6708bb27
AG		 2946 else {
2947 pr_err("WSNZ=1, WRITE_SAME w/sectors=0 not supported\n");
2948 goto out_invalid_cdb_field;
2949 }
dd3a5ad8 2950
5db0753b 2951 cmd->t_task_lba = get_unaligned_be64(&cdb[2]);
706d5860
NB		 2952 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2953
2954 if (target_check_write_same_discard(&cdb[1], dev) < 0)
67236c44 2955 goto out_unsupported_cdb;
5bda90c8
CH		 2956 if (!passthrough)
2957 cmd->execute_task = target_emulate_write_same;
706d5860
NB		 2958 break;
2959 case WRITE_SAME:
2960 sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
2961 if (sector_ret)
2962 goto out_unsupported_cdb;
2963
2964 if (sectors)
12850626 2965 size = transport_get_size(1, cdb, cmd);
706d5860
NB		 2966 else {
2967 pr_err("WSNZ=1, WRITE_SAME w/sectors=0 not supported\n");
2968 goto out_invalid_cdb_field;
c66ac9db 2969 }
706d5860
NB		 2970
2971 cmd->t_task_lba = get_unaligned_be32(&cdb[2]);
c66ac9db 2972 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
706d5860
NB		 2973 /*
2974 * Follow sbcr26 with WRITE_SAME (10) and check for the existence
2975 * of byte 1 bit 3 UNMAP instead of original reserved field
2976 */
2977 if (target_check_write_same_discard(&cdb[1], dev) < 0)
67236c44 2978 goto out_unsupported_cdb;
5bda90c8
CH		 2979 if (!passthrough)
2980 cmd->execute_task = target_emulate_write_same;
c66ac9db
NB		 2981 break;
2982 case ALLOW_MEDIUM_REMOVAL:
c66ac9db 2983 case ERASE:
c66ac9db
NB		 2984 case REZERO_UNIT:
2985 case SEEK_10:
c66ac9db
NB		 2986 case SPACE:
2987 case START_STOP:
2988 case TEST_UNIT_READY:
2989 case VERIFY:
2990 case WRITE_FILEMARKS:
5bda90c8
CH		 2991 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
2992 if (!passthrough)
2993 cmd->execute_task = target_emulate_noop;
2994 break;
2995 case GPCMD_CLOSE_TRACK:
2996 case INITIALIZE_ELEMENT_STATUS:
2997 case GPCMD_LOAD_UNLOAD:
2998 case GPCMD_SET_SPEED:
c66ac9db
NB		 2999 case MOVE_MEDIUM:
3000 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
3001 break;
3002 case REPORT_LUNS:
e76a35d6 3003 cmd->execute_task = target_report_luns;
c66ac9db
NB		 3004 size = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9];
3005 /*
3006 * Do implict HEAD_OF_QUEUE processing for REPORT_LUNS
3007 * See spc4r17 section 5.3
3008 */
5951146d 3009 if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
e66ecd50 3010 cmd->sam_task_attr = MSG_HEAD_TAG;
05d1c7c0 3011 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
c66ac9db
NB		 3012 break;
3013 default:
6708bb27 3014 pr_warn("TARGET_CORE[%s]: Unsupported SCSI Opcode"
c66ac9db 3015 " 0x%02x, sending CHECK_CONDITION.\n",
e3d6f909 3016 cmd->se_tfo->get_fabric_name(), cdb[0]);
c66ac9db
NB		 3017 goto out_unsupported_cdb;
3018 }
3019
3020 if (size != cmd->data_length) {
6708bb27 3021 pr_warn("TARGET_CORE[%s]: Expected Transfer Length:"
c66ac9db 3022 " %u does not match SCSI CDB Length: %u for SAM Opcode:"
e3d6f909 3023 " 0x%02x\n", cmd->se_tfo->get_fabric_name(),
c66ac9db
NB		 3024 cmd->data_length, size, cdb[0]);
3025
3026 cmd->cmd_spdtl = size;
3027
3028 if (cmd->data_direction == DMA_TO_DEVICE) {
6708bb27 3029 pr_err("Rejecting underflow/overflow"
c66ac9db
NB		 3030 " WRITE data\n");
3031 goto out_invalid_cdb_field;
3032 }
3033 /*
3034 * Reject READ_* or WRITE_* with overflow/underflow for
3035 * type SCF_SCSI_DATA_SG_IO_CDB.
3036 */
6708bb27
AG		 3037 if (!ret && (dev->se_sub_dev->se_dev_attrib.block_size != 512)) {
3038 pr_err("Failing OVERFLOW/UNDERFLOW for LBA op"
c66ac9db 3039 " CDB on non 512-byte sector setup subsystem"
e3d6f909 3040 " plugin: %s\n", dev->transport->name);
c66ac9db
NB		 3041 /* Returns CHECK_CONDITION + INVALID_CDB_FIELD */
3042 goto out_invalid_cdb_field;
3043 }
3044
3045 if (size > cmd->data_length) {
3046 cmd->se_cmd_flags |= SCF_OVERFLOW_BIT;
3047 cmd->residual_count = (size - cmd->data_length);
3048 } else {
3049 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
3050 cmd->residual_count = (cmd->data_length - size);
3051 }
3052 cmd->data_length = size;
3053 }
3054
5bda90c8
CH		 3055 /* reject any command that we don't have a handler for */
3056 if (!(passthrough || cmd->execute_task ||
3057 (cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB)))
3058 goto out_unsupported_cdb;
3059
c66ac9db
NB		 3060 transport_set_supported_SAM_opcode(cmd);
3061 return ret;
3062
3063out_unsupported_cdb:
3064 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
3065 cmd->scsi_sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
5951146d 3066 return -EINVAL;
c66ac9db
NB		 3067out_invalid_cdb_field:
3068 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
3069 cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD;
5951146d 3070 return -EINVAL;
c66ac9db
NB		 3071}
3072
c66ac9db 3073/*
35e0e757 3074 * Called from I/O completion to determine which dormant/delayed
c66ac9db
NB		 3075 * and ordered cmds need to have their tasks added to the execution queue.
3076 */
3077static void transport_complete_task_attr(struct se_cmd *cmd)
3078{
5951146d 3079 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 3080 struct se_cmd *cmd_p, *cmd_tmp;
3081 int new_active_tasks = 0;
3082
e66ecd50 3083 if (cmd->sam_task_attr == MSG_SIMPLE_TAG) {
c66ac9db
NB		 3084 atomic_dec(&dev->simple_cmds);
3085 smp_mb__after_atomic_dec();
3086 dev->dev_cur_ordered_id++;
6708bb27 3087 pr_debug("Incremented dev->dev_cur_ordered_id: %u for"
c66ac9db
NB		 3088 " SIMPLE: %u\n", dev->dev_cur_ordered_id,
3089 cmd->se_ordered_id);
e66ecd50 3090 } else if (cmd->sam_task_attr == MSG_HEAD_TAG) {
c66ac9db 3091 dev->dev_cur_ordered_id++;
6708bb27 3092 pr_debug("Incremented dev_cur_ordered_id: %u for"
c66ac9db
NB		 3093 " HEAD_OF_QUEUE: %u\n", dev->dev_cur_ordered_id,
3094 cmd->se_ordered_id);
e66ecd50 3095 } else if (cmd->sam_task_attr == MSG_ORDERED_TAG) {
c66ac9db
NB		 3096 atomic_dec(&dev->dev_ordered_sync);
3097 smp_mb__after_atomic_dec();
c66ac9db
NB		 3098
3099 dev->dev_cur_ordered_id++;
6708bb27 3100 pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED:"
c66ac9db
NB		 3101 " %u\n", dev->dev_cur_ordered_id, cmd->se_ordered_id);
3102 }
3103 /*
3104 * Process all commands up to the last received
3105 * ORDERED task attribute which requires another blocking
3106 * boundary
3107 */
3108 spin_lock(&dev->delayed_cmd_lock);
3109 list_for_each_entry_safe(cmd_p, cmd_tmp,
5951146d 3110 &dev->delayed_cmd_list, se_delayed_node) {
c66ac9db 3111
5951146d 3112 list_del(&cmd_p->se_delayed_node);
c66ac9db
NB		 3113 spin_unlock(&dev->delayed_cmd_lock);
3114
6708bb27 3115 pr_debug("Calling add_tasks() for"
c66ac9db
NB		 3116 " cmd_p: 0x%02x Task Attr: 0x%02x"
3117 " Dormant -> Active, se_ordered_id: %u\n",
6708bb27 3118 cmd_p->t_task_cdb[0],
c66ac9db
NB		 3119 cmd_p->sam_task_attr, cmd_p->se_ordered_id);
3120
3121 transport_add_tasks_from_cmd(cmd_p);
3122 new_active_tasks++;
3123
3124 spin_lock(&dev->delayed_cmd_lock);
e66ecd50 3125 if (cmd_p->sam_task_attr == MSG_ORDERED_TAG)
c66ac9db
NB		 3126 break;
3127 }
3128 spin_unlock(&dev->delayed_cmd_lock);
3129 /*
3130 * If new tasks have become active, wake up the transport thread
3131 * to do the processing of the Active tasks.
3132 */
3133 if (new_active_tasks != 0)
e3d6f909 3134 wake_up_interruptible(&dev->dev_queue_obj.thread_wq);
c66ac9db
NB		 3135}
3136
e057f533 3137static void transport_complete_qf(struct se_cmd *cmd)
07bde79a
NB		 3138{
3139 int ret = 0;
3140
e057f533
CH		 3141 if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
3142 transport_complete_task_attr(cmd);
3143
3144 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
3145 ret = cmd->se_tfo->queue_status(cmd);
3146 if (ret)
3147 goto out;
3148 }
07bde79a
NB		 3149
3150 switch (cmd->data_direction) {
3151 case DMA_FROM_DEVICE:
3152 ret = cmd->se_tfo->queue_data_in(cmd);
3153 break;
3154 case DMA_TO_DEVICE:
ec98f782 3155 if (cmd->t_bidi_data_sg) {
07bde79a
NB		 3156 ret = cmd->se_tfo->queue_data_in(cmd);
3157 if (ret < 0)
e057f533 3158 break;
07bde79a
NB		 3159 }
3160 /* Fall through for DMA_TO_DEVICE */
3161 case DMA_NONE:
3162 ret = cmd->se_tfo->queue_status(cmd);
3163 break;
3164 default:
3165 break;
3166 }
3167
e057f533
CH		 3168out:
3169 if (ret < 0) {
3170 transport_handle_queue_full(cmd, cmd->se_dev);
3171 return;
3172 }
3173 transport_lun_remove_cmd(cmd);
3174 transport_cmd_check_stop_to_fabric(cmd);
07bde79a
NB		 3175}
3176
3177static void transport_handle_queue_full(
3178 struct se_cmd *cmd,
e057f533 3179 struct se_device *dev)
07bde79a
NB		 3180{
3181 spin_lock_irq(&dev->qf_cmd_lock);
07bde79a
NB		 3182 list_add_tail(&cmd->se_qf_node, &cmd->se_dev->qf_cmd_list);
3183 atomic_inc(&dev->dev_qf_count);
3184 smp_mb__after_atomic_inc();
3185 spin_unlock_irq(&cmd->se_dev->qf_cmd_lock);
3186
3187 schedule_work(&cmd->se_dev->qf_work_queue);
3188}
3189
35e0e757 3190static void target_complete_ok_work(struct work_struct *work)
c66ac9db 3191{
35e0e757 3192 struct se_cmd *cmd = container_of(work, struct se_cmd, work);
07bde79a 3193 int reason = 0, ret;
35e0e757 3194
c66ac9db
NB		 3195 /*
3196 * Check if we need to move delayed/dormant tasks from cmds on the
3197 * delayed execution list after a HEAD_OF_QUEUE or ORDERED Task
3198 * Attribute.
3199 */
5951146d 3200 if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
c66ac9db 3201 transport_complete_task_attr(cmd);
07bde79a
NB		 3202 /*
3203 * Check to schedule QUEUE_FULL work, or execute an existing
3204 * cmd->transport_qf_callback()
3205 */
3206 if (atomic_read(&cmd->se_dev->dev_qf_count) != 0)
3207 schedule_work(&cmd->se_dev->qf_work_queue);
3208
c66ac9db
NB		 3209 /*
3210 * Check if we need to retrieve a sense buffer from
3211 * the struct se_cmd in question.
3212 */
3213 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
3214 if (transport_get_sense_data(cmd) < 0)
3215 reason = TCM_NON_EXISTENT_LUN;
3216
3217 /*
3218 * Only set when an struct se_task->task_scsi_status returned
3219 * a non GOOD status.
3220 */
3221 if (cmd->scsi_status) {
07bde79a 3222 ret = transport_send_check_condition_and_sense(
c66ac9db 3223 cmd, reason, 1);
f147abb4 3224 if (ret == -EAGAIN || ret == -ENOMEM)
07bde79a
NB		 3225 goto queue_full;
3226
c66ac9db
NB		 3227 transport_lun_remove_cmd(cmd);
3228 transport_cmd_check_stop_to_fabric(cmd);
3229 return;
3230 }
3231 }
3232 /*
25985edc 3233 * Check for a callback, used by amongst other things
c66ac9db
NB		 3234 * XDWRITE_READ_10 emulation.
3235 */
3236 if (cmd->transport_complete_callback)
3237 cmd->transport_complete_callback(cmd);
3238
3239 switch (cmd->data_direction) {
3240 case DMA_FROM_DEVICE:
3241 spin_lock(&cmd->se_lun->lun_sep_lock);
e3d6f909
AG		 3242 if (cmd->se_lun->lun_sep) {
3243 cmd->se_lun->lun_sep->sep_stats.tx_data_octets +=
c66ac9db
NB		 3244 cmd->data_length;
3245 }
3246 spin_unlock(&cmd->se_lun->lun_sep_lock);
c66ac9db 3247
07bde79a 3248 ret = cmd->se_tfo->queue_data_in(cmd);
f147abb4 3249 if (ret == -EAGAIN || ret == -ENOMEM)
07bde79a 3250 goto queue_full;
c66ac9db
NB		 3251 break;
3252 case DMA_TO_DEVICE:
3253 spin_lock(&cmd->se_lun->lun_sep_lock);
e3d6f909
AG		 3254 if (cmd->se_lun->lun_sep) {
3255 cmd->se_lun->lun_sep->sep_stats.rx_data_octets +=
c66ac9db
NB		 3256 cmd->data_length;
3257 }
3258 spin_unlock(&cmd->se_lun->lun_sep_lock);
3259 /*
3260 * Check if we need to send READ payload for BIDI-COMMAND
3261 */
ec98f782 3262 if (cmd->t_bidi_data_sg) {
c66ac9db 3263 spin_lock(&cmd->se_lun->lun_sep_lock);
e3d6f909
AG		 3264 if (cmd->se_lun->lun_sep) {
3265 cmd->se_lun->lun_sep->sep_stats.tx_data_octets +=
c66ac9db
NB		 3266 cmd->data_length;
3267 }
3268 spin_unlock(&cmd->se_lun->lun_sep_lock);
07bde79a 3269 ret = cmd->se_tfo->queue_data_in(cmd);
f147abb4 3270 if (ret == -EAGAIN || ret == -ENOMEM)
07bde79a 3271 goto queue_full;
c66ac9db
NB		 3272 break;
3273 }
3274 /* Fall through for DMA_TO_DEVICE */
3275 case DMA_NONE:
07bde79a 3276 ret = cmd->se_tfo->queue_status(cmd);
f147abb4 3277 if (ret == -EAGAIN || ret == -ENOMEM)
07bde79a 3278 goto queue_full;
c66ac9db
NB		 3279 break;
3280 default:
3281 break;
3282 }
3283
3284 transport_lun_remove_cmd(cmd);
3285 transport_cmd_check_stop_to_fabric(cmd);
07bde79a
NB		 3286 return;
3287
3288queue_full:
6708bb27 3289 pr_debug("Handling complete_ok QUEUE_FULL: se_cmd: %p,"
07bde79a 3290 " data_direction: %d\n", cmd, cmd->data_direction);
e057f533
CH		 3291 cmd->t_state = TRANSPORT_COMPLETE_QF_OK;
3292 transport_handle_queue_full(cmd, cmd->se_dev);
c66ac9db
NB		 3293}
3294
3295static void transport_free_dev_tasks(struct se_cmd *cmd)
3296{
3297 struct se_task *task, *task_tmp;
3298 unsigned long flags;
0c2cfe5f 3299 LIST_HEAD(dispose_list);
c66ac9db 3300
a1d8b49a 3301 spin_lock_irqsave(&cmd->t_state_lock, flags);
c66ac9db 3302 list_for_each_entry_safe(task, task_tmp,
a1d8b49a 3303 &cmd->t_task_list, t_list) {
0c2cfe5f
CH		 3304 if (!(task->task_flags & TF_ACTIVE))
3305 list_move_tail(&task->t_list, &dispose_list);
3306 }
3307 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3308
3309 while (!list_empty(&dispose_list)) {
3310 task = list_first_entry(&dispose_list, struct se_task, t_list);
c66ac9db 3311
af3f00c7
CH		 3312 if (task->task_sg != cmd->t_data_sg &&
3313 task->task_sg != cmd->t_bidi_data_sg)
3314 kfree(task->task_sg);
c66ac9db
NB		 3315
3316 list_del(&task->t_list);
3317
42bf829e 3318 cmd->se_dev->transport->free_task(task);
c66ac9db 3319 }
c66ac9db
NB		 3320}
3321
6708bb27 3322static inline void transport_free_sgl(struct scatterlist *sgl, int nents)
c66ac9db 3323{
ec98f782 3324 struct scatterlist *sg;
ec98f782 3325 int count;
c66ac9db 3326
6708bb27
AG		 3327 for_each_sg(sgl, sg, nents, count)
3328 __free_page(sg_page(sg));
c66ac9db 3329
6708bb27
AG		 3330 kfree(sgl);
3331}
c66ac9db 3332
6708bb27
AG		 3333static inline void transport_free_pages(struct se_cmd *cmd)
3334{
3335 if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC)
3336 return;
3337
3338 transport_free_sgl(cmd->t_data_sg, cmd->t_data_nents);
ec98f782
AG		 3339 cmd->t_data_sg = NULL;
3340 cmd->t_data_nents = 0;
c66ac9db 3341
6708bb27 3342 transport_free_sgl(cmd->t_bidi_data_sg, cmd->t_bidi_data_nents);
ec98f782
AG		 3343 cmd->t_bidi_data_sg = NULL;
3344 cmd->t_bidi_data_nents = 0;
c66ac9db
NB		 3345}
3346
e26d99ae
CH		 3347/**
3348 * transport_release_cmd - free a command
3349 * @cmd: command to free
3350 *
3351 * This routine unconditionally frees a command, and reference counting
3352 * or list removal must be done in the caller.
3353 */
3354static void transport_release_cmd(struct se_cmd *cmd)
3355{
3356 BUG_ON(!cmd->se_tfo);
3357
c8e31f26 3358 if (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)
e26d99ae
CH		 3359 core_tmr_release_req(cmd->se_tmr_req);
3360 if (cmd->t_task_cdb != cmd->__t_task_cdb)
3361 kfree(cmd->t_task_cdb);
3362 /*
7481deb4
NB		 3363 * If this cmd has been setup with target_get_sess_cmd(), drop
3364 * the kref and call ->release_cmd() in kref callback.
e26d99ae 3365 */
7481deb4
NB		 3366 if (cmd->check_release != 0) {
3367 target_put_sess_cmd(cmd->se_sess, cmd);
3368 return;
3369 }
e26d99ae
CH		 3370 cmd->se_tfo->release_cmd(cmd);
3371}
3372
d3df7825
CH		 3373/**
3374 * transport_put_cmd - release a reference to a command
3375 * @cmd: command to release
3376 *
3377 * This routine releases our reference to the command and frees it if possible.
3378 */
39c05f32 3379static void transport_put_cmd(struct se_cmd *cmd)
c66ac9db
NB		 3380{
3381 unsigned long flags;
4911e3cc 3382 int free_tasks = 0;
c66ac9db 3383
a1d8b49a 3384 spin_lock_irqsave(&cmd->t_state_lock, flags);
4911e3cc
CH		 3385 if (atomic_read(&cmd->t_fe_count)) {
3386 if (!atomic_dec_and_test(&cmd->t_fe_count))
3387 goto out_busy;
3388 }
3389
3390 if (atomic_read(&cmd->t_se_count)) {
3391 if (!atomic_dec_and_test(&cmd->t_se_count))
3392 goto out_busy;
3393 }
3394
7d680f3b
CH		 3395 if (cmd->transport_state & CMD_T_DEV_ACTIVE) {
3396 cmd->transport_state &= ~CMD_T_DEV_ACTIVE;
4911e3cc
CH		 3397 transport_all_task_dev_remove_state(cmd);
3398 free_tasks = 1;
c66ac9db 3399 }
a1d8b49a 3400 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 3401
4911e3cc
CH		 3402 if (free_tasks != 0)
3403 transport_free_dev_tasks(cmd);
d3df7825 3404
c66ac9db 3405 transport_free_pages(cmd);
31afc39c 3406 transport_release_cmd(cmd);
39c05f32 3407 return;
4911e3cc
CH		 3408out_busy:
3409 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 3410}
3411
c66ac9db 3412/*
ec98f782
AG		 3413 * transport_generic_map_mem_to_cmd - Use fabric-alloced pages instead of
3414 * allocating in the core.
c66ac9db
NB		 3415 * @cmd: Associated se_cmd descriptor
3416 * @mem: SGL style memory for TCM WRITE / READ
3417 * @sg_mem_num: Number of SGL elements
3418 * @mem_bidi_in: SGL style memory for TCM BIDI READ
3419 * @sg_mem_bidi_num: Number of BIDI READ SGL elements
3420 *
3421 * Return: nonzero return cmd was rejected for -ENOMEM or inproper usage
3422 * of parameters.
3423 */
3424int transport_generic_map_mem_to_cmd(
3425 struct se_cmd *cmd,
5951146d
AG		 3426 struct scatterlist *sgl,
3427 u32 sgl_count,
3428 struct scatterlist *sgl_bidi,
3429 u32 sgl_bidi_count)
c66ac9db 3430{
5951146d 3431 if (!sgl || !sgl_count)
c66ac9db 3432 return 0;
c66ac9db 3433
c66ac9db
NB		 3434 if ((cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB) ||
3435 (cmd->se_cmd_flags & SCF_SCSI_CONTROL_SG_IO_CDB)) {
fef58a60
NB		 3436 /*
3437 * Reject SCSI data overflow with map_mem_to_cmd() as incoming
3438 * scatterlists already have been set to follow what the fabric
3439 * passes for the original expected data transfer length.
3440 */
3441 if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
3442 pr_warn("Rejecting SCSI DATA overflow for fabric using"
3443 " SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC\n");
3444 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
3445 cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD;
3446 return -EINVAL;
3447 }
c66ac9db 3448
ec98f782
AG		 3449 cmd->t_data_sg = sgl;
3450 cmd->t_data_nents = sgl_count;
c66ac9db 3451
ec98f782
AG		 3452 if (sgl_bidi && sgl_bidi_count) {
3453 cmd->t_bidi_data_sg = sgl_bidi;
3454 cmd->t_bidi_data_nents = sgl_bidi_count;
c66ac9db
NB		 3455 }
3456 cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
c66ac9db
NB		 3457 }
3458
3459 return 0;
3460}
3461EXPORT_SYMBOL(transport_generic_map_mem_to_cmd);
3462
4949314c 3463void *transport_kmap_data_sg(struct se_cmd *cmd)
05d1c7c0 3464{
ec98f782 3465 struct scatterlist *sg = cmd->t_data_sg;
4949314c
AG		 3466 struct page **pages;
3467 int i;
05d1c7c0 3468
ec98f782 3469 BUG_ON(!sg);
05d1c7c0 3470 /*
ec98f782
AG		 3471 * We need to take into account a possible offset here for fabrics like
3472 * tcm_loop who may be using a contig buffer from the SCSI midlayer for
3473 * control CDBs passed as SGLs via transport_generic_map_mem_to_cmd()
05d1c7c0 3474 */
4949314c
AG		 3475 if (!cmd->t_data_nents)
3476 return NULL;
3477 else if (cmd->t_data_nents == 1)
3478 return kmap(sg_page(sg)) + sg->offset;
3479
3480 /* >1 page. use vmap */
3481 pages = kmalloc(sizeof(*pages) * cmd->t_data_nents, GFP_KERNEL);
3482 if (!pages)
3483 return NULL;
3484
3485 /* convert sg[] to pages[] */
3486 for_each_sg(cmd->t_data_sg, sg, cmd->t_data_nents, i) {
3487 pages[i] = sg_page(sg);
3488 }
3489
3490 cmd->t_data_vmap = vmap(pages, cmd->t_data_nents, VM_MAP, PAGE_KERNEL);
3491 kfree(pages);
3492 if (!cmd->t_data_vmap)
3493 return NULL;
3494
3495 return cmd->t_data_vmap + cmd->t_data_sg[0].offset;
05d1c7c0 3496}
4949314c 3497EXPORT_SYMBOL(transport_kmap_data_sg);
05d1c7c0 3498
4949314c 3499void transport_kunmap_data_sg(struct se_cmd *cmd)
05d1c7c0 3500{
4949314c
AG		 3501 if (!cmd->t_data_nents)
3502 return;
3503 else if (cmd->t_data_nents == 1)
3504 kunmap(sg_page(cmd->t_data_sg));
3505
3506 vunmap(cmd->t_data_vmap);
3507 cmd->t_data_vmap = NULL;
05d1c7c0 3508}
4949314c 3509EXPORT_SYMBOL(transport_kunmap_data_sg);
05d1c7c0 3510
c66ac9db 3511static int
05d1c7c0 3512transport_generic_get_mem(struct se_cmd *cmd)
c66ac9db 3513{
ec98f782
AG		 3514 u32 length = cmd->data_length;
3515 unsigned int nents;
3516 struct page *page;
9db9da33 3517 gfp_t zero_flag;
ec98f782 3518 int i = 0;
c66ac9db 3519
ec98f782
AG		 3520 nents = DIV_ROUND_UP(length, PAGE_SIZE);
3521 cmd->t_data_sg = kmalloc(sizeof(struct scatterlist) * nents, GFP_KERNEL);
3522 if (!cmd->t_data_sg)
3523 return -ENOMEM;
c66ac9db 3524
ec98f782
AG		 3525 cmd->t_data_nents = nents;
3526 sg_init_table(cmd->t_data_sg, nents);
c66ac9db 3527
9db9da33 3528 zero_flag = cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB ? 0 : __GFP_ZERO;
3529
ec98f782
AG		 3530 while (length) {
3531 u32 page_len = min_t(u32, length, PAGE_SIZE);
9db9da33 3532 page = alloc_page(GFP_KERNEL | zero_flag);
ec98f782
AG		 3533 if (!page)
3534 goto out;
c66ac9db 3535
ec98f782
AG		 3536 sg_set_page(&cmd->t_data_sg[i], page, page_len, 0);
3537 length -= page_len;
3538 i++;
c66ac9db 3539 }
c66ac9db 3540 return 0;
c66ac9db 3541
ec98f782
AG		 3542out:
3543 while (i >= 0) {
3544 __free_page(sg_page(&cmd->t_data_sg[i]));
3545 i--;
c66ac9db 3546 }
ec98f782
AG		 3547 kfree(cmd->t_data_sg);
3548 cmd->t_data_sg = NULL;
3549 return -ENOMEM;
c66ac9db
NB		 3550}
3551
a1d8b49a
AG		 3552/* Reduce sectors if they are too long for the device */
3553static inline sector_t transport_limit_task_sectors(
c66ac9db
NB		 3554 struct se_device *dev,
3555 unsigned long long lba,
a1d8b49a 3556 sector_t sectors)
c66ac9db 3557{
a1d8b49a 3558 sectors = min_t(sector_t, sectors, dev->se_sub_dev->se_dev_attrib.max_sectors);
c66ac9db 3559
a1d8b49a
AG		 3560 if (dev->transport->get_device_type(dev) == TYPE_DISK)
3561 if ((lba + sectors) > transport_dev_end_lba(dev))
3562 sectors = ((transport_dev_end_lba(dev) - lba) + 1);
c66ac9db 3563
a1d8b49a 3564 return sectors;
c66ac9db
NB		 3565}
3566
c66ac9db
NB		 3567
3568/*
3569 * This function can be used by HW target mode drivers to create a linked
3570 * scatterlist from all contiguously allocated struct se_task->task_sg[].
3571 * This is intended to be called during the completion path by TCM Core
3572 * when struct target_core_fabric_ops->check_task_sg_chaining is enabled.
3573 */
3574void transport_do_task_sg_chain(struct se_cmd *cmd)
3575{
ec98f782
AG		 3576 struct scatterlist *sg_first = NULL;
3577 struct scatterlist *sg_prev = NULL;
3578 int sg_prev_nents = 0;
3579 struct scatterlist *sg;
c66ac9db 3580 struct se_task *task;
ec98f782 3581 u32 chained_nents = 0;
c66ac9db
NB		 3582 int i;
3583
ec98f782
AG		 3584 BUG_ON(!cmd->se_tfo->task_sg_chaining);
3585
c66ac9db
NB		 3586 /*
3587 * Walk the struct se_task list and setup scatterlist chains
a1d8b49a 3588 * for each contiguously allocated struct se_task->task_sg[].
c66ac9db 3589 */
a1d8b49a 3590 list_for_each_entry(task, &cmd->t_task_list, t_list) {
ec98f782 3591 if (!task->task_sg)
c66ac9db
NB		 3592 continue;
3593
ec98f782
AG		 3594 if (!sg_first) {
3595 sg_first = task->task_sg;
6708bb27 3596 chained_nents = task->task_sg_nents;
97868c89 3597 } else {
ec98f782 3598 sg_chain(sg_prev, sg_prev_nents, task->task_sg);
6708bb27 3599 chained_nents += task->task_sg_nents;
97868c89 3600 }
c3c74c7a
NB		 3601 /*
3602 * For the padded tasks, use the extra SGL vector allocated
3603 * in transport_allocate_data_tasks() for the sg_prev_nents
04629b7b
CH		 3604 * offset into sg_chain() above.
3605 *
3606 * We do not need the padding for the last task (or a single
3607 * task), but in that case we will never use the sg_prev_nents
3608 * value below which would be incorrect.
c3c74c7a 3609 */
04629b7b 3610 sg_prev_nents = (task->task_sg_nents + 1);
ec98f782 3611 sg_prev = task->task_sg;
c66ac9db
NB		 3612 }
3613 /*
3614 * Setup the starting pointer and total t_tasks_sg_linked_no including
3615 * padding SGs for linking and to mark the end.
3616 */
a1d8b49a 3617 cmd->t_tasks_sg_chained = sg_first;
ec98f782 3618 cmd->t_tasks_sg_chained_no = chained_nents;
c66ac9db 3619
6708bb27 3620 pr_debug("Setup cmd: %p cmd->t_tasks_sg_chained: %p and"
a1d8b49a
AG		 3621 " t_tasks_sg_chained_no: %u\n", cmd, cmd->t_tasks_sg_chained,
3622 cmd->t_tasks_sg_chained_no);
c66ac9db 3623
a1d8b49a
AG		 3624 for_each_sg(cmd->t_tasks_sg_chained, sg,
3625 cmd->t_tasks_sg_chained_no, i) {
c66ac9db 3626
6708bb27 3627 pr_debug("SG[%d]: %p page: %p length: %d offset: %d\n",
5951146d 3628 i, sg, sg_page(sg), sg->length, sg->offset);
c66ac9db 3629 if (sg_is_chain(sg))
6708bb27 3630 pr_debug("SG: %p sg_is_chain=1\n", sg);
c66ac9db 3631 if (sg_is_last(sg))
6708bb27 3632 pr_debug("SG: %p sg_is_last=1\n", sg);
c66ac9db 3633 }
c66ac9db
NB		 3634}
3635EXPORT_SYMBOL(transport_do_task_sg_chain);
3636
a1d8b49a
AG		 3637/*
3638 * Break up cmd into chunks transport can handle
3639 */
38b40067
CH		 3640static int
3641transport_allocate_data_tasks(struct se_cmd *cmd,
c66ac9db 3642 enum dma_data_direction data_direction,
38b40067 3643 struct scatterlist *cmd_sg, unsigned int sgl_nents)
c66ac9db 3644{
5951146d 3645 struct se_device *dev = cmd->se_dev;
a3eedc22 3646 int task_count, i;
38b40067
CH		 3647 unsigned long long lba;
3648 sector_t sectors, dev_max_sectors;
3649 u32 sector_size;
3650
3651 if (transport_cmd_get_valid_sectors(cmd) < 0)
3652 return -EINVAL;
3653
3654 dev_max_sectors = dev->se_sub_dev->se_dev_attrib.max_sectors;
3655 sector_size = dev->se_sub_dev->se_dev_attrib.block_size;
a1d8b49a 3656
ec98f782 3657 WARN_ON(cmd->data_length % sector_size);
38b40067
CH		 3658
3659 lba = cmd->t_task_lba;
ec98f782 3660 sectors = DIV_ROUND_UP(cmd->data_length, sector_size);
277c5f27 3661 task_count = DIV_ROUND_UP_SECTOR_T(sectors, dev_max_sectors);
af3f00c7
CH		 3662
3663 /*
3664 * If we need just a single task reuse the SG list in the command
3665 * and avoid a lot of work.
3666 */
3667 if (task_count == 1) {
3668 struct se_task *task;
3669 unsigned long flags;
3670
3671 task = transport_generic_get_task(cmd, data_direction);
3672 if (!task)
3673 return -ENOMEM;
3674
3675 task->task_sg = cmd_sg;
3676 task->task_sg_nents = sgl_nents;
3677
3678 task->task_lba = lba;
3679 task->task_sectors = sectors;
3680 task->task_size = task->task_sectors * sector_size;
3681
3682 spin_lock_irqsave(&cmd->t_state_lock, flags);
3683 list_add_tail(&task->t_list, &cmd->t_task_list);
3684 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3685
3686 return task_count;
3687 }
3688
ec98f782 3689 for (i = 0; i < task_count; i++) {
38b40067 3690 struct se_task *task;
c3c74c7a 3691 unsigned int task_size, task_sg_nents_padded;
38b40067
CH		 3692 struct scatterlist *sg;
3693 unsigned long flags;
ec98f782 3694 int count;
a1d8b49a 3695
c66ac9db 3696 task = transport_generic_get_task(cmd, data_direction);
a1d8b49a 3697 if (!task)
ec98f782 3698 return -ENOMEM;
c66ac9db 3699
c66ac9db 3700 task->task_lba = lba;
ec98f782
AG		 3701 task->task_sectors = min(sectors, dev_max_sectors);
3702 task->task_size = task->task_sectors * sector_size;
c66ac9db 3703
525a48a2
NB		 3704 /*
3705 * This now assumes that passed sg_ents are in PAGE_SIZE chunks
3706 * in order to calculate the number per task SGL entries
3707 */
3708 task->task_sg_nents = DIV_ROUND_UP(task->task_size, PAGE_SIZE);
c66ac9db 3709 /*
ec98f782
AG		 3710 * Check if the fabric module driver is requesting that all
3711 * struct se_task->task_sg[] be chained together.. If so,
3712 * then allocate an extra padding SG entry for linking and
c3c74c7a
NB		 3713 * marking the end of the chained SGL for every task except
3714 * the last one for (task_count > 1) operation, or skipping
3715 * the extra padding for the (task_count == 1) case.
c66ac9db 3716 */
c3c74c7a
NB		 3717 if (cmd->se_tfo->task_sg_chaining && (i < (task_count - 1))) {
3718 task_sg_nents_padded = (task->task_sg_nents + 1);
c3c74c7a
NB		 3719 } else
3720 task_sg_nents_padded = task->task_sg_nents;
c66ac9db 3721
1d20bb61 3722 task->task_sg = kmalloc(sizeof(struct scatterlist) *
c3c74c7a 3723 task_sg_nents_padded, GFP_KERNEL);
ec98f782
AG		 3724 if (!task->task_sg) {
3725 cmd->se_dev->transport->free_task(task);
3726 return -ENOMEM;
3727 }
3728
c3c74c7a 3729 sg_init_table(task->task_sg, task_sg_nents_padded);
c66ac9db 3730
ec98f782
AG		 3731 task_size = task->task_size;
3732
3733 /* Build new sgl, only up to task_size */
6708bb27 3734 for_each_sg(task->task_sg, sg, task->task_sg_nents, count) {
ec98f782
AG		 3735 if (cmd_sg->length > task_size)
3736 break;
3737
3738 *sg = *cmd_sg;
3739 task_size -= cmd_sg->length;
3740 cmd_sg = sg_next(cmd_sg);
c66ac9db 3741 }
c66ac9db 3742
ec98f782
AG		 3743 lba += task->task_sectors;
3744 sectors -= task->task_sectors;
c66ac9db 3745
ec98f782
AG		 3746 spin_lock_irqsave(&cmd->t_state_lock, flags);
3747 list_add_tail(&task->t_list, &cmd->t_task_list);
3748 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 3749 }
3750
ec98f782 3751 return task_count;
c66ac9db
NB		 3752}
3753
3754static int
ec98f782 3755transport_allocate_control_task(struct se_cmd *cmd)
c66ac9db 3756{
c66ac9db 3757 struct se_task *task;
ec98f782 3758 unsigned long flags;
c66ac9db 3759
91ec1d35
NB		 3760 /* Workaround for handling zero-length control CDBs */
3761 if ((cmd->se_cmd_flags & SCF_SCSI_CONTROL_SG_IO_CDB) &&
3762 !cmd->data_length)
3763 return 0;
3764
c66ac9db
NB		 3765 task = transport_generic_get_task(cmd, cmd->data_direction);
3766 if (!task)
ec98f782 3767 return -ENOMEM;
c66ac9db 3768
af3f00c7 3769 task->task_sg = cmd->t_data_sg;
c66ac9db 3770 task->task_size = cmd->data_length;
6708bb27 3771 task->task_sg_nents = cmd->t_data_nents;
c66ac9db 3772
ec98f782
AG		 3773 spin_lock_irqsave(&cmd->t_state_lock, flags);
3774 list_add_tail(&task->t_list, &cmd->t_task_list);
3775 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 3776
6708bb27 3777 /* Success! Return number of tasks allocated */
a3eedc22 3778 return 1;
ec98f782
AG		 3779}
3780
da0f7619
CH		 3781/*
3782 * Allocate any required ressources to execute the command, and either place
3783 * it on the execution queue if possible. For writes we might not have the
3784 * payload yet, thus notify the fabric via a call to ->write_pending instead.
c66ac9db 3785 */
a1d8b49a 3786int transport_generic_new_cmd(struct se_cmd *cmd)
c66ac9db 3787{
da0f7619 3788 struct se_device *dev = cmd->se_dev;
9ac54987 3789 int task_cdbs, task_cdbs_bidi = 0;
da0f7619 3790 int set_counts = 1;
c66ac9db
NB		 3791 int ret = 0;
3792
3793 /*
3794 * Determine is the TCM fabric module has already allocated physical
3795 * memory, and is directly calling transport_generic_map_mem_to_cmd()
ec98f782 3796 * beforehand.
c66ac9db 3797 */
ec98f782
AG		 3798 if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) &&
3799 cmd->data_length) {
05d1c7c0 3800 ret = transport_generic_get_mem(cmd);
c66ac9db 3801 if (ret < 0)
03e98c9e 3802 goto out_fail;
c66ac9db 3803 }
da0f7619 3804
1d20bb61 3805 /*
38b40067 3806 * For BIDI command set up the read tasks first.
1d20bb61 3807 */
da0f7619 3808 if (cmd->t_bidi_data_sg &&
38b40067
CH		 3809 dev->transport->transport_type != TRANSPORT_PLUGIN_PHBA_PDEV) {
3810 BUG_ON(!(cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB));
3811
9ac54987
NB		 3812 task_cdbs_bidi = transport_allocate_data_tasks(cmd,
3813 DMA_FROM_DEVICE, cmd->t_bidi_data_sg,
3814 cmd->t_bidi_data_nents);
3815 if (task_cdbs_bidi <= 0)
da0f7619
CH		 3816 goto out_fail;
3817
3818 atomic_inc(&cmd->t_fe_count);
3819 atomic_inc(&cmd->t_se_count);
3820 set_counts = 0;
3821 }
38b40067
CH		 3822
3823 if (cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB) {
3824 task_cdbs = transport_allocate_data_tasks(cmd,
3825 cmd->data_direction, cmd->t_data_sg,
3826 cmd->t_data_nents);
3827 } else {
3828 task_cdbs = transport_allocate_control_task(cmd);
3829 }
3830
410f6702 3831 if (task_cdbs < 0)
da0f7619 3832 goto out_fail;
410f6702 3833 else if (!task_cdbs && (cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB)) {
7d680f3b 3834 spin_lock_irq(&cmd->t_state_lock);
410f6702 3835 cmd->t_state = TRANSPORT_COMPLETE;
7d680f3b
CH		 3836 cmd->transport_state |= CMD_T_ACTIVE;
3837 spin_unlock_irq(&cmd->t_state_lock);
91ec1d35
NB		 3838
3839 if (cmd->t_task_cdb[0] == REQUEST_SENSE) {
3840 u8 ua_asc = 0, ua_ascq = 0;
3841
3842 core_scsi3_ua_clear_for_request_sense(cmd,
3843 &ua_asc, &ua_ascq);
3844 }
3845
410f6702
RD		 3846 INIT_WORK(&cmd->work, target_complete_ok_work);
3847 queue_work(target_completion_wq, &cmd->work);
3848 return 0;
3849 }
da0f7619
CH		 3850
3851 if (set_counts) {
3852 atomic_inc(&cmd->t_fe_count);
3853 atomic_inc(&cmd->t_se_count);
3854 }
3855
9ac54987
NB		 3856 cmd->t_task_list_num = (task_cdbs + task_cdbs_bidi);
3857 atomic_set(&cmd->t_task_cdbs_left, cmd->t_task_list_num);
3858 atomic_set(&cmd->t_task_cdbs_ex_left, cmd->t_task_list_num);
da0f7619 3859
c66ac9db 3860 /*
a1d8b49a 3861 * For WRITEs, let the fabric know its buffer is ready..
c66ac9db
NB		 3862 * This WRITE struct se_cmd (and all of its associated struct se_task's)
3863 * will be added to the struct se_device execution queue after its WRITE
3864 * data has arrived. (ie: It gets handled by the transport processing
3865 * thread a second time)
3866 */
3867 if (cmd->data_direction == DMA_TO_DEVICE) {
3868 transport_add_tasks_to_state_queue(cmd);
3869 return transport_generic_write_pending(cmd);
3870 }
3871 /*
3872 * Everything else but a WRITE, add the struct se_cmd's struct se_task's
3873 * to the execution queue.
3874 */
3875 transport_execute_tasks(cmd);
3876 return 0;
da0f7619
CH		 3877
3878out_fail:
3879 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
3880 cmd->scsi_sense_reason = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
3881 return -EINVAL;
c66ac9db 3882}
a1d8b49a 3883EXPORT_SYMBOL(transport_generic_new_cmd);
c66ac9db
NB		 3884
3885/* transport_generic_process_write():
3886 *
3887 *
3888 */
3889void transport_generic_process_write(struct se_cmd *cmd)
3890{
c66ac9db
NB		 3891 transport_execute_tasks(cmd);
3892}
3893EXPORT_SYMBOL(transport_generic_process_write);
3894
e057f533 3895static void transport_write_pending_qf(struct se_cmd *cmd)
07bde79a 3896{
f147abb4
NB		 3897 int ret;
3898
3899 ret = cmd->se_tfo->write_pending(cmd);
3900 if (ret == -EAGAIN || ret == -ENOMEM) {
e057f533
CH		 3901 pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n",
3902 cmd);
3903 transport_handle_queue_full(cmd, cmd->se_dev);
3904 }
07bde79a
NB		 3905}
3906
c66ac9db
NB		 3907static int transport_generic_write_pending(struct se_cmd *cmd)
3908{
3909 unsigned long flags;
3910 int ret;
3911
a1d8b49a 3912 spin_lock_irqsave(&cmd->t_state_lock, flags);
c66ac9db 3913 cmd->t_state = TRANSPORT_WRITE_PENDING;
a1d8b49a 3914 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
07bde79a 3915
c66ac9db
NB		 3916 /*
3917 * Clear the se_cmd for WRITE_PENDING status in order to set
7d680f3b
CH		 3918 * CMD_T_ACTIVE so that transport_generic_handle_data can be called
3919 * from HW target mode interrupt code. This is safe to be called
3920 * with transport_off=1 before the cmd->se_tfo->write_pending
c66ac9db
NB		 3921 * because the se_cmd->se_lun pointer is not being cleared.
3922 */
3923 transport_cmd_check_stop(cmd, 1, 0);
3924
3925 /*
3926 * Call the fabric write_pending function here to let the
3927 * frontend know that WRITE buffers are ready.
3928 */
e3d6f909 3929 ret = cmd->se_tfo->write_pending(cmd);
f147abb4 3930 if (ret == -EAGAIN || ret == -ENOMEM)
07bde79a
NB		 3931 goto queue_full;
3932 else if (ret < 0)
c66ac9db
NB		 3933 return ret;
3934
03e98c9e 3935 return 1;
07bde79a
NB		 3936
3937queue_full:
6708bb27 3938 pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", cmd);
07bde79a 3939 cmd->t_state = TRANSPORT_COMPLETE_QF_WP;
e057f533 3940 transport_handle_queue_full(cmd, cmd->se_dev);
f147abb4 3941 return 0;
c66ac9db
NB		 3942}
3943
39c05f32 3944void transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks)
c66ac9db 3945{
d14921d6 3946 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD)) {
c8e31f26 3947 if (wait_for_tasks && (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
d14921d6
NB		 3948 transport_wait_for_tasks(cmd);
3949
35462975 3950 transport_release_cmd(cmd);
d14921d6
NB		 3951 } else {
3952 if (wait_for_tasks)
3953 transport_wait_for_tasks(cmd);
3954
c66ac9db
NB		 3955 core_dec_lacl_count(cmd->se_sess->se_node_acl, cmd);
3956
82f1c8a4 3957 if (cmd->se_lun)
c66ac9db 3958 transport_lun_remove_cmd(cmd);
c66ac9db 3959
f4366772
NB		 3960 transport_free_dev_tasks(cmd);
3961
39c05f32 3962 transport_put_cmd(cmd);
c66ac9db
NB		 3963 }
3964}
3965EXPORT_SYMBOL(transport_generic_free_cmd);
3966
a17f091d
NB		 3967/* target_get_sess_cmd - Add command to active ->sess_cmd_list
3968 * @se_sess: session to reference
3969 * @se_cmd: command descriptor to add
a6360785 3970 * @ack_kref: Signal that fabric will perform an ack target_put_sess_cmd()
a17f091d 3971 */
a6360785
NB		 3972void target_get_sess_cmd(struct se_session *se_sess, struct se_cmd *se_cmd,
3973 bool ack_kref)
a17f091d
NB		 3974{
3975 unsigned long flags;
3976
7481deb4 3977 kref_init(&se_cmd->cmd_kref);
a6360785
NB		 3978 /*
3979 * Add a second kref if the fabric caller is expecting to handle
3980 * fabric acknowledgement that requires two target_put_sess_cmd()
3981 * invocations before se_cmd descriptor release.
3982 */
3983 if (ack_kref == true)
3984 kref_get(&se_cmd->cmd_kref);
7481deb4 3985
a17f091d
NB		 3986 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
3987 list_add_tail(&se_cmd->se_cmd_list, &se_sess->sess_cmd_list);
3988 se_cmd->check_release = 1;
3989 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
3990}
3991EXPORT_SYMBOL(target_get_sess_cmd);
3992
7481deb4 3993static void target_release_cmd_kref(struct kref *kref)
a17f091d 3994{
7481deb4
NB		 3995 struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref);
3996 struct se_session *se_sess = se_cmd->se_sess;
a17f091d
NB		 3997 unsigned long flags;
3998
3999 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
4000 if (list_empty(&se_cmd->se_cmd_list)) {
4001 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
4002 WARN_ON(1);
7481deb4 4003 return;
a17f091d 4004 }
a17f091d
NB		 4005 if (se_sess->sess_tearing_down && se_cmd->cmd_wait_set) {
4006 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
4007 complete(&se_cmd->cmd_wait_comp);
7481deb4 4008 return;
a17f091d
NB		 4009 }
4010 list_del(&se_cmd->se_cmd_list);
4011 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
4012
7481deb4
NB		 4013 se_cmd->se_tfo->release_cmd(se_cmd);
4014}
4015
4016/* target_put_sess_cmd - Check for active I/O shutdown via kref_put
4017 * @se_sess: session to reference
4018 * @se_cmd: command descriptor to drop
4019 */
4020int target_put_sess_cmd(struct se_session *se_sess, struct se_cmd *se_cmd)
4021{
4022 return kref_put(&se_cmd->cmd_kref, target_release_cmd_kref);
a17f091d
NB		 4023}
4024EXPORT_SYMBOL(target_put_sess_cmd);
4025
4026/* target_splice_sess_cmd_list - Split active cmds into sess_wait_list
4027 * @se_sess: session to split
4028 */
4029void target_splice_sess_cmd_list(struct se_session *se_sess)
4030{
4031 struct se_cmd *se_cmd;
4032 unsigned long flags;
4033
4034 WARN_ON(!list_empty(&se_sess->sess_wait_list));
4035 INIT_LIST_HEAD(&se_sess->sess_wait_list);
4036
4037 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
4038 se_sess->sess_tearing_down = 1;
4039
4040 list_splice_init(&se_sess->sess_cmd_list, &se_sess->sess_wait_list);
4041
4042 list_for_each_entry(se_cmd, &se_sess->sess_wait_list, se_cmd_list)
4043 se_cmd->cmd_wait_set = 1;
4044
4045 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
4046}
4047EXPORT_SYMBOL(target_splice_sess_cmd_list);
4048
4049/* target_wait_for_sess_cmds - Wait for outstanding descriptors
4050 * @se_sess: session to wait for active I/O
4051 * @wait_for_tasks: Make extra transport_wait_for_tasks call
4052 */
4053void target_wait_for_sess_cmds(
4054 struct se_session *se_sess,
4055 int wait_for_tasks)
4056{
4057 struct se_cmd *se_cmd, *tmp_cmd;
4058 bool rc = false;
4059
4060 list_for_each_entry_safe(se_cmd, tmp_cmd,
4061 &se_sess->sess_wait_list, se_cmd_list) {
4062 list_del(&se_cmd->se_cmd_list);
4063
4064 pr_debug("Waiting for se_cmd: %p t_state: %d, fabric state:"
4065 " %d\n", se_cmd, se_cmd->t_state,
4066 se_cmd->se_tfo->get_cmd_state(se_cmd));
4067
4068 if (wait_for_tasks) {
4069 pr_debug("Calling transport_wait_for_tasks se_cmd: %p t_state: %d,"
4070 " fabric state: %d\n", se_cmd, se_cmd->t_state,
4071 se_cmd->se_tfo->get_cmd_state(se_cmd));
4072
4073 rc = transport_wait_for_tasks(se_cmd);
4074
4075 pr_debug("After transport_wait_for_tasks se_cmd: %p t_state: %d,"
4076 " fabric state: %d\n", se_cmd, se_cmd->t_state,
4077 se_cmd->se_tfo->get_cmd_state(se_cmd));
4078 }
4079
4080 if (!rc) {
4081 wait_for_completion(&se_cmd->cmd_wait_comp);
4082 pr_debug("After cmd_wait_comp: se_cmd: %p t_state: %d"
4083 " fabric state: %d\n", se_cmd, se_cmd->t_state,
4084 se_cmd->se_tfo->get_cmd_state(se_cmd));
4085 }
4086
4087 se_cmd->se_tfo->release_cmd(se_cmd);
4088 }
4089}
4090EXPORT_SYMBOL(target_wait_for_sess_cmds);
4091
c66ac9db
NB		 4092/* transport_lun_wait_for_tasks():
4093 *
4094 * Called from ConfigFS context to stop the passed struct se_cmd to allow
4095 * an struct se_lun to be successfully shutdown.
4096 */
4097static int transport_lun_wait_for_tasks(struct se_cmd *cmd, struct se_lun *lun)
4098{
4099 unsigned long flags;
4100 int ret;
4101 /*
4102 * If the frontend has already requested this struct se_cmd to
4103 * be stopped, we can safely ignore this struct se_cmd.
4104 */
a1d8b49a 4105 spin_lock_irqsave(&cmd->t_state_lock, flags);
7d680f3b
CH		 4106 if (cmd->transport_state & CMD_T_STOP) {
4107 cmd->transport_state &= ~CMD_T_LUN_STOP;
4108
4109 pr_debug("ConfigFS ITT[0x%08x] - CMD_T_STOP, skipping\n",
4110 cmd->se_tfo->get_task_tag(cmd));
a1d8b49a 4111 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 4112 transport_cmd_check_stop(cmd, 1, 0);
e3d6f909 4113 return -EPERM;
c66ac9db 4114 }
7d680f3b 4115 cmd->transport_state |= CMD_T_LUN_FE_STOP;
a1d8b49a 4116 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 4117
5951146d 4118 wake_up_interruptible(&cmd->se_dev->dev_queue_obj.thread_wq);
c66ac9db
NB		 4119
4120 ret = transport_stop_tasks_for_cmd(cmd);
4121
6708bb27
AG		 4122 pr_debug("ConfigFS: cmd: %p t_tasks: %d stop tasks ret:"
4123 " %d\n", cmd, cmd->t_task_list_num, ret);
c66ac9db 4124 if (!ret) {
6708bb27 4125 pr_debug("ConfigFS: ITT[0x%08x] - stopping cmd....\n",
e3d6f909 4126 cmd->se_tfo->get_task_tag(cmd));
a1d8b49a 4127 wait_for_completion(&cmd->transport_lun_stop_comp);
6708bb27 4128 pr_debug("ConfigFS: ITT[0x%08x] - stopped cmd....\n",
e3d6f909 4129 cmd->se_tfo->get_task_tag(cmd));
c66ac9db 4130 }
3df8d40b 4131 transport_remove_cmd_from_queue(cmd);
c66ac9db
NB		 4132
4133 return 0;
4134}
4135
c66ac9db
NB		 4136static void __transport_clear_lun_from_sessions(struct se_lun *lun)
4137{
4138 struct se_cmd *cmd = NULL;
4139 unsigned long lun_flags, cmd_flags;
4140 /*
4141 * Do exception processing and return CHECK_CONDITION status to the
4142 * Initiator Port.
4143 */
4144 spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
5951146d
AG		 4145 while (!list_empty(&lun->lun_cmd_list)) {
4146 cmd = list_first_entry(&lun->lun_cmd_list,
4147 struct se_cmd, se_lun_node);
3d26fea0 4148 list_del_init(&cmd->se_lun_node);
5951146d 4149
c66ac9db
NB		 4150 /*
4151 * This will notify iscsi_target_transport.c:
4152 * transport_cmd_check_stop() that a LUN shutdown is in
4153 * progress for the iscsi_cmd_t.
4154 */
a1d8b49a 4155 spin_lock(&cmd->t_state_lock);
6708bb27 4156 pr_debug("SE_LUN[%d] - Setting cmd->transport"
c66ac9db 4157 "_lun_stop for ITT: 0x%08x\n",
e3d6f909
AG		 4158 cmd->se_lun->unpacked_lun,
4159 cmd->se_tfo->get_task_tag(cmd));
7d680f3b 4160 cmd->transport_state |= CMD_T_LUN_STOP;
a1d8b49a 4161 spin_unlock(&cmd->t_state_lock);
c66ac9db
NB		 4162
4163 spin_unlock_irqrestore(&lun->lun_cmd_lock, lun_flags);
4164
6708bb27
AG		 4165 if (!cmd->se_lun) {
4166 pr_err("ITT: 0x%08x, [i,t]_state: %u/%u\n",
e3d6f909
AG		 4167 cmd->se_tfo->get_task_tag(cmd),
4168 cmd->se_tfo->get_cmd_state(cmd), cmd->t_state);
c66ac9db
NB		 4169 BUG();
4170 }
4171 /*
4172 * If the Storage engine still owns the iscsi_cmd_t, determine
4173 * and/or stop its context.
4174 */
6708bb27 4175 pr_debug("SE_LUN[%d] - ITT: 0x%08x before transport"
e3d6f909
AG		 4176 "_lun_wait_for_tasks()\n", cmd->se_lun->unpacked_lun,
4177 cmd->se_tfo->get_task_tag(cmd));
c66ac9db 4178
e3d6f909 4179 if (transport_lun_wait_for_tasks(cmd, cmd->se_lun) < 0) {
c66ac9db
NB		 4180 spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
4181 continue;
4182 }
4183
6708bb27 4184 pr_debug("SE_LUN[%d] - ITT: 0x%08x after transport_lun"
c66ac9db 4185 "_wait_for_tasks(): SUCCESS\n",
e3d6f909
AG		 4186 cmd->se_lun->unpacked_lun,
4187 cmd->se_tfo->get_task_tag(cmd));
c66ac9db 4188
a1d8b49a 4189 spin_lock_irqsave(&cmd->t_state_lock, cmd_flags);
7d680f3b 4190 if (!(cmd->transport_state & CMD_T_DEV_ACTIVE)) {
a1d8b49a 4191 spin_unlock_irqrestore(&cmd->t_state_lock, cmd_flags);
c66ac9db
NB		 4192 goto check_cond;
4193 }
7d680f3b 4194 cmd->transport_state &= ~CMD_T_DEV_ACTIVE;
c66ac9db 4195 transport_all_task_dev_remove_state(cmd);
a1d8b49a 4196 spin_unlock_irqrestore(&cmd->t_state_lock, cmd_flags);
c66ac9db
NB		 4197
4198 transport_free_dev_tasks(cmd);
4199 /*
4200 * The Storage engine stopped this struct se_cmd before it was
4201 * send to the fabric frontend for delivery back to the
4202 * Initiator Node. Return this SCSI CDB back with an
4203 * CHECK_CONDITION status.
4204 */
4205check_cond:
4206 transport_send_check_condition_and_sense(cmd,
4207 TCM_NON_EXISTENT_LUN, 0);
4208 /*
4209 * If the fabric frontend is waiting for this iscsi_cmd_t to
4210 * be released, notify the waiting thread now that LU has
4211 * finished accessing it.
4212 */
a1d8b49a 4213 spin_lock_irqsave(&cmd->t_state_lock, cmd_flags);
7d680f3b 4214 if (cmd->transport_state & CMD_T_LUN_FE_STOP) {
6708bb27 4215 pr_debug("SE_LUN[%d] - Detected FE stop for"
c66ac9db
NB		 4216 " struct se_cmd: %p ITT: 0x%08x\n",
4217 lun->unpacked_lun,
e3d6f909 4218 cmd, cmd->se_tfo->get_task_tag(cmd));
c66ac9db 4219
a1d8b49a 4220 spin_unlock_irqrestore(&cmd->t_state_lock,
c66ac9db
NB		 4221 cmd_flags);
4222 transport_cmd_check_stop(cmd, 1, 0);
a1d8b49a 4223 complete(&cmd->transport_lun_fe_stop_comp);
c66ac9db
NB		 4224 spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
4225 continue;
4226 }
6708bb27 4227 pr_debug("SE_LUN[%d] - ITT: 0x%08x finished processing\n",
e3d6f909 4228 lun->unpacked_lun, cmd->se_tfo->get_task_tag(cmd));
c66ac9db 4229
a1d8b49a 4230 spin_unlock_irqrestore(&cmd->t_state_lock, cmd_flags);
c66ac9db
NB		 4231 spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
4232 }
4233 spin_unlock_irqrestore(&lun->lun_cmd_lock, lun_flags);
4234}
4235
4236static int transport_clear_lun_thread(void *p)
4237{
8359cf43 4238 struct se_lun *lun = p;
c66ac9db
NB		 4239
4240 __transport_clear_lun_from_sessions(lun);
4241 complete(&lun->lun_shutdown_comp);
4242
4243 return 0;
4244}
4245
4246int transport_clear_lun_from_sessions(struct se_lun *lun)
4247{
4248 struct task_struct *kt;
4249
5951146d 4250 kt = kthread_run(transport_clear_lun_thread, lun,
c66ac9db
NB		 4251 "tcm_cl_%u", lun->unpacked_lun);
4252 if (IS_ERR(kt)) {
6708bb27 4253 pr_err("Unable to start clear_lun thread\n");
e3d6f909 4254 return PTR_ERR(kt);
c66ac9db
NB		 4255 }
4256 wait_for_completion(&lun->lun_shutdown_comp);
4257
4258 return 0;
4259}
4260
d14921d6
NB		 4261/**
4262 * transport_wait_for_tasks - wait for completion to occur
4263 * @cmd: command to wait
c66ac9db 4264 *
d14921d6
NB		 4265 * Called from frontend fabric context to wait for storage engine
4266 * to pause and/or release frontend generated struct se_cmd.
c66ac9db 4267 */
a17f091d 4268bool transport_wait_for_tasks(struct se_cmd *cmd)
c66ac9db
NB		 4269{
4270 unsigned long flags;
4271
a1d8b49a 4272 spin_lock_irqsave(&cmd->t_state_lock, flags);
c8e31f26
AG		 4273 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD) &&
4274 !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) {
d14921d6 4275 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
a17f091d 4276 return false;
d14921d6
NB		 4277 }
4278 /*
4279 * Only perform a possible wait_for_tasks if SCF_SUPPORTED_SAM_OPCODE
4280 * has been set in transport_set_supported_SAM_opcode().
4281 */
c8e31f26
AG		 4282 if (!(cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) &&
4283 !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) {
d14921d6 4284 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
a17f091d 4285 return false;
d14921d6 4286 }
c66ac9db
NB		 4287 /*
4288 * If we are already stopped due to an external event (ie: LUN shutdown)
4289 * sleep until the connection can have the passed struct se_cmd back.
a1d8b49a 4290 * The cmd->transport_lun_stopped_sem will be upped by
c66ac9db
NB		 4291 * transport_clear_lun_from_sessions() once the ConfigFS context caller
4292 * has completed its operation on the struct se_cmd.
4293 */
7d680f3b 4294 if (cmd->transport_state & CMD_T_LUN_STOP) {
6708bb27 4295 pr_debug("wait_for_tasks: Stopping"
e3d6f909 4296 " wait_for_completion(&cmd->t_tasktransport_lun_fe"
c66ac9db 4297 "_stop_comp); for ITT: 0x%08x\n",
e3d6f909 4298 cmd->se_tfo->get_task_tag(cmd));
c66ac9db
NB		 4299 /*
4300 * There is a special case for WRITES where a FE exception +
4301 * LUN shutdown means ConfigFS context is still sleeping on
4302 * transport_lun_stop_comp in transport_lun_wait_for_tasks().
4303 * We go ahead and up transport_lun_stop_comp just to be sure
4304 * here.
4305 */
a1d8b49a
AG		 4306 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
4307 complete(&cmd->transport_lun_stop_comp);
4308 wait_for_completion(&cmd->transport_lun_fe_stop_comp);
4309 spin_lock_irqsave(&cmd->t_state_lock, flags);
c66ac9db
NB		 4310
4311 transport_all_task_dev_remove_state(cmd);
4312 /*
4313 * At this point, the frontend who was the originator of this
4314 * struct se_cmd, now owns the structure and can be released through
4315 * normal means below.
4316 */
6708bb27 4317 pr_debug("wait_for_tasks: Stopped"
e3d6f909 4318 " wait_for_completion(&cmd->t_tasktransport_lun_fe_"
c66ac9db 4319 "stop_comp); for ITT: 0x%08x\n",
e3d6f909 4320 cmd->se_tfo->get_task_tag(cmd));
c66ac9db 4321
7d680f3b 4322 cmd->transport_state &= ~CMD_T_LUN_STOP;
c66ac9db 4323 }
7d680f3b
CH		 4324
4325 if (!(cmd->transport_state & CMD_T_ACTIVE) ||
4326 (cmd->transport_state & CMD_T_ABORTED)) {
d14921d6 4327 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
a17f091d 4328 return false;
d14921d6 4329 }
c66ac9db 4330
7d680f3b 4331 cmd->transport_state |= CMD_T_STOP;
c66ac9db 4332
6708bb27 4333 pr_debug("wait_for_tasks: Stopping %p ITT: 0x%08x"
7d680f3b 4334 " i_state: %d, t_state: %d, CMD_T_STOP\n",
f2da9dbd
CH		 4335 cmd, cmd->se_tfo->get_task_tag(cmd),
4336 cmd->se_tfo->get_cmd_state(cmd), cmd->t_state);
c66ac9db 4337
a1d8b49a 4338 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db 4339
5951146d 4340 wake_up_interruptible(&cmd->se_dev->dev_queue_obj.thread_wq);
c66ac9db 4341
a1d8b49a 4342 wait_for_completion(&cmd->t_transport_stop_comp);
c66ac9db 4343
a1d8b49a 4344 spin_lock_irqsave(&cmd->t_state_lock, flags);
7d680f3b 4345 cmd->transport_state &= ~(CMD_T_ACTIVE | CMD_T_STOP);
c66ac9db 4346
6708bb27 4347 pr_debug("wait_for_tasks: Stopped wait_for_compltion("
a1d8b49a 4348 "&cmd->t_transport_stop_comp) for ITT: 0x%08x\n",
e3d6f909 4349 cmd->se_tfo->get_task_tag(cmd));
c66ac9db 4350
d14921d6 4351 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
a17f091d
NB		 4352
4353 return true;
c66ac9db 4354}
d14921d6 4355EXPORT_SYMBOL(transport_wait_for_tasks);
c66ac9db
NB		 4356
4357static int transport_get_sense_codes(
4358 struct se_cmd *cmd,
4359 u8 *asc,
4360 u8 *ascq)
4361{
4362 *asc = cmd->scsi_asc;
4363 *ascq = cmd->scsi_ascq;
4364
4365 return 0;
4366}
4367
4368static int transport_set_sense_codes(
4369 struct se_cmd *cmd,
4370 u8 asc,
4371 u8 ascq)
4372{
4373 cmd->scsi_asc = asc;
4374 cmd->scsi_ascq = ascq;
4375
4376 return 0;
4377}
4378
4379int transport_send_check_condition_and_sense(
4380 struct se_cmd *cmd,
4381 u8 reason,
4382 int from_transport)
4383{
4384 unsigned char *buffer = cmd->sense_buffer;
4385 unsigned long flags;
4386 int offset;
4387 u8 asc = 0, ascq = 0;
4388
a1d8b49a 4389 spin_lock_irqsave(&cmd->t_state_lock, flags);
c66ac9db 4390 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
a1d8b49a 4391 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 4392 return 0;
4393 }
4394 cmd->se_cmd_flags |= SCF_SENT_CHECK_CONDITION;
a1d8b49a 4395 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
c66ac9db
NB		 4396
4397 if (!reason && from_transport)
4398 goto after_reason;
4399
4400 if (!from_transport)
4401 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE;
4402 /*
4403 * Data Segment and SenseLength of the fabric response PDU.
4404 *
4405 * TRANSPORT_SENSE_BUFFER is now set to SCSI_SENSE_BUFFERSIZE
4406 * from include/scsi/scsi_cmnd.h
4407 */
e3d6f909 4408 offset = cmd->se_tfo->set_fabric_sense_len(cmd,
c66ac9db
NB		 4409 TRANSPORT_SENSE_BUFFER);
4410 /*
4411 * Actual SENSE DATA, see SPC-3 7.23.2 SPC_SENSE_KEY_OFFSET uses
4412 * SENSE KEY values from include/scsi/scsi.h
4413 */
4414 switch (reason) {
4415 case TCM_NON_EXISTENT_LUN:
eb39d340
NB		 4416 /* CURRENT ERROR */
4417 buffer[offset] = 0x70;
895f3022 4418 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
eb39d340
NB		 4419 /* ILLEGAL REQUEST */
4420 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
4421 /* LOGICAL UNIT NOT SUPPORTED */
4422 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x25;
4423 break;
c66ac9db
NB		 4424 case TCM_UNSUPPORTED_SCSI_OPCODE:
4425 case TCM_SECTOR_COUNT_TOO_MANY:
4426 /* CURRENT ERROR */
4427 buffer[offset] = 0x70;
895f3022 4428 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4429 /* ILLEGAL REQUEST */
4430 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
4431 /* INVALID COMMAND OPERATION CODE */
4432 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x20;
4433 break;
4434 case TCM_UNKNOWN_MODE_PAGE:
4435 /* CURRENT ERROR */
4436 buffer[offset] = 0x70;
895f3022 4437 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4438 /* ILLEGAL REQUEST */
4439 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
4440 /* INVALID FIELD IN CDB */
4441 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x24;
4442 break;
4443 case TCM_CHECK_CONDITION_ABORT_CMD:
4444 /* CURRENT ERROR */
4445 buffer[offset] = 0x70;
895f3022 4446 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4447 /* ABORTED COMMAND */
4448 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
4449 /* BUS DEVICE RESET FUNCTION OCCURRED */
4450 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x29;
4451 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x03;
4452 break;
4453 case TCM_INCORRECT_AMOUNT_OF_DATA:
4454 /* CURRENT ERROR */
4455 buffer[offset] = 0x70;
895f3022 4456 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4457 /* ABORTED COMMAND */
4458 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
4459 /* WRITE ERROR */
4460 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x0c;
4461 /* NOT ENOUGH UNSOLICITED DATA */
4462 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x0d;
4463 break;
4464 case TCM_INVALID_CDB_FIELD:
4465 /* CURRENT ERROR */
4466 buffer[offset] = 0x70;
895f3022 4467 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
9fbc8909
RD		 4468 /* ILLEGAL REQUEST */
4469 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
c66ac9db
NB		 4470 /* INVALID FIELD IN CDB */
4471 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x24;
4472 break;
4473 case TCM_INVALID_PARAMETER_LIST:
4474 /* CURRENT ERROR */
4475 buffer[offset] = 0x70;
895f3022 4476 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
9fbc8909
RD		 4477 /* ILLEGAL REQUEST */
4478 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
c66ac9db
NB		 4479 /* INVALID FIELD IN PARAMETER LIST */
4480 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x26;
4481 break;
4482 case TCM_UNEXPECTED_UNSOLICITED_DATA:
4483 /* CURRENT ERROR */
4484 buffer[offset] = 0x70;
895f3022 4485 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4486 /* ABORTED COMMAND */
4487 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
4488 /* WRITE ERROR */
4489 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x0c;
4490 /* UNEXPECTED_UNSOLICITED_DATA */
4491 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x0c;
4492 break;
4493 case TCM_SERVICE_CRC_ERROR:
4494 /* CURRENT ERROR */
4495 buffer[offset] = 0x70;
895f3022 4496 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4497 /* ABORTED COMMAND */
4498 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
4499 /* PROTOCOL SERVICE CRC ERROR */
4500 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x47;
4501 /* N/A */
4502 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x05;
4503 break;
4504 case TCM_SNACK_REJECTED:
4505 /* CURRENT ERROR */
4506 buffer[offset] = 0x70;
895f3022 4507 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4508 /* ABORTED COMMAND */
4509 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
4510 /* READ ERROR */
4511 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x11;
4512 /* FAILED RETRANSMISSION REQUEST */
4513 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x13;
4514 break;
4515 case TCM_WRITE_PROTECTED:
4516 /* CURRENT ERROR */
4517 buffer[offset] = 0x70;
895f3022 4518 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4519 /* DATA PROTECT */
4520 buffer[offset+SPC_SENSE_KEY_OFFSET] = DATA_PROTECT;
4521 /* WRITE PROTECTED */
4522 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x27;
4523 break;
4524 case TCM_CHECK_CONDITION_UNIT_ATTENTION:
4525 /* CURRENT ERROR */
4526 buffer[offset] = 0x70;
895f3022 4527 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4528 /* UNIT ATTENTION */
4529 buffer[offset+SPC_SENSE_KEY_OFFSET] = UNIT_ATTENTION;
4530 core_scsi3_ua_for_check_condition(cmd, &asc, &ascq);
4531 buffer[offset+SPC_ASC_KEY_OFFSET] = asc;
4532 buffer[offset+SPC_ASCQ_KEY_OFFSET] = ascq;
4533 break;
4534 case TCM_CHECK_CONDITION_NOT_READY:
4535 /* CURRENT ERROR */
4536 buffer[offset] = 0x70;
895f3022 4537 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4538 /* Not Ready */
4539 buffer[offset+SPC_SENSE_KEY_OFFSET] = NOT_READY;
4540 transport_get_sense_codes(cmd, &asc, &ascq);
4541 buffer[offset+SPC_ASC_KEY_OFFSET] = asc;
4542 buffer[offset+SPC_ASCQ_KEY_OFFSET] = ascq;
4543 break;
4544 case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE:
4545 default:
4546 /* CURRENT ERROR */
4547 buffer[offset] = 0x70;
895f3022 4548 buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10;
c66ac9db
NB		 4549 /* ILLEGAL REQUEST */
4550 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
4551 /* LOGICAL UNIT COMMUNICATION FAILURE */
4552 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x80;
4553 break;
4554 }
4555 /*
4556 * This code uses linux/include/scsi/scsi.h SAM status codes!
4557 */
4558 cmd->scsi_status = SAM_STAT_CHECK_CONDITION;
4559 /*
4560 * Automatically padded, this value is encoded in the fabric's
4561 * data_length response PDU containing the SCSI defined sense data.
4562 */
4563 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER + offset;
4564
4565after_reason:
07bde79a 4566 return cmd->se_tfo->queue_status(cmd);
c66ac9db
NB		 4567}
4568EXPORT_SYMBOL(transport_send_check_condition_and_sense);
4569
4570int transport_check_aborted_status(struct se_cmd *cmd, int send_status)
4571{
4572 int ret = 0;
4573
7d680f3b 4574 if (cmd->transport_state & CMD_T_ABORTED) {
6708bb27 4575 if (!send_status ||
c66ac9db
NB		 4576 (cmd->se_cmd_flags & SCF_SENT_DELAYED_TAS))
4577 return 1;
4578#if 0
6708bb27 4579 pr_debug("Sending delayed SAM_STAT_TASK_ABORTED"
c66ac9db 4580 " status for CDB: 0x%02x ITT: 0x%08x\n",
a1d8b49a 4581 cmd->t_task_cdb[0],
e3d6f909 4582 cmd->se_tfo->get_task_tag(cmd));
c66ac9db
NB		 4583#endif
4584 cmd->se_cmd_flags |= SCF_SENT_DELAYED_TAS;
e3d6f909 4585 cmd->se_tfo->queue_status(cmd);
c66ac9db
NB		 4586 ret = 1;
4587 }
4588 return ret;
4589}
4590EXPORT_SYMBOL(transport_check_aborted_status);
4591
4592void transport_send_task_abort(struct se_cmd *cmd)
4593{
c252f003
NB		 4594 unsigned long flags;
4595
4596 spin_lock_irqsave(&cmd->t_state_lock, flags);
4597 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
4598 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
4599 return;
4600 }
4601 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
4602
c66ac9db
NB		 4603 /*
4604 * If there are still expected incoming fabric WRITEs, we wait
4605 * until until they have completed before sending a TASK_ABORTED
4606 * response. This response with TASK_ABORTED status will be
4607 * queued back to fabric module by transport_check_aborted_status().
4608 */
4609 if (cmd->data_direction == DMA_TO_DEVICE) {
e3d6f909 4610 if (cmd->se_tfo->write_pending_status(cmd) != 0) {
7d680f3b 4611 cmd->transport_state |= CMD_T_ABORTED;
c66ac9db 4612 smp_mb__after_atomic_inc();
c66ac9db
NB		 4613 }
4614 }
4615 cmd->scsi_status = SAM_STAT_TASK_ABORTED;
4616#if 0
6708bb27 4617 pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x,"
a1d8b49a 4618 " ITT: 0x%08x\n", cmd->t_task_cdb[0],
e3d6f909 4619 cmd->se_tfo->get_task_tag(cmd));
c66ac9db 4620#endif
e3d6f909 4621 cmd->se_tfo->queue_status(cmd);
c66ac9db
NB		 4622}
4623
e26d99ae 4624static int transport_generic_do_tmr(struct se_cmd *cmd)
c66ac9db 4625{
5951146d 4626 struct se_device *dev = cmd->se_dev;
c66ac9db
NB		 4627 struct se_tmr_req *tmr = cmd->se_tmr_req;
4628 int ret;
4629
4630 switch (tmr->function) {
5c6cd613 4631 case TMR_ABORT_TASK:
c66ac9db
NB		 4632 tmr->response = TMR_FUNCTION_REJECTED;
4633 break;
5c6cd613
NB		 4634 case TMR_ABORT_TASK_SET:
4635 case TMR_CLEAR_ACA:
4636 case TMR_CLEAR_TASK_SET:
c66ac9db
NB		 4637 tmr->response = TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED;
4638 break;
5c6cd613 4639 case TMR_LUN_RESET:
c66ac9db
NB		 4640 ret = core_tmr_lun_reset(dev, tmr, NULL, NULL);
4641 tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE :
4642 TMR_FUNCTION_REJECTED;
4643 break;
5c6cd613 4644 case TMR_TARGET_WARM_RESET:
c66ac9db
NB		 4645 tmr->response = TMR_FUNCTION_REJECTED;
4646 break;
5c6cd613 4647 case TMR_TARGET_COLD_RESET:
c66ac9db
NB		 4648 tmr->response = TMR_FUNCTION_REJECTED;
4649 break;
c66ac9db 4650 default:
6708bb27 4651 pr_err("Uknown TMR function: 0x%02x.\n",
c66ac9db
NB		 4652 tmr->function);
4653 tmr->response = TMR_FUNCTION_REJECTED;
4654 break;
4655 }
4656
4657 cmd->t_state = TRANSPORT_ISTATE_PROCESSING;
e3d6f909 4658 cmd->se_tfo->queue_tm_rsp(cmd);
c66ac9db 4659
b7b8bef7 4660 transport_cmd_check_stop_to_fabric(cmd);
c66ac9db
NB		 4661 return 0;
4662}
4663
c66ac9db
NB		 4664/* transport_processing_thread():
4665 *
4666 *
4667 */
4668static int transport_processing_thread(void *param)
4669{
5951146d 4670 int ret;
c66ac9db 4671 struct se_cmd *cmd;
8359cf43 4672 struct se_device *dev = param;
c66ac9db 4673
c66ac9db 4674 while (!kthread_should_stop()) {
e3d6f909
AG		 4675 ret = wait_event_interruptible(dev->dev_queue_obj.thread_wq,
4676 atomic_read(&dev->dev_queue_obj.queue_cnt) ||
c66ac9db
NB		 4677 kthread_should_stop());
4678 if (ret < 0)
4679 goto out;
4680
c66ac9db 4681get_cmd:
5951146d
AG		 4682 cmd = transport_get_cmd_from_queue(&dev->dev_queue_obj);
4683 if (!cmd)
c66ac9db
NB		 4684 continue;
4685
5951146d 4686 switch (cmd->t_state) {
680b73c5
CH		 4687 case TRANSPORT_NEW_CMD:
4688 BUG();
4689 break;
c66ac9db 4690 case TRANSPORT_NEW_CMD_MAP:
6708bb27
AG		 4691 if (!cmd->se_tfo->new_cmd_map) {
4692 pr_err("cmd->se_tfo->new_cmd_map is"
c66ac9db
NB		 4693 " NULL for TRANSPORT_NEW_CMD_MAP\n");
4694 BUG();
4695 }
e3d6f909 4696 ret = cmd->se_tfo->new_cmd_map(cmd);
c66ac9db 4697 if (ret < 0) {
03e98c9e 4698 transport_generic_request_failure(cmd);
c66ac9db
NB		 4699 break;
4700 }
c66ac9db 4701 ret = transport_generic_new_cmd(cmd);
f147abb4 4702 if (ret < 0) {
03e98c9e
NB		 4703 transport_generic_request_failure(cmd);
4704 break;
c66ac9db
NB		 4705 }
4706 break;
4707 case TRANSPORT_PROCESS_WRITE:
4708 transport_generic_process_write(cmd);
4709 break;
c66ac9db
NB		 4710 case TRANSPORT_PROCESS_TMR:
4711 transport_generic_do_tmr(cmd);
4712 break;
07bde79a 4713 case TRANSPORT_COMPLETE_QF_WP:
e057f533
CH		 4714 transport_write_pending_qf(cmd);
4715 break;
4716 case TRANSPORT_COMPLETE_QF_OK:
4717 transport_complete_qf(cmd);
07bde79a 4718 break;
c66ac9db 4719 default:
f2da9dbd
CH		 4720 pr_err("Unknown t_state: %d for ITT: 0x%08x "
4721 "i_state: %d on SE LUN: %u\n",
4722 cmd->t_state,
e3d6f909
AG		 4723 cmd->se_tfo->get_task_tag(cmd),
4724 cmd->se_tfo->get_cmd_state(cmd),
4725 cmd->se_lun->unpacked_lun);
c66ac9db
NB		 4726 BUG();
4727 }
4728
4729 goto get_cmd;
4730 }
4731
4732out:
ce8762f6
NB		 4733 WARN_ON(!list_empty(&dev->state_task_list));
4734 WARN_ON(!list_empty(&dev->dev_queue_obj.qobj_list));
c66ac9db
NB		 4735 dev->process_thread = NULL;
4736 return 0;
4737}
Linux kernel - Deliverables
This page took 0.371495 seconds and 5 git commands to generate.