drivers/target/iscsi/iscsi_target_erl0.c

   1 /******************************************************************************
   2  * This file contains error recovery level zero functions used by
   3  * the iSCSI Target driver.
   4  *
   5  * \u00a9 Copyright 2007-2011 RisingTide Systems LLC.
   6  *
   7  * Licensed to the Linux Foundation under the General Public License (GPL) version 2.
   8  *
   9  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License as published by
  13  * the Free Software Foundation; either version 2 of the License, or
  14  * (at your option) any later version.
  15  *
  16  * This program is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  * GNU General Public License for more details.
  20  ******************************************************************************/
  21
  22 #include <scsi/iscsi_proto.h>
  23 #include <target/target_core_base.h>
  24 #include <target/target_core_fabric.h>
  25
  26 #include "iscsi_target_core.h"
  27 #include "iscsi_target_seq_pdu_list.h"
  28 #include "iscsi_target_tq.h"
  29 #include "iscsi_target_erl0.h"
  30 #include "iscsi_target_erl1.h"
  31 #include "iscsi_target_erl2.h"
  32 #include "iscsi_target_util.h"
  33 #include "iscsi_target.h"
  34
  35 /*
  36  *      Used to set values in struct iscsi_cmd that iscsit_dataout_check_sequence()
  37  *      checks against to determine a PDU's Offset+Length is within the current
  38  *      DataOUT Sequence.  Used for DataSequenceInOrder=Yes only.
  39  */
  40 void iscsit_set_dataout_sequence_values(
  41         struct iscsi_cmd *cmd)
  42 {
  43         struct iscsi_conn *conn = cmd->conn;
  44         /*
  45          * Still set seq_start_offset and seq_end_offset for Unsolicited
  46          * DataOUT, even if DataSequenceInOrder=No.
  47          */
  48         if (cmd->unsolicited_data) {
  49                 cmd->seq_start_offset = cmd->write_data_done;
  50                 cmd->seq_end_offset = (cmd->write_data_done +
  51                         (cmd->se_cmd.data_length >
  52                          conn->sess->sess_ops->FirstBurstLength) ?
  53                         conn->sess->sess_ops->FirstBurstLength : cmd->se_cmd.data_length);
  54                 return;
  55         }
  56
  57         if (!conn->sess->sess_ops->DataSequenceInOrder)
  58                 return;
  59
  60         if (!cmd->seq_start_offset && !cmd->seq_end_offset) {
  61                 cmd->seq_start_offset = cmd->write_data_done;
  62                 cmd->seq_end_offset = (cmd->se_cmd.data_length >
  63                         conn->sess->sess_ops->MaxBurstLength) ?
  64                         (cmd->write_data_done +
  65                         conn->sess->sess_ops->MaxBurstLength) : cmd->se_cmd.data_length;
  66         } else {
  67                 cmd->seq_start_offset = cmd->seq_end_offset;
  68                 cmd->seq_end_offset = ((cmd->seq_end_offset +
  69                         conn->sess->sess_ops->MaxBurstLength) >=
  70                         cmd->se_cmd.data_length) ? cmd->se_cmd.data_length :
  71                         (cmd->seq_end_offset +
  72                          conn->sess->sess_ops->MaxBurstLength);
  73         }
  74 }
  75
  76 static int iscsit_dataout_within_command_recovery_check(
  77         struct iscsi_cmd *cmd,
  78         unsigned char *buf)
  79 {
  80         struct iscsi_conn *conn = cmd->conn;
  81         struct iscsi_data *hdr = (struct iscsi_data *) buf;
  82         u32 payload_length = ntoh24(hdr->dlength);
  83
  84         /*
  85          * We do the within-command recovery checks here as it is
  86          * the first function called in iscsi_check_pre_dataout().
  87          * Basically, if we are in within-command recovery and
  88          * the PDU does not contain the offset the sequence needs,
  89          * dump the payload.
  90          *
  91          * This only applies to DataPDUInOrder=Yes, for
  92          * DataPDUInOrder=No we only re-request the failed PDU
  93          * and check that all PDUs in a sequence are received
  94          * upon end of sequence.
  95          */
  96         if (conn->sess->sess_ops->DataSequenceInOrder) {
  97                 if ((cmd->cmd_flags & ICF_WITHIN_COMMAND_RECOVERY) &&
  98                     (cmd->write_data_done != hdr->offset))
  99                         goto dump;
 100
 101                 cmd->cmd_flags &= ~ICF_WITHIN_COMMAND_RECOVERY;
 102         } else {
 103                 struct iscsi_seq *seq;
 104
 105                 seq = iscsit_get_seq_holder(cmd, hdr->offset, payload_length);
 106                 if (!seq)
 107                         return DATAOUT_CANNOT_RECOVER;
 108                 /*
 109                  * Set the struct iscsi_seq pointer to reuse later.
 110                  */
 111                 cmd->seq_ptr = seq;
 112
 113                 if (conn->sess->sess_ops->DataPDUInOrder) {
 114                         if ((seq->status ==
 115                              DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY) &&
 116                            ((seq->offset != hdr->offset) ||
 117                             (seq->data_sn != hdr->datasn)))
 118                                 goto dump;
 119                 } else {
 120                         if ((seq->status ==
 121                              DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY) &&
 122                             (seq->data_sn != hdr->datasn))
 123                                 goto dump;
 124                 }
 125
 126                 if (seq->status == DATAOUT_SEQUENCE_COMPLETE)
 127                         goto dump;
 128
 129                 if (seq->status != DATAOUT_SEQUENCE_COMPLETE)
 130                         seq->status = 0;
 131         }
 132
 133         return DATAOUT_NORMAL;
 134
 135 dump:
 136         pr_err("Dumping DataOUT PDU Offset: %u Length: %d DataSN:"
 137                 " 0x%08x\n", hdr->offset, payload_length, hdr->datasn);
 138         return iscsit_dump_data_payload(conn, payload_length, 1);
 139 }
 140
 141 static int iscsit_dataout_check_unsolicited_sequence(
 142         struct iscsi_cmd *cmd,
 143         unsigned char *buf)
 144 {
 145         u32 first_burst_len;
 146         struct iscsi_conn *conn = cmd->conn;
 147         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 148         u32 payload_length = ntoh24(hdr->dlength);
 149
 150
 151         if ((hdr->offset < cmd->seq_start_offset) ||
 152            ((hdr->offset + payload_length) > cmd->seq_end_offset)) {
 153                 pr_err("Command ITT: 0x%08x with Offset: %u,"
 154                 " Length: %u outside of Unsolicited Sequence %u:%u while"
 155                 " DataSequenceInOrder=Yes.\n", cmd->init_task_tag,
 156                 hdr->offset, payload_length, cmd->seq_start_offset,
 157                         cmd->seq_end_offset);
 158                 return DATAOUT_CANNOT_RECOVER;
 159         }
 160
 161         first_burst_len = (cmd->first_burst_len + payload_length);
 162
 163         if (first_burst_len > conn->sess->sess_ops->FirstBurstLength) {
 164                 pr_err("Total %u bytes exceeds FirstBurstLength: %u"
 165                         " for this Unsolicited DataOut Burst.\n",
 166                         first_burst_len, conn->sess->sess_ops->FirstBurstLength);
 167                 transport_send_check_condition_and_sense(&cmd->se_cmd,
 168                                 TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 169                 return DATAOUT_CANNOT_RECOVER;
 170         }
 171
 172         /*
 173          * Perform various MaxBurstLength and ISCSI_FLAG_CMD_FINAL sanity
 174          * checks for the current Unsolicited DataOUT Sequence.
 175          */
 176         if (hdr->flags & ISCSI_FLAG_CMD_FINAL) {
 177                 /*
 178                  * Ignore ISCSI_FLAG_CMD_FINAL checks while DataPDUInOrder=No, end of
 179                  * sequence checks are handled in
 180                  * iscsit_dataout_datapduinorder_no_fbit().
 181                  */
 182                 if (!conn->sess->sess_ops->DataPDUInOrder)
 183                         goto out;
 184
 185                 if ((first_burst_len != cmd->se_cmd.data_length) &&
 186                     (first_burst_len != conn->sess->sess_ops->FirstBurstLength)) {
 187                         pr_err("Unsolicited non-immediate data"
 188                         " received %u does not equal FirstBurstLength: %u, and"
 189                         " does not equal ExpXferLen %u.\n", first_burst_len,
 190                                 conn->sess->sess_ops->FirstBurstLength,
 191                                 cmd->se_cmd.data_length);
 192                         transport_send_check_condition_and_sense(&cmd->se_cmd,
 193                                         TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 194                         return DATAOUT_CANNOT_RECOVER;
 195                 }
 196         } else {
 197                 if (first_burst_len == conn->sess->sess_ops->FirstBurstLength) {
 198                         pr_err("Command ITT: 0x%08x reached"
 199                         " FirstBurstLength: %u, but ISCSI_FLAG_CMD_FINAL is not set. protocol"
 200                                 " error.\n", cmd->init_task_tag,
 201                                 conn->sess->sess_ops->FirstBurstLength);
 202                         return DATAOUT_CANNOT_RECOVER;
 203                 }
 204                 if (first_burst_len == cmd->se_cmd.data_length) {
 205                         pr_err("Command ITT: 0x%08x reached"
 206                         " ExpXferLen: %u, but ISCSI_FLAG_CMD_FINAL is not set. protocol"
 207                         " error.\n", cmd->init_task_tag, cmd->se_cmd.data_length);
 208                         return DATAOUT_CANNOT_RECOVER;
 209                 }
 210         }
 211
 212 out:
 213         return DATAOUT_NORMAL;
 214 }
 215
 216 static int iscsit_dataout_check_sequence(
 217         struct iscsi_cmd *cmd,
 218         unsigned char *buf)
 219 {
 220         u32 next_burst_len;
 221         struct iscsi_conn *conn = cmd->conn;
 222         struct iscsi_seq *seq = NULL;
 223         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 224         u32 payload_length = ntoh24(hdr->dlength);
 225
 226         /*
 227          * For DataSequenceInOrder=Yes: Check that the offset and offset+length
 228          * is within range as defined by iscsi_set_dataout_sequence_values().
 229          *
 230          * For DataSequenceInOrder=No: Check that an struct iscsi_seq exists for
 231          * offset+length tuple.
 232          */
 233         if (conn->sess->sess_ops->DataSequenceInOrder) {
 234                 /*
 235                  * Due to possibility of recovery DataOUT sent by the initiator
 236                  * fullfilling an Recovery R2T, it's best to just dump the
 237                  * payload here, instead of erroring out.
 238                  */
 239                 if ((hdr->offset < cmd->seq_start_offset) ||
 240                    ((hdr->offset + payload_length) > cmd->seq_end_offset)) {
 241                         pr_err("Command ITT: 0x%08x with Offset: %u,"
 242                         " Length: %u outside of Sequence %u:%u while"
 243                         " DataSequenceInOrder=Yes.\n", cmd->init_task_tag,
 244                         hdr->offset, payload_length, cmd->seq_start_offset,
 245                                 cmd->seq_end_offset);
 246
 247                         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 248                                 return DATAOUT_CANNOT_RECOVER;
 249                         return DATAOUT_WITHIN_COMMAND_RECOVERY;
 250                 }
 251
 252                 next_burst_len = (cmd->next_burst_len + payload_length);
 253         } else {
 254                 seq = iscsit_get_seq_holder(cmd, hdr->offset, payload_length);
 255                 if (!seq)
 256                         return DATAOUT_CANNOT_RECOVER;
 257                 /*
 258                  * Set the struct iscsi_seq pointer to reuse later.
 259                  */
 260                 cmd->seq_ptr = seq;
 261
 262                 if (seq->status == DATAOUT_SEQUENCE_COMPLETE) {
 263                         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 264                                 return DATAOUT_CANNOT_RECOVER;
 265                         return DATAOUT_WITHIN_COMMAND_RECOVERY;
 266                 }
 267
 268                 next_burst_len = (seq->next_burst_len + payload_length);
 269         }
 270
 271         if (next_burst_len > conn->sess->sess_ops->MaxBurstLength) {
 272                 pr_err("Command ITT: 0x%08x, NextBurstLength: %u and"
 273                         " Length: %u exceeds MaxBurstLength: %u. protocol"
 274                         " error.\n", cmd->init_task_tag,
 275                         (next_burst_len - payload_length),
 276                         payload_length, conn->sess->sess_ops->MaxBurstLength);
 277                 return DATAOUT_CANNOT_RECOVER;
 278         }
 279
 280         /*
 281          * Perform various MaxBurstLength and ISCSI_FLAG_CMD_FINAL sanity
 282          * checks for the current DataOUT Sequence.
 283          */
 284         if (hdr->flags & ISCSI_FLAG_CMD_FINAL) {
 285                 /*
 286                  * Ignore ISCSI_FLAG_CMD_FINAL checks while DataPDUInOrder=No, end of
 287                  * sequence checks are handled in
 288                  * iscsit_dataout_datapduinorder_no_fbit().
 289                  */
 290                 if (!conn->sess->sess_ops->DataPDUInOrder)
 291                         goto out;
 292
 293                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 294                         if ((next_burst_len <
 295                              conn->sess->sess_ops->MaxBurstLength) &&
 296                            ((cmd->write_data_done + payload_length) <
 297                              cmd->se_cmd.data_length)) {
 298                                 pr_err("Command ITT: 0x%08x set ISCSI_FLAG_CMD_FINAL"
 299                                 " before end of DataOUT sequence, protocol"
 300                                 " error.\n", cmd->init_task_tag);
 301                                 return DATAOUT_CANNOT_RECOVER;
 302                         }
 303                 } else {
 304                         if (next_burst_len < seq->xfer_len) {
 305                                 pr_err("Command ITT: 0x%08x set ISCSI_FLAG_CMD_FINAL"
 306                                 " before end of DataOUT sequence, protocol"
 307                                 " error.\n", cmd->init_task_tag);
 308                                 return DATAOUT_CANNOT_RECOVER;
 309                         }
 310                 }
 311         } else {
 312                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 313                         if (next_burst_len ==
 314                                         conn->sess->sess_ops->MaxBurstLength) {
 315                                 pr_err("Command ITT: 0x%08x reached"
 316                                 " MaxBurstLength: %u, but ISCSI_FLAG_CMD_FINAL is"
 317                                 " not set, protocol error.", cmd->init_task_tag,
 318                                         conn->sess->sess_ops->MaxBurstLength);
 319                                 return DATAOUT_CANNOT_RECOVER;
 320                         }
 321                         if ((cmd->write_data_done + payload_length) ==
 322                                         cmd->se_cmd.data_length) {
 323                                 pr_err("Command ITT: 0x%08x reached"
 324                                 " last DataOUT PDU in sequence but ISCSI_FLAG_"
 325                                 "CMD_FINAL is not set, protocol error.\n",
 326                                         cmd->init_task_tag);
 327                                 return DATAOUT_CANNOT_RECOVER;
 328                         }
 329                 } else {
 330                         if (next_burst_len == seq->xfer_len) {
 331                                 pr_err("Command ITT: 0x%08x reached"
 332                                 " last DataOUT PDU in sequence but ISCSI_FLAG_"
 333                                 "CMD_FINAL is not set, protocol error.\n",
 334                                         cmd->init_task_tag);
 335                                 return DATAOUT_CANNOT_RECOVER;
 336                         }
 337                 }
 338         }
 339
 340 out:
 341         return DATAOUT_NORMAL;
 342 }
 343
 344 static int iscsit_dataout_check_datasn(
 345         struct iscsi_cmd *cmd,
 346         unsigned char *buf)
 347 {
 348         int dump = 0, recovery = 0;
 349         u32 data_sn = 0;
 350         struct iscsi_conn *conn = cmd->conn;
 351         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 352         u32 payload_length = ntoh24(hdr->dlength);
 353
 354         /*
 355          * Considering the target has no method of re-requesting DataOUT
 356          * by DataSN, if we receieve a greater DataSN than expected we
 357          * assume the functions for DataPDUInOrder=[Yes,No] below will
 358          * handle it.
 359          *
 360          * If the DataSN is less than expected, dump the payload.
 361          */
 362         if (conn->sess->sess_ops->DataSequenceInOrder)
 363                 data_sn = cmd->data_sn;
 364         else {
 365                 struct iscsi_seq *seq = cmd->seq_ptr;
 366                 data_sn = seq->data_sn;
 367         }
 368
 369         if (hdr->datasn > data_sn) {
 370                 pr_err("Command ITT: 0x%08x, received DataSN: 0x%08x"
 371                         " higher than expected 0x%08x.\n", cmd->init_task_tag,
 372                                 hdr->datasn, data_sn);
 373                 recovery = 1;
 374                 goto recover;
 375         } else if (hdr->datasn < data_sn) {
 376                 pr_err("Command ITT: 0x%08x, received DataSN: 0x%08x"
 377                         " lower than expected 0x%08x, discarding payload.\n",
 378                         cmd->init_task_tag, hdr->datasn, data_sn);
 379                 dump = 1;
 380                 goto dump;
 381         }
 382
 383         return DATAOUT_NORMAL;
 384
 385 recover:
 386         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 387                 pr_err("Unable to perform within-command recovery"
 388                                 " while ERL=0.\n");
 389                 return DATAOUT_CANNOT_RECOVER;
 390         }
 391 dump:
 392         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 393                 return DATAOUT_CANNOT_RECOVER;
 394
 395         return (recovery || dump) ? DATAOUT_WITHIN_COMMAND_RECOVERY :
 396                                 DATAOUT_NORMAL;
 397 }
 398
 399 static int iscsit_dataout_pre_datapduinorder_yes(
 400         struct iscsi_cmd *cmd,
 401         unsigned char *buf)
 402 {
 403         int dump = 0, recovery = 0;
 404         struct iscsi_conn *conn = cmd->conn;
 405         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 406         u32 payload_length = ntoh24(hdr->dlength);
 407
 408         /*
 409          * For DataSequenceInOrder=Yes: If the offset is greater than the global
 410          * DataPDUInOrder=Yes offset counter in struct iscsi_cmd a protcol error has
 411          * occured and fail the connection.
 412          *
 413          * For DataSequenceInOrder=No: If the offset is greater than the per
 414          * sequence DataPDUInOrder=Yes offset counter in struct iscsi_seq a protocol
 415          * error has occured and fail the connection.
 416          */
 417         if (conn->sess->sess_ops->DataSequenceInOrder) {
 418                 if (hdr->offset != cmd->write_data_done) {
 419                         pr_err("Command ITT: 0x%08x, received offset"
 420                         " %u different than expected %u.\n", cmd->init_task_tag,
 421                                 hdr->offset, cmd->write_data_done);
 422                         recovery = 1;
 423                         goto recover;
 424                 }
 425         } else {
 426                 struct iscsi_seq *seq = cmd->seq_ptr;
 427
 428                 if (hdr->offset > seq->offset) {
 429                         pr_err("Command ITT: 0x%08x, received offset"
 430                         " %u greater than expected %u.\n", cmd->init_task_tag,
 431                                 hdr->offset, seq->offset);
 432                         recovery = 1;
 433                         goto recover;
 434                 } else if (hdr->offset < seq->offset) {
 435                         pr_err("Command ITT: 0x%08x, received offset"
 436                         " %u less than expected %u, discarding payload.\n",
 437                                 cmd->init_task_tag, hdr->offset, seq->offset);
 438                         dump = 1;
 439                         goto dump;
 440                 }
 441         }
 442
 443         return DATAOUT_NORMAL;
 444
 445 recover:
 446         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 447                 pr_err("Unable to perform within-command recovery"
 448                                 " while ERL=0.\n");
 449                 return DATAOUT_CANNOT_RECOVER;
 450         }
 451 dump:
 452         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 453                 return DATAOUT_CANNOT_RECOVER;
 454
 455         return (recovery) ? iscsit_recover_dataout_sequence(cmd,
 456                 hdr->offset, payload_length) :
 457                (dump) ? DATAOUT_WITHIN_COMMAND_RECOVERY : DATAOUT_NORMAL;
 458 }
 459
 460 static int iscsit_dataout_pre_datapduinorder_no(
 461         struct iscsi_cmd *cmd,
 462         unsigned char *buf)
 463 {
 464         struct iscsi_pdu *pdu;
 465         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 466         u32 payload_length = ntoh24(hdr->dlength);
 467
 468         pdu = iscsit_get_pdu_holder(cmd, hdr->offset, payload_length);
 469         if (!pdu)
 470                 return DATAOUT_CANNOT_RECOVER;
 471
 472         cmd->pdu_ptr = pdu;
 473
 474         switch (pdu->status) {
 475         case ISCSI_PDU_NOT_RECEIVED:
 476         case ISCSI_PDU_CRC_FAILED:
 477         case ISCSI_PDU_TIMED_OUT:
 478                 break;
 479         case ISCSI_PDU_RECEIVED_OK:
 480                 pr_err("Command ITT: 0x%08x received already gotten"
 481                         " Offset: %u, Length: %u\n", cmd->init_task_tag,
 482                                 hdr->offset, payload_length);
 483                 return iscsit_dump_data_payload(cmd->conn, payload_length, 1);
 484         default:
 485                 return DATAOUT_CANNOT_RECOVER;
 486         }
 487
 488         return DATAOUT_NORMAL;
 489 }
 490
 491 static int iscsit_dataout_update_r2t(struct iscsi_cmd *cmd, u32 offset, u32 length)
 492 {
 493         struct iscsi_r2t *r2t;
 494
 495         if (cmd->unsolicited_data)
 496                 return 0;
 497
 498         r2t = iscsit_get_r2t_for_eos(cmd, offset, length);
 499         if (!r2t)
 500                 return -1;
 501
 502         spin_lock_bh(&cmd->r2t_lock);
 503         r2t->seq_complete = 1;
 504         cmd->outstanding_r2ts--;
 505         spin_unlock_bh(&cmd->r2t_lock);
 506
 507         return 0;
 508 }
 509
 510 static int iscsit_dataout_update_datapduinorder_no(
 511         struct iscsi_cmd *cmd,
 512         u32 data_sn,
 513         int f_bit)
 514 {
 515         int ret = 0;
 516         struct iscsi_pdu *pdu = cmd->pdu_ptr;
 517
 518         pdu->data_sn = data_sn;
 519
 520         switch (pdu->status) {
 521         case ISCSI_PDU_NOT_RECEIVED:
 522                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 523                 break;
 524         case ISCSI_PDU_CRC_FAILED:
 525                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 526                 break;
 527         case ISCSI_PDU_TIMED_OUT:
 528                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 529                 break;
 530         default:
 531                 return DATAOUT_CANNOT_RECOVER;
 532         }
 533
 534         if (f_bit) {
 535                 ret = iscsit_dataout_datapduinorder_no_fbit(cmd, pdu);
 536                 if (ret == DATAOUT_CANNOT_RECOVER)
 537                         return ret;
 538         }
 539
 540         return DATAOUT_NORMAL;
 541 }
 542
 543 static int iscsit_dataout_post_crc_passed(
 544         struct iscsi_cmd *cmd,
 545         unsigned char *buf)
 546 {
 547         int ret, send_r2t = 0;
 548         struct iscsi_conn *conn = cmd->conn;
 549         struct iscsi_seq *seq = NULL;
 550         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 551         u32 payload_length = ntoh24(hdr->dlength);
 552
 553         if (cmd->unsolicited_data) {
 554                 if ((cmd->first_burst_len + payload_length) ==
 555                      conn->sess->sess_ops->FirstBurstLength) {
 556                         if (iscsit_dataout_update_r2t(cmd, hdr->offset,
 557                                         payload_length) < 0)
 558                                 return DATAOUT_CANNOT_RECOVER;
 559                         send_r2t = 1;
 560                 }
 561
 562                 if (!conn->sess->sess_ops->DataPDUInOrder) {
 563                         ret = iscsit_dataout_update_datapduinorder_no(cmd,
 564                                 hdr->datasn, (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 565                         if (ret == DATAOUT_CANNOT_RECOVER)
 566                                 return ret;
 567                 }
 568
 569                 cmd->first_burst_len += payload_length;
 570
 571                 if (conn->sess->sess_ops->DataSequenceInOrder)
 572                         cmd->data_sn++;
 573                 else {
 574                         seq = cmd->seq_ptr;
 575                         seq->data_sn++;
 576                         seq->offset += payload_length;
 577                 }
 578
 579                 if (send_r2t) {
 580                         if (seq)
 581                                 seq->status = DATAOUT_SEQUENCE_COMPLETE;
 582                         cmd->first_burst_len = 0;
 583                         cmd->unsolicited_data = 0;
 584                 }
 585         } else {
 586                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 587                         if ((cmd->next_burst_len + payload_length) ==
 588                              conn->sess->sess_ops->MaxBurstLength) {
 589                                 if (iscsit_dataout_update_r2t(cmd, hdr->offset,
 590                                                 payload_length) < 0)
 591                                         return DATAOUT_CANNOT_RECOVER;
 592                                 send_r2t = 1;
 593                         }
 594
 595                         if (!conn->sess->sess_ops->DataPDUInOrder) {
 596                                 ret = iscsit_dataout_update_datapduinorder_no(
 597                                                 cmd, hdr->datasn,
 598                                                 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 599                                 if (ret == DATAOUT_CANNOT_RECOVER)
 600                                         return ret;
 601                         }
 602
 603                         cmd->next_burst_len += payload_length;
 604                         cmd->data_sn++;
 605
 606                         if (send_r2t)
 607                                 cmd->next_burst_len = 0;
 608                 } else {
 609                         seq = cmd->seq_ptr;
 610
 611                         if ((seq->next_burst_len + payload_length) ==
 612                              seq->xfer_len) {
 613                                 if (iscsit_dataout_update_r2t(cmd, hdr->offset,
 614                                                 payload_length) < 0)
 615                                         return DATAOUT_CANNOT_RECOVER;
 616                                 send_r2t = 1;
 617                         }
 618
 619                         if (!conn->sess->sess_ops->DataPDUInOrder) {
 620                                 ret = iscsit_dataout_update_datapduinorder_no(
 621                                                 cmd, hdr->datasn,
 622                                                 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 623                                 if (ret == DATAOUT_CANNOT_RECOVER)
 624                                         return ret;
 625                         }
 626
 627                         seq->data_sn++;
 628                         seq->offset += payload_length;
 629                         seq->next_burst_len += payload_length;
 630
 631                         if (send_r2t) {
 632                                 seq->next_burst_len = 0;
 633                                 seq->status = DATAOUT_SEQUENCE_COMPLETE;
 634                         }
 635                 }
 636         }
 637
 638         if (send_r2t && conn->sess->sess_ops->DataSequenceInOrder)
 639                 cmd->data_sn = 0;
 640
 641         cmd->write_data_done += payload_length;
 642
 643         if (cmd->write_data_done == cmd->se_cmd.data_length)
 644                 return DATAOUT_SEND_TO_TRANSPORT;
 645         else if (send_r2t)
 646                 return DATAOUT_SEND_R2T;
 647         else
 648                 return DATAOUT_NORMAL;
 649 }
 650
 651 static int iscsit_dataout_post_crc_failed(
 652         struct iscsi_cmd *cmd,
 653         unsigned char *buf)
 654 {
 655         struct iscsi_conn *conn = cmd->conn;
 656         struct iscsi_pdu *pdu;
 657         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 658         u32 payload_length = ntoh24(hdr->dlength);
 659
 660         if (conn->sess->sess_ops->DataPDUInOrder)
 661                 goto recover;
 662         /*
 663          * The rest of this function is only called when DataPDUInOrder=No.
 664          */
 665         pdu = cmd->pdu_ptr;
 666
 667         switch (pdu->status) {
 668         case ISCSI_PDU_NOT_RECEIVED:
 669                 pdu->status = ISCSI_PDU_CRC_FAILED;
 670                 break;
 671         case ISCSI_PDU_CRC_FAILED:
 672                 break;
 673         case ISCSI_PDU_TIMED_OUT:
 674                 pdu->status = ISCSI_PDU_CRC_FAILED;
 675                 break;
 676         default:
 677                 return DATAOUT_CANNOT_RECOVER;
 678         }
 679
 680 recover:
 681         return iscsit_recover_dataout_sequence(cmd, hdr->offset, payload_length);
 682 }
 683
 684 /*
 685  *      Called from iscsit_handle_data_out() before DataOUT Payload is received
 686  *      and CRC computed.
 687  */
 688 extern int iscsit_check_pre_dataout(
 689         struct iscsi_cmd *cmd,
 690         unsigned char *buf)
 691 {
 692         int ret;
 693         struct iscsi_conn *conn = cmd->conn;
 694
 695         ret = iscsit_dataout_within_command_recovery_check(cmd, buf);
 696         if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 697             (ret == DATAOUT_CANNOT_RECOVER))
 698                 return ret;
 699
 700         ret = iscsit_dataout_check_datasn(cmd, buf);
 701         if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 702             (ret == DATAOUT_CANNOT_RECOVER))
 703                 return ret;
 704
 705         if (cmd->unsolicited_data) {
 706                 ret = iscsit_dataout_check_unsolicited_sequence(cmd, buf);
 707                 if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 708                     (ret == DATAOUT_CANNOT_RECOVER))
 709                         return ret;
 710         } else {
 711                 ret = iscsit_dataout_check_sequence(cmd, buf);
 712                 if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 713                     (ret == DATAOUT_CANNOT_RECOVER))
 714                         return ret;
 715         }
 716
 717         return (conn->sess->sess_ops->DataPDUInOrder) ?
 718                 iscsit_dataout_pre_datapduinorder_yes(cmd, buf) :
 719                 iscsit_dataout_pre_datapduinorder_no(cmd, buf);
 720 }
 721
 722 /*
 723  *      Called from iscsit_handle_data_out() after DataOUT Payload is received
 724  *      and CRC computed.
 725  */
 726 int iscsit_check_post_dataout(
 727         struct iscsi_cmd *cmd,
 728         unsigned char *buf,
 729         u8 data_crc_failed)
 730 {
 731         struct iscsi_conn *conn = cmd->conn;
 732
 733         cmd->dataout_timeout_retries = 0;
 734
 735         if (!data_crc_failed)
 736                 return iscsit_dataout_post_crc_passed(cmd, buf);
 737         else {
 738                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 739                         pr_err("Unable to recover from DataOUT CRC"
 740                                 " failure while ERL=0, closing session.\n");
 741                         iscsit_add_reject_from_cmd(ISCSI_REASON_DATA_DIGEST_ERROR,
 742                                         1, 0, buf, cmd);
 743                         return DATAOUT_CANNOT_RECOVER;
 744                 }
 745
 746                 iscsit_add_reject_from_cmd(ISCSI_REASON_DATA_DIGEST_ERROR,
 747                                 0, 0, buf, cmd);
 748                 return iscsit_dataout_post_crc_failed(cmd, buf);
 749         }
 750 }
 751
 752 static void iscsit_handle_time2retain_timeout(unsigned long data)
 753 {
 754         struct iscsi_session *sess = (struct iscsi_session *) data;
 755         struct iscsi_portal_group *tpg = ISCSI_TPG_S(sess);
 756         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
 757
 758         spin_lock_bh(&se_tpg->session_lock);
 759         if (sess->time2retain_timer_flags & ISCSI_TF_STOP) {
 760                 spin_unlock_bh(&se_tpg->session_lock);
 761                 return;
 762         }
 763         if (atomic_read(&sess->session_reinstatement)) {
 764                 pr_err("Exiting Time2Retain handler because"
 765                                 " session_reinstatement=1\n");
 766                 spin_unlock_bh(&se_tpg->session_lock);
 767                 return;
 768         }
 769         sess->time2retain_timer_flags |= ISCSI_TF_EXPIRED;
 770
 771         pr_err("Time2Retain timer expired for SID: %u, cleaning up"
 772                         " iSCSI session.\n", sess->sid);
 773         {
 774         struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
 775
 776         if (tiqn) {
 777                 spin_lock(&tiqn->sess_err_stats.lock);
 778                 strcpy(tiqn->sess_err_stats.last_sess_fail_rem_name,
 779                         (void *)sess->sess_ops->InitiatorName);
 780                 tiqn->sess_err_stats.last_sess_failure_type =
 781                                 ISCSI_SESS_ERR_CXN_TIMEOUT;
 782                 tiqn->sess_err_stats.cxn_timeout_errors++;
 783                 sess->conn_timeout_errors++;
 784                 spin_unlock(&tiqn->sess_err_stats.lock);
 785         }
 786         }
 787
 788         spin_unlock_bh(&se_tpg->session_lock);
 789         target_put_session(sess->se_sess);
 790 }
 791
 792 extern void iscsit_start_time2retain_handler(struct iscsi_session *sess)
 793 {
 794         int tpg_active;
 795         /*
 796          * Only start Time2Retain timer when the assoicated TPG is still in
 797          * an ACTIVE (eg: not disabled or shutdown) state.
 798          */
 799         spin_lock(&ISCSI_TPG_S(sess)->tpg_state_lock);
 800         tpg_active = (ISCSI_TPG_S(sess)->tpg_state == TPG_STATE_ACTIVE);
 801         spin_unlock(&ISCSI_TPG_S(sess)->tpg_state_lock);
 802
 803         if (!tpg_active)
 804                 return;
 805
 806         if (sess->time2retain_timer_flags & ISCSI_TF_RUNNING)
 807                 return;
 808
 809         pr_debug("Starting Time2Retain timer for %u seconds on"
 810                 " SID: %u\n", sess->sess_ops->DefaultTime2Retain, sess->sid);
 811
 812         init_timer(&sess->time2retain_timer);
 813         sess->time2retain_timer.expires =
 814                 (get_jiffies_64() + sess->sess_ops->DefaultTime2Retain * HZ);
 815         sess->time2retain_timer.data = (unsigned long)sess;
 816         sess->time2retain_timer.function = iscsit_handle_time2retain_timeout;
 817         sess->time2retain_timer_flags &= ~ISCSI_TF_STOP;
 818         sess->time2retain_timer_flags |= ISCSI_TF_RUNNING;
 819         add_timer(&sess->time2retain_timer);
 820 }
 821
 822 /*
 823  *      Called with spin_lock_bh(&struct se_portal_group->session_lock) held
 824  */
 825 extern int iscsit_stop_time2retain_timer(struct iscsi_session *sess)
 826 {
 827         struct iscsi_portal_group *tpg = ISCSI_TPG_S(sess);
 828         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
 829
 830         if (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)
 831                 return -1;
 832
 833         if (!(sess->time2retain_timer_flags & ISCSI_TF_RUNNING))
 834                 return 0;
 835
 836         sess->time2retain_timer_flags |= ISCSI_TF_STOP;
 837         spin_unlock_bh(&se_tpg->session_lock);
 838
 839         del_timer_sync(&sess->time2retain_timer);
 840
 841         spin_lock_bh(&se_tpg->session_lock);
 842         sess->time2retain_timer_flags &= ~ISCSI_TF_RUNNING;
 843         pr_debug("Stopped Time2Retain Timer for SID: %u\n",
 844                         sess->sid);
 845         return 0;
 846 }
 847
 848 void iscsit_connection_reinstatement_rcfr(struct iscsi_conn *conn)
 849 {
 850         spin_lock_bh(&conn->state_lock);
 851         if (atomic_read(&conn->connection_exit)) {
 852                 spin_unlock_bh(&conn->state_lock);
 853                 goto sleep;
 854         }
 855
 856         if (atomic_read(&conn->transport_failed)) {
 857                 spin_unlock_bh(&conn->state_lock);
 858                 goto sleep;
 859         }
 860         spin_unlock_bh(&conn->state_lock);
 861
 862         iscsi_thread_set_force_reinstatement(conn);
 863
 864 sleep:
 865         wait_for_completion(&conn->conn_wait_rcfr_comp);
 866         complete(&conn->conn_post_wait_comp);
 867 }
 868
 869 void iscsit_cause_connection_reinstatement(struct iscsi_conn *conn, int sleep)
 870 {
 871         spin_lock_bh(&conn->state_lock);
 872         if (atomic_read(&conn->connection_exit)) {
 873                 spin_unlock_bh(&conn->state_lock);
 874                 return;
 875         }
 876
 877         if (atomic_read(&conn->transport_failed)) {
 878                 spin_unlock_bh(&conn->state_lock);
 879                 return;
 880         }
 881
 882         if (atomic_read(&conn->connection_reinstatement)) {
 883                 spin_unlock_bh(&conn->state_lock);
 884                 return;
 885         }
 886
 887         if (iscsi_thread_set_force_reinstatement(conn) < 0) {
 888                 spin_unlock_bh(&conn->state_lock);
 889                 return;
 890         }
 891
 892         atomic_set(&conn->connection_reinstatement, 1);
 893         if (!sleep) {
 894                 spin_unlock_bh(&conn->state_lock);
 895                 return;
 896         }
 897
 898         atomic_set(&conn->sleep_on_conn_wait_comp, 1);
 899         spin_unlock_bh(&conn->state_lock);
 900
 901         wait_for_completion(&conn->conn_wait_comp);
 902         complete(&conn->conn_post_wait_comp);
 903 }
 904
 905 void iscsit_fall_back_to_erl0(struct iscsi_session *sess)
 906 {
 907         pr_debug("Falling back to ErrorRecoveryLevel=0 for SID:"
 908                         " %u\n", sess->sid);
 909
 910         atomic_set(&sess->session_fall_back_to_erl0, 1);
 911 }
 912
 913 static void iscsit_handle_connection_cleanup(struct iscsi_conn *conn)
 914 {
 915         struct iscsi_session *sess = conn->sess;
 916
 917         if ((sess->sess_ops->ErrorRecoveryLevel == 2) &&
 918             !atomic_read(&sess->session_reinstatement) &&
 919             !atomic_read(&sess->session_fall_back_to_erl0))
 920                 iscsit_connection_recovery_transport_reset(conn);
 921         else {
 922                 pr_debug("Performing cleanup for failed iSCSI"
 923                         " Connection ID: %hu from %s\n", conn->cid,
 924                         sess->sess_ops->InitiatorName);
 925                 iscsit_close_connection(conn);
 926         }
 927 }
 928
 929 extern void iscsit_take_action_for_connection_exit(struct iscsi_conn *conn)
 930 {
 931         spin_lock_bh(&conn->state_lock);
 932         if (atomic_read(&conn->connection_exit)) {
 933                 spin_unlock_bh(&conn->state_lock);
 934                 return;
 935         }
 936         atomic_set(&conn->connection_exit, 1);
 937
 938         if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) {
 939                 spin_unlock_bh(&conn->state_lock);
 940                 iscsit_close_connection(conn);
 941                 return;
 942         }
 943
 944         if (conn->conn_state == TARG_CONN_STATE_CLEANUP_WAIT) {
 945                 spin_unlock_bh(&conn->state_lock);
 946                 return;
 947         }
 948
 949         pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n");
 950         conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT;
 951         spin_unlock_bh(&conn->state_lock);
 952
 953         iscsit_handle_connection_cleanup(conn);
 954 }
 955
 956 /*
 957  *      This is the simple function that makes the magic of
 958  *      sync and steering happen in the follow paradoxical order:
 959  *
 960  *      0) Receive conn->of_marker (bytes left until next OFMarker)
 961  *         bytes into an offload buffer.  When we pass the exact number
 962  *         of bytes in conn->of_marker, iscsit_dump_data_payload() and hence
 963  *         rx_data() will automatically receive the identical u32 marker
 964  *         values and store it in conn->of_marker_offset;
 965  *      1) Now conn->of_marker_offset will contain the offset to the start
 966  *         of the next iSCSI PDU.  Dump these remaining bytes into another
 967  *         offload buffer.
 968  *      2) We are done!
 969  *         Next byte in the TCP stream will contain the next iSCSI PDU!
 970  *         Cool Huh?!
 971  */
 972 int iscsit_recover_from_unknown_opcode(struct iscsi_conn *conn)
 973 {
 974         /*
 975          * Make sure the remaining bytes to next maker is a sane value.
 976          */
 977         if (conn->of_marker > (conn->conn_ops->OFMarkInt * 4)) {
 978                 pr_err("Remaining bytes to OFMarker: %u exceeds"
 979                         " OFMarkInt bytes: %u.\n", conn->of_marker,
 980                                 conn->conn_ops->OFMarkInt * 4);
 981                 return -1;
 982         }
 983
 984         pr_debug("Advancing %u bytes in TCP stream to get to the"
 985                         " next OFMarker.\n", conn->of_marker);
 986
 987         if (iscsit_dump_data_payload(conn, conn->of_marker, 0) < 0)
 988                 return -1;
 989
 990         /*
 991          * Make sure the offset marker we retrived is a valid value.
 992          */
 993         if (conn->of_marker_offset > (ISCSI_HDR_LEN + (ISCSI_CRC_LEN * 2) +
 994             conn->conn_ops->MaxRecvDataSegmentLength)) {
 995                 pr_err("OfMarker offset value: %u exceeds limit.\n",
 996                         conn->of_marker_offset);
 997                 return -1;
 998         }
 999
1000         pr_debug("Discarding %u bytes of TCP stream to get to the"
1001                         " next iSCSI Opcode.\n", conn->of_marker_offset);
1002
1003         if (iscsit_dump_data_payload(conn, conn->of_marker_offset, 0) < 0)
1004                 return -1;
1005
1006         return 0;
1007 }