[deliverable/linux.git] / drivers / infiniband / ulp / iser / iscsi_iser.h

/*
 * iSER transport for the Open iSCSI Initiator & iSER transport internals
 *
 * Copyright (C) 2004 Dmitry Yusupov
 * Copyright (C) 2004 Alex Aizman
 * Copyright (C) 2005 Mike Christie
 * based on code maintained by open-iscsi@googlegroups.com
 *
 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved.
 * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
 * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *	- Redistributions of source code must retain the above
 *	  copyright notice, this list of conditions and the following
 *	  disclaimer.
 *
 *	- Redistributions in binary form must reproduce the above
 *	  copyright notice, this list of conditions and the following
 *	  disclaimer in the documentation and/or other materials
 *	  provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
#ifndef __ISCSI_ISER_H__
#define __ISCSI_ISER_H__

#include <linux/types.h>
#include <linux/net.h>
#include <linux/printk.h>
#include <scsi/libiscsi.h>
#include <scsi/scsi_transport_iscsi.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_device.h>

#include <linux/interrupt.h>
#include <linux/wait.h>
#include <linux/sched.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/dma-mapping.h>
#include <linux/mutex.h>
#include <linux/mempool.h>
#include <linux/uio.h>

#include <linux/socket.h>
#include <linux/in.h>
#include <linux/in6.h>

#include <rdma/ib_verbs.h>
#include <rdma/ib_fmr_pool.h>
#include <rdma/rdma_cm.h>

#define DRV_NAME	"iser"
#define PFX		DRV_NAME ": "
#define DRV_VER		"1.4.1"

#define iser_dbg(fmt, arg...)				\
	do {						\
		if (iser_debug_level > 2)		\
			printk(KERN_DEBUG PFX "%s:" fmt,\
				__func__ , ## arg);	\
	} while (0)

#define iser_warn(fmt, arg...)				\
	do {						\
		if (iser_debug_level > 0)		\
			pr_warn(PFX "%s:" fmt,          \
				__func__ , ## arg);	\
	} while (0)

#define iser_info(fmt, arg...)				\
	do {						\
		if (iser_debug_level > 1)		\
			pr_info(PFX "%s:" fmt,          \
				__func__ , ## arg);	\
	} while (0)

#define iser_err(fmt, arg...)				\
	do {						\
		printk(KERN_ERR PFX "%s:" fmt,          \
		       __func__ , ## arg);		\
	} while (0)

#define SHIFT_4K	12
#define SIZE_4K	(1ULL << SHIFT_4K)
#define MASK_4K	(~(SIZE_4K-1))
					/* support up to 512KB in one RDMA */
#define ISCSI_ISER_SG_TABLESIZE         (0x80000 >> SHIFT_4K)
#define ISER_DEF_XMIT_CMDS_DEFAULT		512
#if ISCSI_DEF_XMIT_CMDS_MAX > ISER_DEF_XMIT_CMDS_DEFAULT
	#define ISER_DEF_XMIT_CMDS_MAX		ISCSI_DEF_XMIT_CMDS_MAX
#else
	#define ISER_DEF_XMIT_CMDS_MAX		ISER_DEF_XMIT_CMDS_DEFAULT
#endif
#define ISER_DEF_CMD_PER_LUN		ISER_DEF_XMIT_CMDS_MAX

/* QP settings */
/* Maximal bounds on received asynchronous PDUs */
#define ISER_MAX_RX_MISC_PDUS		4 /* NOOP_IN(2) , ASYNC_EVENT(2)   */

#define ISER_MAX_TX_MISC_PDUS		6 /* NOOP_OUT(2), TEXT(1),         *
					   * SCSI_TMFUNC(2), LOGOUT(1) */

#define ISER_QP_MAX_RECV_DTOS		(ISER_DEF_XMIT_CMDS_MAX)

#define ISER_MIN_POSTED_RX		(ISER_DEF_XMIT_CMDS_MAX >> 2)

/* the max TX (send) WR supported by the iSER QP is defined by                 *
 * max_send_wr = T * (1 + D) + C ; D is how many inflight dataouts we expect   *
 * to have at max for SCSI command. The tx posting & completion handling code  *
 * supports -EAGAIN scheme where tx is suspended till the QP has room for more *
 * send WR. D=8 comes from 64K/8K                                              */

#define ISER_INFLIGHT_DATAOUTS		8

#define ISER_QP_MAX_REQ_DTOS		(ISER_DEF_XMIT_CMDS_MAX *    \
					(1 + ISER_INFLIGHT_DATAOUTS) + \
					ISER_MAX_TX_MISC_PDUS        + \
					ISER_MAX_RX_MISC_PDUS)

/* Max registration work requests per command */
#define ISER_MAX_REG_WR_PER_CMD		5

/* For Signature we don't support DATAOUTs so no need to make room for them */
#define ISER_QP_SIG_MAX_REQ_DTOS	(ISER_DEF_XMIT_CMDS_MAX	*       \
					(1 + ISER_MAX_REG_WR_PER_CMD) + \
					ISER_MAX_TX_MISC_PDUS         + \
					ISER_MAX_RX_MISC_PDUS)

#define ISER_WC_BATCH_COUNT   16
#define ISER_SIGNAL_CMD_COUNT 32

#define ISER_VER			0x10
#define ISER_WSV			0x08
#define ISER_RSV			0x04

#define ISER_FASTREG_LI_WRID		0xffffffffffffffffULL
#define ISER_BEACON_WRID		0xfffffffffffffffeULL

struct iser_hdr {
	u8      flags;
	u8      rsvd[3];
	__be32  write_stag; /* write rkey */
	__be64  write_va;
	__be32  read_stag;  /* read rkey */
	__be64  read_va;
} __attribute__((packed));


#define ISER_ZBVA_NOT_SUPPORTED		0x80
#define ISER_SEND_W_INV_NOT_SUPPORTED	0x40

struct iser_cm_hdr {
	u8      flags;
	u8      rsvd[3];
} __packed;

/* Constant PDU lengths calculations */
#define ISER_HEADERS_LEN  (sizeof(struct iser_hdr) + sizeof(struct iscsi_hdr))

#define ISER_RECV_DATA_SEG_LEN	128
#define ISER_RX_PAYLOAD_SIZE	(ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN)
#define ISER_RX_LOGIN_SIZE	(ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN)

/* Length of an object name string */
#define ISER_OBJECT_NAME_SIZE		    64

enum iser_conn_state {
	ISER_CONN_INIT,		   /* descriptor allocd, no conn          */
	ISER_CONN_PENDING,	   /* in the process of being established */
	ISER_CONN_UP,		   /* up and running                      */
	ISER_CONN_TERMINATING,	   /* in the process of being terminated  */
	ISER_CONN_DOWN,		   /* shut down                           */
	ISER_CONN_STATES_NUM
};

enum iser_task_status {
	ISER_TASK_STATUS_INIT = 0,
	ISER_TASK_STATUS_STARTED,
	ISER_TASK_STATUS_COMPLETED
};

enum iser_data_dir {
	ISER_DIR_IN = 0,	   /* to initiator */
	ISER_DIR_OUT,		   /* from initiator */
	ISER_DIRS_NUM
};

struct iser_data_buf {
	void               *buf;      /* pointer to the sg list               */
	unsigned int       size;      /* num entries of this sg               */
	unsigned long      data_len;  /* total data len                       */
	unsigned int       dma_nents; /* returned by dma_map_sg               */
	char       	   *copy_buf; /* allocated copy buf for SGs unaligned *
	                               * for rdma which are copied            */
	struct scatterlist sg_single; /* SG-ified clone of a non SG SC or     *
				       * unaligned SG                         */
  };

/* fwd declarations */
struct iser_device;
struct iscsi_iser_task;
struct iscsi_endpoint;

struct iser_mem_reg {
	u32  lkey;
	u32  rkey;
	u64  va;
	u64  len;
	void *mem_h;
	int  is_mr;
};

struct iser_regd_buf {
	struct iser_mem_reg     reg;        /* memory registration info        */
	void                    *virt_addr;
	struct iser_device      *device;    /* device->device for dma_unmap    */
	enum dma_data_direction direction;  /* direction for dma_unmap	       */
	unsigned int            data_size;
};

enum iser_desc_type {
	ISCSI_TX_CONTROL ,
	ISCSI_TX_SCSI_COMMAND,
	ISCSI_TX_DATAOUT
};

struct iser_tx_desc {
	struct iser_hdr              iser_header;
	struct iscsi_hdr             iscsi_header;
	enum   iser_desc_type        type;
	u64		             dma_addr;
	/* sg[0] points to iser/iscsi headers, sg[1] optionally points to either
	of immediate data, unsolicited data-out or control (login,text) */
	struct ib_sge		     tx_sg[2];
	int                          num_sge;
};

#define ISER_RX_PAD_SIZE	(256 - (ISER_RX_PAYLOAD_SIZE + \
					sizeof(u64) + sizeof(struct ib_sge)))
struct iser_rx_desc {
	struct iser_hdr              iser_header;
	struct iscsi_hdr             iscsi_header;
	char		             data[ISER_RECV_DATA_SEG_LEN];
	u64		             dma_addr;
	struct ib_sge		     rx_sg;
	char		             pad[ISER_RX_PAD_SIZE];
} __attribute__((packed));

#define ISER_MAX_CQ 4

struct iser_conn;
struct ib_conn;
struct iscsi_iser_task;

/**
 * struct iser_comp - iSER completion context
 *
 * @device:     pointer to device handle
 * @cq:         completion queue
 * @wcs:        work completion array
 * @tasklet:    Tasklet handle
 * @active_qps: Number of active QPs attached
 *              to completion context
 */
struct iser_comp {
	struct iser_device      *device;
	struct ib_cq		*cq;
	struct ib_wc		 wcs[ISER_WC_BATCH_COUNT];
	struct tasklet_struct	 tasklet;
	int                      active_qps;
};

struct iser_device {
	struct ib_device             *ib_device;
	struct ib_pd	             *pd;
	struct ib_device_attr	     dev_attr;
	struct ib_mr	             *mr;
	struct ib_event_handler      event_handler;
	struct list_head             ig_list; /* entry in ig devices list */
	int                          refcount;
	int			     comps_used;
	struct iser_comp	     comps[ISER_MAX_CQ];
	int                          (*iser_alloc_rdma_reg_res)(struct ib_conn *ib_conn,
								unsigned cmds_max);
	void                         (*iser_free_rdma_reg_res)(struct ib_conn *ib_conn);
	int                          (*iser_reg_rdma_mem)(struct iscsi_iser_task *iser_task,
							  enum iser_data_dir cmd_dir);
	void                         (*iser_unreg_rdma_mem)(struct iscsi_iser_task *iser_task,
							    enum iser_data_dir cmd_dir);
};

#define ISER_CHECK_GUARD	0xc0
#define ISER_CHECK_REFTAG	0x0f
#define ISER_CHECK_APPTAG	0x30

enum iser_reg_indicator {
	ISER_DATA_KEY_VALID	= 1 << 0,
	ISER_PROT_KEY_VALID	= 1 << 1,
	ISER_SIG_KEY_VALID	= 1 << 2,
	ISER_FASTREG_PROTECTED	= 1 << 3,
};

struct iser_pi_context {
	struct ib_mr                   *prot_mr;
	struct ib_fast_reg_page_list   *prot_frpl;
	struct ib_mr                   *sig_mr;
};

struct fast_reg_descriptor {
	struct list_head		  list;
	/* For fast registration - FRWR */
	struct ib_mr			 *data_mr;
	struct ib_fast_reg_page_list     *data_frpl;
	struct iser_pi_context		 *pi_ctx;
	/* registration indicators container */
	u8				  reg_indicators;
};

/**
 * struct ib_conn - Infiniband related objects
 *
 * @cma_id:              rdma_cm connection maneger handle
 * @qp:                  Connection Queue-pair
 * @post_recv_buf_count: post receive counter
 * @rx_wr:               receive work request for batch posts
 * @device:              reference to iser device
 * @comp:                iser completion context
 * @pi_support:          Indicate device T10-PI support
 * @beacon:              beacon send wr to signal all flush errors were drained
 * @flush_comp:          completes when all connection completions consumed
 * @lock:                protects fmr/fastreg pool
 * @union.fmr:
 *     @pool:            FMR pool for fast registrations
 *     @page_vec:        page vector to hold mapped commands pages
 *                       used for registration
 * @union.fastreg:
 *     @pool:            Fast registration descriptors pool for fast
 *                       registrations
 *     @pool_size:       Size of pool
 */
struct ib_conn {
	struct rdma_cm_id           *cma_id;
	struct ib_qp	            *qp;
	int                          post_recv_buf_count;
	struct ib_recv_wr	     rx_wr[ISER_MIN_POSTED_RX];
	struct iser_device          *device;
	struct iser_comp	    *comp;
	bool			     pi_support;
	struct ib_send_wr	     beacon;
	struct completion	     flush_comp;
	spinlock_t		     lock;
	union {
		struct {
			struct ib_fmr_pool      *pool;
			struct iser_page_vec	*page_vec;
		} fmr;
		struct {
			struct list_head	 pool;
			int			 pool_size;
		} fastreg;
	};
};

struct iser_conn {
	struct ib_conn		     ib_conn;
	struct iscsi_conn	     *iscsi_conn;
	struct iscsi_endpoint	     *ep;
	enum iser_conn_state	     state;	    /* rdma connection state   */
	unsigned		     qp_max_recv_dtos; /* num of rx buffers */
	unsigned		     qp_max_recv_dtos_mask; /* above minus 1 */
	unsigned		     min_posted_rx; /* qp_max_recv_dtos >> 2 */
	char 			     name[ISER_OBJECT_NAME_SIZE];
	struct work_struct	     release_work;
	struct mutex		     state_mutex;
	struct completion	     stop_completion;
	struct completion	     ib_completion;
	struct completion	     up_completion;
	struct list_head	     conn_list;       /* entry in ig conn list */

	char  			     *login_buf;
	char			     *login_req_buf, *login_resp_buf;
	u64			     login_req_dma, login_resp_dma;
	unsigned int 		     rx_desc_head;
	struct iser_rx_desc	     *rx_descs;
	u32                          num_rx_descs;
};

struct iscsi_iser_task {
	struct iser_tx_desc          desc;
	struct iser_conn	     *iser_conn;
	enum iser_task_status 	     status;
	struct scsi_cmnd	     *sc;
	int                          command_sent;  /* set if command  sent  */
	int                          dir[ISER_DIRS_NUM];      /* set if dir use*/
	struct iser_regd_buf         rdma_regd[ISER_DIRS_NUM];/* regd rdma buf */
	struct iser_data_buf         data[ISER_DIRS_NUM];     /* orig. data des*/
	struct iser_data_buf         data_copy[ISER_DIRS_NUM];/* contig. copy  */
	struct iser_data_buf         prot[ISER_DIRS_NUM];     /* prot desc     */
	struct iser_data_buf         prot_copy[ISER_DIRS_NUM];/* prot copy     */
};

struct iser_page_vec {
	u64 *pages;
	int length;
	int offset;
	int data_size;
};

struct iser_global {
	struct mutex      device_list_mutex;/*                   */
	struct list_head  device_list;	     /* all iSER devices */
	struct mutex      connlist_mutex;
	struct list_head  connlist;		/* all iSER IB connections */

	struct kmem_cache *desc_cache;
};

extern struct iser_global ig;
extern int iser_debug_level;
extern bool iser_pi_enable;
extern int iser_pi_guard;

int iser_send_control(struct iscsi_conn *conn,
		      struct iscsi_task *task);

int iser_send_command(struct iscsi_conn *conn,
		      struct iscsi_task *task);

int iser_send_data_out(struct iscsi_conn *conn,
		       struct iscsi_task *task,
		       struct iscsi_data *hdr);

void iscsi_iser_recv(struct iscsi_conn *conn,
		     struct iscsi_hdr       *hdr,
		     char                   *rx_data,
		     int                    rx_data_len);

void iser_conn_init(struct iser_conn *iser_conn);

void iser_conn_release(struct iser_conn *iser_conn);

int iser_conn_terminate(struct iser_conn *iser_conn);

void iser_release_work(struct work_struct *work);

void iser_rcv_completion(struct iser_rx_desc *desc,
			 unsigned long    dto_xfer_len,
			 struct ib_conn *ib_conn);

void iser_snd_completion(struct iser_tx_desc *desc,
			 struct ib_conn *ib_conn);

void iser_task_rdma_init(struct iscsi_iser_task *task);

void iser_task_rdma_finalize(struct iscsi_iser_task *task);

void iser_free_rx_descriptors(struct iser_conn *iser_conn);

void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
				     struct iser_data_buf *mem,
				     struct iser_data_buf *mem_copy,
				     enum iser_data_dir cmd_dir);

int  iser_reg_rdma_mem_fmr(struct iscsi_iser_task *task,
			   enum iser_data_dir cmd_dir);
int  iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *task,
			       enum iser_data_dir cmd_dir);

int  iser_connect(struct iser_conn   *iser_conn,
		  struct sockaddr    *src_addr,
		  struct sockaddr    *dst_addr,
		  int                non_blocking);

int  iser_reg_page_vec(struct ib_conn *ib_conn,
		       struct iser_page_vec *page_vec,
		       struct iser_mem_reg  *mem_reg);

void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task,
			enum iser_data_dir cmd_dir);
void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
			    enum iser_data_dir cmd_dir);

int  iser_post_recvl(struct iser_conn *iser_conn);
int  iser_post_recvm(struct iser_conn *iser_conn, int count);
int  iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,
		    bool signal);

int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
			    struct iser_data_buf       *data,
			    enum   iser_data_dir       iser_dir,
			    enum   dma_data_direction  dma_dir);

void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task,
			      struct iser_data_buf *data);
int  iser_initialize_task_headers(struct iscsi_task *task,
			struct iser_tx_desc *tx_desc);
int iser_alloc_rx_descriptors(struct iser_conn *iser_conn,
			      struct iscsi_session *session);
int iser_create_fmr_pool(struct ib_conn *ib_conn, unsigned cmds_max);
void iser_free_fmr_pool(struct ib_conn *ib_conn);
int iser_create_fastreg_pool(struct ib_conn *ib_conn, unsigned cmds_max);
void iser_free_fastreg_pool(struct ib_conn *ib_conn);
u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,
			     enum iser_data_dir cmd_dir, sector_t *sector);
#endif
Commit	Line	Data
49cd5382 OG	1	/*
	2	* iSER transport for the Open iSCSI Initiator & iSER transport internals
	3	*
	4	* Copyright (C) 2004 Dmitry Yusupov
	5	* Copyright (C) 2004 Alex Aizman
	6	* Copyright (C) 2005 Mike Christie
	7	* based on code maintained by open-iscsi@googlegroups.com
	8	*
	9	* Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved.
	10	* Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
3ee07d27	11	* Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved.
49cd5382 OG	12	*
	13	* This software is available to you under a choice of one of two
	14	* licenses. You may choose to be licensed under the terms of the GNU
	15	* General Public License (GPL) Version 2, available from the file
	16	* COPYING in the main directory of this source tree, or the
	17	* OpenIB.org BSD license below:
	18	*
	19	* Redistribution and use in source and binary forms, with or
	20	* without modification, are permitted provided that the following
	21	* conditions are met:
	22	*
	23	* - Redistributions of source code must retain the above
	24	* copyright notice, this list of conditions and the following
	25	* disclaimer.
	26	*
	27	* - Redistributions in binary form must reproduce the above
	28	* copyright notice, this list of conditions and the following
	29	* disclaimer in the documentation and/or other materials
	30	* provided with the distribution.
	31	*
	32	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	33	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	34	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	35	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	36	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	37	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	38	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	39	* SOFTWARE.
49cd5382 OG	40	*/
	41	#ifndef __ISCSI_ISER_H__
	42	#define __ISCSI_ISER_H__
	43
	44	#include <linux/types.h>
	45	#include <linux/net.h>
4f363882	46	#include <linux/printk.h>
49cd5382 OG	47	#include <scsi/libiscsi.h>
49cd5382 OG	48	#include <scsi/scsi_transport_iscsi.h>
177e31bd SG	49	#include <scsi/scsi_cmnd.h>
177e31bd SG	50	#include <scsi/scsi_device.h>
49cd5382	51
a6b7a407	52	#include <linux/interrupt.h>
49cd5382 OG	53	#include <linux/wait.h>
	54	#include <linux/sched.h>
	55	#include <linux/list.h>
	56	#include <linux/slab.h>
	57	#include <linux/dma-mapping.h>
	58	#include <linux/mutex.h>
	59	#include <linux/mempool.h>
	60	#include <linux/uio.h>
	61
	62	#include <linux/socket.h>
	63	#include <linux/in.h>
	64	#include <linux/in6.h>
	65
	66	#include <rdma/ib_verbs.h>
	67	#include <rdma/ib_fmr_pool.h>
	68	#include <rdma/rdma_cm.h>
	69
	70	#define DRV_NAME "iser"
	71	#define PFX DRV_NAME ": "
61aabb3c	72	#define DRV_VER "1.4.1"
49cd5382 OG	73
49cd5382 OG	74	#define iser_dbg(fmt, arg...) \
6f735e36	75	do { \
4f363882	76	if (iser_debug_level > 2) \
6f735e36 ED	77	printk(KERN_DEBUG PFX "%s:" fmt,\
	78	__func__ , ## arg); \
	79	} while (0)
	80
	81	#define iser_warn(fmt, arg...) \
4f363882	82	do { \
f91424cf	83	if (iser_debug_level > 0) \
4f363882 RD	84	pr_warn(PFX "%s:" fmt, \
	85	__func__ , ## arg); \
	86	} while (0)
	87
	88	#define iser_info(fmt, arg...) \
49cd5382	89	do { \
f91424cf	90	if (iser_debug_level > 1) \
4f363882	91	pr_info(PFX "%s:" fmt, \
49cd5382 OG	92	__func__ , ## arg); \
	93	} while (0)
	94
	95	#define iser_err(fmt, arg...) \
	96	do { \
	97	printk(KERN_ERR PFX "%s:" fmt, \
	98	__func__ , ## arg); \
	99	} while (0)
	100
8dfa0876	101	#define SHIFT_4K 12
0ace64b8	102	#define SIZE_4K (1ULL << SHIFT_4K)
8dfa0876	103	#define MASK_4K (~(SIZE_4K-1))
25985edc	104	/* support up to 512KB in one RDMA */
8dfa0876	105	#define ISCSI_ISER_SG_TABLESIZE (0x80000 >> SHIFT_4K)
b7f04513 SP	106	#define ISER_DEF_XMIT_CMDS_DEFAULT 512
	107	#if ISCSI_DEF_XMIT_CMDS_MAX > ISER_DEF_XMIT_CMDS_DEFAULT
	108	#define ISER_DEF_XMIT_CMDS_MAX ISCSI_DEF_XMIT_CMDS_MAX
	109	#else
	110	#define ISER_DEF_XMIT_CMDS_MAX ISER_DEF_XMIT_CMDS_DEFAULT
	111	#endif
	112	#define ISER_DEF_CMD_PER_LUN ISER_DEF_XMIT_CMDS_MAX
49cd5382 OG	113
	114	/* QP settings */
	115	/* Maximal bounds on received asynchronous PDUs */
	116	#define ISER_MAX_RX_MISC_PDUS 4 /* NOOP_IN(2) , ASYNC_EVENT(2) */
	117
	118	#define ISER_MAX_TX_MISC_PDUS 6 /* NOOP_OUT(2), TEXT(1), *
	119	* SCSI_TMFUNC(2), LOGOUT(1) */
	120
b7f04513	121	#define ISER_QP_MAX_RECV_DTOS (ISER_DEF_XMIT_CMDS_MAX)
bcc60c38	122
b7f04513	123	#define ISER_MIN_POSTED_RX (ISER_DEF_XMIT_CMDS_MAX >> 2)
49cd5382 OG	124
	125	/* the max TX (send) WR supported by the iSER QP is defined by *
	126	* max_send_wr = T * (1 + D) + C ; D is how many inflight dataouts we expect *
	127	* to have at max for SCSI command. The tx posting & completion handling code *
	128	* supports -EAGAIN scheme where tx is suspended till the QP has room for more *
	129	* send WR. D=8 comes from 64K/8K */
	130
	131	#define ISER_INFLIGHT_DATAOUTS 8
	132
b7f04513	133	#define ISER_QP_MAX_REQ_DTOS (ISER_DEF_XMIT_CMDS_MAX * \
49cd5382 OG	134	(1 + ISER_INFLIGHT_DATAOUTS) + \
	135	ISER_MAX_TX_MISC_PDUS + \
	136	ISER_MAX_RX_MISC_PDUS)
	137
6b5a8fb0 AT	138	/* Max registration work requests per command */
	139	#define ISER_MAX_REG_WR_PER_CMD 5
	140
	141	/* For Signature we don't support DATAOUTs so no need to make room for them */
	142	#define ISER_QP_SIG_MAX_REQ_DTOS (ISER_DEF_XMIT_CMDS_MAX * \
	143	(1 + ISER_MAX_REG_WR_PER_CMD) + \
	144	ISER_MAX_TX_MISC_PDUS + \
	145	ISER_MAX_RX_MISC_PDUS)
	146
6e6fe2fb	147	#define ISER_WC_BATCH_COUNT 16
6df5a128	148	#define ISER_SIGNAL_CMD_COUNT 32
6e6fe2fb	149
49cd5382 OG	150	#define ISER_VER 0x10
	151	#define ISER_WSV 0x08
	152	#define ISER_RSV 0x04
	153
7306b8fa	154	#define ISER_FASTREG_LI_WRID 0xffffffffffffffffULL
ff3dd52d	155	#define ISER_BEACON_WRID 0xfffffffffffffffeULL
db523b8d	156
49cd5382 OG	157	struct iser_hdr {
	158	u8 flags;
	159	u8 rsvd[3];
	160	__be32 write_stag; /* write rkey */
	161	__be64 write_va;
	162	__be32 read_stag; /* read rkey */
	163	__be64 read_va;
	164	} __attribute__((packed));
	165
8d8399de OG	166
	167	#define ISER_ZBVA_NOT_SUPPORTED 0x80
	168	#define ISER_SEND_W_INV_NOT_SUPPORTED 0x40
	169
	170	struct iser_cm_hdr {
	171	u8 flags;
	172	u8 rsvd[3];
	173	} __packed;
	174
bcc60c38 OG	175	/* Constant PDU lengths calculations */
	176	#define ISER_HEADERS_LEN (sizeof(struct iser_hdr) + sizeof(struct iscsi_hdr))
	177
	178	#define ISER_RECV_DATA_SEG_LEN 128
	179	#define ISER_RX_PAYLOAD_SIZE (ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN)
	180	#define ISER_RX_LOGIN_SIZE (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN)
49cd5382 OG	181
	182	/* Length of an object name string */
	183	#define ISER_OBJECT_NAME_SIZE 64
	184
5716af6e	185	enum iser_conn_state {
49cd5382 OG	186	ISER_CONN_INIT, /* descriptor allocd, no conn */
	187	ISER_CONN_PENDING, /* in the process of being established */
	188	ISER_CONN_UP, /* up and running */
	189	ISER_CONN_TERMINATING, /* in the process of being terminated */
	190	ISER_CONN_DOWN, /* shut down */
	191	ISER_CONN_STATES_NUM
	192	};
	193
	194	enum iser_task_status {
	195	ISER_TASK_STATUS_INIT = 0,
	196	ISER_TASK_STATUS_STARTED,
	197	ISER_TASK_STATUS_COMPLETED
	198	};
	199
	200	enum iser_data_dir {
	201	ISER_DIR_IN = 0, /* to initiator */
	202	ISER_DIR_OUT, /* from initiator */
	203	ISER_DIRS_NUM
	204	};
	205
	206	struct iser_data_buf {
	207	void buf; / pointer to the sg list */
	208	unsigned int size; /* num entries of this sg */
	209	unsigned long data_len; /* total data len */
	210	unsigned int dma_nents; /* returned by dma_map_sg */
	211	char copy_buf; / allocated copy buf for SGs unaligned *
	212	* for rdma which are copied */
	213	struct scatterlist sg_single; /* SG-ified clone of a non SG SC or *
	214	* unaligned SG */
	215	};
	216
	217	/* fwd declarations */
	218	struct iser_device;
2261ec3d	219	struct iscsi_iser_task;
412eeafa	220	struct iscsi_endpoint;
49cd5382 OG	221
	222	struct iser_mem_reg {
	223	u32 lkey;
	224	u32 rkey;
	225	u64 va;
	226	u64 len;
	227	void *mem_h;
5587856c	228	int is_mr;
49cd5382 OG	229	};
	230
	231	struct iser_regd_buf {
	232	struct iser_mem_reg reg; /* memory registration info */
	233	void *virt_addr;
	234	struct iser_device device; / device->device for dma_unmap */
49cd5382 OG	235	enum dma_data_direction direction; /* direction for dma_unmap */
49cd5382 OG	236	unsigned int data_size;
49cd5382 OG	237	};
	238
	239	enum iser_desc_type {
49cd5382 OG	240	ISCSI_TX_CONTROL ,
	241	ISCSI_TX_SCSI_COMMAND,
	242	ISCSI_TX_DATAOUT
	243	};
	244
f19624aa	245	struct iser_tx_desc {
49cd5382 OG	246	struct iser_hdr iser_header;
49cd5382 OG	247	struct iscsi_hdr iscsi_header;
49cd5382	248	enum iser_desc_type type;
f19624aa OG	249	u64 dma_addr;
	250	/* sg[0] points to iser/iscsi headers, sg[1] optionally points to either
	251	of immediate data, unsolicited data-out or control (login,text) */
	252	struct ib_sge tx_sg[2];
	253	int num_sge;
49cd5382 OG	254	};
49cd5382 OG	255
bcc60c38 OG	256	#define ISER_RX_PAD_SIZE (256 - (ISER_RX_PAYLOAD_SIZE + \
	257	sizeof(u64) + sizeof(struct ib_sge)))
	258	struct iser_rx_desc {
	259	struct iser_hdr iser_header;
	260	struct iscsi_hdr iscsi_header;
	261	char data[ISER_RECV_DATA_SEG_LEN];
	262	u64 dma_addr;
	263	struct ib_sge rx_sg;
	264	char pad[ISER_RX_PAD_SIZE];
	265	} __attribute__((packed));
	266
5a33a669 AT	267	#define ISER_MAX_CQ 4
5a33a669 AT	268
b4e155ff	269	struct iser_conn;
a4ee3539	270	struct ib_conn;
b4e155ff SG	271	struct iscsi_iser_task;
b4e155ff SG	272
bf175540 SG	273	/**
	274	* struct iser_comp - iSER completion context
	275	*
	276	* @device: pointer to device handle
6aabfa76	277	* @cq: completion queue
6e6fe2fb	278	* @wcs: work completion array
bf175540 SG	279	* @tasklet: Tasklet handle
	280	* @active_qps: Number of active QPs attached
	281	* to completion context
	282	*/
	283	struct iser_comp {
	284	struct iser_device *device;
6aabfa76	285	struct ib_cq *cq;
6e6fe2fb	286	struct ib_wc wcs[ISER_WC_BATCH_COUNT];
bf175540 SG	287	struct tasklet_struct tasklet;
	288	int active_qps;
	289	};
	290
49cd5382 OG	291	struct iser_device {
	292	struct ib_device *ib_device;
	293	struct ib_pd *pd;
65198d6b	294	struct ib_device_attr dev_attr;
49cd5382	295	struct ib_mr *mr;
2110f9bf	296	struct ib_event_handler event_handler;
49cd5382 OG	297	struct list_head ig_list; /* entry in ig devices list */
49cd5382 OG	298	int refcount;
bf175540 SG	299	int comps_used;
bf175540 SG	300	struct iser_comp comps[ISER_MAX_CQ];
a4ee3539	301	int (iser_alloc_rdma_reg_res)(struct ib_conn ib_conn,
b4e155ff	302	unsigned cmds_max);
a4ee3539	303	void (iser_free_rdma_reg_res)(struct ib_conn ib_conn);
b4e155ff SG	304	int (iser_reg_rdma_mem)(struct iscsi_iser_task iser_task,
	305	enum iser_data_dir cmd_dir);
	306	void (iser_unreg_rdma_mem)(struct iscsi_iser_task iser_task,
	307	enum iser_data_dir cmd_dir);
49cd5382 OG	308	};
49cd5382 OG	309
177e31bd SG	310	#define ISER_CHECK_GUARD 0xc0
	311	#define ISER_CHECK_REFTAG 0x0f
	312	#define ISER_CHECK_APPTAG 0x30
	313
73bc06b7	314	enum iser_reg_indicator {
6b5a8fb0 AT	315	ISER_DATA_KEY_VALID = 1 << 0,
	316	ISER_PROT_KEY_VALID = 1 << 1,
	317	ISER_SIG_KEY_VALID = 1 << 2,
	318	ISER_FASTREG_PROTECTED = 1 << 3,
	319	};
	320
	321	struct iser_pi_context {
	322	struct ib_mr *prot_mr;
	323	struct ib_fast_reg_page_list *prot_frpl;
	324	struct ib_mr *sig_mr;
73bc06b7 SG	325	};
73bc06b7 SG	326
5587856c SG	327	struct fast_reg_descriptor {
	328	struct list_head list;
	329	/* For fast registration - FRWR */
	330	struct ib_mr *data_mr;
	331	struct ib_fast_reg_page_list *data_frpl;
6b5a8fb0	332	struct iser_pi_context *pi_ctx;
73bc06b7 SG	333	/* registration indicators container */
73bc06b7 SG	334	u8 reg_indicators;
5587856c SG	335	};
5587856c SG	336
a4ee3539 SG	337	/**
	338	* struct ib_conn - Infiniband related objects
	339	*
	340	* @cma_id: rdma_cm connection maneger handle
	341	* @qp: Connection Queue-pair
	342	* @post_recv_buf_count: post receive counter
a4ee3539 SG	343	* @rx_wr: receive work request for batch posts
a4ee3539 SG	344	* @device: reference to iser device
bf175540	345	* @comp: iser completion context
a4ee3539	346	* @pi_support: Indicate device T10-PI support
ff3dd52d	347	* @beacon: beacon send wr to signal all flush errors were drained
6aabfa76	348	* @flush_comp: completes when all connection completions consumed
a4ee3539 SG	349	* @lock: protects fmr/fastreg pool
	350	* @union.fmr:
	351	* @pool: FMR pool for fast registrations
	352	* @page_vec: page vector to hold mapped commands pages
	353	* used for registration
	354	* @union.fastreg:
	355	* @pool: Fast registration descriptors pool for fast
	356	* registrations
	357	* @pool_size: Size of pool
	358	*/
	359	struct ib_conn {
	360	struct rdma_cm_id *cma_id;
	361	struct ib_qp *qp;
	362	int post_recv_buf_count;
a4ee3539 SG	363	struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX];
a4ee3539 SG	364	struct iser_device *device;
bf175540	365	struct iser_comp *comp;
a4ee3539	366	bool pi_support;
ff3dd52d	367	struct ib_send_wr beacon;
6aabfa76	368	struct completion flush_comp;
a4ee3539 SG	369	spinlock_t lock;
	370	union {
	371	struct {
	372	struct ib_fmr_pool *pool;
	373	struct iser_page_vec *page_vec;
	374	} fmr;
	375	struct {
	376	struct list_head pool;
	377	int pool_size;
	378	} fastreg;
	379	};
	380	};
	381
49cd5382	382	struct iser_conn {
a4ee3539	383	struct ib_conn ib_conn;
4667f5df	384	struct iscsi_conn *iscsi_conn;
412eeafa	385	struct iscsi_endpoint *ep;
5716af6e	386	enum iser_conn_state state; /* rdma connection state */
b7f04513 SP	387	unsigned qp_max_recv_dtos; /* num of rx buffers */
	388	unsigned qp_max_recv_dtos_mask; /* above minus 1 */
	389	unsigned min_posted_rx; /* qp_max_recv_dtos >> 2 */
49cd5382	390	char name[ISER_OBJECT_NAME_SIZE];
b73c3ada	391	struct work_struct release_work;
504130c0	392	struct mutex state_mutex;
c47a3c9e SG	393	struct completion stop_completion;
c47a3c9e SG	394	struct completion ib_completion;
9a6d3234	395	struct completion up_completion;
49cd5382	396	struct list_head conn_list; /* entry in ig conn list */
bcc60c38 OG	397
bcc60c38 OG	398	char *login_buf;
2c4ce609 OG	399	char login_req_buf, login_resp_buf;
2c4ce609 OG	400	u64 login_req_dma, login_resp_dma;
bcc60c38 OG	401	unsigned int rx_desc_head;
bcc60c38 OG	402	struct iser_rx_desc *rx_descs;
6aabfa76	403	u32 num_rx_descs;
49cd5382 OG	404	};
49cd5382 OG	405
2261ec3d	406	struct iscsi_iser_task {
f19624aa	407	struct iser_tx_desc desc;
5716af6e	408	struct iser_conn *iser_conn;
49cd5382	409	enum iser_task_status status;
177e31bd	410	struct scsi_cmnd *sc;
49cd5382 OG	411	int command_sent; /* set if command sent */
	412	int dir[ISER_DIRS_NUM]; /* set if dir use*/
	413	struct iser_regd_buf rdma_regd[ISER_DIRS_NUM];/* regd rdma buf */
	414	struct iser_data_buf data[ISER_DIRS_NUM]; /* orig. data des*/
	415	struct iser_data_buf data_copy[ISER_DIRS_NUM];/* contig. copy */
177e31bd SG	416	struct iser_data_buf prot[ISER_DIRS_NUM]; /* prot desc */
177e31bd SG	417	struct iser_data_buf prot_copy[ISER_DIRS_NUM];/* prot copy */
49cd5382 OG	418	};
	419
	420	struct iser_page_vec {
	421	u64 *pages;
	422	int length;
	423	int offset;
	424	int data_size;
	425	};
	426
	427	struct iser_global {
	428	struct mutex device_list_mutex;/* */
	429	struct list_head device_list; /* all iSER devices */
	430	struct mutex connlist_mutex;
	431	struct list_head connlist; /* all iSER IB connections */
	432
e54f8188	433	struct kmem_cache *desc_cache;
49cd5382 OG	434	};
	435
	436	extern struct iser_global ig;
	437	extern int iser_debug_level;
7f733847 AT	438	extern bool iser_pi_enable;
7f733847 AT	439	extern int iser_pi_guard;
49cd5382	440
2747fdb2	441	int iser_send_control(struct iscsi_conn *conn,
2261ec3d	442	struct iscsi_task *task);
49cd5382	443
2747fdb2	444	int iser_send_command(struct iscsi_conn *conn,
2261ec3d	445	struct iscsi_task *task);
49cd5382	446
2747fdb2	447	int iser_send_data_out(struct iscsi_conn *conn,
2261ec3d	448	struct iscsi_task *task,
2747fdb2	449	struct iscsi_data *hdr);
49cd5382 OG	450
	451	void iscsi_iser_recv(struct iscsi_conn *conn,
	452	struct iscsi_hdr *hdr,
	453	char *rx_data,
	454	int rx_data_len);
	455
5716af6e	456	void iser_conn_init(struct iser_conn *iser_conn);
49cd5382	457
5716af6e	458	void iser_conn_release(struct iser_conn *iser_conn);
49cd5382	459
c47a3c9e	460	int iser_conn_terminate(struct iser_conn *iser_conn);
49cd5382	461
b73c3ada AN	462	void iser_release_work(struct work_struct *work);
b73c3ada AN	463
bcc60c38 OG	464	void iser_rcv_completion(struct iser_rx_desc *desc,
bcc60c38 OG	465	unsigned long dto_xfer_len,
a4ee3539	466	struct ib_conn *ib_conn);
49cd5382	467
5716af6e	468	void iser_snd_completion(struct iser_tx_desc *desc,
a4ee3539	469	struct ib_conn *ib_conn);
49cd5382	470
2261ec3d	471	void iser_task_rdma_init(struct iscsi_iser_task *task);
49cd5382	472
2261ec3d	473	void iser_task_rdma_finalize(struct iscsi_iser_task *task);
49cd5382	474
5716af6e	475	void iser_free_rx_descriptors(struct iser_conn *iser_conn);
bcc60c38	476
9a8b08fa SG	477	void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
	478	struct iser_data_buf *mem,
	479	struct iser_data_buf *mem_copy,
	480	enum iser_data_dir cmd_dir);
49cd5382	481
e657571b SG	482	int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *task,
e657571b SG	483	enum iser_data_dir cmd_dir);
7306b8fa SG	484	int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *task,
7306b8fa SG	485	enum iser_data_dir cmd_dir);
49cd5382	486
5716af6e	487	int iser_connect(struct iser_conn *iser_conn,
96ed02d4 RD	488	struct sockaddr *src_addr,
96ed02d4 RD	489	struct sockaddr *dst_addr,
49cd5382 OG	490	int non_blocking);
49cd5382 OG	491
a4ee3539	492	int iser_reg_page_vec(struct ib_conn *ib_conn,
49cd5382 OG	493	struct iser_page_vec *page_vec,
	494	struct iser_mem_reg *mem_reg);
	495
e657571b SG	496	void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task,
e657571b SG	497	enum iser_data_dir cmd_dir);
7306b8fa SG	498	void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
7306b8fa SG	499	enum iser_data_dir cmd_dir);
49cd5382	500
5716af6e SG	501	int iser_post_recvl(struct iser_conn *iser_conn);
5716af6e SG	502	int iser_post_recvm(struct iser_conn *iser_conn, int count);
6df5a128 SG	503	int iser_post_send(struct ib_conn ib_conn, struct iser_tx_desc tx_desc,
6df5a128 SG	504	bool signal);
49cd5382	505
2261ec3d	506	int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
74a20780 EZ	507	struct iser_data_buf *data,
	508	enum iser_data_dir iser_dir,
	509	enum dma_data_direction dma_dir);
	510
9a8b08fa SG	511	void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task,
9a8b08fa SG	512	struct iser_data_buf *data);
f19624aa OG	513	int iser_initialize_task_headers(struct iscsi_task *task,
f19624aa OG	514	struct iser_tx_desc *tx_desc);
5716af6e SG	515	int iser_alloc_rx_descriptors(struct iser_conn *iser_conn,
5716af6e SG	516	struct iscsi_session *session);
a4ee3539 SG	517	int iser_create_fmr_pool(struct ib_conn *ib_conn, unsigned cmds_max);
	518	void iser_free_fmr_pool(struct ib_conn *ib_conn);
	519	int iser_create_fastreg_pool(struct ib_conn *ib_conn, unsigned cmds_max);
	520	void iser_free_fastreg_pool(struct ib_conn *ib_conn);
0a7a08ad SG	521	u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,
0a7a08ad SG	522	enum iser_data_dir cmd_dir, sector_t *sector);
49cd5382	523	#endif