projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
skd: remove SKD_OMIT_FROM_SRC_DIST ifdefs
[deliverable/linux.git] / drivers / block / skd_main.c
CommitLineData
e67f86b3
AB		 1/* Copyright 2012 STEC, Inc.
2 *
3 * This file is licensed under the terms of the 3-clause
4 * BSD License (http://opensource.org/licenses/BSD-3-Clause)
5 * or the GNU GPL-2.0 (http://www.gnu.org/licenses/gpl-2.0.html),
6 * at your option. Both licenses are also available in the LICENSE file
7 * distributed with this project. This file may not be copied, modified,
8 * or distributed except in accordance with those terms.
9 * Gordoni Waidhofer <gwaidhofer@stec-inc.com>
10 * Initial Driver Design!
11 * Thomas Swann <tswann@stec-inc.com>
12 * Interrupt handling.
13 * Ramprasad Chinthekindi <rchinthekindi@stec-inc.com>
14 * biomode implementation.
15 * Akhil Bhansali <abhansali@stec-inc.com>
16 * Added support for DISCARD / FLUSH and FUA.
17 */
18
19#include <linux/kernel.h>
20#include <linux/module.h>
21#include <linux/init.h>
22#include <linux/pci.h>
23#include <linux/slab.h>
24#include <linux/spinlock.h>
25#include <linux/blkdev.h>
26#include <linux/sched.h>
27#include <linux/interrupt.h>
28#include <linux/compiler.h>
29#include <linux/workqueue.h>
30#include <linux/bitops.h>
31#include <linux/delay.h>
32#include <linux/time.h>
33#include <linux/hdreg.h>
34#include <linux/dma-mapping.h>
35#include <linux/completion.h>
36#include <linux/scatterlist.h>
37#include <linux/version.h>
38#include <linux/err.h>
39#include <linux/scatterlist.h>
40#include <linux/aer.h>
41#include <linux/ctype.h>
42#include <linux/wait.h>
43#include <linux/uio.h>
44#include <scsi/scsi.h>
e67f86b3
AB		 45#include <scsi/sg.h>
46#include <linux/io.h>
47#include <linux/uaccess.h>
4ca90b53 48#include <asm/unaligned.h>
e67f86b3
AB		 49
50#include "skd_s1120.h"
51
52static int skd_dbg_level;
53static int skd_isr_comp_limit = 4;
54
55enum {
56 STEC_LINK_2_5GTS = 0,
57 STEC_LINK_5GTS = 1,
58 STEC_LINK_8GTS = 2,
59 STEC_LINK_UNKNOWN = 0xFF
60};
61
62enum {
63 SKD_FLUSH_INITIALIZER,
64 SKD_FLUSH_ZERO_SIZE_FIRST,
65 SKD_FLUSH_DATA_SECOND,
66};
67
e67f86b3
AB		 68#define SKD_ASSERT(expr) \
69 do { \
70 if (unlikely(!(expr))) { \
71 pr_err("Assertion failed! %s,%s,%s,line=%d\n", \
72 # expr, __FILE__, __func__, __LINE__); \
73 } \
74 } while (0)
75
e67f86b3
AB		 76#define DRV_NAME "skd"
77#define DRV_VERSION "2.2.1"
78#define DRV_BUILD_ID "0260"
79#define PFX DRV_NAME ": "
80#define DRV_BIN_VERSION 0x100
81#define DRV_VER_COMPL "2.2.1." DRV_BUILD_ID
82
83MODULE_AUTHOR("bug-reports: support@stec-inc.com");
84MODULE_LICENSE("Dual BSD/GPL");
85
38d4a1bb 86MODULE_DESCRIPTION("STEC s1120 PCIe SSD block driver (b" DRV_BUILD_ID ")");
e67f86b3
AB		 87MODULE_VERSION(DRV_VERSION "-" DRV_BUILD_ID);
88
89#define PCI_VENDOR_ID_STEC 0x1B39
90#define PCI_DEVICE_ID_S1120 0x0001
91
92#define SKD_FUA_NV (1 << 1)
93#define SKD_MINORS_PER_DEVICE 16
94
95#define SKD_MAX_QUEUE_DEPTH 200u
96
97#define SKD_PAUSE_TIMEOUT (5 * 1000)
98
99#define SKD_N_FITMSG_BYTES (512u)
100
101#define SKD_N_SPECIAL_CONTEXT 32u
102#define SKD_N_SPECIAL_FITMSG_BYTES (128u)
103
104/* SG elements are 32 bytes, so we can make this 4096 and still be under the
105 * 128KB limit. That allows 4096*4K = 16M xfer size
106 */
107#define SKD_N_SG_PER_REQ_DEFAULT 256u
108#define SKD_N_SG_PER_SPECIAL 256u
109
110#define SKD_N_COMPLETION_ENTRY 256u
111#define SKD_N_READ_CAP_BYTES (8u)
112
113#define SKD_N_INTERNAL_BYTES (512u)
114
115/* 5 bits of uniqifier, 0xF800 */
116#define SKD_ID_INCR (0x400)
117#define SKD_ID_TABLE_MASK (3u << 8u)
118#define SKD_ID_RW_REQUEST (0u << 8u)
119#define SKD_ID_INTERNAL (1u << 8u)
120#define SKD_ID_SPECIAL_REQUEST (2u << 8u)
121#define SKD_ID_FIT_MSG (3u << 8u)
122#define SKD_ID_SLOT_MASK 0x00FFu
123#define SKD_ID_SLOT_AND_TABLE_MASK 0x03FFu
124
125#define SKD_N_TIMEOUT_SLOT 4u
126#define SKD_TIMEOUT_SLOT_MASK 3u
127
128#define SKD_N_MAX_SECTORS 2048u
129
130#define SKD_MAX_RETRIES 2u
131
132#define SKD_TIMER_SECONDS(seconds) (seconds)
133#define SKD_TIMER_MINUTES(minutes) ((minutes) * (60))
134
135#define INQ_STD_NBYTES 36
136#define SKD_DISCARD_CDB_LENGTH 24
137
138enum skd_drvr_state {
139 SKD_DRVR_STATE_LOAD,
140 SKD_DRVR_STATE_IDLE,
141 SKD_DRVR_STATE_BUSY,
142 SKD_DRVR_STATE_STARTING,
143 SKD_DRVR_STATE_ONLINE,
144 SKD_DRVR_STATE_PAUSING,
145 SKD_DRVR_STATE_PAUSED,
146 SKD_DRVR_STATE_DRAINING_TIMEOUT,
147 SKD_DRVR_STATE_RESTARTING,
148 SKD_DRVR_STATE_RESUMING,
149 SKD_DRVR_STATE_STOPPING,
150 SKD_DRVR_STATE_FAULT,
151 SKD_DRVR_STATE_DISAPPEARED,
152 SKD_DRVR_STATE_PROTOCOL_MISMATCH,
153 SKD_DRVR_STATE_BUSY_ERASE,
154 SKD_DRVR_STATE_BUSY_SANITIZE,
155 SKD_DRVR_STATE_BUSY_IMMINENT,
156 SKD_DRVR_STATE_WAIT_BOOT,
157 SKD_DRVR_STATE_SYNCING,
158};
159
160#define SKD_WAIT_BOOT_TIMO SKD_TIMER_SECONDS(90u)
161#define SKD_STARTING_TIMO SKD_TIMER_SECONDS(8u)
162#define SKD_RESTARTING_TIMO SKD_TIMER_MINUTES(4u)
163#define SKD_DRAINING_TIMO SKD_TIMER_SECONDS(6u)
164#define SKD_BUSY_TIMO SKD_TIMER_MINUTES(20u)
165#define SKD_STARTED_BUSY_TIMO SKD_TIMER_SECONDS(60u)
166#define SKD_START_WAIT_SECONDS 90u
167
168enum skd_req_state {
169 SKD_REQ_STATE_IDLE,
170 SKD_REQ_STATE_SETUP,
171 SKD_REQ_STATE_BUSY,
172 SKD_REQ_STATE_COMPLETED,
173 SKD_REQ_STATE_TIMEOUT,
174 SKD_REQ_STATE_ABORTED,
175};
176
177enum skd_fit_msg_state {
178 SKD_MSG_STATE_IDLE,
179 SKD_MSG_STATE_BUSY,
180};
181
182enum skd_check_status_action {
183 SKD_CHECK_STATUS_REPORT_GOOD,
184 SKD_CHECK_STATUS_REPORT_SMART_ALERT,
185 SKD_CHECK_STATUS_REQUEUE_REQUEST,
186 SKD_CHECK_STATUS_REPORT_ERROR,
187 SKD_CHECK_STATUS_BUSY_IMMINENT,
188};
189
190struct skd_fitmsg_context {
191 enum skd_fit_msg_state state;
192
193 struct skd_fitmsg_context *next;
194
195 u32 id;
196 u16 outstanding;
197
198 u32 length;
199 u32 offset;
200
201 u8 *msg_buf;
202 dma_addr_t mb_dma_address;
203};
204
205struct skd_request_context {
206 enum skd_req_state state;
207
208 struct skd_request_context *next;
209
210 u16 id;
211 u32 fitmsg_id;
212
213 struct request *req;
e67f86b3
AB		 214 u8 flush_cmd;
215 u8 discard_page;
216
217 u32 timeout_stamp;
218 u8 sg_data_dir;
219 struct scatterlist *sg;
220 u32 n_sg;
221 u32 sg_byte_count;
222
223 struct fit_sg_descriptor *sksg_list;
224 dma_addr_t sksg_dma_address;
225
226 struct fit_completion_entry_v1 completion;
227
228 struct fit_comp_error_info err_info;
229
230};
231#define SKD_DATA_DIR_HOST_TO_CARD 1
232#define SKD_DATA_DIR_CARD_TO_HOST 2
233#define SKD_DATA_DIR_NONE 3 /* especially for DISCARD requests. */
234
235struct skd_special_context {
236 struct skd_request_context req;
237
238 u8 orphaned;
239
240 void *data_buf;
241 dma_addr_t db_dma_address;
242
243 u8 *msg_buf;
244 dma_addr_t mb_dma_address;
245};
246
247struct skd_sg_io {
248 fmode_t mode;
249 void __user *argp;
250
251 struct sg_io_hdr sg;
252
253 u8 cdb[16];
254
255 u32 dxfer_len;
256 u32 iovcnt;
257 struct sg_iovec *iov;
258 struct sg_iovec no_iov_iov;
259
260 struct skd_special_context *skspcl;
261};
262
263typedef enum skd_irq_type {
264 SKD_IRQ_LEGACY,
265 SKD_IRQ_MSI,
266 SKD_IRQ_MSIX
267} skd_irq_type_t;
268
269#define SKD_MAX_BARS 2
270
271struct skd_device {
272 volatile void __iomem *mem_map[SKD_MAX_BARS];
273 resource_size_t mem_phys[SKD_MAX_BARS];
274 u32 mem_size[SKD_MAX_BARS];
275
276 skd_irq_type_t irq_type;
277 u32 msix_count;
278 struct skd_msix_entry *msix_entries;
279
280 struct pci_dev *pdev;
281 int pcie_error_reporting_is_enabled;
282
283 spinlock_t lock;
284 struct gendisk *disk;
285 struct request_queue *queue;
286 struct device *class_dev;
287 int gendisk_on;
288 int sync_done;
289
290 atomic_t device_count;
291 u32 devno;
292 u32 major;
293 char name[32];
294 char isr_name[30];
295
296 enum skd_drvr_state state;
297 u32 drive_state;
298
299 u32 in_flight;
300 u32 cur_max_queue_depth;
301 u32 queue_low_water_mark;
302 u32 dev_max_queue_depth;
303
304 u32 num_fitmsg_context;
305 u32 num_req_context;
306
307 u32 timeout_slot[SKD_N_TIMEOUT_SLOT];
308 u32 timeout_stamp;
309 struct skd_fitmsg_context *skmsg_free_list;
310 struct skd_fitmsg_context *skmsg_table;
311
312 struct skd_request_context *skreq_free_list;
313 struct skd_request_context *skreq_table;
314
315 struct skd_special_context *skspcl_free_list;
316 struct skd_special_context *skspcl_table;
317
318 struct skd_special_context internal_skspcl;
319 u32 read_cap_blocksize;
320 u32 read_cap_last_lba;
321 int read_cap_is_valid;
322 int inquiry_is_valid;
323 u8 inq_serial_num[13]; /*12 chars plus null term */
324 u8 id_str[80]; /* holds a composite name (pci + sernum) */
325
326 u8 skcomp_cycle;
327 u32 skcomp_ix;
328 struct fit_completion_entry_v1 *skcomp_table;
329 struct fit_comp_error_info *skerr_table;
330 dma_addr_t cq_dma_address;
331
332 wait_queue_head_t waitq;
333
334 struct timer_list timer;
335 u32 timer_countdown;
336 u32 timer_substate;
337
338 int n_special;
339 int sgs_per_request;
340 u32 last_mtd;
341
342 u32 proto_ver;
343
344 int dbg_level;
345 u32 connect_time_stamp;
346 int connect_retries;
347#define SKD_MAX_CONNECT_RETRIES 16
348 u32 drive_jiffies;
349
350 u32 timo_slot;
351
e67f86b3 352
38d4a1bb 353 struct work_struct completion_worker;
e67f86b3
AB		 354};
355
356#define SKD_WRITEL(DEV, VAL, OFF) skd_reg_write32(DEV, VAL, OFF)
357#define SKD_READL(DEV, OFF) skd_reg_read32(DEV, OFF)
358#define SKD_WRITEQ(DEV, VAL, OFF) skd_reg_write64(DEV, VAL, OFF)
359
360static inline u32 skd_reg_read32(struct skd_device *skdev, u32 offset)
361{
362 u32 val;
363
364 if (likely(skdev->dbg_level < 2))
365 return readl(skdev->mem_map[1] + offset);
366 else {
367 barrier();
368 val = readl(skdev->mem_map[1] + offset);
369 barrier();
2e44b427 370 pr_debug("%s:%s:%d offset %x = %x\n",
371 skdev->name, __func__, __LINE__, offset, val);
e67f86b3
AB		 372 return val;
373 }
374
375}
376
377static inline void skd_reg_write32(struct skd_device *skdev, u32 val,
378 u32 offset)
379{
380 if (likely(skdev->dbg_level < 2)) {
381 writel(val, skdev->mem_map[1] + offset);
382 barrier();
e67f86b3
AB		 383 } else {
384 barrier();
385 writel(val, skdev->mem_map[1] + offset);
386 barrier();
2e44b427 387 pr_debug("%s:%s:%d offset %x = %x\n",
388 skdev->name, __func__, __LINE__, offset, val);
e67f86b3
AB		 389 }
390}
391
392static inline void skd_reg_write64(struct skd_device *skdev, u64 val,
393 u32 offset)
394{
395 if (likely(skdev->dbg_level < 2)) {
396 writeq(val, skdev->mem_map[1] + offset);
397 barrier();
e67f86b3
AB		 398 } else {
399 barrier();
400 writeq(val, skdev->mem_map[1] + offset);
401 barrier();
2e44b427 402 pr_debug("%s:%s:%d offset %x = %016llx\n",
403 skdev->name, __func__, __LINE__, offset, val);
e67f86b3
AB		 404 }
405}
406
407
408#define SKD_IRQ_DEFAULT SKD_IRQ_MSI
409static int skd_isr_type = SKD_IRQ_DEFAULT;
410
411module_param(skd_isr_type, int, 0444);
412MODULE_PARM_DESC(skd_isr_type, "Interrupt type capability."
413 " (0==legacy, 1==MSI, 2==MSI-X, default==1)");
414
415#define SKD_MAX_REQ_PER_MSG_DEFAULT 1
416static int skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
417
418module_param(skd_max_req_per_msg, int, 0444);
419MODULE_PARM_DESC(skd_max_req_per_msg,
420 "Maximum SCSI requests packed in a single message."
421 " (1-14, default==1)");
422
423#define SKD_MAX_QUEUE_DEPTH_DEFAULT 64
424#define SKD_MAX_QUEUE_DEPTH_DEFAULT_STR "64"
425static int skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
426
427module_param(skd_max_queue_depth, int, 0444);
428MODULE_PARM_DESC(skd_max_queue_depth,
429 "Maximum SCSI requests issued to s1120."
430 " (1-200, default==" SKD_MAX_QUEUE_DEPTH_DEFAULT_STR ")");
431
432static int skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
433module_param(skd_sgs_per_request, int, 0444);
434MODULE_PARM_DESC(skd_sgs_per_request,
435 "Maximum SG elements per block request."
436 " (1-4096, default==256)");
437
438static int skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
439module_param(skd_max_pass_thru, int, 0444);
440MODULE_PARM_DESC(skd_max_pass_thru,
441 "Maximum SCSI pass-thru at a time." " (1-50, default==32)");
442
443module_param(skd_dbg_level, int, 0444);
444MODULE_PARM_DESC(skd_dbg_level, "s1120 debug level (0,1,2)");
445
446module_param(skd_isr_comp_limit, int, 0444);
447MODULE_PARM_DESC(skd_isr_comp_limit, "s1120 isr comp limit (0=none) default=4");
448
e67f86b3
AB		 449/* Major device number dynamically assigned. */
450static u32 skd_major;
451
452static struct skd_device *skd_construct(struct pci_dev *pdev);
453static void skd_destruct(struct skd_device *skdev);
454static const struct block_device_operations skd_blockdev_ops;
455static void skd_send_fitmsg(struct skd_device *skdev,
456 struct skd_fitmsg_context *skmsg);
457static void skd_send_special_fitmsg(struct skd_device *skdev,
458 struct skd_special_context *skspcl);
459static void skd_request_fn(struct request_queue *rq);
460static void skd_end_request(struct skd_device *skdev,
461 struct skd_request_context *skreq, int error);
462static int skd_preop_sg_list(struct skd_device *skdev,
463 struct skd_request_context *skreq);
464static void skd_postop_sg_list(struct skd_device *skdev,
465 struct skd_request_context *skreq);
466
467static void skd_restart_device(struct skd_device *skdev);
468static int skd_quiesce_dev(struct skd_device *skdev);
469static int skd_unquiesce_dev(struct skd_device *skdev);
470static void skd_release_special(struct skd_device *skdev,
471 struct skd_special_context *skspcl);
472static void skd_disable_interrupts(struct skd_device *skdev);
473static void skd_isr_fwstate(struct skd_device *skdev);
474static void skd_recover_requests(struct skd_device *skdev, int requeue);
475static void skd_soft_reset(struct skd_device *skdev);
476
477static const char *skd_name(struct skd_device *skdev);
478const char *skd_drive_state_to_str(int state);
479const char *skd_skdev_state_to_str(enum skd_drvr_state state);
480static void skd_log_skdev(struct skd_device *skdev, const char *event);
481static void skd_log_skmsg(struct skd_device *skdev,
482 struct skd_fitmsg_context *skmsg, const char *event);
483static void skd_log_skreq(struct skd_device *skdev,
484 struct skd_request_context *skreq, const char *event);
485
e67f86b3
AB		 486/*
487 *****************************************************************************
488 * READ/WRITE REQUESTS
489 *****************************************************************************
490 */
fcd37eb3 491static void skd_fail_all_pending(struct skd_device *skdev)
e67f86b3
AB		 492{
493 struct request_queue *q = skdev->queue;
494 struct request *req;
495
496 for (;; ) {
497 req = blk_peek_request(q);
498 if (req == NULL)
499 break;
500 blk_start_request(req);
501 __blk_end_request_all(req, -EIO);
502 }
503}
504
e67f86b3
AB		 505static void
506skd_prep_rw_cdb(struct skd_scsi_request *scsi_req,
507 int data_dir, unsigned lba,
508 unsigned count)
509{
510 if (data_dir == READ)
511 scsi_req->cdb[0] = 0x28;
512 else
513 scsi_req->cdb[0] = 0x2a;
514
515 scsi_req->cdb[1] = 0;
516 scsi_req->cdb[2] = (lba & 0xff000000) >> 24;
517 scsi_req->cdb[3] = (lba & 0xff0000) >> 16;
518 scsi_req->cdb[4] = (lba & 0xff00) >> 8;
519 scsi_req->cdb[5] = (lba & 0xff);
520 scsi_req->cdb[6] = 0;
521 scsi_req->cdb[7] = (count & 0xff00) >> 8;
522 scsi_req->cdb[8] = count & 0xff;
523 scsi_req->cdb[9] = 0;
524}
525
526static void
527skd_prep_zerosize_flush_cdb(struct skd_scsi_request *scsi_req,
38d4a1bb 528 struct skd_request_context *skreq)
e67f86b3
AB		 529{
530 skreq->flush_cmd = 1;
531
532 scsi_req->cdb[0] = 0x35;
533 scsi_req->cdb[1] = 0;
534 scsi_req->cdb[2] = 0;
535 scsi_req->cdb[3] = 0;
536 scsi_req->cdb[4] = 0;
537 scsi_req->cdb[5] = 0;
538 scsi_req->cdb[6] = 0;
539 scsi_req->cdb[7] = 0;
540 scsi_req->cdb[8] = 0;
541 scsi_req->cdb[9] = 0;
542}
543
544static void
545skd_prep_discard_cdb(struct skd_scsi_request *scsi_req,
38d4a1bb
MS		 546 struct skd_request_context *skreq,
547 struct page *page,
548 u32 lba, u32 count)
e67f86b3
AB		 549{
550 char *buf;
551 unsigned long len;
552 struct request *req;
553
554 buf = page_address(page);
555 len = SKD_DISCARD_CDB_LENGTH;
556
557 scsi_req->cdb[0] = UNMAP;
558 scsi_req->cdb[8] = len;
559
560 put_unaligned_be16(6 + 16, &buf[0]);
561 put_unaligned_be16(16, &buf[2]);
562 put_unaligned_be64(lba, &buf[8]);
563 put_unaligned_be32(count, &buf[16]);
564
fcd37eb3
JA		 565 req = skreq->req;
566 blk_add_request_payload(req, page, len);
567 req->buffer = buf;
e67f86b3
AB		 568}
569
570static void skd_request_fn_not_online(struct request_queue *q);
571
572static void skd_request_fn(struct request_queue *q)
573{
574 struct skd_device *skdev = q->queuedata;
575 struct skd_fitmsg_context *skmsg = NULL;
576 struct fit_msg_hdr *fmh = NULL;
577 struct skd_request_context *skreq;
578 struct request *req = NULL;
e67f86b3
AB		 579 struct skd_scsi_request *scsi_req;
580 struct page *page;
581 unsigned long io_flags;
582 int error;
583 u32 lba;
584 u32 count;
585 int data_dir;
586 u32 be_lba;
587 u32 be_count;
588 u64 be_dmaa;
589 u64 cmdctxt;
590 u32 timo_slot;
591 void *cmd_ptr;
592 int flush, fua;
593
594 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
595 skd_request_fn_not_online(q);
596 return;
597 }
598
6a5ec65b 599 if (blk_queue_stopped(skdev->queue)) {
e67f86b3
AB		 600 if (skdev->skmsg_free_list == NULL ||
601 skdev->skreq_free_list == NULL ||
602 skdev->in_flight >= skdev->queue_low_water_mark)
603 /* There is still some kind of shortage */
604 return;
605
6a5ec65b 606 queue_flag_clear(QUEUE_FLAG_STOPPED, skdev->queue);
e67f86b3
AB		 607 }
608
609 /*
610 * Stop conditions:
611 * - There are no more native requests
612 * - There are already the maximum number of requests in progress
613 * - There are no more skd_request_context entries
614 * - There are no more FIT msg buffers
615 */
616 for (;; ) {
617
618 flush = fua = 0;
619
fcd37eb3 620 req = blk_peek_request(q);
e67f86b3 621
fcd37eb3
JA		 622 /* Are there any native requests to start? */
623 if (req == NULL)
624 break;
e67f86b3 625
fcd37eb3
JA		 626 lba = (u32)blk_rq_pos(req);
627 count = blk_rq_sectors(req);
628 data_dir = rq_data_dir(req);
629 io_flags = req->cmd_flags;
e67f86b3 630
fcd37eb3
JA		 631 if (io_flags & REQ_FLUSH)
632 flush++;
e67f86b3 633
fcd37eb3
JA		 634 if (io_flags & REQ_FUA)
635 fua++;
e67f86b3 636
fcd37eb3
JA		 637 pr_debug("%s:%s:%d new req=%p lba=%u(0x%x) "
638 "count=%u(0x%x) dir=%d\n",
639 skdev->name, __func__, __LINE__,
640 req, lba, lba, count, count, data_dir);
e67f86b3 641
38d4a1bb 642 /* At this point we know there is a request */
e67f86b3
AB		 643
644 /* Are too many requets already in progress? */
645 if (skdev->in_flight >= skdev->cur_max_queue_depth) {
2e44b427 646 pr_debug("%s:%s:%d qdepth %d, limit %d\n",
647 skdev->name, __func__, __LINE__,
648 skdev->in_flight, skdev->cur_max_queue_depth);
e67f86b3
AB		 649 break;
650 }
651
652 /* Is a skd_request_context available? */
653 skreq = skdev->skreq_free_list;
654 if (skreq == NULL) {
2e44b427 655 pr_debug("%s:%s:%d Out of req=%p\n",
656 skdev->name, __func__, __LINE__, q);
e67f86b3
AB		 657 break;
658 }
659 SKD_ASSERT(skreq->state == SKD_REQ_STATE_IDLE);
660 SKD_ASSERT((skreq->id & SKD_ID_INCR) == 0);
661
662 /* Now we check to see if we can get a fit msg */
663 if (skmsg == NULL) {
664 if (skdev->skmsg_free_list == NULL) {
2e44b427 665 pr_debug("%s:%s:%d Out of msg\n",
666 skdev->name, __func__, __LINE__);
e67f86b3
AB		 667 break;
668 }
669 }
670
671 skreq->flush_cmd = 0;
672 skreq->n_sg = 0;
673 skreq->sg_byte_count = 0;
674 skreq->discard_page = 0;
675
676 /*
38d4a1bb 677 * OK to now dequeue request from q.
e67f86b3
AB		 678 *
679 * At this point we are comitted to either start or reject
680 * the native request. Note that skd_request_context is
681 * available but is still at the head of the free list.
682 */
fcd37eb3
JA		 683 blk_start_request(req);
684 skreq->req = req;
685 skreq->fitmsg_id = 0;
e67f86b3
AB		 686
687 /* Either a FIT msg is in progress or we have to start one. */
688 if (skmsg == NULL) {
689 /* Are there any FIT msg buffers available? */
690 skmsg = skdev->skmsg_free_list;
691 if (skmsg == NULL) {
2e44b427 692 pr_debug("%s:%s:%d Out of msg skdev=%p\n",
693 skdev->name, __func__, __LINE__,
694 skdev);
e67f86b3
AB		 695 break;
696 }
697 SKD_ASSERT(skmsg->state == SKD_MSG_STATE_IDLE);
698 SKD_ASSERT((skmsg->id & SKD_ID_INCR) == 0);
699
700 skdev->skmsg_free_list = skmsg->next;
701
702 skmsg->state = SKD_MSG_STATE_BUSY;
703 skmsg->id += SKD_ID_INCR;
704
705 /* Initialize the FIT msg header */
706 fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
707 memset(fmh, 0, sizeof(*fmh));
708 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
709 skmsg->length = sizeof(*fmh);
710 }
711
712 skreq->fitmsg_id = skmsg->id;
713
714 /*
715 * Note that a FIT msg may have just been started
716 * but contains no SoFIT requests yet.
717 */
718
719 /*
720 * Transcode the request, checking as we go. The outcome of
721 * the transcoding is represented by the error variable.
722 */
723 cmd_ptr = &skmsg->msg_buf[skmsg->length];
724 memset(cmd_ptr, 0, 32);
725
726 be_lba = cpu_to_be32(lba);
727 be_count = cpu_to_be32(count);
728 be_dmaa = cpu_to_be64((u64)skreq->sksg_dma_address);
729 cmdctxt = skreq->id + SKD_ID_INCR;
730
731 scsi_req = cmd_ptr;
732 scsi_req->hdr.tag = cmdctxt;
733 scsi_req->hdr.sg_list_dma_address = be_dmaa;
734
735 if (data_dir == READ)
736 skreq->sg_data_dir = SKD_DATA_DIR_CARD_TO_HOST;
737 else
738 skreq->sg_data_dir = SKD_DATA_DIR_HOST_TO_CARD;
739
740 if (io_flags & REQ_DISCARD) {
741 page = alloc_page(GFP_ATOMIC | __GFP_ZERO);
742 if (!page) {
743 pr_err("request_fn:Page allocation failed.\n");
744 skd_end_request(skdev, skreq, -ENOMEM);
745 break;
746 }
747 skreq->discard_page = 1;
748 skd_prep_discard_cdb(scsi_req, skreq, page, lba, count);
749
750 } else if (flush == SKD_FLUSH_ZERO_SIZE_FIRST) {
751 skd_prep_zerosize_flush_cdb(scsi_req, skreq);
752 SKD_ASSERT(skreq->flush_cmd == 1);
753
754 } else {
755 skd_prep_rw_cdb(scsi_req, data_dir, lba, count);
756 }
757
758 if (fua)
759 scsi_req->cdb[1] |= SKD_FUA_NV;
760
fcd37eb3 761 if (!req->bio)
e67f86b3
AB		 762 goto skip_sg;
763
764 error = skd_preop_sg_list(skdev, skreq);
765
766 if (error != 0) {
767 /*
768 * Complete the native request with error.
769 * Note that the request context is still at the
770 * head of the free list, and that the SoFIT request
771 * was encoded into the FIT msg buffer but the FIT
772 * msg length has not been updated. In short, the
773 * only resource that has been allocated but might
774 * not be used is that the FIT msg could be empty.
775 */
2e44b427 776 pr_debug("%s:%s:%d error Out\n",
777 skdev->name, __func__, __LINE__);
e67f86b3
AB		 778 skd_end_request(skdev, skreq, error);
779 continue;
780 }
781
782skip_sg:
783 scsi_req->hdr.sg_list_len_bytes =
784 cpu_to_be32(skreq->sg_byte_count);
785
786 /* Complete resource allocations. */
787 skdev->skreq_free_list = skreq->next;
788 skreq->state = SKD_REQ_STATE_BUSY;
789 skreq->id += SKD_ID_INCR;
790
791 skmsg->length += sizeof(struct skd_scsi_request);
792 fmh->num_protocol_cmds_coalesced++;
793
794 /*
795 * Update the active request counts.
796 * Capture the timeout timestamp.
797 */
798 skreq->timeout_stamp = skdev->timeout_stamp;
799 timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
800 skdev->timeout_slot[timo_slot]++;
801 skdev->in_flight++;
2e44b427 802 pr_debug("%s:%s:%d req=0x%x busy=%d\n",
803 skdev->name, __func__, __LINE__,
804 skreq->id, skdev->in_flight);
e67f86b3
AB		 805
806 /*
807 * If the FIT msg buffer is full send it.
808 */
809 if (skmsg->length >= SKD_N_FITMSG_BYTES ||
810 fmh->num_protocol_cmds_coalesced >= skd_max_req_per_msg) {
811 skd_send_fitmsg(skdev, skmsg);
812 skmsg = NULL;
813 fmh = NULL;
814 }
815 }
816
817 /*
818 * Is a FIT msg in progress? If it is empty put the buffer back
819 * on the free list. If it is non-empty send what we got.
820 * This minimizes latency when there are fewer requests than
821 * what fits in a FIT msg.
822 */
823 if (skmsg != NULL) {
824 /* Bigger than just a FIT msg header? */
825 if (skmsg->length > sizeof(struct fit_msg_hdr)) {
2e44b427 826 pr_debug("%s:%s:%d sending msg=%p, len %d\n",
827 skdev->name, __func__, __LINE__,
828 skmsg, skmsg->length);
e67f86b3
AB		 829 skd_send_fitmsg(skdev, skmsg);
830 } else {
831 /*
832 * The FIT msg is empty. It means we got started
833 * on the msg, but the requests were rejected.
834 */
835 skmsg->state = SKD_MSG_STATE_IDLE;
836 skmsg->id += SKD_ID_INCR;
837 skmsg->next = skdev->skmsg_free_list;
838 skdev->skmsg_free_list = skmsg;
839 }
840 skmsg = NULL;
841 fmh = NULL;
842 }
843
844 /*
845 * If req is non-NULL it means there is something to do but
846 * we are out of a resource.
847 */
fcd37eb3 848 if (req)
6a5ec65b 849 blk_stop_queue(skdev->queue);
e67f86b3
AB		 850}
851
38d4a1bb
MS		 852static void skd_end_request(struct skd_device *skdev,
853 struct skd_request_context *skreq, int error)
e67f86b3
AB		 854{
855 struct request *req = skreq->req;
856 unsigned int io_flags = req->cmd_flags;
857
858 if ((io_flags & REQ_DISCARD) &&
859 (skreq->discard_page == 1)) {
38d4a1bb 860 pr_debug("%s:%s:%d, free the page!",
2e44b427 861 skdev->name, __func__, __LINE__);
e67f86b3
AB		 862 free_page((unsigned long)req->buffer);
863 req->buffer = NULL;
864 }
865
866 if (unlikely(error)) {
867 struct request *req = skreq->req;
868 char *cmd = (rq_data_dir(req) == READ) ? "read" : "write";
869 u32 lba = (u32)blk_rq_pos(req);
870 u32 count = blk_rq_sectors(req);
871
872 pr_err("(%s): Error cmd=%s sect=%u count=%u id=0x%x\n",
873 skd_name(skdev), cmd, lba, count, skreq->id);
874 } else
2e44b427 875 pr_debug("%s:%s:%d id=0x%x error=%d\n",
876 skdev->name, __func__, __LINE__, skreq->id, error);
e67f86b3
AB		 877
878 __blk_end_request_all(skreq->req, error);
879}
880
fcd37eb3 881static int skd_preop_sg_list(struct skd_device *skdev,
38d4a1bb 882 struct skd_request_context *skreq)
e67f86b3
AB		 883{
884 struct request *req = skreq->req;
885 int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
886 int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
887 struct scatterlist *sg = &skreq->sg[0];
888 int n_sg;
889 int i;
890
891 skreq->sg_byte_count = 0;
892
893 /* SKD_ASSERT(skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD ||
894 skreq->sg_data_dir == SKD_DATA_DIR_CARD_TO_HOST); */
895
896 n_sg = blk_rq_map_sg(skdev->queue, req, sg);
897 if (n_sg <= 0)
898 return -EINVAL;
899
900 /*
901 * Map scatterlist to PCI bus addresses.
902 * Note PCI might change the number of entries.
903 */
904 n_sg = pci_map_sg(skdev->pdev, sg, n_sg, pci_dir);
905 if (n_sg <= 0)
906 return -EINVAL;
907
908 SKD_ASSERT(n_sg <= skdev->sgs_per_request);
909
910 skreq->n_sg = n_sg;
911
912 for (i = 0; i < n_sg; i++) {
913 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
914 u32 cnt = sg_dma_len(&sg[i]);
915 uint64_t dma_addr = sg_dma_address(&sg[i]);
916
917 sgd->control = FIT_SGD_CONTROL_NOT_LAST;
918 sgd->byte_count = cnt;
919 skreq->sg_byte_count += cnt;
920 sgd->host_side_addr = dma_addr;
921 sgd->dev_side_addr = 0;
922 }
923
924 skreq->sksg_list[n_sg - 1].next_desc_ptr = 0LL;
925 skreq->sksg_list[n_sg - 1].control = FIT_SGD_CONTROL_LAST;
926
927 if (unlikely(skdev->dbg_level > 1)) {
2e44b427 928 pr_debug("%s:%s:%d skreq=%x sksg_list=%p sksg_dma=%llx\n",
929 skdev->name, __func__, __LINE__,
930 skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
e67f86b3
AB		 931 for (i = 0; i < n_sg; i++) {
932 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
2e44b427 933 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
934 "addr=0x%llx next=0x%llx\n",
935 skdev->name, __func__, __LINE__,
936 i, sgd->byte_count, sgd->control,
937 sgd->host_side_addr, sgd->next_desc_ptr);
e67f86b3
AB		 938 }
939 }
940
941 return 0;
942}
943
fcd37eb3 944static void skd_postop_sg_list(struct skd_device *skdev,
38d4a1bb 945 struct skd_request_context *skreq)
e67f86b3
AB		 946{
947 int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
948 int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
949
950 /*
951 * restore the next ptr for next IO request so we
952 * don't have to set it every time.
953 */
954 skreq->sksg_list[skreq->n_sg - 1].next_desc_ptr =
955 skreq->sksg_dma_address +
956 ((skreq->n_sg) * sizeof(struct fit_sg_descriptor));
957 pci_unmap_sg(skdev->pdev, &skreq->sg[0], skreq->n_sg, pci_dir);
958}
959
e67f86b3
AB		 960static void skd_request_fn_not_online(struct request_queue *q)
961{
962 struct skd_device *skdev = q->queuedata;
963 int error;
964
965 SKD_ASSERT(skdev->state != SKD_DRVR_STATE_ONLINE);
966
967 skd_log_skdev(skdev, "req_not_online");
968 switch (skdev->state) {
969 case SKD_DRVR_STATE_PAUSING:
970 case SKD_DRVR_STATE_PAUSED:
971 case SKD_DRVR_STATE_STARTING:
972 case SKD_DRVR_STATE_RESTARTING:
973 case SKD_DRVR_STATE_WAIT_BOOT:
974 /* In case of starting, we haven't started the queue,
975 * so we can't get here... but requests are
976 * possibly hanging out waiting for us because we
977 * reported the dev/skd0 already. They'll wait
978 * forever if connect doesn't complete.
979 * What to do??? delay dev/skd0 ??
980 */
981 case SKD_DRVR_STATE_BUSY:
982 case SKD_DRVR_STATE_BUSY_IMMINENT:
983 case SKD_DRVR_STATE_BUSY_ERASE:
984 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
985 return;
986
987 case SKD_DRVR_STATE_BUSY_SANITIZE:
988 case SKD_DRVR_STATE_STOPPING:
989 case SKD_DRVR_STATE_SYNCING:
990 case SKD_DRVR_STATE_FAULT:
991 case SKD_DRVR_STATE_DISAPPEARED:
992 default:
993 error = -EIO;
994 break;
995 }
996
997 /* If we get here, terminate all pending block requeusts
998 * with EIO and any scsi pass thru with appropriate sense
999 */
1000
1001 skd_fail_all_pending(skdev);
1002}
1003
1004/*
1005 *****************************************************************************
1006 * TIMER
1007 *****************************************************************************
1008 */
1009
1010static void skd_timer_tick_not_online(struct skd_device *skdev);
1011
1012static void skd_timer_tick(ulong arg)
1013{
1014 struct skd_device *skdev = (struct skd_device *)arg;
1015
1016 u32 timo_slot;
1017 u32 overdue_timestamp;
1018 unsigned long reqflags;
1019 u32 state;
1020
1021 if (skdev->state == SKD_DRVR_STATE_FAULT)
1022 /* The driver has declared fault, and we want it to
1023 * stay that way until driver is reloaded.
1024 */
1025 return;
1026
1027 spin_lock_irqsave(&skdev->lock, reqflags);
1028
1029 state = SKD_READL(skdev, FIT_STATUS);
1030 state &= FIT_SR_DRIVE_STATE_MASK;
1031 if (state != skdev->drive_state)
1032 skd_isr_fwstate(skdev);
1033
1034 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
1035 skd_timer_tick_not_online(skdev);
1036 goto timer_func_out;
1037 }
1038 skdev->timeout_stamp++;
1039 timo_slot = skdev->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
1040
1041 /*
1042 * All requests that happened during the previous use of
1043 * this slot should be done by now. The previous use was
1044 * over 7 seconds ago.
1045 */
1046 if (skdev->timeout_slot[timo_slot] == 0)
1047 goto timer_func_out;
1048
1049 /* Something is overdue */
1050 overdue_timestamp = skdev->timeout_stamp - SKD_N_TIMEOUT_SLOT;
1051
2e44b427 1052 pr_debug("%s:%s:%d found %d timeouts, draining busy=%d\n",
1053 skdev->name, __func__, __LINE__,
1054 skdev->timeout_slot[timo_slot], skdev->in_flight);
e67f86b3
AB		 1055 pr_err("(%s): Overdue IOs (%d), busy %d\n",
1056 skd_name(skdev), skdev->timeout_slot[timo_slot],
1057 skdev->in_flight);
1058
1059 skdev->timer_countdown = SKD_DRAINING_TIMO;
1060 skdev->state = SKD_DRVR_STATE_DRAINING_TIMEOUT;
1061 skdev->timo_slot = timo_slot;
6a5ec65b 1062 blk_stop_queue(skdev->queue);
e67f86b3
AB		 1063
1064timer_func_out:
1065 mod_timer(&skdev->timer, (jiffies + HZ));
1066
1067 spin_unlock_irqrestore(&skdev->lock, reqflags);
1068}
1069
1070static void skd_timer_tick_not_online(struct skd_device *skdev)
1071{
1072 switch (skdev->state) {
1073 case SKD_DRVR_STATE_IDLE:
1074 case SKD_DRVR_STATE_LOAD:
1075 break;
1076 case SKD_DRVR_STATE_BUSY_SANITIZE:
2e44b427 1077 pr_debug("%s:%s:%d drive busy sanitize[%x], driver[%x]\n",
1078 skdev->name, __func__, __LINE__,
1079 skdev->drive_state, skdev->state);
e67f86b3
AB		 1080 /* If we've been in sanitize for 3 seconds, we figure we're not
1081 * going to get anymore completions, so recover requests now
1082 */
1083 if (skdev->timer_countdown > 0) {
1084 skdev->timer_countdown--;
1085 return;
1086 }
1087 skd_recover_requests(skdev, 0);
1088 break;
1089
1090 case SKD_DRVR_STATE_BUSY:
1091 case SKD_DRVR_STATE_BUSY_IMMINENT:
1092 case SKD_DRVR_STATE_BUSY_ERASE:
2e44b427 1093 pr_debug("%s:%s:%d busy[%x], countdown=%d\n",
1094 skdev->name, __func__, __LINE__,
1095 skdev->state, skdev->timer_countdown);
e67f86b3
AB		 1096 if (skdev->timer_countdown > 0) {
1097 skdev->timer_countdown--;
1098 return;
1099 }
2e44b427 1100 pr_debug("%s:%s:%d busy[%x], timedout=%d, restarting device.",
1101 skdev->name, __func__, __LINE__,
1102 skdev->state, skdev->timer_countdown);
e67f86b3
AB		 1103 skd_restart_device(skdev);
1104 break;
1105
1106 case SKD_DRVR_STATE_WAIT_BOOT:
1107 case SKD_DRVR_STATE_STARTING:
1108 if (skdev->timer_countdown > 0) {
1109 skdev->timer_countdown--;
1110 return;
1111 }
1112 /* For now, we fault the drive. Could attempt resets to
1113 * revcover at some point. */
1114 skdev->state = SKD_DRVR_STATE_FAULT;
1115
1116 pr_err("(%s): DriveFault Connect Timeout (%x)\n",
1117 skd_name(skdev), skdev->drive_state);
1118
1119 /*start the queue so we can respond with error to requests */
1120 /* wakeup anyone waiting for startup complete */
6a5ec65b 1121 blk_start_queue(skdev->queue);
e67f86b3
AB		 1122 skdev->gendisk_on = -1;
1123 wake_up_interruptible(&skdev->waitq);
1124 break;
1125
1126 case SKD_DRVR_STATE_ONLINE:
1127 /* shouldn't get here. */
1128 break;
1129
1130 case SKD_DRVR_STATE_PAUSING:
1131 case SKD_DRVR_STATE_PAUSED:
1132 break;
1133
1134 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
2e44b427 1135 pr_debug("%s:%s:%d "
1136 "draining busy [%d] tick[%d] qdb[%d] tmls[%d]\n",
1137 skdev->name, __func__, __LINE__,
1138 skdev->timo_slot,
1139 skdev->timer_countdown,
1140 skdev->in_flight,
1141 skdev->timeout_slot[skdev->timo_slot]);
e67f86b3
AB		 1142 /* if the slot has cleared we can let the I/O continue */
1143 if (skdev->timeout_slot[skdev->timo_slot] == 0) {
2e44b427 1144 pr_debug("%s:%s:%d Slot drained, starting queue.\n",
1145 skdev->name, __func__, __LINE__);
e67f86b3 1146 skdev->state = SKD_DRVR_STATE_ONLINE;
6a5ec65b 1147 blk_start_queue(skdev->queue);
e67f86b3
AB		 1148 return;
1149 }
1150 if (skdev->timer_countdown > 0) {
1151 skdev->timer_countdown--;
1152 return;
1153 }
1154 skd_restart_device(skdev);
1155 break;
1156
1157 case SKD_DRVR_STATE_RESTARTING:
1158 if (skdev->timer_countdown > 0) {
1159 skdev->timer_countdown--;
1160 return;
1161 }
1162 /* For now, we fault the drive. Could attempt resets to
1163 * revcover at some point. */
1164 skdev->state = SKD_DRVR_STATE_FAULT;
1165 pr_err("(%s): DriveFault Reconnect Timeout (%x)\n",
1166 skd_name(skdev), skdev->drive_state);
1167
1168 /*
1169 * Recovering does two things:
1170 * 1. completes IO with error
1171 * 2. reclaims dma resources
1172 * When is it safe to recover requests?
1173 * - if the drive state is faulted
1174 * - if the state is still soft reset after out timeout
1175 * - if the drive registers are dead (state = FF)
1176 * If it is "unsafe", we still need to recover, so we will
1177 * disable pci bus mastering and disable our interrupts.
1178 */
1179
1180 if ((skdev->drive_state == FIT_SR_DRIVE_SOFT_RESET) ||
1181 (skdev->drive_state == FIT_SR_DRIVE_FAULT) ||
1182 (skdev->drive_state == FIT_SR_DRIVE_STATE_MASK))
1183 /* It never came out of soft reset. Try to
1184 * recover the requests and then let them
1185 * fail. This is to mitigate hung processes. */
1186 skd_recover_requests(skdev, 0);
1187 else {
1188 pr_err("(%s): Disable BusMaster (%x)\n",
1189 skd_name(skdev), skdev->drive_state);
1190 pci_disable_device(skdev->pdev);
1191 skd_disable_interrupts(skdev);
1192 skd_recover_requests(skdev, 0);
1193 }
1194
1195 /*start the queue so we can respond with error to requests */
1196 /* wakeup anyone waiting for startup complete */
6a5ec65b 1197 blk_start_queue(skdev->queue);
e67f86b3
AB		 1198 skdev->gendisk_on = -1;
1199 wake_up_interruptible(&skdev->waitq);
1200 break;
1201
1202 case SKD_DRVR_STATE_RESUMING:
1203 case SKD_DRVR_STATE_STOPPING:
1204 case SKD_DRVR_STATE_SYNCING:
1205 case SKD_DRVR_STATE_FAULT:
1206 case SKD_DRVR_STATE_DISAPPEARED:
1207 default:
1208 break;
1209 }
1210}
1211
1212static int skd_start_timer(struct skd_device *skdev)
1213{
1214 int rc;
1215
1216 init_timer(&skdev->timer);
1217 setup_timer(&skdev->timer, skd_timer_tick, (ulong)skdev);
1218
1219 rc = mod_timer(&skdev->timer, (jiffies + HZ));
1220 if (rc)
1221 pr_err("%s: failed to start timer %d\n",
1222 __func__, rc);
1223 return rc;
1224}
1225
1226static void skd_kill_timer(struct skd_device *skdev)
1227{
1228 del_timer_sync(&skdev->timer);
1229}
1230
1231/*
1232 *****************************************************************************
1233 * IOCTL
1234 *****************************************************************************
1235 */
1236static int skd_ioctl_sg_io(struct skd_device *skdev,
1237 fmode_t mode, void __user *argp);
1238static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
1239 struct skd_sg_io *sksgio);
1240static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
1241 struct skd_sg_io *sksgio);
1242static int skd_sg_io_prep_buffering(struct skd_device *skdev,
1243 struct skd_sg_io *sksgio);
1244static int skd_sg_io_copy_buffer(struct skd_device *skdev,
1245 struct skd_sg_io *sksgio, int dxfer_dir);
1246static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
1247 struct skd_sg_io *sksgio);
1248static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio);
1249static int skd_sg_io_release_skspcl(struct skd_device *skdev,
1250 struct skd_sg_io *sksgio);
1251static int skd_sg_io_put_status(struct skd_device *skdev,
1252 struct skd_sg_io *sksgio);
1253
1254static void skd_complete_special(struct skd_device *skdev,
1255 volatile struct fit_completion_entry_v1
1256 *skcomp,
1257 volatile struct fit_comp_error_info *skerr,
1258 struct skd_special_context *skspcl);
1259
1260static int skd_bdev_ioctl(struct block_device *bdev, fmode_t mode,
1261 uint cmd_in, ulong arg)
1262{
1263 int rc = 0;
1264 struct gendisk *disk = bdev->bd_disk;
1265 struct skd_device *skdev = disk->private_data;
1266 void __user *p = (void *)arg;
1267
2e44b427 1268 pr_debug("%s:%s:%d %s: CMD[%s] ioctl mode 0x%x, cmd 0x%x arg %0lx\n",
1269 skdev->name, __func__, __LINE__,
1270 disk->disk_name, current->comm, mode, cmd_in, arg);
e67f86b3
AB		 1271
1272 if (!capable(CAP_SYS_ADMIN))
1273 return -EPERM;
1274
1275 switch (cmd_in) {
1276 case SG_SET_TIMEOUT:
1277 case SG_GET_TIMEOUT:
1278 case SG_GET_VERSION_NUM:
1279 rc = scsi_cmd_ioctl(disk->queue, disk, mode, cmd_in, p);
1280 break;
1281 case SG_IO:
1282 rc = skd_ioctl_sg_io(skdev, mode, p);
1283 break;
1284
1285 default:
1286 rc = -ENOTTY;
1287 break;
1288 }
1289
2e44b427 1290 pr_debug("%s:%s:%d %s: completion rc %d\n",
1291 skdev->name, __func__, __LINE__, disk->disk_name, rc);
e67f86b3
AB		 1292 return rc;
1293}
1294
1295static int skd_ioctl_sg_io(struct skd_device *skdev, fmode_t mode,
1296 void __user *argp)
1297{
1298 int rc;
1299 struct skd_sg_io sksgio;
1300
1301 memset(&sksgio, 0, sizeof(sksgio));
1302 sksgio.mode = mode;
1303 sksgio.argp = argp;
1304 sksgio.iov = &sksgio.no_iov_iov;
1305
1306 switch (skdev->state) {
1307 case SKD_DRVR_STATE_ONLINE:
1308 case SKD_DRVR_STATE_BUSY_IMMINENT:
1309 break;
1310
1311 default:
2e44b427 1312 pr_debug("%s:%s:%d drive not online\n",
1313 skdev->name, __func__, __LINE__);
e67f86b3
AB		 1314 rc = -ENXIO;
1315 goto out;
1316 }
1317
f721bb0d
AB		 1318 rc = skd_sg_io_get_and_check_args(skdev, &sksgio);
1319 if (rc)
1320 goto out;
1321
1322 rc = skd_sg_io_obtain_skspcl(skdev, &sksgio);
1323 if (rc)
1324 goto out;
1325
1326 rc = skd_sg_io_prep_buffering(skdev, &sksgio);
1327 if (rc)
1328 goto out;
1329
1330 rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_TO_DEV);
1331 if (rc)
e67f86b3
AB		 1332 goto out;
1333
f721bb0d
AB		 1334 rc = skd_sg_io_send_fitmsg(skdev, &sksgio);
1335 if (rc)
e67f86b3
AB		 1336 goto out;
1337
f721bb0d
AB		 1338 rc = skd_sg_io_await(skdev, &sksgio);
1339 if (rc)
1340 goto out;
1341
1342 rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_FROM_DEV);
1343 if (rc)
1344 goto out;
1345
1346 rc = skd_sg_io_put_status(skdev, &sksgio);
1347 if (rc)
e67f86b3
AB		 1348 goto out;
1349
1350 rc = 0;
1351
1352out:
1353 skd_sg_io_release_skspcl(skdev, &sksgio);
1354
1355 if (sksgio.iov != NULL && sksgio.iov != &sksgio.no_iov_iov)
1356 kfree(sksgio.iov);
1357 return rc;
1358}
1359
1360static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
1361 struct skd_sg_io *sksgio)
1362{
1363 struct sg_io_hdr *sgp = &sksgio->sg;
1364 int i, acc;
1365
1366 if (!access_ok(VERIFY_WRITE, sksgio->argp, sizeof(sg_io_hdr_t))) {
2e44b427 1367 pr_debug("%s:%s:%d access sg failed %p\n",
1368 skdev->name, __func__, __LINE__, sksgio->argp);
e67f86b3
AB		 1369 return -EFAULT;
1370 }
1371
1372 if (__copy_from_user(sgp, sksgio->argp, sizeof(sg_io_hdr_t))) {
2e44b427 1373 pr_debug("%s:%s:%d copy_from_user sg failed %p\n",
1374 skdev->name, __func__, __LINE__, sksgio->argp);
e67f86b3
AB		 1375 return -EFAULT;
1376 }
1377
1378 if (sgp->interface_id != SG_INTERFACE_ID_ORIG) {
2e44b427 1379 pr_debug("%s:%s:%d interface_id invalid 0x%x\n",
1380 skdev->name, __func__, __LINE__, sgp->interface_id);
e67f86b3
AB		 1381 return -EINVAL;
1382 }
1383
1384 if (sgp->cmd_len > sizeof(sksgio->cdb)) {
2e44b427 1385 pr_debug("%s:%s:%d cmd_len invalid %d\n",
1386 skdev->name, __func__, __LINE__, sgp->cmd_len);
e67f86b3
AB		 1387 return -EINVAL;
1388 }
1389
1390 if (sgp->iovec_count > 256) {
2e44b427 1391 pr_debug("%s:%s:%d iovec_count invalid %d\n",
1392 skdev->name, __func__, __LINE__, sgp->iovec_count);
e67f86b3
AB		 1393 return -EINVAL;
1394 }
1395
1396 if (sgp->dxfer_len > (PAGE_SIZE * SKD_N_SG_PER_SPECIAL)) {
2e44b427 1397 pr_debug("%s:%s:%d dxfer_len invalid %d\n",
1398 skdev->name, __func__, __LINE__, sgp->dxfer_len);
e67f86b3
AB		 1399 return -EINVAL;
1400 }
1401
1402 switch (sgp->dxfer_direction) {
1403 case SG_DXFER_NONE:
1404 acc = -1;
1405 break;
1406
1407 case SG_DXFER_TO_DEV:
1408 acc = VERIFY_READ;
1409 break;
1410
1411 case SG_DXFER_FROM_DEV:
1412 case SG_DXFER_TO_FROM_DEV:
1413 acc = VERIFY_WRITE;
1414 break;
1415
1416 default:
2e44b427 1417 pr_debug("%s:%s:%d dxfer_dir invalid %d\n",
1418 skdev->name, __func__, __LINE__, sgp->dxfer_direction);
e67f86b3
AB		 1419 return -EINVAL;
1420 }
1421
1422 if (copy_from_user(sksgio->cdb, sgp->cmdp, sgp->cmd_len)) {
2e44b427 1423 pr_debug("%s:%s:%d copy_from_user cmdp failed %p\n",
1424 skdev->name, __func__, __LINE__, sgp->cmdp);
e67f86b3
AB		 1425 return -EFAULT;
1426 }
1427
1428 if (sgp->mx_sb_len != 0) {
1429 if (!access_ok(VERIFY_WRITE, sgp->sbp, sgp->mx_sb_len)) {
2e44b427 1430 pr_debug("%s:%s:%d access sbp failed %p\n",
1431 skdev->name, __func__, __LINE__, sgp->sbp);
e67f86b3
AB		 1432 return -EFAULT;
1433 }
1434 }
1435
1436 if (sgp->iovec_count == 0) {
1437 sksgio->iov[0].iov_base = sgp->dxferp;
1438 sksgio->iov[0].iov_len = sgp->dxfer_len;
1439 sksgio->iovcnt = 1;
1440 sksgio->dxfer_len = sgp->dxfer_len;
1441 } else {
1442 struct sg_iovec *iov;
1443 uint nbytes = sizeof(*iov) * sgp->iovec_count;
1444 size_t iov_data_len;
1445
1446 iov = kmalloc(nbytes, GFP_KERNEL);
1447 if (iov == NULL) {
2e44b427 1448 pr_debug("%s:%s:%d alloc iovec failed %d\n",
1449 skdev->name, __func__, __LINE__,
1450 sgp->iovec_count);
e67f86b3
AB		 1451 return -ENOMEM;
1452 }
1453 sksgio->iov = iov;
1454 sksgio->iovcnt = sgp->iovec_count;
1455
1456 if (copy_from_user(iov, sgp->dxferp, nbytes)) {
2e44b427 1457 pr_debug("%s:%s:%d copy_from_user iovec failed %p\n",
1458 skdev->name, __func__, __LINE__, sgp->dxferp);
e67f86b3
AB		 1459 return -EFAULT;
1460 }
1461
1462 /*
1463 * Sum up the vecs, making sure they don't overflow
1464 */
1465 iov_data_len = 0;
1466 for (i = 0; i < sgp->iovec_count; i++) {
1467 if (iov_data_len + iov[i].iov_len < iov_data_len)
1468 return -EINVAL;
1469 iov_data_len += iov[i].iov_len;
1470 }
1471
1472 /* SG_IO howto says that the shorter of the two wins */
1473 if (sgp->dxfer_len < iov_data_len) {
1474 sksgio->iovcnt = iov_shorten((struct iovec *)iov,
1475 sgp->iovec_count,
1476 sgp->dxfer_len);
1477 sksgio->dxfer_len = sgp->dxfer_len;
1478 } else
1479 sksgio->dxfer_len = iov_data_len;
1480 }
1481
1482 if (sgp->dxfer_direction != SG_DXFER_NONE) {
1483 struct sg_iovec *iov = sksgio->iov;
1484 for (i = 0; i < sksgio->iovcnt; i++, iov++) {
1485 if (!access_ok(acc, iov->iov_base, iov->iov_len)) {
2e44b427 1486 pr_debug("%s:%s:%d access data failed %p/%d\n",
1487 skdev->name, __func__, __LINE__,
1488 iov->iov_base, (int)iov->iov_len);
e67f86b3
AB		 1489 return -EFAULT;
1490 }
1491 }
1492 }
1493
1494 return 0;
1495}
1496
1497static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
1498 struct skd_sg_io *sksgio)
1499{
1500 struct skd_special_context *skspcl = NULL;
1501 int rc;
1502
38d4a1bb 1503 for (;;) {
e67f86b3
AB		 1504 ulong flags;
1505
1506 spin_lock_irqsave(&skdev->lock, flags);
1507 skspcl = skdev->skspcl_free_list;
1508 if (skspcl != NULL) {
1509 skdev->skspcl_free_list =
1510 (struct skd_special_context *)skspcl->req.next;
1511 skspcl->req.id += SKD_ID_INCR;
1512 skspcl->req.state = SKD_REQ_STATE_SETUP;
1513 skspcl->orphaned = 0;
1514 skspcl->req.n_sg = 0;
1515 }
1516 spin_unlock_irqrestore(&skdev->lock, flags);
1517
1518 if (skspcl != NULL) {
1519 rc = 0;
1520 break;
1521 }
1522
2e44b427 1523 pr_debug("%s:%s:%d blocking\n",
1524 skdev->name, __func__, __LINE__);
e67f86b3
AB		 1525
1526 rc = wait_event_interruptible_timeout(
1527 skdev->waitq,
1528 (skdev->skspcl_free_list != NULL),
1529 msecs_to_jiffies(sksgio->sg.timeout));
1530
2e44b427 1531 pr_debug("%s:%s:%d unblocking, rc=%d\n",
1532 skdev->name, __func__, __LINE__, rc);
e67f86b3
AB		 1533
1534 if (rc <= 0) {
1535 if (rc == 0)
1536 rc = -ETIMEDOUT;
1537 else
1538 rc = -EINTR;
1539 break;
1540 }
1541 /*
1542 * If we get here rc > 0 meaning the timeout to
1543 * wait_event_interruptible_timeout() had time left, hence the
1544 * sought event -- non-empty free list -- happened.
1545 * Retry the allocation.
1546 */
1547 }
1548 sksgio->skspcl = skspcl;
1549
1550 return rc;
1551}
1552
1553static int skd_skreq_prep_buffering(struct skd_device *skdev,
1554 struct skd_request_context *skreq,
1555 u32 dxfer_len)
1556{
1557 u32 resid = dxfer_len;
1558
1559 /*
1560 * The DMA engine must have aligned addresses and byte counts.
1561 */
1562 resid += (-resid) & 3;
1563 skreq->sg_byte_count = resid;
1564
1565 skreq->n_sg = 0;
1566
1567 while (resid > 0) {
1568 u32 nbytes = PAGE_SIZE;
1569 u32 ix = skreq->n_sg;
1570 struct scatterlist *sg = &skreq->sg[ix];
1571 struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
1572 struct page *page;
1573
1574 if (nbytes > resid)
1575 nbytes = resid;
1576
1577 page = alloc_page(GFP_KERNEL);
1578 if (page == NULL)
1579 return -ENOMEM;
1580
1581 sg_set_page(sg, page, nbytes, 0);
1582
1583 /* TODO: This should be going through a pci_???()
1584 * routine to do proper mapping. */
1585 sksg->control = FIT_SGD_CONTROL_NOT_LAST;
1586 sksg->byte_count = nbytes;
1587
1588 sksg->host_side_addr = sg_phys(sg);
1589
1590 sksg->dev_side_addr = 0;
1591 sksg->next_desc_ptr = skreq->sksg_dma_address +
1592 (ix + 1) * sizeof(*sksg);
1593
1594 skreq->n_sg++;
1595 resid -= nbytes;
1596 }
1597
1598 if (skreq->n_sg > 0) {
1599 u32 ix = skreq->n_sg - 1;
1600 struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
1601
1602 sksg->control = FIT_SGD_CONTROL_LAST;
1603 sksg->next_desc_ptr = 0;
1604 }
1605
1606 if (unlikely(skdev->dbg_level > 1)) {
1607 u32 i;
1608
2e44b427 1609 pr_debug("%s:%s:%d skreq=%x sksg_list=%p sksg_dma=%llx\n",
1610 skdev->name, __func__, __LINE__,
1611 skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
e67f86b3
AB		 1612 for (i = 0; i < skreq->n_sg; i++) {
1613 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
1614
2e44b427 1615 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
1616 "addr=0x%llx next=0x%llx\n",
1617 skdev->name, __func__, __LINE__,
1618 i, sgd->byte_count, sgd->control,
1619 sgd->host_side_addr, sgd->next_desc_ptr);
e67f86b3
AB		 1620 }
1621 }
1622
1623 return 0;
1624}
1625
1626static int skd_sg_io_prep_buffering(struct skd_device *skdev,
1627 struct skd_sg_io *sksgio)
1628{
1629 struct skd_special_context *skspcl = sksgio->skspcl;
1630 struct skd_request_context *skreq = &skspcl->req;
1631 u32 dxfer_len = sksgio->dxfer_len;
1632 int rc;
1633
1634 rc = skd_skreq_prep_buffering(skdev, skreq, dxfer_len);
1635 /*
1636 * Eventually, errors or not, skd_release_special() is called
1637 * to recover allocations including partial allocations.
1638 */
1639 return rc;
1640}
1641
1642static int skd_sg_io_copy_buffer(struct skd_device *skdev,
1643 struct skd_sg_io *sksgio, int dxfer_dir)
1644{
1645 struct skd_special_context *skspcl = sksgio->skspcl;
1646 u32 iov_ix = 0;
1647 struct sg_iovec curiov;
1648 u32 sksg_ix = 0;
1649 u8 *bufp = NULL;
1650 u32 buf_len = 0;
1651 u32 resid = sksgio->dxfer_len;
1652 int rc;
1653
1654 curiov.iov_len = 0;
1655 curiov.iov_base = NULL;
1656
1657 if (dxfer_dir != sksgio->sg.dxfer_direction) {
1658 if (dxfer_dir != SG_DXFER_TO_DEV ||
1659 sksgio->sg.dxfer_direction != SG_DXFER_TO_FROM_DEV)
1660 return 0;
1661 }
1662
1663 while (resid > 0) {
1664 u32 nbytes = PAGE_SIZE;
1665
1666 if (curiov.iov_len == 0) {
1667 curiov = sksgio->iov[iov_ix++];
1668 continue;
1669 }
1670
1671 if (buf_len == 0) {
1672 struct page *page;
1673 page = sg_page(&skspcl->req.sg[sksg_ix++]);
1674 bufp = page_address(page);
1675 buf_len = PAGE_SIZE;
1676 }
1677
1678 nbytes = min_t(u32, nbytes, resid);
1679 nbytes = min_t(u32, nbytes, curiov.iov_len);
1680 nbytes = min_t(u32, nbytes, buf_len);
1681
1682 if (dxfer_dir == SG_DXFER_TO_DEV)
1683 rc = __copy_from_user(bufp, curiov.iov_base, nbytes);
1684 else
1685 rc = __copy_to_user(curiov.iov_base, bufp, nbytes);
1686
1687 if (rc)
1688 return -EFAULT;
1689
1690 resid -= nbytes;
1691 curiov.iov_len -= nbytes;
1692 curiov.iov_base += nbytes;
1693 buf_len -= nbytes;
1694 }
1695
1696 return 0;
1697}
1698
1699static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
1700 struct skd_sg_io *sksgio)
1701{
1702 struct skd_special_context *skspcl = sksgio->skspcl;
1703 struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
1704 struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
1705
1706 memset(skspcl->msg_buf, 0, SKD_N_SPECIAL_FITMSG_BYTES);
1707
1708 /* Initialize the FIT msg header */
1709 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
1710 fmh->num_protocol_cmds_coalesced = 1;
1711
1712 /* Initialize the SCSI request */
1713 if (sksgio->sg.dxfer_direction != SG_DXFER_NONE)
1714 scsi_req->hdr.sg_list_dma_address =
1715 cpu_to_be64(skspcl->req.sksg_dma_address);
1716 scsi_req->hdr.tag = skspcl->req.id;
1717 scsi_req->hdr.sg_list_len_bytes =
1718 cpu_to_be32(skspcl->req.sg_byte_count);
1719 memcpy(scsi_req->cdb, sksgio->cdb, sizeof(scsi_req->cdb));
1720
1721 skspcl->req.state = SKD_REQ_STATE_BUSY;
1722 skd_send_special_fitmsg(skdev, skspcl);
1723
1724 return 0;
1725}
1726
1727static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio)
1728{
1729 unsigned long flags;
1730 int rc;
1731
1732 rc = wait_event_interruptible_timeout(skdev->waitq,
1733 (sksgio->skspcl->req.state !=
1734 SKD_REQ_STATE_BUSY),
1735 msecs_to_jiffies(sksgio->sg.
1736 timeout));
1737
1738 spin_lock_irqsave(&skdev->lock, flags);
1739
1740 if (sksgio->skspcl->req.state == SKD_REQ_STATE_ABORTED) {
2e44b427 1741 pr_debug("%s:%s:%d skspcl %p aborted\n",
1742 skdev->name, __func__, __LINE__, sksgio->skspcl);
e67f86b3
AB		 1743
1744 /* Build check cond, sense and let command finish. */
1745 /* For a timeout, we must fabricate completion and sense
1746 * data to complete the command */
1747 sksgio->skspcl->req.completion.status =
1748 SAM_STAT_CHECK_CONDITION;
1749
1750 memset(&sksgio->skspcl->req.err_info, 0,
1751 sizeof(sksgio->skspcl->req.err_info));
1752 sksgio->skspcl->req.err_info.type = 0x70;
1753 sksgio->skspcl->req.err_info.key = ABORTED_COMMAND;
1754 sksgio->skspcl->req.err_info.code = 0x44;
1755 sksgio->skspcl->req.err_info.qual = 0;
1756 rc = 0;
1757 } else if (sksgio->skspcl->req.state != SKD_REQ_STATE_BUSY)
1758 /* No longer on the adapter. We finish. */
1759 rc = 0;
1760 else {
1761 /* Something's gone wrong. Still busy. Timeout or
1762 * user interrupted (control-C). Mark as an orphan
1763 * so it will be disposed when completed. */
1764 sksgio->skspcl->orphaned = 1;
1765 sksgio->skspcl = NULL;
1766 if (rc == 0) {
2e44b427 1767 pr_debug("%s:%s:%d timed out %p (%u ms)\n",
1768 skdev->name, __func__, __LINE__,
1769 sksgio, sksgio->sg.timeout);
e67f86b3
AB		 1770 rc = -ETIMEDOUT;
1771 } else {
2e44b427 1772 pr_debug("%s:%s:%d cntlc %p\n",
1773 skdev->name, __func__, __LINE__, sksgio);
e67f86b3
AB		 1774 rc = -EINTR;
1775 }
1776 }
1777
1778 spin_unlock_irqrestore(&skdev->lock, flags);
1779
1780 return rc;
1781}
1782
1783static int skd_sg_io_put_status(struct skd_device *skdev,
1784 struct skd_sg_io *sksgio)
1785{
1786 struct sg_io_hdr *sgp = &sksgio->sg;
1787 struct skd_special_context *skspcl = sksgio->skspcl;
1788 int resid = 0;
1789
1790 u32 nb = be32_to_cpu(skspcl->req.completion.num_returned_bytes);
1791
1792 sgp->status = skspcl->req.completion.status;
1793 resid = sksgio->dxfer_len - nb;
1794
1795 sgp->masked_status = sgp->status & STATUS_MASK;
1796 sgp->msg_status = 0;
1797 sgp->host_status = 0;
1798 sgp->driver_status = 0;
1799 sgp->resid = resid;
1800 if (sgp->masked_status || sgp->host_status || sgp->driver_status)
1801 sgp->info |= SG_INFO_CHECK;
1802
2e44b427 1803 pr_debug("%s:%s:%d status %x masked %x resid 0x%x\n",
1804 skdev->name, __func__, __LINE__,
1805 sgp->status, sgp->masked_status, sgp->resid);
e67f86b3
AB		 1806
1807 if (sgp->masked_status == SAM_STAT_CHECK_CONDITION) {
1808 if (sgp->mx_sb_len > 0) {
1809 struct fit_comp_error_info *ei = &skspcl->req.err_info;
1810 u32 nbytes = sizeof(*ei);
1811
1812 nbytes = min_t(u32, nbytes, sgp->mx_sb_len);
1813
1814 sgp->sb_len_wr = nbytes;
1815
1816 if (__copy_to_user(sgp->sbp, ei, nbytes)) {
2e44b427 1817 pr_debug("%s:%s:%d copy_to_user sense failed %p\n",
1818 skdev->name, __func__, __LINE__,
1819 sgp->sbp);
e67f86b3
AB		 1820 return -EFAULT;
1821 }
1822 }
1823 }
1824
1825 if (__copy_to_user(sksgio->argp, sgp, sizeof(sg_io_hdr_t))) {
2e44b427 1826 pr_debug("%s:%s:%d copy_to_user sg failed %p\n",
1827 skdev->name, __func__, __LINE__, sksgio->argp);
e67f86b3
AB		 1828 return -EFAULT;
1829 }
1830
1831 return 0;
1832}
1833
1834static int skd_sg_io_release_skspcl(struct skd_device *skdev,
1835 struct skd_sg_io *sksgio)
1836{
1837 struct skd_special_context *skspcl = sksgio->skspcl;
1838
1839 if (skspcl != NULL) {
1840 ulong flags;
1841
1842 sksgio->skspcl = NULL;
1843
1844 spin_lock_irqsave(&skdev->lock, flags);
1845 skd_release_special(skdev, skspcl);
1846 spin_unlock_irqrestore(&skdev->lock, flags);
1847 }
1848
1849 return 0;
1850}
1851
1852/*
1853 *****************************************************************************
1854 * INTERNAL REQUESTS -- generated by driver itself
1855 *****************************************************************************
1856 */
1857
1858static int skd_format_internal_skspcl(struct skd_device *skdev)
1859{
1860 struct skd_special_context *skspcl = &skdev->internal_skspcl;
1861 struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
1862 struct fit_msg_hdr *fmh;
1863 uint64_t dma_address;
1864 struct skd_scsi_request *scsi;
1865
1866 fmh = (struct fit_msg_hdr *)&skspcl->msg_buf[0];
1867 fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
1868 fmh->num_protocol_cmds_coalesced = 1;
1869
1870 scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
1871 memset(scsi, 0, sizeof(*scsi));
1872 dma_address = skspcl->req.sksg_dma_address;
1873 scsi->hdr.sg_list_dma_address = cpu_to_be64(dma_address);
1874 sgd->control = FIT_SGD_CONTROL_LAST;
1875 sgd->byte_count = 0;
1876 sgd->host_side_addr = skspcl->db_dma_address;
1877 sgd->dev_side_addr = 0;
1878 sgd->next_desc_ptr = 0LL;
1879
1880 return 1;
1881}
1882
1883#define WR_BUF_SIZE SKD_N_INTERNAL_BYTES
1884
1885static void skd_send_internal_skspcl(struct skd_device *skdev,
1886 struct skd_special_context *skspcl,
1887 u8 opcode)
1888{
1889 struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
1890 struct skd_scsi_request *scsi;
1891 unsigned char *buf = skspcl->data_buf;
1892 int i;
1893
1894 if (skspcl->req.state != SKD_REQ_STATE_IDLE)
1895 /*
1896 * A refresh is already in progress.
1897 * Just wait for it to finish.
1898 */
1899 return;
1900
1901 SKD_ASSERT((skspcl->req.id & SKD_ID_INCR) == 0);
1902 skspcl->req.state = SKD_REQ_STATE_BUSY;
1903 skspcl->req.id += SKD_ID_INCR;
1904
1905 scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
1906 scsi->hdr.tag = skspcl->req.id;
1907
1908 memset(scsi->cdb, 0, sizeof(scsi->cdb));
1909
1910 switch (opcode) {
1911 case TEST_UNIT_READY:
1912 scsi->cdb[0] = TEST_UNIT_READY;
1913 sgd->byte_count = 0;
1914 scsi->hdr.sg_list_len_bytes = 0;
1915 break;
1916
1917 case READ_CAPACITY:
1918 scsi->cdb[0] = READ_CAPACITY;
1919 sgd->byte_count = SKD_N_READ_CAP_BYTES;
1920 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1921 break;
1922
1923 case INQUIRY:
1924 scsi->cdb[0] = INQUIRY;
1925 scsi->cdb[1] = 0x01; /* evpd */
1926 scsi->cdb[2] = 0x80; /* serial number page */
1927 scsi->cdb[4] = 0x10;
1928 sgd->byte_count = 16;
1929 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1930 break;
1931
1932 case SYNCHRONIZE_CACHE:
1933 scsi->cdb[0] = SYNCHRONIZE_CACHE;
1934 sgd->byte_count = 0;
1935 scsi->hdr.sg_list_len_bytes = 0;
1936 break;
1937
1938 case WRITE_BUFFER:
1939 scsi->cdb[0] = WRITE_BUFFER;
1940 scsi->cdb[1] = 0x02;
1941 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
1942 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
1943 sgd->byte_count = WR_BUF_SIZE;
1944 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1945 /* fill incrementing byte pattern */
1946 for (i = 0; i < sgd->byte_count; i++)
1947 buf[i] = i & 0xFF;
1948 break;
1949
1950 case READ_BUFFER:
1951 scsi->cdb[0] = READ_BUFFER;
1952 scsi->cdb[1] = 0x02;
1953 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
1954 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
1955 sgd->byte_count = WR_BUF_SIZE;
1956 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1957 memset(skspcl->data_buf, 0, sgd->byte_count);
1958 break;
1959
1960 default:
1961 SKD_ASSERT("Don't know what to send");
1962 return;
1963
1964 }
1965 skd_send_special_fitmsg(skdev, skspcl);
1966}
1967
1968static void skd_refresh_device_data(struct skd_device *skdev)
1969{
1970 struct skd_special_context *skspcl = &skdev->internal_skspcl;
1971
1972 skd_send_internal_skspcl(skdev, skspcl, TEST_UNIT_READY);
1973}
1974
1975static int skd_chk_read_buf(struct skd_device *skdev,
1976 struct skd_special_context *skspcl)
1977{
1978 unsigned char *buf = skspcl->data_buf;
1979 int i;
1980
1981 /* check for incrementing byte pattern */
1982 for (i = 0; i < WR_BUF_SIZE; i++)
1983 if (buf[i] != (i & 0xFF))
1984 return 1;
1985
1986 return 0;
1987}
1988
1989static void skd_log_check_status(struct skd_device *skdev, u8 status, u8 key,
1990 u8 code, u8 qual, u8 fruc)
1991{
1992 /* If the check condition is of special interest, log a message */
1993 if ((status == SAM_STAT_CHECK_CONDITION) && (key == 0x02)
1994 && (code == 0x04) && (qual == 0x06)) {
1995 pr_err("(%s): *** LOST_WRITE_DATA ERROR *** key/asc/"
1996 "ascq/fruc %02x/%02x/%02x/%02x\n",
1997 skd_name(skdev), key, code, qual, fruc);
1998 }
1999}
2000
2001static void skd_complete_internal(struct skd_device *skdev,
2002 volatile struct fit_completion_entry_v1
2003 *skcomp,
2004 volatile struct fit_comp_error_info *skerr,
2005 struct skd_special_context *skspcl)
2006{
2007 u8 *buf = skspcl->data_buf;
2008 u8 status;
2009 int i;
2010 struct skd_scsi_request *scsi =
2011 (struct skd_scsi_request *)&skspcl->msg_buf[64];
2012
2013 SKD_ASSERT(skspcl == &skdev->internal_skspcl);
2014
2e44b427 2015 pr_debug("%s:%s:%d complete internal %x\n",
2016 skdev->name, __func__, __LINE__, scsi->cdb[0]);
e67f86b3
AB		 2017
2018 skspcl->req.completion = *skcomp;
2019 skspcl->req.state = SKD_REQ_STATE_IDLE;
2020 skspcl->req.id += SKD_ID_INCR;
2021
2022 status = skspcl->req.completion.status;
2023
2024 skd_log_check_status(skdev, status, skerr->key, skerr->code,
2025 skerr->qual, skerr->fruc);
2026
2027 switch (scsi->cdb[0]) {
2028 case TEST_UNIT_READY:
2029 if (status == SAM_STAT_GOOD)
2030 skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
2031 else if ((status == SAM_STAT_CHECK_CONDITION) &&
2032 (skerr->key == MEDIUM_ERROR))
2033 skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
2034 else {
2035 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2e44b427 2036 pr_debug("%s:%s:%d TUR failed, don't send anymore state 0x%x\n",
2037 skdev->name, __func__, __LINE__,
2038 skdev->state);
e67f86b3
AB		 2039 return;
2040 }
2e44b427 2041 pr_debug("%s:%s:%d **** TUR failed, retry skerr\n",
2042 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2043 skd_send_internal_skspcl(skdev, skspcl, 0x00);
2044 }
2045 break;
2046
2047 case WRITE_BUFFER:
2048 if (status == SAM_STAT_GOOD)
2049 skd_send_internal_skspcl(skdev, skspcl, READ_BUFFER);
2050 else {
2051 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2e44b427 2052 pr_debug("%s:%s:%d write buffer failed, don't send anymore state 0x%x\n",
2053 skdev->name, __func__, __LINE__,
2054 skdev->state);
e67f86b3
AB		 2055 return;
2056 }
2e44b427 2057 pr_debug("%s:%s:%d **** write buffer failed, retry skerr\n",
2058 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2059 skd_send_internal_skspcl(skdev, skspcl, 0x00);
2060 }
2061 break;
2062
2063 case READ_BUFFER:
2064 if (status == SAM_STAT_GOOD) {
2065 if (skd_chk_read_buf(skdev, skspcl) == 0)
2066 skd_send_internal_skspcl(skdev, skspcl,
2067 READ_CAPACITY);
2068 else {
2069 pr_err(
2070 "(%s):*** W/R Buffer mismatch %d ***\n",
2071 skd_name(skdev), skdev->connect_retries);
2072 if (skdev->connect_retries <
2073 SKD_MAX_CONNECT_RETRIES) {
2074 skdev->connect_retries++;
2075 skd_soft_reset(skdev);
2076 } else {
2077 pr_err(
2078 "(%s): W/R Buffer Connect Error\n",
2079 skd_name(skdev));
2080 return;
2081 }
2082 }
2083
2084 } else {
2085 if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2e44b427 2086 pr_debug("%s:%s:%d "
2087 "read buffer failed, don't send anymore state 0x%x\n",
2088 skdev->name, __func__, __LINE__,
2089 skdev->state);
e67f86b3
AB		 2090 return;
2091 }
2e44b427 2092 pr_debug("%s:%s:%d "
2093 "**** read buffer failed, retry skerr\n",
2094 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2095 skd_send_internal_skspcl(skdev, skspcl, 0x00);
2096 }
2097 break;
2098
2099 case READ_CAPACITY:
2100 skdev->read_cap_is_valid = 0;
2101 if (status == SAM_STAT_GOOD) {
2102 skdev->read_cap_last_lba =
2103 (buf[0] << 24) | (buf[1] << 16) |
2104 (buf[2] << 8) | buf[3];
2105 skdev->read_cap_blocksize =
2106 (buf[4] << 24) | (buf[5] << 16) |
2107 (buf[6] << 8) | buf[7];
2108
2e44b427 2109 pr_debug("%s:%s:%d last lba %d, bs %d\n",
2110 skdev->name, __func__, __LINE__,
2111 skdev->read_cap_last_lba,
2112 skdev->read_cap_blocksize);
e67f86b3
AB		 2113
2114 set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
2115
2116 skdev->read_cap_is_valid = 1;
2117
2118 skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
2119 } else if ((status == SAM_STAT_CHECK_CONDITION) &&
2120 (skerr->key == MEDIUM_ERROR)) {
2121 skdev->read_cap_last_lba = ~0;
2122 set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
2e44b427 2123 pr_debug("%s:%s:%d "
2124 "**** MEDIUM ERROR caused READCAP to fail, ignore failure and continue to inquiry\n",
2125 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2126 skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
2127 } else {
2e44b427 2128 pr_debug("%s:%s:%d **** READCAP failed, retry TUR\n",
2129 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2130 skd_send_internal_skspcl(skdev, skspcl,
2131 TEST_UNIT_READY);
2132 }
2133 break;
2134
2135 case INQUIRY:
2136 skdev->inquiry_is_valid = 0;
2137 if (status == SAM_STAT_GOOD) {
2138 skdev->inquiry_is_valid = 1;
2139
2140 for (i = 0; i < 12; i++)
2141 skdev->inq_serial_num[i] = buf[i + 4];
2142 skdev->inq_serial_num[12] = 0;
2143 }
2144
2145 if (skd_unquiesce_dev(skdev) < 0)
2e44b427 2146 pr_debug("%s:%s:%d **** failed, to ONLINE device\n",
2147 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2148 /* connection is complete */
2149 skdev->connect_retries = 0;
2150 break;
2151
2152 case SYNCHRONIZE_CACHE:
2153 if (status == SAM_STAT_GOOD)
2154 skdev->sync_done = 1;
2155 else
2156 skdev->sync_done = -1;
2157 wake_up_interruptible(&skdev->waitq);
2158 break;
2159
2160 default:
2161 SKD_ASSERT("we didn't send this");
2162 }
2163}
2164
2165/*
2166 *****************************************************************************
2167 * FIT MESSAGES
2168 *****************************************************************************
2169 */
2170
2171static void skd_send_fitmsg(struct skd_device *skdev,
2172 struct skd_fitmsg_context *skmsg)
2173{
2174 u64 qcmd;
2175 struct fit_msg_hdr *fmh;
2176
2e44b427 2177 pr_debug("%s:%s:%d dma address 0x%llx, busy=%d\n",
2178 skdev->name, __func__, __LINE__,
2179 skmsg->mb_dma_address, skdev->in_flight);
2180 pr_debug("%s:%s:%d msg_buf 0x%p, offset %x\n",
2181 skdev->name, __func__, __LINE__,
2182 skmsg->msg_buf, skmsg->offset);
e67f86b3
AB		 2183
2184 qcmd = skmsg->mb_dma_address;
2185 qcmd |= FIT_QCMD_QID_NORMAL;
2186
2187 fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
2188 skmsg->outstanding = fmh->num_protocol_cmds_coalesced;
2189
2190 if (unlikely(skdev->dbg_level > 1)) {
2191 u8 *bp = (u8 *)skmsg->msg_buf;
2192 int i;
2193 for (i = 0; i < skmsg->length; i += 8) {
2e44b427 2194 pr_debug("%s:%s:%d msg[%2d] %02x %02x %02x %02x "
2195 "%02x %02x %02x %02x\n",
2196 skdev->name, __func__, __LINE__,
2197 i, bp[i + 0], bp[i + 1], bp[i + 2],
2198 bp[i + 3], bp[i + 4], bp[i + 5],
2199 bp[i + 6], bp[i + 7]);
e67f86b3
AB		 2200 if (i == 0)
2201 i = 64 - 8;
2202 }
2203 }
2204
2205 if (skmsg->length > 256)
2206 qcmd |= FIT_QCMD_MSGSIZE_512;
2207 else if (skmsg->length > 128)
2208 qcmd |= FIT_QCMD_MSGSIZE_256;
2209 else if (skmsg->length > 64)
2210 qcmd |= FIT_QCMD_MSGSIZE_128;
2211 else
2212 /*
2213 * This makes no sense because the FIT msg header is
2214 * 64 bytes. If the msg is only 64 bytes long it has
2215 * no payload.
2216 */
2217 qcmd |= FIT_QCMD_MSGSIZE_64;
2218
2219 SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
2220
2221}
2222
2223static void skd_send_special_fitmsg(struct skd_device *skdev,
2224 struct skd_special_context *skspcl)
2225{
2226 u64 qcmd;
2227
2228 if (unlikely(skdev->dbg_level > 1)) {
2229 u8 *bp = (u8 *)skspcl->msg_buf;
2230 int i;
2231
2232 for (i = 0; i < SKD_N_SPECIAL_FITMSG_BYTES; i += 8) {
2e44b427 2233 pr_debug("%s:%s:%d spcl[%2d] %02x %02x %02x %02x "
2234 "%02x %02x %02x %02x\n",
2235 skdev->name, __func__, __LINE__, i,
2236 bp[i + 0], bp[i + 1], bp[i + 2], bp[i + 3],
2237 bp[i + 4], bp[i + 5], bp[i + 6], bp[i + 7]);
e67f86b3
AB		 2238 if (i == 0)
2239 i = 64 - 8;
2240 }
2241
2e44b427 2242 pr_debug("%s:%s:%d skspcl=%p id=%04x sksg_list=%p sksg_dma=%llx\n",
2243 skdev->name, __func__, __LINE__,
2244 skspcl, skspcl->req.id, skspcl->req.sksg_list,
2245 skspcl->req.sksg_dma_address);
e67f86b3
AB		 2246 for (i = 0; i < skspcl->req.n_sg; i++) {
2247 struct fit_sg_descriptor *sgd =
2248 &skspcl->req.sksg_list[i];
2249
2e44b427 2250 pr_debug("%s:%s:%d sg[%d] count=%u ctrl=0x%x "
2251 "addr=0x%llx next=0x%llx\n",
2252 skdev->name, __func__, __LINE__,
2253 i, sgd->byte_count, sgd->control,
2254 sgd->host_side_addr, sgd->next_desc_ptr);
e67f86b3
AB		 2255 }
2256 }
2257
2258 /*
2259 * Special FIT msgs are always 128 bytes: a 64-byte FIT hdr
2260 * and one 64-byte SSDI command.
2261 */
2262 qcmd = skspcl->mb_dma_address;
2263 qcmd |= FIT_QCMD_QID_NORMAL + FIT_QCMD_MSGSIZE_128;
2264
2265 SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
2266}
2267
2268/*
2269 *****************************************************************************
2270 * COMPLETION QUEUE
2271 *****************************************************************************
2272 */
2273
2274static void skd_complete_other(struct skd_device *skdev,
2275 volatile struct fit_completion_entry_v1 *skcomp,
2276 volatile struct fit_comp_error_info *skerr);
2277
e67f86b3
AB		 2278struct sns_info {
2279 u8 type;
2280 u8 stat;
2281 u8 key;
2282 u8 asc;
2283 u8 ascq;
2284 u8 mask;
2285 enum skd_check_status_action action;
2286};
2287
2288static struct sns_info skd_chkstat_table[] = {
2289 /* Good */
2290 { 0x70, 0x02, RECOVERED_ERROR, 0, 0, 0x1c,
2291 SKD_CHECK_STATUS_REPORT_GOOD },
2292
2293 /* Smart alerts */
2294 { 0x70, 0x02, NO_SENSE, 0x0B, 0x00, 0x1E, /* warnings */
2295 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2296 { 0x70, 0x02, NO_SENSE, 0x5D, 0x00, 0x1E, /* thresholds */
2297 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2298 { 0x70, 0x02, RECOVERED_ERROR, 0x0B, 0x01, 0x1F, /* temperature over trigger */
2299 SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2300
2301 /* Retry (with limits) */
2302 { 0x70, 0x02, 0x0B, 0, 0, 0x1C, /* This one is for DMA ERROR */
2303 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2304 { 0x70, 0x02, 0x06, 0x0B, 0x00, 0x1E, /* warnings */
2305 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2306 { 0x70, 0x02, 0x06, 0x5D, 0x00, 0x1E, /* thresholds */
2307 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2308 { 0x70, 0x02, 0x06, 0x80, 0x30, 0x1F, /* backup power */
2309 SKD_CHECK_STATUS_REQUEUE_REQUEST },
2310
2311 /* Busy (or about to be) */
2312 { 0x70, 0x02, 0x06, 0x3f, 0x01, 0x1F, /* fw changed */
2313 SKD_CHECK_STATUS_BUSY_IMMINENT },
2314};
2315
2316/*
2317 * Look up status and sense data to decide how to handle the error
2318 * from the device.
2319 * mask says which fields must match e.g., mask=0x18 means check
2320 * type and stat, ignore key, asc, ascq.
2321 */
2322
38d4a1bb
MS		 2323static enum skd_check_status_action
2324skd_check_status(struct skd_device *skdev,
2325 u8 cmp_status, volatile struct fit_comp_error_info *skerr)
e67f86b3
AB		 2326{
2327 int i, n;
2328
2329 pr_err("(%s): key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
2330 skd_name(skdev), skerr->key, skerr->code, skerr->qual,
2331 skerr->fruc);
2332
2e44b427 2333 pr_debug("%s:%s:%d stat: t=%02x stat=%02x k=%02x c=%02x q=%02x fruc=%02x\n",
2334 skdev->name, __func__, __LINE__, skerr->type, cmp_status,
2335 skerr->key, skerr->code, skerr->qual, skerr->fruc);
e67f86b3
AB		 2336
2337 /* Does the info match an entry in the good category? */
2338 n = sizeof(skd_chkstat_table) / sizeof(skd_chkstat_table[0]);
2339 for (i = 0; i < n; i++) {
2340 struct sns_info *sns = &skd_chkstat_table[i];
2341
2342 if (sns->mask & 0x10)
2343 if (skerr->type != sns->type)
2344 continue;
2345
2346 if (sns->mask & 0x08)
2347 if (cmp_status != sns->stat)
2348 continue;
2349
2350 if (sns->mask & 0x04)
2351 if (skerr->key != sns->key)
2352 continue;
2353
2354 if (sns->mask & 0x02)
2355 if (skerr->code != sns->asc)
2356 continue;
2357
2358 if (sns->mask & 0x01)
2359 if (skerr->qual != sns->ascq)
2360 continue;
2361
2362 if (sns->action == SKD_CHECK_STATUS_REPORT_SMART_ALERT) {
2363 pr_err("(%s): SMART Alert: sense key/asc/ascq "
2364 "%02x/%02x/%02x\n",
2365 skd_name(skdev), skerr->key,
2366 skerr->code, skerr->qual);
2367 }
2368 return sns->action;
2369 }
2370
2371 /* No other match, so nonzero status means error,
2372 * zero status means good
2373 */
2374 if (cmp_status) {
2e44b427 2375 pr_debug("%s:%s:%d status check: error\n",
2376 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2377 return SKD_CHECK_STATUS_REPORT_ERROR;
2378 }
2379
2e44b427 2380 pr_debug("%s:%s:%d status check good default\n",
2381 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2382 return SKD_CHECK_STATUS_REPORT_GOOD;
2383}
2384
2385static void skd_resolve_req_exception(struct skd_device *skdev,
2386 struct skd_request_context *skreq)
2387{
2388 u8 cmp_status = skreq->completion.status;
2389
2390 switch (skd_check_status(skdev, cmp_status, &skreq->err_info)) {
2391 case SKD_CHECK_STATUS_REPORT_GOOD:
2392 case SKD_CHECK_STATUS_REPORT_SMART_ALERT:
2393 skd_end_request(skdev, skreq, 0);
2394 break;
2395
2396 case SKD_CHECK_STATUS_BUSY_IMMINENT:
2397 skd_log_skreq(skdev, skreq, "retry(busy)");
38d4a1bb 2398 blk_requeue_request(skdev->queue, skreq->req);
e67f86b3
AB		 2399 pr_info("(%s) drive BUSY imminent\n", skd_name(skdev));
2400 skdev->state = SKD_DRVR_STATE_BUSY_IMMINENT;
2401 skdev->timer_countdown = SKD_TIMER_MINUTES(20);
2402 skd_quiesce_dev(skdev);
2403 break;
2404
2405 case SKD_CHECK_STATUS_REQUEUE_REQUEST:
fcd37eb3
JA		 2406 if ((unsigned long) ++skreq->req->special < SKD_MAX_RETRIES) {
2407 skd_log_skreq(skdev, skreq, "retry");
38d4a1bb 2408 blk_requeue_request(skdev->queue, skreq->req);
fcd37eb3 2409 break;
e67f86b3
AB		 2410 }
2411 /* fall through to report error */
2412
2413 case SKD_CHECK_STATUS_REPORT_ERROR:
2414 default:
2415 skd_end_request(skdev, skreq, -EIO);
2416 break;
2417 }
2418}
2419
e67f86b3
AB		 2420/* assume spinlock is already held */
2421static void skd_release_skreq(struct skd_device *skdev,
2422 struct skd_request_context *skreq)
2423{
2424 u32 msg_slot;
2425 struct skd_fitmsg_context *skmsg;
2426
2427 u32 timo_slot;
2428
2429 /*
2430 * Reclaim the FIT msg buffer if this is
2431 * the first of the requests it carried to
2432 * be completed. The FIT msg buffer used to
2433 * send this request cannot be reused until
2434 * we are sure the s1120 card has copied
2435 * it to its memory. The FIT msg might have
2436 * contained several requests. As soon as
2437 * any of them are completed we know that
2438 * the entire FIT msg was transferred.
2439 * Only the first completed request will
2440 * match the FIT msg buffer id. The FIT
2441 * msg buffer id is immediately updated.
2442 * When subsequent requests complete the FIT
2443 * msg buffer id won't match, so we know
2444 * quite cheaply that it is already done.
2445 */
2446 msg_slot = skreq->fitmsg_id & SKD_ID_SLOT_MASK;
2447 SKD_ASSERT(msg_slot < skdev->num_fitmsg_context);
2448
2449 skmsg = &skdev->skmsg_table[msg_slot];
2450 if (skmsg->id == skreq->fitmsg_id) {
2451 SKD_ASSERT(skmsg->state == SKD_MSG_STATE_BUSY);
2452 SKD_ASSERT(skmsg->outstanding > 0);
2453 skmsg->outstanding--;
2454 if (skmsg->outstanding == 0) {
2455 skmsg->state = SKD_MSG_STATE_IDLE;
2456 skmsg->id += SKD_ID_INCR;
2457 skmsg->next = skdev->skmsg_free_list;
2458 skdev->skmsg_free_list = skmsg;
2459 }
2460 }
2461
2462 /*
2463 * Decrease the number of active requests.
2464 * Also decrements the count in the timeout slot.
2465 */
2466 SKD_ASSERT(skdev->in_flight > 0);
2467 skdev->in_flight -= 1;
2468
2469 timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
2470 SKD_ASSERT(skdev->timeout_slot[timo_slot] > 0);
2471 skdev->timeout_slot[timo_slot] -= 1;
2472
2473 /*
2474 * Reset backpointer
2475 */
fcd37eb3 2476 skreq->req = NULL;
e67f86b3
AB		 2477
2478 /*
2479 * Reclaim the skd_request_context
2480 */
2481 skreq->state = SKD_REQ_STATE_IDLE;
2482 skreq->id += SKD_ID_INCR;
2483 skreq->next = skdev->skreq_free_list;
2484 skdev->skreq_free_list = skreq;
2485}
2486
2487#define DRIVER_INQ_EVPD_PAGE_CODE 0xDA
2488
2489static void skd_do_inq_page_00(struct skd_device *skdev,
2490 volatile struct fit_completion_entry_v1 *skcomp,
2491 volatile struct fit_comp_error_info *skerr,
2492 uint8_t *cdb, uint8_t *buf)
2493{
2494 uint16_t insert_pt, max_bytes, drive_pages, drive_bytes, new_size;
2495
2496 /* Caller requested "supported pages". The driver needs to insert
2497 * its page.
2498 */
2e44b427 2499 pr_debug("%s:%s:%d skd_do_driver_inquiry: modify supported pages.\n",
2500 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2501
2502 /* If the device rejected the request because the CDB was
2503 * improperly formed, then just leave.
2504 */
2505 if (skcomp->status == SAM_STAT_CHECK_CONDITION &&
2506 skerr->key == ILLEGAL_REQUEST && skerr->code == 0x24)
2507 return;
2508
2509 /* Get the amount of space the caller allocated */
2510 max_bytes = (cdb[3] << 8) | cdb[4];
2511
2512 /* Get the number of pages actually returned by the device */
2513 drive_pages = (buf[2] << 8) | buf[3];
2514 drive_bytes = drive_pages + 4;
2515 new_size = drive_pages + 1;
2516
2517 /* Supported pages must be in numerical order, so find where
2518 * the driver page needs to be inserted into the list of
2519 * pages returned by the device.
2520 */
2521 for (insert_pt = 4; insert_pt < drive_bytes; insert_pt++) {
2522 if (buf[insert_pt] == DRIVER_INQ_EVPD_PAGE_CODE)
2523 return; /* Device using this page code. abort */
2524 else if (buf[insert_pt] > DRIVER_INQ_EVPD_PAGE_CODE)
2525 break;
2526 }
2527
2528 if (insert_pt < max_bytes) {
2529 uint16_t u;
2530
2531 /* Shift everything up one byte to make room. */
2532 for (u = new_size + 3; u > insert_pt; u--)
2533 buf[u] = buf[u - 1];
2534 buf[insert_pt] = DRIVER_INQ_EVPD_PAGE_CODE;
2535
2536 /* SCSI byte order increment of num_returned_bytes by 1 */
2537 skcomp->num_returned_bytes =
2538 be32_to_cpu(skcomp->num_returned_bytes) + 1;
2539 skcomp->num_returned_bytes =
2540 be32_to_cpu(skcomp->num_returned_bytes);
2541 }
2542
2543 /* update page length field to reflect the driver's page too */
2544 buf[2] = (uint8_t)((new_size >> 8) & 0xFF);
2545 buf[3] = (uint8_t)((new_size >> 0) & 0xFF);
2546}
2547
2548static void skd_get_link_info(struct pci_dev *pdev, u8 *speed, u8 *width)
2549{
2550 int pcie_reg;
2551 u16 pci_bus_speed;
2552 u8 pci_lanes;
2553
2554 pcie_reg = pci_find_capability(pdev, PCI_CAP_ID_EXP);
2555 if (pcie_reg) {
2556 u16 linksta;
2557 pci_read_config_word(pdev, pcie_reg + PCI_EXP_LNKSTA, &linksta);
2558
2559 pci_bus_speed = linksta & 0xF;
2560 pci_lanes = (linksta & 0x3F0) >> 4;
2561 } else {
2562 *speed = STEC_LINK_UNKNOWN;
2563 *width = 0xFF;
2564 return;
2565 }
2566
2567 switch (pci_bus_speed) {
2568 case 1:
2569 *speed = STEC_LINK_2_5GTS;
2570 break;
2571 case 2:
2572 *speed = STEC_LINK_5GTS;
2573 break;
2574 case 3:
2575 *speed = STEC_LINK_8GTS;
2576 break;
2577 default:
2578 *speed = STEC_LINK_UNKNOWN;
2579 break;
2580 }
2581
2582 if (pci_lanes <= 0x20)
2583 *width = pci_lanes;
2584 else
2585 *width = 0xFF;
2586}
2587
2588static void skd_do_inq_page_da(struct skd_device *skdev,
2589 volatile struct fit_completion_entry_v1 *skcomp,
2590 volatile struct fit_comp_error_info *skerr,
2591 uint8_t *cdb, uint8_t *buf)
2592{
2593 unsigned max_bytes;
2594 struct driver_inquiry_data inq;
2595 u16 val;
2596
2e44b427 2597 pr_debug("%s:%s:%d skd_do_driver_inquiry: return driver page\n",
2598 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2599
2600 memset(&inq, 0, sizeof(inq));
2601
2602 inq.page_code = DRIVER_INQ_EVPD_PAGE_CODE;
2603
2604 if (skdev->pdev && skdev->pdev->bus) {
2605 skd_get_link_info(skdev->pdev,
2606 &inq.pcie_link_speed, &inq.pcie_link_lanes);
2607 inq.pcie_bus_number = cpu_to_be16(skdev->pdev->bus->number);
2608 inq.pcie_device_number = PCI_SLOT(skdev->pdev->devfn);
2609 inq.pcie_function_number = PCI_FUNC(skdev->pdev->devfn);
2610
2611 pci_read_config_word(skdev->pdev, PCI_VENDOR_ID, &val);
2612 inq.pcie_vendor_id = cpu_to_be16(val);
2613
2614 pci_read_config_word(skdev->pdev, PCI_DEVICE_ID, &val);
2615 inq.pcie_device_id = cpu_to_be16(val);
2616
2617 pci_read_config_word(skdev->pdev, PCI_SUBSYSTEM_VENDOR_ID,
2618 &val);
2619 inq.pcie_subsystem_vendor_id = cpu_to_be16(val);
2620
2621 pci_read_config_word(skdev->pdev, PCI_SUBSYSTEM_ID, &val);
2622 inq.pcie_subsystem_device_id = cpu_to_be16(val);
2623 } else {
2624 inq.pcie_bus_number = 0xFFFF;
2625 inq.pcie_device_number = 0xFF;
2626 inq.pcie_function_number = 0xFF;
2627 inq.pcie_link_speed = 0xFF;
2628 inq.pcie_link_lanes = 0xFF;
2629 inq.pcie_vendor_id = 0xFFFF;
2630 inq.pcie_device_id = 0xFFFF;
2631 inq.pcie_subsystem_vendor_id = 0xFFFF;
2632 inq.pcie_subsystem_device_id = 0xFFFF;
2633 }
2634
2635 /* Driver version, fixed lenth, padded with spaces on the right */
2636 inq.driver_version_length = sizeof(inq.driver_version);
2637 memset(&inq.driver_version, ' ', sizeof(inq.driver_version));
2638 memcpy(inq.driver_version, DRV_VER_COMPL,
2639 min(sizeof(inq.driver_version), strlen(DRV_VER_COMPL)));
2640
2641 inq.page_length = cpu_to_be16((sizeof(inq) - 4));
2642
2643 /* Clear the error set by the device */
2644 skcomp->status = SAM_STAT_GOOD;
2645 memset((void *)skerr, 0, sizeof(*skerr));
2646
2647 /* copy response into output buffer */
2648 max_bytes = (cdb[3] << 8) | cdb[4];
2649 memcpy(buf, &inq, min_t(unsigned, max_bytes, sizeof(inq)));
2650
2651 skcomp->num_returned_bytes =
2652 be32_to_cpu(min_t(uint16_t, max_bytes, sizeof(inq)));
2653}
2654
2655static void skd_do_driver_inq(struct skd_device *skdev,
2656 volatile struct fit_completion_entry_v1 *skcomp,
2657 volatile struct fit_comp_error_info *skerr,
2658 uint8_t *cdb, uint8_t *buf)
2659{
2660 if (!buf)
2661 return;
2662 else if (cdb[0] != INQUIRY)
2663 return; /* Not an INQUIRY */
2664 else if ((cdb[1] & 1) == 0)
2665 return; /* EVPD not set */
2666 else if (cdb[2] == 0)
2667 /* Need to add driver's page to supported pages list */
2668 skd_do_inq_page_00(skdev, skcomp, skerr, cdb, buf);
2669 else if (cdb[2] == DRIVER_INQ_EVPD_PAGE_CODE)
2670 /* Caller requested driver's page */
2671 skd_do_inq_page_da(skdev, skcomp, skerr, cdb, buf);
2672}
2673
2674static unsigned char *skd_sg_1st_page_ptr(struct scatterlist *sg)
2675{
2676 if (!sg)
2677 return NULL;
2678 if (!sg_page(sg))
2679 return NULL;
2680 return sg_virt(sg);
2681}
2682
2683static void skd_process_scsi_inq(struct skd_device *skdev,
2684 volatile struct fit_completion_entry_v1
2685 *skcomp,
2686 volatile struct fit_comp_error_info *skerr,
2687 struct skd_special_context *skspcl)
2688{
2689 uint8_t *buf;
2690 struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
2691 struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
2692
2693 dma_sync_sg_for_cpu(skdev->class_dev, skspcl->req.sg, skspcl->req.n_sg,
2694 skspcl->req.sg_data_dir);
2695 buf = skd_sg_1st_page_ptr(skspcl->req.sg);
2696
2697 if (buf)
2698 skd_do_driver_inq(skdev, skcomp, skerr, scsi_req->cdb, buf);
2699}
2700
2701
2702static int skd_isr_completion_posted(struct skd_device *skdev,
2703 int limit, int *enqueued)
2704{
2705 volatile struct fit_completion_entry_v1 *skcmp = NULL;
2706 volatile struct fit_comp_error_info *skerr;
2707 u16 req_id;
2708 u32 req_slot;
2709 struct skd_request_context *skreq;
2710 u16 cmp_cntxt = 0;
2711 u8 cmp_status = 0;
2712 u8 cmp_cycle = 0;
2713 u32 cmp_bytes = 0;
2714 int rc = 0;
2715 int processed = 0;
e67f86b3
AB		 2716
2717 for (;; ) {
2718 SKD_ASSERT(skdev->skcomp_ix < SKD_N_COMPLETION_ENTRY);
2719
2720 skcmp = &skdev->skcomp_table[skdev->skcomp_ix];
2721 cmp_cycle = skcmp->cycle;
2722 cmp_cntxt = skcmp->tag;
2723 cmp_status = skcmp->status;
2724 cmp_bytes = be32_to_cpu(skcmp->num_returned_bytes);
2725
2726 skerr = &skdev->skerr_table[skdev->skcomp_ix];
2727
2e44b427 2728 pr_debug("%s:%s:%d "
2729 "cycle=%d ix=%d got cycle=%d cmdctxt=0x%x stat=%d "
2730 "busy=%d rbytes=0x%x proto=%d\n",
2731 skdev->name, __func__, __LINE__, skdev->skcomp_cycle,
2732 skdev->skcomp_ix, cmp_cycle, cmp_cntxt, cmp_status,
2733 skdev->in_flight, cmp_bytes, skdev->proto_ver);
e67f86b3
AB		 2734
2735 if (cmp_cycle != skdev->skcomp_cycle) {
2e44b427 2736 pr_debug("%s:%s:%d end of completions\n",
2737 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2738 break;
2739 }
2740 /*
2741 * Update the completion queue head index and possibly
2742 * the completion cycle count. 8-bit wrap-around.
2743 */
2744 skdev->skcomp_ix++;
2745 if (skdev->skcomp_ix >= SKD_N_COMPLETION_ENTRY) {
2746 skdev->skcomp_ix = 0;
2747 skdev->skcomp_cycle++;
2748 }
2749
2750 /*
2751 * The command context is a unique 32-bit ID. The low order
2752 * bits help locate the request. The request is usually a
2753 * r/w request (see skd_start() above) or a special request.
2754 */
2755 req_id = cmp_cntxt;
2756 req_slot = req_id & SKD_ID_SLOT_AND_TABLE_MASK;
2757
2758 /* Is this other than a r/w request? */
2759 if (req_slot >= skdev->num_req_context) {
2760 /*
2761 * This is not a completion for a r/w request.
2762 */
2763 skd_complete_other(skdev, skcmp, skerr);
2764 continue;
2765 }
2766
2767 skreq = &skdev->skreq_table[req_slot];
2768
2769 /*
2770 * Make sure the request ID for the slot matches.
2771 */
2772 if (skreq->id != req_id) {
2e44b427 2773 pr_debug("%s:%s:%d mismatch comp_id=0x%x req_id=0x%x\n",
2774 skdev->name, __func__, __LINE__,
2775 req_id, skreq->id);
e67f86b3
AB		 2776 {
2777 u16 new_id = cmp_cntxt;
2778 pr_err("(%s): Completion mismatch "
2779 "comp_id=0x%04x skreq=0x%04x new=0x%04x\n",
2780 skd_name(skdev), req_id,
2781 skreq->id, new_id);
2782
2783 continue;
2784 }
2785 }
2786
2787 SKD_ASSERT(skreq->state == SKD_REQ_STATE_BUSY);
2788
2789 if (skreq->state == SKD_REQ_STATE_ABORTED) {
2e44b427 2790 pr_debug("%s:%s:%d reclaim req %p id=%04x\n",
2791 skdev->name, __func__, __LINE__,
2792 skreq, skreq->id);
e67f86b3
AB		 2793 /* a previously timed out command can
2794 * now be cleaned up */
2795 skd_release_skreq(skdev, skreq);
2796 continue;
2797 }
2798
2799 skreq->completion = *skcmp;
2800 if (unlikely(cmp_status == SAM_STAT_CHECK_CONDITION)) {
2801 skreq->err_info = *skerr;
2802 skd_log_check_status(skdev, cmp_status, skerr->key,
2803 skerr->code, skerr->qual,
2804 skerr->fruc);
2805 }
2806 /* Release DMA resources for the request. */
2807 if (skreq->n_sg > 0)
2808 skd_postop_sg_list(skdev, skreq);
2809
fcd37eb3 2810 if (!skreq->req) {
2e44b427 2811 pr_debug("%s:%s:%d NULL backptr skdreq %p, "
2812 "req=0x%x req_id=0x%x\n",
2813 skdev->name, __func__, __LINE__,
2814 skreq, skreq->id, req_id);
e67f86b3
AB		 2815 } else {
2816 /*
2817 * Capture the outcome and post it back to the
2818 * native request.
2819 */
fcd37eb3
JA		 2820 if (likely(cmp_status == SAM_STAT_GOOD))
2821 skd_end_request(skdev, skreq, 0);
2822 else
e67f86b3 2823 skd_resolve_req_exception(skdev, skreq);
e67f86b3
AB		 2824 }
2825
2826 /*
2827 * Release the skreq, its FIT msg (if one), timeout slot,
2828 * and queue depth.
2829 */
2830 skd_release_skreq(skdev, skreq);
2831
2832 /* skd_isr_comp_limit equal zero means no limit */
2833 if (limit) {
2834 if (++processed >= limit) {
2835 rc = 1;
2836 break;
2837 }
2838 }
2839 }
2840
2841 if ((skdev->state == SKD_DRVR_STATE_PAUSING)
2842 && (skdev->in_flight) == 0) {
2843 skdev->state = SKD_DRVR_STATE_PAUSED;
2844 wake_up_interruptible(&skdev->waitq);
2845 }
2846
2847 return rc;
2848}
2849
2850static void skd_complete_other(struct skd_device *skdev,
2851 volatile struct fit_completion_entry_v1 *skcomp,
2852 volatile struct fit_comp_error_info *skerr)
2853{
2854 u32 req_id = 0;
2855 u32 req_table;
2856 u32 req_slot;
2857 struct skd_special_context *skspcl;
2858
2859 req_id = skcomp->tag;
2860 req_table = req_id & SKD_ID_TABLE_MASK;
2861 req_slot = req_id & SKD_ID_SLOT_MASK;
2862
2e44b427 2863 pr_debug("%s:%s:%d table=0x%x id=0x%x slot=%d\n",
2864 skdev->name, __func__, __LINE__,
2865 req_table, req_id, req_slot);
e67f86b3
AB		 2866
2867 /*
2868 * Based on the request id, determine how to dispatch this completion.
2869 * This swich/case is finding the good cases and forwarding the
2870 * completion entry. Errors are reported below the switch.
2871 */
2872 switch (req_table) {
2873 case SKD_ID_RW_REQUEST:
2874 /*
2875 * The caller, skd_completion_posted_isr() above,
2876 * handles r/w requests. The only way we get here
2877 * is if the req_slot is out of bounds.
2878 */
2879 break;
2880
2881 case SKD_ID_SPECIAL_REQUEST:
2882 /*
2883 * Make sure the req_slot is in bounds and that the id
2884 * matches.
2885 */
2886 if (req_slot < skdev->n_special) {
2887 skspcl = &skdev->skspcl_table[req_slot];
2888 if (skspcl->req.id == req_id &&
2889 skspcl->req.state == SKD_REQ_STATE_BUSY) {
2890 skd_complete_special(skdev,
2891 skcomp, skerr, skspcl);
2892 return;
2893 }
2894 }
2895 break;
2896
2897 case SKD_ID_INTERNAL:
2898 if (req_slot == 0) {
2899 skspcl = &skdev->internal_skspcl;
2900 if (skspcl->req.id == req_id &&
2901 skspcl->req.state == SKD_REQ_STATE_BUSY) {
2902 skd_complete_internal(skdev,
2903 skcomp, skerr, skspcl);
2904 return;
2905 }
2906 }
2907 break;
2908
2909 case SKD_ID_FIT_MSG:
2910 /*
2911 * These id's should never appear in a completion record.
2912 */
2913 break;
2914
2915 default:
2916 /*
2917 * These id's should never appear anywhere;
2918 */
2919 break;
2920 }
2921
2922 /*
2923 * If we get here it is a bad or stale id.
2924 */
2925}
2926
2927static void skd_complete_special(struct skd_device *skdev,
2928 volatile struct fit_completion_entry_v1
2929 *skcomp,
2930 volatile struct fit_comp_error_info *skerr,
2931 struct skd_special_context *skspcl)
2932{
2e44b427 2933 pr_debug("%s:%s:%d completing special request %p\n",
2934 skdev->name, __func__, __LINE__, skspcl);
e67f86b3
AB		 2935 if (skspcl->orphaned) {
2936 /* Discard orphaned request */
2937 /* ?: Can this release directly or does it need
2938 * to use a worker? */
2e44b427 2939 pr_debug("%s:%s:%d release orphaned %p\n",
2940 skdev->name, __func__, __LINE__, skspcl);
e67f86b3
AB		 2941 skd_release_special(skdev, skspcl);
2942 return;
2943 }
2944
2945 skd_process_scsi_inq(skdev, skcomp, skerr, skspcl);
2946
2947 skspcl->req.state = SKD_REQ_STATE_COMPLETED;
2948 skspcl->req.completion = *skcomp;
2949 skspcl->req.err_info = *skerr;
2950
2951 skd_log_check_status(skdev, skspcl->req.completion.status, skerr->key,
2952 skerr->code, skerr->qual, skerr->fruc);
2953
2954 wake_up_interruptible(&skdev->waitq);
2955}
2956
2957/* assume spinlock is already held */
2958static void skd_release_special(struct skd_device *skdev,
2959 struct skd_special_context *skspcl)
2960{
2961 int i, was_depleted;
2962
2963 for (i = 0; i < skspcl->req.n_sg; i++) {
e67f86b3
AB		 2964 struct page *page = sg_page(&skspcl->req.sg[i]);
2965 __free_page(page);
2966 }
2967
2968 was_depleted = (skdev->skspcl_free_list == NULL);
2969
2970 skspcl->req.state = SKD_REQ_STATE_IDLE;
2971 skspcl->req.id += SKD_ID_INCR;
2972 skspcl->req.next =
2973 (struct skd_request_context *)skdev->skspcl_free_list;
2974 skdev->skspcl_free_list = (struct skd_special_context *)skspcl;
2975
2976 if (was_depleted) {
2e44b427 2977 pr_debug("%s:%s:%d skspcl was depleted\n",
2978 skdev->name, __func__, __LINE__);
e67f86b3
AB		 2979 /* Free list was depleted. Their might be waiters. */
2980 wake_up_interruptible(&skdev->waitq);
2981 }
2982}
2983
2984static void skd_reset_skcomp(struct skd_device *skdev)
2985{
2986 u32 nbytes;
2987 struct fit_completion_entry_v1 *skcomp;
2988
2989 nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
2990 nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
2991
2992 memset(skdev->skcomp_table, 0, nbytes);
2993
2994 skdev->skcomp_ix = 0;
2995 skdev->skcomp_cycle = 1;
2996}
2997
2998/*
2999 *****************************************************************************
3000 * INTERRUPTS
3001 *****************************************************************************
3002 */
3003static void skd_completion_worker(struct work_struct *work)
3004{
3005 struct skd_device *skdev =
3006 container_of(work, struct skd_device, completion_worker);
3007 unsigned long flags;
3008 int flush_enqueued = 0;
3009
3010 spin_lock_irqsave(&skdev->lock, flags);
3011
3012 /*
3013 * pass in limit=0, which means no limit..
3014 * process everything in compq
3015 */
3016 skd_isr_completion_posted(skdev, 0, &flush_enqueued);
3017 skd_request_fn(skdev->queue);
3018
3019 spin_unlock_irqrestore(&skdev->lock, flags);
3020}
3021
3022static void skd_isr_msg_from_dev(struct skd_device *skdev);
3023
3024irqreturn_t
3025static skd_isr(int irq, void *ptr)
3026{
3027 struct skd_device *skdev;
3028 u32 intstat;
3029 u32 ack;
3030 int rc = 0;
3031 int deferred = 0;
3032 int flush_enqueued = 0;
3033
3034 skdev = (struct skd_device *)ptr;
3035 spin_lock(&skdev->lock);
3036
3037 for (;; ) {
3038 intstat = SKD_READL(skdev, FIT_INT_STATUS_HOST);
3039
3040 ack = FIT_INT_DEF_MASK;
3041 ack &= intstat;
3042
2e44b427 3043 pr_debug("%s:%s:%d intstat=0x%x ack=0x%x\n",
3044 skdev->name, __func__, __LINE__, intstat, ack);
e67f86b3
AB		 3045
3046 /* As long as there is an int pending on device, keep
3047 * running loop. When none, get out, but if we've never
3048 * done any processing, call completion handler?
3049 */
3050 if (ack == 0) {
3051 /* No interrupts on device, but run the completion
3052 * processor anyway?
3053 */
3054 if (rc == 0)
3055 if (likely (skdev->state
3056 == SKD_DRVR_STATE_ONLINE))
3057 deferred = 1;
3058 break;
3059 }
3060
3061 rc = IRQ_HANDLED;
3062
3063 SKD_WRITEL(skdev, ack, FIT_INT_STATUS_HOST);
3064
3065 if (likely((skdev->state != SKD_DRVR_STATE_LOAD) &&
3066 (skdev->state != SKD_DRVR_STATE_STOPPING))) {
3067 if (intstat & FIT_ISH_COMPLETION_POSTED) {
3068 /*
3069 * If we have already deferred completion
3070 * processing, don't bother running it again
3071 */
3072 if (deferred == 0)
3073 deferred =
3074 skd_isr_completion_posted(skdev,
3075 skd_isr_comp_limit, &flush_enqueued);
3076 }
3077
3078 if (intstat & FIT_ISH_FW_STATE_CHANGE) {
3079 skd_isr_fwstate(skdev);
3080 if (skdev->state == SKD_DRVR_STATE_FAULT ||
3081 skdev->state ==
3082 SKD_DRVR_STATE_DISAPPEARED) {
3083 spin_unlock(&skdev->lock);
3084 return rc;
3085 }
3086 }
3087
3088 if (intstat & FIT_ISH_MSG_FROM_DEV)
3089 skd_isr_msg_from_dev(skdev);
3090 }
3091 }
3092
3093 if (unlikely(flush_enqueued))
3094 skd_request_fn(skdev->queue);
3095
3096 if (deferred)
3097 schedule_work(&skdev->completion_worker);
3098 else if (!flush_enqueued)
3099 skd_request_fn(skdev->queue);
3100
3101 spin_unlock(&skdev->lock);
3102
3103 return rc;
3104}
3105
e67f86b3
AB		 3106static void skd_drive_fault(struct skd_device *skdev)
3107{
3108 skdev->state = SKD_DRVR_STATE_FAULT;
3109 pr_err("(%s): Drive FAULT\n", skd_name(skdev));
3110}
3111
3112static void skd_drive_disappeared(struct skd_device *skdev)
3113{
3114 skdev->state = SKD_DRVR_STATE_DISAPPEARED;
3115 pr_err("(%s): Drive DISAPPEARED\n", skd_name(skdev));
3116}
3117
3118static void skd_isr_fwstate(struct skd_device *skdev)
3119{
3120 u32 sense;
3121 u32 state;
3122 u32 mtd;
3123 int prev_driver_state = skdev->state;
3124
3125 sense = SKD_READL(skdev, FIT_STATUS);
3126 state = sense & FIT_SR_DRIVE_STATE_MASK;
3127
3128 pr_err("(%s): s1120 state %s(%d)=>%s(%d)\n",
3129 skd_name(skdev),
3130 skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
3131 skd_drive_state_to_str(state), state);
3132
3133 skdev->drive_state = state;
3134
3135 switch (skdev->drive_state) {
3136 case FIT_SR_DRIVE_INIT:
3137 if (skdev->state == SKD_DRVR_STATE_PROTOCOL_MISMATCH) {
3138 skd_disable_interrupts(skdev);
3139 break;
3140 }
3141 if (skdev->state == SKD_DRVR_STATE_RESTARTING)
3142 skd_recover_requests(skdev, 0);
3143 if (skdev->state == SKD_DRVR_STATE_WAIT_BOOT) {
3144 skdev->timer_countdown = SKD_STARTING_TIMO;
3145 skdev->state = SKD_DRVR_STATE_STARTING;
3146 skd_soft_reset(skdev);
3147 break;
3148 }
3149 mtd = FIT_MXD_CONS(FIT_MTD_FITFW_INIT, 0, 0);
3150 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3151 skdev->last_mtd = mtd;
3152 break;
3153
3154 case FIT_SR_DRIVE_ONLINE:
3155 skdev->cur_max_queue_depth = skd_max_queue_depth;
3156 if (skdev->cur_max_queue_depth > skdev->dev_max_queue_depth)
3157 skdev->cur_max_queue_depth = skdev->dev_max_queue_depth;
3158
3159 skdev->queue_low_water_mark =
3160 skdev->cur_max_queue_depth * 2 / 3 + 1;
3161 if (skdev->queue_low_water_mark < 1)
3162 skdev->queue_low_water_mark = 1;
3163 pr_info(
3164 "(%s): Queue depth limit=%d dev=%d lowat=%d\n",
3165 skd_name(skdev),
3166 skdev->cur_max_queue_depth,
3167 skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
3168
3169 skd_refresh_device_data(skdev);
3170 break;
3171
3172 case FIT_SR_DRIVE_BUSY:
3173 skdev->state = SKD_DRVR_STATE_BUSY;
3174 skdev->timer_countdown = SKD_BUSY_TIMO;
3175 skd_quiesce_dev(skdev);
3176 break;
3177 case FIT_SR_DRIVE_BUSY_SANITIZE:
3178 /* set timer for 3 seconds, we'll abort any unfinished
3179 * commands after that expires
3180 */
3181 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
3182 skdev->timer_countdown = SKD_TIMER_SECONDS(3);
6a5ec65b 3183 blk_start_queue(skdev->queue);
e67f86b3
AB		 3184 break;
3185 case FIT_SR_DRIVE_BUSY_ERASE:
3186 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
3187 skdev->timer_countdown = SKD_BUSY_TIMO;
3188 break;
3189 case FIT_SR_DRIVE_OFFLINE:
3190 skdev->state = SKD_DRVR_STATE_IDLE;
3191 break;
3192 case FIT_SR_DRIVE_SOFT_RESET:
3193 switch (skdev->state) {
3194 case SKD_DRVR_STATE_STARTING:
3195 case SKD_DRVR_STATE_RESTARTING:
3196 /* Expected by a caller of skd_soft_reset() */
3197 break;
3198 default:
3199 skdev->state = SKD_DRVR_STATE_RESTARTING;
3200 break;
3201 }
3202 break;
3203 case FIT_SR_DRIVE_FW_BOOTING:
2e44b427 3204 pr_debug("%s:%s:%d ISR FIT_SR_DRIVE_FW_BOOTING %s\n",
3205 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3
AB		 3206 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
3207 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
3208 break;
3209
3210 case FIT_SR_DRIVE_DEGRADED:
3211 case FIT_SR_PCIE_LINK_DOWN:
3212 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
3213 break;
3214
3215 case FIT_SR_DRIVE_FAULT:
3216 skd_drive_fault(skdev);
3217 skd_recover_requests(skdev, 0);
6a5ec65b 3218 blk_start_queue(skdev->queue);
e67f86b3
AB		 3219 break;
3220
3221 /* PCIe bus returned all Fs? */
3222 case 0xFF:
3223 pr_info("(%s): state=0x%x sense=0x%x\n",
3224 skd_name(skdev), state, sense);
3225 skd_drive_disappeared(skdev);
3226 skd_recover_requests(skdev, 0);
6a5ec65b 3227 blk_start_queue(skdev->queue);
e67f86b3
AB		 3228 break;
3229 default:
3230 /*
3231 * Uknown FW State. Wait for a state we recognize.
3232 */
3233 break;
3234 }
3235 pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
3236 skd_name(skdev),
3237 skd_skdev_state_to_str(prev_driver_state), prev_driver_state,
3238 skd_skdev_state_to_str(skdev->state), skdev->state);
3239}
3240
3241static void skd_recover_requests(struct skd_device *skdev, int requeue)
3242{
3243 int i;
3244
3245 for (i = 0; i < skdev->num_req_context; i++) {
3246 struct skd_request_context *skreq = &skdev->skreq_table[i];
3247
3248 if (skreq->state == SKD_REQ_STATE_BUSY) {
3249 skd_log_skreq(skdev, skreq, "recover");
3250
3251 SKD_ASSERT((skreq->id & SKD_ID_INCR) != 0);
fcd37eb3 3252 SKD_ASSERT(skreq->req != NULL);
e67f86b3
AB		 3253
3254 /* Release DMA resources for the request. */
3255 if (skreq->n_sg > 0)
3256 skd_postop_sg_list(skdev, skreq);
3257
fcd37eb3
JA		 3258 if (requeue &&
3259 (unsigned long) ++skreq->req->special <
3260 SKD_MAX_RETRIES)
38d4a1bb 3261 blk_requeue_request(skdev->queue, skreq->req);
fcd37eb3 3262 else
e67f86b3
AB		 3263 skd_end_request(skdev, skreq, -EIO);
3264
fcd37eb3 3265 skreq->req = NULL;
e67f86b3
AB		 3266
3267 skreq->state = SKD_REQ_STATE_IDLE;
3268 skreq->id += SKD_ID_INCR;
e67f86b3
AB		 3269 }
3270 if (i > 0)
3271 skreq[-1].next = skreq;
3272 skreq->next = NULL;
3273 }
3274 skdev->skreq_free_list = skdev->skreq_table;
3275
3276 for (i = 0; i < skdev->num_fitmsg_context; i++) {
3277 struct skd_fitmsg_context *skmsg = &skdev->skmsg_table[i];
3278
3279 if (skmsg->state == SKD_MSG_STATE_BUSY) {
3280 skd_log_skmsg(skdev, skmsg, "salvaged");
3281 SKD_ASSERT((skmsg->id & SKD_ID_INCR) != 0);
3282 skmsg->state = SKD_MSG_STATE_IDLE;
3283 skmsg->id += SKD_ID_INCR;
3284 }
3285 if (i > 0)
3286 skmsg[-1].next = skmsg;
3287 skmsg->next = NULL;
3288 }
3289 skdev->skmsg_free_list = skdev->skmsg_table;
3290
3291 for (i = 0; i < skdev->n_special; i++) {
3292 struct skd_special_context *skspcl = &skdev->skspcl_table[i];
3293
3294 /* If orphaned, reclaim it because it has already been reported
3295 * to the process as an error (it was just waiting for
3296 * a completion that didn't come, and now it will never come)
3297 * If busy, change to a state that will cause it to error
3298 * out in the wait routine and let it do the normal
3299 * reporting and reclaiming
3300 */
3301 if (skspcl->req.state == SKD_REQ_STATE_BUSY) {
3302 if (skspcl->orphaned) {
2e44b427 3303 pr_debug("%s:%s:%d orphaned %p\n",
3304 skdev->name, __func__, __LINE__,
3305 skspcl);
e67f86b3
AB		 3306 skd_release_special(skdev, skspcl);
3307 } else {
2e44b427 3308 pr_debug("%s:%s:%d not orphaned %p\n",
3309 skdev->name, __func__, __LINE__,
3310 skspcl);
e67f86b3
AB		 3311 skspcl->req.state = SKD_REQ_STATE_ABORTED;
3312 }
3313 }
3314 }
3315 skdev->skspcl_free_list = skdev->skspcl_table;
3316
3317 for (i = 0; i < SKD_N_TIMEOUT_SLOT; i++)
3318 skdev->timeout_slot[i] = 0;
3319
3320 skdev->in_flight = 0;
3321}
3322
3323static void skd_isr_msg_from_dev(struct skd_device *skdev)
3324{
3325 u32 mfd;
3326 u32 mtd;
3327 u32 data;
3328
3329 mfd = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
3330
2e44b427 3331 pr_debug("%s:%s:%d mfd=0x%x last_mtd=0x%x\n",
3332 skdev->name, __func__, __LINE__, mfd, skdev->last_mtd);
e67f86b3
AB		 3333
3334 /* ignore any mtd that is an ack for something we didn't send */
3335 if (FIT_MXD_TYPE(mfd) != FIT_MXD_TYPE(skdev->last_mtd))
3336 return;
3337
3338 switch (FIT_MXD_TYPE(mfd)) {
3339 case FIT_MTD_FITFW_INIT:
3340 skdev->proto_ver = FIT_PROTOCOL_MAJOR_VER(mfd);
3341
3342 if (skdev->proto_ver != FIT_PROTOCOL_VERSION_1) {
3343 pr_err("(%s): protocol mismatch\n",
3344 skdev->name);
3345 pr_err("(%s): got=%d support=%d\n",
3346 skdev->name, skdev->proto_ver,
3347 FIT_PROTOCOL_VERSION_1);
3348 pr_err("(%s): please upgrade driver\n",
3349 skdev->name);
3350 skdev->state = SKD_DRVR_STATE_PROTOCOL_MISMATCH;
3351 skd_soft_reset(skdev);
3352 break;
3353 }
3354 mtd = FIT_MXD_CONS(FIT_MTD_GET_CMDQ_DEPTH, 0, 0);
3355 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3356 skdev->last_mtd = mtd;
3357 break;
3358
3359 case FIT_MTD_GET_CMDQ_DEPTH:
3360 skdev->dev_max_queue_depth = FIT_MXD_DATA(mfd);
3361 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_DEPTH, 0,
3362 SKD_N_COMPLETION_ENTRY);
3363 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3364 skdev->last_mtd = mtd;
3365 break;
3366
3367 case FIT_MTD_SET_COMPQ_DEPTH:
3368 SKD_WRITEQ(skdev, skdev->cq_dma_address, FIT_MSG_TO_DEVICE_ARG);
3369 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_ADDR, 0, 0);
3370 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3371 skdev->last_mtd = mtd;
3372 break;
3373
3374 case FIT_MTD_SET_COMPQ_ADDR:
3375 skd_reset_skcomp(skdev);
3376 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_HOST_ID, 0, skdev->devno);
3377 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3378 skdev->last_mtd = mtd;
3379 break;
3380
3381 case FIT_MTD_CMD_LOG_HOST_ID:
3382 skdev->connect_time_stamp = get_seconds();
3383 data = skdev->connect_time_stamp & 0xFFFF;
3384 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_LO, 0, data);
3385 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3386 skdev->last_mtd = mtd;
3387 break;
3388
3389 case FIT_MTD_CMD_LOG_TIME_STAMP_LO:
3390 skdev->drive_jiffies = FIT_MXD_DATA(mfd);
3391 data = (skdev->connect_time_stamp >> 16) & 0xFFFF;
3392 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_HI, 0, data);
3393 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3394 skdev->last_mtd = mtd;
3395 break;
3396
3397 case FIT_MTD_CMD_LOG_TIME_STAMP_HI:
3398 skdev->drive_jiffies |= (FIT_MXD_DATA(mfd) << 16);
3399 mtd = FIT_MXD_CONS(FIT_MTD_ARM_QUEUE, 0, 0);
3400 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3401 skdev->last_mtd = mtd;
3402
3403 pr_err("(%s): Time sync driver=0x%x device=0x%x\n",
3404 skd_name(skdev),
3405 skdev->connect_time_stamp, skdev->drive_jiffies);
3406 break;
3407
3408 case FIT_MTD_ARM_QUEUE:
3409 skdev->last_mtd = 0;
3410 /*
3411 * State should be, or soon will be, FIT_SR_DRIVE_ONLINE.
3412 */
3413 break;
3414
3415 default:
3416 break;
3417 }
3418}
3419
3420static void skd_disable_interrupts(struct skd_device *skdev)
3421{
3422 u32 sense;
3423
3424 sense = SKD_READL(skdev, FIT_CONTROL);
3425 sense &= ~FIT_CR_ENABLE_INTERRUPTS;
3426 SKD_WRITEL(skdev, sense, FIT_CONTROL);
2e44b427 3427 pr_debug("%s:%s:%d sense 0x%x\n",
3428 skdev->name, __func__, __LINE__, sense);
e67f86b3
AB		 3429
3430 /* Note that the 1s is written. A 1-bit means
3431 * disable, a 0 means enable.
3432 */
3433 SKD_WRITEL(skdev, ~0, FIT_INT_MASK_HOST);
3434}
3435
3436static void skd_enable_interrupts(struct skd_device *skdev)
3437{
3438 u32 val;
3439
3440 /* unmask interrupts first */
3441 val = FIT_ISH_FW_STATE_CHANGE +
3442 FIT_ISH_COMPLETION_POSTED + FIT_ISH_MSG_FROM_DEV;
3443
3444 /* Note that the compliment of mask is written. A 1-bit means
3445 * disable, a 0 means enable. */
3446 SKD_WRITEL(skdev, ~val, FIT_INT_MASK_HOST);
2e44b427 3447 pr_debug("%s:%s:%d interrupt mask=0x%x\n",
3448 skdev->name, __func__, __LINE__, ~val);
e67f86b3
AB		 3449
3450 val = SKD_READL(skdev, FIT_CONTROL);
3451 val |= FIT_CR_ENABLE_INTERRUPTS;
2e44b427 3452 pr_debug("%s:%s:%d control=0x%x\n",
3453 skdev->name, __func__, __LINE__, val);
e67f86b3
AB		 3454 SKD_WRITEL(skdev, val, FIT_CONTROL);
3455}
3456
3457/*
3458 *****************************************************************************
3459 * START, STOP, RESTART, QUIESCE, UNQUIESCE
3460 *****************************************************************************
3461 */
3462
3463static void skd_soft_reset(struct skd_device *skdev)
3464{
3465 u32 val;
3466
3467 val = SKD_READL(skdev, FIT_CONTROL);
3468 val |= (FIT_CR_SOFT_RESET);
2e44b427 3469 pr_debug("%s:%s:%d control=0x%x\n",
3470 skdev->name, __func__, __LINE__, val);
e67f86b3
AB		 3471 SKD_WRITEL(skdev, val, FIT_CONTROL);
3472}
3473
3474static void skd_start_device(struct skd_device *skdev)
3475{
3476 unsigned long flags;
3477 u32 sense;
3478 u32 state;
3479
3480 spin_lock_irqsave(&skdev->lock, flags);
3481
3482 /* ack all ghost interrupts */
3483 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3484
3485 sense = SKD_READL(skdev, FIT_STATUS);
3486
2e44b427 3487 pr_debug("%s:%s:%d initial status=0x%x\n",
3488 skdev->name, __func__, __LINE__, sense);
e67f86b3
AB		 3489
3490 state = sense & FIT_SR_DRIVE_STATE_MASK;
3491 skdev->drive_state = state;
3492 skdev->last_mtd = 0;
3493
3494 skdev->state = SKD_DRVR_STATE_STARTING;
3495 skdev->timer_countdown = SKD_STARTING_TIMO;
3496
3497 skd_enable_interrupts(skdev);
3498
3499 switch (skdev->drive_state) {
3500 case FIT_SR_DRIVE_OFFLINE:
3501 pr_err("(%s): Drive offline...\n", skd_name(skdev));
3502 break;
3503
3504 case FIT_SR_DRIVE_FW_BOOTING:
2e44b427 3505 pr_debug("%s:%s:%d FIT_SR_DRIVE_FW_BOOTING %s\n",
3506 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3
AB		 3507 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
3508 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
3509 break;
3510
3511 case FIT_SR_DRIVE_BUSY_SANITIZE:
3512 pr_info("(%s): Start: BUSY_SANITIZE\n",
3513 skd_name(skdev));
3514 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
3515 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3516 break;
3517
3518 case FIT_SR_DRIVE_BUSY_ERASE:
3519 pr_info("(%s): Start: BUSY_ERASE\n", skd_name(skdev));
3520 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
3521 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3522 break;
3523
3524 case FIT_SR_DRIVE_INIT:
3525 case FIT_SR_DRIVE_ONLINE:
3526 skd_soft_reset(skdev);
3527 break;
3528
3529 case FIT_SR_DRIVE_BUSY:
3530 pr_err("(%s): Drive Busy...\n", skd_name(skdev));
3531 skdev->state = SKD_DRVR_STATE_BUSY;
3532 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3533 break;
3534
3535 case FIT_SR_DRIVE_SOFT_RESET:
3536 pr_err("(%s) drive soft reset in prog\n",
3537 skd_name(skdev));
3538 break;
3539
3540 case FIT_SR_DRIVE_FAULT:
3541 /* Fault state is bad...soft reset won't do it...
3542 * Hard reset, maybe, but does it work on device?
3543 * For now, just fault so the system doesn't hang.
3544 */
3545 skd_drive_fault(skdev);
3546 /*start the queue so we can respond with error to requests */
2e44b427 3547 pr_debug("%s:%s:%d starting %s queue\n",
3548 skdev->name, __func__, __LINE__, skdev->name);
6a5ec65b 3549 blk_start_queue(skdev->queue);
e67f86b3
AB		 3550 skdev->gendisk_on = -1;
3551 wake_up_interruptible(&skdev->waitq);
3552 break;
3553
3554 case 0xFF:
3555 /* Most likely the device isn't there or isn't responding
3556 * to the BAR1 addresses. */
3557 skd_drive_disappeared(skdev);
3558 /*start the queue so we can respond with error to requests */
2e44b427 3559 pr_debug("%s:%s:%d starting %s queue to error-out reqs\n",
3560 skdev->name, __func__, __LINE__, skdev->name);
6a5ec65b 3561 blk_start_queue(skdev->queue);
e67f86b3
AB		 3562 skdev->gendisk_on = -1;
3563 wake_up_interruptible(&skdev->waitq);
3564 break;
3565
3566 default:
3567 pr_err("(%s) Start: unknown state %x\n",
3568 skd_name(skdev), skdev->drive_state);
3569 break;
3570 }
3571
3572 state = SKD_READL(skdev, FIT_CONTROL);
2e44b427 3573 pr_debug("%s:%s:%d FIT Control Status=0x%x\n",
3574 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3575
3576 state = SKD_READL(skdev, FIT_INT_STATUS_HOST);
2e44b427 3577 pr_debug("%s:%s:%d Intr Status=0x%x\n",
3578 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3579
3580 state = SKD_READL(skdev, FIT_INT_MASK_HOST);
2e44b427 3581 pr_debug("%s:%s:%d Intr Mask=0x%x\n",
3582 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3583
3584 state = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
2e44b427 3585 pr_debug("%s:%s:%d Msg from Dev=0x%x\n",
3586 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3587
3588 state = SKD_READL(skdev, FIT_HW_VERSION);
2e44b427 3589 pr_debug("%s:%s:%d HW version=0x%x\n",
3590 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3591
3592 spin_unlock_irqrestore(&skdev->lock, flags);
3593}
3594
3595static void skd_stop_device(struct skd_device *skdev)
3596{
3597 unsigned long flags;
3598 struct skd_special_context *skspcl = &skdev->internal_skspcl;
3599 u32 dev_state;
3600 int i;
3601
3602 spin_lock_irqsave(&skdev->lock, flags);
3603
3604 if (skdev->state != SKD_DRVR_STATE_ONLINE) {
3605 pr_err("(%s): skd_stop_device not online no sync\n",
3606 skd_name(skdev));
3607 goto stop_out;
3608 }
3609
3610 if (skspcl->req.state != SKD_REQ_STATE_IDLE) {
3611 pr_err("(%s): skd_stop_device no special\n",
3612 skd_name(skdev));
3613 goto stop_out;
3614 }
3615
3616 skdev->state = SKD_DRVR_STATE_SYNCING;
3617 skdev->sync_done = 0;
3618
3619 skd_send_internal_skspcl(skdev, skspcl, SYNCHRONIZE_CACHE);
3620
3621 spin_unlock_irqrestore(&skdev->lock, flags);
3622
3623 wait_event_interruptible_timeout(skdev->waitq,
3624 (skdev->sync_done), (10 * HZ));
3625
3626 spin_lock_irqsave(&skdev->lock, flags);
3627
3628 switch (skdev->sync_done) {
3629 case 0:
3630 pr_err("(%s): skd_stop_device no sync\n",
3631 skd_name(skdev));
3632 break;
3633 case 1:
3634 pr_err("(%s): skd_stop_device sync done\n",
3635 skd_name(skdev));
3636 break;
3637 default:
3638 pr_err("(%s): skd_stop_device sync error\n",
3639 skd_name(skdev));
3640 }
3641
3642stop_out:
3643 skdev->state = SKD_DRVR_STATE_STOPPING;
3644 spin_unlock_irqrestore(&skdev->lock, flags);
3645
3646 skd_kill_timer(skdev);
3647
3648 spin_lock_irqsave(&skdev->lock, flags);
3649 skd_disable_interrupts(skdev);
3650
3651 /* ensure all ints on device are cleared */
3652 /* soft reset the device to unload with a clean slate */
3653 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3654 SKD_WRITEL(skdev, FIT_CR_SOFT_RESET, FIT_CONTROL);
3655
3656 spin_unlock_irqrestore(&skdev->lock, flags);
3657
3658 /* poll every 100ms, 1 second timeout */
3659 for (i = 0; i < 10; i++) {
3660 dev_state =
3661 SKD_READL(skdev, FIT_STATUS) & FIT_SR_DRIVE_STATE_MASK;
3662 if (dev_state == FIT_SR_DRIVE_INIT)
3663 break;
3664 set_current_state(TASK_INTERRUPTIBLE);
3665 schedule_timeout(msecs_to_jiffies(100));
3666 }
3667
3668 if (dev_state != FIT_SR_DRIVE_INIT)
3669 pr_err("(%s): skd_stop_device state error 0x%02x\n",
3670 skd_name(skdev), dev_state);
3671}
3672
3673/* assume spinlock is held */
3674static void skd_restart_device(struct skd_device *skdev)
3675{
3676 u32 state;
3677
3678 /* ack all ghost interrupts */
3679 SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3680
3681 state = SKD_READL(skdev, FIT_STATUS);
3682
2e44b427 3683 pr_debug("%s:%s:%d drive status=0x%x\n",
3684 skdev->name, __func__, __LINE__, state);
e67f86b3
AB		 3685
3686 state &= FIT_SR_DRIVE_STATE_MASK;
3687 skdev->drive_state = state;
3688 skdev->last_mtd = 0;
3689
3690 skdev->state = SKD_DRVR_STATE_RESTARTING;
3691 skdev->timer_countdown = SKD_RESTARTING_TIMO;
3692
3693 skd_soft_reset(skdev);
3694}
3695
3696/* assume spinlock is held */
3697static int skd_quiesce_dev(struct skd_device *skdev)
3698{
3699 int rc = 0;
3700
3701 switch (skdev->state) {
3702 case SKD_DRVR_STATE_BUSY:
3703 case SKD_DRVR_STATE_BUSY_IMMINENT:
2e44b427 3704 pr_debug("%s:%s:%d stopping %s queue\n",
3705 skdev->name, __func__, __LINE__, skdev->name);
6a5ec65b 3706 blk_stop_queue(skdev->queue);
e67f86b3
AB		 3707 break;
3708 case SKD_DRVR_STATE_ONLINE:
3709 case SKD_DRVR_STATE_STOPPING:
3710 case SKD_DRVR_STATE_SYNCING:
3711 case SKD_DRVR_STATE_PAUSING:
3712 case SKD_DRVR_STATE_PAUSED:
3713 case SKD_DRVR_STATE_STARTING:
3714 case SKD_DRVR_STATE_RESTARTING:
3715 case SKD_DRVR_STATE_RESUMING:
3716 default:
3717 rc = -EINVAL;
2e44b427 3718 pr_debug("%s:%s:%d state [%d] not implemented\n",
3719 skdev->name, __func__, __LINE__, skdev->state);
e67f86b3
AB		 3720 }
3721 return rc;
3722}
3723
3724/* assume spinlock is held */
3725static int skd_unquiesce_dev(struct skd_device *skdev)
3726{
3727 int prev_driver_state = skdev->state;
3728
3729 skd_log_skdev(skdev, "unquiesce");
3730 if (skdev->state == SKD_DRVR_STATE_ONLINE) {
2e44b427 3731 pr_debug("%s:%s:%d **** device already ONLINE\n",
3732 skdev->name, __func__, __LINE__);
e67f86b3
AB		 3733 return 0;
3734 }
3735 if (skdev->drive_state != FIT_SR_DRIVE_ONLINE) {
3736 /*
3737 * If there has been an state change to other than
3738 * ONLINE, we will rely on controller state change
3739 * to come back online and restart the queue.
3740 * The BUSY state means that driver is ready to
3741 * continue normal processing but waiting for controller
3742 * to become available.
3743 */
3744 skdev->state = SKD_DRVR_STATE_BUSY;
2e44b427 3745 pr_debug("%s:%s:%d drive BUSY state\n",
3746 skdev->name, __func__, __LINE__);
e67f86b3
AB		 3747 return 0;
3748 }
3749
3750 /*
3751 * Drive has just come online, driver is either in startup,
3752 * paused performing a task, or bust waiting for hardware.
3753 */
3754 switch (skdev->state) {
3755 case SKD_DRVR_STATE_PAUSED:
3756 case SKD_DRVR_STATE_BUSY:
3757 case SKD_DRVR_STATE_BUSY_IMMINENT:
3758 case SKD_DRVR_STATE_BUSY_ERASE:
3759 case SKD_DRVR_STATE_STARTING:
3760 case SKD_DRVR_STATE_RESTARTING:
3761 case SKD_DRVR_STATE_FAULT:
3762 case SKD_DRVR_STATE_IDLE:
3763 case SKD_DRVR_STATE_LOAD:
3764 skdev->state = SKD_DRVR_STATE_ONLINE;
3765 pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
3766 skd_name(skdev),
3767 skd_skdev_state_to_str(prev_driver_state),
3768 prev_driver_state, skd_skdev_state_to_str(skdev->state),
3769 skdev->state);
2e44b427 3770 pr_debug("%s:%s:%d **** device ONLINE...starting block queue\n",
3771 skdev->name, __func__, __LINE__);
3772 pr_debug("%s:%s:%d starting %s queue\n",
3773 skdev->name, __func__, __LINE__, skdev->name);
e67f86b3 3774 pr_info("(%s): STEC s1120 ONLINE\n", skd_name(skdev));
6a5ec65b 3775 blk_start_queue(skdev->queue);
e67f86b3
AB		 3776 skdev->gendisk_on = 1;
3777 wake_up_interruptible(&skdev->waitq);
3778 break;
3779
3780 case SKD_DRVR_STATE_DISAPPEARED:
3781 default:
2e44b427 3782 pr_debug("%s:%s:%d **** driver state %d, not implemented \n",
3783 skdev->name, __func__, __LINE__,
3784 skdev->state);
e67f86b3
AB		 3785 return -EBUSY;
3786 }
3787 return 0;
3788}
3789
3790/*
3791 *****************************************************************************
3792 * PCIe MSI/MSI-X INTERRUPT HANDLERS
3793 *****************************************************************************
3794 */
3795
3796static irqreturn_t skd_reserved_isr(int irq, void *skd_host_data)
3797{
3798 struct skd_device *skdev = skd_host_data;
3799 unsigned long flags;
3800
3801 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3802 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3803 skdev->name, __func__, __LINE__,
3804 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3805 pr_err("(%s): MSIX reserved irq %d = 0x%x\n", skd_name(skdev),
3806 irq, SKD_READL(skdev, FIT_INT_STATUS_HOST));
3807 SKD_WRITEL(skdev, FIT_INT_RESERVED_MASK, FIT_INT_STATUS_HOST);
3808 spin_unlock_irqrestore(&skdev->lock, flags);
3809 return IRQ_HANDLED;
3810}
3811
3812static irqreturn_t skd_statec_isr(int irq, void *skd_host_data)
3813{
3814 struct skd_device *skdev = skd_host_data;
3815 unsigned long flags;
3816
3817 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3818 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3819 skdev->name, __func__, __LINE__,
3820 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3821 SKD_WRITEL(skdev, FIT_ISH_FW_STATE_CHANGE, FIT_INT_STATUS_HOST);
3822 skd_isr_fwstate(skdev);
3823 spin_unlock_irqrestore(&skdev->lock, flags);
3824 return IRQ_HANDLED;
3825}
3826
3827static irqreturn_t skd_comp_q(int irq, void *skd_host_data)
3828{
3829 struct skd_device *skdev = skd_host_data;
3830 unsigned long flags;
3831 int flush_enqueued = 0;
3832 int deferred;
3833
3834 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3835 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3836 skdev->name, __func__, __LINE__,
3837 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3838 SKD_WRITEL(skdev, FIT_ISH_COMPLETION_POSTED, FIT_INT_STATUS_HOST);
3839 deferred = skd_isr_completion_posted(skdev, skd_isr_comp_limit,
3840 &flush_enqueued);
e67f86b3
AB		 3841 if (flush_enqueued)
3842 skd_request_fn(skdev->queue);
3843
3844 if (deferred)
3845 schedule_work(&skdev->completion_worker);
3846 else if (!flush_enqueued)
3847 skd_request_fn(skdev->queue);
3848
3849 spin_unlock_irqrestore(&skdev->lock, flags);
3850
3851 return IRQ_HANDLED;
3852}
3853
3854static irqreturn_t skd_msg_isr(int irq, void *skd_host_data)
3855{
3856 struct skd_device *skdev = skd_host_data;
3857 unsigned long flags;
3858
3859 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3860 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3861 skdev->name, __func__, __LINE__,
3862 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3863 SKD_WRITEL(skdev, FIT_ISH_MSG_FROM_DEV, FIT_INT_STATUS_HOST);
3864 skd_isr_msg_from_dev(skdev);
3865 spin_unlock_irqrestore(&skdev->lock, flags);
3866 return IRQ_HANDLED;
3867}
3868
3869static irqreturn_t skd_qfull_isr(int irq, void *skd_host_data)
3870{
3871 struct skd_device *skdev = skd_host_data;
3872 unsigned long flags;
3873
3874 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 3875 pr_debug("%s:%s:%d MSIX = 0x%x\n",
3876 skdev->name, __func__, __LINE__,
3877 SKD_READL(skdev, FIT_INT_STATUS_HOST));
e67f86b3
AB		 3878 SKD_WRITEL(skdev, FIT_INT_QUEUE_FULL, FIT_INT_STATUS_HOST);
3879 spin_unlock_irqrestore(&skdev->lock, flags);
3880 return IRQ_HANDLED;
3881}
3882
3883/*
3884 *****************************************************************************
3885 * PCIe MSI/MSI-X SETUP
3886 *****************************************************************************
3887 */
3888
3889struct skd_msix_entry {
3890 int have_irq;
3891 u32 vector;
3892 u32 entry;
3893 struct skd_device *rsp;
3894 char isr_name[30];
3895};
3896
3897struct skd_init_msix_entry {
3898 const char *name;
3899 irq_handler_t handler;
3900};
3901
3902#define SKD_MAX_MSIX_COUNT 13
3903#define SKD_MIN_MSIX_COUNT 7
3904#define SKD_BASE_MSIX_IRQ 4
3905
3906static struct skd_init_msix_entry msix_entries[SKD_MAX_MSIX_COUNT] = {
3907 { "(DMA 0)", skd_reserved_isr },
3908 { "(DMA 1)", skd_reserved_isr },
3909 { "(DMA 2)", skd_reserved_isr },
3910 { "(DMA 3)", skd_reserved_isr },
3911 { "(State Change)", skd_statec_isr },
3912 { "(COMPL_Q)", skd_comp_q },
3913 { "(MSG)", skd_msg_isr },
3914 { "(Reserved)", skd_reserved_isr },
3915 { "(Reserved)", skd_reserved_isr },
3916 { "(Queue Full 0)", skd_qfull_isr },
3917 { "(Queue Full 1)", skd_qfull_isr },
3918 { "(Queue Full 2)", skd_qfull_isr },
3919 { "(Queue Full 3)", skd_qfull_isr },
3920};
3921
3922static void skd_release_msix(struct skd_device *skdev)
3923{
3924 struct skd_msix_entry *qentry;
3925 int i;
3926
3927 if (skdev->msix_entries == NULL)
3928 return;
3929 for (i = 0; i < skdev->msix_count; i++) {
3930 qentry = &skdev->msix_entries[i];
3931 skdev = qentry->rsp;
3932
3933 if (qentry->have_irq)
3934 devm_free_irq(&skdev->pdev->dev,
3935 qentry->vector, qentry->rsp);
3936 }
3937 pci_disable_msix(skdev->pdev);
3938 kfree(skdev->msix_entries);
3939 skdev->msix_count = 0;
3940 skdev->msix_entries = NULL;
3941}
3942
3943static int skd_acquire_msix(struct skd_device *skdev)
3944{
3945 int i, rc;
3946 struct pci_dev *pdev;
3947 struct msix_entry *entries = NULL;
3948 struct skd_msix_entry *qentry;
3949
3950 pdev = skdev->pdev;
3951 skdev->msix_count = SKD_MAX_MSIX_COUNT;
3952 entries = kzalloc(sizeof(struct msix_entry) * SKD_MAX_MSIX_COUNT,
3953 GFP_KERNEL);
3954 if (!entries)
3955 return -ENOMEM;
3956
3957 for (i = 0; i < SKD_MAX_MSIX_COUNT; i++)
3958 entries[i].entry = i;
3959
3960 rc = pci_enable_msix(pdev, entries, SKD_MAX_MSIX_COUNT);
3961 if (rc < 0)
3962 goto msix_out;
3963 if (rc) {
3964 if (rc < SKD_MIN_MSIX_COUNT) {
3965 pr_err("(%s): failed to enable MSI-X %d\n",
3966 skd_name(skdev), rc);
3967 goto msix_out;
3968 }
2e44b427 3969 pr_debug("%s:%s:%d %s: <%s> allocated %d MSI-X vectors\n",
3970 skdev->name, __func__, __LINE__,
3971 pci_name(pdev), skdev->name, rc);
e67f86b3
AB		 3972
3973 skdev->msix_count = rc;
3974 rc = pci_enable_msix(pdev, entries, skdev->msix_count);
3975 if (rc) {
3976 pr_err("(%s): failed to enable MSI-X "
3977 "support (%d) %d\n",
3978 skd_name(skdev), skdev->msix_count, rc);
3979 goto msix_out;
3980 }
3981 }
3982 skdev->msix_entries = kzalloc(sizeof(struct skd_msix_entry) *
3983 skdev->msix_count, GFP_KERNEL);
3984 if (!skdev->msix_entries) {
3985 rc = -ENOMEM;
3986 skdev->msix_count = 0;
3987 pr_err("(%s): msix table allocation error\n",
3988 skd_name(skdev));
3989 goto msix_out;
3990 }
3991
3992 qentry = skdev->msix_entries;
3993 for (i = 0; i < skdev->msix_count; i++) {
3994 qentry->vector = entries[i].vector;
3995 qentry->entry = entries[i].entry;
3996 qentry->rsp = NULL;
3997 qentry->have_irq = 0;
2e44b427 3998 pr_debug("%s:%s:%d %s: <%s> msix (%d) vec %d, entry %x\n",
3999 skdev->name, __func__, __LINE__,
4000 pci_name(pdev), skdev->name,
4001 i, qentry->vector, qentry->entry);
e67f86b3
AB		 4002 qentry++;
4003 }
4004
4005 /* Enable MSI-X vectors for the base queue */
4006 for (i = 0; i < SKD_MAX_MSIX_COUNT; i++) {
4007 qentry = &skdev->msix_entries[i];
4008 snprintf(qentry->isr_name, sizeof(qentry->isr_name),
4009 "%s%d-msix %s", DRV_NAME, skdev->devno,
4010 msix_entries[i].name);
4011 rc = devm_request_irq(&skdev->pdev->dev, qentry->vector,
4012 msix_entries[i].handler, 0,
4013 qentry->isr_name, skdev);
4014 if (rc) {
4015 pr_err("(%s): Unable to register(%d) MSI-X "
4016 "handler %d: %s\n",
4017 skd_name(skdev), rc, i, qentry->isr_name);
4018 goto msix_out;
4019 } else {
4020 qentry->have_irq = 1;
4021 qentry->rsp = skdev;
4022 }
4023 }
2e44b427 4024 pr_debug("%s:%s:%d %s: <%s> msix %d irq(s) enabled\n",
4025 skdev->name, __func__, __LINE__,
4026 pci_name(pdev), skdev->name, skdev->msix_count);
e67f86b3
AB		 4027 return 0;
4028
4029msix_out:
4030 if (entries)
4031 kfree(entries);
4032 skd_release_msix(skdev);
4033 return rc;
4034}
4035
4036static int skd_acquire_irq(struct skd_device *skdev)
4037{
4038 int rc;
4039 struct pci_dev *pdev;
4040
4041 pdev = skdev->pdev;
4042 skdev->msix_count = 0;
4043
4044RETRY_IRQ_TYPE:
4045 switch (skdev->irq_type) {
4046 case SKD_IRQ_MSIX:
4047 rc = skd_acquire_msix(skdev);
4048 if (!rc)
4049 pr_info("(%s): MSI-X %d irqs enabled\n",
4050 skd_name(skdev), skdev->msix_count);
4051 else {
4052 pr_err(
4053 "(%s): failed to enable MSI-X, re-trying with MSI %d\n",
4054 skd_name(skdev), rc);
4055 skdev->irq_type = SKD_IRQ_MSI;
4056 goto RETRY_IRQ_TYPE;
4057 }
4058 break;
4059 case SKD_IRQ_MSI:
4060 snprintf(skdev->isr_name, sizeof(skdev->isr_name), "%s%d-msi",
4061 DRV_NAME, skdev->devno);
4062 rc = pci_enable_msi(pdev);
4063 if (!rc) {
4064 rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr, 0,
4065 skdev->isr_name, skdev);
4066 if (rc) {
4067 pci_disable_msi(pdev);
4068 pr_err(
4069 "(%s): failed to allocate the MSI interrupt %d\n",
4070 skd_name(skdev), rc);
4071 goto RETRY_IRQ_LEGACY;
4072 }
4073 pr_info("(%s): MSI irq %d enabled\n",
4074 skd_name(skdev), pdev->irq);
4075 } else {
4076RETRY_IRQ_LEGACY:
4077 pr_err(
4078 "(%s): failed to enable MSI, re-trying with LEGACY %d\n",
4079 skd_name(skdev), rc);
4080 skdev->irq_type = SKD_IRQ_LEGACY;
4081 goto RETRY_IRQ_TYPE;
4082 }
4083 break;
4084 case SKD_IRQ_LEGACY:
4085 snprintf(skdev->isr_name, sizeof(skdev->isr_name),
4086 "%s%d-legacy", DRV_NAME, skdev->devno);
4087 rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr,
4088 IRQF_SHARED, skdev->isr_name, skdev);
4089 if (!rc)
4090 pr_info("(%s): LEGACY irq %d enabled\n",
4091 skd_name(skdev), pdev->irq);
4092 else
4093 pr_err("(%s): request LEGACY irq error %d\n",
4094 skd_name(skdev), rc);
4095 break;
4096 default:
4097 pr_info("(%s): irq_type %d invalid, re-set to %d\n",
4098 skd_name(skdev), skdev->irq_type, SKD_IRQ_DEFAULT);
4099 skdev->irq_type = SKD_IRQ_LEGACY;
4100 goto RETRY_IRQ_TYPE;
4101 }
4102 return rc;
4103}
4104
4105static void skd_release_irq(struct skd_device *skdev)
4106{
4107 switch (skdev->irq_type) {
4108 case SKD_IRQ_MSIX:
4109 skd_release_msix(skdev);
4110 break;
4111 case SKD_IRQ_MSI:
4112 devm_free_irq(&skdev->pdev->dev, skdev->pdev->irq, skdev);
4113 pci_disable_msi(skdev->pdev);
4114 break;
4115 case SKD_IRQ_LEGACY:
4116 devm_free_irq(&skdev->pdev->dev, skdev->pdev->irq, skdev);
4117 break;
4118 default:
4119 pr_err("(%s): wrong irq type %d!",
4120 skd_name(skdev), skdev->irq_type);
4121 break;
4122 }
4123}
4124
4125/*
4126 *****************************************************************************
4127 * CONSTRUCT
4128 *****************************************************************************
4129 */
4130
4131static int skd_cons_skcomp(struct skd_device *skdev);
4132static int skd_cons_skmsg(struct skd_device *skdev);
4133static int skd_cons_skreq(struct skd_device *skdev);
4134static int skd_cons_skspcl(struct skd_device *skdev);
4135static int skd_cons_sksb(struct skd_device *skdev);
4136static struct fit_sg_descriptor *skd_cons_sg_list(struct skd_device *skdev,
4137 u32 n_sg,
4138 dma_addr_t *ret_dma_addr);
4139static int skd_cons_disk(struct skd_device *skdev);
4140
4141#define SKD_N_DEV_TABLE 16u
4142static u32 skd_next_devno;
4143
4144static struct skd_device *skd_construct(struct pci_dev *pdev)
4145{
4146 struct skd_device *skdev;
4147 int blk_major = skd_major;
4148 int rc;
4149
4150 skdev = kzalloc(sizeof(*skdev), GFP_KERNEL);
4151
4152 if (!skdev) {
4153 pr_err(PFX "(%s): memory alloc failure\n",
4154 pci_name(pdev));
4155 return NULL;
4156 }
4157
4158 skdev->state = SKD_DRVR_STATE_LOAD;
4159 skdev->pdev = pdev;
4160 skdev->devno = skd_next_devno++;
4161 skdev->major = blk_major;
4162 skdev->irq_type = skd_isr_type;
4163 sprintf(skdev->name, DRV_NAME "%d", skdev->devno);
4164 skdev->dev_max_queue_depth = 0;
4165
4166 skdev->num_req_context = skd_max_queue_depth;
4167 skdev->num_fitmsg_context = skd_max_queue_depth;
4168 skdev->n_special = skd_max_pass_thru;
4169 skdev->cur_max_queue_depth = 1;
4170 skdev->queue_low_water_mark = 1;
4171 skdev->proto_ver = 99;
4172 skdev->sgs_per_request = skd_sgs_per_request;
4173 skdev->dbg_level = skd_dbg_level;
4174
e67f86b3
AB		 4175 atomic_set(&skdev->device_count, 0);
4176
4177 spin_lock_init(&skdev->lock);
4178
4179 INIT_WORK(&skdev->completion_worker, skd_completion_worker);
e67f86b3 4180
2e44b427 4181 pr_debug("%s:%s:%d skcomp\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4182 rc = skd_cons_skcomp(skdev);
4183 if (rc < 0)
4184 goto err_out;
4185
2e44b427 4186 pr_debug("%s:%s:%d skmsg\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4187 rc = skd_cons_skmsg(skdev);
4188 if (rc < 0)
4189 goto err_out;
4190
2e44b427 4191 pr_debug("%s:%s:%d skreq\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4192 rc = skd_cons_skreq(skdev);
4193 if (rc < 0)
4194 goto err_out;
4195
2e44b427 4196 pr_debug("%s:%s:%d skspcl\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4197 rc = skd_cons_skspcl(skdev);
4198 if (rc < 0)
4199 goto err_out;
4200
2e44b427 4201 pr_debug("%s:%s:%d sksb\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4202 rc = skd_cons_sksb(skdev);
4203 if (rc < 0)
4204 goto err_out;
4205
2e44b427 4206 pr_debug("%s:%s:%d disk\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4207 rc = skd_cons_disk(skdev);
4208 if (rc < 0)
4209 goto err_out;
4210
2e44b427 4211 pr_debug("%s:%s:%d VICTORY\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4212 return skdev;
4213
4214err_out:
2e44b427 4215 pr_debug("%s:%s:%d construct failed\n",
4216 skdev->name, __func__, __LINE__);
e67f86b3
AB		 4217 skd_destruct(skdev);
4218 return NULL;
4219}
4220
4221static int skd_cons_skcomp(struct skd_device *skdev)
4222{
4223 int rc = 0;
4224 struct fit_completion_entry_v1 *skcomp;
4225 u32 nbytes;
4226
4227 nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
4228 nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
4229
2e44b427 4230 pr_debug("%s:%s:%d comp pci_alloc, total bytes %d entries %d\n",
4231 skdev->name, __func__, __LINE__,
4232 nbytes, SKD_N_COMPLETION_ENTRY);
e67f86b3
AB		 4233
4234 skcomp = pci_alloc_consistent(skdev->pdev, nbytes,
4235 &skdev->cq_dma_address);
4236
4237 if (skcomp == NULL) {
4238 rc = -ENOMEM;
4239 goto err_out;
4240 }
4241
4242 memset(skcomp, 0, nbytes);
4243
4244 skdev->skcomp_table = skcomp;
4245 skdev->skerr_table = (struct fit_comp_error_info *)((char *)skcomp +
4246 sizeof(*skcomp) *
4247 SKD_N_COMPLETION_ENTRY);
4248
4249err_out:
4250 return rc;
4251}
4252
4253static int skd_cons_skmsg(struct skd_device *skdev)
4254{
4255 int rc = 0;
4256 u32 i;
4257
2e44b427 4258 pr_debug("%s:%s:%d skmsg_table kzalloc, struct %lu, count %u total %lu\n",
4259 skdev->name, __func__, __LINE__,
4260 sizeof(struct skd_fitmsg_context),
4261 skdev->num_fitmsg_context,
4262 sizeof(struct skd_fitmsg_context) * skdev->num_fitmsg_context);
e67f86b3
AB		 4263
4264 skdev->skmsg_table = kzalloc(sizeof(struct skd_fitmsg_context)
4265 *skdev->num_fitmsg_context, GFP_KERNEL);
4266 if (skdev->skmsg_table == NULL) {
4267 rc = -ENOMEM;
4268 goto err_out;
4269 }
4270
4271 for (i = 0; i < skdev->num_fitmsg_context; i++) {
4272 struct skd_fitmsg_context *skmsg;
4273
4274 skmsg = &skdev->skmsg_table[i];
4275
4276 skmsg->id = i + SKD_ID_FIT_MSG;
4277
4278 skmsg->state = SKD_MSG_STATE_IDLE;
4279 skmsg->msg_buf = pci_alloc_consistent(skdev->pdev,
4280 SKD_N_FITMSG_BYTES + 64,
4281 &skmsg->mb_dma_address);
4282
4283 if (skmsg->msg_buf == NULL) {
4284 rc = -ENOMEM;
4285 goto err_out;
4286 }
4287
4288 skmsg->offset = (u32)((u64)skmsg->msg_buf &
4289 (~FIT_QCMD_BASE_ADDRESS_MASK));
4290 skmsg->msg_buf += ~FIT_QCMD_BASE_ADDRESS_MASK;
4291 skmsg->msg_buf = (u8 *)((u64)skmsg->msg_buf &
4292 FIT_QCMD_BASE_ADDRESS_MASK);
4293 skmsg->mb_dma_address += ~FIT_QCMD_BASE_ADDRESS_MASK;
4294 skmsg->mb_dma_address &= FIT_QCMD_BASE_ADDRESS_MASK;
4295 memset(skmsg->msg_buf, 0, SKD_N_FITMSG_BYTES);
4296
4297 skmsg->next = &skmsg[1];
4298 }
4299
4300 /* Free list is in order starting with the 0th entry. */
4301 skdev->skmsg_table[i - 1].next = NULL;
4302 skdev->skmsg_free_list = skdev->skmsg_table;
4303
4304err_out:
4305 return rc;
4306}
4307
4308static int skd_cons_skreq(struct skd_device *skdev)
4309{
4310 int rc = 0;
4311 u32 i;
4312
2e44b427 4313 pr_debug("%s:%s:%d skreq_table kzalloc, struct %lu, count %u total %lu\n",
4314 skdev->name, __func__, __LINE__,
4315 sizeof(struct skd_request_context),
4316 skdev->num_req_context,
4317 sizeof(struct skd_request_context) * skdev->num_req_context);
e67f86b3
AB		 4318
4319 skdev->skreq_table = kzalloc(sizeof(struct skd_request_context)
4320 * skdev->num_req_context, GFP_KERNEL);
4321 if (skdev->skreq_table == NULL) {
4322 rc = -ENOMEM;
4323 goto err_out;
4324 }
4325
2e44b427 4326 pr_debug("%s:%s:%d alloc sg_table sg_per_req %u scatlist %lu total %lu\n",
4327 skdev->name, __func__, __LINE__,
4328 skdev->sgs_per_request, sizeof(struct scatterlist),
4329 skdev->sgs_per_request * sizeof(struct scatterlist));
e67f86b3
AB		 4330
4331 for (i = 0; i < skdev->num_req_context; i++) {
4332 struct skd_request_context *skreq;
4333
4334 skreq = &skdev->skreq_table[i];
4335
4336 skreq->id = i + SKD_ID_RW_REQUEST;
4337 skreq->state = SKD_REQ_STATE_IDLE;
4338
4339 skreq->sg = kzalloc(sizeof(struct scatterlist) *
4340 skdev->sgs_per_request, GFP_KERNEL);
4341 if (skreq->sg == NULL) {
4342 rc = -ENOMEM;
4343 goto err_out;
4344 }
4345 sg_init_table(skreq->sg, skdev->sgs_per_request);
4346
4347 skreq->sksg_list = skd_cons_sg_list(skdev,
4348 skdev->sgs_per_request,
4349 &skreq->sksg_dma_address);
4350
4351 if (skreq->sksg_list == NULL) {
4352 rc = -ENOMEM;
4353 goto err_out;
4354 }
4355
4356 skreq->next = &skreq[1];
4357 }
4358
4359 /* Free list is in order starting with the 0th entry. */
4360 skdev->skreq_table[i - 1].next = NULL;
4361 skdev->skreq_free_list = skdev->skreq_table;
4362
4363err_out:
4364 return rc;
4365}
4366
4367static int skd_cons_skspcl(struct skd_device *skdev)
4368{
4369 int rc = 0;
4370 u32 i, nbytes;
4371
2e44b427 4372 pr_debug("%s:%s:%d skspcl_table kzalloc, struct %lu, count %u total %lu\n",
4373 skdev->name, __func__, __LINE__,
4374 sizeof(struct skd_special_context),
4375 skdev->n_special,
4376 sizeof(struct skd_special_context) * skdev->n_special);
e67f86b3
AB		 4377
4378 skdev->skspcl_table = kzalloc(sizeof(struct skd_special_context)
4379 * skdev->n_special, GFP_KERNEL);
4380 if (skdev->skspcl_table == NULL) {
4381 rc = -ENOMEM;
4382 goto err_out;
4383 }
4384
4385 for (i = 0; i < skdev->n_special; i++) {
4386 struct skd_special_context *skspcl;
4387
4388 skspcl = &skdev->skspcl_table[i];
4389
4390 skspcl->req.id = i + SKD_ID_SPECIAL_REQUEST;
4391 skspcl->req.state = SKD_REQ_STATE_IDLE;
4392
4393 skspcl->req.next = &skspcl[1].req;
4394
4395 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4396
4397 skspcl->msg_buf = pci_alloc_consistent(skdev->pdev, nbytes,
4398 &skspcl->mb_dma_address);
4399 if (skspcl->msg_buf == NULL) {
4400 rc = -ENOMEM;
4401 goto err_out;
4402 }
4403
4404 memset(skspcl->msg_buf, 0, nbytes);
4405
4406 skspcl->req.sg = kzalloc(sizeof(struct scatterlist) *
4407 SKD_N_SG_PER_SPECIAL, GFP_KERNEL);
4408 if (skspcl->req.sg == NULL) {
4409 rc = -ENOMEM;
4410 goto err_out;
4411 }
4412
4413 skspcl->req.sksg_list = skd_cons_sg_list(skdev,
4414 SKD_N_SG_PER_SPECIAL,
4415 &skspcl->req.
4416 sksg_dma_address);
4417 if (skspcl->req.sksg_list == NULL) {
4418 rc = -ENOMEM;
4419 goto err_out;
4420 }
4421 }
4422
4423 /* Free list is in order starting with the 0th entry. */
4424 skdev->skspcl_table[i - 1].req.next = NULL;
4425 skdev->skspcl_free_list = skdev->skspcl_table;
4426
4427 return rc;
4428
4429err_out:
4430 return rc;
4431}
4432
4433static int skd_cons_sksb(struct skd_device *skdev)
4434{
4435 int rc = 0;
4436 struct skd_special_context *skspcl;
4437 u32 nbytes;
4438
4439 skspcl = &skdev->internal_skspcl;
4440
4441 skspcl->req.id = 0 + SKD_ID_INTERNAL;
4442 skspcl->req.state = SKD_REQ_STATE_IDLE;
4443
4444 nbytes = SKD_N_INTERNAL_BYTES;
4445
4446 skspcl->data_buf = pci_alloc_consistent(skdev->pdev, nbytes,
4447 &skspcl->db_dma_address);
4448 if (skspcl->data_buf == NULL) {
4449 rc = -ENOMEM;
4450 goto err_out;
4451 }
4452
4453 memset(skspcl->data_buf, 0, nbytes);
4454
4455 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4456 skspcl->msg_buf = pci_alloc_consistent(skdev->pdev, nbytes,
4457 &skspcl->mb_dma_address);
4458 if (skspcl->msg_buf == NULL) {
4459 rc = -ENOMEM;
4460 goto err_out;
4461 }
4462
4463 memset(skspcl->msg_buf, 0, nbytes);
4464
4465 skspcl->req.sksg_list = skd_cons_sg_list(skdev, 1,
4466 &skspcl->req.sksg_dma_address);
4467 if (skspcl->req.sksg_list == NULL) {
4468 rc = -ENOMEM;
4469 goto err_out;
4470 }
4471
4472 if (!skd_format_internal_skspcl(skdev)) {
4473 rc = -EINVAL;
4474 goto err_out;
4475 }
4476
4477err_out:
4478 return rc;
4479}
4480
4481static struct fit_sg_descriptor *skd_cons_sg_list(struct skd_device *skdev,
4482 u32 n_sg,
4483 dma_addr_t *ret_dma_addr)
4484{
4485 struct fit_sg_descriptor *sg_list;
4486 u32 nbytes;
4487
4488 nbytes = sizeof(*sg_list) * n_sg;
4489
4490 sg_list = pci_alloc_consistent(skdev->pdev, nbytes, ret_dma_addr);
4491
4492 if (sg_list != NULL) {
4493 uint64_t dma_address = *ret_dma_addr;
4494 u32 i;
4495
4496 memset(sg_list, 0, nbytes);
4497
4498 for (i = 0; i < n_sg - 1; i++) {
4499 uint64_t ndp_off;
4500 ndp_off = (i + 1) * sizeof(struct fit_sg_descriptor);
4501
4502 sg_list[i].next_desc_ptr = dma_address + ndp_off;
4503 }
4504 sg_list[i].next_desc_ptr = 0LL;
4505 }
4506
4507 return sg_list;
4508}
4509
4510static int skd_cons_disk(struct skd_device *skdev)
4511{
4512 int rc = 0;
4513 struct gendisk *disk;
4514 struct request_queue *q;
4515 unsigned long flags;
4516
4517 disk = alloc_disk(SKD_MINORS_PER_DEVICE);
4518 if (!disk) {
4519 rc = -ENOMEM;
4520 goto err_out;
4521 }
4522
4523 skdev->disk = disk;
4524 sprintf(disk->disk_name, DRV_NAME "%u", skdev->devno);
4525
4526 disk->major = skdev->major;
4527 disk->first_minor = skdev->devno * SKD_MINORS_PER_DEVICE;
4528 disk->fops = &skd_blockdev_ops;
4529 disk->private_data = skdev;
4530
fcd37eb3 4531 q = blk_init_queue(skd_request_fn, &skdev->lock);
e67f86b3
AB		 4532 if (!q) {
4533 rc = -ENOMEM;
4534 goto err_out;
4535 }
4536
4537 skdev->queue = q;
4538 disk->queue = q;
4539 q->queuedata = skdev;
4540
e67f86b3
AB		 4541 blk_queue_flush(q, REQ_FLUSH | REQ_FUA);
4542 blk_queue_max_segments(q, skdev->sgs_per_request);
4543 blk_queue_max_hw_sectors(q, SKD_N_MAX_SECTORS);
4544
4545 /* set sysfs ptimal_io_size to 8K */
4546 blk_queue_io_opt(q, 8192);
4547
4548 /* DISCARD Flag initialization. */
4549 q->limits.discard_granularity = 8192;
4550 q->limits.discard_alignment = 0;
4551 q->limits.max_discard_sectors = UINT_MAX >> 9;
4552 q->limits.discard_zeroes_data = 1;
4553 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
4554 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
4555
4556 spin_lock_irqsave(&skdev->lock, flags);
2e44b427 4557 pr_debug("%s:%s:%d stopping %s queue\n",
4558 skdev->name, __func__, __LINE__, skdev->name);
6a5ec65b 4559 blk_stop_queue(skdev->queue);
e67f86b3
AB		 4560 spin_unlock_irqrestore(&skdev->lock, flags);
4561
4562err_out:
4563 return rc;
4564}
4565
4566/*
4567 *****************************************************************************
4568 * DESTRUCT (FREE)
4569 *****************************************************************************
4570 */
4571
4572static void skd_free_skcomp(struct skd_device *skdev);
4573static void skd_free_skmsg(struct skd_device *skdev);
4574static void skd_free_skreq(struct skd_device *skdev);
4575static void skd_free_skspcl(struct skd_device *skdev);
4576static void skd_free_sksb(struct skd_device *skdev);
4577static void skd_free_sg_list(struct skd_device *skdev,
4578 struct fit_sg_descriptor *sg_list,
4579 u32 n_sg, dma_addr_t dma_addr);
4580static void skd_free_disk(struct skd_device *skdev);
4581
4582static void skd_destruct(struct skd_device *skdev)
4583{
4584 if (skdev == NULL)
4585 return;
4586
4587
2e44b427 4588 pr_debug("%s:%s:%d disk\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4589 skd_free_disk(skdev);
4590
2e44b427 4591 pr_debug("%s:%s:%d sksb\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4592 skd_free_sksb(skdev);
4593
2e44b427 4594 pr_debug("%s:%s:%d skspcl\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4595 skd_free_skspcl(skdev);
4596
2e44b427 4597 pr_debug("%s:%s:%d skreq\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4598 skd_free_skreq(skdev);
4599
2e44b427 4600 pr_debug("%s:%s:%d skmsg\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4601 skd_free_skmsg(skdev);
4602
2e44b427 4603 pr_debug("%s:%s:%d skcomp\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4604 skd_free_skcomp(skdev);
4605
2e44b427 4606 pr_debug("%s:%s:%d skdev\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4607 kfree(skdev);
4608}
4609
4610static void skd_free_skcomp(struct skd_device *skdev)
4611{
4612 if (skdev->skcomp_table != NULL) {
4613 u32 nbytes;
4614
4615 nbytes = sizeof(skdev->skcomp_table[0]) *
4616 SKD_N_COMPLETION_ENTRY;
4617 pci_free_consistent(skdev->pdev, nbytes,
4618 skdev->skcomp_table, skdev->cq_dma_address);
4619 }
4620
4621 skdev->skcomp_table = NULL;
4622 skdev->cq_dma_address = 0;
4623}
4624
4625static void skd_free_skmsg(struct skd_device *skdev)
4626{
4627 u32 i;
4628
4629 if (skdev->skmsg_table == NULL)
4630 return;
4631
4632 for (i = 0; i < skdev->num_fitmsg_context; i++) {
4633 struct skd_fitmsg_context *skmsg;
4634
4635 skmsg = &skdev->skmsg_table[i];
4636
4637 if (skmsg->msg_buf != NULL) {
4638 skmsg->msg_buf += skmsg->offset;
4639 skmsg->mb_dma_address += skmsg->offset;
4640 pci_free_consistent(skdev->pdev, SKD_N_FITMSG_BYTES,
4641 skmsg->msg_buf,
4642 skmsg->mb_dma_address);
4643 }
4644 skmsg->msg_buf = NULL;
4645 skmsg->mb_dma_address = 0;
4646 }
4647
4648 kfree(skdev->skmsg_table);
4649 skdev->skmsg_table = NULL;
4650}
4651
4652static void skd_free_skreq(struct skd_device *skdev)
4653{
4654 u32 i;
4655
4656 if (skdev->skreq_table == NULL)
4657 return;
4658
4659 for (i = 0; i < skdev->num_req_context; i++) {
4660 struct skd_request_context *skreq;
4661
4662 skreq = &skdev->skreq_table[i];
4663
4664 skd_free_sg_list(skdev, skreq->sksg_list,
4665 skdev->sgs_per_request,
4666 skreq->sksg_dma_address);
4667
4668 skreq->sksg_list = NULL;
4669 skreq->sksg_dma_address = 0;
4670
4671 kfree(skreq->sg);
4672 }
4673
4674 kfree(skdev->skreq_table);
4675 skdev->skreq_table = NULL;
4676}
4677
4678static void skd_free_skspcl(struct skd_device *skdev)
4679{
4680 u32 i;
4681 u32 nbytes;
4682
4683 if (skdev->skspcl_table == NULL)
4684 return;
4685
4686 for (i = 0; i < skdev->n_special; i++) {
4687 struct skd_special_context *skspcl;
4688
4689 skspcl = &skdev->skspcl_table[i];
4690
4691 if (skspcl->msg_buf != NULL) {
4692 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4693 pci_free_consistent(skdev->pdev, nbytes,
4694 skspcl->msg_buf,
4695 skspcl->mb_dma_address);
4696 }
4697
4698 skspcl->msg_buf = NULL;
4699 skspcl->mb_dma_address = 0;
4700
4701 skd_free_sg_list(skdev, skspcl->req.sksg_list,
4702 SKD_N_SG_PER_SPECIAL,
4703 skspcl->req.sksg_dma_address);
4704
4705 skspcl->req.sksg_list = NULL;
4706 skspcl->req.sksg_dma_address = 0;
4707
4708 kfree(skspcl->req.sg);
4709 }
4710
4711 kfree(skdev->skspcl_table);
4712 skdev->skspcl_table = NULL;
4713}
4714
4715static void skd_free_sksb(struct skd_device *skdev)
4716{
4717 struct skd_special_context *skspcl;
4718 u32 nbytes;
4719
4720 skspcl = &skdev->internal_skspcl;
4721
4722 if (skspcl->data_buf != NULL) {
4723 nbytes = SKD_N_INTERNAL_BYTES;
4724
4725 pci_free_consistent(skdev->pdev, nbytes,
4726 skspcl->data_buf, skspcl->db_dma_address);
4727 }
4728
4729 skspcl->data_buf = NULL;
4730 skspcl->db_dma_address = 0;
4731
4732 if (skspcl->msg_buf != NULL) {
4733 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4734 pci_free_consistent(skdev->pdev, nbytes,
4735 skspcl->msg_buf, skspcl->mb_dma_address);
4736 }
4737
4738 skspcl->msg_buf = NULL;
4739 skspcl->mb_dma_address = 0;
4740
4741 skd_free_sg_list(skdev, skspcl->req.sksg_list, 1,
4742 skspcl->req.sksg_dma_address);
4743
4744 skspcl->req.sksg_list = NULL;
4745 skspcl->req.sksg_dma_address = 0;
4746}
4747
4748static void skd_free_sg_list(struct skd_device *skdev,
4749 struct fit_sg_descriptor *sg_list,
4750 u32 n_sg, dma_addr_t dma_addr)
4751{
4752 if (sg_list != NULL) {
4753 u32 nbytes;
4754
4755 nbytes = sizeof(*sg_list) * n_sg;
4756
4757 pci_free_consistent(skdev->pdev, nbytes, sg_list, dma_addr);
4758 }
4759}
4760
4761static void skd_free_disk(struct skd_device *skdev)
4762{
4763 struct gendisk *disk = skdev->disk;
4764
4765 if (disk != NULL) {
4766 struct request_queue *q = disk->queue;
4767
4768 if (disk->flags & GENHD_FL_UP)
4769 del_gendisk(disk);
4770 if (q)
4771 blk_cleanup_queue(q);
4772 put_disk(disk);
4773 }
4774 skdev->disk = NULL;
4775}
4776
4777
4778
4779/*
4780 *****************************************************************************
4781 * BLOCK DEVICE (BDEV) GLUE
4782 *****************************************************************************
4783 */
4784
4785static int skd_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo)
4786{
4787 struct skd_device *skdev;
4788 u64 capacity;
4789
4790 skdev = bdev->bd_disk->private_data;
4791
2e44b427 4792 pr_debug("%s:%s:%d %s: CMD[%s] getgeo device\n",
4793 skdev->name, __func__, __LINE__,
4794 bdev->bd_disk->disk_name, current->comm);
e67f86b3
AB		 4795
4796 if (skdev->read_cap_is_valid) {
4797 capacity = get_capacity(skdev->disk);
4798 geo->heads = 64;
4799 geo->sectors = 255;
4800 geo->cylinders = (capacity) / (255 * 64);
4801
4802 return 0;
4803 }
4804 return -EIO;
4805}
4806
4807static int skd_bdev_attach(struct skd_device *skdev)
4808{
2e44b427 4809 pr_debug("%s:%s:%d add_disk\n", skdev->name, __func__, __LINE__);
e67f86b3
AB		 4810 add_disk(skdev->disk);
4811 return 0;
4812}
4813
4814static const struct block_device_operations skd_blockdev_ops = {
4815 .owner = THIS_MODULE,
4816 .ioctl = skd_bdev_ioctl,
4817 .getgeo = skd_bdev_getgeo,
4818};
4819
4820
4821/*
4822 *****************************************************************************
4823 * PCIe DRIVER GLUE
4824 *****************************************************************************
4825 */
4826
4827static DEFINE_PCI_DEVICE_TABLE(skd_pci_tbl) = {
4828 { PCI_VENDOR_ID_STEC, PCI_DEVICE_ID_S1120,
4829 PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
4830 { 0 } /* terminate list */
4831};
4832
4833MODULE_DEVICE_TABLE(pci, skd_pci_tbl);
4834
4835static char *skd_pci_info(struct skd_device *skdev, char *str)
4836{
4837 int pcie_reg;
4838
4839 strcpy(str, "PCIe (");
4840 pcie_reg = pci_find_capability(skdev->pdev, PCI_CAP_ID_EXP);
4841
4842 if (pcie_reg) {
4843
4844 char lwstr[6];
4845 uint16_t pcie_lstat, lspeed, lwidth;
4846
4847 pcie_reg += 0x12;
4848 pci_read_config_word(skdev->pdev, pcie_reg, &pcie_lstat);
4849 lspeed = pcie_lstat & (0xF);
4850 lwidth = (pcie_lstat & 0x3F0) >> 4;
4851
4852 if (lspeed == 1)
4853 strcat(str, "2.5GT/s ");
4854 else if (lspeed == 2)
4855 strcat(str, "5.0GT/s ");
4856 else
4857 strcat(str, "<unknown> ");
4858 snprintf(lwstr, sizeof(lwstr), "%dX)", lwidth);
4859 strcat(str, lwstr);
4860 }
4861 return str;
4862}
4863
4864static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
4865{
4866 int i;
4867 int rc = 0;
4868 char pci_str[32];
4869 struct skd_device *skdev;
4870
4871 pr_info("STEC s1120 Driver(%s) version %s-b%s\n",
4872 DRV_NAME, DRV_VERSION, DRV_BUILD_ID);
4873 pr_info("(skd?:??:[%s]): vendor=%04X device=%04x\n",
4874 pci_name(pdev), pdev->vendor, pdev->device);
4875
4876 rc = pci_enable_device(pdev);
4877 if (rc)
4878 return rc;
4879 rc = pci_request_regions(pdev, DRV_NAME);
4880 if (rc)
4881 goto err_out;
4882 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
4883 if (!rc) {
4884 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
4885
4886 pr_err("(%s): consistent DMA mask error %d\n",
4887 pci_name(pdev), rc);
4888 }
4889 } else {
4890 (rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)));
4891 if (rc) {
4892
4893 pr_err("(%s): DMA mask error %d\n",
4894 pci_name(pdev), rc);
4895 goto err_out_regions;
4896 }
4897 }
4898
b8df6647
BZ		 4899 if (!skd_major) {
4900 rc = register_blkdev(0, DRV_NAME);
4901 if (rc < 0)
4902 goto err_out_regions;
4903 BUG_ON(!rc);
4904 skd_major = rc;
4905 }
4906
e67f86b3 4907 skdev = skd_construct(pdev);
1762b57f
WY		 4908 if (skdev == NULL) {
4909 rc = -ENOMEM;
e67f86b3 4910 goto err_out_regions;
1762b57f 4911 }
e67f86b3
AB		 4912
4913 skd_pci_info(skdev, pci_str);
4914 pr_info("(%s): %s 64bit\n", skd_name(skdev), pci_str);
4915
4916 pci_set_master(pdev);
4917 rc = pci_enable_pcie_error_reporting(pdev);
4918 if (rc) {
4919 pr_err(
4920 "(%s): bad enable of PCIe error reporting rc=%d\n",
4921 skd_name(skdev), rc);
4922 skdev->pcie_error_reporting_is_enabled = 0;
4923 } else
4924 skdev->pcie_error_reporting_is_enabled = 1;
4925
4926
4927 pci_set_drvdata(pdev, skdev);
ebedd16d 4928
e67f86b3
AB		 4929 skdev->disk->driverfs_dev = &pdev->dev;
4930
4931 for (i = 0; i < SKD_MAX_BARS; i++) {
4932 skdev->mem_phys[i] = pci_resource_start(pdev, i);
4933 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
4934 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
4935 skdev->mem_size[i]);
4936 if (!skdev->mem_map[i]) {
4937 pr_err("(%s): Unable to map adapter memory!\n",
4938 skd_name(skdev));
4939 rc = -ENODEV;
4940 goto err_out_iounmap;
4941 }
2e44b427 4942 pr_debug("%s:%s:%d mem_map=%p, phyd=%016llx, size=%d\n",
4943 skdev->name, __func__, __LINE__,
4944 skdev->mem_map[i],
4945 (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
e67f86b3
AB		 4946 }
4947
4948 rc = skd_acquire_irq(skdev);
4949 if (rc) {
4950 pr_err("(%s): interrupt resource error %d\n",
4951 skd_name(skdev), rc);
4952 goto err_out_iounmap;
4953 }
4954
4955 rc = skd_start_timer(skdev);
4956 if (rc)
4957 goto err_out_timer;
4958
4959 init_waitqueue_head(&skdev->waitq);
4960
4961 skd_start_device(skdev);
4962
4963 rc = wait_event_interruptible_timeout(skdev->waitq,
4964 (skdev->gendisk_on),
4965 (SKD_START_WAIT_SECONDS * HZ));
4966 if (skdev->gendisk_on > 0) {
4967 /* device came on-line after reset */
4968 skd_bdev_attach(skdev);
4969 rc = 0;
4970 } else {
4971 /* we timed out, something is wrong with the device,
4972 don't add the disk structure */
4973 pr_err(
4974 "(%s): error: waiting for s1120 timed out %d!\n",
4975 skd_name(skdev), rc);
4976 /* in case of no error; we timeout with ENXIO */
4977 if (!rc)
4978 rc = -ENXIO;
4979 goto err_out_timer;
4980 }
4981
4982
4983#ifdef SKD_VMK_POLL_HANDLER
4984 if (skdev->irq_type == SKD_IRQ_MSIX) {
4985 /* MSIX completion handler is being used for coredump */
4986 vmklnx_scsi_register_poll_handler(skdev->scsi_host,
4987 skdev->msix_entries[5].vector,
4988 skd_comp_q, skdev);
4989 } else {
4990 vmklnx_scsi_register_poll_handler(skdev->scsi_host,
4991 skdev->pdev->irq, skd_isr,
4992 skdev);
4993 }
4994#endif /* SKD_VMK_POLL_HANDLER */
4995
4996 return rc;
4997
4998err_out_timer:
4999 skd_stop_device(skdev);
5000 skd_release_irq(skdev);
5001
5002err_out_iounmap:
5003 for (i = 0; i < SKD_MAX_BARS; i++)
5004 if (skdev->mem_map[i])
5005 iounmap(skdev->mem_map[i]);
5006
5007 if (skdev->pcie_error_reporting_is_enabled)
5008 pci_disable_pcie_error_reporting(pdev);
5009
5010 skd_destruct(skdev);
5011
5012err_out_regions:
5013 pci_release_regions(pdev);
5014
5015err_out:
5016 pci_disable_device(pdev);
5017 pci_set_drvdata(pdev, NULL);
5018 return rc;
5019}
5020
5021static void skd_pci_remove(struct pci_dev *pdev)
5022{
5023 int i;
5024 struct skd_device *skdev;
5025
5026 skdev = pci_get_drvdata(pdev);
5027 if (!skdev) {
5028 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5029 return;
5030 }
5031 skd_stop_device(skdev);
5032 skd_release_irq(skdev);
5033
5034 for (i = 0; i < SKD_MAX_BARS; i++)
5035 if (skdev->mem_map[i])
5036 iounmap((u32 *)skdev->mem_map[i]);
5037
5038 if (skdev->pcie_error_reporting_is_enabled)
5039 pci_disable_pcie_error_reporting(pdev);
5040
5041 skd_destruct(skdev);
5042
5043 pci_release_regions(pdev);
5044 pci_disable_device(pdev);
5045 pci_set_drvdata(pdev, NULL);
5046
5047 return;
5048}
5049
5050static int skd_pci_suspend(struct pci_dev *pdev, pm_message_t state)
5051{
5052 int i;
5053 struct skd_device *skdev;
5054
5055 skdev = pci_get_drvdata(pdev);
5056 if (!skdev) {
5057 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5058 return -EIO;
5059 }
5060
5061 skd_stop_device(skdev);
5062
5063 skd_release_irq(skdev);
5064
5065 for (i = 0; i < SKD_MAX_BARS; i++)
5066 if (skdev->mem_map[i])
5067 iounmap((u32 *)skdev->mem_map[i]);
5068
5069 if (skdev->pcie_error_reporting_is_enabled)
5070 pci_disable_pcie_error_reporting(pdev);
5071
5072 pci_release_regions(pdev);
5073 pci_save_state(pdev);
5074 pci_disable_device(pdev);
5075 pci_set_power_state(pdev, pci_choose_state(pdev, state));
5076 return 0;
5077}
5078
5079static int skd_pci_resume(struct pci_dev *pdev)
5080{
5081 int i;
5082 int rc = 0;
5083 struct skd_device *skdev;
5084
5085 skdev = pci_get_drvdata(pdev);
5086 if (!skdev) {
5087 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5088 return -1;
5089 }
5090
5091 pci_set_power_state(pdev, PCI_D0);
5092 pci_enable_wake(pdev, PCI_D0, 0);
5093 pci_restore_state(pdev);
5094
5095 rc = pci_enable_device(pdev);
5096 if (rc)
5097 return rc;
5098 rc = pci_request_regions(pdev, DRV_NAME);
5099 if (rc)
5100 goto err_out;
5101 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
5102 if (!rc) {
5103 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
5104
5105 pr_err("(%s): consistent DMA mask error %d\n",
5106 pci_name(pdev), rc);
5107 }
5108 } else {
5109 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
5110 if (rc) {
5111
5112 pr_err("(%s): DMA mask error %d\n",
5113 pci_name(pdev), rc);
5114 goto err_out_regions;
5115 }
5116 }
5117
5118 pci_set_master(pdev);
5119 rc = pci_enable_pcie_error_reporting(pdev);
5120 if (rc) {
5121 pr_err("(%s): bad enable of PCIe error reporting rc=%d\n",
5122 skdev->name, rc);
5123 skdev->pcie_error_reporting_is_enabled = 0;
5124 } else
5125 skdev->pcie_error_reporting_is_enabled = 1;
5126
5127 for (i = 0; i < SKD_MAX_BARS; i++) {
5128
5129 skdev->mem_phys[i] = pci_resource_start(pdev, i);
5130 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
5131 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
5132 skdev->mem_size[i]);
5133 if (!skdev->mem_map[i]) {
5134 pr_err("(%s): Unable to map adapter memory!\n",
5135 skd_name(skdev));
5136 rc = -ENODEV;
5137 goto err_out_iounmap;
5138 }
2e44b427 5139 pr_debug("%s:%s:%d mem_map=%p, phyd=%016llx, size=%d\n",
5140 skdev->name, __func__, __LINE__,
5141 skdev->mem_map[i],
5142 (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
e67f86b3
AB		 5143 }
5144 rc = skd_acquire_irq(skdev);
5145 if (rc) {
5146
5147 pr_err("(%s): interrupt resource error %d\n",
5148 pci_name(pdev), rc);
5149 goto err_out_iounmap;
5150 }
5151
5152 rc = skd_start_timer(skdev);
5153 if (rc)
5154 goto err_out_timer;
5155
5156 init_waitqueue_head(&skdev->waitq);
5157
5158 skd_start_device(skdev);
5159
5160 return rc;
5161
5162err_out_timer:
5163 skd_stop_device(skdev);
5164 skd_release_irq(skdev);
5165
5166err_out_iounmap:
5167 for (i = 0; i < SKD_MAX_BARS; i++)
5168 if (skdev->mem_map[i])
5169 iounmap(skdev->mem_map[i]);
5170
5171 if (skdev->pcie_error_reporting_is_enabled)
5172 pci_disable_pcie_error_reporting(pdev);
5173
5174err_out_regions:
5175 pci_release_regions(pdev);
5176
5177err_out:
5178 pci_disable_device(pdev);
5179 return rc;
5180}
5181
5182static void skd_pci_shutdown(struct pci_dev *pdev)
5183{
5184 struct skd_device *skdev;
5185
5186 pr_err("skd_pci_shutdown called\n");
5187
5188 skdev = pci_get_drvdata(pdev);
5189 if (!skdev) {
5190 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5191 return;
5192 }
5193
5194 pr_err("%s: calling stop\n", skd_name(skdev));
5195 skd_stop_device(skdev);
5196}
5197
5198static struct pci_driver skd_driver = {
5199 .name = DRV_NAME,
5200 .id_table = skd_pci_tbl,
5201 .probe = skd_pci_probe,
5202 .remove = skd_pci_remove,
5203 .suspend = skd_pci_suspend,
5204 .resume = skd_pci_resume,
5205 .shutdown = skd_pci_shutdown,
5206};
5207
5208/*
5209 *****************************************************************************
5210 * LOGGING SUPPORT
5211 *****************************************************************************
5212 */
5213
5214static const char *skd_name(struct skd_device *skdev)
5215{
5216 memset(skdev->id_str, 0, sizeof(skdev->id_str));
5217
5218 if (skdev->inquiry_is_valid)
5219 snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:%s:[%s]",
5220 skdev->name, skdev->inq_serial_num,
5221 pci_name(skdev->pdev));
5222 else
5223 snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:??:[%s]",
5224 skdev->name, pci_name(skdev->pdev));
5225
5226 return skdev->id_str;
5227}
5228
5229const char *skd_drive_state_to_str(int state)
5230{
5231 switch (state) {
5232 case FIT_SR_DRIVE_OFFLINE:
5233 return "OFFLINE";
5234 case FIT_SR_DRIVE_INIT:
5235 return "INIT";
5236 case FIT_SR_DRIVE_ONLINE:
5237 return "ONLINE";
5238 case FIT_SR_DRIVE_BUSY:
5239 return "BUSY";
5240 case FIT_SR_DRIVE_FAULT:
5241 return "FAULT";
5242 case FIT_SR_DRIVE_DEGRADED:
5243 return "DEGRADED";
5244 case FIT_SR_PCIE_LINK_DOWN:
5245 return "INK_DOWN";
5246 case FIT_SR_DRIVE_SOFT_RESET:
5247 return "SOFT_RESET";
5248 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
5249 return "NEED_FW";
5250 case FIT_SR_DRIVE_INIT_FAULT:
5251 return "INIT_FAULT";
5252 case FIT_SR_DRIVE_BUSY_SANITIZE:
5253 return "BUSY_SANITIZE";
5254 case FIT_SR_DRIVE_BUSY_ERASE:
5255 return "BUSY_ERASE";
5256 case FIT_SR_DRIVE_FW_BOOTING:
5257 return "FW_BOOTING";
5258 default:
5259 return "???";
5260 }
5261}
5262
5263const char *skd_skdev_state_to_str(enum skd_drvr_state state)
5264{
5265 switch (state) {
5266 case SKD_DRVR_STATE_LOAD:
5267 return "LOAD";
5268 case SKD_DRVR_STATE_IDLE:
5269 return "IDLE";
5270 case SKD_DRVR_STATE_BUSY:
5271 return "BUSY";
5272 case SKD_DRVR_STATE_STARTING:
5273 return "STARTING";
5274 case SKD_DRVR_STATE_ONLINE:
5275 return "ONLINE";
5276 case SKD_DRVR_STATE_PAUSING:
5277 return "PAUSING";
5278 case SKD_DRVR_STATE_PAUSED:
5279 return "PAUSED";
5280 case SKD_DRVR_STATE_DRAINING_TIMEOUT:
5281 return "DRAINING_TIMEOUT";
5282 case SKD_DRVR_STATE_RESTARTING:
5283 return "RESTARTING";
5284 case SKD_DRVR_STATE_RESUMING:
5285 return "RESUMING";
5286 case SKD_DRVR_STATE_STOPPING:
5287 return "STOPPING";
5288 case SKD_DRVR_STATE_SYNCING:
5289 return "SYNCING";
5290 case SKD_DRVR_STATE_FAULT:
5291 return "FAULT";
5292 case SKD_DRVR_STATE_DISAPPEARED:
5293 return "DISAPPEARED";
5294 case SKD_DRVR_STATE_BUSY_ERASE:
5295 return "BUSY_ERASE";
5296 case SKD_DRVR_STATE_BUSY_SANITIZE:
5297 return "BUSY_SANITIZE";
5298 case SKD_DRVR_STATE_BUSY_IMMINENT:
5299 return "BUSY_IMMINENT";
5300 case SKD_DRVR_STATE_WAIT_BOOT:
5301 return "WAIT_BOOT";
5302
5303 default:
5304 return "???";
5305 }
5306}
5307
5308const char *skd_skmsg_state_to_str(enum skd_fit_msg_state state)
5309{
5310 switch (state) {
5311 case SKD_MSG_STATE_IDLE:
5312 return "IDLE";
5313 case SKD_MSG_STATE_BUSY:
5314 return "BUSY";
5315 default:
5316 return "???";
5317 }
5318}
5319
5320const char *skd_skreq_state_to_str(enum skd_req_state state)
5321{
5322 switch (state) {
5323 case SKD_REQ_STATE_IDLE:
5324 return "IDLE";
5325 case SKD_REQ_STATE_SETUP:
5326 return "SETUP";
5327 case SKD_REQ_STATE_BUSY:
5328 return "BUSY";
5329 case SKD_REQ_STATE_COMPLETED:
5330 return "COMPLETED";
5331 case SKD_REQ_STATE_TIMEOUT:
5332 return "TIMEOUT";
5333 case SKD_REQ_STATE_ABORTED:
5334 return "ABORTED";
5335 default:
5336 return "???";
5337 }
5338}
5339
5340static void skd_log_skdev(struct skd_device *skdev, const char *event)
5341{
2e44b427 5342 pr_debug("%s:%s:%d (%s) skdev=%p event='%s'\n",
5343 skdev->name, __func__, __LINE__, skdev->name, skdev, event);
5344 pr_debug("%s:%s:%d drive_state=%s(%d) driver_state=%s(%d)\n",
5345 skdev->name, __func__, __LINE__,
5346 skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
5347 skd_skdev_state_to_str(skdev->state), skdev->state);
5348 pr_debug("%s:%s:%d busy=%d limit=%d dev=%d lowat=%d\n",
5349 skdev->name, __func__, __LINE__,
5350 skdev->in_flight, skdev->cur_max_queue_depth,
5351 skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
5352 pr_debug("%s:%s:%d timestamp=0x%x cycle=%d cycle_ix=%d\n",
5353 skdev->name, __func__, __LINE__,
5354 skdev->timeout_stamp, skdev->skcomp_cycle, skdev->skcomp_ix);
e67f86b3
AB		 5355}
5356
5357static void skd_log_skmsg(struct skd_device *skdev,
5358 struct skd_fitmsg_context *skmsg, const char *event)
5359{
2e44b427 5360 pr_debug("%s:%s:%d (%s) skmsg=%p event='%s'\n",
5361 skdev->name, __func__, __LINE__, skdev->name, skmsg, event);
5362 pr_debug("%s:%s:%d state=%s(%d) id=0x%04x length=%d\n",
5363 skdev->name, __func__, __LINE__,
5364 skd_skmsg_state_to_str(skmsg->state), skmsg->state,
5365 skmsg->id, skmsg->length);
e67f86b3
AB		 5366}
5367
5368static void skd_log_skreq(struct skd_device *skdev,
5369 struct skd_request_context *skreq, const char *event)
5370{
2e44b427 5371 pr_debug("%s:%s:%d (%s) skreq=%p event='%s'\n",
5372 skdev->name, __func__, __LINE__, skdev->name, skreq, event);
5373 pr_debug("%s:%s:%d state=%s(%d) id=0x%04x fitmsg=0x%04x\n",
5374 skdev->name, __func__, __LINE__,
5375 skd_skreq_state_to_str(skreq->state), skreq->state,
5376 skreq->id, skreq->fitmsg_id);
5377 pr_debug("%s:%s:%d timo=0x%x sg_dir=%d n_sg=%d\n",
5378 skdev->name, __func__, __LINE__,
5379 skreq->timeout_stamp, skreq->sg_data_dir, skreq->n_sg);
e67f86b3 5380
fcd37eb3
JA		 5381 if (skreq->req != NULL) {
5382 struct request *req = skreq->req;
5383 u32 lba = (u32)blk_rq_pos(req);
5384 u32 count = blk_rq_sectors(req);
e67f86b3 5385
fcd37eb3
JA		 5386 pr_debug("%s:%s:%d "
5387 "req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n",
5388 skdev->name, __func__, __LINE__,
5389 req, lba, lba, count, count,
5390 (int)rq_data_dir(req));
5391 } else
5392 pr_debug("%s:%s:%d req=NULL\n",
5393 skdev->name, __func__, __LINE__);
e67f86b3
AB		 5394}
5395
5396/*
5397 *****************************************************************************
5398 * MODULE GLUE
5399 *****************************************************************************
5400 */
5401
5402static int __init skd_init(void)
5403{
e67f86b3
AB		 5404 pr_info(PFX " v%s-b%s loaded\n", DRV_VERSION, DRV_BUILD_ID);
5405
5406 switch (skd_isr_type) {
5407 case SKD_IRQ_LEGACY:
5408 case SKD_IRQ_MSI:
5409 case SKD_IRQ_MSIX:
5410 break;
5411 default:
fbed149a 5412 pr_err(PFX "skd_isr_type %d invalid, re-set to %d\n",
e67f86b3
AB		 5413 skd_isr_type, SKD_IRQ_DEFAULT);
5414 skd_isr_type = SKD_IRQ_DEFAULT;
5415 }
5416
fbed149a
BZ		 5417 if (skd_max_queue_depth < 1 ||
5418 skd_max_queue_depth > SKD_MAX_QUEUE_DEPTH) {
5419 pr_err(PFX "skd_max_queue_depth %d invalid, re-set to %d\n",
e67f86b3
AB		 5420 skd_max_queue_depth, SKD_MAX_QUEUE_DEPTH_DEFAULT);
5421 skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
5422 }
5423
5424 if (skd_max_req_per_msg < 1 || skd_max_req_per_msg > 14) {
fbed149a 5425 pr_err(PFX "skd_max_req_per_msg %d invalid, re-set to %d\n",
e67f86b3
AB		 5426 skd_max_req_per_msg, SKD_MAX_REQ_PER_MSG_DEFAULT);
5427 skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
5428 }
5429
5430 if (skd_sgs_per_request < 1 || skd_sgs_per_request > 4096) {
fbed149a 5431 pr_err(PFX "skd_sg_per_request %d invalid, re-set to %d\n",
e67f86b3
AB		 5432 skd_sgs_per_request, SKD_N_SG_PER_REQ_DEFAULT);
5433 skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
5434 }
5435
5436 if (skd_dbg_level < 0 || skd_dbg_level > 2) {
fbed149a 5437 pr_err(PFX "skd_dbg_level %d invalid, re-set to %d\n",
e67f86b3
AB		 5438 skd_dbg_level, 0);
5439 skd_dbg_level = 0;
5440 }
5441
5442 if (skd_isr_comp_limit < 0) {
fbed149a 5443 pr_err(PFX "skd_isr_comp_limit %d invalid, set to %d\n",
e67f86b3
AB		 5444 skd_isr_comp_limit, 0);
5445 skd_isr_comp_limit = 0;
5446 }
5447
5448 if (skd_max_pass_thru < 1 || skd_max_pass_thru > 50) {
fbed149a 5449 pr_err(PFX "skd_max_pass_thru %d invalid, re-set to %d\n",
e67f86b3
AB		 5450 skd_max_pass_thru, SKD_N_SPECIAL_CONTEXT);
5451 skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
5452 }
5453
b8df6647 5454 return pci_register_driver(&skd_driver);
e67f86b3
AB		 5455}
5456
5457static void __exit skd_exit(void)
5458{
5459 pr_info(PFX " v%s-b%s unloading\n", DRV_VERSION, DRV_BUILD_ID);
5460
e67f86b3 5461 pci_unregister_driver(&skd_driver);
b8df6647
BZ		 5462
5463 if (skd_major)
5464 unregister_blkdev(skd_major, DRV_NAME);
e67f86b3
AB		 5465}
5466
e67f86b3
AB		 5467module_init(skd_init);
5468module_exit(skd_exit);
Linux kernel - Deliverables
This page took 0.245059 seconds and 5 git commands to generate.