projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
rsxx: Restructured DMA cancel scheme.
[deliverable/linux.git] / drivers / block / rsxx / dma.c
CommitLineData
8722ff8c 1/*
2* Filename: dma.c
3*
4*
5* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7*
8* (C) Copyright 2013 IBM Corporation
9*
10* This program is free software; you can redistribute it and/or
11* modify it under the terms of the GNU General Public License as
12* published by the Free Software Foundation; either version 2 of the
13* License, or (at your option) any later version.
14*
15* This program is distributed in the hope that it will be useful, but
16* WITHOUT ANY WARRANTY; without even the implied warranty of
17* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18* General Public License for more details.
19*
20* You should have received a copy of the GNU General Public License
21* along with this program; if not, write to the Free Software Foundation,
22* Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23*/
24
e5e9fdaa 25#include <linux/slab.h>
8722ff8c 26#include "rsxx_priv.h"
27
28struct rsxx_dma {
29 struct list_head list;
30 u8 cmd;
9bb3c446 31 unsigned int laddr; /* Logical address */
8722ff8c 32 struct {
33 u32 off;
34 u32 cnt;
35 } sub_page;
36 dma_addr_t dma_addr;
37 struct page *page;
38 unsigned int pg_off; /* Page Offset */
39 rsxx_dma_cb cb;
40 void *cb_data;
41};
42
43/* This timeout is used to detect a stalled DMA channel */
44#define DMA_ACTIVITY_TIMEOUT msecs_to_jiffies(10000)
45
46struct hw_status {
47 u8 status;
48 u8 tag;
49 __le16 count;
50 __le32 _rsvd2;
51 __le64 _rsvd3;
52} __packed;
53
54enum rsxx_dma_status {
55 DMA_SW_ERR = 0x1,
56 DMA_HW_FAULT = 0x2,
57 DMA_CANCELLED = 0x4,
58};
59
60struct hw_cmd {
61 u8 command;
62 u8 tag;
63 u8 _rsvd;
64 u8 sub_page; /* Bit[0:2]: 512byte offset */
65 /* Bit[4:6]: 512byte count */
66 __le32 device_addr;
67 __le64 host_addr;
68} __packed;
69
70enum rsxx_hw_cmd {
71 HW_CMD_BLK_DISCARD = 0x70,
72 HW_CMD_BLK_WRITE = 0x80,
73 HW_CMD_BLK_READ = 0xC0,
74 HW_CMD_BLK_RECON_READ = 0xE0,
75};
76
77enum rsxx_hw_status {
78 HW_STATUS_CRC = 0x01,
79 HW_STATUS_HARD_ERR = 0x02,
80 HW_STATUS_SOFT_ERR = 0x04,
81 HW_STATUS_FAULT = 0x08,
82};
83
8722ff8c 84static struct kmem_cache *rsxx_dma_pool;
85
86struct dma_tracker {
87 int next_tag;
88 struct rsxx_dma *dma;
89};
90
91#define DMA_TRACKER_LIST_SIZE8 (sizeof(struct dma_tracker_list) + \
92 (sizeof(struct dma_tracker) * RSXX_MAX_OUTSTANDING_CMDS))
93
94struct dma_tracker_list {
95 spinlock_t lock;
96 int head;
97 struct dma_tracker list[0];
98};
99
100
101/*----------------- Misc Utility Functions -------------------*/
c206c709 102static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card)
8722ff8c 103{
104 unsigned long long tgt_addr8;
105
106 tgt_addr8 = ((addr8 >> card->_stripe.upper_shift) &
107 card->_stripe.upper_mask) |
108 ((addr8) & card->_stripe.lower_mask);
109 do_div(tgt_addr8, RSXX_HW_BLK_SIZE);
110 return tgt_addr8;
111}
112
c206c709 113static unsigned int rsxx_get_dma_tgt(struct rsxx_cardinfo *card, u64 addr8)
8722ff8c 114{
115 unsigned int tgt;
116
117 tgt = (addr8 >> card->_stripe.target_shift) & card->_stripe.target_mask;
118
119 return tgt;
120}
121
c95246c3 122void rsxx_dma_queue_reset(struct rsxx_cardinfo *card)
8722ff8c 123{
124 /* Reset all DMA Command/Status Queues */
125 iowrite32(DMA_QUEUE_RESET, card->regmap + RESET);
126}
127
128static unsigned int get_dma_size(struct rsxx_dma *dma)
129{
130 if (dma->sub_page.cnt)
131 return dma->sub_page.cnt << 9;
132 else
133 return RSXX_HW_BLK_SIZE;
134}
135
136
137/*----------------- DMA Tracker -------------------*/
138static void set_tracker_dma(struct dma_tracker_list *trackers,
139 int tag,
140 struct rsxx_dma *dma)
141{
142 trackers->list[tag].dma = dma;
143}
144
145static struct rsxx_dma *get_tracker_dma(struct dma_tracker_list *trackers,
146 int tag)
147{
148 return trackers->list[tag].dma;
149}
150
151static int pop_tracker(struct dma_tracker_list *trackers)
152{
153 int tag;
154
155 spin_lock(&trackers->lock);
156 tag = trackers->head;
157 if (tag != -1) {
158 trackers->head = trackers->list[tag].next_tag;
159 trackers->list[tag].next_tag = -1;
160 }
161 spin_unlock(&trackers->lock);
162
163 return tag;
164}
165
166static void push_tracker(struct dma_tracker_list *trackers, int tag)
167{
168 spin_lock(&trackers->lock);
169 trackers->list[tag].next_tag = trackers->head;
170 trackers->head = tag;
171 trackers->list[tag].dma = NULL;
172 spin_unlock(&trackers->lock);
173}
174
175
176/*----------------- Interrupt Coalescing -------------*/
177/*
178 * Interrupt Coalescing Register Format:
179 * Interrupt Timer (64ns units) [15:0]
180 * Interrupt Count [24:16]
181 * Reserved [31:25]
182*/
183#define INTR_COAL_LATENCY_MASK (0x0000ffff)
184
185#define INTR_COAL_COUNT_SHIFT 16
186#define INTR_COAL_COUNT_BITS 9
187#define INTR_COAL_COUNT_MASK (((1 << INTR_COAL_COUNT_BITS) - 1) << \
188 INTR_COAL_COUNT_SHIFT)
189#define INTR_COAL_LATENCY_UNITS_NS 64
190
191
192static u32 dma_intr_coal_val(u32 mode, u32 count, u32 latency)
193{
194 u32 latency_units = latency / INTR_COAL_LATENCY_UNITS_NS;
195
196 if (mode == RSXX_INTR_COAL_DISABLED)
197 return 0;
198
199 return ((count << INTR_COAL_COUNT_SHIFT) & INTR_COAL_COUNT_MASK) |
200 (latency_units & INTR_COAL_LATENCY_MASK);
201
202}
203
204static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card)
205{
206 int i;
207 u32 q_depth = 0;
208 u32 intr_coal;
209
c95246c3
PK		 210 if (card->config.data.intr_coal.mode != RSXX_INTR_COAL_AUTO_TUNE ||
211 unlikely(card->eeh_state))
8722ff8c 212 return;
213
214 for (i = 0; i < card->n_targets; i++)
215 q_depth += atomic_read(&card->ctrl[i].stats.hw_q_depth);
216
217 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
218 q_depth / 2,
219 card->config.data.intr_coal.latency);
220 iowrite32(intr_coal, card->regmap + INTR_COAL);
221}
222
223/*----------------- RSXX DMA Handling -------------------*/
c95246c3 224static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl,
8722ff8c 225 struct rsxx_dma *dma,
226 unsigned int status)
227{
228 if (status & DMA_SW_ERR)
c95246c3 229 ctrl->stats.dma_sw_err++;
8722ff8c 230 if (status & DMA_HW_FAULT)
c95246c3 231 ctrl->stats.dma_hw_fault++;
8722ff8c 232 if (status & DMA_CANCELLED)
c95246c3 233 ctrl->stats.dma_cancelled++;
8722ff8c 234
235 if (dma->dma_addr)
c95246c3
PK		 236 pci_unmap_page(ctrl->card->dev, dma->dma_addr,
237 get_dma_size(dma),
8722ff8c 238 dma->cmd == HW_CMD_BLK_WRITE ?
239 PCI_DMA_TODEVICE :
240 PCI_DMA_FROMDEVICE);
241
242 if (dma->cb)
c95246c3 243 dma->cb(ctrl->card, dma->cb_data, status ? 1 : 0);
8722ff8c 244
245 kmem_cache_free(rsxx_dma_pool, dma);
246}
247
0ab4743e
PK		 248int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl,
249 struct list_head *q)
250{
251 struct rsxx_dma *dma;
252 struct rsxx_dma *tmp;
253 int cnt = 0;
254
255 list_for_each_entry_safe(dma, tmp, q, list) {
256 list_del(&dma->list);
257 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
258 cnt++;
259 }
260
261 return cnt;
262}
263
8722ff8c 264static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl,
265 struct rsxx_dma *dma)
266{
267 /*
268 * Requeued DMAs go to the front of the queue so they are issued
269 * first.
270 */
0ab4743e 271 spin_lock_bh(&ctrl->queue_lock);
8722ff8c 272 list_add(&dma->list, &ctrl->queue);
0ab4743e 273 spin_unlock_bh(&ctrl->queue_lock);
8722ff8c 274}
275
276static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl,
277 struct rsxx_dma *dma,
278 u8 hw_st)
279{
280 unsigned int status = 0;
281 int requeue_cmd = 0;
282
283 dev_dbg(CARD_TO_DEV(ctrl->card),
284 "Handling DMA error(cmd x%02x, laddr x%08x st:x%02x)\n",
285 dma->cmd, dma->laddr, hw_st);
286
287 if (hw_st & HW_STATUS_CRC)
288 ctrl->stats.crc_errors++;
289 if (hw_st & HW_STATUS_HARD_ERR)
290 ctrl->stats.hard_errors++;
291 if (hw_st & HW_STATUS_SOFT_ERR)
292 ctrl->stats.soft_errors++;
293
294 switch (dma->cmd) {
295 case HW_CMD_BLK_READ:
296 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
297 if (ctrl->card->scrub_hard) {
298 dma->cmd = HW_CMD_BLK_RECON_READ;
299 requeue_cmd = 1;
300 ctrl->stats.reads_retried++;
301 } else {
302 status |= DMA_HW_FAULT;
303 ctrl->stats.reads_failed++;
304 }
305 } else if (hw_st & HW_STATUS_FAULT) {
306 status |= DMA_HW_FAULT;
307 ctrl->stats.reads_failed++;
308 }
309
310 break;
311 case HW_CMD_BLK_RECON_READ:
312 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
313 /* Data could not be reconstructed. */
314 status |= DMA_HW_FAULT;
315 ctrl->stats.reads_failed++;
316 }
317
318 break;
319 case HW_CMD_BLK_WRITE:
320 status |= DMA_HW_FAULT;
321 ctrl->stats.writes_failed++;
322
323 break;
324 case HW_CMD_BLK_DISCARD:
325 status |= DMA_HW_FAULT;
326 ctrl->stats.discards_failed++;
327
328 break;
329 default:
330 dev_err(CARD_TO_DEV(ctrl->card),
331 "Unknown command in DMA!(cmd: x%02x "
332 "laddr x%08x st: x%02x\n",
333 dma->cmd, dma->laddr, hw_st);
334 status |= DMA_SW_ERR;
335
336 break;
337 }
338
339 if (requeue_cmd)
340 rsxx_requeue_dma(ctrl, dma);
341 else
c95246c3 342 rsxx_complete_dma(ctrl, dma, status);
8722ff8c 343}
344
345static void dma_engine_stalled(unsigned long data)
346{
347 struct rsxx_dma_ctrl *ctrl = (struct rsxx_dma_ctrl *)data;
0ab4743e 348 int cnt;
8722ff8c 349
c95246c3
PK		 350 if (atomic_read(&ctrl->stats.hw_q_depth) == 0 ||
351 unlikely(ctrl->card->eeh_state))
8722ff8c 352 return;
353
354 if (ctrl->cmd.idx != ioread32(ctrl->regmap + SW_CMD_IDX)) {
355 /*
356 * The dma engine was stalled because the SW_CMD_IDX write
357 * was lost. Issue it again to recover.
358 */
359 dev_warn(CARD_TO_DEV(ctrl->card),
360 "SW_CMD_IDX write was lost, re-writing...\n");
361 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
362 mod_timer(&ctrl->activity_timer,
363 jiffies + DMA_ACTIVITY_TIMEOUT);
364 } else {
365 dev_warn(CARD_TO_DEV(ctrl->card),
366 "DMA channel %d has stalled, faulting interface.\n",
367 ctrl->id);
368 ctrl->card->dma_fault = 1;
0ab4743e
PK		 369
370 /* Clean up the DMA queue */
371 spin_lock(&ctrl->queue_lock);
372 cnt = rsxx_cleanup_dma_queue(ctrl, &ctrl->queue);
373 spin_unlock(&ctrl->queue_lock);
374
375 cnt += rsxx_dma_cancel(ctrl);
376
377 if (cnt)
378 dev_info(CARD_TO_DEV(ctrl->card),
379 "Freed %d queued DMAs on channel %d\n",
380 cnt, ctrl->id);
8722ff8c 381 }
382}
383
384static void rsxx_issue_dmas(struct work_struct *work)
385{
386 struct rsxx_dma_ctrl *ctrl;
387 struct rsxx_dma *dma;
388 int tag;
389 int cmds_pending = 0;
390 struct hw_cmd *hw_cmd_buf;
391
392 ctrl = container_of(work, struct rsxx_dma_ctrl, issue_dma_work);
393 hw_cmd_buf = ctrl->cmd.buf;
394
c95246c3
PK		 395 if (unlikely(ctrl->card->halt) ||
396 unlikely(ctrl->card->eeh_state))
8722ff8c 397 return;
398
399 while (1) {
0ab4743e 400 spin_lock_bh(&ctrl->queue_lock);
8722ff8c 401 if (list_empty(&ctrl->queue)) {
0ab4743e 402 spin_unlock_bh(&ctrl->queue_lock);
8722ff8c 403 break;
404 }
0ab4743e 405 spin_unlock_bh(&ctrl->queue_lock);
8722ff8c 406
407 tag = pop_tracker(ctrl->trackers);
408 if (tag == -1)
409 break;
410
0ab4743e 411 spin_lock_bh(&ctrl->queue_lock);
8722ff8c 412 dma = list_entry(ctrl->queue.next, struct rsxx_dma, list);
413 list_del(&dma->list);
414 ctrl->stats.sw_q_depth--;
0ab4743e 415 spin_unlock_bh(&ctrl->queue_lock);
8722ff8c 416
417 /*
418 * This will catch any DMAs that slipped in right before the
419 * fault, but was queued after all the other DMAs were
420 * cancelled.
421 */
422 if (unlikely(ctrl->card->dma_fault)) {
423 push_tracker(ctrl->trackers, tag);
c95246c3 424 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
8722ff8c 425 continue;
426 }
427
428 set_tracker_dma(ctrl->trackers, tag, dma);
429 hw_cmd_buf[ctrl->cmd.idx].command = dma->cmd;
430 hw_cmd_buf[ctrl->cmd.idx].tag = tag;
431 hw_cmd_buf[ctrl->cmd.idx]._rsvd = 0;
432 hw_cmd_buf[ctrl->cmd.idx].sub_page =
433 ((dma->sub_page.cnt & 0x7) << 4) |
434 (dma->sub_page.off & 0x7);
435
436 hw_cmd_buf[ctrl->cmd.idx].device_addr =
437 cpu_to_le32(dma->laddr);
438
439 hw_cmd_buf[ctrl->cmd.idx].host_addr =
440 cpu_to_le64(dma->dma_addr);
441
442 dev_dbg(CARD_TO_DEV(ctrl->card),
443 "Issue DMA%d(laddr %d tag %d) to idx %d\n",
444 ctrl->id, dma->laddr, tag, ctrl->cmd.idx);
445
446 ctrl->cmd.idx = (ctrl->cmd.idx + 1) & RSXX_CS_IDX_MASK;
447 cmds_pending++;
448
449 if (dma->cmd == HW_CMD_BLK_WRITE)
450 ctrl->stats.writes_issued++;
451 else if (dma->cmd == HW_CMD_BLK_DISCARD)
452 ctrl->stats.discards_issued++;
453 else
454 ctrl->stats.reads_issued++;
455 }
456
457 /* Let HW know we've queued commands. */
458 if (cmds_pending) {
8722ff8c 459 atomic_add(cmds_pending, &ctrl->stats.hw_q_depth);
460 mod_timer(&ctrl->activity_timer,
461 jiffies + DMA_ACTIVITY_TIMEOUT);
c95246c3
PK		 462
463 if (unlikely(ctrl->card->eeh_state)) {
464 del_timer_sync(&ctrl->activity_timer);
465 return;
466 }
467
8722ff8c 468 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
469 }
470}
471
472static void rsxx_dma_done(struct work_struct *work)
473{
474 struct rsxx_dma_ctrl *ctrl;
475 struct rsxx_dma *dma;
476 unsigned long flags;
477 u16 count;
478 u8 status;
479 u8 tag;
480 struct hw_status *hw_st_buf;
481
482 ctrl = container_of(work, struct rsxx_dma_ctrl, dma_done_work);
483 hw_st_buf = ctrl->status.buf;
484
485 if (unlikely(ctrl->card->halt) ||
c95246c3
PK		 486 unlikely(ctrl->card->dma_fault) ||
487 unlikely(ctrl->card->eeh_state))
8722ff8c 488 return;
489
490 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
491
492 while (count == ctrl->e_cnt) {
493 /*
494 * The read memory-barrier is necessary to keep aggressive
495 * processors/optimizers (such as the PPC Apple G5) from
496 * reordering the following status-buffer tag & status read
497 * *before* the count read on subsequent iterations of the
498 * loop!
499 */
500 rmb();
501
502 status = hw_st_buf[ctrl->status.idx].status;
503 tag = hw_st_buf[ctrl->status.idx].tag;
504
505 dma = get_tracker_dma(ctrl->trackers, tag);
506 if (dma == NULL) {
507 spin_lock_irqsave(&ctrl->card->irq_lock, flags);
508 rsxx_disable_ier(ctrl->card, CR_INTR_DMA_ALL);
509 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
510
511 dev_err(CARD_TO_DEV(ctrl->card),
512 "No tracker for tag %d "
513 "(idx %d id %d)\n",
514 tag, ctrl->status.idx, ctrl->id);
515 return;
516 }
517
518 dev_dbg(CARD_TO_DEV(ctrl->card),
519 "Completing DMA%d"
520 "(laddr x%x tag %d st: x%x cnt: x%04x) from idx %d.\n",
521 ctrl->id, dma->laddr, tag, status, count,
522 ctrl->status.idx);
523
524 atomic_dec(&ctrl->stats.hw_q_depth);
525
526 mod_timer(&ctrl->activity_timer,
527 jiffies + DMA_ACTIVITY_TIMEOUT);
528
529 if (status)
530 rsxx_handle_dma_error(ctrl, dma, status);
531 else
c95246c3 532 rsxx_complete_dma(ctrl, dma, 0);
8722ff8c 533
534 push_tracker(ctrl->trackers, tag);
535
536 ctrl->status.idx = (ctrl->status.idx + 1) &
537 RSXX_CS_IDX_MASK;
538 ctrl->e_cnt++;
539
540 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
541 }
542
543 dma_intr_coal_auto_tune(ctrl->card);
544
545 if (atomic_read(&ctrl->stats.hw_q_depth) == 0)
546 del_timer_sync(&ctrl->activity_timer);
547
548 spin_lock_irqsave(&ctrl->card->irq_lock, flags);
549 rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id));
550 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
551
0ab4743e 552 spin_lock_bh(&ctrl->queue_lock);
8722ff8c 553 if (ctrl->stats.sw_q_depth)
554 queue_work(ctrl->issue_wq, &ctrl->issue_dma_work);
0ab4743e 555 spin_unlock_bh(&ctrl->queue_lock);
8722ff8c 556}
557
558static int rsxx_queue_discard(struct rsxx_cardinfo *card,
559 struct list_head *q,
560 unsigned int laddr,
561 rsxx_dma_cb cb,
562 void *cb_data)
563{
564 struct rsxx_dma *dma;
565
566 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
567 if (!dma)
568 return -ENOMEM;
569
570 dma->cmd = HW_CMD_BLK_DISCARD;
571 dma->laddr = laddr;
572 dma->dma_addr = 0;
573 dma->sub_page.off = 0;
574 dma->sub_page.cnt = 0;
575 dma->page = NULL;
576 dma->pg_off = 0;
577 dma->cb = cb;
578 dma->cb_data = cb_data;
579
580 dev_dbg(CARD_TO_DEV(card), "Queuing[D] laddr %x\n", dma->laddr);
581
582 list_add_tail(&dma->list, q);
583
584 return 0;
585}
586
587static int rsxx_queue_dma(struct rsxx_cardinfo *card,
588 struct list_head *q,
589 int dir,
590 unsigned int dma_off,
591 unsigned int dma_len,
592 unsigned int laddr,
593 struct page *page,
594 unsigned int pg_off,
595 rsxx_dma_cb cb,
596 void *cb_data)
597{
598 struct rsxx_dma *dma;
599
600 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
601 if (!dma)
602 return -ENOMEM;
603
604 dma->dma_addr = pci_map_page(card->dev, page, pg_off, dma_len,
605 dir ? PCI_DMA_TODEVICE :
606 PCI_DMA_FROMDEVICE);
607 if (!dma->dma_addr) {
608 kmem_cache_free(rsxx_dma_pool, dma);
609 return -ENOMEM;
610 }
611
612 dma->cmd = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ;
613 dma->laddr = laddr;
614 dma->sub_page.off = (dma_off >> 9);
615 dma->sub_page.cnt = (dma_len >> 9);
616 dma->page = page;
617 dma->pg_off = pg_off;
618 dma->cb = cb;
619 dma->cb_data = cb_data;
620
621 dev_dbg(CARD_TO_DEV(card),
622 "Queuing[%c] laddr %x off %d cnt %d page %p pg_off %d\n",
623 dir ? 'W' : 'R', dma->laddr, dma->sub_page.off,
624 dma->sub_page.cnt, dma->page, dma->pg_off);
625
626 /* Queue the DMA */
627 list_add_tail(&dma->list, q);
628
629 return 0;
630}
631
632int rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
633 struct bio *bio,
634 atomic_t *n_dmas,
635 rsxx_dma_cb cb,
636 void *cb_data)
637{
638 struct list_head dma_list[RSXX_MAX_TARGETS];
639 struct bio_vec *bvec;
640 unsigned long long addr8;
641 unsigned int laddr;
642 unsigned int bv_len;
643 unsigned int bv_off;
644 unsigned int dma_off;
645 unsigned int dma_len;
646 int dma_cnt[RSXX_MAX_TARGETS];
647 int tgt;
648 int st;
649 int i;
650
651 addr8 = bio->bi_sector << 9; /* sectors are 512 bytes */
652 atomic_set(n_dmas, 0);
653
654 for (i = 0; i < card->n_targets; i++) {
655 INIT_LIST_HEAD(&dma_list[i]);
656 dma_cnt[i] = 0;
657 }
658
659 if (bio->bi_rw & REQ_DISCARD) {
660 bv_len = bio->bi_size;
661
662 while (bv_len > 0) {
663 tgt = rsxx_get_dma_tgt(card, addr8);
664 laddr = rsxx_addr8_to_laddr(addr8, card);
665
666 st = rsxx_queue_discard(card, &dma_list[tgt], laddr,
667 cb, cb_data);
668 if (st)
669 goto bvec_err;
670
671 dma_cnt[tgt]++;
672 atomic_inc(n_dmas);
673 addr8 += RSXX_HW_BLK_SIZE;
674 bv_len -= RSXX_HW_BLK_SIZE;
675 }
676 } else {
677 bio_for_each_segment(bvec, bio, i) {
678 bv_len = bvec->bv_len;
679 bv_off = bvec->bv_offset;
680
681 while (bv_len > 0) {
682 tgt = rsxx_get_dma_tgt(card, addr8);
683 laddr = rsxx_addr8_to_laddr(addr8, card);
684 dma_off = addr8 & RSXX_HW_BLK_MASK;
685 dma_len = min(bv_len,
686 RSXX_HW_BLK_SIZE - dma_off);
687
688 st = rsxx_queue_dma(card, &dma_list[tgt],
689 bio_data_dir(bio),
690 dma_off, dma_len,
691 laddr, bvec->bv_page,
692 bv_off, cb, cb_data);
693 if (st)
694 goto bvec_err;
695
696 dma_cnt[tgt]++;
697 atomic_inc(n_dmas);
698 addr8 += dma_len;
699 bv_off += dma_len;
700 bv_len -= dma_len;
701 }
702 }
703 }
704
705 for (i = 0; i < card->n_targets; i++) {
706 if (!list_empty(&dma_list[i])) {
0ab4743e 707 spin_lock_bh(&card->ctrl[i].queue_lock);
8722ff8c 708 card->ctrl[i].stats.sw_q_depth += dma_cnt[i];
709 list_splice_tail(&dma_list[i], &card->ctrl[i].queue);
0ab4743e 710 spin_unlock_bh(&card->ctrl[i].queue_lock);
8722ff8c 711
712 queue_work(card->ctrl[i].issue_wq,
713 &card->ctrl[i].issue_dma_work);
714 }
715 }
716
717 return 0;
718
719bvec_err:
0ab4743e
PK		 720 for (i = 0; i < card->n_targets; i++) {
721 spin_lock_bh(&card->ctrl[i].queue_lock);
722 rsxx_cleanup_dma_queue(&card->ctrl[i], &dma_list[i]);
723 spin_unlock_bh(&card->ctrl[i].queue_lock);
724 }
8722ff8c 725
726 return st;
727}
728
729
730/*----------------- DMA Engine Initialization & Setup -------------------*/
c95246c3
PK		 731int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl)
732{
733 ctrl->status.buf = pci_alloc_consistent(dev, STATUS_BUFFER_SIZE8,
734 &ctrl->status.dma_addr);
735 ctrl->cmd.buf = pci_alloc_consistent(dev, COMMAND_BUFFER_SIZE8,
736 &ctrl->cmd.dma_addr);
737 if (ctrl->status.buf == NULL || ctrl->cmd.buf == NULL)
738 return -ENOMEM;
739
740 memset(ctrl->status.buf, 0xac, STATUS_BUFFER_SIZE8);
741 iowrite32(lower_32_bits(ctrl->status.dma_addr),
742 ctrl->regmap + SB_ADD_LO);
743 iowrite32(upper_32_bits(ctrl->status.dma_addr),
744 ctrl->regmap + SB_ADD_HI);
745
746 memset(ctrl->cmd.buf, 0x83, COMMAND_BUFFER_SIZE8);
747 iowrite32(lower_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_LO);
748 iowrite32(upper_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_HI);
749
750 ctrl->status.idx = ioread32(ctrl->regmap + HW_STATUS_CNT);
751 if (ctrl->status.idx > RSXX_MAX_OUTSTANDING_CMDS) {
752 dev_crit(&dev->dev, "Failed reading status cnt x%x\n",
753 ctrl->status.idx);
754 return -EINVAL;
755 }
756 iowrite32(ctrl->status.idx, ctrl->regmap + HW_STATUS_CNT);
757 iowrite32(ctrl->status.idx, ctrl->regmap + SW_STATUS_CNT);
758
759 ctrl->cmd.idx = ioread32(ctrl->regmap + HW_CMD_IDX);
760 if (ctrl->cmd.idx > RSXX_MAX_OUTSTANDING_CMDS) {
761 dev_crit(&dev->dev, "Failed reading cmd cnt x%x\n",
762 ctrl->status.idx);
763 return -EINVAL;
764 }
765 iowrite32(ctrl->cmd.idx, ctrl->regmap + HW_CMD_IDX);
766 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
767
768 return 0;
769}
770
8722ff8c 771static int rsxx_dma_ctrl_init(struct pci_dev *dev,
772 struct rsxx_dma_ctrl *ctrl)
773{
774 int i;
c95246c3 775 int st;
8722ff8c 776
777 memset(&ctrl->stats, 0, sizeof(ctrl->stats));
778
8722ff8c 779 ctrl->trackers = vmalloc(DMA_TRACKER_LIST_SIZE8);
780 if (!ctrl->trackers)
781 return -ENOMEM;
782
783 ctrl->trackers->head = 0;
784 for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
785 ctrl->trackers->list[i].next_tag = i + 1;
786 ctrl->trackers->list[i].dma = NULL;
787 }
788 ctrl->trackers->list[RSXX_MAX_OUTSTANDING_CMDS-1].next_tag = -1;
789 spin_lock_init(&ctrl->trackers->lock);
790
791 spin_lock_init(&ctrl->queue_lock);
792 INIT_LIST_HEAD(&ctrl->queue);
793
794 setup_timer(&ctrl->activity_timer, dma_engine_stalled,
c206c709 795 (unsigned long)ctrl);
8722ff8c 796
797 ctrl->issue_wq = alloc_ordered_workqueue(DRIVER_NAME"_issue", 0);
798 if (!ctrl->issue_wq)
799 return -ENOMEM;
800
801 ctrl->done_wq = alloc_ordered_workqueue(DRIVER_NAME"_done", 0);
802 if (!ctrl->done_wq)
803 return -ENOMEM;
804
805 INIT_WORK(&ctrl->issue_dma_work, rsxx_issue_dmas);
806 INIT_WORK(&ctrl->dma_done_work, rsxx_dma_done);
807
c95246c3
PK		 808 st = rsxx_hw_buffers_init(dev, ctrl);
809 if (st)
810 return st;
8722ff8c 811
8722ff8c 812 return 0;
813}
814
c206c709 815static int rsxx_dma_stripe_setup(struct rsxx_cardinfo *card,
8722ff8c 816 unsigned int stripe_size8)
817{
818 if (!is_power_of_2(stripe_size8)) {
819 dev_err(CARD_TO_DEV(card),
820 "stripe_size is NOT a power of 2!\n");
821 return -EINVAL;
822 }
823
824 card->_stripe.lower_mask = stripe_size8 - 1;
825
826 card->_stripe.upper_mask = ~(card->_stripe.lower_mask);
827 card->_stripe.upper_shift = ffs(card->n_targets) - 1;
828
829 card->_stripe.target_mask = card->n_targets - 1;
830 card->_stripe.target_shift = ffs(stripe_size8) - 1;
831
832 dev_dbg(CARD_TO_DEV(card), "_stripe.lower_mask = x%016llx\n",
833 card->_stripe.lower_mask);
834 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_shift = x%016llx\n",
835 card->_stripe.upper_shift);
836 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_mask = x%016llx\n",
837 card->_stripe.upper_mask);
838 dev_dbg(CARD_TO_DEV(card), "_stripe.target_mask = x%016llx\n",
839 card->_stripe.target_mask);
840 dev_dbg(CARD_TO_DEV(card), "_stripe.target_shift = x%016llx\n",
841 card->_stripe.target_shift);
842
843 return 0;
844}
845
c95246c3 846int rsxx_dma_configure(struct rsxx_cardinfo *card)
8722ff8c 847{
848 u32 intr_coal;
849
850 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
851 card->config.data.intr_coal.count,
852 card->config.data.intr_coal.latency);
853 iowrite32(intr_coal, card->regmap + INTR_COAL);
854
855 return rsxx_dma_stripe_setup(card, card->config.data.stripe_size);
856}
857
858int rsxx_dma_setup(struct rsxx_cardinfo *card)
859{
860 unsigned long flags;
861 int st;
862 int i;
863
864 dev_info(CARD_TO_DEV(card),
865 "Initializing %d DMA targets\n",
866 card->n_targets);
867
868 /* Regmap is divided up into 4K chunks. One for each DMA channel */
869 for (i = 0; i < card->n_targets; i++)
870 card->ctrl[i].regmap = card->regmap + (i * 4096);
871
872 card->dma_fault = 0;
873
874 /* Reset the DMA queues */
875 rsxx_dma_queue_reset(card);
876
877 /************* Setup DMA Control *************/
878 for (i = 0; i < card->n_targets; i++) {
879 st = rsxx_dma_ctrl_init(card->dev, &card->ctrl[i]);
880 if (st)
881 goto failed_dma_setup;
882
883 card->ctrl[i].card = card;
884 card->ctrl[i].id = i;
885 }
886
887 card->scrub_hard = 1;
888
889 if (card->config_valid)
890 rsxx_dma_configure(card);
891
892 /* Enable the interrupts after all setup has completed. */
893 for (i = 0; i < card->n_targets; i++) {
894 spin_lock_irqsave(&card->irq_lock, flags);
895 rsxx_enable_ier_and_isr(card, CR_INTR_DMA(i));
896 spin_unlock_irqrestore(&card->irq_lock, flags);
897 }
898
899 return 0;
900
901failed_dma_setup:
902 for (i = 0; i < card->n_targets; i++) {
903 struct rsxx_dma_ctrl *ctrl = &card->ctrl[i];
904
905 if (ctrl->issue_wq) {
906 destroy_workqueue(ctrl->issue_wq);
907 ctrl->issue_wq = NULL;
908 }
909
910 if (ctrl->done_wq) {
911 destroy_workqueue(ctrl->done_wq);
912 ctrl->done_wq = NULL;
913 }
914
915 if (ctrl->trackers)
916 vfree(ctrl->trackers);
917
918 if (ctrl->status.buf)
919 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
920 ctrl->status.buf,
921 ctrl->status.dma_addr);
922 if (ctrl->cmd.buf)
923 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
924 ctrl->cmd.buf, ctrl->cmd.dma_addr);
925 }
926
927 return st;
928}
929
0ab4743e
PK		 930int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl)
931{
932 struct rsxx_dma *dma;
933 int i;
934 int cnt = 0;
935
936 /* Clean up issued DMAs */
937 for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
938 dma = get_tracker_dma(ctrl->trackers, i);
939 if (dma) {
940 atomic_dec(&ctrl->stats.hw_q_depth);
941 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
942 push_tracker(ctrl->trackers, i);
943 cnt++;
944 }
945 }
946
947 return cnt;
948}
8722ff8c 949
950void rsxx_dma_destroy(struct rsxx_cardinfo *card)
951{
952 struct rsxx_dma_ctrl *ctrl;
0ab4743e 953 int i;
8722ff8c 954
955 for (i = 0; i < card->n_targets; i++) {
956 ctrl = &card->ctrl[i];
957
958 if (ctrl->issue_wq) {
959 destroy_workqueue(ctrl->issue_wq);
960 ctrl->issue_wq = NULL;
961 }
962
963 if (ctrl->done_wq) {
964 destroy_workqueue(ctrl->done_wq);
965 ctrl->done_wq = NULL;
966 }
967
968 if (timer_pending(&ctrl->activity_timer))
969 del_timer_sync(&ctrl->activity_timer);
970
971 /* Clean up the DMA queue */
0ab4743e
PK		 972 spin_lock_bh(&ctrl->queue_lock);
973 rsxx_cleanup_dma_queue(ctrl, &ctrl->queue);
974 spin_unlock_bh(&ctrl->queue_lock);
8722ff8c 975
0ab4743e 976 rsxx_dma_cancel(ctrl);
8722ff8c 977
978 vfree(ctrl->trackers);
979
980 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
981 ctrl->status.buf, ctrl->status.dma_addr);
982 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
983 ctrl->cmd.buf, ctrl->cmd.dma_addr);
984 }
985}
986
4dcaf472 987int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card)
c95246c3
PK		 988{
989 int i;
990 int j;
991 int cnt;
992 struct rsxx_dma *dma;
d8d595df
PK		 993 struct list_head *issued_dmas;
994
995 issued_dmas = kzalloc(sizeof(*issued_dmas) * card->n_targets,
996 GFP_KERNEL);
4dcaf472
PK		 997 if (!issued_dmas)
998 return -ENOMEM;
c95246c3
PK		 999
1000 for (i = 0; i < card->n_targets; i++) {
1001 INIT_LIST_HEAD(&issued_dmas[i]);
1002 cnt = 0;
1003 for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) {
1004 dma = get_tracker_dma(card->ctrl[i].trackers, j);
1005 if (dma == NULL)
1006 continue;
1007
1008 if (dma->cmd == HW_CMD_BLK_WRITE)
1009 card->ctrl[i].stats.writes_issued--;
1010 else if (dma->cmd == HW_CMD_BLK_DISCARD)
1011 card->ctrl[i].stats.discards_issued--;
1012 else
1013 card->ctrl[i].stats.reads_issued--;
1014
1015 list_add_tail(&dma->list, &issued_dmas[i]);
1016 push_tracker(card->ctrl[i].trackers, j);
1017 cnt++;
1018 }
1019
0ab4743e 1020 spin_lock_bh(&card->ctrl[i].queue_lock);
c95246c3
PK		 1021 list_splice(&issued_dmas[i], &card->ctrl[i].queue);
1022
1023 atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth);
1024 card->ctrl[i].stats.sw_q_depth += cnt;
1025 card->ctrl[i].e_cnt = 0;
1026
1027 list_for_each_entry(dma, &card->ctrl[i].queue, list) {
1028 if (dma->dma_addr)
1029 pci_unmap_page(card->dev, dma->dma_addr,
1030 get_dma_size(dma),
1031 dma->cmd == HW_CMD_BLK_WRITE ?
1032 PCI_DMA_TODEVICE :
1033 PCI_DMA_FROMDEVICE);
1034 }
0ab4743e 1035 spin_unlock_bh(&card->ctrl[i].queue_lock);
c95246c3 1036 }
d8d595df
PK		 1037
1038 kfree(issued_dmas);
4dcaf472
PK		 1039
1040 return 0;
c95246c3
PK		 1041}
1042
c95246c3
PK		 1043int rsxx_eeh_remap_dmas(struct rsxx_cardinfo *card)
1044{
1045 struct rsxx_dma *dma;
c95246c3
PK		 1046 int i;
1047
1048 for (i = 0; i < card->n_targets; i++) {
0ab4743e 1049 spin_lock_bh(&card->ctrl[i].queue_lock);
c95246c3
PK		 1050 list_for_each_entry(dma, &card->ctrl[i].queue, list) {
1051 dma->dma_addr = pci_map_page(card->dev, dma->page,
1052 dma->pg_off, get_dma_size(dma),
1053 dma->cmd == HW_CMD_BLK_WRITE ?
1054 PCI_DMA_TODEVICE :
1055 PCI_DMA_FROMDEVICE);
1056 if (!dma->dma_addr) {
0ab4743e 1057 spin_unlock_bh(&card->ctrl[i].queue_lock);
c95246c3
PK		 1058 kmem_cache_free(rsxx_dma_pool, dma);
1059 return -ENOMEM;
1060 }
1061 }
0ab4743e 1062 spin_unlock_bh(&card->ctrl[i].queue_lock);
c95246c3
PK		 1063 }
1064
1065 return 0;
1066}
8722ff8c 1067
1068int rsxx_dma_init(void)
1069{
1070 rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN);
1071 if (!rsxx_dma_pool)
1072 return -ENOMEM;
1073
1074 return 0;
1075}
1076
1077
1078void rsxx_dma_cleanup(void)
1079{
1080 kmem_cache_destroy(rsxx_dma_pool);
1081}
1082
Linux kernel - Deliverables
This page took 0.235287 seconds and 5 git commands to generate.