Commit | Line | Data |
---|---|---|
8722ff8c | 1 | /* |
2 | * Filename: dma.c | |
3 | * | |
4 | * | |
5 | * Authors: Joshua Morris <josh.h.morris@us.ibm.com> | |
6 | * Philip Kelleher <pjk1939@linux.vnet.ibm.com> | |
7 | * | |
8 | * (C) Copyright 2013 IBM Corporation | |
9 | * | |
10 | * This program is free software; you can redistribute it and/or | |
11 | * modify it under the terms of the GNU General Public License as | |
12 | * published by the Free Software Foundation; either version 2 of the | |
13 | * License, or (at your option) any later version. | |
14 | * | |
15 | * This program is distributed in the hope that it will be useful, but | |
16 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
17 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
18 | * General Public License for more details. | |
19 | * | |
20 | * You should have received a copy of the GNU General Public License | |
21 | * along with this program; if not, write to the Free Software Foundation, | |
22 | * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | |
23 | */ | |
24 | ||
e5e9fdaa | 25 | #include <linux/slab.h> |
8722ff8c | 26 | #include "rsxx_priv.h" |
27 | ||
28 | struct rsxx_dma { | |
29 | struct list_head list; | |
30 | u8 cmd; | |
9bb3c446 | 31 | unsigned int laddr; /* Logical address */ |
8722ff8c | 32 | struct { |
33 | u32 off; | |
34 | u32 cnt; | |
35 | } sub_page; | |
36 | dma_addr_t dma_addr; | |
37 | struct page *page; | |
38 | unsigned int pg_off; /* Page Offset */ | |
39 | rsxx_dma_cb cb; | |
40 | void *cb_data; | |
41 | }; | |
42 | ||
43 | /* This timeout is used to detect a stalled DMA channel */ | |
44 | #define DMA_ACTIVITY_TIMEOUT msecs_to_jiffies(10000) | |
45 | ||
46 | struct hw_status { | |
47 | u8 status; | |
48 | u8 tag; | |
49 | __le16 count; | |
50 | __le32 _rsvd2; | |
51 | __le64 _rsvd3; | |
52 | } __packed; | |
53 | ||
54 | enum rsxx_dma_status { | |
55 | DMA_SW_ERR = 0x1, | |
56 | DMA_HW_FAULT = 0x2, | |
57 | DMA_CANCELLED = 0x4, | |
58 | }; | |
59 | ||
60 | struct hw_cmd { | |
61 | u8 command; | |
62 | u8 tag; | |
63 | u8 _rsvd; | |
64 | u8 sub_page; /* Bit[0:2]: 512byte offset */ | |
65 | /* Bit[4:6]: 512byte count */ | |
66 | __le32 device_addr; | |
67 | __le64 host_addr; | |
68 | } __packed; | |
69 | ||
70 | enum rsxx_hw_cmd { | |
71 | HW_CMD_BLK_DISCARD = 0x70, | |
72 | HW_CMD_BLK_WRITE = 0x80, | |
73 | HW_CMD_BLK_READ = 0xC0, | |
74 | HW_CMD_BLK_RECON_READ = 0xE0, | |
75 | }; | |
76 | ||
77 | enum rsxx_hw_status { | |
78 | HW_STATUS_CRC = 0x01, | |
79 | HW_STATUS_HARD_ERR = 0x02, | |
80 | HW_STATUS_SOFT_ERR = 0x04, | |
81 | HW_STATUS_FAULT = 0x08, | |
82 | }; | |
83 | ||
8722ff8c | 84 | static struct kmem_cache *rsxx_dma_pool; |
85 | ||
86 | struct dma_tracker { | |
87 | int next_tag; | |
88 | struct rsxx_dma *dma; | |
89 | }; | |
90 | ||
91 | #define DMA_TRACKER_LIST_SIZE8 (sizeof(struct dma_tracker_list) + \ | |
92 | (sizeof(struct dma_tracker) * RSXX_MAX_OUTSTANDING_CMDS)) | |
93 | ||
94 | struct dma_tracker_list { | |
95 | spinlock_t lock; | |
96 | int head; | |
97 | struct dma_tracker list[0]; | |
98 | }; | |
99 | ||
100 | ||
101 | /*----------------- Misc Utility Functions -------------------*/ | |
c206c709 | 102 | static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card) |
8722ff8c | 103 | { |
104 | unsigned long long tgt_addr8; | |
105 | ||
106 | tgt_addr8 = ((addr8 >> card->_stripe.upper_shift) & | |
107 | card->_stripe.upper_mask) | | |
108 | ((addr8) & card->_stripe.lower_mask); | |
109 | do_div(tgt_addr8, RSXX_HW_BLK_SIZE); | |
110 | return tgt_addr8; | |
111 | } | |
112 | ||
c206c709 | 113 | static unsigned int rsxx_get_dma_tgt(struct rsxx_cardinfo *card, u64 addr8) |
8722ff8c | 114 | { |
115 | unsigned int tgt; | |
116 | ||
117 | tgt = (addr8 >> card->_stripe.target_shift) & card->_stripe.target_mask; | |
118 | ||
119 | return tgt; | |
120 | } | |
121 | ||
c95246c3 | 122 | void rsxx_dma_queue_reset(struct rsxx_cardinfo *card) |
8722ff8c | 123 | { |
124 | /* Reset all DMA Command/Status Queues */ | |
125 | iowrite32(DMA_QUEUE_RESET, card->regmap + RESET); | |
126 | } | |
127 | ||
128 | static unsigned int get_dma_size(struct rsxx_dma *dma) | |
129 | { | |
130 | if (dma->sub_page.cnt) | |
131 | return dma->sub_page.cnt << 9; | |
132 | else | |
133 | return RSXX_HW_BLK_SIZE; | |
134 | } | |
135 | ||
136 | ||
137 | /*----------------- DMA Tracker -------------------*/ | |
138 | static void set_tracker_dma(struct dma_tracker_list *trackers, | |
139 | int tag, | |
140 | struct rsxx_dma *dma) | |
141 | { | |
142 | trackers->list[tag].dma = dma; | |
143 | } | |
144 | ||
145 | static struct rsxx_dma *get_tracker_dma(struct dma_tracker_list *trackers, | |
146 | int tag) | |
147 | { | |
148 | return trackers->list[tag].dma; | |
149 | } | |
150 | ||
151 | static int pop_tracker(struct dma_tracker_list *trackers) | |
152 | { | |
153 | int tag; | |
154 | ||
155 | spin_lock(&trackers->lock); | |
156 | tag = trackers->head; | |
157 | if (tag != -1) { | |
158 | trackers->head = trackers->list[tag].next_tag; | |
159 | trackers->list[tag].next_tag = -1; | |
160 | } | |
161 | spin_unlock(&trackers->lock); | |
162 | ||
163 | return tag; | |
164 | } | |
165 | ||
166 | static void push_tracker(struct dma_tracker_list *trackers, int tag) | |
167 | { | |
168 | spin_lock(&trackers->lock); | |
169 | trackers->list[tag].next_tag = trackers->head; | |
170 | trackers->head = tag; | |
171 | trackers->list[tag].dma = NULL; | |
172 | spin_unlock(&trackers->lock); | |
173 | } | |
174 | ||
175 | ||
176 | /*----------------- Interrupt Coalescing -------------*/ | |
177 | /* | |
178 | * Interrupt Coalescing Register Format: | |
179 | * Interrupt Timer (64ns units) [15:0] | |
180 | * Interrupt Count [24:16] | |
181 | * Reserved [31:25] | |
182 | */ | |
183 | #define INTR_COAL_LATENCY_MASK (0x0000ffff) | |
184 | ||
185 | #define INTR_COAL_COUNT_SHIFT 16 | |
186 | #define INTR_COAL_COUNT_BITS 9 | |
187 | #define INTR_COAL_COUNT_MASK (((1 << INTR_COAL_COUNT_BITS) - 1) << \ | |
188 | INTR_COAL_COUNT_SHIFT) | |
189 | #define INTR_COAL_LATENCY_UNITS_NS 64 | |
190 | ||
191 | ||
192 | static u32 dma_intr_coal_val(u32 mode, u32 count, u32 latency) | |
193 | { | |
194 | u32 latency_units = latency / INTR_COAL_LATENCY_UNITS_NS; | |
195 | ||
196 | if (mode == RSXX_INTR_COAL_DISABLED) | |
197 | return 0; | |
198 | ||
199 | return ((count << INTR_COAL_COUNT_SHIFT) & INTR_COAL_COUNT_MASK) | | |
200 | (latency_units & INTR_COAL_LATENCY_MASK); | |
201 | ||
202 | } | |
203 | ||
204 | static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card) | |
205 | { | |
206 | int i; | |
207 | u32 q_depth = 0; | |
208 | u32 intr_coal; | |
209 | ||
c95246c3 PK |
210 | if (card->config.data.intr_coal.mode != RSXX_INTR_COAL_AUTO_TUNE || |
211 | unlikely(card->eeh_state)) | |
8722ff8c | 212 | return; |
213 | ||
214 | for (i = 0; i < card->n_targets; i++) | |
215 | q_depth += atomic_read(&card->ctrl[i].stats.hw_q_depth); | |
216 | ||
217 | intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode, | |
218 | q_depth / 2, | |
219 | card->config.data.intr_coal.latency); | |
220 | iowrite32(intr_coal, card->regmap + INTR_COAL); | |
221 | } | |
222 | ||
223 | /*----------------- RSXX DMA Handling -------------------*/ | |
e5feab22 PK |
224 | static void rsxx_free_dma(struct rsxx_dma_ctrl *ctrl, struct rsxx_dma *dma) |
225 | { | |
e35f38bf PK |
226 | if (dma->cmd != HW_CMD_BLK_DISCARD) { |
227 | if (!pci_dma_mapping_error(ctrl->card->dev, dma->dma_addr)) { | |
228 | pci_unmap_page(ctrl->card->dev, dma->dma_addr, | |
229 | get_dma_size(dma), | |
230 | dma->cmd == HW_CMD_BLK_WRITE ? | |
231 | PCI_DMA_TODEVICE : | |
232 | PCI_DMA_FROMDEVICE); | |
233 | } | |
e5feab22 PK |
234 | } |
235 | ||
236 | kmem_cache_free(rsxx_dma_pool, dma); | |
237 | } | |
238 | ||
c95246c3 | 239 | static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl, |
8722ff8c | 240 | struct rsxx_dma *dma, |
241 | unsigned int status) | |
242 | { | |
243 | if (status & DMA_SW_ERR) | |
c95246c3 | 244 | ctrl->stats.dma_sw_err++; |
8722ff8c | 245 | if (status & DMA_HW_FAULT) |
c95246c3 | 246 | ctrl->stats.dma_hw_fault++; |
8722ff8c | 247 | if (status & DMA_CANCELLED) |
c95246c3 | 248 | ctrl->stats.dma_cancelled++; |
8722ff8c | 249 | |
8722ff8c | 250 | if (dma->cb) |
c95246c3 | 251 | dma->cb(ctrl->card, dma->cb_data, status ? 1 : 0); |
8722ff8c | 252 | |
e5feab22 | 253 | rsxx_free_dma(ctrl, dma); |
8722ff8c | 254 | } |
255 | ||
0ab4743e | 256 | int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl, |
e5feab22 | 257 | struct list_head *q, unsigned int done) |
0ab4743e PK |
258 | { |
259 | struct rsxx_dma *dma; | |
260 | struct rsxx_dma *tmp; | |
261 | int cnt = 0; | |
262 | ||
263 | list_for_each_entry_safe(dma, tmp, q, list) { | |
264 | list_del(&dma->list); | |
e5feab22 PK |
265 | if (done & COMPLETE_DMA) |
266 | rsxx_complete_dma(ctrl, dma, DMA_CANCELLED); | |
267 | else | |
268 | rsxx_free_dma(ctrl, dma); | |
0ab4743e PK |
269 | cnt++; |
270 | } | |
271 | ||
272 | return cnt; | |
273 | } | |
274 | ||
8722ff8c | 275 | static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl, |
276 | struct rsxx_dma *dma) | |
277 | { | |
278 | /* | |
279 | * Requeued DMAs go to the front of the queue so they are issued | |
280 | * first. | |
281 | */ | |
0ab4743e | 282 | spin_lock_bh(&ctrl->queue_lock); |
62302508 | 283 | ctrl->stats.sw_q_depth++; |
8722ff8c | 284 | list_add(&dma->list, &ctrl->queue); |
0ab4743e | 285 | spin_unlock_bh(&ctrl->queue_lock); |
8722ff8c | 286 | } |
287 | ||
288 | static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl, | |
289 | struct rsxx_dma *dma, | |
290 | u8 hw_st) | |
291 | { | |
292 | unsigned int status = 0; | |
293 | int requeue_cmd = 0; | |
294 | ||
295 | dev_dbg(CARD_TO_DEV(ctrl->card), | |
296 | "Handling DMA error(cmd x%02x, laddr x%08x st:x%02x)\n", | |
297 | dma->cmd, dma->laddr, hw_st); | |
298 | ||
299 | if (hw_st & HW_STATUS_CRC) | |
300 | ctrl->stats.crc_errors++; | |
301 | if (hw_st & HW_STATUS_HARD_ERR) | |
302 | ctrl->stats.hard_errors++; | |
303 | if (hw_st & HW_STATUS_SOFT_ERR) | |
304 | ctrl->stats.soft_errors++; | |
305 | ||
306 | switch (dma->cmd) { | |
307 | case HW_CMD_BLK_READ: | |
308 | if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) { | |
309 | if (ctrl->card->scrub_hard) { | |
310 | dma->cmd = HW_CMD_BLK_RECON_READ; | |
311 | requeue_cmd = 1; | |
312 | ctrl->stats.reads_retried++; | |
313 | } else { | |
314 | status |= DMA_HW_FAULT; | |
315 | ctrl->stats.reads_failed++; | |
316 | } | |
317 | } else if (hw_st & HW_STATUS_FAULT) { | |
318 | status |= DMA_HW_FAULT; | |
319 | ctrl->stats.reads_failed++; | |
320 | } | |
321 | ||
322 | break; | |
323 | case HW_CMD_BLK_RECON_READ: | |
324 | if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) { | |
325 | /* Data could not be reconstructed. */ | |
326 | status |= DMA_HW_FAULT; | |
327 | ctrl->stats.reads_failed++; | |
328 | } | |
329 | ||
330 | break; | |
331 | case HW_CMD_BLK_WRITE: | |
332 | status |= DMA_HW_FAULT; | |
333 | ctrl->stats.writes_failed++; | |
334 | ||
335 | break; | |
336 | case HW_CMD_BLK_DISCARD: | |
337 | status |= DMA_HW_FAULT; | |
338 | ctrl->stats.discards_failed++; | |
339 | ||
340 | break; | |
341 | default: | |
342 | dev_err(CARD_TO_DEV(ctrl->card), | |
343 | "Unknown command in DMA!(cmd: x%02x " | |
344 | "laddr x%08x st: x%02x\n", | |
345 | dma->cmd, dma->laddr, hw_st); | |
346 | status |= DMA_SW_ERR; | |
347 | ||
348 | break; | |
349 | } | |
350 | ||
351 | if (requeue_cmd) | |
352 | rsxx_requeue_dma(ctrl, dma); | |
353 | else | |
c95246c3 | 354 | rsxx_complete_dma(ctrl, dma, status); |
8722ff8c | 355 | } |
356 | ||
357 | static void dma_engine_stalled(unsigned long data) | |
358 | { | |
359 | struct rsxx_dma_ctrl *ctrl = (struct rsxx_dma_ctrl *)data; | |
0ab4743e | 360 | int cnt; |
8722ff8c | 361 | |
c95246c3 PK |
362 | if (atomic_read(&ctrl->stats.hw_q_depth) == 0 || |
363 | unlikely(ctrl->card->eeh_state)) | |
8722ff8c | 364 | return; |
365 | ||
366 | if (ctrl->cmd.idx != ioread32(ctrl->regmap + SW_CMD_IDX)) { | |
367 | /* | |
368 | * The dma engine was stalled because the SW_CMD_IDX write | |
369 | * was lost. Issue it again to recover. | |
370 | */ | |
371 | dev_warn(CARD_TO_DEV(ctrl->card), | |
372 | "SW_CMD_IDX write was lost, re-writing...\n"); | |
373 | iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX); | |
374 | mod_timer(&ctrl->activity_timer, | |
375 | jiffies + DMA_ACTIVITY_TIMEOUT); | |
376 | } else { | |
377 | dev_warn(CARD_TO_DEV(ctrl->card), | |
378 | "DMA channel %d has stalled, faulting interface.\n", | |
379 | ctrl->id); | |
380 | ctrl->card->dma_fault = 1; | |
0ab4743e PK |
381 | |
382 | /* Clean up the DMA queue */ | |
383 | spin_lock(&ctrl->queue_lock); | |
e5feab22 | 384 | cnt = rsxx_cleanup_dma_queue(ctrl, &ctrl->queue, COMPLETE_DMA); |
0ab4743e PK |
385 | spin_unlock(&ctrl->queue_lock); |
386 | ||
387 | cnt += rsxx_dma_cancel(ctrl); | |
388 | ||
389 | if (cnt) | |
390 | dev_info(CARD_TO_DEV(ctrl->card), | |
391 | "Freed %d queued DMAs on channel %d\n", | |
392 | cnt, ctrl->id); | |
8722ff8c | 393 | } |
394 | } | |
395 | ||
31a70bb4 | 396 | static void rsxx_issue_dmas(struct rsxx_dma_ctrl *ctrl) |
8722ff8c | 397 | { |
8722ff8c | 398 | struct rsxx_dma *dma; |
399 | int tag; | |
400 | int cmds_pending = 0; | |
401 | struct hw_cmd *hw_cmd_buf; | |
1b21f5b2 | 402 | int dir; |
8722ff8c | 403 | |
8722ff8c | 404 | hw_cmd_buf = ctrl->cmd.buf; |
405 | ||
c95246c3 PK |
406 | if (unlikely(ctrl->card->halt) || |
407 | unlikely(ctrl->card->eeh_state)) | |
8722ff8c | 408 | return; |
409 | ||
410 | while (1) { | |
0ab4743e | 411 | spin_lock_bh(&ctrl->queue_lock); |
8722ff8c | 412 | if (list_empty(&ctrl->queue)) { |
0ab4743e | 413 | spin_unlock_bh(&ctrl->queue_lock); |
8722ff8c | 414 | break; |
415 | } | |
0ab4743e | 416 | spin_unlock_bh(&ctrl->queue_lock); |
8722ff8c | 417 | |
418 | tag = pop_tracker(ctrl->trackers); | |
419 | if (tag == -1) | |
420 | break; | |
421 | ||
0ab4743e | 422 | spin_lock_bh(&ctrl->queue_lock); |
8722ff8c | 423 | dma = list_entry(ctrl->queue.next, struct rsxx_dma, list); |
424 | list_del(&dma->list); | |
425 | ctrl->stats.sw_q_depth--; | |
0ab4743e | 426 | spin_unlock_bh(&ctrl->queue_lock); |
8722ff8c | 427 | |
428 | /* | |
429 | * This will catch any DMAs that slipped in right before the | |
430 | * fault, but was queued after all the other DMAs were | |
431 | * cancelled. | |
432 | */ | |
433 | if (unlikely(ctrl->card->dma_fault)) { | |
434 | push_tracker(ctrl->trackers, tag); | |
c95246c3 | 435 | rsxx_complete_dma(ctrl, dma, DMA_CANCELLED); |
8722ff8c | 436 | continue; |
437 | } | |
438 | ||
0317cd6d PK |
439 | if (dma->cmd != HW_CMD_BLK_DISCARD) { |
440 | if (dma->cmd == HW_CMD_BLK_WRITE) | |
441 | dir = PCI_DMA_TODEVICE; | |
442 | else | |
443 | dir = PCI_DMA_FROMDEVICE; | |
444 | ||
445 | /* | |
446 | * The function pci_map_page is placed here because we | |
447 | * can only, by design, issue up to 255 commands to the | |
448 | * hardware at one time per DMA channel. So the maximum | |
449 | * amount of mapped memory would be 255 * 4 channels * | |
450 | * 4096 Bytes which is less than 2GB, the limit of a x8 | |
451 | * Non-HWWD PCIe slot. This way the pci_map_page | |
452 | * function should never fail because of a lack of | |
453 | * mappable memory. | |
454 | */ | |
455 | dma->dma_addr = pci_map_page(ctrl->card->dev, dma->page, | |
456 | dma->pg_off, dma->sub_page.cnt << 9, dir); | |
457 | if (pci_dma_mapping_error(ctrl->card->dev, dma->dma_addr)) { | |
458 | push_tracker(ctrl->trackers, tag); | |
459 | rsxx_complete_dma(ctrl, dma, DMA_CANCELLED); | |
460 | continue; | |
461 | } | |
1b21f5b2 PK |
462 | } |
463 | ||
8722ff8c | 464 | set_tracker_dma(ctrl->trackers, tag, dma); |
465 | hw_cmd_buf[ctrl->cmd.idx].command = dma->cmd; | |
466 | hw_cmd_buf[ctrl->cmd.idx].tag = tag; | |
467 | hw_cmd_buf[ctrl->cmd.idx]._rsvd = 0; | |
468 | hw_cmd_buf[ctrl->cmd.idx].sub_page = | |
469 | ((dma->sub_page.cnt & 0x7) << 4) | | |
470 | (dma->sub_page.off & 0x7); | |
471 | ||
472 | hw_cmd_buf[ctrl->cmd.idx].device_addr = | |
473 | cpu_to_le32(dma->laddr); | |
474 | ||
475 | hw_cmd_buf[ctrl->cmd.idx].host_addr = | |
476 | cpu_to_le64(dma->dma_addr); | |
477 | ||
478 | dev_dbg(CARD_TO_DEV(ctrl->card), | |
479 | "Issue DMA%d(laddr %d tag %d) to idx %d\n", | |
480 | ctrl->id, dma->laddr, tag, ctrl->cmd.idx); | |
481 | ||
482 | ctrl->cmd.idx = (ctrl->cmd.idx + 1) & RSXX_CS_IDX_MASK; | |
483 | cmds_pending++; | |
484 | ||
485 | if (dma->cmd == HW_CMD_BLK_WRITE) | |
486 | ctrl->stats.writes_issued++; | |
487 | else if (dma->cmd == HW_CMD_BLK_DISCARD) | |
488 | ctrl->stats.discards_issued++; | |
489 | else | |
490 | ctrl->stats.reads_issued++; | |
491 | } | |
492 | ||
493 | /* Let HW know we've queued commands. */ | |
494 | if (cmds_pending) { | |
8722ff8c | 495 | atomic_add(cmds_pending, &ctrl->stats.hw_q_depth); |
496 | mod_timer(&ctrl->activity_timer, | |
497 | jiffies + DMA_ACTIVITY_TIMEOUT); | |
c95246c3 PK |
498 | |
499 | if (unlikely(ctrl->card->eeh_state)) { | |
500 | del_timer_sync(&ctrl->activity_timer); | |
501 | return; | |
502 | } | |
503 | ||
8722ff8c | 504 | iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX); |
505 | } | |
506 | } | |
507 | ||
31a70bb4 | 508 | static void rsxx_dma_done(struct rsxx_dma_ctrl *ctrl) |
8722ff8c | 509 | { |
8722ff8c | 510 | struct rsxx_dma *dma; |
511 | unsigned long flags; | |
512 | u16 count; | |
513 | u8 status; | |
514 | u8 tag; | |
515 | struct hw_status *hw_st_buf; | |
516 | ||
8722ff8c | 517 | hw_st_buf = ctrl->status.buf; |
518 | ||
519 | if (unlikely(ctrl->card->halt) || | |
c95246c3 PK |
520 | unlikely(ctrl->card->dma_fault) || |
521 | unlikely(ctrl->card->eeh_state)) | |
8722ff8c | 522 | return; |
523 | ||
524 | count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count); | |
525 | ||
526 | while (count == ctrl->e_cnt) { | |
527 | /* | |
528 | * The read memory-barrier is necessary to keep aggressive | |
529 | * processors/optimizers (such as the PPC Apple G5) from | |
530 | * reordering the following status-buffer tag & status read | |
531 | * *before* the count read on subsequent iterations of the | |
532 | * loop! | |
533 | */ | |
534 | rmb(); | |
535 | ||
536 | status = hw_st_buf[ctrl->status.idx].status; | |
537 | tag = hw_st_buf[ctrl->status.idx].tag; | |
538 | ||
539 | dma = get_tracker_dma(ctrl->trackers, tag); | |
540 | if (dma == NULL) { | |
541 | spin_lock_irqsave(&ctrl->card->irq_lock, flags); | |
542 | rsxx_disable_ier(ctrl->card, CR_INTR_DMA_ALL); | |
543 | spin_unlock_irqrestore(&ctrl->card->irq_lock, flags); | |
544 | ||
545 | dev_err(CARD_TO_DEV(ctrl->card), | |
546 | "No tracker for tag %d " | |
547 | "(idx %d id %d)\n", | |
548 | tag, ctrl->status.idx, ctrl->id); | |
549 | return; | |
550 | } | |
551 | ||
552 | dev_dbg(CARD_TO_DEV(ctrl->card), | |
553 | "Completing DMA%d" | |
554 | "(laddr x%x tag %d st: x%x cnt: x%04x) from idx %d.\n", | |
555 | ctrl->id, dma->laddr, tag, status, count, | |
556 | ctrl->status.idx); | |
557 | ||
558 | atomic_dec(&ctrl->stats.hw_q_depth); | |
559 | ||
560 | mod_timer(&ctrl->activity_timer, | |
561 | jiffies + DMA_ACTIVITY_TIMEOUT); | |
562 | ||
563 | if (status) | |
564 | rsxx_handle_dma_error(ctrl, dma, status); | |
565 | else | |
c95246c3 | 566 | rsxx_complete_dma(ctrl, dma, 0); |
8722ff8c | 567 | |
568 | push_tracker(ctrl->trackers, tag); | |
569 | ||
570 | ctrl->status.idx = (ctrl->status.idx + 1) & | |
571 | RSXX_CS_IDX_MASK; | |
572 | ctrl->e_cnt++; | |
573 | ||
574 | count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count); | |
575 | } | |
576 | ||
577 | dma_intr_coal_auto_tune(ctrl->card); | |
578 | ||
579 | if (atomic_read(&ctrl->stats.hw_q_depth) == 0) | |
580 | del_timer_sync(&ctrl->activity_timer); | |
581 | ||
582 | spin_lock_irqsave(&ctrl->card->irq_lock, flags); | |
583 | rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id)); | |
584 | spin_unlock_irqrestore(&ctrl->card->irq_lock, flags); | |
585 | ||
0ab4743e | 586 | spin_lock_bh(&ctrl->queue_lock); |
8722ff8c | 587 | if (ctrl->stats.sw_q_depth) |
588 | queue_work(ctrl->issue_wq, &ctrl->issue_dma_work); | |
0ab4743e | 589 | spin_unlock_bh(&ctrl->queue_lock); |
8722ff8c | 590 | } |
591 | ||
31a70bb4 PK |
592 | static void rsxx_schedule_issue(struct work_struct *work) |
593 | { | |
594 | struct rsxx_dma_ctrl *ctrl; | |
595 | ||
596 | ctrl = container_of(work, struct rsxx_dma_ctrl, issue_dma_work); | |
597 | ||
598 | mutex_lock(&ctrl->work_lock); | |
599 | rsxx_issue_dmas(ctrl); | |
600 | mutex_unlock(&ctrl->work_lock); | |
601 | } | |
602 | ||
603 | static void rsxx_schedule_done(struct work_struct *work) | |
604 | { | |
605 | struct rsxx_dma_ctrl *ctrl; | |
606 | ||
607 | ctrl = container_of(work, struct rsxx_dma_ctrl, dma_done_work); | |
608 | ||
609 | mutex_lock(&ctrl->work_lock); | |
610 | rsxx_dma_done(ctrl); | |
611 | mutex_unlock(&ctrl->work_lock); | |
612 | } | |
613 | ||
8722ff8c | 614 | static int rsxx_queue_discard(struct rsxx_cardinfo *card, |
615 | struct list_head *q, | |
616 | unsigned int laddr, | |
617 | rsxx_dma_cb cb, | |
618 | void *cb_data) | |
619 | { | |
620 | struct rsxx_dma *dma; | |
621 | ||
622 | dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL); | |
623 | if (!dma) | |
624 | return -ENOMEM; | |
625 | ||
626 | dma->cmd = HW_CMD_BLK_DISCARD; | |
627 | dma->laddr = laddr; | |
628 | dma->dma_addr = 0; | |
629 | dma->sub_page.off = 0; | |
630 | dma->sub_page.cnt = 0; | |
631 | dma->page = NULL; | |
632 | dma->pg_off = 0; | |
633 | dma->cb = cb; | |
634 | dma->cb_data = cb_data; | |
635 | ||
636 | dev_dbg(CARD_TO_DEV(card), "Queuing[D] laddr %x\n", dma->laddr); | |
637 | ||
638 | list_add_tail(&dma->list, q); | |
639 | ||
640 | return 0; | |
641 | } | |
642 | ||
643 | static int rsxx_queue_dma(struct rsxx_cardinfo *card, | |
644 | struct list_head *q, | |
645 | int dir, | |
646 | unsigned int dma_off, | |
647 | unsigned int dma_len, | |
648 | unsigned int laddr, | |
649 | struct page *page, | |
650 | unsigned int pg_off, | |
651 | rsxx_dma_cb cb, | |
652 | void *cb_data) | |
653 | { | |
654 | struct rsxx_dma *dma; | |
655 | ||
656 | dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL); | |
657 | if (!dma) | |
658 | return -ENOMEM; | |
659 | ||
8722ff8c | 660 | dma->cmd = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ; |
661 | dma->laddr = laddr; | |
662 | dma->sub_page.off = (dma_off >> 9); | |
663 | dma->sub_page.cnt = (dma_len >> 9); | |
664 | dma->page = page; | |
665 | dma->pg_off = pg_off; | |
666 | dma->cb = cb; | |
667 | dma->cb_data = cb_data; | |
668 | ||
669 | dev_dbg(CARD_TO_DEV(card), | |
670 | "Queuing[%c] laddr %x off %d cnt %d page %p pg_off %d\n", | |
671 | dir ? 'W' : 'R', dma->laddr, dma->sub_page.off, | |
672 | dma->sub_page.cnt, dma->page, dma->pg_off); | |
673 | ||
674 | /* Queue the DMA */ | |
675 | list_add_tail(&dma->list, q); | |
676 | ||
677 | return 0; | |
678 | } | |
679 | ||
680 | int rsxx_dma_queue_bio(struct rsxx_cardinfo *card, | |
681 | struct bio *bio, | |
682 | atomic_t *n_dmas, | |
683 | rsxx_dma_cb cb, | |
684 | void *cb_data) | |
685 | { | |
686 | struct list_head dma_list[RSXX_MAX_TARGETS]; | |
7988613b KO |
687 | struct bio_vec bvec; |
688 | struct bvec_iter iter; | |
8722ff8c | 689 | unsigned long long addr8; |
690 | unsigned int laddr; | |
691 | unsigned int bv_len; | |
692 | unsigned int bv_off; | |
693 | unsigned int dma_off; | |
694 | unsigned int dma_len; | |
695 | int dma_cnt[RSXX_MAX_TARGETS]; | |
696 | int tgt; | |
697 | int st; | |
698 | int i; | |
699 | ||
4f024f37 | 700 | addr8 = bio->bi_iter.bi_sector << 9; /* sectors are 512 bytes */ |
8722ff8c | 701 | atomic_set(n_dmas, 0); |
702 | ||
703 | for (i = 0; i < card->n_targets; i++) { | |
704 | INIT_LIST_HEAD(&dma_list[i]); | |
705 | dma_cnt[i] = 0; | |
706 | } | |
707 | ||
708 | if (bio->bi_rw & REQ_DISCARD) { | |
4f024f37 | 709 | bv_len = bio->bi_iter.bi_size; |
8722ff8c | 710 | |
711 | while (bv_len > 0) { | |
712 | tgt = rsxx_get_dma_tgt(card, addr8); | |
713 | laddr = rsxx_addr8_to_laddr(addr8, card); | |
714 | ||
715 | st = rsxx_queue_discard(card, &dma_list[tgt], laddr, | |
716 | cb, cb_data); | |
717 | if (st) | |
718 | goto bvec_err; | |
719 | ||
720 | dma_cnt[tgt]++; | |
721 | atomic_inc(n_dmas); | |
722 | addr8 += RSXX_HW_BLK_SIZE; | |
723 | bv_len -= RSXX_HW_BLK_SIZE; | |
724 | } | |
725 | } else { | |
7988613b KO |
726 | bio_for_each_segment(bvec, bio, iter) { |
727 | bv_len = bvec.bv_len; | |
728 | bv_off = bvec.bv_offset; | |
8722ff8c | 729 | |
730 | while (bv_len > 0) { | |
731 | tgt = rsxx_get_dma_tgt(card, addr8); | |
732 | laddr = rsxx_addr8_to_laddr(addr8, card); | |
733 | dma_off = addr8 & RSXX_HW_BLK_MASK; | |
734 | dma_len = min(bv_len, | |
735 | RSXX_HW_BLK_SIZE - dma_off); | |
736 | ||
737 | st = rsxx_queue_dma(card, &dma_list[tgt], | |
738 | bio_data_dir(bio), | |
739 | dma_off, dma_len, | |
7988613b | 740 | laddr, bvec.bv_page, |
8722ff8c | 741 | bv_off, cb, cb_data); |
742 | if (st) | |
743 | goto bvec_err; | |
744 | ||
745 | dma_cnt[tgt]++; | |
746 | atomic_inc(n_dmas); | |
747 | addr8 += dma_len; | |
748 | bv_off += dma_len; | |
749 | bv_len -= dma_len; | |
750 | } | |
751 | } | |
752 | } | |
753 | ||
754 | for (i = 0; i < card->n_targets; i++) { | |
755 | if (!list_empty(&dma_list[i])) { | |
0ab4743e | 756 | spin_lock_bh(&card->ctrl[i].queue_lock); |
8722ff8c | 757 | card->ctrl[i].stats.sw_q_depth += dma_cnt[i]; |
758 | list_splice_tail(&dma_list[i], &card->ctrl[i].queue); | |
0ab4743e | 759 | spin_unlock_bh(&card->ctrl[i].queue_lock); |
8722ff8c | 760 | |
761 | queue_work(card->ctrl[i].issue_wq, | |
762 | &card->ctrl[i].issue_dma_work); | |
763 | } | |
764 | } | |
765 | ||
766 | return 0; | |
767 | ||
768 | bvec_err: | |
e5feab22 PK |
769 | for (i = 0; i < card->n_targets; i++) |
770 | rsxx_cleanup_dma_queue(&card->ctrl[i], &dma_list[i], | |
771 | FREE_DMA); | |
8722ff8c | 772 | |
773 | return st; | |
774 | } | |
775 | ||
776 | ||
777 | /*----------------- DMA Engine Initialization & Setup -------------------*/ | |
c95246c3 PK |
778 | int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl) |
779 | { | |
780 | ctrl->status.buf = pci_alloc_consistent(dev, STATUS_BUFFER_SIZE8, | |
781 | &ctrl->status.dma_addr); | |
782 | ctrl->cmd.buf = pci_alloc_consistent(dev, COMMAND_BUFFER_SIZE8, | |
783 | &ctrl->cmd.dma_addr); | |
784 | if (ctrl->status.buf == NULL || ctrl->cmd.buf == NULL) | |
785 | return -ENOMEM; | |
786 | ||
787 | memset(ctrl->status.buf, 0xac, STATUS_BUFFER_SIZE8); | |
788 | iowrite32(lower_32_bits(ctrl->status.dma_addr), | |
789 | ctrl->regmap + SB_ADD_LO); | |
790 | iowrite32(upper_32_bits(ctrl->status.dma_addr), | |
791 | ctrl->regmap + SB_ADD_HI); | |
792 | ||
793 | memset(ctrl->cmd.buf, 0x83, COMMAND_BUFFER_SIZE8); | |
794 | iowrite32(lower_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_LO); | |
795 | iowrite32(upper_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_HI); | |
796 | ||
797 | ctrl->status.idx = ioread32(ctrl->regmap + HW_STATUS_CNT); | |
798 | if (ctrl->status.idx > RSXX_MAX_OUTSTANDING_CMDS) { | |
799 | dev_crit(&dev->dev, "Failed reading status cnt x%x\n", | |
800 | ctrl->status.idx); | |
801 | return -EINVAL; | |
802 | } | |
803 | iowrite32(ctrl->status.idx, ctrl->regmap + HW_STATUS_CNT); | |
804 | iowrite32(ctrl->status.idx, ctrl->regmap + SW_STATUS_CNT); | |
805 | ||
806 | ctrl->cmd.idx = ioread32(ctrl->regmap + HW_CMD_IDX); | |
807 | if (ctrl->cmd.idx > RSXX_MAX_OUTSTANDING_CMDS) { | |
808 | dev_crit(&dev->dev, "Failed reading cmd cnt x%x\n", | |
809 | ctrl->status.idx); | |
810 | return -EINVAL; | |
811 | } | |
812 | iowrite32(ctrl->cmd.idx, ctrl->regmap + HW_CMD_IDX); | |
813 | iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX); | |
814 | ||
815 | return 0; | |
816 | } | |
817 | ||
8722ff8c | 818 | static int rsxx_dma_ctrl_init(struct pci_dev *dev, |
819 | struct rsxx_dma_ctrl *ctrl) | |
820 | { | |
821 | int i; | |
c95246c3 | 822 | int st; |
8722ff8c | 823 | |
824 | memset(&ctrl->stats, 0, sizeof(ctrl->stats)); | |
825 | ||
8722ff8c | 826 | ctrl->trackers = vmalloc(DMA_TRACKER_LIST_SIZE8); |
827 | if (!ctrl->trackers) | |
828 | return -ENOMEM; | |
829 | ||
830 | ctrl->trackers->head = 0; | |
831 | for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) { | |
832 | ctrl->trackers->list[i].next_tag = i + 1; | |
833 | ctrl->trackers->list[i].dma = NULL; | |
834 | } | |
835 | ctrl->trackers->list[RSXX_MAX_OUTSTANDING_CMDS-1].next_tag = -1; | |
836 | spin_lock_init(&ctrl->trackers->lock); | |
837 | ||
838 | spin_lock_init(&ctrl->queue_lock); | |
31a70bb4 | 839 | mutex_init(&ctrl->work_lock); |
8722ff8c | 840 | INIT_LIST_HEAD(&ctrl->queue); |
841 | ||
842 | setup_timer(&ctrl->activity_timer, dma_engine_stalled, | |
c206c709 | 843 | (unsigned long)ctrl); |
8722ff8c | 844 | |
845 | ctrl->issue_wq = alloc_ordered_workqueue(DRIVER_NAME"_issue", 0); | |
846 | if (!ctrl->issue_wq) | |
847 | return -ENOMEM; | |
848 | ||
849 | ctrl->done_wq = alloc_ordered_workqueue(DRIVER_NAME"_done", 0); | |
850 | if (!ctrl->done_wq) | |
851 | return -ENOMEM; | |
852 | ||
31a70bb4 PK |
853 | INIT_WORK(&ctrl->issue_dma_work, rsxx_schedule_issue); |
854 | INIT_WORK(&ctrl->dma_done_work, rsxx_schedule_done); | |
8722ff8c | 855 | |
c95246c3 PK |
856 | st = rsxx_hw_buffers_init(dev, ctrl); |
857 | if (st) | |
858 | return st; | |
8722ff8c | 859 | |
8722ff8c | 860 | return 0; |
861 | } | |
862 | ||
c206c709 | 863 | static int rsxx_dma_stripe_setup(struct rsxx_cardinfo *card, |
8722ff8c | 864 | unsigned int stripe_size8) |
865 | { | |
866 | if (!is_power_of_2(stripe_size8)) { | |
867 | dev_err(CARD_TO_DEV(card), | |
868 | "stripe_size is NOT a power of 2!\n"); | |
869 | return -EINVAL; | |
870 | } | |
871 | ||
872 | card->_stripe.lower_mask = stripe_size8 - 1; | |
873 | ||
874 | card->_stripe.upper_mask = ~(card->_stripe.lower_mask); | |
875 | card->_stripe.upper_shift = ffs(card->n_targets) - 1; | |
876 | ||
877 | card->_stripe.target_mask = card->n_targets - 1; | |
878 | card->_stripe.target_shift = ffs(stripe_size8) - 1; | |
879 | ||
880 | dev_dbg(CARD_TO_DEV(card), "_stripe.lower_mask = x%016llx\n", | |
881 | card->_stripe.lower_mask); | |
882 | dev_dbg(CARD_TO_DEV(card), "_stripe.upper_shift = x%016llx\n", | |
883 | card->_stripe.upper_shift); | |
884 | dev_dbg(CARD_TO_DEV(card), "_stripe.upper_mask = x%016llx\n", | |
885 | card->_stripe.upper_mask); | |
886 | dev_dbg(CARD_TO_DEV(card), "_stripe.target_mask = x%016llx\n", | |
887 | card->_stripe.target_mask); | |
888 | dev_dbg(CARD_TO_DEV(card), "_stripe.target_shift = x%016llx\n", | |
889 | card->_stripe.target_shift); | |
890 | ||
891 | return 0; | |
892 | } | |
893 | ||
c95246c3 | 894 | int rsxx_dma_configure(struct rsxx_cardinfo *card) |
8722ff8c | 895 | { |
896 | u32 intr_coal; | |
897 | ||
898 | intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode, | |
899 | card->config.data.intr_coal.count, | |
900 | card->config.data.intr_coal.latency); | |
901 | iowrite32(intr_coal, card->regmap + INTR_COAL); | |
902 | ||
903 | return rsxx_dma_stripe_setup(card, card->config.data.stripe_size); | |
904 | } | |
905 | ||
906 | int rsxx_dma_setup(struct rsxx_cardinfo *card) | |
907 | { | |
908 | unsigned long flags; | |
909 | int st; | |
910 | int i; | |
911 | ||
912 | dev_info(CARD_TO_DEV(card), | |
913 | "Initializing %d DMA targets\n", | |
914 | card->n_targets); | |
915 | ||
916 | /* Regmap is divided up into 4K chunks. One for each DMA channel */ | |
917 | for (i = 0; i < card->n_targets; i++) | |
918 | card->ctrl[i].regmap = card->regmap + (i * 4096); | |
919 | ||
920 | card->dma_fault = 0; | |
921 | ||
922 | /* Reset the DMA queues */ | |
923 | rsxx_dma_queue_reset(card); | |
924 | ||
925 | /************* Setup DMA Control *************/ | |
926 | for (i = 0; i < card->n_targets; i++) { | |
927 | st = rsxx_dma_ctrl_init(card->dev, &card->ctrl[i]); | |
928 | if (st) | |
929 | goto failed_dma_setup; | |
930 | ||
931 | card->ctrl[i].card = card; | |
932 | card->ctrl[i].id = i; | |
933 | } | |
934 | ||
935 | card->scrub_hard = 1; | |
936 | ||
937 | if (card->config_valid) | |
938 | rsxx_dma_configure(card); | |
939 | ||
940 | /* Enable the interrupts after all setup has completed. */ | |
941 | for (i = 0; i < card->n_targets; i++) { | |
942 | spin_lock_irqsave(&card->irq_lock, flags); | |
943 | rsxx_enable_ier_and_isr(card, CR_INTR_DMA(i)); | |
944 | spin_unlock_irqrestore(&card->irq_lock, flags); | |
945 | } | |
946 | ||
947 | return 0; | |
948 | ||
949 | failed_dma_setup: | |
950 | for (i = 0; i < card->n_targets; i++) { | |
951 | struct rsxx_dma_ctrl *ctrl = &card->ctrl[i]; | |
952 | ||
953 | if (ctrl->issue_wq) { | |
954 | destroy_workqueue(ctrl->issue_wq); | |
955 | ctrl->issue_wq = NULL; | |
956 | } | |
957 | ||
958 | if (ctrl->done_wq) { | |
959 | destroy_workqueue(ctrl->done_wq); | |
960 | ctrl->done_wq = NULL; | |
961 | } | |
962 | ||
963 | if (ctrl->trackers) | |
964 | vfree(ctrl->trackers); | |
965 | ||
966 | if (ctrl->status.buf) | |
967 | pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8, | |
968 | ctrl->status.buf, | |
969 | ctrl->status.dma_addr); | |
970 | if (ctrl->cmd.buf) | |
971 | pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8, | |
972 | ctrl->cmd.buf, ctrl->cmd.dma_addr); | |
973 | } | |
974 | ||
975 | return st; | |
976 | } | |
977 | ||
0ab4743e PK |
978 | int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl) |
979 | { | |
980 | struct rsxx_dma *dma; | |
981 | int i; | |
982 | int cnt = 0; | |
983 | ||
984 | /* Clean up issued DMAs */ | |
985 | for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) { | |
986 | dma = get_tracker_dma(ctrl->trackers, i); | |
987 | if (dma) { | |
988 | atomic_dec(&ctrl->stats.hw_q_depth); | |
989 | rsxx_complete_dma(ctrl, dma, DMA_CANCELLED); | |
990 | push_tracker(ctrl->trackers, i); | |
991 | cnt++; | |
992 | } | |
993 | } | |
994 | ||
995 | return cnt; | |
996 | } | |
8722ff8c | 997 | |
998 | void rsxx_dma_destroy(struct rsxx_cardinfo *card) | |
999 | { | |
1000 | struct rsxx_dma_ctrl *ctrl; | |
0ab4743e | 1001 | int i; |
8722ff8c | 1002 | |
1003 | for (i = 0; i < card->n_targets; i++) { | |
1004 | ctrl = &card->ctrl[i]; | |
1005 | ||
1006 | if (ctrl->issue_wq) { | |
1007 | destroy_workqueue(ctrl->issue_wq); | |
1008 | ctrl->issue_wq = NULL; | |
1009 | } | |
1010 | ||
1011 | if (ctrl->done_wq) { | |
1012 | destroy_workqueue(ctrl->done_wq); | |
1013 | ctrl->done_wq = NULL; | |
1014 | } | |
1015 | ||
1016 | if (timer_pending(&ctrl->activity_timer)) | |
1017 | del_timer_sync(&ctrl->activity_timer); | |
1018 | ||
1019 | /* Clean up the DMA queue */ | |
0ab4743e | 1020 | spin_lock_bh(&ctrl->queue_lock); |
e5feab22 | 1021 | rsxx_cleanup_dma_queue(ctrl, &ctrl->queue, COMPLETE_DMA); |
0ab4743e | 1022 | spin_unlock_bh(&ctrl->queue_lock); |
8722ff8c | 1023 | |
0ab4743e | 1024 | rsxx_dma_cancel(ctrl); |
8722ff8c | 1025 | |
1026 | vfree(ctrl->trackers); | |
1027 | ||
1028 | pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8, | |
1029 | ctrl->status.buf, ctrl->status.dma_addr); | |
1030 | pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8, | |
1031 | ctrl->cmd.buf, ctrl->cmd.dma_addr); | |
1032 | } | |
1033 | } | |
1034 | ||
4dcaf472 | 1035 | int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card) |
c95246c3 PK |
1036 | { |
1037 | int i; | |
1038 | int j; | |
1039 | int cnt; | |
1040 | struct rsxx_dma *dma; | |
d8d595df PK |
1041 | struct list_head *issued_dmas; |
1042 | ||
1043 | issued_dmas = kzalloc(sizeof(*issued_dmas) * card->n_targets, | |
1044 | GFP_KERNEL); | |
4dcaf472 PK |
1045 | if (!issued_dmas) |
1046 | return -ENOMEM; | |
c95246c3 PK |
1047 | |
1048 | for (i = 0; i < card->n_targets; i++) { | |
1049 | INIT_LIST_HEAD(&issued_dmas[i]); | |
1050 | cnt = 0; | |
1051 | for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) { | |
1052 | dma = get_tracker_dma(card->ctrl[i].trackers, j); | |
1053 | if (dma == NULL) | |
1054 | continue; | |
1055 | ||
1056 | if (dma->cmd == HW_CMD_BLK_WRITE) | |
1057 | card->ctrl[i].stats.writes_issued--; | |
1058 | else if (dma->cmd == HW_CMD_BLK_DISCARD) | |
1059 | card->ctrl[i].stats.discards_issued--; | |
1060 | else | |
1061 | card->ctrl[i].stats.reads_issued--; | |
1062 | ||
e35f38bf PK |
1063 | if (dma->cmd != HW_CMD_BLK_DISCARD) { |
1064 | pci_unmap_page(card->dev, dma->dma_addr, | |
1065 | get_dma_size(dma), | |
1066 | dma->cmd == HW_CMD_BLK_WRITE ? | |
1067 | PCI_DMA_TODEVICE : | |
1068 | PCI_DMA_FROMDEVICE); | |
1069 | } | |
1070 | ||
c95246c3 PK |
1071 | list_add_tail(&dma->list, &issued_dmas[i]); |
1072 | push_tracker(card->ctrl[i].trackers, j); | |
1073 | cnt++; | |
1074 | } | |
1075 | ||
0ab4743e | 1076 | spin_lock_bh(&card->ctrl[i].queue_lock); |
c95246c3 PK |
1077 | list_splice(&issued_dmas[i], &card->ctrl[i].queue); |
1078 | ||
1079 | atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth); | |
1080 | card->ctrl[i].stats.sw_q_depth += cnt; | |
1081 | card->ctrl[i].e_cnt = 0; | |
0ab4743e | 1082 | spin_unlock_bh(&card->ctrl[i].queue_lock); |
c95246c3 | 1083 | } |
d8d595df PK |
1084 | |
1085 | kfree(issued_dmas); | |
4dcaf472 PK |
1086 | |
1087 | return 0; | |
c95246c3 PK |
1088 | } |
1089 | ||
8722ff8c | 1090 | int rsxx_dma_init(void) |
1091 | { | |
1092 | rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN); | |
1093 | if (!rsxx_dma_pool) | |
1094 | return -ENOMEM; | |
1095 | ||
1096 | return 0; | |
1097 | } | |
1098 | ||
1099 | ||
1100 | void rsxx_dma_cleanup(void) | |
1101 | { | |
1102 | kmem_cache_destroy(rsxx_dma_pool); | |
1103 | } | |
1104 |