projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'for-4.9/block' into for-next
[deliverable/linux.git] / drivers / net / ethernet / cavium / liquidio / octeon_droq.c
1 /**********************************************************************
2 * Author: Cavium, Inc.
3 *
4 * Contact: support@cavium.com
5 * Please include "LiquidIO" in the subject.
6 *
7 * Copyright (c) 2003-2015 Cavium, Inc.
8 *
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
12 *
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more
17 * details.
18 *
19 * This file may also be available under a different license from Cavium.
20 * Contact Cavium, Inc. for more information
21 **********************************************************************/
22 #include <linux/pci.h>
23 #include <linux/netdevice.h>
24 #include <linux/vmalloc.h>
25 #include "liquidio_common.h"
26 #include "octeon_droq.h"
27 #include "octeon_iq.h"
28 #include "response_manager.h"
29 #include "octeon_device.h"
30 #include "octeon_main.h"
31 #include "octeon_network.h"
32 #include "cn66xx_regs.h"
33 #include "cn66xx_device.h"
34
35 #define CVM_MIN(d1, d2) (((d1) < (d2)) ? (d1) : (d2))
36 #define CVM_MAX(d1, d2) (((d1) > (d2)) ? (d1) : (d2))
37
38 struct niclist {
39 struct list_head list;
40 void *ptr;
41 };
42
43 struct __dispatch {
44 struct list_head list;
45 struct octeon_recv_info *rinfo;
46 octeon_dispatch_fn_t disp_fn;
47 };
48
49 /** Get the argument that the user set when registering dispatch
50 * function for a given opcode/subcode.
51 * @param octeon_dev - the octeon device pointer.
52 * @param opcode - the opcode for which the dispatch argument
53 * is to be checked.
54 * @param subcode - the subcode for which the dispatch argument
55 * is to be checked.
56 * @return Success: void * (argument to the dispatch function)
57 * @return Failure: NULL
58 *
59 */
60 static inline void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev,
61 u16 opcode, u16 subcode)
62 {
63 int idx;
64 struct list_head *dispatch;
65 void *fn_arg = NULL;
66 u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode);
67
68 idx = combined_opcode & OCTEON_OPCODE_MASK;
69
70 spin_lock_bh(&octeon_dev->dispatch.lock);
71
72 if (octeon_dev->dispatch.count == 0) {
73 spin_unlock_bh(&octeon_dev->dispatch.lock);
74 return NULL;
75 }
76
77 if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) {
78 fn_arg = octeon_dev->dispatch.dlist[idx].arg;
79 } else {
80 list_for_each(dispatch,
81 &octeon_dev->dispatch.dlist[idx].list) {
82 if (((struct octeon_dispatch *)dispatch)->opcode ==
83 combined_opcode) {
84 fn_arg = ((struct octeon_dispatch *)
85 dispatch)->arg;
86 break;
87 }
88 }
89 }
90
91 spin_unlock_bh(&octeon_dev->dispatch.lock);
92 return fn_arg;
93 }
94
95 /** Check for packets on Droq. This function should be called with
96 * lock held.
97 * @param droq - Droq on which count is checked.
98 * @return Returns packet count.
99 */
100 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq)
101 {
102 u32 pkt_count = 0;
103
104 pkt_count = readl(droq->pkts_sent_reg);
105 if (pkt_count) {
106 atomic_add(pkt_count, &droq->pkts_pending);
107 writel(pkt_count, droq->pkts_sent_reg);
108 }
109
110 return pkt_count;
111 }
112
113 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq)
114 {
115 u32 count = 0;
116
117 /* max_empty_descs is the max. no. of descs that can have no buffers.
118 * If the empty desc count goes beyond this value, we cannot safely
119 * read in a 64K packet sent by Octeon
120 * (64K is max pkt size from Octeon)
121 */
122 droq->max_empty_descs = 0;
123
124 do {
125 droq->max_empty_descs++;
126 count += droq->buffer_size;
127 } while (count < (64 * 1024));
128
129 droq->max_empty_descs = droq->max_count - droq->max_empty_descs;
130 }
131
132 static void octeon_droq_reset_indices(struct octeon_droq *droq)
133 {
134 droq->read_idx = 0;
135 droq->write_idx = 0;
136 droq->refill_idx = 0;
137 droq->refill_count = 0;
138 atomic_set(&droq->pkts_pending, 0);
139 }
140
141 static void
142 octeon_droq_destroy_ring_buffers(struct octeon_device *oct,
143 struct octeon_droq *droq)
144 {
145 u32 i;
146 struct octeon_skb_page_info *pg_info;
147
148 for (i = 0; i < droq->max_count; i++) {
149 pg_info = &droq->recv_buf_list[i].pg_info;
150
151 if (pg_info->dma)
152 lio_unmap_ring(oct->pci_dev,
153 (u64)pg_info->dma);
154 pg_info->dma = 0;
155
156 if (pg_info->page)
157 recv_buffer_destroy(droq->recv_buf_list[i].buffer,
158 pg_info);
159
160 if (droq->desc_ring && droq->desc_ring[i].info_ptr)
161 lio_unmap_ring_info(oct->pci_dev,
162 (u64)droq->
163 desc_ring[i].info_ptr,
164 OCT_DROQ_INFO_SIZE);
165 droq->recv_buf_list[i].buffer = NULL;
166 }
167
168 octeon_droq_reset_indices(droq);
169 }
170
171 static int
172 octeon_droq_setup_ring_buffers(struct octeon_device *oct,
173 struct octeon_droq *droq)
174 {
175 u32 i;
176 void *buf;
177 struct octeon_droq_desc *desc_ring = droq->desc_ring;
178
179 for (i = 0; i < droq->max_count; i++) {
180 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info);
181
182 if (!buf) {
183 dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n",
184 __func__);
185 droq->stats.rx_alloc_failure++;
186 return -ENOMEM;
187 }
188
189 droq->recv_buf_list[i].buffer = buf;
190 droq->recv_buf_list[i].data = get_rbd(buf);
191 droq->info_list[i].length = 0;
192
193 /* map ring buffers into memory */
194 desc_ring[i].info_ptr = lio_map_ring_info(droq, i);
195 desc_ring[i].buffer_ptr =
196 lio_map_ring(droq->recv_buf_list[i].buffer);
197 }
198
199 octeon_droq_reset_indices(droq);
200
201 octeon_droq_compute_max_packet_bufs(droq);
202
203 return 0;
204 }
205
206 int octeon_delete_droq(struct octeon_device *oct, u32 q_no)
207 {
208 struct octeon_droq *droq = oct->droq[q_no];
209
210 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
211
212 octeon_droq_destroy_ring_buffers(oct, droq);
213 vfree(droq->recv_buf_list);
214
215 if (droq->info_base_addr)
216 cnnic_free_aligned_dma(oct->pci_dev, droq->info_list,
217 droq->info_alloc_size,
218 droq->info_base_addr,
219 droq->info_list_dma);
220
221 if (droq->desc_ring)
222 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
223 droq->desc_ring, droq->desc_ring_dma);
224
225 memset(droq, 0, OCT_DROQ_SIZE);
226
227 return 0;
228 }
229
230 int octeon_init_droq(struct octeon_device *oct,
231 u32 q_no,
232 u32 num_descs,
233 u32 desc_size,
234 void *app_ctx)
235 {
236 struct octeon_droq *droq;
237 u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0;
238 u32 c_pkts_per_intr = 0, c_refill_threshold = 0;
239 int orig_node = dev_to_node(&oct->pci_dev->dev);
240 int numa_node = cpu_to_node(q_no % num_online_cpus());
241
242 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
243
244 droq = oct->droq[q_no];
245 memset(droq, 0, OCT_DROQ_SIZE);
246
247 droq->oct_dev = oct;
248 droq->q_no = q_no;
249 if (app_ctx)
250 droq->app_ctx = app_ctx;
251 else
252 droq->app_ctx = (void *)(size_t)q_no;
253
254 c_num_descs = num_descs;
255 c_buf_size = desc_size;
256 if (OCTEON_CN6XXX(oct)) {
257 struct octeon_config *conf6x = CHIP_FIELD(oct, cn6xxx, conf);
258
259 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x);
260 c_refill_threshold =
261 (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x);
262 } else {
263 return 1;
264 }
265
266 droq->max_count = c_num_descs;
267 droq->buffer_size = c_buf_size;
268
269 desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE;
270 set_dev_node(&oct->pci_dev->dev, numa_node);
271 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
272 (dma_addr_t *)&droq->desc_ring_dma);
273 set_dev_node(&oct->pci_dev->dev, orig_node);
274 if (!droq->desc_ring)
275 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
276 (dma_addr_t *)&droq->desc_ring_dma);
277
278 if (!droq->desc_ring) {
279 dev_err(&oct->pci_dev->dev,
280 "Output queue %d ring alloc failed\n", q_no);
281 return 1;
282 }
283
284 dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n",
285 q_no, droq->desc_ring, droq->desc_ring_dma);
286 dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no,
287 droq->max_count);
288
289 droq->info_list =
290 cnnic_numa_alloc_aligned_dma((droq->max_count *
291 OCT_DROQ_INFO_SIZE),
292 &droq->info_alloc_size,
293 &droq->info_base_addr,
294 numa_node);
295 if (!droq->info_list) {
296 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for info list.\n");
297 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
298 droq->desc_ring, droq->desc_ring_dma);
299 return 1;
300 }
301
302 droq->recv_buf_list = (struct octeon_recv_buffer *)
303 vmalloc_node(droq->max_count *
304 OCT_DROQ_RECVBUF_SIZE,
305 numa_node);
306 if (!droq->recv_buf_list)
307 droq->recv_buf_list = (struct octeon_recv_buffer *)
308 vmalloc(droq->max_count *
309 OCT_DROQ_RECVBUF_SIZE);
310 if (!droq->recv_buf_list) {
311 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n");
312 goto init_droq_fail;
313 }
314
315 if (octeon_droq_setup_ring_buffers(oct, droq))
316 goto init_droq_fail;
317
318 droq->pkts_per_intr = c_pkts_per_intr;
319 droq->refill_threshold = c_refill_threshold;
320
321 dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n",
322 droq->max_empty_descs);
323
324 spin_lock_init(&droq->lock);
325
326 INIT_LIST_HEAD(&droq->dispatch_list);
327
328 /* For 56xx Pass1, this function won't be called, so no checks. */
329 oct->fn_list.setup_oq_regs(oct, q_no);
330
331 oct->io_qmask.oq |= (1ULL << q_no);
332
333 return 0;
334
335 init_droq_fail:
336 octeon_delete_droq(oct, q_no);
337 return 1;
338 }
339
340 /* octeon_create_recv_info
341 * Parameters:
342 * octeon_dev - pointer to the octeon device structure
343 * droq - droq in which the packet arrived.
344 * buf_cnt - no. of buffers used by the packet.
345 * idx - index in the descriptor for the first buffer in the packet.
346 * Description:
347 * Allocates a recv_info_t and copies the buffer addresses for packet data
348 * into the recv_pkt space which starts at an 8B offset from recv_info_t.
349 * Flags the descriptors for refill later. If available descriptors go
350 * below the threshold to receive a 64K pkt, new buffers are first allocated
351 * before the recv_pkt_t is created.
352 * This routine will be called in interrupt context.
353 * Returns:
354 * Success: Pointer to recv_info_t
355 * Failure: NULL.
356 * Locks:
357 * The droq->lock is held when this routine is called.
358 */
359 static inline struct octeon_recv_info *octeon_create_recv_info(
360 struct octeon_device *octeon_dev,
361 struct octeon_droq *droq,
362 u32 buf_cnt,
363 u32 idx)
364 {
365 struct octeon_droq_info *info;
366 struct octeon_recv_pkt *recv_pkt;
367 struct octeon_recv_info *recv_info;
368 u32 i, bytes_left;
369 struct octeon_skb_page_info *pg_info;
370
371 info = &droq->info_list[idx];
372
373 recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch));
374 if (!recv_info)
375 return NULL;
376
377 recv_pkt = recv_info->recv_pkt;
378 recv_pkt->rh = info->rh;
379 recv_pkt->length = (u32)info->length;
380 recv_pkt->buffer_count = (u16)buf_cnt;
381 recv_pkt->octeon_id = (u16)octeon_dev->octeon_id;
382
383 i = 0;
384 bytes_left = (u32)info->length;
385
386 while (buf_cnt) {
387 {
388 pg_info = &droq->recv_buf_list[idx].pg_info;
389
390 lio_unmap_ring(octeon_dev->pci_dev,
391 (u64)pg_info->dma);
392 pg_info->page = NULL;
393 pg_info->dma = 0;
394 }
395
396 recv_pkt->buffer_size[i] =
397 (bytes_left >=
398 droq->buffer_size) ? droq->buffer_size : bytes_left;
399
400 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer;
401 droq->recv_buf_list[idx].buffer = NULL;
402
403 INCR_INDEX_BY1(idx, droq->max_count);
404 bytes_left -= droq->buffer_size;
405 i++;
406 buf_cnt--;
407 }
408
409 return recv_info;
410 }
411
412 /* If we were not able to refill all buffers, try to move around
413 * the buffers that were not dispatched.
414 */
415 static inline u32
416 octeon_droq_refill_pullup_descs(struct octeon_droq *droq,
417 struct octeon_droq_desc *desc_ring)
418 {
419 u32 desc_refilled = 0;
420
421 u32 refill_index = droq->refill_idx;
422
423 while (refill_index != droq->read_idx) {
424 if (droq->recv_buf_list[refill_index].buffer) {
425 droq->recv_buf_list[droq->refill_idx].buffer =
426 droq->recv_buf_list[refill_index].buffer;
427 droq->recv_buf_list[droq->refill_idx].data =
428 droq->recv_buf_list[refill_index].data;
429 desc_ring[droq->refill_idx].buffer_ptr =
430 desc_ring[refill_index].buffer_ptr;
431 droq->recv_buf_list[refill_index].buffer = NULL;
432 desc_ring[refill_index].buffer_ptr = 0;
433 do {
434 INCR_INDEX_BY1(droq->refill_idx,
435 droq->max_count);
436 desc_refilled++;
437 droq->refill_count--;
438 } while (droq->recv_buf_list[droq->refill_idx].
439 buffer);
440 }
441 INCR_INDEX_BY1(refill_index, droq->max_count);
442 } /* while */
443 return desc_refilled;
444 }
445
446 /* octeon_droq_refill
447 * Parameters:
448 * droq - droq in which descriptors require new buffers.
449 * Description:
450 * Called during normal DROQ processing in interrupt mode or by the poll
451 * thread to refill the descriptors from which buffers were dispatched
452 * to upper layers. Attempts to allocate new buffers. If that fails, moves
453 * up buffers (that were not dispatched) to form a contiguous ring.
454 * Returns:
455 * No of descriptors refilled.
456 * Locks:
457 * This routine is called with droq->lock held.
458 */
459 static u32
460 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq)
461 {
462 struct octeon_droq_desc *desc_ring;
463 void *buf = NULL;
464 u8 *data;
465 u32 desc_refilled = 0;
466 struct octeon_skb_page_info *pg_info;
467
468 desc_ring = droq->desc_ring;
469
470 while (droq->refill_count && (desc_refilled < droq->max_count)) {
471 /* If a valid buffer exists (happens if there is no dispatch),
472 * reuse
473 * the buffer, else allocate.
474 */
475 if (!droq->recv_buf_list[droq->refill_idx].buffer) {
476 pg_info =
477 &droq->recv_buf_list[droq->refill_idx].pg_info;
478 /* Either recycle the existing pages or go for
479 * new page alloc
480 */
481 if (pg_info->page)
482 buf = recv_buffer_reuse(octeon_dev, pg_info);
483 else
484 buf = recv_buffer_alloc(octeon_dev, pg_info);
485 /* If a buffer could not be allocated, no point in
486 * continuing
487 */
488 if (!buf) {
489 droq->stats.rx_alloc_failure++;
490 break;
491 }
492 droq->recv_buf_list[droq->refill_idx].buffer =
493 buf;
494 data = get_rbd(buf);
495 } else {
496 data = get_rbd(droq->recv_buf_list
497 [droq->refill_idx].buffer);
498 }
499
500 droq->recv_buf_list[droq->refill_idx].data = data;
501
502 desc_ring[droq->refill_idx].buffer_ptr =
503 lio_map_ring(droq->recv_buf_list[droq->
504 refill_idx].buffer);
505 /* Reset any previous values in the length field. */
506 droq->info_list[droq->refill_idx].length = 0;
507
508 INCR_INDEX_BY1(droq->refill_idx, droq->max_count);
509 desc_refilled++;
510 droq->refill_count--;
511 }
512
513 if (droq->refill_count)
514 desc_refilled +=
515 octeon_droq_refill_pullup_descs(droq, desc_ring);
516
517 /* if droq->refill_count
518 * The refill count would not change in pass two. We only moved buffers
519 * to close the gap in the ring, but we would still have the same no. of
520 * buffers to refill.
521 */
522 return desc_refilled;
523 }
524
525 static inline u32
526 octeon_droq_get_bufcount(u32 buf_size, u32 total_len)
527 {
528 u32 buf_cnt = 0;
529
530 while (total_len > (buf_size * buf_cnt))
531 buf_cnt++;
532 return buf_cnt;
533 }
534
535 static int
536 octeon_droq_dispatch_pkt(struct octeon_device *oct,
537 struct octeon_droq *droq,
538 union octeon_rh *rh,
539 struct octeon_droq_info *info)
540 {
541 u32 cnt;
542 octeon_dispatch_fn_t disp_fn;
543 struct octeon_recv_info *rinfo;
544
545 cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length);
546
547 disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode,
548 (u16)rh->r.subcode);
549 if (disp_fn) {
550 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx);
551 if (rinfo) {
552 struct __dispatch *rdisp = rinfo->rsvd;
553
554 rdisp->rinfo = rinfo;
555 rdisp->disp_fn = disp_fn;
556 rinfo->recv_pkt->rh = *rh;
557 list_add_tail(&rdisp->list,
558 &droq->dispatch_list);
559 } else {
560 droq->stats.dropped_nomem++;
561 }
562 } else {
563 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n",
564 (unsigned int)rh->r.opcode,
565 (unsigned int)rh->r.subcode);
566 droq->stats.dropped_nodispatch++;
567 } /* else (dispatch_fn ... */
568
569 return cnt;
570 }
571
572 static inline void octeon_droq_drop_packets(struct octeon_device *oct,
573 struct octeon_droq *droq,
574 u32 cnt)
575 {
576 u32 i = 0, buf_cnt;
577 struct octeon_droq_info *info;
578
579 for (i = 0; i < cnt; i++) {
580 info = &droq->info_list[droq->read_idx];
581 octeon_swap_8B_data((u64 *)info, 2);
582
583 if (info->length) {
584 info->length -= OCT_RH_SIZE;
585 droq->stats.bytes_received += info->length;
586 buf_cnt = octeon_droq_get_bufcount(droq->buffer_size,
587 (u32)info->length);
588 } else {
589 dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n");
590 buf_cnt = 1;
591 }
592
593 INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count);
594 droq->refill_count += buf_cnt;
595 }
596 }
597
598 static u32
599 octeon_droq_fast_process_packets(struct octeon_device *oct,
600 struct octeon_droq *droq,
601 u32 pkts_to_process)
602 {
603 struct octeon_droq_info *info;
604 union octeon_rh *rh;
605 u32 pkt, total_len = 0, pkt_count;
606
607 pkt_count = pkts_to_process;
608
609 for (pkt = 0; pkt < pkt_count; pkt++) {
610 u32 pkt_len = 0;
611 struct sk_buff *nicbuf = NULL;
612 struct octeon_skb_page_info *pg_info;
613 void *buf;
614
615 info = &droq->info_list[droq->read_idx];
616 octeon_swap_8B_data((u64 *)info, 2);
617
618 if (!info->length) {
619 dev_err(&oct->pci_dev->dev,
620 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n",
621 droq->q_no, droq->read_idx, pkt_count);
622 print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS,
623 (u8 *)info,
624 OCT_DROQ_INFO_SIZE);
625 break;
626 }
627
628 /* Len of resp hdr in included in the received data len. */
629 info->length -= OCT_RH_SIZE;
630 rh = &info->rh;
631
632 total_len += (u32)info->length;
633 if (OPCODE_SLOW_PATH(rh)) {
634 u32 buf_cnt;
635
636 buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info);
637 INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count);
638 droq->refill_count += buf_cnt;
639 } else {
640 if (info->length <= droq->buffer_size) {
641 pkt_len = (u32)info->length;
642 nicbuf = droq->recv_buf_list[
643 droq->read_idx].buffer;
644 pg_info = &droq->recv_buf_list[
645 droq->read_idx].pg_info;
646 if (recv_buffer_recycle(oct, pg_info))
647 pg_info->page = NULL;
648 droq->recv_buf_list[droq->read_idx].buffer =
649 NULL;
650
651 INCR_INDEX_BY1(droq->read_idx, droq->max_count);
652 droq->refill_count++;
653 } else {
654 nicbuf = octeon_fast_packet_alloc((u32)
655 info->length);
656 pkt_len = 0;
657 /* nicbuf allocation can fail. We'll handle it
658 * inside the loop.
659 */
660 while (pkt_len < info->length) {
661 int cpy_len, idx = droq->read_idx;
662
663 cpy_len = ((pkt_len + droq->buffer_size)
664 > info->length) ?
665 ((u32)info->length - pkt_len) :
666 droq->buffer_size;
667
668 if (nicbuf) {
669 octeon_fast_packet_next(droq,
670 nicbuf,
671 cpy_len,
672 idx);
673 buf = droq->recv_buf_list[idx].
674 buffer;
675 recv_buffer_fast_free(buf);
676 droq->recv_buf_list[idx].buffer
677 = NULL;
678 } else {
679 droq->stats.rx_alloc_failure++;
680 }
681
682 pkt_len += cpy_len;
683 INCR_INDEX_BY1(droq->read_idx,
684 droq->max_count);
685 droq->refill_count++;
686 }
687 }
688
689 if (nicbuf) {
690 if (droq->ops.fptr) {
691 droq->ops.fptr(oct->octeon_id,
692 nicbuf, pkt_len,
693 rh, &droq->napi,
694 droq->ops.farg);
695 } else {
696 recv_buffer_free(nicbuf);
697 }
698 }
699 }
700
701 if (droq->refill_count >= droq->refill_threshold) {
702 int desc_refilled = octeon_droq_refill(oct, droq);
703
704 /* Flush the droq descriptor data to memory to be sure
705 * that when we update the credits the data in memory
706 * is accurate.
707 */
708 wmb();
709 writel((desc_refilled), droq->pkts_credit_reg);
710 /* make sure mmio write completes */
711 mmiowb();
712 }
713
714 } /* for (each packet)... */
715
716 /* Increment refill_count by the number of buffers processed. */
717 droq->stats.pkts_received += pkt;
718 droq->stats.bytes_received += total_len;
719
720 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) {
721 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt));
722
723 droq->stats.dropped_toomany += (pkts_to_process - pkt);
724 return pkts_to_process;
725 }
726
727 return pkt;
728 }
729
730 int
731 octeon_droq_process_packets(struct octeon_device *oct,
732 struct octeon_droq *droq,
733 u32 budget)
734 {
735 u32 pkt_count = 0, pkts_processed = 0;
736 struct list_head *tmp, *tmp2;
737
738 pkt_count = atomic_read(&droq->pkts_pending);
739 if (!pkt_count)
740 return 0;
741
742 if (pkt_count > budget)
743 pkt_count = budget;
744
745 /* Grab the droq lock */
746 spin_lock(&droq->lock);
747
748 pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count);
749
750 atomic_sub(pkts_processed, &droq->pkts_pending);
751
752 /* Release the spin lock */
753 spin_unlock(&droq->lock);
754
755 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
756 struct __dispatch *rdisp = (struct __dispatch *)tmp;
757
758 list_del(tmp);
759 rdisp->disp_fn(rdisp->rinfo,
760 octeon_get_dispatch_arg
761 (oct,
762 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
763 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
764 }
765
766 /* If there are packets pending. schedule tasklet again */
767 if (atomic_read(&droq->pkts_pending))
768 return 1;
769
770 return 0;
771 }
772
773 /**
774 * Utility function to poll for packets. check_hw_for_packets must be
775 * called before calling this routine.
776 */
777
778 static int
779 octeon_droq_process_poll_pkts(struct octeon_device *oct,
780 struct octeon_droq *droq, u32 budget)
781 {
782 struct list_head *tmp, *tmp2;
783 u32 pkts_available = 0, pkts_processed = 0;
784 u32 total_pkts_processed = 0;
785
786 if (budget > droq->max_count)
787 budget = droq->max_count;
788
789 spin_lock(&droq->lock);
790
791 while (total_pkts_processed < budget) {
792 pkts_available =
793 CVM_MIN((budget - total_pkts_processed),
794 (u32)(atomic_read(&droq->pkts_pending)));
795
796 if (pkts_available == 0)
797 break;
798
799 pkts_processed =
800 octeon_droq_fast_process_packets(oct, droq,
801 pkts_available);
802
803 atomic_sub(pkts_processed, &droq->pkts_pending);
804
805 total_pkts_processed += pkts_processed;
806
807 octeon_droq_check_hw_for_pkts(droq);
808 }
809
810 spin_unlock(&droq->lock);
811
812 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
813 struct __dispatch *rdisp = (struct __dispatch *)tmp;
814
815 list_del(tmp);
816 rdisp->disp_fn(rdisp->rinfo,
817 octeon_get_dispatch_arg
818 (oct,
819 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
820 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
821 }
822
823 return total_pkts_processed;
824 }
825
826 int
827 octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, int cmd,
828 u32 arg)
829 {
830 struct octeon_droq *droq;
831
832 droq = oct->droq[q_no];
833
834 if (cmd == POLL_EVENT_PROCESS_PKTS)
835 return octeon_droq_process_poll_pkts(oct, droq, arg);
836
837 if (cmd == POLL_EVENT_PENDING_PKTS) {
838 u32 pkt_cnt = atomic_read(&droq->pkts_pending);
839
840 return octeon_droq_process_packets(oct, droq, pkt_cnt);
841 }
842
843 if (cmd == POLL_EVENT_ENABLE_INTR) {
844 u32 value;
845 unsigned long flags;
846
847 /* Enable Pkt Interrupt */
848 switch (oct->chip_id) {
849 case OCTEON_CN66XX:
850 case OCTEON_CN68XX: {
851 struct octeon_cn6xxx *cn6xxx =
852 (struct octeon_cn6xxx *)oct->chip;
853 spin_lock_irqsave
854 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
855 value =
856 octeon_read_csr(oct,
857 CN6XXX_SLI_PKT_TIME_INT_ENB);
858 value |= (1 << q_no);
859 octeon_write_csr(oct,
860 CN6XXX_SLI_PKT_TIME_INT_ENB,
861 value);
862 value =
863 octeon_read_csr(oct,
864 CN6XXX_SLI_PKT_CNT_INT_ENB);
865 value |= (1 << q_no);
866 octeon_write_csr(oct,
867 CN6XXX_SLI_PKT_CNT_INT_ENB,
868 value);
869
870 /* don't bother flushing the enables */
871
872 spin_unlock_irqrestore
873 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
874 return 0;
875 }
876 break;
877 }
878
879 return 0;
880 }
881
882 dev_err(&oct->pci_dev->dev, "%s Unknown command: %d\n", __func__, cmd);
883 return -EINVAL;
884 }
885
886 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no,
887 struct octeon_droq_ops *ops)
888 {
889 struct octeon_droq *droq;
890 unsigned long flags;
891 struct octeon_config *oct_cfg = NULL;
892
893 oct_cfg = octeon_get_conf(oct);
894
895 if (!oct_cfg)
896 return -EINVAL;
897
898 if (!(ops)) {
899 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n",
900 __func__);
901 return -EINVAL;
902 }
903
904 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
905 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
906 __func__, q_no, (oct->num_oqs - 1));
907 return -EINVAL;
908 }
909
910 droq = oct->droq[q_no];
911
912 spin_lock_irqsave(&droq->lock, flags);
913
914 memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops));
915
916 spin_unlock_irqrestore(&droq->lock, flags);
917
918 return 0;
919 }
920
921 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no)
922 {
923 unsigned long flags;
924 struct octeon_droq *droq;
925 struct octeon_config *oct_cfg = NULL;
926
927 oct_cfg = octeon_get_conf(oct);
928
929 if (!oct_cfg)
930 return -EINVAL;
931
932 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
933 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
934 __func__, q_no, oct->num_oqs - 1);
935 return -EINVAL;
936 }
937
938 droq = oct->droq[q_no];
939
940 if (!droq) {
941 dev_info(&oct->pci_dev->dev,
942 "Droq id (%d) not available.\n", q_no);
943 return 0;
944 }
945
946 spin_lock_irqsave(&droq->lock, flags);
947
948 droq->ops.fptr = NULL;
949 droq->ops.farg = NULL;
950 droq->ops.drop_on_max = 0;
951
952 spin_unlock_irqrestore(&droq->lock, flags);
953
954 return 0;
955 }
956
957 int octeon_create_droq(struct octeon_device *oct,
958 u32 q_no, u32 num_descs,
959 u32 desc_size, void *app_ctx)
960 {
961 struct octeon_droq *droq;
962 int numa_node = cpu_to_node(q_no % num_online_cpus());
963
964 if (oct->droq[q_no]) {
965 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n",
966 q_no);
967 return 1;
968 }
969
970 /* Allocate the DS for the new droq. */
971 droq = vmalloc_node(sizeof(*droq), numa_node);
972 if (!droq)
973 droq = vmalloc(sizeof(*droq));
974 if (!droq)
975 goto create_droq_fail;
976 memset(droq, 0, sizeof(struct octeon_droq));
977
978 /*Disable the pkt o/p for this Q */
979 octeon_set_droq_pkt_op(oct, q_no, 0);
980 oct->droq[q_no] = droq;
981
982 /* Initialize the Droq */
983 octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx);
984
985 oct->num_oqs++;
986
987 dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__,
988 oct->num_oqs);
989
990 /* Global Droq register settings */
991
992 /* As of now not required, as setting are done for all 32 Droqs at
993 * the same time.
994 */
995 return 0;
996
997 create_droq_fail:
998 octeon_delete_droq(oct, q_no);
999 return -ENOMEM;
1000 }
Linux kernel - Deliverables
This page took 0.055498 seconds and 5 git commands to generate.