projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge remote-tracking branch 'mmc-uh/next'
[deliverable/linux.git] / drivers / net / ethernet / cavium / liquidio / octeon_droq.c
1 /**********************************************************************
2 * Author: Cavium, Inc.
3 *
4 * Contact: support@cavium.com
5 * Please include "LiquidIO" in the subject.
6 *
7 * Copyright (c) 2003-2015 Cavium, Inc.
8 *
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
12 *
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more
17 * details.
18 *
19 * This file may also be available under a different license from Cavium.
20 * Contact Cavium, Inc. for more information
21 **********************************************************************/
22 #include <linux/pci.h>
23 #include <linux/netdevice.h>
24 #include <linux/vmalloc.h>
25 #include "liquidio_common.h"
26 #include "octeon_droq.h"
27 #include "octeon_iq.h"
28 #include "response_manager.h"
29 #include "octeon_device.h"
30 #include "octeon_main.h"
31 #include "octeon_network.h"
32 #include "cn66xx_regs.h"
33 #include "cn66xx_device.h"
34 #include "cn23xx_pf_device.h"
35
36 #define CVM_MIN(d1, d2) (((d1) < (d2)) ? (d1) : (d2))
37 #define CVM_MAX(d1, d2) (((d1) > (d2)) ? (d1) : (d2))
38
39 struct niclist {
40 struct list_head list;
41 void *ptr;
42 };
43
44 struct __dispatch {
45 struct list_head list;
46 struct octeon_recv_info *rinfo;
47 octeon_dispatch_fn_t disp_fn;
48 };
49
50 /** Get the argument that the user set when registering dispatch
51 * function for a given opcode/subcode.
52 * @param octeon_dev - the octeon device pointer.
53 * @param opcode - the opcode for which the dispatch argument
54 * is to be checked.
55 * @param subcode - the subcode for which the dispatch argument
56 * is to be checked.
57 * @return Success: void * (argument to the dispatch function)
58 * @return Failure: NULL
59 *
60 */
61 static inline void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev,
62 u16 opcode, u16 subcode)
63 {
64 int idx;
65 struct list_head *dispatch;
66 void *fn_arg = NULL;
67 u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode);
68
69 idx = combined_opcode & OCTEON_OPCODE_MASK;
70
71 spin_lock_bh(&octeon_dev->dispatch.lock);
72
73 if (octeon_dev->dispatch.count == 0) {
74 spin_unlock_bh(&octeon_dev->dispatch.lock);
75 return NULL;
76 }
77
78 if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) {
79 fn_arg = octeon_dev->dispatch.dlist[idx].arg;
80 } else {
81 list_for_each(dispatch,
82 &octeon_dev->dispatch.dlist[idx].list) {
83 if (((struct octeon_dispatch *)dispatch)->opcode ==
84 combined_opcode) {
85 fn_arg = ((struct octeon_dispatch *)
86 dispatch)->arg;
87 break;
88 }
89 }
90 }
91
92 spin_unlock_bh(&octeon_dev->dispatch.lock);
93 return fn_arg;
94 }
95
96 /** Check for packets on Droq. This function should be called with lock held.
97 * @param droq - Droq on which count is checked.
98 * @return Returns packet count.
99 */
100 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq)
101 {
102 u32 pkt_count = 0;
103 u32 last_count;
104
105 pkt_count = readl(droq->pkts_sent_reg);
106
107 last_count = pkt_count - droq->pkt_count;
108 droq->pkt_count = pkt_count;
109
110 /* we shall write to cnts at napi irq enable or end of droq tasklet */
111 if (last_count)
112 atomic_add(last_count, &droq->pkts_pending);
113
114 return last_count;
115 }
116
117 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq)
118 {
119 u32 count = 0;
120
121 /* max_empty_descs is the max. no. of descs that can have no buffers.
122 * If the empty desc count goes beyond this value, we cannot safely
123 * read in a 64K packet sent by Octeon
124 * (64K is max pkt size from Octeon)
125 */
126 droq->max_empty_descs = 0;
127
128 do {
129 droq->max_empty_descs++;
130 count += droq->buffer_size;
131 } while (count < (64 * 1024));
132
133 droq->max_empty_descs = droq->max_count - droq->max_empty_descs;
134 }
135
136 static void octeon_droq_reset_indices(struct octeon_droq *droq)
137 {
138 droq->read_idx = 0;
139 droq->write_idx = 0;
140 droq->refill_idx = 0;
141 droq->refill_count = 0;
142 atomic_set(&droq->pkts_pending, 0);
143 }
144
145 static void
146 octeon_droq_destroy_ring_buffers(struct octeon_device *oct,
147 struct octeon_droq *droq)
148 {
149 u32 i;
150 struct octeon_skb_page_info *pg_info;
151
152 for (i = 0; i < droq->max_count; i++) {
153 pg_info = &droq->recv_buf_list[i].pg_info;
154
155 if (pg_info->dma)
156 lio_unmap_ring(oct->pci_dev,
157 (u64)pg_info->dma);
158 pg_info->dma = 0;
159
160 if (pg_info->page)
161 recv_buffer_destroy(droq->recv_buf_list[i].buffer,
162 pg_info);
163
164 if (droq->desc_ring && droq->desc_ring[i].info_ptr)
165 lio_unmap_ring_info(oct->pci_dev,
166 (u64)droq->
167 desc_ring[i].info_ptr,
168 OCT_DROQ_INFO_SIZE);
169 droq->recv_buf_list[i].buffer = NULL;
170 }
171
172 octeon_droq_reset_indices(droq);
173 }
174
175 static int
176 octeon_droq_setup_ring_buffers(struct octeon_device *oct,
177 struct octeon_droq *droq)
178 {
179 u32 i;
180 void *buf;
181 struct octeon_droq_desc *desc_ring = droq->desc_ring;
182
183 for (i = 0; i < droq->max_count; i++) {
184 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info);
185
186 if (!buf) {
187 dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n",
188 __func__);
189 droq->stats.rx_alloc_failure++;
190 return -ENOMEM;
191 }
192
193 droq->recv_buf_list[i].buffer = buf;
194 droq->recv_buf_list[i].data = get_rbd(buf);
195 droq->info_list[i].length = 0;
196
197 /* map ring buffers into memory */
198 desc_ring[i].info_ptr = lio_map_ring_info(droq, i);
199 desc_ring[i].buffer_ptr =
200 lio_map_ring(droq->recv_buf_list[i].buffer);
201 }
202
203 octeon_droq_reset_indices(droq);
204
205 octeon_droq_compute_max_packet_bufs(droq);
206
207 return 0;
208 }
209
210 int octeon_delete_droq(struct octeon_device *oct, u32 q_no)
211 {
212 struct octeon_droq *droq = oct->droq[q_no];
213
214 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
215
216 octeon_droq_destroy_ring_buffers(oct, droq);
217 vfree(droq->recv_buf_list);
218
219 if (droq->info_base_addr)
220 cnnic_free_aligned_dma(oct->pci_dev, droq->info_list,
221 droq->info_alloc_size,
222 droq->info_base_addr,
223 droq->info_list_dma);
224
225 if (droq->desc_ring)
226 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
227 droq->desc_ring, droq->desc_ring_dma);
228
229 memset(droq, 0, OCT_DROQ_SIZE);
230
231 return 0;
232 }
233
234 int octeon_init_droq(struct octeon_device *oct,
235 u32 q_no,
236 u32 num_descs,
237 u32 desc_size,
238 void *app_ctx)
239 {
240 struct octeon_droq *droq;
241 u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0;
242 u32 c_pkts_per_intr = 0, c_refill_threshold = 0;
243 int orig_node = dev_to_node(&oct->pci_dev->dev);
244 int numa_node = cpu_to_node(q_no % num_online_cpus());
245
246 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
247
248 droq = oct->droq[q_no];
249 memset(droq, 0, OCT_DROQ_SIZE);
250
251 droq->oct_dev = oct;
252 droq->q_no = q_no;
253 if (app_ctx)
254 droq->app_ctx = app_ctx;
255 else
256 droq->app_ctx = (void *)(size_t)q_no;
257
258 c_num_descs = num_descs;
259 c_buf_size = desc_size;
260 if (OCTEON_CN6XXX(oct)) {
261 struct octeon_config *conf6x = CHIP_FIELD(oct, cn6xxx, conf);
262
263 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x);
264 c_refill_threshold =
265 (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x);
266 } else if (OCTEON_CN23XX_PF(oct)) {
267 struct octeon_config *conf23 = CHIP_FIELD(oct, cn23xx_pf, conf);
268
269 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23);
270 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23);
271 } else {
272 return 1;
273 }
274
275 droq->max_count = c_num_descs;
276 droq->buffer_size = c_buf_size;
277
278 desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE;
279 set_dev_node(&oct->pci_dev->dev, numa_node);
280 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
281 (dma_addr_t *)&droq->desc_ring_dma);
282 set_dev_node(&oct->pci_dev->dev, orig_node);
283 if (!droq->desc_ring)
284 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
285 (dma_addr_t *)&droq->desc_ring_dma);
286
287 if (!droq->desc_ring) {
288 dev_err(&oct->pci_dev->dev,
289 "Output queue %d ring alloc failed\n", q_no);
290 return 1;
291 }
292
293 dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n",
294 q_no, droq->desc_ring, droq->desc_ring_dma);
295 dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no,
296 droq->max_count);
297
298 droq->info_list =
299 cnnic_numa_alloc_aligned_dma((droq->max_count *
300 OCT_DROQ_INFO_SIZE),
301 &droq->info_alloc_size,
302 &droq->info_base_addr,
303 numa_node);
304 if (!droq->info_list) {
305 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for info list.\n");
306 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
307 droq->desc_ring, droq->desc_ring_dma);
308 return 1;
309 }
310
311 droq->recv_buf_list = (struct octeon_recv_buffer *)
312 vmalloc_node(droq->max_count *
313 OCT_DROQ_RECVBUF_SIZE,
314 numa_node);
315 if (!droq->recv_buf_list)
316 droq->recv_buf_list = (struct octeon_recv_buffer *)
317 vmalloc(droq->max_count *
318 OCT_DROQ_RECVBUF_SIZE);
319 if (!droq->recv_buf_list) {
320 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n");
321 goto init_droq_fail;
322 }
323
324 if (octeon_droq_setup_ring_buffers(oct, droq))
325 goto init_droq_fail;
326
327 droq->pkts_per_intr = c_pkts_per_intr;
328 droq->refill_threshold = c_refill_threshold;
329
330 dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n",
331 droq->max_empty_descs);
332
333 spin_lock_init(&droq->lock);
334
335 INIT_LIST_HEAD(&droq->dispatch_list);
336
337 /* For 56xx Pass1, this function won't be called, so no checks. */
338 oct->fn_list.setup_oq_regs(oct, q_no);
339
340 oct->io_qmask.oq |= (1ULL << q_no);
341
342 return 0;
343
344 init_droq_fail:
345 octeon_delete_droq(oct, q_no);
346 return 1;
347 }
348
349 /* octeon_create_recv_info
350 * Parameters:
351 * octeon_dev - pointer to the octeon device structure
352 * droq - droq in which the packet arrived.
353 * buf_cnt - no. of buffers used by the packet.
354 * idx - index in the descriptor for the first buffer in the packet.
355 * Description:
356 * Allocates a recv_info_t and copies the buffer addresses for packet data
357 * into the recv_pkt space which starts at an 8B offset from recv_info_t.
358 * Flags the descriptors for refill later. If available descriptors go
359 * below the threshold to receive a 64K pkt, new buffers are first allocated
360 * before the recv_pkt_t is created.
361 * This routine will be called in interrupt context.
362 * Returns:
363 * Success: Pointer to recv_info_t
364 * Failure: NULL.
365 * Locks:
366 * The droq->lock is held when this routine is called.
367 */
368 static inline struct octeon_recv_info *octeon_create_recv_info(
369 struct octeon_device *octeon_dev,
370 struct octeon_droq *droq,
371 u32 buf_cnt,
372 u32 idx)
373 {
374 struct octeon_droq_info *info;
375 struct octeon_recv_pkt *recv_pkt;
376 struct octeon_recv_info *recv_info;
377 u32 i, bytes_left;
378 struct octeon_skb_page_info *pg_info;
379
380 info = &droq->info_list[idx];
381
382 recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch));
383 if (!recv_info)
384 return NULL;
385
386 recv_pkt = recv_info->recv_pkt;
387 recv_pkt->rh = info->rh;
388 recv_pkt->length = (u32)info->length;
389 recv_pkt->buffer_count = (u16)buf_cnt;
390 recv_pkt->octeon_id = (u16)octeon_dev->octeon_id;
391
392 i = 0;
393 bytes_left = (u32)info->length;
394
395 while (buf_cnt) {
396 {
397 pg_info = &droq->recv_buf_list[idx].pg_info;
398
399 lio_unmap_ring(octeon_dev->pci_dev,
400 (u64)pg_info->dma);
401 pg_info->page = NULL;
402 pg_info->dma = 0;
403 }
404
405 recv_pkt->buffer_size[i] =
406 (bytes_left >=
407 droq->buffer_size) ? droq->buffer_size : bytes_left;
408
409 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer;
410 droq->recv_buf_list[idx].buffer = NULL;
411
412 INCR_INDEX_BY1(idx, droq->max_count);
413 bytes_left -= droq->buffer_size;
414 i++;
415 buf_cnt--;
416 }
417
418 return recv_info;
419 }
420
421 /* If we were not able to refill all buffers, try to move around
422 * the buffers that were not dispatched.
423 */
424 static inline u32
425 octeon_droq_refill_pullup_descs(struct octeon_droq *droq,
426 struct octeon_droq_desc *desc_ring)
427 {
428 u32 desc_refilled = 0;
429
430 u32 refill_index = droq->refill_idx;
431
432 while (refill_index != droq->read_idx) {
433 if (droq->recv_buf_list[refill_index].buffer) {
434 droq->recv_buf_list[droq->refill_idx].buffer =
435 droq->recv_buf_list[refill_index].buffer;
436 droq->recv_buf_list[droq->refill_idx].data =
437 droq->recv_buf_list[refill_index].data;
438 desc_ring[droq->refill_idx].buffer_ptr =
439 desc_ring[refill_index].buffer_ptr;
440 droq->recv_buf_list[refill_index].buffer = NULL;
441 desc_ring[refill_index].buffer_ptr = 0;
442 do {
443 INCR_INDEX_BY1(droq->refill_idx,
444 droq->max_count);
445 desc_refilled++;
446 droq->refill_count--;
447 } while (droq->recv_buf_list[droq->refill_idx].
448 buffer);
449 }
450 INCR_INDEX_BY1(refill_index, droq->max_count);
451 } /* while */
452 return desc_refilled;
453 }
454
455 /* octeon_droq_refill
456 * Parameters:
457 * droq - droq in which descriptors require new buffers.
458 * Description:
459 * Called during normal DROQ processing in interrupt mode or by the poll
460 * thread to refill the descriptors from which buffers were dispatched
461 * to upper layers. Attempts to allocate new buffers. If that fails, moves
462 * up buffers (that were not dispatched) to form a contiguous ring.
463 * Returns:
464 * No of descriptors refilled.
465 * Locks:
466 * This routine is called with droq->lock held.
467 */
468 static u32
469 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq)
470 {
471 struct octeon_droq_desc *desc_ring;
472 void *buf = NULL;
473 u8 *data;
474 u32 desc_refilled = 0;
475 struct octeon_skb_page_info *pg_info;
476
477 desc_ring = droq->desc_ring;
478
479 while (droq->refill_count && (desc_refilled < droq->max_count)) {
480 /* If a valid buffer exists (happens if there is no dispatch),
481 * reuse
482 * the buffer, else allocate.
483 */
484 if (!droq->recv_buf_list[droq->refill_idx].buffer) {
485 pg_info =
486 &droq->recv_buf_list[droq->refill_idx].pg_info;
487 /* Either recycle the existing pages or go for
488 * new page alloc
489 */
490 if (pg_info->page)
491 buf = recv_buffer_reuse(octeon_dev, pg_info);
492 else
493 buf = recv_buffer_alloc(octeon_dev, pg_info);
494 /* If a buffer could not be allocated, no point in
495 * continuing
496 */
497 if (!buf) {
498 droq->stats.rx_alloc_failure++;
499 break;
500 }
501 droq->recv_buf_list[droq->refill_idx].buffer =
502 buf;
503 data = get_rbd(buf);
504 } else {
505 data = get_rbd(droq->recv_buf_list
506 [droq->refill_idx].buffer);
507 }
508
509 droq->recv_buf_list[droq->refill_idx].data = data;
510
511 desc_ring[droq->refill_idx].buffer_ptr =
512 lio_map_ring(droq->recv_buf_list[droq->
513 refill_idx].buffer);
514 /* Reset any previous values in the length field. */
515 droq->info_list[droq->refill_idx].length = 0;
516
517 INCR_INDEX_BY1(droq->refill_idx, droq->max_count);
518 desc_refilled++;
519 droq->refill_count--;
520 }
521
522 if (droq->refill_count)
523 desc_refilled +=
524 octeon_droq_refill_pullup_descs(droq, desc_ring);
525
526 /* if droq->refill_count
527 * The refill count would not change in pass two. We only moved buffers
528 * to close the gap in the ring, but we would still have the same no. of
529 * buffers to refill.
530 */
531 return desc_refilled;
532 }
533
534 static inline u32
535 octeon_droq_get_bufcount(u32 buf_size, u32 total_len)
536 {
537 u32 buf_cnt = 0;
538
539 while (total_len > (buf_size * buf_cnt))
540 buf_cnt++;
541 return buf_cnt;
542 }
543
544 static int
545 octeon_droq_dispatch_pkt(struct octeon_device *oct,
546 struct octeon_droq *droq,
547 union octeon_rh *rh,
548 struct octeon_droq_info *info)
549 {
550 u32 cnt;
551 octeon_dispatch_fn_t disp_fn;
552 struct octeon_recv_info *rinfo;
553
554 cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length);
555
556 disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode,
557 (u16)rh->r.subcode);
558 if (disp_fn) {
559 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx);
560 if (rinfo) {
561 struct __dispatch *rdisp = rinfo->rsvd;
562
563 rdisp->rinfo = rinfo;
564 rdisp->disp_fn = disp_fn;
565 rinfo->recv_pkt->rh = *rh;
566 list_add_tail(&rdisp->list,
567 &droq->dispatch_list);
568 } else {
569 droq->stats.dropped_nomem++;
570 }
571 } else {
572 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n",
573 (unsigned int)rh->r.opcode,
574 (unsigned int)rh->r.subcode);
575 droq->stats.dropped_nodispatch++;
576 }
577
578 return cnt;
579 }
580
581 static inline void octeon_droq_drop_packets(struct octeon_device *oct,
582 struct octeon_droq *droq,
583 u32 cnt)
584 {
585 u32 i = 0, buf_cnt;
586 struct octeon_droq_info *info;
587
588 for (i = 0; i < cnt; i++) {
589 info = &droq->info_list[droq->read_idx];
590 octeon_swap_8B_data((u64 *)info, 2);
591
592 if (info->length) {
593 info->length -= OCT_RH_SIZE;
594 droq->stats.bytes_received += info->length;
595 buf_cnt = octeon_droq_get_bufcount(droq->buffer_size,
596 (u32)info->length);
597 } else {
598 dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n");
599 buf_cnt = 1;
600 }
601
602 INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count);
603 droq->refill_count += buf_cnt;
604 }
605 }
606
607 static u32
608 octeon_droq_fast_process_packets(struct octeon_device *oct,
609 struct octeon_droq *droq,
610 u32 pkts_to_process)
611 {
612 struct octeon_droq_info *info;
613 union octeon_rh *rh;
614 u32 pkt, total_len = 0, pkt_count;
615
616 pkt_count = pkts_to_process;
617
618 for (pkt = 0; pkt < pkt_count; pkt++) {
619 u32 pkt_len = 0;
620 struct sk_buff *nicbuf = NULL;
621 struct octeon_skb_page_info *pg_info;
622 void *buf;
623
624 info = &droq->info_list[droq->read_idx];
625 octeon_swap_8B_data((u64 *)info, 2);
626
627 if (!info->length) {
628 dev_err(&oct->pci_dev->dev,
629 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n",
630 droq->q_no, droq->read_idx, pkt_count);
631 print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS,
632 (u8 *)info,
633 OCT_DROQ_INFO_SIZE);
634 break;
635 }
636
637 /* Len of resp hdr in included in the received data len. */
638 info->length -= OCT_RH_SIZE;
639 rh = &info->rh;
640
641 total_len += (u32)info->length;
642 if (OPCODE_SLOW_PATH(rh)) {
643 u32 buf_cnt;
644
645 buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info);
646 INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count);
647 droq->refill_count += buf_cnt;
648 } else {
649 if (info->length <= droq->buffer_size) {
650 pkt_len = (u32)info->length;
651 nicbuf = droq->recv_buf_list[
652 droq->read_idx].buffer;
653 pg_info = &droq->recv_buf_list[
654 droq->read_idx].pg_info;
655 if (recv_buffer_recycle(oct, pg_info))
656 pg_info->page = NULL;
657 droq->recv_buf_list[droq->read_idx].buffer =
658 NULL;
659
660 INCR_INDEX_BY1(droq->read_idx, droq->max_count);
661 droq->refill_count++;
662 } else {
663 nicbuf = octeon_fast_packet_alloc((u32)
664 info->length);
665 pkt_len = 0;
666 /* nicbuf allocation can fail. We'll handle it
667 * inside the loop.
668 */
669 while (pkt_len < info->length) {
670 int cpy_len, idx = droq->read_idx;
671
672 cpy_len = ((pkt_len + droq->buffer_size)
673 > info->length) ?
674 ((u32)info->length - pkt_len) :
675 droq->buffer_size;
676
677 if (nicbuf) {
678 octeon_fast_packet_next(droq,
679 nicbuf,
680 cpy_len,
681 idx);
682 buf = droq->recv_buf_list[idx].
683 buffer;
684 recv_buffer_fast_free(buf);
685 droq->recv_buf_list[idx].buffer
686 = NULL;
687 } else {
688 droq->stats.rx_alloc_failure++;
689 }
690
691 pkt_len += cpy_len;
692 INCR_INDEX_BY1(droq->read_idx,
693 droq->max_count);
694 droq->refill_count++;
695 }
696 }
697
698 if (nicbuf) {
699 if (droq->ops.fptr) {
700 droq->ops.fptr(oct->octeon_id,
701 nicbuf, pkt_len,
702 rh, &droq->napi,
703 droq->ops.farg);
704 } else {
705 recv_buffer_free(nicbuf);
706 }
707 }
708 }
709
710 if (droq->refill_count >= droq->refill_threshold) {
711 int desc_refilled = octeon_droq_refill(oct, droq);
712
713 /* Flush the droq descriptor data to memory to be sure
714 * that when we update the credits the data in memory
715 * is accurate.
716 */
717 wmb();
718 writel((desc_refilled), droq->pkts_credit_reg);
719 /* make sure mmio write completes */
720 mmiowb();
721 }
722
723 } /* for (each packet)... */
724
725 /* Increment refill_count by the number of buffers processed. */
726 droq->stats.pkts_received += pkt;
727 droq->stats.bytes_received += total_len;
728
729 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) {
730 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt));
731
732 droq->stats.dropped_toomany += (pkts_to_process - pkt);
733 return pkts_to_process;
734 }
735
736 return pkt;
737 }
738
739 int
740 octeon_droq_process_packets(struct octeon_device *oct,
741 struct octeon_droq *droq,
742 u32 budget)
743 {
744 u32 pkt_count = 0, pkts_processed = 0;
745 struct list_head *tmp, *tmp2;
746
747 /* Grab the droq lock */
748 spin_lock(&droq->lock);
749
750 octeon_droq_check_hw_for_pkts(droq);
751 pkt_count = atomic_read(&droq->pkts_pending);
752
753 if (!pkt_count) {
754 spin_unlock(&droq->lock);
755 return 0;
756 }
757
758 if (pkt_count > budget)
759 pkt_count = budget;
760
761 pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count);
762
763 atomic_sub(pkts_processed, &droq->pkts_pending);
764
765 /* Release the spin lock */
766 spin_unlock(&droq->lock);
767
768 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
769 struct __dispatch *rdisp = (struct __dispatch *)tmp;
770
771 list_del(tmp);
772 rdisp->disp_fn(rdisp->rinfo,
773 octeon_get_dispatch_arg
774 (oct,
775 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
776 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
777 }
778
779 /* If there are packets pending. schedule tasklet again */
780 if (atomic_read(&droq->pkts_pending))
781 return 1;
782
783 return 0;
784 }
785
786 /**
787 * Utility function to poll for packets. check_hw_for_packets must be
788 * called before calling this routine.
789 */
790
791 static int
792 octeon_droq_process_poll_pkts(struct octeon_device *oct,
793 struct octeon_droq *droq, u32 budget)
794 {
795 struct list_head *tmp, *tmp2;
796 u32 pkts_available = 0, pkts_processed = 0;
797 u32 total_pkts_processed = 0;
798
799 if (budget > droq->max_count)
800 budget = droq->max_count;
801
802 spin_lock(&droq->lock);
803
804 while (total_pkts_processed < budget) {
805 octeon_droq_check_hw_for_pkts(droq);
806
807 pkts_available =
808 CVM_MIN((budget - total_pkts_processed),
809 (u32)(atomic_read(&droq->pkts_pending)));
810
811 if (pkts_available == 0)
812 break;
813
814 pkts_processed =
815 octeon_droq_fast_process_packets(oct, droq,
816 pkts_available);
817
818 atomic_sub(pkts_processed, &droq->pkts_pending);
819
820 total_pkts_processed += pkts_processed;
821 }
822
823 spin_unlock(&droq->lock);
824
825 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
826 struct __dispatch *rdisp = (struct __dispatch *)tmp;
827
828 list_del(tmp);
829 rdisp->disp_fn(rdisp->rinfo,
830 octeon_get_dispatch_arg
831 (oct,
832 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
833 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
834 }
835
836 return total_pkts_processed;
837 }
838
839 int
840 octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, int cmd,
841 u32 arg)
842 {
843 struct octeon_droq *droq;
844
845 droq = oct->droq[q_no];
846
847 if (cmd == POLL_EVENT_PROCESS_PKTS)
848 return octeon_droq_process_poll_pkts(oct, droq, arg);
849
850 if (cmd == POLL_EVENT_PENDING_PKTS) {
851 u32 pkt_cnt = atomic_read(&droq->pkts_pending);
852
853 return octeon_droq_process_packets(oct, droq, pkt_cnt);
854 }
855
856 if (cmd == POLL_EVENT_ENABLE_INTR) {
857 u32 value;
858 unsigned long flags;
859
860 /* Enable Pkt Interrupt */
861 switch (oct->chip_id) {
862 case OCTEON_CN66XX:
863 case OCTEON_CN68XX: {
864 struct octeon_cn6xxx *cn6xxx =
865 (struct octeon_cn6xxx *)oct->chip;
866 spin_lock_irqsave
867 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
868 value =
869 octeon_read_csr(oct,
870 CN6XXX_SLI_PKT_TIME_INT_ENB);
871 value |= (1 << q_no);
872 octeon_write_csr(oct,
873 CN6XXX_SLI_PKT_TIME_INT_ENB,
874 value);
875 value =
876 octeon_read_csr(oct,
877 CN6XXX_SLI_PKT_CNT_INT_ENB);
878 value |= (1 << q_no);
879 octeon_write_csr(oct,
880 CN6XXX_SLI_PKT_CNT_INT_ENB,
881 value);
882
883 /* don't bother flushing the enables */
884
885 spin_unlock_irqrestore
886 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
887 return 0;
888 }
889 break;
890 case OCTEON_CN23XX_PF_VID: {
891 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]);
892 }
893 break;
894 }
895 return 0;
896 }
897
898 dev_err(&oct->pci_dev->dev, "%s Unknown command: %d\n", __func__, cmd);
899 return -EINVAL;
900 }
901
902 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no,
903 struct octeon_droq_ops *ops)
904 {
905 struct octeon_droq *droq;
906 unsigned long flags;
907 struct octeon_config *oct_cfg = NULL;
908
909 oct_cfg = octeon_get_conf(oct);
910
911 if (!oct_cfg)
912 return -EINVAL;
913
914 if (!(ops)) {
915 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n",
916 __func__);
917 return -EINVAL;
918 }
919
920 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
921 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
922 __func__, q_no, (oct->num_oqs - 1));
923 return -EINVAL;
924 }
925
926 droq = oct->droq[q_no];
927
928 spin_lock_irqsave(&droq->lock, flags);
929
930 memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops));
931
932 spin_unlock_irqrestore(&droq->lock, flags);
933
934 return 0;
935 }
936
937 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no)
938 {
939 unsigned long flags;
940 struct octeon_droq *droq;
941 struct octeon_config *oct_cfg = NULL;
942
943 oct_cfg = octeon_get_conf(oct);
944
945 if (!oct_cfg)
946 return -EINVAL;
947
948 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
949 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
950 __func__, q_no, oct->num_oqs - 1);
951 return -EINVAL;
952 }
953
954 droq = oct->droq[q_no];
955
956 if (!droq) {
957 dev_info(&oct->pci_dev->dev,
958 "Droq id (%d) not available.\n", q_no);
959 return 0;
960 }
961
962 spin_lock_irqsave(&droq->lock, flags);
963
964 droq->ops.fptr = NULL;
965 droq->ops.farg = NULL;
966 droq->ops.drop_on_max = 0;
967
968 spin_unlock_irqrestore(&droq->lock, flags);
969
970 return 0;
971 }
972
973 int octeon_create_droq(struct octeon_device *oct,
974 u32 q_no, u32 num_descs,
975 u32 desc_size, void *app_ctx)
976 {
977 struct octeon_droq *droq;
978 int numa_node = cpu_to_node(q_no % num_online_cpus());
979
980 if (oct->droq[q_no]) {
981 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n",
982 q_no);
983 return 1;
984 }
985
986 /* Allocate the DS for the new droq. */
987 droq = vmalloc_node(sizeof(*droq), numa_node);
988 if (!droq)
989 droq = vmalloc(sizeof(*droq));
990 if (!droq)
991 goto create_droq_fail;
992 memset(droq, 0, sizeof(struct octeon_droq));
993
994 /*Disable the pkt o/p for this Q */
995 octeon_set_droq_pkt_op(oct, q_no, 0);
996 oct->droq[q_no] = droq;
997
998 /* Initialize the Droq */
999 octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx);
1000
1001 oct->num_oqs++;
1002
1003 dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__,
1004 oct->num_oqs);
1005
1006 /* Global Droq register settings */
1007
1008 /* As of now not required, as setting are done for all 32 Droqs at
1009 * the same time.
1010 */
1011 return 0;
1012
1013 create_droq_fail:
1014 octeon_delete_droq(oct, q_no);
1015 return -ENOMEM;
1016 }
Linux kernel - Deliverables
This page took 0.080503 seconds and 5 git commands to generate.