projects / deliverable / linux.git / blob
? search:


dd0b32c58a2440fad21199aeae4772140aa2296f
[deliverable/linux.git] / drivers / net / ethernet / broadcom / bnxt / bnxt.c
1 /* Broadcom NetXtreme-C/E network driver.
2 *
3 * Copyright (c) 2014-2016 Broadcom Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
8 */
9
10 #include <linux/module.h>
11
12 #include <linux/stringify.h>
13 #include <linux/kernel.h>
14 #include <linux/timer.h>
15 #include <linux/errno.h>
16 #include <linux/ioport.h>
17 #include <linux/slab.h>
18 #include <linux/vmalloc.h>
19 #include <linux/interrupt.h>
20 #include <linux/pci.h>
21 #include <linux/netdevice.h>
22 #include <linux/etherdevice.h>
23 #include <linux/skbuff.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/bitops.h>
26 #include <linux/io.h>
27 #include <linux/irq.h>
28 #include <linux/delay.h>
29 #include <asm/byteorder.h>
30 #include <asm/page.h>
31 #include <linux/time.h>
32 #include <linux/mii.h>
33 #include <linux/if.h>
34 #include <linux/if_vlan.h>
35 #include <net/ip.h>
36 #include <net/tcp.h>
37 #include <net/udp.h>
38 #include <net/checksum.h>
39 #include <net/ip6_checksum.h>
40 #if defined(CONFIG_VXLAN) || defined(CONFIG_VXLAN_MODULE)
41 #include <net/vxlan.h>
42 #endif
43 #ifdef CONFIG_NET_RX_BUSY_POLL
44 #include <net/busy_poll.h>
45 #endif
46 #include <linux/workqueue.h>
47 #include <linux/prefetch.h>
48 #include <linux/cache.h>
49 #include <linux/log2.h>
50 #include <linux/aer.h>
51 #include <linux/bitmap.h>
52 #include <linux/cpu_rmap.h>
53
54 #include "bnxt_hsi.h"
55 #include "bnxt.h"
56 #include "bnxt_sriov.h"
57 #include "bnxt_ethtool.h"
58
59 #define BNXT_TX_TIMEOUT (5 * HZ)
60
61 static const char version[] =
62 "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n";
63
64 MODULE_LICENSE("GPL");
65 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
66 MODULE_VERSION(DRV_MODULE_VERSION);
67
68 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
69 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
70 #define BNXT_RX_COPY_THRESH 256
71
72 #define BNXT_TX_PUSH_THRESH 164
73
74 enum board_idx {
75 BCM57301,
76 BCM57302,
77 BCM57304,
78 BCM57402,
79 BCM57404,
80 BCM57406,
81 BCM57304_VF,
82 BCM57404_VF,
83 };
84
85 /* indexed by enum above */
86 static const struct {
87 char *name;
88 } board_info[] = {
89 { "Broadcom BCM57301 NetXtreme-C Single-port 10Gb Ethernet" },
90 { "Broadcom BCM57302 NetXtreme-C Dual-port 10Gb/25Gb Ethernet" },
91 { "Broadcom BCM57304 NetXtreme-C Dual-port 10Gb/25Gb/40Gb/50Gb Ethernet" },
92 { "Broadcom BCM57402 NetXtreme-E Dual-port 10Gb Ethernet" },
93 { "Broadcom BCM57404 NetXtreme-E Dual-port 10Gb/25Gb Ethernet" },
94 { "Broadcom BCM57406 NetXtreme-E Dual-port 10GBase-T Ethernet" },
95 { "Broadcom BCM57304 NetXtreme-C Ethernet Virtual Function" },
96 { "Broadcom BCM57404 NetXtreme-E Ethernet Virtual Function" },
97 };
98
99 static const struct pci_device_id bnxt_pci_tbl[] = {
100 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
101 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
102 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
103 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
104 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
105 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
106 #ifdef CONFIG_BNXT_SRIOV
107 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = BCM57304_VF },
108 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = BCM57404_VF },
109 #endif
110 { 0 }
111 };
112
113 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
114
115 static const u16 bnxt_vf_req_snif[] = {
116 HWRM_FUNC_CFG,
117 HWRM_PORT_PHY_QCFG,
118 HWRM_CFA_L2_FILTER_ALLOC,
119 };
120
121 static const u16 bnxt_async_events_arr[] = {
122 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
123 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
124 };
125
126 static bool bnxt_vf_pciid(enum board_idx idx)
127 {
128 return (idx == BCM57304_VF || idx == BCM57404_VF);
129 }
130
131 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID)
132 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
133 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS)
134
135 #define BNXT_CP_DB_REARM(db, raw_cons) \
136 writel(DB_CP_REARM_FLAGS | RING_CMP(raw_cons), db)
137
138 #define BNXT_CP_DB(db, raw_cons) \
139 writel(DB_CP_FLAGS | RING_CMP(raw_cons), db)
140
141 #define BNXT_CP_DB_IRQ_DIS(db) \
142 writel(DB_CP_IRQ_DIS_FLAGS, db)
143
144 static inline u32 bnxt_tx_avail(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
145 {
146 /* Tell compiler to fetch tx indices from memory. */
147 barrier();
148
149 return bp->tx_ring_size -
150 ((txr->tx_prod - txr->tx_cons) & bp->tx_ring_mask);
151 }
152
153 static const u16 bnxt_lhint_arr[] = {
154 TX_BD_FLAGS_LHINT_512_AND_SMALLER,
155 TX_BD_FLAGS_LHINT_512_TO_1023,
156 TX_BD_FLAGS_LHINT_1024_TO_2047,
157 TX_BD_FLAGS_LHINT_1024_TO_2047,
158 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
159 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
160 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
161 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
162 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
163 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
164 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
165 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
166 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
167 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
168 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
169 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
170 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
171 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
172 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
173 };
174
175 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
176 {
177 struct bnxt *bp = netdev_priv(dev);
178 struct tx_bd *txbd;
179 struct tx_bd_ext *txbd1;
180 struct netdev_queue *txq;
181 int i;
182 dma_addr_t mapping;
183 unsigned int length, pad = 0;
184 u32 len, free_size, vlan_tag_flags, cfa_action, flags;
185 u16 prod, last_frag;
186 struct pci_dev *pdev = bp->pdev;
187 struct bnxt_tx_ring_info *txr;
188 struct bnxt_sw_tx_bd *tx_buf;
189
190 i = skb_get_queue_mapping(skb);
191 if (unlikely(i >= bp->tx_nr_rings)) {
192 dev_kfree_skb_any(skb);
193 return NETDEV_TX_OK;
194 }
195
196 txr = &bp->tx_ring[i];
197 txq = netdev_get_tx_queue(dev, i);
198 prod = txr->tx_prod;
199
200 free_size = bnxt_tx_avail(bp, txr);
201 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
202 netif_tx_stop_queue(txq);
203 return NETDEV_TX_BUSY;
204 }
205
206 length = skb->len;
207 len = skb_headlen(skb);
208 last_frag = skb_shinfo(skb)->nr_frags;
209
210 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
211
212 txbd->tx_bd_opaque = prod;
213
214 tx_buf = &txr->tx_buf_ring[prod];
215 tx_buf->skb = skb;
216 tx_buf->nr_frags = last_frag;
217
218 vlan_tag_flags = 0;
219 cfa_action = 0;
220 if (skb_vlan_tag_present(skb)) {
221 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
222 skb_vlan_tag_get(skb);
223 /* Currently supports 8021Q, 8021AD vlan offloads
224 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
225 */
226 if (skb->vlan_proto == htons(ETH_P_8021Q))
227 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
228 }
229
230 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
231 struct tx_push_buffer *tx_push_buf = txr->tx_push;
232 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
233 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
234 void *pdata = tx_push_buf->data;
235 u64 *end;
236 int j, push_len;
237
238 /* Set COAL_NOW to be ready quickly for the next push */
239 tx_push->tx_bd_len_flags_type =
240 cpu_to_le32((length << TX_BD_LEN_SHIFT) |
241 TX_BD_TYPE_LONG_TX_BD |
242 TX_BD_FLAGS_LHINT_512_AND_SMALLER |
243 TX_BD_FLAGS_COAL_NOW |
244 TX_BD_FLAGS_PACKET_END |
245 (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
246
247 if (skb->ip_summed == CHECKSUM_PARTIAL)
248 tx_push1->tx_bd_hsize_lflags =
249 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
250 else
251 tx_push1->tx_bd_hsize_lflags = 0;
252
253 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
254 tx_push1->tx_bd_cfa_action = cpu_to_le32(cfa_action);
255
256 end = pdata + length;
257 end = PTR_ALIGN(end, 8) - 1;
258 *end = 0;
259
260 skb_copy_from_linear_data(skb, pdata, len);
261 pdata += len;
262 for (j = 0; j < last_frag; j++) {
263 skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
264 void *fptr;
265
266 fptr = skb_frag_address_safe(frag);
267 if (!fptr)
268 goto normal_tx;
269
270 memcpy(pdata, fptr, skb_frag_size(frag));
271 pdata += skb_frag_size(frag);
272 }
273
274 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
275 txbd->tx_bd_haddr = txr->data_mapping;
276 prod = NEXT_TX(prod);
277 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
278 memcpy(txbd, tx_push1, sizeof(*txbd));
279 prod = NEXT_TX(prod);
280 tx_push->doorbell =
281 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
282 txr->tx_prod = prod;
283
284 netdev_tx_sent_queue(txq, skb->len);
285
286 push_len = (length + sizeof(*tx_push) + 7) / 8;
287 if (push_len > 16) {
288 __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 16);
289 __iowrite64_copy(txr->tx_doorbell + 4, tx_push_buf + 1,
290 push_len - 16);
291 } else {
292 __iowrite64_copy(txr->tx_doorbell, tx_push_buf,
293 push_len);
294 }
295
296 tx_buf->is_push = 1;
297 goto tx_done;
298 }
299
300 normal_tx:
301 if (length < BNXT_MIN_PKT_SIZE) {
302 pad = BNXT_MIN_PKT_SIZE - length;
303 if (skb_pad(skb, pad)) {
304 /* SKB already freed. */
305 tx_buf->skb = NULL;
306 return NETDEV_TX_OK;
307 }
308 length = BNXT_MIN_PKT_SIZE;
309 }
310
311 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
312
313 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
314 dev_kfree_skb_any(skb);
315 tx_buf->skb = NULL;
316 return NETDEV_TX_OK;
317 }
318
319 dma_unmap_addr_set(tx_buf, mapping, mapping);
320 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
321 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
322
323 txbd->tx_bd_haddr = cpu_to_le64(mapping);
324
325 prod = NEXT_TX(prod);
326 txbd1 = (struct tx_bd_ext *)
327 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
328
329 txbd1->tx_bd_hsize_lflags = 0;
330 if (skb_is_gso(skb)) {
331 u32 hdr_len;
332
333 if (skb->encapsulation)
334 hdr_len = skb_inner_network_offset(skb) +
335 skb_inner_network_header_len(skb) +
336 inner_tcp_hdrlen(skb);
337 else
338 hdr_len = skb_transport_offset(skb) +
339 tcp_hdrlen(skb);
340
341 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
342 TX_BD_FLAGS_T_IPID |
343 (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
344 length = skb_shinfo(skb)->gso_size;
345 txbd1->tx_bd_mss = cpu_to_le32(length);
346 length += hdr_len;
347 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
348 txbd1->tx_bd_hsize_lflags =
349 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
350 txbd1->tx_bd_mss = 0;
351 }
352
353 length >>= 9;
354 flags |= bnxt_lhint_arr[length];
355 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
356
357 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
358 txbd1->tx_bd_cfa_action = cpu_to_le32(cfa_action);
359 for (i = 0; i < last_frag; i++) {
360 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
361
362 prod = NEXT_TX(prod);
363 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
364
365 len = skb_frag_size(frag);
366 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
367 DMA_TO_DEVICE);
368
369 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
370 goto tx_dma_error;
371
372 tx_buf = &txr->tx_buf_ring[prod];
373 dma_unmap_addr_set(tx_buf, mapping, mapping);
374
375 txbd->tx_bd_haddr = cpu_to_le64(mapping);
376
377 flags = len << TX_BD_LEN_SHIFT;
378 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
379 }
380
381 flags &= ~TX_BD_LEN;
382 txbd->tx_bd_len_flags_type =
383 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
384 TX_BD_FLAGS_PACKET_END);
385
386 netdev_tx_sent_queue(txq, skb->len);
387
388 /* Sync BD data before updating doorbell */
389 wmb();
390
391 prod = NEXT_TX(prod);
392 txr->tx_prod = prod;
393
394 writel(DB_KEY_TX | prod, txr->tx_doorbell);
395 writel(DB_KEY_TX | prod, txr->tx_doorbell);
396
397 tx_done:
398
399 mmiowb();
400
401 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
402 netif_tx_stop_queue(txq);
403
404 /* netif_tx_stop_queue() must be done before checking
405 * tx index in bnxt_tx_avail() below, because in
406 * bnxt_tx_int(), we update tx index before checking for
407 * netif_tx_queue_stopped().
408 */
409 smp_mb();
410 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
411 netif_tx_wake_queue(txq);
412 }
413 return NETDEV_TX_OK;
414
415 tx_dma_error:
416 last_frag = i;
417
418 /* start back at beginning and unmap skb */
419 prod = txr->tx_prod;
420 tx_buf = &txr->tx_buf_ring[prod];
421 tx_buf->skb = NULL;
422 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
423 skb_headlen(skb), PCI_DMA_TODEVICE);
424 prod = NEXT_TX(prod);
425
426 /* unmap remaining mapped pages */
427 for (i = 0; i < last_frag; i++) {
428 prod = NEXT_TX(prod);
429 tx_buf = &txr->tx_buf_ring[prod];
430 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
431 skb_frag_size(&skb_shinfo(skb)->frags[i]),
432 PCI_DMA_TODEVICE);
433 }
434
435 dev_kfree_skb_any(skb);
436 return NETDEV_TX_OK;
437 }
438
439 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
440 {
441 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
442 int index = txr - &bp->tx_ring[0];
443 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, index);
444 u16 cons = txr->tx_cons;
445 struct pci_dev *pdev = bp->pdev;
446 int i;
447 unsigned int tx_bytes = 0;
448
449 for (i = 0; i < nr_pkts; i++) {
450 struct bnxt_sw_tx_bd *tx_buf;
451 struct sk_buff *skb;
452 int j, last;
453
454 tx_buf = &txr->tx_buf_ring[cons];
455 cons = NEXT_TX(cons);
456 skb = tx_buf->skb;
457 tx_buf->skb = NULL;
458
459 if (tx_buf->is_push) {
460 tx_buf->is_push = 0;
461 goto next_tx_int;
462 }
463
464 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
465 skb_headlen(skb), PCI_DMA_TODEVICE);
466 last = tx_buf->nr_frags;
467
468 for (j = 0; j < last; j++) {
469 cons = NEXT_TX(cons);
470 tx_buf = &txr->tx_buf_ring[cons];
471 dma_unmap_page(
472 &pdev->dev,
473 dma_unmap_addr(tx_buf, mapping),
474 skb_frag_size(&skb_shinfo(skb)->frags[j]),
475 PCI_DMA_TODEVICE);
476 }
477
478 next_tx_int:
479 cons = NEXT_TX(cons);
480
481 tx_bytes += skb->len;
482 dev_kfree_skb_any(skb);
483 }
484
485 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
486 txr->tx_cons = cons;
487
488 /* Need to make the tx_cons update visible to bnxt_start_xmit()
489 * before checking for netif_tx_queue_stopped(). Without the
490 * memory barrier, there is a small possibility that bnxt_start_xmit()
491 * will miss it and cause the queue to be stopped forever.
492 */
493 smp_mb();
494
495 if (unlikely(netif_tx_queue_stopped(txq)) &&
496 (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
497 __netif_tx_lock(txq, smp_processor_id());
498 if (netif_tx_queue_stopped(txq) &&
499 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
500 txr->dev_state != BNXT_DEV_STATE_CLOSING)
501 netif_tx_wake_queue(txq);
502 __netif_tx_unlock(txq);
503 }
504 }
505
506 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
507 gfp_t gfp)
508 {
509 u8 *data;
510 struct pci_dev *pdev = bp->pdev;
511
512 data = kmalloc(bp->rx_buf_size, gfp);
513 if (!data)
514 return NULL;
515
516 *mapping = dma_map_single(&pdev->dev, data + BNXT_RX_DMA_OFFSET,
517 bp->rx_buf_use_size, PCI_DMA_FROMDEVICE);
518
519 if (dma_mapping_error(&pdev->dev, *mapping)) {
520 kfree(data);
521 data = NULL;
522 }
523 return data;
524 }
525
526 static inline int bnxt_alloc_rx_data(struct bnxt *bp,
527 struct bnxt_rx_ring_info *rxr,
528 u16 prod, gfp_t gfp)
529 {
530 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
531 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
532 u8 *data;
533 dma_addr_t mapping;
534
535 data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
536 if (!data)
537 return -ENOMEM;
538
539 rx_buf->data = data;
540 dma_unmap_addr_set(rx_buf, mapping, mapping);
541
542 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
543
544 return 0;
545 }
546
547 static void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons,
548 u8 *data)
549 {
550 u16 prod = rxr->rx_prod;
551 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
552 struct rx_bd *cons_bd, *prod_bd;
553
554 prod_rx_buf = &rxr->rx_buf_ring[prod];
555 cons_rx_buf = &rxr->rx_buf_ring[cons];
556
557 prod_rx_buf->data = data;
558
559 dma_unmap_addr_set(prod_rx_buf, mapping,
560 dma_unmap_addr(cons_rx_buf, mapping));
561
562 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
563 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
564
565 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
566 }
567
568 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
569 {
570 u16 next, max = rxr->rx_agg_bmap_size;
571
572 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
573 if (next >= max)
574 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
575 return next;
576 }
577
578 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
579 struct bnxt_rx_ring_info *rxr,
580 u16 prod, gfp_t gfp)
581 {
582 struct rx_bd *rxbd =
583 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
584 struct bnxt_sw_rx_agg_bd *rx_agg_buf;
585 struct pci_dev *pdev = bp->pdev;
586 struct page *page;
587 dma_addr_t mapping;
588 u16 sw_prod = rxr->rx_sw_agg_prod;
589
590 page = alloc_page(gfp);
591 if (!page)
592 return -ENOMEM;
593
594 mapping = dma_map_page(&pdev->dev, page, 0, PAGE_SIZE,
595 PCI_DMA_FROMDEVICE);
596 if (dma_mapping_error(&pdev->dev, mapping)) {
597 __free_page(page);
598 return -EIO;
599 }
600
601 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
602 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
603
604 __set_bit(sw_prod, rxr->rx_agg_bmap);
605 rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
606 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
607
608 rx_agg_buf->page = page;
609 rx_agg_buf->mapping = mapping;
610 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
611 rxbd->rx_bd_opaque = sw_prod;
612 return 0;
613 }
614
615 static void bnxt_reuse_rx_agg_bufs(struct bnxt_napi *bnapi, u16 cp_cons,
616 u32 agg_bufs)
617 {
618 struct bnxt *bp = bnapi->bp;
619 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
620 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
621 u16 prod = rxr->rx_agg_prod;
622 u16 sw_prod = rxr->rx_sw_agg_prod;
623 u32 i;
624
625 for (i = 0; i < agg_bufs; i++) {
626 u16 cons;
627 struct rx_agg_cmp *agg;
628 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
629 struct rx_bd *prod_bd;
630 struct page *page;
631
632 agg = (struct rx_agg_cmp *)
633 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
634 cons = agg->rx_agg_cmp_opaque;
635 __clear_bit(cons, rxr->rx_agg_bmap);
636
637 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
638 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
639
640 __set_bit(sw_prod, rxr->rx_agg_bmap);
641 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
642 cons_rx_buf = &rxr->rx_agg_ring[cons];
643
644 /* It is possible for sw_prod to be equal to cons, so
645 * set cons_rx_buf->page to NULL first.
646 */
647 page = cons_rx_buf->page;
648 cons_rx_buf->page = NULL;
649 prod_rx_buf->page = page;
650
651 prod_rx_buf->mapping = cons_rx_buf->mapping;
652
653 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
654
655 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
656 prod_bd->rx_bd_opaque = sw_prod;
657
658 prod = NEXT_RX_AGG(prod);
659 sw_prod = NEXT_RX_AGG(sw_prod);
660 cp_cons = NEXT_CMP(cp_cons);
661 }
662 rxr->rx_agg_prod = prod;
663 rxr->rx_sw_agg_prod = sw_prod;
664 }
665
666 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
667 struct bnxt_rx_ring_info *rxr, u16 cons,
668 u16 prod, u8 *data, dma_addr_t dma_addr,
669 unsigned int len)
670 {
671 int err;
672 struct sk_buff *skb;
673
674 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
675 if (unlikely(err)) {
676 bnxt_reuse_rx_data(rxr, cons, data);
677 return NULL;
678 }
679
680 skb = build_skb(data, 0);
681 dma_unmap_single(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
682 PCI_DMA_FROMDEVICE);
683 if (!skb) {
684 kfree(data);
685 return NULL;
686 }
687
688 skb_reserve(skb, BNXT_RX_OFFSET);
689 skb_put(skb, len);
690 return skb;
691 }
692
693 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, struct bnxt_napi *bnapi,
694 struct sk_buff *skb, u16 cp_cons,
695 u32 agg_bufs)
696 {
697 struct pci_dev *pdev = bp->pdev;
698 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
699 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
700 u16 prod = rxr->rx_agg_prod;
701 u32 i;
702
703 for (i = 0; i < agg_bufs; i++) {
704 u16 cons, frag_len;
705 struct rx_agg_cmp *agg;
706 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
707 struct page *page;
708 dma_addr_t mapping;
709
710 agg = (struct rx_agg_cmp *)
711 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
712 cons = agg->rx_agg_cmp_opaque;
713 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
714 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
715
716 cons_rx_buf = &rxr->rx_agg_ring[cons];
717 skb_fill_page_desc(skb, i, cons_rx_buf->page, 0, frag_len);
718 __clear_bit(cons, rxr->rx_agg_bmap);
719
720 /* It is possible for bnxt_alloc_rx_page() to allocate
721 * a sw_prod index that equals the cons index, so we
722 * need to clear the cons entry now.
723 */
724 mapping = dma_unmap_addr(cons_rx_buf, mapping);
725 page = cons_rx_buf->page;
726 cons_rx_buf->page = NULL;
727
728 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
729 struct skb_shared_info *shinfo;
730 unsigned int nr_frags;
731
732 shinfo = skb_shinfo(skb);
733 nr_frags = --shinfo->nr_frags;
734 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
735
736 dev_kfree_skb(skb);
737
738 cons_rx_buf->page = page;
739
740 /* Update prod since possibly some pages have been
741 * allocated already.
742 */
743 rxr->rx_agg_prod = prod;
744 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs - i);
745 return NULL;
746 }
747
748 dma_unmap_page(&pdev->dev, mapping, PAGE_SIZE,
749 PCI_DMA_FROMDEVICE);
750
751 skb->data_len += frag_len;
752 skb->len += frag_len;
753 skb->truesize += PAGE_SIZE;
754
755 prod = NEXT_RX_AGG(prod);
756 cp_cons = NEXT_CMP(cp_cons);
757 }
758 rxr->rx_agg_prod = prod;
759 return skb;
760 }
761
762 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
763 u8 agg_bufs, u32 *raw_cons)
764 {
765 u16 last;
766 struct rx_agg_cmp *agg;
767
768 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
769 last = RING_CMP(*raw_cons);
770 agg = (struct rx_agg_cmp *)
771 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
772 return RX_AGG_CMP_VALID(agg, *raw_cons);
773 }
774
775 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
776 unsigned int len,
777 dma_addr_t mapping)
778 {
779 struct bnxt *bp = bnapi->bp;
780 struct pci_dev *pdev = bp->pdev;
781 struct sk_buff *skb;
782
783 skb = napi_alloc_skb(&bnapi->napi, len);
784 if (!skb)
785 return NULL;
786
787 dma_sync_single_for_cpu(&pdev->dev, mapping,
788 bp->rx_copy_thresh, PCI_DMA_FROMDEVICE);
789
790 memcpy(skb->data - BNXT_RX_OFFSET, data, len + BNXT_RX_OFFSET);
791
792 dma_sync_single_for_device(&pdev->dev, mapping,
793 bp->rx_copy_thresh,
794 PCI_DMA_FROMDEVICE);
795
796 skb_put(skb, len);
797 return skb;
798 }
799
800 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
801 struct rx_tpa_start_cmp *tpa_start,
802 struct rx_tpa_start_cmp_ext *tpa_start1)
803 {
804 u8 agg_id = TPA_START_AGG_ID(tpa_start);
805 u16 cons, prod;
806 struct bnxt_tpa_info *tpa_info;
807 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
808 struct rx_bd *prod_bd;
809 dma_addr_t mapping;
810
811 cons = tpa_start->rx_tpa_start_cmp_opaque;
812 prod = rxr->rx_prod;
813 cons_rx_buf = &rxr->rx_buf_ring[cons];
814 prod_rx_buf = &rxr->rx_buf_ring[prod];
815 tpa_info = &rxr->rx_tpa[agg_id];
816
817 prod_rx_buf->data = tpa_info->data;
818
819 mapping = tpa_info->mapping;
820 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
821
822 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
823
824 prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
825
826 tpa_info->data = cons_rx_buf->data;
827 cons_rx_buf->data = NULL;
828 tpa_info->mapping = dma_unmap_addr(cons_rx_buf, mapping);
829
830 tpa_info->len =
831 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
832 RX_TPA_START_CMP_LEN_SHIFT;
833 if (likely(TPA_START_HASH_VALID(tpa_start))) {
834 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
835
836 tpa_info->hash_type = PKT_HASH_TYPE_L4;
837 tpa_info->gso_type = SKB_GSO_TCPV4;
838 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
839 if (hash_type == 3)
840 tpa_info->gso_type = SKB_GSO_TCPV6;
841 tpa_info->rss_hash =
842 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
843 } else {
844 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
845 tpa_info->gso_type = 0;
846 if (netif_msg_rx_err(bp))
847 netdev_warn(bp->dev, "TPA packet without valid hash\n");
848 }
849 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
850 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
851
852 rxr->rx_prod = NEXT_RX(prod);
853 cons = NEXT_RX(cons);
854 cons_rx_buf = &rxr->rx_buf_ring[cons];
855
856 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
857 rxr->rx_prod = NEXT_RX(rxr->rx_prod);
858 cons_rx_buf->data = NULL;
859 }
860
861 static void bnxt_abort_tpa(struct bnxt *bp, struct bnxt_napi *bnapi,
862 u16 cp_cons, u32 agg_bufs)
863 {
864 if (agg_bufs)
865 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
866 }
867
868 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr))
869 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
870
871 static inline struct sk_buff *bnxt_gro_skb(struct bnxt_tpa_info *tpa_info,
872 struct rx_tpa_end_cmp *tpa_end,
873 struct rx_tpa_end_cmp_ext *tpa_end1,
874 struct sk_buff *skb)
875 {
876 #ifdef CONFIG_INET
877 struct tcphdr *th;
878 int payload_off, tcp_opt_len = 0;
879 int len, nw_off;
880 u16 segs;
881
882 segs = TPA_END_TPA_SEGS(tpa_end);
883 if (segs == 1)
884 return skb;
885
886 NAPI_GRO_CB(skb)->count = segs;
887 skb_shinfo(skb)->gso_size =
888 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
889 skb_shinfo(skb)->gso_type = tpa_info->gso_type;
890 payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
891 RX_TPA_END_CMP_PAYLOAD_OFFSET) >>
892 RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT;
893 if (TPA_END_GRO_TS(tpa_end))
894 tcp_opt_len = 12;
895
896 if (tpa_info->gso_type == SKB_GSO_TCPV4) {
897 struct iphdr *iph;
898
899 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
900 ETH_HLEN;
901 skb_set_network_header(skb, nw_off);
902 iph = ip_hdr(skb);
903 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
904 len = skb->len - skb_transport_offset(skb);
905 th = tcp_hdr(skb);
906 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
907 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
908 struct ipv6hdr *iph;
909
910 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
911 ETH_HLEN;
912 skb_set_network_header(skb, nw_off);
913 iph = ipv6_hdr(skb);
914 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
915 len = skb->len - skb_transport_offset(skb);
916 th = tcp_hdr(skb);
917 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
918 } else {
919 dev_kfree_skb_any(skb);
920 return NULL;
921 }
922 tcp_gro_complete(skb);
923
924 if (nw_off) { /* tunnel */
925 struct udphdr *uh = NULL;
926
927 if (skb->protocol == htons(ETH_P_IP)) {
928 struct iphdr *iph = (struct iphdr *)skb->data;
929
930 if (iph->protocol == IPPROTO_UDP)
931 uh = (struct udphdr *)(iph + 1);
932 } else {
933 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
934
935 if (iph->nexthdr == IPPROTO_UDP)
936 uh = (struct udphdr *)(iph + 1);
937 }
938 if (uh) {
939 if (uh->check)
940 skb_shinfo(skb)->gso_type |=
941 SKB_GSO_UDP_TUNNEL_CSUM;
942 else
943 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
944 }
945 }
946 #endif
947 return skb;
948 }
949
950 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
951 struct bnxt_napi *bnapi,
952 u32 *raw_cons,
953 struct rx_tpa_end_cmp *tpa_end,
954 struct rx_tpa_end_cmp_ext *tpa_end1,
955 bool *agg_event)
956 {
957 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
958 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
959 u8 agg_id = TPA_END_AGG_ID(tpa_end);
960 u8 *data, agg_bufs;
961 u16 cp_cons = RING_CMP(*raw_cons);
962 unsigned int len;
963 struct bnxt_tpa_info *tpa_info;
964 dma_addr_t mapping;
965 struct sk_buff *skb;
966
967 tpa_info = &rxr->rx_tpa[agg_id];
968 data = tpa_info->data;
969 prefetch(data);
970 len = tpa_info->len;
971 mapping = tpa_info->mapping;
972
973 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
974 RX_TPA_END_CMP_AGG_BUFS) >> RX_TPA_END_CMP_AGG_BUFS_SHIFT;
975
976 if (agg_bufs) {
977 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
978 return ERR_PTR(-EBUSY);
979
980 *agg_event = true;
981 cp_cons = NEXT_CMP(cp_cons);
982 }
983
984 if (unlikely(agg_bufs > MAX_SKB_FRAGS)) {
985 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
986 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
987 agg_bufs, (int)MAX_SKB_FRAGS);
988 return NULL;
989 }
990
991 if (len <= bp->rx_copy_thresh) {
992 skb = bnxt_copy_skb(bnapi, data, len, mapping);
993 if (!skb) {
994 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
995 return NULL;
996 }
997 } else {
998 u8 *new_data;
999 dma_addr_t new_mapping;
1000
1001 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1002 if (!new_data) {
1003 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1004 return NULL;
1005 }
1006
1007 tpa_info->data = new_data;
1008 tpa_info->mapping = new_mapping;
1009
1010 skb = build_skb(data, 0);
1011 dma_unmap_single(&bp->pdev->dev, mapping, bp->rx_buf_use_size,
1012 PCI_DMA_FROMDEVICE);
1013
1014 if (!skb) {
1015 kfree(data);
1016 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1017 return NULL;
1018 }
1019 skb_reserve(skb, BNXT_RX_OFFSET);
1020 skb_put(skb, len);
1021 }
1022
1023 if (agg_bufs) {
1024 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1025 if (!skb) {
1026 /* Page reuse already handled by bnxt_rx_pages(). */
1027 return NULL;
1028 }
1029 }
1030 skb->protocol = eth_type_trans(skb, bp->dev);
1031
1032 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1033 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1034
1035 if (tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) {
1036 netdev_features_t features = skb->dev->features;
1037 u16 vlan_proto = tpa_info->metadata >>
1038 RX_CMP_FLAGS2_METADATA_TPID_SFT;
1039
1040 if (((features & NETIF_F_HW_VLAN_CTAG_RX) &&
1041 vlan_proto == ETH_P_8021Q) ||
1042 ((features & NETIF_F_HW_VLAN_STAG_RX) &&
1043 vlan_proto == ETH_P_8021AD)) {
1044 __vlan_hwaccel_put_tag(skb, htons(vlan_proto),
1045 tpa_info->metadata &
1046 RX_CMP_FLAGS2_METADATA_VID_MASK);
1047 }
1048 }
1049
1050 skb_checksum_none_assert(skb);
1051 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1052 skb->ip_summed = CHECKSUM_UNNECESSARY;
1053 skb->csum_level =
1054 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1055 }
1056
1057 if (TPA_END_GRO(tpa_end))
1058 skb = bnxt_gro_skb(tpa_info, tpa_end, tpa_end1, skb);
1059
1060 return skb;
1061 }
1062
1063 /* returns the following:
1064 * 1 - 1 packet successfully received
1065 * 0 - successful TPA_START, packet not completed yet
1066 * -EBUSY - completion ring does not have all the agg buffers yet
1067 * -ENOMEM - packet aborted due to out of memory
1068 * -EIO - packet aborted due to hw error indicated in BD
1069 */
1070 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
1071 bool *agg_event)
1072 {
1073 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1074 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1075 struct net_device *dev = bp->dev;
1076 struct rx_cmp *rxcmp;
1077 struct rx_cmp_ext *rxcmp1;
1078 u32 tmp_raw_cons = *raw_cons;
1079 u16 cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1080 struct bnxt_sw_rx_bd *rx_buf;
1081 unsigned int len;
1082 u8 *data, agg_bufs, cmp_type;
1083 dma_addr_t dma_addr;
1084 struct sk_buff *skb;
1085 int rc = 0;
1086
1087 rxcmp = (struct rx_cmp *)
1088 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1089
1090 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1091 cp_cons = RING_CMP(tmp_raw_cons);
1092 rxcmp1 = (struct rx_cmp_ext *)
1093 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1094
1095 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1096 return -EBUSY;
1097
1098 cmp_type = RX_CMP_TYPE(rxcmp);
1099
1100 prod = rxr->rx_prod;
1101
1102 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1103 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1104 (struct rx_tpa_start_cmp_ext *)rxcmp1);
1105
1106 goto next_rx_no_prod;
1107
1108 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1109 skb = bnxt_tpa_end(bp, bnapi, &tmp_raw_cons,
1110 (struct rx_tpa_end_cmp *)rxcmp,
1111 (struct rx_tpa_end_cmp_ext *)rxcmp1,
1112 agg_event);
1113
1114 if (unlikely(IS_ERR(skb)))
1115 return -EBUSY;
1116
1117 rc = -ENOMEM;
1118 if (likely(skb)) {
1119 skb_record_rx_queue(skb, bnapi->index);
1120 skb_mark_napi_id(skb, &bnapi->napi);
1121 if (bnxt_busy_polling(bnapi))
1122 netif_receive_skb(skb);
1123 else
1124 napi_gro_receive(&bnapi->napi, skb);
1125 rc = 1;
1126 }
1127 goto next_rx_no_prod;
1128 }
1129
1130 cons = rxcmp->rx_cmp_opaque;
1131 rx_buf = &rxr->rx_buf_ring[cons];
1132 data = rx_buf->data;
1133 prefetch(data);
1134
1135 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & RX_CMP_AGG_BUFS) >>
1136 RX_CMP_AGG_BUFS_SHIFT;
1137
1138 if (agg_bufs) {
1139 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1140 return -EBUSY;
1141
1142 cp_cons = NEXT_CMP(cp_cons);
1143 *agg_event = true;
1144 }
1145
1146 rx_buf->data = NULL;
1147 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1148 bnxt_reuse_rx_data(rxr, cons, data);
1149 if (agg_bufs)
1150 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
1151
1152 rc = -EIO;
1153 goto next_rx;
1154 }
1155
1156 len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1157 dma_addr = dma_unmap_addr(rx_buf, mapping);
1158
1159 if (len <= bp->rx_copy_thresh) {
1160 skb = bnxt_copy_skb(bnapi, data, len, dma_addr);
1161 bnxt_reuse_rx_data(rxr, cons, data);
1162 if (!skb) {
1163 rc = -ENOMEM;
1164 goto next_rx;
1165 }
1166 } else {
1167 skb = bnxt_rx_skb(bp, rxr, cons, prod, data, dma_addr, len);
1168 if (!skb) {
1169 rc = -ENOMEM;
1170 goto next_rx;
1171 }
1172 }
1173
1174 if (agg_bufs) {
1175 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1176 if (!skb) {
1177 rc = -ENOMEM;
1178 goto next_rx;
1179 }
1180 }
1181
1182 if (RX_CMP_HASH_VALID(rxcmp)) {
1183 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1184 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1185
1186 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1187 if (hash_type != 1 && hash_type != 3)
1188 type = PKT_HASH_TYPE_L3;
1189 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1190 }
1191
1192 skb->protocol = eth_type_trans(skb, dev);
1193
1194 if (rxcmp1->rx_cmp_flags2 &
1195 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) {
1196 netdev_features_t features = skb->dev->features;
1197 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1198 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1199
1200 if (((features & NETIF_F_HW_VLAN_CTAG_RX) &&
1201 vlan_proto == ETH_P_8021Q) ||
1202 ((features & NETIF_F_HW_VLAN_STAG_RX) &&
1203 vlan_proto == ETH_P_8021AD))
1204 __vlan_hwaccel_put_tag(skb, htons(vlan_proto),
1205 meta_data &
1206 RX_CMP_FLAGS2_METADATA_VID_MASK);
1207 }
1208
1209 skb_checksum_none_assert(skb);
1210 if (RX_CMP_L4_CS_OK(rxcmp1)) {
1211 if (dev->features & NETIF_F_RXCSUM) {
1212 skb->ip_summed = CHECKSUM_UNNECESSARY;
1213 skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1214 }
1215 } else {
1216 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1217 if (dev->features & NETIF_F_RXCSUM)
1218 cpr->rx_l4_csum_errors++;
1219 }
1220 }
1221
1222 skb_record_rx_queue(skb, bnapi->index);
1223 skb_mark_napi_id(skb, &bnapi->napi);
1224 if (bnxt_busy_polling(bnapi))
1225 netif_receive_skb(skb);
1226 else
1227 napi_gro_receive(&bnapi->napi, skb);
1228 rc = 1;
1229
1230 next_rx:
1231 rxr->rx_prod = NEXT_RX(prod);
1232
1233 next_rx_no_prod:
1234 *raw_cons = tmp_raw_cons;
1235
1236 return rc;
1237 }
1238
1239 static int bnxt_async_event_process(struct bnxt *bp,
1240 struct hwrm_async_event_cmpl *cmpl)
1241 {
1242 u16 event_id = le16_to_cpu(cmpl->event_id);
1243
1244 /* TODO CHIMP_FW: Define event id's for link change, error etc */
1245 switch (event_id) {
1246 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
1247 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
1248 break;
1249 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
1250 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
1251 break;
1252 default:
1253 netdev_err(bp->dev, "unhandled ASYNC event (id 0x%x)\n",
1254 event_id);
1255 goto async_event_process_exit;
1256 }
1257 schedule_work(&bp->sp_task);
1258 async_event_process_exit:
1259 return 0;
1260 }
1261
1262 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
1263 {
1264 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
1265 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
1266 struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
1267 (struct hwrm_fwd_req_cmpl *)txcmp;
1268
1269 switch (cmpl_type) {
1270 case CMPL_BASE_TYPE_HWRM_DONE:
1271 seq_id = le16_to_cpu(h_cmpl->sequence_id);
1272 if (seq_id == bp->hwrm_intr_seq_id)
1273 bp->hwrm_intr_seq_id = HWRM_SEQ_ID_INVALID;
1274 else
1275 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
1276 break;
1277
1278 case CMPL_BASE_TYPE_HWRM_FWD_REQ:
1279 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
1280
1281 if ((vf_id < bp->pf.first_vf_id) ||
1282 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
1283 netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
1284 vf_id);
1285 return -EINVAL;
1286 }
1287
1288 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
1289 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
1290 schedule_work(&bp->sp_task);
1291 break;
1292
1293 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
1294 bnxt_async_event_process(bp,
1295 (struct hwrm_async_event_cmpl *)txcmp);
1296
1297 default:
1298 break;
1299 }
1300
1301 return 0;
1302 }
1303
1304 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
1305 {
1306 struct bnxt_napi *bnapi = dev_instance;
1307 struct bnxt *bp = bnapi->bp;
1308 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1309 u32 cons = RING_CMP(cpr->cp_raw_cons);
1310
1311 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1312 napi_schedule(&bnapi->napi);
1313 return IRQ_HANDLED;
1314 }
1315
1316 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
1317 {
1318 u32 raw_cons = cpr->cp_raw_cons;
1319 u16 cons = RING_CMP(raw_cons);
1320 struct tx_cmp *txcmp;
1321
1322 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1323
1324 return TX_CMP_VALID(txcmp, raw_cons);
1325 }
1326
1327 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
1328 {
1329 struct bnxt_napi *bnapi = dev_instance;
1330 struct bnxt *bp = bnapi->bp;
1331 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1332 u32 cons = RING_CMP(cpr->cp_raw_cons);
1333 u32 int_status;
1334
1335 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1336
1337 if (!bnxt_has_work(bp, cpr)) {
1338 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
1339 /* return if erroneous interrupt */
1340 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
1341 return IRQ_NONE;
1342 }
1343
1344 /* disable ring IRQ */
1345 BNXT_CP_DB_IRQ_DIS(cpr->cp_doorbell);
1346
1347 /* Return here if interrupt is shared and is disabled. */
1348 if (unlikely(atomic_read(&bp->intr_sem) != 0))
1349 return IRQ_HANDLED;
1350
1351 napi_schedule(&bnapi->napi);
1352 return IRQ_HANDLED;
1353 }
1354
1355 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
1356 {
1357 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1358 u32 raw_cons = cpr->cp_raw_cons;
1359 u32 cons;
1360 int tx_pkts = 0;
1361 int rx_pkts = 0;
1362 bool rx_event = false;
1363 bool agg_event = false;
1364 struct tx_cmp *txcmp;
1365
1366 while (1) {
1367 int rc;
1368
1369 cons = RING_CMP(raw_cons);
1370 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1371
1372 if (!TX_CMP_VALID(txcmp, raw_cons))
1373 break;
1374
1375 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
1376 tx_pkts++;
1377 /* return full budget so NAPI will complete. */
1378 if (unlikely(tx_pkts > bp->tx_wake_thresh))
1379 rx_pkts = budget;
1380 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
1381 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &agg_event);
1382 if (likely(rc >= 0))
1383 rx_pkts += rc;
1384 else if (rc == -EBUSY) /* partial completion */
1385 break;
1386 rx_event = true;
1387 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
1388 CMPL_BASE_TYPE_HWRM_DONE) ||
1389 (TX_CMP_TYPE(txcmp) ==
1390 CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
1391 (TX_CMP_TYPE(txcmp) ==
1392 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
1393 bnxt_hwrm_handler(bp, txcmp);
1394 }
1395 raw_cons = NEXT_RAW_CMP(raw_cons);
1396
1397 if (rx_pkts == budget)
1398 break;
1399 }
1400
1401 cpr->cp_raw_cons = raw_cons;
1402 /* ACK completion ring before freeing tx ring and producing new
1403 * buffers in rx/agg rings to prevent overflowing the completion
1404 * ring.
1405 */
1406 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
1407
1408 if (tx_pkts)
1409 bnxt_tx_int(bp, bnapi, tx_pkts);
1410
1411 if (rx_event) {
1412 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1413
1414 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
1415 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
1416 if (agg_event) {
1417 writel(DB_KEY_RX | rxr->rx_agg_prod,
1418 rxr->rx_agg_doorbell);
1419 writel(DB_KEY_RX | rxr->rx_agg_prod,
1420 rxr->rx_agg_doorbell);
1421 }
1422 }
1423 return rx_pkts;
1424 }
1425
1426 static int bnxt_poll(struct napi_struct *napi, int budget)
1427 {
1428 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
1429 struct bnxt *bp = bnapi->bp;
1430 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1431 int work_done = 0;
1432
1433 if (!bnxt_lock_napi(bnapi))
1434 return budget;
1435
1436 while (1) {
1437 work_done += bnxt_poll_work(bp, bnapi, budget - work_done);
1438
1439 if (work_done >= budget)
1440 break;
1441
1442 if (!bnxt_has_work(bp, cpr)) {
1443 napi_complete(napi);
1444 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
1445 break;
1446 }
1447 }
1448 mmiowb();
1449 bnxt_unlock_napi(bnapi);
1450 return work_done;
1451 }
1452
1453 #ifdef CONFIG_NET_RX_BUSY_POLL
1454 static int bnxt_busy_poll(struct napi_struct *napi)
1455 {
1456 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
1457 struct bnxt *bp = bnapi->bp;
1458 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1459 int rx_work, budget = 4;
1460
1461 if (atomic_read(&bp->intr_sem) != 0)
1462 return LL_FLUSH_FAILED;
1463
1464 if (!bnxt_lock_poll(bnapi))
1465 return LL_FLUSH_BUSY;
1466
1467 rx_work = bnxt_poll_work(bp, bnapi, budget);
1468
1469 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
1470
1471 bnxt_unlock_poll(bnapi);
1472 return rx_work;
1473 }
1474 #endif
1475
1476 static void bnxt_free_tx_skbs(struct bnxt *bp)
1477 {
1478 int i, max_idx;
1479 struct pci_dev *pdev = bp->pdev;
1480
1481 if (!bp->tx_ring)
1482 return;
1483
1484 max_idx = bp->tx_nr_pages * TX_DESC_CNT;
1485 for (i = 0; i < bp->tx_nr_rings; i++) {
1486 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
1487 int j;
1488
1489 for (j = 0; j < max_idx;) {
1490 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
1491 struct sk_buff *skb = tx_buf->skb;
1492 int k, last;
1493
1494 if (!skb) {
1495 j++;
1496 continue;
1497 }
1498
1499 tx_buf->skb = NULL;
1500
1501 if (tx_buf->is_push) {
1502 dev_kfree_skb(skb);
1503 j += 2;
1504 continue;
1505 }
1506
1507 dma_unmap_single(&pdev->dev,
1508 dma_unmap_addr(tx_buf, mapping),
1509 skb_headlen(skb),
1510 PCI_DMA_TODEVICE);
1511
1512 last = tx_buf->nr_frags;
1513 j += 2;
1514 for (k = 0; k < last; k++, j++) {
1515 int ring_idx = j & bp->tx_ring_mask;
1516 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
1517
1518 tx_buf = &txr->tx_buf_ring[ring_idx];
1519 dma_unmap_page(
1520 &pdev->dev,
1521 dma_unmap_addr(tx_buf, mapping),
1522 skb_frag_size(frag), PCI_DMA_TODEVICE);
1523 }
1524 dev_kfree_skb(skb);
1525 }
1526 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
1527 }
1528 }
1529
1530 static void bnxt_free_rx_skbs(struct bnxt *bp)
1531 {
1532 int i, max_idx, max_agg_idx;
1533 struct pci_dev *pdev = bp->pdev;
1534
1535 if (!bp->rx_ring)
1536 return;
1537
1538 max_idx = bp->rx_nr_pages * RX_DESC_CNT;
1539 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
1540 for (i = 0; i < bp->rx_nr_rings; i++) {
1541 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
1542 int j;
1543
1544 if (rxr->rx_tpa) {
1545 for (j = 0; j < MAX_TPA; j++) {
1546 struct bnxt_tpa_info *tpa_info =
1547 &rxr->rx_tpa[j];
1548 u8 *data = tpa_info->data;
1549
1550 if (!data)
1551 continue;
1552
1553 dma_unmap_single(
1554 &pdev->dev,
1555 dma_unmap_addr(tpa_info, mapping),
1556 bp->rx_buf_use_size,
1557 PCI_DMA_FROMDEVICE);
1558
1559 tpa_info->data = NULL;
1560
1561 kfree(data);
1562 }
1563 }
1564
1565 for (j = 0; j < max_idx; j++) {
1566 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
1567 u8 *data = rx_buf->data;
1568
1569 if (!data)
1570 continue;
1571
1572 dma_unmap_single(&pdev->dev,
1573 dma_unmap_addr(rx_buf, mapping),
1574 bp->rx_buf_use_size,
1575 PCI_DMA_FROMDEVICE);
1576
1577 rx_buf->data = NULL;
1578
1579 kfree(data);
1580 }
1581
1582 for (j = 0; j < max_agg_idx; j++) {
1583 struct bnxt_sw_rx_agg_bd *rx_agg_buf =
1584 &rxr->rx_agg_ring[j];
1585 struct page *page = rx_agg_buf->page;
1586
1587 if (!page)
1588 continue;
1589
1590 dma_unmap_page(&pdev->dev,
1591 dma_unmap_addr(rx_agg_buf, mapping),
1592 PAGE_SIZE, PCI_DMA_FROMDEVICE);
1593
1594 rx_agg_buf->page = NULL;
1595 __clear_bit(j, rxr->rx_agg_bmap);
1596
1597 __free_page(page);
1598 }
1599 }
1600 }
1601
1602 static void bnxt_free_skbs(struct bnxt *bp)
1603 {
1604 bnxt_free_tx_skbs(bp);
1605 bnxt_free_rx_skbs(bp);
1606 }
1607
1608 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
1609 {
1610 struct pci_dev *pdev = bp->pdev;
1611 int i;
1612
1613 for (i = 0; i < ring->nr_pages; i++) {
1614 if (!ring->pg_arr[i])
1615 continue;
1616
1617 dma_free_coherent(&pdev->dev, ring->page_size,
1618 ring->pg_arr[i], ring->dma_arr[i]);
1619
1620 ring->pg_arr[i] = NULL;
1621 }
1622 if (ring->pg_tbl) {
1623 dma_free_coherent(&pdev->dev, ring->nr_pages * 8,
1624 ring->pg_tbl, ring->pg_tbl_map);
1625 ring->pg_tbl = NULL;
1626 }
1627 if (ring->vmem_size && *ring->vmem) {
1628 vfree(*ring->vmem);
1629 *ring->vmem = NULL;
1630 }
1631 }
1632
1633 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
1634 {
1635 int i;
1636 struct pci_dev *pdev = bp->pdev;
1637
1638 if (ring->nr_pages > 1) {
1639 ring->pg_tbl = dma_alloc_coherent(&pdev->dev,
1640 ring->nr_pages * 8,
1641 &ring->pg_tbl_map,
1642 GFP_KERNEL);
1643 if (!ring->pg_tbl)
1644 return -ENOMEM;
1645 }
1646
1647 for (i = 0; i < ring->nr_pages; i++) {
1648 ring->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
1649 ring->page_size,
1650 &ring->dma_arr[i],
1651 GFP_KERNEL);
1652 if (!ring->pg_arr[i])
1653 return -ENOMEM;
1654
1655 if (ring->nr_pages > 1)
1656 ring->pg_tbl[i] = cpu_to_le64(ring->dma_arr[i]);
1657 }
1658
1659 if (ring->vmem_size) {
1660 *ring->vmem = vzalloc(ring->vmem_size);
1661 if (!(*ring->vmem))
1662 return -ENOMEM;
1663 }
1664 return 0;
1665 }
1666
1667 static void bnxt_free_rx_rings(struct bnxt *bp)
1668 {
1669 int i;
1670
1671 if (!bp->rx_ring)
1672 return;
1673
1674 for (i = 0; i < bp->rx_nr_rings; i++) {
1675 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
1676 struct bnxt_ring_struct *ring;
1677
1678 kfree(rxr->rx_tpa);
1679 rxr->rx_tpa = NULL;
1680
1681 kfree(rxr->rx_agg_bmap);
1682 rxr->rx_agg_bmap = NULL;
1683
1684 ring = &rxr->rx_ring_struct;
1685 bnxt_free_ring(bp, ring);
1686
1687 ring = &rxr->rx_agg_ring_struct;
1688 bnxt_free_ring(bp, ring);
1689 }
1690 }
1691
1692 static int bnxt_alloc_rx_rings(struct bnxt *bp)
1693 {
1694 int i, rc, agg_rings = 0, tpa_rings = 0;
1695
1696 if (!bp->rx_ring)
1697 return -ENOMEM;
1698
1699 if (bp->flags & BNXT_FLAG_AGG_RINGS)
1700 agg_rings = 1;
1701
1702 if (bp->flags & BNXT_FLAG_TPA)
1703 tpa_rings = 1;
1704
1705 for (i = 0; i < bp->rx_nr_rings; i++) {
1706 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
1707 struct bnxt_ring_struct *ring;
1708
1709 ring = &rxr->rx_ring_struct;
1710
1711 rc = bnxt_alloc_ring(bp, ring);
1712 if (rc)
1713 return rc;
1714
1715 if (agg_rings) {
1716 u16 mem_size;
1717
1718 ring = &rxr->rx_agg_ring_struct;
1719 rc = bnxt_alloc_ring(bp, ring);
1720 if (rc)
1721 return rc;
1722
1723 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
1724 mem_size = rxr->rx_agg_bmap_size / 8;
1725 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
1726 if (!rxr->rx_agg_bmap)
1727 return -ENOMEM;
1728
1729 if (tpa_rings) {
1730 rxr->rx_tpa = kcalloc(MAX_TPA,
1731 sizeof(struct bnxt_tpa_info),
1732 GFP_KERNEL);
1733 if (!rxr->rx_tpa)
1734 return -ENOMEM;
1735 }
1736 }
1737 }
1738 return 0;
1739 }
1740
1741 static void bnxt_free_tx_rings(struct bnxt *bp)
1742 {
1743 int i;
1744 struct pci_dev *pdev = bp->pdev;
1745
1746 if (!bp->tx_ring)
1747 return;
1748
1749 for (i = 0; i < bp->tx_nr_rings; i++) {
1750 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
1751 struct bnxt_ring_struct *ring;
1752
1753 if (txr->tx_push) {
1754 dma_free_coherent(&pdev->dev, bp->tx_push_size,
1755 txr->tx_push, txr->tx_push_mapping);
1756 txr->tx_push = NULL;
1757 }
1758
1759 ring = &txr->tx_ring_struct;
1760
1761 bnxt_free_ring(bp, ring);
1762 }
1763 }
1764
1765 static int bnxt_alloc_tx_rings(struct bnxt *bp)
1766 {
1767 int i, j, rc;
1768 struct pci_dev *pdev = bp->pdev;
1769
1770 bp->tx_push_size = 0;
1771 if (bp->tx_push_thresh) {
1772 int push_size;
1773
1774 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
1775 bp->tx_push_thresh);
1776
1777 if (push_size > 256) {
1778 push_size = 0;
1779 bp->tx_push_thresh = 0;
1780 }
1781
1782 bp->tx_push_size = push_size;
1783 }
1784
1785 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
1786 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
1787 struct bnxt_ring_struct *ring;
1788
1789 ring = &txr->tx_ring_struct;
1790
1791 rc = bnxt_alloc_ring(bp, ring);
1792 if (rc)
1793 return rc;
1794
1795 if (bp->tx_push_size) {
1796 dma_addr_t mapping;
1797
1798 /* One pre-allocated DMA buffer to backup
1799 * TX push operation
1800 */
1801 txr->tx_push = dma_alloc_coherent(&pdev->dev,
1802 bp->tx_push_size,
1803 &txr->tx_push_mapping,
1804 GFP_KERNEL);
1805
1806 if (!txr->tx_push)
1807 return -ENOMEM;
1808
1809 mapping = txr->tx_push_mapping +
1810 sizeof(struct tx_push_bd);
1811 txr->data_mapping = cpu_to_le64(mapping);
1812
1813 memset(txr->tx_push, 0, sizeof(struct tx_push_bd));
1814 }
1815 ring->queue_id = bp->q_info[j].queue_id;
1816 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
1817 j++;
1818 }
1819 return 0;
1820 }
1821
1822 static void bnxt_free_cp_rings(struct bnxt *bp)
1823 {
1824 int i;
1825
1826 if (!bp->bnapi)
1827 return;
1828
1829 for (i = 0; i < bp->cp_nr_rings; i++) {
1830 struct bnxt_napi *bnapi = bp->bnapi[i];
1831 struct bnxt_cp_ring_info *cpr;
1832 struct bnxt_ring_struct *ring;
1833
1834 if (!bnapi)
1835 continue;
1836
1837 cpr = &bnapi->cp_ring;
1838 ring = &cpr->cp_ring_struct;
1839
1840 bnxt_free_ring(bp, ring);
1841 }
1842 }
1843
1844 static int bnxt_alloc_cp_rings(struct bnxt *bp)
1845 {
1846 int i, rc;
1847
1848 for (i = 0; i < bp->cp_nr_rings; i++) {
1849 struct bnxt_napi *bnapi = bp->bnapi[i];
1850 struct bnxt_cp_ring_info *cpr;
1851 struct bnxt_ring_struct *ring;
1852
1853 if (!bnapi)
1854 continue;
1855
1856 cpr = &bnapi->cp_ring;
1857 ring = &cpr->cp_ring_struct;
1858
1859 rc = bnxt_alloc_ring(bp, ring);
1860 if (rc)
1861 return rc;
1862 }
1863 return 0;
1864 }
1865
1866 static void bnxt_init_ring_struct(struct bnxt *bp)
1867 {
1868 int i;
1869
1870 for (i = 0; i < bp->cp_nr_rings; i++) {
1871 struct bnxt_napi *bnapi = bp->bnapi[i];
1872 struct bnxt_cp_ring_info *cpr;
1873 struct bnxt_rx_ring_info *rxr;
1874 struct bnxt_tx_ring_info *txr;
1875 struct bnxt_ring_struct *ring;
1876
1877 if (!bnapi)
1878 continue;
1879
1880 cpr = &bnapi->cp_ring;
1881 ring = &cpr->cp_ring_struct;
1882 ring->nr_pages = bp->cp_nr_pages;
1883 ring->page_size = HW_CMPD_RING_SIZE;
1884 ring->pg_arr = (void **)cpr->cp_desc_ring;
1885 ring->dma_arr = cpr->cp_desc_mapping;
1886 ring->vmem_size = 0;
1887
1888 rxr = bnapi->rx_ring;
1889 if (!rxr)
1890 goto skip_rx;
1891
1892 ring = &rxr->rx_ring_struct;
1893 ring->nr_pages = bp->rx_nr_pages;
1894 ring->page_size = HW_RXBD_RING_SIZE;
1895 ring->pg_arr = (void **)rxr->rx_desc_ring;
1896 ring->dma_arr = rxr->rx_desc_mapping;
1897 ring->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
1898 ring->vmem = (void **)&rxr->rx_buf_ring;
1899
1900 ring = &rxr->rx_agg_ring_struct;
1901 ring->nr_pages = bp->rx_agg_nr_pages;
1902 ring->page_size = HW_RXBD_RING_SIZE;
1903 ring->pg_arr = (void **)rxr->rx_agg_desc_ring;
1904 ring->dma_arr = rxr->rx_agg_desc_mapping;
1905 ring->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
1906 ring->vmem = (void **)&rxr->rx_agg_ring;
1907
1908 skip_rx:
1909 txr = bnapi->tx_ring;
1910 if (!txr)
1911 continue;
1912
1913 ring = &txr->tx_ring_struct;
1914 ring->nr_pages = bp->tx_nr_pages;
1915 ring->page_size = HW_RXBD_RING_SIZE;
1916 ring->pg_arr = (void **)txr->tx_desc_ring;
1917 ring->dma_arr = txr->tx_desc_mapping;
1918 ring->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
1919 ring->vmem = (void **)&txr->tx_buf_ring;
1920 }
1921 }
1922
1923 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
1924 {
1925 int i;
1926 u32 prod;
1927 struct rx_bd **rx_buf_ring;
1928
1929 rx_buf_ring = (struct rx_bd **)ring->pg_arr;
1930 for (i = 0, prod = 0; i < ring->nr_pages; i++) {
1931 int j;
1932 struct rx_bd *rxbd;
1933
1934 rxbd = rx_buf_ring[i];
1935 if (!rxbd)
1936 continue;
1937
1938 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
1939 rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
1940 rxbd->rx_bd_opaque = prod;
1941 }
1942 }
1943 }
1944
1945 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
1946 {
1947 struct net_device *dev = bp->dev;
1948 struct bnxt_rx_ring_info *rxr;
1949 struct bnxt_ring_struct *ring;
1950 u32 prod, type;
1951 int i;
1952
1953 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
1954 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
1955
1956 if (NET_IP_ALIGN == 2)
1957 type |= RX_BD_FLAGS_SOP;
1958
1959 rxr = &bp->rx_ring[ring_nr];
1960 ring = &rxr->rx_ring_struct;
1961 bnxt_init_rxbd_pages(ring, type);
1962
1963 prod = rxr->rx_prod;
1964 for (i = 0; i < bp->rx_ring_size; i++) {
1965 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
1966 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
1967 ring_nr, i, bp->rx_ring_size);
1968 break;
1969 }
1970 prod = NEXT_RX(prod);
1971 }
1972 rxr->rx_prod = prod;
1973 ring->fw_ring_id = INVALID_HW_RING_ID;
1974
1975 ring = &rxr->rx_agg_ring_struct;
1976 ring->fw_ring_id = INVALID_HW_RING_ID;
1977
1978 if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
1979 return 0;
1980
1981 type = ((u32)PAGE_SIZE << RX_BD_LEN_SHIFT) |
1982 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
1983
1984 bnxt_init_rxbd_pages(ring, type);
1985
1986 prod = rxr->rx_agg_prod;
1987 for (i = 0; i < bp->rx_agg_ring_size; i++) {
1988 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
1989 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
1990 ring_nr, i, bp->rx_ring_size);
1991 break;
1992 }
1993 prod = NEXT_RX_AGG(prod);
1994 }
1995 rxr->rx_agg_prod = prod;
1996
1997 if (bp->flags & BNXT_FLAG_TPA) {
1998 if (rxr->rx_tpa) {
1999 u8 *data;
2000 dma_addr_t mapping;
2001
2002 for (i = 0; i < MAX_TPA; i++) {
2003 data = __bnxt_alloc_rx_data(bp, &mapping,
2004 GFP_KERNEL);
2005 if (!data)
2006 return -ENOMEM;
2007
2008 rxr->rx_tpa[i].data = data;
2009 rxr->rx_tpa[i].mapping = mapping;
2010 }
2011 } else {
2012 netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
2013 return -ENOMEM;
2014 }
2015 }
2016
2017 return 0;
2018 }
2019
2020 static int bnxt_init_rx_rings(struct bnxt *bp)
2021 {
2022 int i, rc = 0;
2023
2024 for (i = 0; i < bp->rx_nr_rings; i++) {
2025 rc = bnxt_init_one_rx_ring(bp, i);
2026 if (rc)
2027 break;
2028 }
2029
2030 return rc;
2031 }
2032
2033 static int bnxt_init_tx_rings(struct bnxt *bp)
2034 {
2035 u16 i;
2036
2037 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
2038 MAX_SKB_FRAGS + 1);
2039
2040 for (i = 0; i < bp->tx_nr_rings; i++) {
2041 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2042 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
2043
2044 ring->fw_ring_id = INVALID_HW_RING_ID;
2045 }
2046
2047 return 0;
2048 }
2049
2050 static void bnxt_free_ring_grps(struct bnxt *bp)
2051 {
2052 kfree(bp->grp_info);
2053 bp->grp_info = NULL;
2054 }
2055
2056 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
2057 {
2058 int i;
2059
2060 if (irq_re_init) {
2061 bp->grp_info = kcalloc(bp->cp_nr_rings,
2062 sizeof(struct bnxt_ring_grp_info),
2063 GFP_KERNEL);
2064 if (!bp->grp_info)
2065 return -ENOMEM;
2066 }
2067 for (i = 0; i < bp->cp_nr_rings; i++) {
2068 if (irq_re_init)
2069 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
2070 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
2071 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
2072 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
2073 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
2074 }
2075 return 0;
2076 }
2077
2078 static void bnxt_free_vnics(struct bnxt *bp)
2079 {
2080 kfree(bp->vnic_info);
2081 bp->vnic_info = NULL;
2082 bp->nr_vnics = 0;
2083 }
2084
2085 static int bnxt_alloc_vnics(struct bnxt *bp)
2086 {
2087 int num_vnics = 1;
2088
2089 #ifdef CONFIG_RFS_ACCEL
2090 if (bp->flags & BNXT_FLAG_RFS)
2091 num_vnics += bp->rx_nr_rings;
2092 #endif
2093
2094 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
2095 GFP_KERNEL);
2096 if (!bp->vnic_info)
2097 return -ENOMEM;
2098
2099 bp->nr_vnics = num_vnics;
2100 return 0;
2101 }
2102
2103 static void bnxt_init_vnics(struct bnxt *bp)
2104 {
2105 int i;
2106
2107 for (i = 0; i < bp->nr_vnics; i++) {
2108 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
2109
2110 vnic->fw_vnic_id = INVALID_HW_RING_ID;
2111 vnic->fw_rss_cos_lb_ctx = INVALID_HW_RING_ID;
2112 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
2113
2114 if (bp->vnic_info[i].rss_hash_key) {
2115 if (i == 0)
2116 prandom_bytes(vnic->rss_hash_key,
2117 HW_HASH_KEY_SIZE);
2118 else
2119 memcpy(vnic->rss_hash_key,
2120 bp->vnic_info[0].rss_hash_key,
2121 HW_HASH_KEY_SIZE);
2122 }
2123 }
2124 }
2125
2126 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
2127 {
2128 int pages;
2129
2130 pages = ring_size / desc_per_pg;
2131
2132 if (!pages)
2133 return 1;
2134
2135 pages++;
2136
2137 while (pages & (pages - 1))
2138 pages++;
2139
2140 return pages;
2141 }
2142
2143 static void bnxt_set_tpa_flags(struct bnxt *bp)
2144 {
2145 bp->flags &= ~BNXT_FLAG_TPA;
2146 if (bp->dev->features & NETIF_F_LRO)
2147 bp->flags |= BNXT_FLAG_LRO;
2148 if ((bp->dev->features & NETIF_F_GRO) && (bp->pdev->revision > 0))
2149 bp->flags |= BNXT_FLAG_GRO;
2150 }
2151
2152 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
2153 * be set on entry.
2154 */
2155 void bnxt_set_ring_params(struct bnxt *bp)
2156 {
2157 u32 ring_size, rx_size, rx_space;
2158 u32 agg_factor = 0, agg_ring_size = 0;
2159
2160 /* 8 for CRC and VLAN */
2161 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
2162
2163 rx_space = rx_size + NET_SKB_PAD +
2164 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2165
2166 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
2167 ring_size = bp->rx_ring_size;
2168 bp->rx_agg_ring_size = 0;
2169 bp->rx_agg_nr_pages = 0;
2170
2171 if (bp->flags & BNXT_FLAG_TPA)
2172 agg_factor = 4;
2173
2174 bp->flags &= ~BNXT_FLAG_JUMBO;
2175 if (rx_space > PAGE_SIZE) {
2176 u32 jumbo_factor;
2177
2178 bp->flags |= BNXT_FLAG_JUMBO;
2179 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
2180 if (jumbo_factor > agg_factor)
2181 agg_factor = jumbo_factor;
2182 }
2183 agg_ring_size = ring_size * agg_factor;
2184
2185 if (agg_ring_size) {
2186 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
2187 RX_DESC_CNT);
2188 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
2189 u32 tmp = agg_ring_size;
2190
2191 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
2192 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
2193 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
2194 tmp, agg_ring_size);
2195 }
2196 bp->rx_agg_ring_size = agg_ring_size;
2197 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
2198 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
2199 rx_space = rx_size + NET_SKB_PAD +
2200 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2201 }
2202
2203 bp->rx_buf_use_size = rx_size;
2204 bp->rx_buf_size = rx_space;
2205
2206 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
2207 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
2208
2209 ring_size = bp->tx_ring_size;
2210 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
2211 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
2212
2213 ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size;
2214 bp->cp_ring_size = ring_size;
2215
2216 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
2217 if (bp->cp_nr_pages > MAX_CP_PAGES) {
2218 bp->cp_nr_pages = MAX_CP_PAGES;
2219 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
2220 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
2221 ring_size, bp->cp_ring_size);
2222 }
2223 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
2224 bp->cp_ring_mask = bp->cp_bit - 1;
2225 }
2226
2227 static void bnxt_free_vnic_attributes(struct bnxt *bp)
2228 {
2229 int i;
2230 struct bnxt_vnic_info *vnic;
2231 struct pci_dev *pdev = bp->pdev;
2232
2233 if (!bp->vnic_info)
2234 return;
2235
2236 for (i = 0; i < bp->nr_vnics; i++) {
2237 vnic = &bp->vnic_info[i];
2238
2239 kfree(vnic->fw_grp_ids);
2240 vnic->fw_grp_ids = NULL;
2241
2242 kfree(vnic->uc_list);
2243 vnic->uc_list = NULL;
2244
2245 if (vnic->mc_list) {
2246 dma_free_coherent(&pdev->dev, vnic->mc_list_size,
2247 vnic->mc_list, vnic->mc_list_mapping);
2248 vnic->mc_list = NULL;
2249 }
2250
2251 if (vnic->rss_table) {
2252 dma_free_coherent(&pdev->dev, PAGE_SIZE,
2253 vnic->rss_table,
2254 vnic->rss_table_dma_addr);
2255 vnic->rss_table = NULL;
2256 }
2257
2258 vnic->rss_hash_key = NULL;
2259 vnic->flags = 0;
2260 }
2261 }
2262
2263 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
2264 {
2265 int i, rc = 0, size;
2266 struct bnxt_vnic_info *vnic;
2267 struct pci_dev *pdev = bp->pdev;
2268 int max_rings;
2269
2270 for (i = 0; i < bp->nr_vnics; i++) {
2271 vnic = &bp->vnic_info[i];
2272
2273 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
2274 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
2275
2276 if (mem_size > 0) {
2277 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
2278 if (!vnic->uc_list) {
2279 rc = -ENOMEM;
2280 goto out;
2281 }
2282 }
2283 }
2284
2285 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
2286 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
2287 vnic->mc_list =
2288 dma_alloc_coherent(&pdev->dev,
2289 vnic->mc_list_size,
2290 &vnic->mc_list_mapping,
2291 GFP_KERNEL);
2292 if (!vnic->mc_list) {
2293 rc = -ENOMEM;
2294 goto out;
2295 }
2296 }
2297
2298 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
2299 max_rings = bp->rx_nr_rings;
2300 else
2301 max_rings = 1;
2302
2303 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
2304 if (!vnic->fw_grp_ids) {
2305 rc = -ENOMEM;
2306 goto out;
2307 }
2308
2309 /* Allocate rss table and hash key */
2310 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
2311 &vnic->rss_table_dma_addr,
2312 GFP_KERNEL);
2313 if (!vnic->rss_table) {
2314 rc = -ENOMEM;
2315 goto out;
2316 }
2317
2318 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
2319
2320 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
2321 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
2322 }
2323 return 0;
2324
2325 out:
2326 return rc;
2327 }
2328
2329 static void bnxt_free_hwrm_resources(struct bnxt *bp)
2330 {
2331 struct pci_dev *pdev = bp->pdev;
2332
2333 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
2334 bp->hwrm_cmd_resp_dma_addr);
2335
2336 bp->hwrm_cmd_resp_addr = NULL;
2337 if (bp->hwrm_dbg_resp_addr) {
2338 dma_free_coherent(&pdev->dev, HWRM_DBG_REG_BUF_SIZE,
2339 bp->hwrm_dbg_resp_addr,
2340 bp->hwrm_dbg_resp_dma_addr);
2341
2342 bp->hwrm_dbg_resp_addr = NULL;
2343 }
2344 }
2345
2346 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
2347 {
2348 struct pci_dev *pdev = bp->pdev;
2349
2350 bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
2351 &bp->hwrm_cmd_resp_dma_addr,
2352 GFP_KERNEL);
2353 if (!bp->hwrm_cmd_resp_addr)
2354 return -ENOMEM;
2355 bp->hwrm_dbg_resp_addr = dma_alloc_coherent(&pdev->dev,
2356 HWRM_DBG_REG_BUF_SIZE,
2357 &bp->hwrm_dbg_resp_dma_addr,
2358 GFP_KERNEL);
2359 if (!bp->hwrm_dbg_resp_addr)
2360 netdev_warn(bp->dev, "fail to alloc debug register dma mem\n");
2361
2362 return 0;
2363 }
2364
2365 static void bnxt_free_stats(struct bnxt *bp)
2366 {
2367 u32 size, i;
2368 struct pci_dev *pdev = bp->pdev;
2369
2370 if (bp->hw_rx_port_stats) {
2371 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size,
2372 bp->hw_rx_port_stats,
2373 bp->hw_rx_port_stats_map);
2374 bp->hw_rx_port_stats = NULL;
2375 bp->flags &= ~BNXT_FLAG_PORT_STATS;
2376 }
2377
2378 if (!bp->bnapi)
2379 return;
2380
2381 size = sizeof(struct ctx_hw_stats);
2382
2383 for (i = 0; i < bp->cp_nr_rings; i++) {
2384 struct bnxt_napi *bnapi = bp->bnapi[i];
2385 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2386
2387 if (cpr->hw_stats) {
2388 dma_free_coherent(&pdev->dev, size, cpr->hw_stats,
2389 cpr->hw_stats_map);
2390 cpr->hw_stats = NULL;
2391 }
2392 }
2393 }
2394
2395 static int bnxt_alloc_stats(struct bnxt *bp)
2396 {
2397 u32 size, i;
2398 struct pci_dev *pdev = bp->pdev;
2399
2400 size = sizeof(struct ctx_hw_stats);
2401
2402 for (i = 0; i < bp->cp_nr_rings; i++) {
2403 struct bnxt_napi *bnapi = bp->bnapi[i];
2404 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2405
2406 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size,
2407 &cpr->hw_stats_map,
2408 GFP_KERNEL);
2409 if (!cpr->hw_stats)
2410 return -ENOMEM;
2411
2412 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
2413 }
2414
2415 if (BNXT_PF(bp)) {
2416 bp->hw_port_stats_size = sizeof(struct rx_port_stats) +
2417 sizeof(struct tx_port_stats) + 1024;
2418
2419 bp->hw_rx_port_stats =
2420 dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size,
2421 &bp->hw_rx_port_stats_map,
2422 GFP_KERNEL);
2423 if (!bp->hw_rx_port_stats)
2424 return -ENOMEM;
2425
2426 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) +
2427 512;
2428 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map +
2429 sizeof(struct rx_port_stats) + 512;
2430 bp->flags |= BNXT_FLAG_PORT_STATS;
2431 }
2432 return 0;
2433 }
2434
2435 static void bnxt_clear_ring_indices(struct bnxt *bp)
2436 {
2437 int i;
2438
2439 if (!bp->bnapi)
2440 return;
2441
2442 for (i = 0; i < bp->cp_nr_rings; i++) {
2443 struct bnxt_napi *bnapi = bp->bnapi[i];
2444 struct bnxt_cp_ring_info *cpr;
2445 struct bnxt_rx_ring_info *rxr;
2446 struct bnxt_tx_ring_info *txr;
2447
2448 if (!bnapi)
2449 continue;
2450
2451 cpr = &bnapi->cp_ring;
2452 cpr->cp_raw_cons = 0;
2453
2454 txr = bnapi->tx_ring;
2455 if (txr) {
2456 txr->tx_prod = 0;
2457 txr->tx_cons = 0;
2458 }
2459
2460 rxr = bnapi->rx_ring;
2461 if (rxr) {
2462 rxr->rx_prod = 0;
2463 rxr->rx_agg_prod = 0;
2464 rxr->rx_sw_agg_prod = 0;
2465 }
2466 }
2467 }
2468
2469 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
2470 {
2471 #ifdef CONFIG_RFS_ACCEL
2472 int i;
2473
2474 /* Under rtnl_lock and all our NAPIs have been disabled. It's
2475 * safe to delete the hash table.
2476 */
2477 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
2478 struct hlist_head *head;
2479 struct hlist_node *tmp;
2480 struct bnxt_ntuple_filter *fltr;
2481
2482 head = &bp->ntp_fltr_hash_tbl[i];
2483 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
2484 hlist_del(&fltr->hash);
2485 kfree(fltr);
2486 }
2487 }
2488 if (irq_reinit) {
2489 kfree(bp->ntp_fltr_bmap);
2490 bp->ntp_fltr_bmap = NULL;
2491 }
2492 bp->ntp_fltr_count = 0;
2493 #endif
2494 }
2495
2496 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
2497 {
2498 #ifdef CONFIG_RFS_ACCEL
2499 int i, rc = 0;
2500
2501 if (!(bp->flags & BNXT_FLAG_RFS))
2502 return 0;
2503
2504 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
2505 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
2506
2507 bp->ntp_fltr_count = 0;
2508 bp->ntp_fltr_bmap = kzalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
2509 GFP_KERNEL);
2510
2511 if (!bp->ntp_fltr_bmap)
2512 rc = -ENOMEM;
2513
2514 return rc;
2515 #else
2516 return 0;
2517 #endif
2518 }
2519
2520 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
2521 {
2522 bnxt_free_vnic_attributes(bp);
2523 bnxt_free_tx_rings(bp);
2524 bnxt_free_rx_rings(bp);
2525 bnxt_free_cp_rings(bp);
2526 bnxt_free_ntp_fltrs(bp, irq_re_init);
2527 if (irq_re_init) {
2528 bnxt_free_stats(bp);
2529 bnxt_free_ring_grps(bp);
2530 bnxt_free_vnics(bp);
2531 kfree(bp->tx_ring);
2532 bp->tx_ring = NULL;
2533 kfree(bp->rx_ring);
2534 bp->rx_ring = NULL;
2535 kfree(bp->bnapi);
2536 bp->bnapi = NULL;
2537 } else {
2538 bnxt_clear_ring_indices(bp);
2539 }
2540 }
2541
2542 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
2543 {
2544 int i, j, rc, size, arr_size;
2545 void *bnapi;
2546
2547 if (irq_re_init) {
2548 /* Allocate bnapi mem pointer array and mem block for
2549 * all queues
2550 */
2551 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
2552 bp->cp_nr_rings);
2553 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
2554 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
2555 if (!bnapi)
2556 return -ENOMEM;
2557
2558 bp->bnapi = bnapi;
2559 bnapi += arr_size;
2560 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
2561 bp->bnapi[i] = bnapi;
2562 bp->bnapi[i]->index = i;
2563 bp->bnapi[i]->bp = bp;
2564 }
2565
2566 bp->rx_ring = kcalloc(bp->rx_nr_rings,
2567 sizeof(struct bnxt_rx_ring_info),
2568 GFP_KERNEL);
2569 if (!bp->rx_ring)
2570 return -ENOMEM;
2571
2572 for (i = 0; i < bp->rx_nr_rings; i++) {
2573 bp->rx_ring[i].bnapi = bp->bnapi[i];
2574 bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
2575 }
2576
2577 bp->tx_ring = kcalloc(bp->tx_nr_rings,
2578 sizeof(struct bnxt_tx_ring_info),
2579 GFP_KERNEL);
2580 if (!bp->tx_ring)
2581 return -ENOMEM;
2582
2583 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
2584 j = 0;
2585 else
2586 j = bp->rx_nr_rings;
2587
2588 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
2589 bp->tx_ring[i].bnapi = bp->bnapi[j];
2590 bp->bnapi[j]->tx_ring = &bp->tx_ring[i];
2591 }
2592
2593 rc = bnxt_alloc_stats(bp);
2594 if (rc)
2595 goto alloc_mem_err;
2596
2597 rc = bnxt_alloc_ntp_fltrs(bp);
2598 if (rc)
2599 goto alloc_mem_err;
2600
2601 rc = bnxt_alloc_vnics(bp);
2602 if (rc)
2603 goto alloc_mem_err;
2604 }
2605
2606 bnxt_init_ring_struct(bp);
2607
2608 rc = bnxt_alloc_rx_rings(bp);
2609 if (rc)
2610 goto alloc_mem_err;
2611
2612 rc = bnxt_alloc_tx_rings(bp);
2613 if (rc)
2614 goto alloc_mem_err;
2615
2616 rc = bnxt_alloc_cp_rings(bp);
2617 if (rc)
2618 goto alloc_mem_err;
2619
2620 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
2621 BNXT_VNIC_UCAST_FLAG;
2622 rc = bnxt_alloc_vnic_attributes(bp);
2623 if (rc)
2624 goto alloc_mem_err;
2625 return 0;
2626
2627 alloc_mem_err:
2628 bnxt_free_mem(bp, true);
2629 return rc;
2630 }
2631
2632 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
2633 u16 cmpl_ring, u16 target_id)
2634 {
2635 struct input *req = request;
2636
2637 req->req_type = cpu_to_le16(req_type);
2638 req->cmpl_ring = cpu_to_le16(cmpl_ring);
2639 req->target_id = cpu_to_le16(target_id);
2640 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
2641 }
2642
2643 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
2644 int timeout, bool silent)
2645 {
2646 int i, intr_process, rc;
2647 struct input *req = msg;
2648 u32 *data = msg;
2649 __le32 *resp_len, *valid;
2650 u16 cp_ring_id, len = 0;
2651 struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
2652
2653 req->seq_id = cpu_to_le16(bp->hwrm_cmd_seq++);
2654 memset(resp, 0, PAGE_SIZE);
2655 cp_ring_id = le16_to_cpu(req->cmpl_ring);
2656 intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
2657
2658 /* Write request msg to hwrm channel */
2659 __iowrite32_copy(bp->bar0, data, msg_len / 4);
2660
2661 for (i = msg_len; i < BNXT_HWRM_MAX_REQ_LEN; i += 4)
2662 writel(0, bp->bar0 + i);
2663
2664 /* currently supports only one outstanding message */
2665 if (intr_process)
2666 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
2667
2668 /* Ring channel doorbell */
2669 writel(1, bp->bar0 + 0x100);
2670
2671 if (!timeout)
2672 timeout = DFLT_HWRM_CMD_TIMEOUT;
2673
2674 i = 0;
2675 if (intr_process) {
2676 /* Wait until hwrm response cmpl interrupt is processed */
2677 while (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID &&
2678 i++ < timeout) {
2679 usleep_range(600, 800);
2680 }
2681
2682 if (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID) {
2683 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
2684 le16_to_cpu(req->req_type));
2685 return -1;
2686 }
2687 } else {
2688 /* Check if response len is updated */
2689 resp_len = bp->hwrm_cmd_resp_addr + HWRM_RESP_LEN_OFFSET;
2690 for (i = 0; i < timeout; i++) {
2691 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
2692 HWRM_RESP_LEN_SFT;
2693 if (len)
2694 break;
2695 usleep_range(600, 800);
2696 }
2697
2698 if (i >= timeout) {
2699 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
2700 timeout, le16_to_cpu(req->req_type),
2701 le16_to_cpu(req->seq_id), *resp_len);
2702 return -1;
2703 }
2704
2705 /* Last word of resp contains valid bit */
2706 valid = bp->hwrm_cmd_resp_addr + len - 4;
2707 for (i = 0; i < timeout; i++) {
2708 if (le32_to_cpu(*valid) & HWRM_RESP_VALID_MASK)
2709 break;
2710 usleep_range(600, 800);
2711 }
2712
2713 if (i >= timeout) {
2714 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
2715 timeout, le16_to_cpu(req->req_type),
2716 le16_to_cpu(req->seq_id), len, *valid);
2717 return -1;
2718 }
2719 }
2720
2721 rc = le16_to_cpu(resp->error_code);
2722 if (rc && !silent)
2723 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
2724 le16_to_cpu(resp->req_type),
2725 le16_to_cpu(resp->seq_id), rc);
2726 return rc;
2727 }
2728
2729 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
2730 {
2731 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
2732 }
2733
2734 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
2735 {
2736 int rc;
2737
2738 mutex_lock(&bp->hwrm_cmd_lock);
2739 rc = _hwrm_send_message(bp, msg, msg_len, timeout);
2740 mutex_unlock(&bp->hwrm_cmd_lock);
2741 return rc;
2742 }
2743
2744 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
2745 int timeout)
2746 {
2747 int rc;
2748
2749 mutex_lock(&bp->hwrm_cmd_lock);
2750 rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
2751 mutex_unlock(&bp->hwrm_cmd_lock);
2752 return rc;
2753 }
2754
2755 static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp)
2756 {
2757 struct hwrm_func_drv_rgtr_input req = {0};
2758 int i;
2759 DECLARE_BITMAP(async_events_bmap, 256);
2760 u32 *events = (u32 *)async_events_bmap;
2761
2762 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
2763
2764 req.enables =
2765 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
2766 FUNC_DRV_RGTR_REQ_ENABLES_VER |
2767 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
2768
2769 memset(async_events_bmap, 0, sizeof(async_events_bmap));
2770 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++)
2771 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
2772
2773 for (i = 0; i < 8; i++)
2774 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
2775
2776 req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
2777 req.ver_maj = DRV_VER_MAJ;
2778 req.ver_min = DRV_VER_MIN;
2779 req.ver_upd = DRV_VER_UPD;
2780
2781 if (BNXT_PF(bp)) {
2782 DECLARE_BITMAP(vf_req_snif_bmap, 256);
2783 u32 *data = (u32 *)vf_req_snif_bmap;
2784
2785 memset(vf_req_snif_bmap, 0, sizeof(vf_req_snif_bmap));
2786 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++)
2787 __set_bit(bnxt_vf_req_snif[i], vf_req_snif_bmap);
2788
2789 for (i = 0; i < 8; i++)
2790 req.vf_req_fwd[i] = cpu_to_le32(data[i]);
2791
2792 req.enables |=
2793 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
2794 }
2795
2796 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2797 }
2798
2799 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
2800 {
2801 struct hwrm_func_drv_unrgtr_input req = {0};
2802
2803 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
2804 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2805 }
2806
2807 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
2808 {
2809 u32 rc = 0;
2810 struct hwrm_tunnel_dst_port_free_input req = {0};
2811
2812 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
2813 req.tunnel_type = tunnel_type;
2814
2815 switch (tunnel_type) {
2816 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
2817 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id;
2818 break;
2819 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
2820 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id;
2821 break;
2822 default:
2823 break;
2824 }
2825
2826 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2827 if (rc)
2828 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
2829 rc);
2830 return rc;
2831 }
2832
2833 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
2834 u8 tunnel_type)
2835 {
2836 u32 rc = 0;
2837 struct hwrm_tunnel_dst_port_alloc_input req = {0};
2838 struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
2839
2840 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
2841
2842 req.tunnel_type = tunnel_type;
2843 req.tunnel_dst_port_val = port;
2844
2845 mutex_lock(&bp->hwrm_cmd_lock);
2846 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2847 if (rc) {
2848 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
2849 rc);
2850 goto err_out;
2851 }
2852
2853 if (tunnel_type & TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN)
2854 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id;
2855
2856 else if (tunnel_type & TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE)
2857 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id;
2858 err_out:
2859 mutex_unlock(&bp->hwrm_cmd_lock);
2860 return rc;
2861 }
2862
2863 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
2864 {
2865 struct hwrm_cfa_l2_set_rx_mask_input req = {0};
2866 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
2867
2868 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
2869 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
2870
2871 req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
2872 req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
2873 req.mask = cpu_to_le32(vnic->rx_mask);
2874 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2875 }
2876
2877 #ifdef CONFIG_RFS_ACCEL
2878 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
2879 struct bnxt_ntuple_filter *fltr)
2880 {
2881 struct hwrm_cfa_ntuple_filter_free_input req = {0};
2882
2883 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
2884 req.ntuple_filter_id = fltr->filter_id;
2885 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2886 }
2887
2888 #define BNXT_NTP_FLTR_FLAGS \
2889 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \
2890 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \
2891 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \
2892 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \
2893 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \
2894 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \
2895 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \
2896 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \
2897 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \
2898 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \
2899 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \
2900 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \
2901 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \
2902 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
2903
2904 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
2905 struct bnxt_ntuple_filter *fltr)
2906 {
2907 int rc = 0;
2908 struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
2909 struct hwrm_cfa_ntuple_filter_alloc_output *resp =
2910 bp->hwrm_cmd_resp_addr;
2911 struct flow_keys *keys = &fltr->fkeys;
2912 struct bnxt_vnic_info *vnic = &bp->vnic_info[fltr->rxq + 1];
2913
2914 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
2915 req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[0];
2916
2917 req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
2918
2919 req.ethertype = htons(ETH_P_IP);
2920 memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
2921 req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
2922 req.ip_protocol = keys->basic.ip_proto;
2923
2924 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
2925 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
2926 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
2927 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
2928
2929 req.src_port = keys->ports.src;
2930 req.src_port_mask = cpu_to_be16(0xffff);
2931 req.dst_port = keys->ports.dst;
2932 req.dst_port_mask = cpu_to_be16(0xffff);
2933
2934 req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
2935 mutex_lock(&bp->hwrm_cmd_lock);
2936 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2937 if (!rc)
2938 fltr->filter_id = resp->ntuple_filter_id;
2939 mutex_unlock(&bp->hwrm_cmd_lock);
2940 return rc;
2941 }
2942 #endif
2943
2944 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
2945 u8 *mac_addr)
2946 {
2947 u32 rc = 0;
2948 struct hwrm_cfa_l2_filter_alloc_input req = {0};
2949 struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
2950
2951 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
2952 req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX |
2953 CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
2954 req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
2955 req.enables =
2956 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
2957 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
2958 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
2959 memcpy(req.l2_addr, mac_addr, ETH_ALEN);
2960 req.l2_addr_mask[0] = 0xff;
2961 req.l2_addr_mask[1] = 0xff;
2962 req.l2_addr_mask[2] = 0xff;
2963 req.l2_addr_mask[3] = 0xff;
2964 req.l2_addr_mask[4] = 0xff;
2965 req.l2_addr_mask[5] = 0xff;
2966
2967 mutex_lock(&bp->hwrm_cmd_lock);
2968 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2969 if (!rc)
2970 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
2971 resp->l2_filter_id;
2972 mutex_unlock(&bp->hwrm_cmd_lock);
2973 return rc;
2974 }
2975
2976 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
2977 {
2978 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
2979 int rc = 0;
2980
2981 /* Any associated ntuple filters will also be cleared by firmware. */
2982 mutex_lock(&bp->hwrm_cmd_lock);
2983 for (i = 0; i < num_of_vnics; i++) {
2984 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
2985
2986 for (j = 0; j < vnic->uc_filter_count; j++) {
2987 struct hwrm_cfa_l2_filter_free_input req = {0};
2988
2989 bnxt_hwrm_cmd_hdr_init(bp, &req,
2990 HWRM_CFA_L2_FILTER_FREE, -1, -1);
2991
2992 req.l2_filter_id = vnic->fw_l2_filter_id[j];
2993
2994 rc = _hwrm_send_message(bp, &req, sizeof(req),
2995 HWRM_CMD_TIMEOUT);
2996 }
2997 vnic->uc_filter_count = 0;
2998 }
2999 mutex_unlock(&bp->hwrm_cmd_lock);
3000
3001 return rc;
3002 }
3003
3004 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
3005 {
3006 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3007 struct hwrm_vnic_tpa_cfg_input req = {0};
3008
3009 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
3010
3011 if (tpa_flags) {
3012 u16 mss = bp->dev->mtu - 40;
3013 u32 nsegs, n, segs = 0, flags;
3014
3015 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
3016 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
3017 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
3018 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
3019 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
3020 if (tpa_flags & BNXT_FLAG_GRO)
3021 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
3022
3023 req.flags = cpu_to_le32(flags);
3024
3025 req.enables =
3026 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
3027 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
3028 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
3029
3030 /* Number of segs are log2 units, and first packet is not
3031 * included as part of this units.
3032 */
3033 if (mss <= PAGE_SIZE) {
3034 n = PAGE_SIZE / mss;
3035 nsegs = (MAX_SKB_FRAGS - 1) * n;
3036 } else {
3037 n = mss / PAGE_SIZE;
3038 if (mss & (PAGE_SIZE - 1))
3039 n++;
3040 nsegs = (MAX_SKB_FRAGS - n) / n;
3041 }
3042
3043 segs = ilog2(nsegs);
3044 req.max_agg_segs = cpu_to_le16(segs);
3045 req.max_aggs = cpu_to_le16(VNIC_TPA_CFG_REQ_MAX_AGGS_MAX);
3046
3047 req.min_agg_len = cpu_to_le32(512);
3048 }
3049 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
3050
3051 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3052 }
3053
3054 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
3055 {
3056 u32 i, j, max_rings;
3057 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3058 struct hwrm_vnic_rss_cfg_input req = {0};
3059
3060 if (vnic->fw_rss_cos_lb_ctx == INVALID_HW_RING_ID)
3061 return 0;
3062
3063 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
3064 if (set_rss) {
3065 vnic->hash_type = BNXT_RSS_HASH_TYPE_FLAG_IPV4 |
3066 BNXT_RSS_HASH_TYPE_FLAG_TCP_IPV4 |
3067 BNXT_RSS_HASH_TYPE_FLAG_IPV6 |
3068 BNXT_RSS_HASH_TYPE_FLAG_TCP_IPV6;
3069
3070 req.hash_type = cpu_to_le32(vnic->hash_type);
3071
3072 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3073 max_rings = bp->rx_nr_rings;
3074 else
3075 max_rings = 1;
3076
3077 /* Fill the RSS indirection table with ring group ids */
3078 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) {
3079 if (j == max_rings)
3080 j = 0;
3081 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
3082 }
3083
3084 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
3085 req.hash_key_tbl_addr =
3086 cpu_to_le64(vnic->rss_hash_key_dma_addr);
3087 }
3088 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx);
3089 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3090 }
3091
3092 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
3093 {
3094 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3095 struct hwrm_vnic_plcmodes_cfg_input req = {0};
3096
3097 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
3098 req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
3099 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
3100 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
3101 req.enables =
3102 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
3103 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
3104 /* thresholds not implemented in firmware yet */
3105 req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
3106 req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
3107 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
3108 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3109 }
3110
3111 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id)
3112 {
3113 struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
3114
3115 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
3116 req.rss_cos_lb_ctx_id =
3117 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx);
3118
3119 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3120 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx = INVALID_HW_RING_ID;
3121 }
3122
3123 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
3124 {
3125 int i;
3126
3127 for (i = 0; i < bp->nr_vnics; i++) {
3128 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3129
3130 if (vnic->fw_rss_cos_lb_ctx != INVALID_HW_RING_ID)
3131 bnxt_hwrm_vnic_ctx_free_one(bp, i);
3132 }
3133 bp->rsscos_nr_ctxs = 0;
3134 }
3135
3136 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id)
3137 {
3138 int rc;
3139 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
3140 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
3141 bp->hwrm_cmd_resp_addr;
3142
3143 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
3144 -1);
3145
3146 mutex_lock(&bp->hwrm_cmd_lock);
3147 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3148 if (!rc)
3149 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx =
3150 le16_to_cpu(resp->rss_cos_lb_ctx_id);
3151 mutex_unlock(&bp->hwrm_cmd_lock);
3152
3153 return rc;
3154 }
3155
3156 static int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
3157 {
3158 unsigned int ring = 0, grp_idx;
3159 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3160 struct hwrm_vnic_cfg_input req = {0};
3161
3162 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
3163 /* Only RSS support for now TBD: COS & LB */
3164 req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP |
3165 VNIC_CFG_REQ_ENABLES_RSS_RULE);
3166 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx);
3167 req.cos_rule = cpu_to_le16(0xffff);
3168 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3169 ring = 0;
3170 else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
3171 ring = vnic_id - 1;
3172
3173 grp_idx = bp->rx_ring[ring].bnapi->index;
3174 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
3175 req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
3176
3177 req.lb_rule = cpu_to_le16(0xffff);
3178 req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN +
3179 VLAN_HLEN);
3180
3181 if (bp->flags & BNXT_FLAG_STRIP_VLAN)
3182 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
3183
3184 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3185 }
3186
3187 static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
3188 {
3189 u32 rc = 0;
3190
3191 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
3192 struct hwrm_vnic_free_input req = {0};
3193
3194 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
3195 req.vnic_id =
3196 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
3197
3198 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3199 if (rc)
3200 return rc;
3201 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
3202 }
3203 return rc;
3204 }
3205
3206 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
3207 {
3208 u16 i;
3209
3210 for (i = 0; i < bp->nr_vnics; i++)
3211 bnxt_hwrm_vnic_free_one(bp, i);
3212 }
3213
3214 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
3215 unsigned int start_rx_ring_idx,
3216 unsigned int nr_rings)
3217 {
3218 int rc = 0;
3219 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
3220 struct hwrm_vnic_alloc_input req = {0};
3221 struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3222
3223 /* map ring groups to this vnic */
3224 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
3225 grp_idx = bp->rx_ring[i].bnapi->index;
3226 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
3227 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
3228 j, nr_rings);
3229 break;
3230 }
3231 bp->vnic_info[vnic_id].fw_grp_ids[j] =
3232 bp->grp_info[grp_idx].fw_grp_id;
3233 }
3234
3235 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx = INVALID_HW_RING_ID;
3236 if (vnic_id == 0)
3237 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
3238
3239 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
3240
3241 mutex_lock(&bp->hwrm_cmd_lock);
3242 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3243 if (!rc)
3244 bp->vnic_info[vnic_id].fw_vnic_id = le32_to_cpu(resp->vnic_id);
3245 mutex_unlock(&bp->hwrm_cmd_lock);
3246 return rc;
3247 }
3248
3249 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
3250 {
3251 u16 i;
3252 u32 rc = 0;
3253
3254 mutex_lock(&bp->hwrm_cmd_lock);
3255 for (i = 0; i < bp->rx_nr_rings; i++) {
3256 struct hwrm_ring_grp_alloc_input req = {0};
3257 struct hwrm_ring_grp_alloc_output *resp =
3258 bp->hwrm_cmd_resp_addr;
3259 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
3260
3261 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
3262
3263 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
3264 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
3265 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
3266 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
3267
3268 rc = _hwrm_send_message(bp, &req, sizeof(req),
3269 HWRM_CMD_TIMEOUT);
3270 if (rc)
3271 break;
3272
3273 bp->grp_info[grp_idx].fw_grp_id =
3274 le32_to_cpu(resp->ring_group_id);
3275 }
3276 mutex_unlock(&bp->hwrm_cmd_lock);
3277 return rc;
3278 }
3279
3280 static int bnxt_hwrm_ring_grp_free(struct bnxt *bp)
3281 {
3282 u16 i;
3283 u32 rc = 0;
3284 struct hwrm_ring_grp_free_input req = {0};
3285
3286 if (!bp->grp_info)
3287 return 0;
3288
3289 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
3290
3291 mutex_lock(&bp->hwrm_cmd_lock);
3292 for (i = 0; i < bp->cp_nr_rings; i++) {
3293 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
3294 continue;
3295 req.ring_group_id =
3296 cpu_to_le32(bp->grp_info[i].fw_grp_id);
3297
3298 rc = _hwrm_send_message(bp, &req, sizeof(req),
3299 HWRM_CMD_TIMEOUT);
3300 if (rc)
3301 break;
3302 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3303 }
3304 mutex_unlock(&bp->hwrm_cmd_lock);
3305 return rc;
3306 }
3307
3308 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
3309 struct bnxt_ring_struct *ring,
3310 u32 ring_type, u32 map_index,
3311 u32 stats_ctx_id)
3312 {
3313 int rc = 0, err = 0;
3314 struct hwrm_ring_alloc_input req = {0};
3315 struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3316 u16 ring_id;
3317
3318 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
3319
3320 req.enables = 0;
3321 if (ring->nr_pages > 1) {
3322 req.page_tbl_addr = cpu_to_le64(ring->pg_tbl_map);
3323 /* Page size is in log2 units */
3324 req.page_size = BNXT_PAGE_SHIFT;
3325 req.page_tbl_depth = 1;
3326 } else {
3327 req.page_tbl_addr = cpu_to_le64(ring->dma_arr[0]);
3328 }
3329 req.fbo = 0;
3330 /* Association of ring index with doorbell index and MSIX number */
3331 req.logical_id = cpu_to_le16(map_index);
3332
3333 switch (ring_type) {
3334 case HWRM_RING_ALLOC_TX:
3335 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
3336 /* Association of transmit ring with completion ring */
3337 req.cmpl_ring_id =
3338 cpu_to_le16(bp->grp_info[map_index].cp_fw_ring_id);
3339 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
3340 req.stat_ctx_id = cpu_to_le32(stats_ctx_id);
3341 req.queue_id = cpu_to_le16(ring->queue_id);
3342 break;
3343 case HWRM_RING_ALLOC_RX:
3344 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
3345 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
3346 break;
3347 case HWRM_RING_ALLOC_AGG:
3348 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
3349 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
3350 break;
3351 case HWRM_RING_ALLOC_CMPL:
3352 req.ring_type = RING_ALLOC_REQ_RING_TYPE_CMPL;
3353 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
3354 if (bp->flags & BNXT_FLAG_USING_MSIX)
3355 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
3356 break;
3357 default:
3358 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
3359 ring_type);
3360 return -1;
3361 }
3362
3363 mutex_lock(&bp->hwrm_cmd_lock);
3364 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3365 err = le16_to_cpu(resp->error_code);
3366 ring_id = le16_to_cpu(resp->ring_id);
3367 mutex_unlock(&bp->hwrm_cmd_lock);
3368
3369 if (rc || err) {
3370 switch (ring_type) {
3371 case RING_FREE_REQ_RING_TYPE_CMPL:
3372 netdev_err(bp->dev, "hwrm_ring_alloc cp failed. rc:%x err:%x\n",
3373 rc, err);
3374 return -1;
3375
3376 case RING_FREE_REQ_RING_TYPE_RX:
3377 netdev_err(bp->dev, "hwrm_ring_alloc rx failed. rc:%x err:%x\n",
3378 rc, err);
3379 return -1;
3380
3381 case RING_FREE_REQ_RING_TYPE_TX:
3382 netdev_err(bp->dev, "hwrm_ring_alloc tx failed. rc:%x err:%x\n",
3383 rc, err);
3384 return -1;
3385
3386 default:
3387 netdev_err(bp->dev, "Invalid ring\n");
3388 return -1;
3389 }
3390 }
3391 ring->fw_ring_id = ring_id;
3392 return rc;
3393 }
3394
3395 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
3396 {
3397 int i, rc = 0;
3398
3399 for (i = 0; i < bp->cp_nr_rings; i++) {
3400 struct bnxt_napi *bnapi = bp->bnapi[i];
3401 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3402 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3403
3404 cpr->cp_doorbell = bp->bar1 + i * 0x80;
3405 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_CMPL, i,
3406 INVALID_STATS_CTX_ID);
3407 if (rc)
3408 goto err_out;
3409 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
3410 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
3411 }
3412
3413 for (i = 0; i < bp->tx_nr_rings; i++) {
3414 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3415 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3416 u32 map_idx = txr->bnapi->index;
3417 u16 fw_stats_ctx = bp->grp_info[map_idx].fw_stats_ctx;
3418
3419 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_TX,
3420 map_idx, fw_stats_ctx);
3421 if (rc)
3422 goto err_out;
3423 txr->tx_doorbell = bp->bar1 + map_idx * 0x80;
3424 }
3425
3426 for (i = 0; i < bp->rx_nr_rings; i++) {
3427 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3428 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
3429 u32 map_idx = rxr->bnapi->index;
3430
3431 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_RX,
3432 map_idx, INVALID_STATS_CTX_ID);
3433 if (rc)
3434 goto err_out;
3435 rxr->rx_doorbell = bp->bar1 + map_idx * 0x80;
3436 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
3437 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
3438 }
3439
3440 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
3441 for (i = 0; i < bp->rx_nr_rings; i++) {
3442 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3443 struct bnxt_ring_struct *ring =
3444 &rxr->rx_agg_ring_struct;
3445 u32 grp_idx = rxr->bnapi->index;
3446 u32 map_idx = grp_idx + bp->rx_nr_rings;
3447
3448 rc = hwrm_ring_alloc_send_msg(bp, ring,
3449 HWRM_RING_ALLOC_AGG,
3450 map_idx,
3451 INVALID_STATS_CTX_ID);
3452 if (rc)
3453 goto err_out;
3454
3455 rxr->rx_agg_doorbell = bp->bar1 + map_idx * 0x80;
3456 writel(DB_KEY_RX | rxr->rx_agg_prod,
3457 rxr->rx_agg_doorbell);
3458 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
3459 }
3460 }
3461 err_out:
3462 return rc;
3463 }
3464
3465 static int hwrm_ring_free_send_msg(struct bnxt *bp,
3466 struct bnxt_ring_struct *ring,
3467 u32 ring_type, int cmpl_ring_id)
3468 {
3469 int rc;
3470 struct hwrm_ring_free_input req = {0};
3471 struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
3472 u16 error_code;
3473
3474 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
3475 req.ring_type = ring_type;
3476 req.ring_id = cpu_to_le16(ring->fw_ring_id);
3477
3478 mutex_lock(&bp->hwrm_cmd_lock);
3479 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3480 error_code = le16_to_cpu(resp->error_code);
3481 mutex_unlock(&bp->hwrm_cmd_lock);
3482
3483 if (rc || error_code) {
3484 switch (ring_type) {
3485 case RING_FREE_REQ_RING_TYPE_CMPL:
3486 netdev_err(bp->dev, "hwrm_ring_free cp failed. rc:%d\n",
3487 rc);
3488 return rc;
3489 case RING_FREE_REQ_RING_TYPE_RX:
3490 netdev_err(bp->dev, "hwrm_ring_free rx failed. rc:%d\n",
3491 rc);
3492 return rc;
3493 case RING_FREE_REQ_RING_TYPE_TX:
3494 netdev_err(bp->dev, "hwrm_ring_free tx failed. rc:%d\n",
3495 rc);
3496 return rc;
3497 default:
3498 netdev_err(bp->dev, "Invalid ring\n");
3499 return -1;
3500 }
3501 }
3502 return 0;
3503 }
3504
3505 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
3506 {
3507 int i;
3508
3509 if (!bp->bnapi)
3510 return;
3511
3512 for (i = 0; i < bp->tx_nr_rings; i++) {
3513 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3514 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3515 u32 grp_idx = txr->bnapi->index;
3516 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
3517
3518 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3519 hwrm_ring_free_send_msg(bp, ring,
3520 RING_FREE_REQ_RING_TYPE_TX,
3521 close_path ? cmpl_ring_id :
3522 INVALID_HW_RING_ID);
3523 ring->fw_ring_id = INVALID_HW_RING_ID;
3524 }
3525 }
3526
3527 for (i = 0; i < bp->rx_nr_rings; i++) {
3528 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3529 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
3530 u32 grp_idx = rxr->bnapi->index;
3531 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
3532
3533 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3534 hwrm_ring_free_send_msg(bp, ring,
3535 RING_FREE_REQ_RING_TYPE_RX,
3536 close_path ? cmpl_ring_id :
3537 INVALID_HW_RING_ID);
3538 ring->fw_ring_id = INVALID_HW_RING_ID;
3539 bp->grp_info[grp_idx].rx_fw_ring_id =
3540 INVALID_HW_RING_ID;
3541 }
3542 }
3543
3544 for (i = 0; i < bp->rx_nr_rings; i++) {
3545 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3546 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
3547 u32 grp_idx = rxr->bnapi->index;
3548 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
3549
3550 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3551 hwrm_ring_free_send_msg(bp, ring,
3552 RING_FREE_REQ_RING_TYPE_RX,
3553 close_path ? cmpl_ring_id :
3554 INVALID_HW_RING_ID);
3555 ring->fw_ring_id = INVALID_HW_RING_ID;
3556 bp->grp_info[grp_idx].agg_fw_ring_id =
3557 INVALID_HW_RING_ID;
3558 }
3559 }
3560
3561 for (i = 0; i < bp->cp_nr_rings; i++) {
3562 struct bnxt_napi *bnapi = bp->bnapi[i];
3563 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3564 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3565
3566 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3567 hwrm_ring_free_send_msg(bp, ring,
3568 RING_FREE_REQ_RING_TYPE_CMPL,
3569 INVALID_HW_RING_ID);
3570 ring->fw_ring_id = INVALID_HW_RING_ID;
3571 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3572 }
3573 }
3574 }
3575
3576 static void bnxt_hwrm_set_coal_params(struct bnxt *bp, u32 max_bufs,
3577 u32 buf_tmrs, u16 flags,
3578 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
3579 {
3580 req->flags = cpu_to_le16(flags);
3581 req->num_cmpl_dma_aggr = cpu_to_le16((u16)max_bufs);
3582 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(max_bufs >> 16);
3583 req->cmpl_aggr_dma_tmr = cpu_to_le16((u16)buf_tmrs);
3584 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(buf_tmrs >> 16);
3585 /* Minimum time between 2 interrupts set to buf_tmr x 2 */
3586 req->int_lat_tmr_min = cpu_to_le16((u16)buf_tmrs * 2);
3587 req->int_lat_tmr_max = cpu_to_le16((u16)buf_tmrs * 4);
3588 req->num_cmpl_aggr_int = cpu_to_le16((u16)max_bufs * 4);
3589 }
3590
3591 int bnxt_hwrm_set_coal(struct bnxt *bp)
3592 {
3593 int i, rc = 0;
3594 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
3595 req_tx = {0}, *req;
3596 u16 max_buf, max_buf_irq;
3597 u16 buf_tmr, buf_tmr_irq;
3598 u32 flags;
3599
3600 bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
3601 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
3602 bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
3603 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
3604
3605 /* Each rx completion (2 records) should be DMAed immediately.
3606 * DMA 1/4 of the completion buffers at a time.
3607 */
3608 max_buf = min_t(u16, bp->rx_coal_bufs / 4, 2);
3609 /* max_buf must not be zero */
3610 max_buf = clamp_t(u16, max_buf, 1, 63);
3611 max_buf_irq = clamp_t(u16, bp->rx_coal_bufs_irq, 1, 63);
3612 buf_tmr = BNXT_USEC_TO_COAL_TIMER(bp->rx_coal_ticks);
3613 /* buf timer set to 1/4 of interrupt timer */
3614 buf_tmr = max_t(u16, buf_tmr / 4, 1);
3615 buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(bp->rx_coal_ticks_irq);
3616 buf_tmr_irq = max_t(u16, buf_tmr_irq, 1);
3617
3618 flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
3619
3620 /* RING_IDLE generates more IRQs for lower latency. Enable it only
3621 * if coal_ticks is less than 25 us.
3622 */
3623 if (bp->rx_coal_ticks < 25)
3624 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
3625
3626 bnxt_hwrm_set_coal_params(bp, max_buf_irq << 16 | max_buf,
3627 buf_tmr_irq << 16 | buf_tmr, flags, &req_rx);
3628
3629 /* max_buf must not be zero */
3630 max_buf = clamp_t(u16, bp->tx_coal_bufs, 1, 63);
3631 max_buf_irq = clamp_t(u16, bp->tx_coal_bufs_irq, 1, 63);
3632 buf_tmr = BNXT_USEC_TO_COAL_TIMER(bp->tx_coal_ticks);
3633 /* buf timer set to 1/4 of interrupt timer */
3634 buf_tmr = max_t(u16, buf_tmr / 4, 1);
3635 buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(bp->tx_coal_ticks_irq);
3636 buf_tmr_irq = max_t(u16, buf_tmr_irq, 1);
3637
3638 flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
3639 bnxt_hwrm_set_coal_params(bp, max_buf_irq << 16 | max_buf,
3640 buf_tmr_irq << 16 | buf_tmr, flags, &req_tx);
3641
3642 mutex_lock(&bp->hwrm_cmd_lock);
3643 for (i = 0; i < bp->cp_nr_rings; i++) {
3644 struct bnxt_napi *bnapi = bp->bnapi[i];
3645
3646 req = &req_rx;
3647 if (!bnapi->rx_ring)
3648 req = &req_tx;
3649 req->ring_id = cpu_to_le16(bp->grp_info[i].cp_fw_ring_id);
3650
3651 rc = _hwrm_send_message(bp, req, sizeof(*req),
3652 HWRM_CMD_TIMEOUT);
3653 if (rc)
3654 break;
3655 }
3656 mutex_unlock(&bp->hwrm_cmd_lock);
3657 return rc;
3658 }
3659
3660 static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
3661 {
3662 int rc = 0, i;
3663 struct hwrm_stat_ctx_free_input req = {0};
3664
3665 if (!bp->bnapi)
3666 return 0;
3667
3668 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
3669
3670 mutex_lock(&bp->hwrm_cmd_lock);
3671 for (i = 0; i < bp->cp_nr_rings; i++) {
3672 struct bnxt_napi *bnapi = bp->bnapi[i];
3673 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3674
3675 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
3676 req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
3677
3678 rc = _hwrm_send_message(bp, &req, sizeof(req),
3679 HWRM_CMD_TIMEOUT);
3680 if (rc)
3681 break;
3682
3683 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3684 }
3685 }
3686 mutex_unlock(&bp->hwrm_cmd_lock);
3687 return rc;
3688 }
3689
3690 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
3691 {
3692 int rc = 0, i;
3693 struct hwrm_stat_ctx_alloc_input req = {0};
3694 struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3695
3696 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
3697
3698 req.update_period_ms = cpu_to_le32(1000);
3699
3700 mutex_lock(&bp->hwrm_cmd_lock);
3701 for (i = 0; i < bp->cp_nr_rings; i++) {
3702 struct bnxt_napi *bnapi = bp->bnapi[i];
3703 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3704
3705 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map);
3706
3707 rc = _hwrm_send_message(bp, &req, sizeof(req),
3708 HWRM_CMD_TIMEOUT);
3709 if (rc)
3710 break;
3711
3712 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
3713
3714 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
3715 }
3716 mutex_unlock(&bp->hwrm_cmd_lock);
3717 return 0;
3718 }
3719
3720 int bnxt_hwrm_func_qcaps(struct bnxt *bp)
3721 {
3722 int rc = 0;
3723 struct hwrm_func_qcaps_input req = {0};
3724 struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
3725
3726 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
3727 req.fid = cpu_to_le16(0xffff);
3728
3729 mutex_lock(&bp->hwrm_cmd_lock);
3730 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3731 if (rc)
3732 goto hwrm_func_qcaps_exit;
3733
3734 if (BNXT_PF(bp)) {
3735 struct bnxt_pf_info *pf = &bp->pf;
3736
3737 pf->fw_fid = le16_to_cpu(resp->fid);
3738 pf->port_id = le16_to_cpu(resp->port_id);
3739 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
3740 memcpy(bp->dev->dev_addr, pf->mac_addr, ETH_ALEN);
3741 pf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
3742 pf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
3743 pf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
3744 pf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
3745 pf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
3746 if (!pf->max_hw_ring_grps)
3747 pf->max_hw_ring_grps = pf->max_tx_rings;
3748 pf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
3749 pf->max_vnics = le16_to_cpu(resp->max_vnics);
3750 pf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
3751 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
3752 pf->max_vfs = le16_to_cpu(resp->max_vfs);
3753 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
3754 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
3755 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
3756 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
3757 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
3758 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
3759 } else {
3760 #ifdef CONFIG_BNXT_SRIOV
3761 struct bnxt_vf_info *vf = &bp->vf;
3762
3763 vf->fw_fid = le16_to_cpu(resp->fid);
3764 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
3765 if (is_valid_ether_addr(vf->mac_addr))
3766 /* overwrite netdev dev_adr with admin VF MAC */
3767 memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
3768 else
3769 random_ether_addr(bp->dev->dev_addr);
3770
3771 vf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
3772 vf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
3773 vf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
3774 vf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
3775 vf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
3776 if (!vf->max_hw_ring_grps)
3777 vf->max_hw_ring_grps = vf->max_tx_rings;
3778 vf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
3779 vf->max_vnics = le16_to_cpu(resp->max_vnics);
3780 vf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
3781 #endif
3782 }
3783
3784 bp->tx_push_thresh = 0;
3785 if (resp->flags &
3786 cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED))
3787 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
3788
3789 hwrm_func_qcaps_exit:
3790 mutex_unlock(&bp->hwrm_cmd_lock);
3791 return rc;
3792 }
3793
3794 static int bnxt_hwrm_func_reset(struct bnxt *bp)
3795 {
3796 struct hwrm_func_reset_input req = {0};
3797
3798 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
3799 req.enables = 0;
3800
3801 return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
3802 }
3803
3804 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
3805 {
3806 int rc = 0;
3807 struct hwrm_queue_qportcfg_input req = {0};
3808 struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
3809 u8 i, *qptr;
3810
3811 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
3812
3813 mutex_lock(&bp->hwrm_cmd_lock);
3814 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3815 if (rc)
3816 goto qportcfg_exit;
3817
3818 if (!resp->max_configurable_queues) {
3819 rc = -EINVAL;
3820 goto qportcfg_exit;
3821 }
3822 bp->max_tc = resp->max_configurable_queues;
3823 if (bp->max_tc > BNXT_MAX_QUEUE)
3824 bp->max_tc = BNXT_MAX_QUEUE;
3825
3826 qptr = &resp->queue_id0;
3827 for (i = 0; i < bp->max_tc; i++) {
3828 bp->q_info[i].queue_id = *qptr++;
3829 bp->q_info[i].queue_profile = *qptr++;
3830 }
3831
3832 qportcfg_exit:
3833 mutex_unlock(&bp->hwrm_cmd_lock);
3834 return rc;
3835 }
3836
3837 static int bnxt_hwrm_ver_get(struct bnxt *bp)
3838 {
3839 int rc;
3840 struct hwrm_ver_get_input req = {0};
3841 struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
3842
3843 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
3844 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
3845 req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
3846 req.hwrm_intf_min = HWRM_VERSION_MINOR;
3847 req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
3848 mutex_lock(&bp->hwrm_cmd_lock);
3849 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3850 if (rc)
3851 goto hwrm_ver_get_exit;
3852
3853 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
3854
3855 bp->hwrm_spec_code = resp->hwrm_intf_maj << 16 |
3856 resp->hwrm_intf_min << 8 | resp->hwrm_intf_upd;
3857 if (resp->hwrm_intf_maj < 1) {
3858 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
3859 resp->hwrm_intf_maj, resp->hwrm_intf_min,
3860 resp->hwrm_intf_upd);
3861 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
3862 }
3863 snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d/%d.%d.%d",
3864 resp->hwrm_fw_maj, resp->hwrm_fw_min, resp->hwrm_fw_bld,
3865 resp->hwrm_intf_maj, resp->hwrm_intf_min, resp->hwrm_intf_upd);
3866
3867 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
3868 if (!bp->hwrm_cmd_timeout)
3869 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
3870
3871 if (resp->hwrm_intf_maj >= 1)
3872 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
3873
3874 hwrm_ver_get_exit:
3875 mutex_unlock(&bp->hwrm_cmd_lock);
3876 return rc;
3877 }
3878
3879 static int bnxt_hwrm_port_qstats(struct bnxt *bp)
3880 {
3881 int rc;
3882 struct bnxt_pf_info *pf = &bp->pf;
3883 struct hwrm_port_qstats_input req = {0};
3884
3885 if (!(bp->flags & BNXT_FLAG_PORT_STATS))
3886 return 0;
3887
3888 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
3889 req.port_id = cpu_to_le16(pf->port_id);
3890 req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map);
3891 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map);
3892 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3893 return rc;
3894 }
3895
3896 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
3897 {
3898 if (bp->vxlan_port_cnt) {
3899 bnxt_hwrm_tunnel_dst_port_free(
3900 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
3901 }
3902 bp->vxlan_port_cnt = 0;
3903 if (bp->nge_port_cnt) {
3904 bnxt_hwrm_tunnel_dst_port_free(
3905 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
3906 }
3907 bp->nge_port_cnt = 0;
3908 }
3909
3910 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
3911 {
3912 int rc, i;
3913 u32 tpa_flags = 0;
3914
3915 if (set_tpa)
3916 tpa_flags = bp->flags & BNXT_FLAG_TPA;
3917 for (i = 0; i < bp->nr_vnics; i++) {
3918 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
3919 if (rc) {
3920 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
3921 rc, i);
3922 return rc;
3923 }
3924 }
3925 return 0;
3926 }
3927
3928 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
3929 {
3930 int i;
3931
3932 for (i = 0; i < bp->nr_vnics; i++)
3933 bnxt_hwrm_vnic_set_rss(bp, i, false);
3934 }
3935
3936 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
3937 bool irq_re_init)
3938 {
3939 if (bp->vnic_info) {
3940 bnxt_hwrm_clear_vnic_filter(bp);
3941 /* clear all RSS setting before free vnic ctx */
3942 bnxt_hwrm_clear_vnic_rss(bp);
3943 bnxt_hwrm_vnic_ctx_free(bp);
3944 /* before free the vnic, undo the vnic tpa settings */
3945 if (bp->flags & BNXT_FLAG_TPA)
3946 bnxt_set_tpa(bp, false);
3947 bnxt_hwrm_vnic_free(bp);
3948 }
3949 bnxt_hwrm_ring_free(bp, close_path);
3950 bnxt_hwrm_ring_grp_free(bp);
3951 if (irq_re_init) {
3952 bnxt_hwrm_stat_ctx_free(bp);
3953 bnxt_hwrm_free_tunnel_ports(bp);
3954 }
3955 }
3956
3957 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
3958 {
3959 int rc;
3960
3961 /* allocate context for vnic */
3962 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id);
3963 if (rc) {
3964 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
3965 vnic_id, rc);
3966 goto vnic_setup_err;
3967 }
3968 bp->rsscos_nr_ctxs++;
3969
3970 /* configure default vnic, ring grp */
3971 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
3972 if (rc) {
3973 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
3974 vnic_id, rc);
3975 goto vnic_setup_err;
3976 }
3977
3978 /* Enable RSS hashing on vnic */
3979 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
3980 if (rc) {
3981 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
3982 vnic_id, rc);
3983 goto vnic_setup_err;
3984 }
3985
3986 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
3987 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
3988 if (rc) {
3989 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
3990 vnic_id, rc);
3991 }
3992 }
3993
3994 vnic_setup_err:
3995 return rc;
3996 }
3997
3998 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
3999 {
4000 #ifdef CONFIG_RFS_ACCEL
4001 int i, rc = 0;
4002
4003 for (i = 0; i < bp->rx_nr_rings; i++) {
4004 u16 vnic_id = i + 1;
4005 u16 ring_id = i;
4006
4007 if (vnic_id >= bp->nr_vnics)
4008 break;
4009
4010 bp->vnic_info[vnic_id].flags |= BNXT_VNIC_RFS_FLAG;
4011 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
4012 if (rc) {
4013 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
4014 vnic_id, rc);
4015 break;
4016 }
4017 rc = bnxt_setup_vnic(bp, vnic_id);
4018 if (rc)
4019 break;
4020 }
4021 return rc;
4022 #else
4023 return 0;
4024 #endif
4025 }
4026
4027 static int bnxt_cfg_rx_mode(struct bnxt *);
4028
4029 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
4030 {
4031 int rc = 0;
4032
4033 if (irq_re_init) {
4034 rc = bnxt_hwrm_stat_ctx_alloc(bp);
4035 if (rc) {
4036 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
4037 rc);
4038 goto err_out;
4039 }
4040 }
4041
4042 rc = bnxt_hwrm_ring_alloc(bp);
4043 if (rc) {
4044 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
4045 goto err_out;
4046 }
4047
4048 rc = bnxt_hwrm_ring_grp_alloc(bp);
4049 if (rc) {
4050 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
4051 goto err_out;
4052 }
4053
4054 /* default vnic 0 */
4055 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, bp->rx_nr_rings);
4056 if (rc) {
4057 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
4058 goto err_out;
4059 }
4060
4061 rc = bnxt_setup_vnic(bp, 0);
4062 if (rc)
4063 goto err_out;
4064
4065 if (bp->flags & BNXT_FLAG_RFS) {
4066 rc = bnxt_alloc_rfs_vnics(bp);
4067 if (rc)
4068 goto err_out;
4069 }
4070
4071 if (bp->flags & BNXT_FLAG_TPA) {
4072 rc = bnxt_set_tpa(bp, true);
4073 if (rc)
4074 goto err_out;
4075 }
4076
4077 if (BNXT_VF(bp))
4078 bnxt_update_vf_mac(bp);
4079
4080 /* Filter for default vnic 0 */
4081 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
4082 if (rc) {
4083 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
4084 goto err_out;
4085 }
4086 bp->vnic_info[0].uc_filter_count = 1;
4087
4088 bp->vnic_info[0].rx_mask = CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
4089
4090 if ((bp->dev->flags & IFF_PROMISC) && BNXT_PF(bp))
4091 bp->vnic_info[0].rx_mask |=
4092 CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
4093
4094 rc = bnxt_cfg_rx_mode(bp);
4095 if (rc)
4096 goto err_out;
4097
4098 rc = bnxt_hwrm_set_coal(bp);
4099 if (rc)
4100 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
4101 rc);
4102
4103 return 0;
4104
4105 err_out:
4106 bnxt_hwrm_resource_free(bp, 0, true);
4107
4108 return rc;
4109 }
4110
4111 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
4112 {
4113 bnxt_hwrm_resource_free(bp, 1, irq_re_init);
4114 return 0;
4115 }
4116
4117 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
4118 {
4119 bnxt_init_rx_rings(bp);
4120 bnxt_init_tx_rings(bp);
4121 bnxt_init_ring_grps(bp, irq_re_init);
4122 bnxt_init_vnics(bp);
4123
4124 return bnxt_init_chip(bp, irq_re_init);
4125 }
4126
4127 static void bnxt_disable_int(struct bnxt *bp)
4128 {
4129 int i;
4130
4131 if (!bp->bnapi)
4132 return;
4133
4134 for (i = 0; i < bp->cp_nr_rings; i++) {
4135 struct bnxt_napi *bnapi = bp->bnapi[i];
4136 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4137
4138 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
4139 }
4140 }
4141
4142 static void bnxt_enable_int(struct bnxt *bp)
4143 {
4144 int i;
4145
4146 atomic_set(&bp->intr_sem, 0);
4147 for (i = 0; i < bp->cp_nr_rings; i++) {
4148 struct bnxt_napi *bnapi = bp->bnapi[i];
4149 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4150
4151 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
4152 }
4153 }
4154
4155 static int bnxt_set_real_num_queues(struct bnxt *bp)
4156 {
4157 int rc;
4158 struct net_device *dev = bp->dev;
4159
4160 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings);
4161 if (rc)
4162 return rc;
4163
4164 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
4165 if (rc)
4166 return rc;
4167
4168 #ifdef CONFIG_RFS_ACCEL
4169 if (bp->flags & BNXT_FLAG_RFS)
4170 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
4171 #endif
4172
4173 return rc;
4174 }
4175
4176 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
4177 bool shared)
4178 {
4179 int _rx = *rx, _tx = *tx;
4180
4181 if (shared) {
4182 *rx = min_t(int, _rx, max);
4183 *tx = min_t(int, _tx, max);
4184 } else {
4185 if (max < 2)
4186 return -ENOMEM;
4187
4188 while (_rx + _tx > max) {
4189 if (_rx > _tx && _rx > 1)
4190 _rx--;
4191 else if (_tx > 1)
4192 _tx--;
4193 }
4194 *rx = _rx;
4195 *tx = _tx;
4196 }
4197 return 0;
4198 }
4199
4200 static int bnxt_setup_msix(struct bnxt *bp)
4201 {
4202 struct msix_entry *msix_ent;
4203 struct net_device *dev = bp->dev;
4204 int i, total_vecs, rc = 0, min = 1;
4205 const int len = sizeof(bp->irq_tbl[0].name);
4206
4207 bp->flags &= ~BNXT_FLAG_USING_MSIX;
4208 total_vecs = bp->cp_nr_rings;
4209
4210 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
4211 if (!msix_ent)
4212 return -ENOMEM;
4213
4214 for (i = 0; i < total_vecs; i++) {
4215 msix_ent[i].entry = i;
4216 msix_ent[i].vector = 0;
4217 }
4218
4219 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
4220 min = 2;
4221
4222 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
4223 if (total_vecs < 0) {
4224 rc = -ENODEV;
4225 goto msix_setup_exit;
4226 }
4227
4228 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
4229 if (bp->irq_tbl) {
4230 int tcs;
4231
4232 /* Trim rings based upon num of vectors allocated */
4233 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
4234 total_vecs, min == 1);
4235 if (rc)
4236 goto msix_setup_exit;
4237
4238 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
4239 tcs = netdev_get_num_tc(dev);
4240 if (tcs > 1) {
4241 bp->tx_nr_rings_per_tc = bp->tx_nr_rings / tcs;
4242 if (bp->tx_nr_rings_per_tc == 0) {
4243 netdev_reset_tc(dev);
4244 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
4245 } else {
4246 int i, off, count;
4247
4248 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tcs;
4249 for (i = 0; i < tcs; i++) {
4250 count = bp->tx_nr_rings_per_tc;
4251 off = i * count;
4252 netdev_set_tc_queue(dev, i, count, off);
4253 }
4254 }
4255 }
4256 bp->cp_nr_rings = total_vecs;
4257
4258 for (i = 0; i < bp->cp_nr_rings; i++) {
4259 char *attr;
4260
4261 bp->irq_tbl[i].vector = msix_ent[i].vector;
4262 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4263 attr = "TxRx";
4264 else if (i < bp->rx_nr_rings)
4265 attr = "rx";
4266 else
4267 attr = "tx";
4268
4269 snprintf(bp->irq_tbl[i].name, len,
4270 "%s-%s-%d", dev->name, attr, i);
4271 bp->irq_tbl[i].handler = bnxt_msix;
4272 }
4273 rc = bnxt_set_real_num_queues(bp);
4274 if (rc)
4275 goto msix_setup_exit;
4276 } else {
4277 rc = -ENOMEM;
4278 goto msix_setup_exit;
4279 }
4280 bp->flags |= BNXT_FLAG_USING_MSIX;
4281 kfree(msix_ent);
4282 return 0;
4283
4284 msix_setup_exit:
4285 netdev_err(bp->dev, "bnxt_setup_msix err: %x\n", rc);
4286 pci_disable_msix(bp->pdev);
4287 kfree(msix_ent);
4288 return rc;
4289 }
4290
4291 static int bnxt_setup_inta(struct bnxt *bp)
4292 {
4293 int rc;
4294 const int len = sizeof(bp->irq_tbl[0].name);
4295
4296 if (netdev_get_num_tc(bp->dev))
4297 netdev_reset_tc(bp->dev);
4298
4299 bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
4300 if (!bp->irq_tbl) {
4301 rc = -ENOMEM;
4302 return rc;
4303 }
4304 bp->rx_nr_rings = 1;
4305 bp->tx_nr_rings = 1;
4306 bp->cp_nr_rings = 1;
4307 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
4308 bp->flags |= BNXT_FLAG_SHARED_RINGS;
4309 bp->irq_tbl[0].vector = bp->pdev->irq;
4310 snprintf(bp->irq_tbl[0].name, len,
4311 "%s-%s-%d", bp->dev->name, "TxRx", 0);
4312 bp->irq_tbl[0].handler = bnxt_inta;
4313 rc = bnxt_set_real_num_queues(bp);
4314 return rc;
4315 }
4316
4317 static int bnxt_setup_int_mode(struct bnxt *bp)
4318 {
4319 int rc = 0;
4320
4321 if (bp->flags & BNXT_FLAG_MSIX_CAP)
4322 rc = bnxt_setup_msix(bp);
4323
4324 if (!(bp->flags & BNXT_FLAG_USING_MSIX)) {
4325 /* fallback to INTA */
4326 rc = bnxt_setup_inta(bp);
4327 }
4328 return rc;
4329 }
4330
4331 static void bnxt_free_irq(struct bnxt *bp)
4332 {
4333 struct bnxt_irq *irq;
4334 int i;
4335
4336 #ifdef CONFIG_RFS_ACCEL
4337 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
4338 bp->dev->rx_cpu_rmap = NULL;
4339 #endif
4340 if (!bp->irq_tbl)
4341 return;
4342
4343 for (i = 0; i < bp->cp_nr_rings; i++) {
4344 irq = &bp->irq_tbl[i];
4345 if (irq->requested)
4346 free_irq(irq->vector, bp->bnapi[i]);
4347 irq->requested = 0;
4348 }
4349 if (bp->flags & BNXT_FLAG_USING_MSIX)
4350 pci_disable_msix(bp->pdev);
4351 kfree(bp->irq_tbl);
4352 bp->irq_tbl = NULL;
4353 }
4354
4355 static int bnxt_request_irq(struct bnxt *bp)
4356 {
4357 int i, j, rc = 0;
4358 unsigned long flags = 0;
4359 #ifdef CONFIG_RFS_ACCEL
4360 struct cpu_rmap *rmap = bp->dev->rx_cpu_rmap;
4361 #endif
4362
4363 if (!(bp->flags & BNXT_FLAG_USING_MSIX))
4364 flags = IRQF_SHARED;
4365
4366 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
4367 struct bnxt_irq *irq = &bp->irq_tbl[i];
4368 #ifdef CONFIG_RFS_ACCEL
4369 if (rmap && bp->bnapi[i]->rx_ring) {
4370 rc = irq_cpu_rmap_add(rmap, irq->vector);
4371 if (rc)
4372 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
4373 j);
4374 j++;
4375 }
4376 #endif
4377 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
4378 bp->bnapi[i]);
4379 if (rc)
4380 break;
4381
4382 irq->requested = 1;
4383 }
4384 return rc;
4385 }
4386
4387 static void bnxt_del_napi(struct bnxt *bp)
4388 {
4389 int i;
4390
4391 if (!bp->bnapi)
4392 return;
4393
4394 for (i = 0; i < bp->cp_nr_rings; i++) {
4395 struct bnxt_napi *bnapi = bp->bnapi[i];
4396
4397 napi_hash_del(&bnapi->napi);
4398 netif_napi_del(&bnapi->napi);
4399 }
4400 }
4401
4402 static void bnxt_init_napi(struct bnxt *bp)
4403 {
4404 int i;
4405 struct bnxt_napi *bnapi;
4406
4407 if (bp->flags & BNXT_FLAG_USING_MSIX) {
4408 for (i = 0; i < bp->cp_nr_rings; i++) {
4409 bnapi = bp->bnapi[i];
4410 netif_napi_add(bp->dev, &bnapi->napi,
4411 bnxt_poll, 64);
4412 }
4413 } else {
4414 bnapi = bp->bnapi[0];
4415 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
4416 }
4417 }
4418
4419 static void bnxt_disable_napi(struct bnxt *bp)
4420 {
4421 int i;
4422
4423 if (!bp->bnapi)
4424 return;
4425
4426 for (i = 0; i < bp->cp_nr_rings; i++) {
4427 napi_disable(&bp->bnapi[i]->napi);
4428 bnxt_disable_poll(bp->bnapi[i]);
4429 }
4430 }
4431
4432 static void bnxt_enable_napi(struct bnxt *bp)
4433 {
4434 int i;
4435
4436 for (i = 0; i < bp->cp_nr_rings; i++) {
4437 bnxt_enable_poll(bp->bnapi[i]);
4438 napi_enable(&bp->bnapi[i]->napi);
4439 }
4440 }
4441
4442 static void bnxt_tx_disable(struct bnxt *bp)
4443 {
4444 int i;
4445 struct bnxt_tx_ring_info *txr;
4446 struct netdev_queue *txq;
4447
4448 if (bp->tx_ring) {
4449 for (i = 0; i < bp->tx_nr_rings; i++) {
4450 txr = &bp->tx_ring[i];
4451 txq = netdev_get_tx_queue(bp->dev, i);
4452 __netif_tx_lock(txq, smp_processor_id());
4453 txr->dev_state = BNXT_DEV_STATE_CLOSING;
4454 __netif_tx_unlock(txq);
4455 }
4456 }
4457 /* Stop all TX queues */
4458 netif_tx_disable(bp->dev);
4459 netif_carrier_off(bp->dev);
4460 }
4461
4462 static void bnxt_tx_enable(struct bnxt *bp)
4463 {
4464 int i;
4465 struct bnxt_tx_ring_info *txr;
4466 struct netdev_queue *txq;
4467
4468 for (i = 0; i < bp->tx_nr_rings; i++) {
4469 txr = &bp->tx_ring[i];
4470 txq = netdev_get_tx_queue(bp->dev, i);
4471 txr->dev_state = 0;
4472 }
4473 netif_tx_wake_all_queues(bp->dev);
4474 if (bp->link_info.link_up)
4475 netif_carrier_on(bp->dev);
4476 }
4477
4478 static void bnxt_report_link(struct bnxt *bp)
4479 {
4480 if (bp->link_info.link_up) {
4481 const char *duplex;
4482 const char *flow_ctrl;
4483 u16 speed;
4484
4485 netif_carrier_on(bp->dev);
4486 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
4487 duplex = "full";
4488 else
4489 duplex = "half";
4490 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
4491 flow_ctrl = "ON - receive & transmit";
4492 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
4493 flow_ctrl = "ON - transmit";
4494 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
4495 flow_ctrl = "ON - receive";
4496 else
4497 flow_ctrl = "none";
4498 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
4499 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
4500 speed, duplex, flow_ctrl);
4501 if (bp->flags & BNXT_FLAG_EEE_CAP)
4502 netdev_info(bp->dev, "EEE is %s\n",
4503 bp->eee.eee_active ? "active" :
4504 "not active");
4505 } else {
4506 netif_carrier_off(bp->dev);
4507 netdev_err(bp->dev, "NIC Link is Down\n");
4508 }
4509 }
4510
4511 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
4512 {
4513 int rc = 0;
4514 struct hwrm_port_phy_qcaps_input req = {0};
4515 struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
4516
4517 if (bp->hwrm_spec_code < 0x10201)
4518 return 0;
4519
4520 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
4521
4522 mutex_lock(&bp->hwrm_cmd_lock);
4523 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4524 if (rc)
4525 goto hwrm_phy_qcaps_exit;
4526
4527 if (resp->eee_supported & PORT_PHY_QCAPS_RESP_EEE_SUPPORTED) {
4528 struct ethtool_eee *eee = &bp->eee;
4529 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
4530
4531 bp->flags |= BNXT_FLAG_EEE_CAP;
4532 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
4533 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
4534 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
4535 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
4536 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
4537 }
4538
4539 hwrm_phy_qcaps_exit:
4540 mutex_unlock(&bp->hwrm_cmd_lock);
4541 return rc;
4542 }
4543
4544 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
4545 {
4546 int rc = 0;
4547 struct bnxt_link_info *link_info = &bp->link_info;
4548 struct hwrm_port_phy_qcfg_input req = {0};
4549 struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
4550 u8 link_up = link_info->link_up;
4551
4552 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
4553
4554 mutex_lock(&bp->hwrm_cmd_lock);
4555 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4556 if (rc) {
4557 mutex_unlock(&bp->hwrm_cmd_lock);
4558 return rc;
4559 }
4560
4561 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
4562 link_info->phy_link_status = resp->link;
4563 link_info->duplex = resp->duplex;
4564 link_info->pause = resp->pause;
4565 link_info->auto_mode = resp->auto_mode;
4566 link_info->auto_pause_setting = resp->auto_pause;
4567 link_info->lp_pause = resp->link_partner_adv_pause;
4568 link_info->force_pause_setting = resp->force_pause;
4569 link_info->duplex_setting = resp->duplex;
4570 if (link_info->phy_link_status == BNXT_LINK_LINK)
4571 link_info->link_speed = le16_to_cpu(resp->link_speed);
4572 else
4573 link_info->link_speed = 0;
4574 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
4575 link_info->support_speeds = le16_to_cpu(resp->support_speeds);
4576 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
4577 link_info->lp_auto_link_speeds =
4578 le16_to_cpu(resp->link_partner_adv_speeds);
4579 link_info->preemphasis = le32_to_cpu(resp->preemphasis);
4580 link_info->phy_ver[0] = resp->phy_maj;
4581 link_info->phy_ver[1] = resp->phy_min;
4582 link_info->phy_ver[2] = resp->phy_bld;
4583 link_info->media_type = resp->media_type;
4584 link_info->transceiver = resp->xcvr_pkg_type;
4585 link_info->phy_addr = resp->eee_config_phy_addr &
4586 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
4587
4588 if (bp->flags & BNXT_FLAG_EEE_CAP) {
4589 struct ethtool_eee *eee = &bp->eee;
4590 u16 fw_speeds;
4591
4592 eee->eee_active = 0;
4593 if (resp->eee_config_phy_addr &
4594 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
4595 eee->eee_active = 1;
4596 fw_speeds = le16_to_cpu(
4597 resp->link_partner_adv_eee_link_speed_mask);
4598 eee->lp_advertised =
4599 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
4600 }
4601
4602 /* Pull initial EEE config */
4603 if (!chng_link_state) {
4604 if (resp->eee_config_phy_addr &
4605 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
4606 eee->eee_enabled = 1;
4607
4608 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
4609 eee->advertised =
4610 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
4611
4612 if (resp->eee_config_phy_addr &
4613 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
4614 __le32 tmr;
4615
4616 eee->tx_lpi_enabled = 1;
4617 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
4618 eee->tx_lpi_timer = le32_to_cpu(tmr) &
4619 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
4620 }
4621 }
4622 }
4623 /* TODO: need to add more logic to report VF link */
4624 if (chng_link_state) {
4625 if (link_info->phy_link_status == BNXT_LINK_LINK)
4626 link_info->link_up = 1;
4627 else
4628 link_info->link_up = 0;
4629 if (link_up != link_info->link_up)
4630 bnxt_report_link(bp);
4631 } else {
4632 /* alwasy link down if not require to update link state */
4633 link_info->link_up = 0;
4634 }
4635 mutex_unlock(&bp->hwrm_cmd_lock);
4636 return 0;
4637 }
4638
4639 static void
4640 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
4641 {
4642 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
4643 if (bp->hwrm_spec_code >= 0x10201)
4644 req->auto_pause =
4645 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
4646 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
4647 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
4648 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
4649 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
4650 req->enables |=
4651 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
4652 } else {
4653 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
4654 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
4655 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
4656 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
4657 req->enables |=
4658 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
4659 if (bp->hwrm_spec_code >= 0x10201) {
4660 req->auto_pause = req->force_pause;
4661 req->enables |= cpu_to_le32(
4662 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
4663 }
4664 }
4665 }
4666
4667 static void bnxt_hwrm_set_link_common(struct bnxt *bp,
4668 struct hwrm_port_phy_cfg_input *req)
4669 {
4670 u8 autoneg = bp->link_info.autoneg;
4671 u16 fw_link_speed = bp->link_info.req_link_speed;
4672 u32 advertising = bp->link_info.advertising;
4673
4674 if (autoneg & BNXT_AUTONEG_SPEED) {
4675 req->auto_mode |=
4676 PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
4677
4678 req->enables |= cpu_to_le32(
4679 PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
4680 req->auto_link_speed_mask = cpu_to_le16(advertising);
4681
4682 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
4683 req->flags |=
4684 cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
4685 } else {
4686 req->force_link_speed = cpu_to_le16(fw_link_speed);
4687 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
4688 }
4689
4690 /* tell chimp that the setting takes effect immediately */
4691 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
4692 }
4693
4694 int bnxt_hwrm_set_pause(struct bnxt *bp)
4695 {
4696 struct hwrm_port_phy_cfg_input req = {0};
4697 int rc;
4698
4699 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
4700 bnxt_hwrm_set_pause_common(bp, &req);
4701
4702 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
4703 bp->link_info.force_link_chng)
4704 bnxt_hwrm_set_link_common(bp, &req);
4705
4706 mutex_lock(&bp->hwrm_cmd_lock);
4707 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4708 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
4709 /* since changing of pause setting doesn't trigger any link
4710 * change event, the driver needs to update the current pause
4711 * result upon successfully return of the phy_cfg command
4712 */
4713 bp->link_info.pause =
4714 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
4715 bp->link_info.auto_pause_setting = 0;
4716 if (!bp->link_info.force_link_chng)
4717 bnxt_report_link(bp);
4718 }
4719 bp->link_info.force_link_chng = false;
4720 mutex_unlock(&bp->hwrm_cmd_lock);
4721 return rc;
4722 }
4723
4724 static void bnxt_hwrm_set_eee(struct bnxt *bp,
4725 struct hwrm_port_phy_cfg_input *req)
4726 {
4727 struct ethtool_eee *eee = &bp->eee;
4728
4729 if (eee->eee_enabled) {
4730 u16 eee_speeds;
4731 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
4732
4733 if (eee->tx_lpi_enabled)
4734 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
4735 else
4736 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
4737
4738 req->flags |= cpu_to_le32(flags);
4739 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
4740 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
4741 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
4742 } else {
4743 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
4744 }
4745 }
4746
4747 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
4748 {
4749 struct hwrm_port_phy_cfg_input req = {0};
4750
4751 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
4752 if (set_pause)
4753 bnxt_hwrm_set_pause_common(bp, &req);
4754
4755 bnxt_hwrm_set_link_common(bp, &req);
4756
4757 if (set_eee)
4758 bnxt_hwrm_set_eee(bp, &req);
4759 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4760 }
4761
4762 static bool bnxt_eee_config_ok(struct bnxt *bp)
4763 {
4764 struct ethtool_eee *eee = &bp->eee;
4765 struct bnxt_link_info *link_info = &bp->link_info;
4766
4767 if (!(bp->flags & BNXT_FLAG_EEE_CAP))
4768 return true;
4769
4770 if (eee->eee_enabled) {
4771 u32 advertising =
4772 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
4773
4774 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
4775 eee->eee_enabled = 0;
4776 return false;
4777 }
4778 if (eee->advertised & ~advertising) {
4779 eee->advertised = advertising & eee->supported;
4780 return false;
4781 }
4782 }
4783 return true;
4784 }
4785
4786 static int bnxt_update_phy_setting(struct bnxt *bp)
4787 {
4788 int rc;
4789 bool update_link = false;
4790 bool update_pause = false;
4791 bool update_eee = false;
4792 struct bnxt_link_info *link_info = &bp->link_info;
4793
4794 rc = bnxt_update_link(bp, true);
4795 if (rc) {
4796 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
4797 rc);
4798 return rc;
4799 }
4800 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
4801 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
4802 link_info->req_flow_ctrl)
4803 update_pause = true;
4804 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
4805 link_info->force_pause_setting != link_info->req_flow_ctrl)
4806 update_pause = true;
4807 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
4808 if (BNXT_AUTO_MODE(link_info->auto_mode))
4809 update_link = true;
4810 if (link_info->req_link_speed != link_info->force_link_speed)
4811 update_link = true;
4812 if (link_info->req_duplex != link_info->duplex_setting)
4813 update_link = true;
4814 } else {
4815 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
4816 update_link = true;
4817 if (link_info->advertising != link_info->auto_link_speeds)
4818 update_link = true;
4819 }
4820
4821 if (!bnxt_eee_config_ok(bp))
4822 update_eee = true;
4823
4824 if (update_link)
4825 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
4826 else if (update_pause)
4827 rc = bnxt_hwrm_set_pause(bp);
4828 if (rc) {
4829 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
4830 rc);
4831 return rc;
4832 }
4833
4834 return rc;
4835 }
4836
4837 /* Common routine to pre-map certain register block to different GRC window.
4838 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
4839 * in PF and 3 windows in VF that can be customized to map in different
4840 * register blocks.
4841 */
4842 static void bnxt_preset_reg_win(struct bnxt *bp)
4843 {
4844 if (BNXT_PF(bp)) {
4845 /* CAG registers map to GRC window #4 */
4846 writel(BNXT_CAG_REG_BASE,
4847 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
4848 }
4849 }
4850
4851 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
4852 {
4853 int rc = 0;
4854
4855 bnxt_preset_reg_win(bp);
4856 netif_carrier_off(bp->dev);
4857 if (irq_re_init) {
4858 rc = bnxt_setup_int_mode(bp);
4859 if (rc) {
4860 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
4861 rc);
4862 return rc;
4863 }
4864 }
4865 if ((bp->flags & BNXT_FLAG_RFS) &&
4866 !(bp->flags & BNXT_FLAG_USING_MSIX)) {
4867 /* disable RFS if falling back to INTA */
4868 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
4869 bp->flags &= ~BNXT_FLAG_RFS;
4870 }
4871
4872 rc = bnxt_alloc_mem(bp, irq_re_init);
4873 if (rc) {
4874 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
4875 goto open_err_free_mem;
4876 }
4877
4878 if (irq_re_init) {
4879 bnxt_init_napi(bp);
4880 rc = bnxt_request_irq(bp);
4881 if (rc) {
4882 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
4883 goto open_err;
4884 }
4885 }
4886
4887 bnxt_enable_napi(bp);
4888
4889 rc = bnxt_init_nic(bp, irq_re_init);
4890 if (rc) {
4891 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
4892 goto open_err;
4893 }
4894
4895 if (link_re_init) {
4896 rc = bnxt_update_phy_setting(bp);
4897 if (rc)
4898 netdev_warn(bp->dev, "failed to update phy settings\n");
4899 }
4900
4901 if (irq_re_init) {
4902 #if defined(CONFIG_VXLAN) || defined(CONFIG_VXLAN_MODULE)
4903 vxlan_get_rx_port(bp->dev);
4904 #endif
4905 if (!bnxt_hwrm_tunnel_dst_port_alloc(
4906 bp, htons(0x17c1),
4907 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE))
4908 bp->nge_port_cnt = 1;
4909 }
4910
4911 set_bit(BNXT_STATE_OPEN, &bp->state);
4912 bnxt_enable_int(bp);
4913 /* Enable TX queues */
4914 bnxt_tx_enable(bp);
4915 mod_timer(&bp->timer, jiffies + bp->current_interval);
4916 bnxt_update_link(bp, true);
4917
4918 return 0;
4919
4920 open_err:
4921 bnxt_disable_napi(bp);
4922 bnxt_del_napi(bp);
4923
4924 open_err_free_mem:
4925 bnxt_free_skbs(bp);
4926 bnxt_free_irq(bp);
4927 bnxt_free_mem(bp, true);
4928 return rc;
4929 }
4930
4931 /* rtnl_lock held */
4932 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
4933 {
4934 int rc = 0;
4935
4936 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
4937 if (rc) {
4938 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
4939 dev_close(bp->dev);
4940 }
4941 return rc;
4942 }
4943
4944 static int bnxt_open(struct net_device *dev)
4945 {
4946 struct bnxt *bp = netdev_priv(dev);
4947 int rc = 0;
4948
4949 rc = bnxt_hwrm_func_reset(bp);
4950 if (rc) {
4951 netdev_err(bp->dev, "hwrm chip reset failure rc: %x\n",
4952 rc);
4953 rc = -1;
4954 return rc;
4955 }
4956 return __bnxt_open_nic(bp, true, true);
4957 }
4958
4959 static void bnxt_disable_int_sync(struct bnxt *bp)
4960 {
4961 int i;
4962
4963 atomic_inc(&bp->intr_sem);
4964 if (!netif_running(bp->dev))
4965 return;
4966
4967 bnxt_disable_int(bp);
4968 for (i = 0; i < bp->cp_nr_rings; i++)
4969 synchronize_irq(bp->irq_tbl[i].vector);
4970 }
4971
4972 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
4973 {
4974 int rc = 0;
4975
4976 #ifdef CONFIG_BNXT_SRIOV
4977 if (bp->sriov_cfg) {
4978 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
4979 !bp->sriov_cfg,
4980 BNXT_SRIOV_CFG_WAIT_TMO);
4981 if (rc)
4982 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
4983 }
4984 #endif
4985 /* Change device state to avoid TX queue wake up's */
4986 bnxt_tx_disable(bp);
4987
4988 clear_bit(BNXT_STATE_OPEN, &bp->state);
4989 smp_mb__after_atomic();
4990 while (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state))
4991 msleep(20);
4992
4993 /* Flush rings before disabling interrupts */
4994 bnxt_shutdown_nic(bp, irq_re_init);
4995
4996 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
4997
4998 bnxt_disable_napi(bp);
4999 bnxt_disable_int_sync(bp);
5000 del_timer_sync(&bp->timer);
5001 bnxt_free_skbs(bp);
5002
5003 if (irq_re_init) {
5004 bnxt_free_irq(bp);
5005 bnxt_del_napi(bp);
5006 }
5007 bnxt_free_mem(bp, irq_re_init);
5008 return rc;
5009 }
5010
5011 static int bnxt_close(struct net_device *dev)
5012 {
5013 struct bnxt *bp = netdev_priv(dev);
5014
5015 bnxt_close_nic(bp, true, true);
5016 return 0;
5017 }
5018
5019 /* rtnl_lock held */
5020 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
5021 {
5022 switch (cmd) {
5023 case SIOCGMIIPHY:
5024 /* fallthru */
5025 case SIOCGMIIREG: {
5026 if (!netif_running(dev))
5027 return -EAGAIN;
5028
5029 return 0;
5030 }
5031
5032 case SIOCSMIIREG:
5033 if (!netif_running(dev))
5034 return -EAGAIN;
5035
5036 return 0;
5037
5038 default:
5039 /* do nothing */
5040 break;
5041 }
5042 return -EOPNOTSUPP;
5043 }
5044
5045 static struct rtnl_link_stats64 *
5046 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
5047 {
5048 u32 i;
5049 struct bnxt *bp = netdev_priv(dev);
5050
5051 memset(stats, 0, sizeof(struct rtnl_link_stats64));
5052
5053 if (!bp->bnapi)
5054 return stats;
5055
5056 /* TODO check if we need to synchronize with bnxt_close path */
5057 for (i = 0; i < bp->cp_nr_rings; i++) {
5058 struct bnxt_napi *bnapi = bp->bnapi[i];
5059 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5060 struct ctx_hw_stats *hw_stats = cpr->hw_stats;
5061
5062 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts);
5063 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts);
5064 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts);
5065
5066 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts);
5067 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts);
5068 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts);
5069
5070 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes);
5071 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes);
5072 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes);
5073
5074 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes);
5075 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes);
5076 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes);
5077
5078 stats->rx_missed_errors +=
5079 le64_to_cpu(hw_stats->rx_discard_pkts);
5080
5081 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts);
5082
5083 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts);
5084 }
5085
5086 if (bp->flags & BNXT_FLAG_PORT_STATS) {
5087 struct rx_port_stats *rx = bp->hw_rx_port_stats;
5088 struct tx_port_stats *tx = bp->hw_tx_port_stats;
5089
5090 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames);
5091 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames);
5092 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) +
5093 le64_to_cpu(rx->rx_ovrsz_frames) +
5094 le64_to_cpu(rx->rx_runt_frames);
5095 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) +
5096 le64_to_cpu(rx->rx_jbr_frames);
5097 stats->collisions = le64_to_cpu(tx->tx_total_collisions);
5098 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns);
5099 stats->tx_errors = le64_to_cpu(tx->tx_err);
5100 }
5101
5102 return stats;
5103 }
5104
5105 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
5106 {
5107 struct net_device *dev = bp->dev;
5108 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5109 struct netdev_hw_addr *ha;
5110 u8 *haddr;
5111 int mc_count = 0;
5112 bool update = false;
5113 int off = 0;
5114
5115 netdev_for_each_mc_addr(ha, dev) {
5116 if (mc_count >= BNXT_MAX_MC_ADDRS) {
5117 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
5118 vnic->mc_list_count = 0;
5119 return false;
5120 }
5121 haddr = ha->addr;
5122 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
5123 memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
5124 update = true;
5125 }
5126 off += ETH_ALEN;
5127 mc_count++;
5128 }
5129 if (mc_count)
5130 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
5131
5132 if (mc_count != vnic->mc_list_count) {
5133 vnic->mc_list_count = mc_count;
5134 update = true;
5135 }
5136 return update;
5137 }
5138
5139 static bool bnxt_uc_list_updated(struct bnxt *bp)
5140 {
5141 struct net_device *dev = bp->dev;
5142 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5143 struct netdev_hw_addr *ha;
5144 int off = 0;
5145
5146 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
5147 return true;
5148
5149 netdev_for_each_uc_addr(ha, dev) {
5150 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
5151 return true;
5152
5153 off += ETH_ALEN;
5154 }
5155 return false;
5156 }
5157
5158 static void bnxt_set_rx_mode(struct net_device *dev)
5159 {
5160 struct bnxt *bp = netdev_priv(dev);
5161 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5162 u32 mask = vnic->rx_mask;
5163 bool mc_update = false;
5164 bool uc_update;
5165
5166 if (!netif_running(dev))
5167 return;
5168
5169 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
5170 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
5171 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST);
5172
5173 /* Only allow PF to be in promiscuous mode */
5174 if ((dev->flags & IFF_PROMISC) && BNXT_PF(bp))
5175 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
5176
5177 uc_update = bnxt_uc_list_updated(bp);
5178
5179 if (dev->flags & IFF_ALLMULTI) {
5180 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
5181 vnic->mc_list_count = 0;
5182 } else {
5183 mc_update = bnxt_mc_list_updated(bp, &mask);
5184 }
5185
5186 if (mask != vnic->rx_mask || uc_update || mc_update) {
5187 vnic->rx_mask = mask;
5188
5189 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
5190 schedule_work(&bp->sp_task);
5191 }
5192 }
5193
5194 static int bnxt_cfg_rx_mode(struct bnxt *bp)
5195 {
5196 struct net_device *dev = bp->dev;
5197 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5198 struct netdev_hw_addr *ha;
5199 int i, off = 0, rc;
5200 bool uc_update;
5201
5202 netif_addr_lock_bh(dev);
5203 uc_update = bnxt_uc_list_updated(bp);
5204 netif_addr_unlock_bh(dev);
5205
5206 if (!uc_update)
5207 goto skip_uc;
5208
5209 mutex_lock(&bp->hwrm_cmd_lock);
5210 for (i = 1; i < vnic->uc_filter_count; i++) {
5211 struct hwrm_cfa_l2_filter_free_input req = {0};
5212
5213 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
5214 -1);
5215
5216 req.l2_filter_id = vnic->fw_l2_filter_id[i];
5217
5218 rc = _hwrm_send_message(bp, &req, sizeof(req),
5219 HWRM_CMD_TIMEOUT);
5220 }
5221 mutex_unlock(&bp->hwrm_cmd_lock);
5222
5223 vnic->uc_filter_count = 1;
5224
5225 netif_addr_lock_bh(dev);
5226 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
5227 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
5228 } else {
5229 netdev_for_each_uc_addr(ha, dev) {
5230 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
5231 off += ETH_ALEN;
5232 vnic->uc_filter_count++;
5233 }
5234 }
5235 netif_addr_unlock_bh(dev);
5236
5237 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
5238 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
5239 if (rc) {
5240 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
5241 rc);
5242 vnic->uc_filter_count = i;
5243 return rc;
5244 }
5245 }
5246
5247 skip_uc:
5248 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
5249 if (rc)
5250 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %x\n",
5251 rc);
5252
5253 return rc;
5254 }
5255
5256 static bool bnxt_rfs_capable(struct bnxt *bp)
5257 {
5258 #ifdef CONFIG_RFS_ACCEL
5259 struct bnxt_pf_info *pf = &bp->pf;
5260 int vnics;
5261
5262 if (BNXT_VF(bp) || !(bp->flags & BNXT_FLAG_MSIX_CAP))
5263 return false;
5264
5265 vnics = 1 + bp->rx_nr_rings;
5266 if (vnics > pf->max_rsscos_ctxs || vnics > pf->max_vnics)
5267 return false;
5268
5269 return true;
5270 #else
5271 return false;
5272 #endif
5273 }
5274
5275 static netdev_features_t bnxt_fix_features(struct net_device *dev,
5276 netdev_features_t features)
5277 {
5278 struct bnxt *bp = netdev_priv(dev);
5279
5280 if (!bnxt_rfs_capable(bp))
5281 features &= ~NETIF_F_NTUPLE;
5282 return features;
5283 }
5284
5285 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
5286 {
5287 struct bnxt *bp = netdev_priv(dev);
5288 u32 flags = bp->flags;
5289 u32 changes;
5290 int rc = 0;
5291 bool re_init = false;
5292 bool update_tpa = false;
5293
5294 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
5295 if ((features & NETIF_F_GRO) && (bp->pdev->revision > 0))
5296 flags |= BNXT_FLAG_GRO;
5297 if (features & NETIF_F_LRO)
5298 flags |= BNXT_FLAG_LRO;
5299
5300 if (features & NETIF_F_HW_VLAN_CTAG_RX)
5301 flags |= BNXT_FLAG_STRIP_VLAN;
5302
5303 if (features & NETIF_F_NTUPLE)
5304 flags |= BNXT_FLAG_RFS;
5305
5306 changes = flags ^ bp->flags;
5307 if (changes & BNXT_FLAG_TPA) {
5308 update_tpa = true;
5309 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
5310 (flags & BNXT_FLAG_TPA) == 0)
5311 re_init = true;
5312 }
5313
5314 if (changes & ~BNXT_FLAG_TPA)
5315 re_init = true;
5316
5317 if (flags != bp->flags) {
5318 u32 old_flags = bp->flags;
5319
5320 bp->flags = flags;
5321
5322 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
5323 if (update_tpa)
5324 bnxt_set_ring_params(bp);
5325 return rc;
5326 }
5327
5328 if (re_init) {
5329 bnxt_close_nic(bp, false, false);
5330 if (update_tpa)
5331 bnxt_set_ring_params(bp);
5332
5333 return bnxt_open_nic(bp, false, false);
5334 }
5335 if (update_tpa) {
5336 rc = bnxt_set_tpa(bp,
5337 (flags & BNXT_FLAG_TPA) ?
5338 true : false);
5339 if (rc)
5340 bp->flags = old_flags;
5341 }
5342 }
5343 return rc;
5344 }
5345
5346 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
5347 {
5348 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
5349 int i = bnapi->index;
5350
5351 if (!txr)
5352 return;
5353
5354 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
5355 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
5356 txr->tx_cons);
5357 }
5358
5359 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
5360 {
5361 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
5362 int i = bnapi->index;
5363
5364 if (!rxr)
5365 return;
5366
5367 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
5368 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
5369 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
5370 rxr->rx_sw_agg_prod);
5371 }
5372
5373 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
5374 {
5375 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5376 int i = bnapi->index;
5377
5378 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
5379 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
5380 }
5381
5382 static void bnxt_dbg_dump_states(struct bnxt *bp)
5383 {
5384 int i;
5385 struct bnxt_napi *bnapi;
5386
5387 for (i = 0; i < bp->cp_nr_rings; i++) {
5388 bnapi = bp->bnapi[i];
5389 if (netif_msg_drv(bp)) {
5390 bnxt_dump_tx_sw_state(bnapi);
5391 bnxt_dump_rx_sw_state(bnapi);
5392 bnxt_dump_cp_sw_state(bnapi);
5393 }
5394 }
5395 }
5396
5397 static void bnxt_reset_task(struct bnxt *bp)
5398 {
5399 bnxt_dbg_dump_states(bp);
5400 if (netif_running(bp->dev)) {
5401 bnxt_close_nic(bp, false, false);
5402 bnxt_open_nic(bp, false, false);
5403 }
5404 }
5405
5406 static void bnxt_tx_timeout(struct net_device *dev)
5407 {
5408 struct bnxt *bp = netdev_priv(dev);
5409
5410 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n");
5411 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
5412 schedule_work(&bp->sp_task);
5413 }
5414
5415 #ifdef CONFIG_NET_POLL_CONTROLLER
5416 static void bnxt_poll_controller(struct net_device *dev)
5417 {
5418 struct bnxt *bp = netdev_priv(dev);
5419 int i;
5420
5421 for (i = 0; i < bp->cp_nr_rings; i++) {
5422 struct bnxt_irq *irq = &bp->irq_tbl[i];
5423
5424 disable_irq(irq->vector);
5425 irq->handler(irq->vector, bp->bnapi[i]);
5426 enable_irq(irq->vector);
5427 }
5428 }
5429 #endif
5430
5431 static void bnxt_timer(unsigned long data)
5432 {
5433 struct bnxt *bp = (struct bnxt *)data;
5434 struct net_device *dev = bp->dev;
5435
5436 if (!netif_running(dev))
5437 return;
5438
5439 if (atomic_read(&bp->intr_sem) != 0)
5440 goto bnxt_restart_timer;
5441
5442 if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS)) {
5443 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
5444 schedule_work(&bp->sp_task);
5445 }
5446 bnxt_restart_timer:
5447 mod_timer(&bp->timer, jiffies + bp->current_interval);
5448 }
5449
5450 static void bnxt_cfg_ntp_filters(struct bnxt *);
5451
5452 static void bnxt_sp_task(struct work_struct *work)
5453 {
5454 struct bnxt *bp = container_of(work, struct bnxt, sp_task);
5455 int rc;
5456
5457 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5458 smp_mb__after_atomic();
5459 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
5460 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5461 return;
5462 }
5463
5464 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
5465 bnxt_cfg_rx_mode(bp);
5466
5467 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
5468 bnxt_cfg_ntp_filters(bp);
5469 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
5470 rc = bnxt_update_link(bp, true);
5471 if (rc)
5472 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
5473 rc);
5474 }
5475 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
5476 bnxt_hwrm_exec_fwd_req(bp);
5477 if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) {
5478 bnxt_hwrm_tunnel_dst_port_alloc(
5479 bp, bp->vxlan_port,
5480 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
5481 }
5482 if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) {
5483 bnxt_hwrm_tunnel_dst_port_free(
5484 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
5485 }
5486 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) {
5487 /* bnxt_reset_task() calls bnxt_close_nic() which waits
5488 * for BNXT_STATE_IN_SP_TASK to clear.
5489 */
5490 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5491 rtnl_lock();
5492 bnxt_reset_task(bp);
5493 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5494 rtnl_unlock();
5495 }
5496
5497 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event))
5498 bnxt_hwrm_port_qstats(bp);
5499
5500 smp_mb__before_atomic();
5501 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5502 }
5503
5504 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
5505 {
5506 int rc;
5507 struct bnxt *bp = netdev_priv(dev);
5508
5509 SET_NETDEV_DEV(dev, &pdev->dev);
5510
5511 /* enable device (incl. PCI PM wakeup), and bus-mastering */
5512 rc = pci_enable_device(pdev);
5513 if (rc) {
5514 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
5515 goto init_err;
5516 }
5517
5518 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
5519 dev_err(&pdev->dev,
5520 "Cannot find PCI device base address, aborting\n");
5521 rc = -ENODEV;
5522 goto init_err_disable;
5523 }
5524
5525 rc = pci_request_regions(pdev, DRV_MODULE_NAME);
5526 if (rc) {
5527 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
5528 goto init_err_disable;
5529 }
5530
5531 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
5532 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
5533 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
5534 goto init_err_disable;
5535 }
5536
5537 pci_set_master(pdev);
5538
5539 bp->dev = dev;
5540 bp->pdev = pdev;
5541
5542 bp->bar0 = pci_ioremap_bar(pdev, 0);
5543 if (!bp->bar0) {
5544 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
5545 rc = -ENOMEM;
5546 goto init_err_release;
5547 }
5548
5549 bp->bar1 = pci_ioremap_bar(pdev, 2);
5550 if (!bp->bar1) {
5551 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n");
5552 rc = -ENOMEM;
5553 goto init_err_release;
5554 }
5555
5556 bp->bar2 = pci_ioremap_bar(pdev, 4);
5557 if (!bp->bar2) {
5558 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
5559 rc = -ENOMEM;
5560 goto init_err_release;
5561 }
5562
5563 pci_enable_pcie_error_reporting(pdev);
5564
5565 INIT_WORK(&bp->sp_task, bnxt_sp_task);
5566
5567 spin_lock_init(&bp->ntp_fltr_lock);
5568
5569 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
5570 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
5571
5572 /* tick values in micro seconds */
5573 bp->rx_coal_ticks = 12;
5574 bp->rx_coal_bufs = 30;
5575 bp->rx_coal_ticks_irq = 1;
5576 bp->rx_coal_bufs_irq = 2;
5577
5578 bp->tx_coal_ticks = 25;
5579 bp->tx_coal_bufs = 30;
5580 bp->tx_coal_ticks_irq = 2;
5581 bp->tx_coal_bufs_irq = 2;
5582
5583 init_timer(&bp->timer);
5584 bp->timer.data = (unsigned long)bp;
5585 bp->timer.function = bnxt_timer;
5586 bp->current_interval = BNXT_TIMER_INTERVAL;
5587
5588 clear_bit(BNXT_STATE_OPEN, &bp->state);
5589
5590 return 0;
5591
5592 init_err_release:
5593 if (bp->bar2) {
5594 pci_iounmap(pdev, bp->bar2);
5595 bp->bar2 = NULL;
5596 }
5597
5598 if (bp->bar1) {
5599 pci_iounmap(pdev, bp->bar1);
5600 bp->bar1 = NULL;
5601 }
5602
5603 if (bp->bar0) {
5604 pci_iounmap(pdev, bp->bar0);
5605 bp->bar0 = NULL;
5606 }
5607
5608 pci_release_regions(pdev);
5609
5610 init_err_disable:
5611 pci_disable_device(pdev);
5612
5613 init_err:
5614 return rc;
5615 }
5616
5617 /* rtnl_lock held */
5618 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
5619 {
5620 struct sockaddr *addr = p;
5621 struct bnxt *bp = netdev_priv(dev);
5622 int rc = 0;
5623
5624 if (!is_valid_ether_addr(addr->sa_data))
5625 return -EADDRNOTAVAIL;
5626
5627 #ifdef CONFIG_BNXT_SRIOV
5628 if (BNXT_VF(bp) && is_valid_ether_addr(bp->vf.mac_addr))
5629 return -EADDRNOTAVAIL;
5630 #endif
5631
5632 if (ether_addr_equal(addr->sa_data, dev->dev_addr))
5633 return 0;
5634
5635 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
5636 if (netif_running(dev)) {
5637 bnxt_close_nic(bp, false, false);
5638 rc = bnxt_open_nic(bp, false, false);
5639 }
5640
5641 return rc;
5642 }
5643
5644 /* rtnl_lock held */
5645 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
5646 {
5647 struct bnxt *bp = netdev_priv(dev);
5648
5649 if (new_mtu < 60 || new_mtu > 9000)
5650 return -EINVAL;
5651
5652 if (netif_running(dev))
5653 bnxt_close_nic(bp, false, false);
5654
5655 dev->mtu = new_mtu;
5656 bnxt_set_ring_params(bp);
5657
5658 if (netif_running(dev))
5659 return bnxt_open_nic(bp, false, false);
5660
5661 return 0;
5662 }
5663
5664 static int bnxt_setup_tc(struct net_device *dev, u32 handle, __be16 proto,
5665 struct tc_to_netdev *ntc)
5666 {
5667 struct bnxt *bp = netdev_priv(dev);
5668 u8 tc;
5669
5670 if (ntc->type != TC_SETUP_MQPRIO)
5671 return -EINVAL;
5672
5673 tc = ntc->tc;
5674
5675 if (tc > bp->max_tc) {
5676 netdev_err(dev, "too many traffic classes requested: %d Max supported is %d\n",
5677 tc, bp->max_tc);
5678 return -EINVAL;
5679 }
5680
5681 if (netdev_get_num_tc(dev) == tc)
5682 return 0;
5683
5684 if (tc) {
5685 int max_rx_rings, max_tx_rings, rc;
5686 bool sh = false;
5687
5688 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5689 sh = true;
5690
5691 rc = bnxt_get_max_rings(bp, &max_rx_rings, &max_tx_rings, sh);
5692 if (rc || bp->tx_nr_rings_per_tc * tc > max_tx_rings)
5693 return -ENOMEM;
5694 }
5695
5696 /* Needs to close the device and do hw resource re-allocations */
5697 if (netif_running(bp->dev))
5698 bnxt_close_nic(bp, true, false);
5699
5700 if (tc) {
5701 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
5702 netdev_set_num_tc(dev, tc);
5703 } else {
5704 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
5705 netdev_reset_tc(dev);
5706 }
5707 bp->cp_nr_rings = max_t(int, bp->tx_nr_rings, bp->rx_nr_rings);
5708 bp->num_stat_ctxs = bp->cp_nr_rings;
5709
5710 if (netif_running(bp->dev))
5711 return bnxt_open_nic(bp, true, false);
5712
5713 return 0;
5714 }
5715
5716 #ifdef CONFIG_RFS_ACCEL
5717 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
5718 struct bnxt_ntuple_filter *f2)
5719 {
5720 struct flow_keys *keys1 = &f1->fkeys;
5721 struct flow_keys *keys2 = &f2->fkeys;
5722
5723 if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src &&
5724 keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst &&
5725 keys1->ports.ports == keys2->ports.ports &&
5726 keys1->basic.ip_proto == keys2->basic.ip_proto &&
5727 keys1->basic.n_proto == keys2->basic.n_proto &&
5728 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr))
5729 return true;
5730
5731 return false;
5732 }
5733
5734 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
5735 u16 rxq_index, u32 flow_id)
5736 {
5737 struct bnxt *bp = netdev_priv(dev);
5738 struct bnxt_ntuple_filter *fltr, *new_fltr;
5739 struct flow_keys *fkeys;
5740 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
5741 int rc = 0, idx, bit_id;
5742 struct hlist_head *head;
5743
5744 if (skb->encapsulation)
5745 return -EPROTONOSUPPORT;
5746
5747 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
5748 if (!new_fltr)
5749 return -ENOMEM;
5750
5751 fkeys = &new_fltr->fkeys;
5752 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
5753 rc = -EPROTONOSUPPORT;
5754 goto err_free;
5755 }
5756
5757 if ((fkeys->basic.n_proto != htons(ETH_P_IP)) ||
5758 ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
5759 (fkeys->basic.ip_proto != IPPROTO_UDP))) {
5760 rc = -EPROTONOSUPPORT;
5761 goto err_free;
5762 }
5763
5764 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
5765
5766 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
5767 head = &bp->ntp_fltr_hash_tbl[idx];
5768 rcu_read_lock();
5769 hlist_for_each_entry_rcu(fltr, head, hash) {
5770 if (bnxt_fltr_match(fltr, new_fltr)) {
5771 rcu_read_unlock();
5772 rc = 0;
5773 goto err_free;
5774 }
5775 }
5776 rcu_read_unlock();
5777
5778 spin_lock_bh(&bp->ntp_fltr_lock);
5779 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
5780 BNXT_NTP_FLTR_MAX_FLTR, 0);
5781 if (bit_id < 0) {
5782 spin_unlock_bh(&bp->ntp_fltr_lock);
5783 rc = -ENOMEM;
5784 goto err_free;
5785 }
5786
5787 new_fltr->sw_id = (u16)bit_id;
5788 new_fltr->flow_id = flow_id;
5789 new_fltr->rxq = rxq_index;
5790 hlist_add_head_rcu(&new_fltr->hash, head);
5791 bp->ntp_fltr_count++;
5792 spin_unlock_bh(&bp->ntp_fltr_lock);
5793
5794 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
5795 schedule_work(&bp->sp_task);
5796
5797 return new_fltr->sw_id;
5798
5799 err_free:
5800 kfree(new_fltr);
5801 return rc;
5802 }
5803
5804 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
5805 {
5806 int i;
5807
5808 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
5809 struct hlist_head *head;
5810 struct hlist_node *tmp;
5811 struct bnxt_ntuple_filter *fltr;
5812 int rc;
5813
5814 head = &bp->ntp_fltr_hash_tbl[i];
5815 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
5816 bool del = false;
5817
5818 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
5819 if (rps_may_expire_flow(bp->dev, fltr->rxq,
5820 fltr->flow_id,
5821 fltr->sw_id)) {
5822 bnxt_hwrm_cfa_ntuple_filter_free(bp,
5823 fltr);
5824 del = true;
5825 }
5826 } else {
5827 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
5828 fltr);
5829 if (rc)
5830 del = true;
5831 else
5832 set_bit(BNXT_FLTR_VALID, &fltr->state);
5833 }
5834
5835 if (del) {
5836 spin_lock_bh(&bp->ntp_fltr_lock);
5837 hlist_del_rcu(&fltr->hash);
5838 bp->ntp_fltr_count--;
5839 spin_unlock_bh(&bp->ntp_fltr_lock);
5840 synchronize_rcu();
5841 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
5842 kfree(fltr);
5843 }
5844 }
5845 }
5846 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
5847 netdev_info(bp->dev, "Receive PF driver unload event!");
5848 }
5849
5850 #else
5851
5852 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
5853 {
5854 }
5855
5856 #endif /* CONFIG_RFS_ACCEL */
5857
5858 static void bnxt_add_vxlan_port(struct net_device *dev, sa_family_t sa_family,
5859 __be16 port)
5860 {
5861 struct bnxt *bp = netdev_priv(dev);
5862
5863 if (!netif_running(dev))
5864 return;
5865
5866 if (sa_family != AF_INET6 && sa_family != AF_INET)
5867 return;
5868
5869 if (bp->vxlan_port_cnt && bp->vxlan_port != port)
5870 return;
5871
5872 bp->vxlan_port_cnt++;
5873 if (bp->vxlan_port_cnt == 1) {
5874 bp->vxlan_port = port;
5875 set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
5876 schedule_work(&bp->sp_task);
5877 }
5878 }
5879
5880 static void bnxt_del_vxlan_port(struct net_device *dev, sa_family_t sa_family,
5881 __be16 port)
5882 {
5883 struct bnxt *bp = netdev_priv(dev);
5884
5885 if (!netif_running(dev))
5886 return;
5887
5888 if (sa_family != AF_INET6 && sa_family != AF_INET)
5889 return;
5890
5891 if (bp->vxlan_port_cnt && bp->vxlan_port == port) {
5892 bp->vxlan_port_cnt--;
5893
5894 if (bp->vxlan_port_cnt == 0) {
5895 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
5896 schedule_work(&bp->sp_task);
5897 }
5898 }
5899 }
5900
5901 static const struct net_device_ops bnxt_netdev_ops = {
5902 .ndo_open = bnxt_open,
5903 .ndo_start_xmit = bnxt_start_xmit,
5904 .ndo_stop = bnxt_close,
5905 .ndo_get_stats64 = bnxt_get_stats64,
5906 .ndo_set_rx_mode = bnxt_set_rx_mode,
5907 .ndo_do_ioctl = bnxt_ioctl,
5908 .ndo_validate_addr = eth_validate_addr,
5909 .ndo_set_mac_address = bnxt_change_mac_addr,
5910 .ndo_change_mtu = bnxt_change_mtu,
5911 .ndo_fix_features = bnxt_fix_features,
5912 .ndo_set_features = bnxt_set_features,
5913 .ndo_tx_timeout = bnxt_tx_timeout,
5914 #ifdef CONFIG_BNXT_SRIOV
5915 .ndo_get_vf_config = bnxt_get_vf_config,
5916 .ndo_set_vf_mac = bnxt_set_vf_mac,
5917 .ndo_set_vf_vlan = bnxt_set_vf_vlan,
5918 .ndo_set_vf_rate = bnxt_set_vf_bw,
5919 .ndo_set_vf_link_state = bnxt_set_vf_link_state,
5920 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk,
5921 #endif
5922 #ifdef CONFIG_NET_POLL_CONTROLLER
5923 .ndo_poll_controller = bnxt_poll_controller,
5924 #endif
5925 .ndo_setup_tc = bnxt_setup_tc,
5926 #ifdef CONFIG_RFS_ACCEL
5927 .ndo_rx_flow_steer = bnxt_rx_flow_steer,
5928 #endif
5929 .ndo_add_vxlan_port = bnxt_add_vxlan_port,
5930 .ndo_del_vxlan_port = bnxt_del_vxlan_port,
5931 #ifdef CONFIG_NET_RX_BUSY_POLL
5932 .ndo_busy_poll = bnxt_busy_poll,
5933 #endif
5934 };
5935
5936 static void bnxt_remove_one(struct pci_dev *pdev)
5937 {
5938 struct net_device *dev = pci_get_drvdata(pdev);
5939 struct bnxt *bp = netdev_priv(dev);
5940
5941 if (BNXT_PF(bp))
5942 bnxt_sriov_disable(bp);
5943
5944 pci_disable_pcie_error_reporting(pdev);
5945 unregister_netdev(dev);
5946 cancel_work_sync(&bp->sp_task);
5947 bp->sp_event = 0;
5948
5949 bnxt_hwrm_func_drv_unrgtr(bp);
5950 bnxt_free_hwrm_resources(bp);
5951 pci_iounmap(pdev, bp->bar2);
5952 pci_iounmap(pdev, bp->bar1);
5953 pci_iounmap(pdev, bp->bar0);
5954 free_netdev(dev);
5955
5956 pci_release_regions(pdev);
5957 pci_disable_device(pdev);
5958 }
5959
5960 static int bnxt_probe_phy(struct bnxt *bp)
5961 {
5962 int rc = 0;
5963 struct bnxt_link_info *link_info = &bp->link_info;
5964
5965 rc = bnxt_hwrm_phy_qcaps(bp);
5966 if (rc) {
5967 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
5968 rc);
5969 return rc;
5970 }
5971
5972 rc = bnxt_update_link(bp, false);
5973 if (rc) {
5974 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
5975 rc);
5976 return rc;
5977 }
5978
5979 /*initialize the ethool setting copy with NVM settings */
5980 if (BNXT_AUTO_MODE(link_info->auto_mode)) {
5981 link_info->autoneg = BNXT_AUTONEG_SPEED;
5982 if (bp->hwrm_spec_code >= 0x10201) {
5983 if (link_info->auto_pause_setting &
5984 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
5985 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
5986 } else {
5987 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
5988 }
5989 link_info->advertising = link_info->auto_link_speeds;
5990 } else {
5991 link_info->req_link_speed = link_info->force_link_speed;
5992 link_info->req_duplex = link_info->duplex_setting;
5993 }
5994 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
5995 link_info->req_flow_ctrl =
5996 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
5997 else
5998 link_info->req_flow_ctrl = link_info->force_pause_setting;
5999 return rc;
6000 }
6001
6002 static int bnxt_get_max_irq(struct pci_dev *pdev)
6003 {
6004 u16 ctrl;
6005
6006 if (!pdev->msix_cap)
6007 return 1;
6008
6009 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
6010 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
6011 }
6012
6013 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
6014 int *max_cp)
6015 {
6016 int max_ring_grps = 0;
6017
6018 #ifdef CONFIG_BNXT_SRIOV
6019 if (!BNXT_PF(bp)) {
6020 *max_tx = bp->vf.max_tx_rings;
6021 *max_rx = bp->vf.max_rx_rings;
6022 *max_cp = min_t(int, bp->vf.max_irqs, bp->vf.max_cp_rings);
6023 *max_cp = min_t(int, *max_cp, bp->vf.max_stat_ctxs);
6024 max_ring_grps = bp->vf.max_hw_ring_grps;
6025 } else
6026 #endif
6027 {
6028 *max_tx = bp->pf.max_tx_rings;
6029 *max_rx = bp->pf.max_rx_rings;
6030 *max_cp = min_t(int, bp->pf.max_irqs, bp->pf.max_cp_rings);
6031 *max_cp = min_t(int, *max_cp, bp->pf.max_stat_ctxs);
6032 max_ring_grps = bp->pf.max_hw_ring_grps;
6033 }
6034
6035 if (bp->flags & BNXT_FLAG_AGG_RINGS)
6036 *max_rx >>= 1;
6037 *max_rx = min_t(int, *max_rx, max_ring_grps);
6038 }
6039
6040 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
6041 {
6042 int rx, tx, cp;
6043
6044 _bnxt_get_max_rings(bp, &rx, &tx, &cp);
6045 if (!rx || !tx || !cp)
6046 return -ENOMEM;
6047
6048 *max_rx = rx;
6049 *max_tx = tx;
6050 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
6051 }
6052
6053 static int bnxt_set_dflt_rings(struct bnxt *bp)
6054 {
6055 int dflt_rings, max_rx_rings, max_tx_rings, rc;
6056 bool sh = true;
6057
6058 if (sh)
6059 bp->flags |= BNXT_FLAG_SHARED_RINGS;
6060 dflt_rings = netif_get_num_default_rss_queues();
6061 rc = bnxt_get_max_rings(bp, &max_rx_rings, &max_tx_rings, sh);
6062 if (rc)
6063 return rc;
6064 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
6065 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
6066 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
6067 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
6068 bp->tx_nr_rings + bp->rx_nr_rings;
6069 bp->num_stat_ctxs = bp->cp_nr_rings;
6070 return rc;
6071 }
6072
6073 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
6074 {
6075 static int version_printed;
6076 struct net_device *dev;
6077 struct bnxt *bp;
6078 int rc, max_irqs;
6079
6080 if (version_printed++ == 0)
6081 pr_info("%s", version);
6082
6083 max_irqs = bnxt_get_max_irq(pdev);
6084 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
6085 if (!dev)
6086 return -ENOMEM;
6087
6088 bp = netdev_priv(dev);
6089
6090 if (bnxt_vf_pciid(ent->driver_data))
6091 bp->flags |= BNXT_FLAG_VF;
6092
6093 if (pdev->msix_cap)
6094 bp->flags |= BNXT_FLAG_MSIX_CAP;
6095
6096 rc = bnxt_init_board(pdev, dev);
6097 if (rc < 0)
6098 goto init_err_free;
6099
6100 dev->netdev_ops = &bnxt_netdev_ops;
6101 dev->watchdog_timeo = BNXT_TX_TIMEOUT;
6102 dev->ethtool_ops = &bnxt_ethtool_ops;
6103
6104 pci_set_drvdata(pdev, dev);
6105
6106 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
6107 NETIF_F_TSO | NETIF_F_TSO6 |
6108 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
6109 NETIF_F_GSO_IPIP | NETIF_F_GSO_SIT |
6110 NETIF_F_RXHASH |
6111 NETIF_F_RXCSUM | NETIF_F_LRO | NETIF_F_GRO;
6112
6113 dev->hw_enc_features =
6114 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
6115 NETIF_F_TSO | NETIF_F_TSO6 |
6116 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
6117 NETIF_F_GSO_IPIP | NETIF_F_GSO_SIT;
6118 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
6119 dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |
6120 NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX;
6121 dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
6122 dev->priv_flags |= IFF_UNICAST_FLT;
6123
6124 #ifdef CONFIG_BNXT_SRIOV
6125 init_waitqueue_head(&bp->sriov_cfg_wait);
6126 #endif
6127 rc = bnxt_alloc_hwrm_resources(bp);
6128 if (rc)
6129 goto init_err;
6130
6131 mutex_init(&bp->hwrm_cmd_lock);
6132 bnxt_hwrm_ver_get(bp);
6133
6134 rc = bnxt_hwrm_func_drv_rgtr(bp);
6135 if (rc)
6136 goto init_err;
6137
6138 /* Get the MAX capabilities for this function */
6139 rc = bnxt_hwrm_func_qcaps(bp);
6140 if (rc) {
6141 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
6142 rc);
6143 rc = -1;
6144 goto init_err;
6145 }
6146
6147 rc = bnxt_hwrm_queue_qportcfg(bp);
6148 if (rc) {
6149 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n",
6150 rc);
6151 rc = -1;
6152 goto init_err;
6153 }
6154
6155 bnxt_set_tpa_flags(bp);
6156 bnxt_set_ring_params(bp);
6157 if (BNXT_PF(bp))
6158 bp->pf.max_irqs = max_irqs;
6159 #if defined(CONFIG_BNXT_SRIOV)
6160 else
6161 bp->vf.max_irqs = max_irqs;
6162 #endif
6163 bnxt_set_dflt_rings(bp);
6164
6165 if (BNXT_PF(bp)) {
6166 dev->hw_features |= NETIF_F_NTUPLE;
6167 if (bnxt_rfs_capable(bp)) {
6168 bp->flags |= BNXT_FLAG_RFS;
6169 dev->features |= NETIF_F_NTUPLE;
6170 }
6171 }
6172
6173 if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX)
6174 bp->flags |= BNXT_FLAG_STRIP_VLAN;
6175
6176 rc = bnxt_probe_phy(bp);
6177 if (rc)
6178 goto init_err;
6179
6180 rc = register_netdev(dev);
6181 if (rc)
6182 goto init_err;
6183
6184 netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
6185 board_info[ent->driver_data].name,
6186 (long)pci_resource_start(pdev, 0), dev->dev_addr);
6187
6188 return 0;
6189
6190 init_err:
6191 pci_iounmap(pdev, bp->bar0);
6192 pci_release_regions(pdev);
6193 pci_disable_device(pdev);
6194
6195 init_err_free:
6196 free_netdev(dev);
6197 return rc;
6198 }
6199
6200 /**
6201 * bnxt_io_error_detected - called when PCI error is detected
6202 * @pdev: Pointer to PCI device
6203 * @state: The current pci connection state
6204 *
6205 * This function is called after a PCI bus error affecting
6206 * this device has been detected.
6207 */
6208 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
6209 pci_channel_state_t state)
6210 {
6211 struct net_device *netdev = pci_get_drvdata(pdev);
6212
6213 netdev_info(netdev, "PCI I/O error detected\n");
6214
6215 rtnl_lock();
6216 netif_device_detach(netdev);
6217
6218 if (state == pci_channel_io_perm_failure) {
6219 rtnl_unlock();
6220 return PCI_ERS_RESULT_DISCONNECT;
6221 }
6222
6223 if (netif_running(netdev))
6224 bnxt_close(netdev);
6225
6226 pci_disable_device(pdev);
6227 rtnl_unlock();
6228
6229 /* Request a slot slot reset. */
6230 return PCI_ERS_RESULT_NEED_RESET;
6231 }
6232
6233 /**
6234 * bnxt_io_slot_reset - called after the pci bus has been reset.
6235 * @pdev: Pointer to PCI device
6236 *
6237 * Restart the card from scratch, as if from a cold-boot.
6238 * At this point, the card has exprienced a hard reset,
6239 * followed by fixups by BIOS, and has its config space
6240 * set up identically to what it was at cold boot.
6241 */
6242 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
6243 {
6244 struct net_device *netdev = pci_get_drvdata(pdev);
6245 struct bnxt *bp = netdev_priv(netdev);
6246 int err = 0;
6247 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
6248
6249 netdev_info(bp->dev, "PCI Slot Reset\n");
6250
6251 rtnl_lock();
6252
6253 if (pci_enable_device(pdev)) {
6254 dev_err(&pdev->dev,
6255 "Cannot re-enable PCI device after reset.\n");
6256 } else {
6257 pci_set_master(pdev);
6258
6259 if (netif_running(netdev))
6260 err = bnxt_open(netdev);
6261
6262 if (!err)
6263 result = PCI_ERS_RESULT_RECOVERED;
6264 }
6265
6266 if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev))
6267 dev_close(netdev);
6268
6269 rtnl_unlock();
6270
6271 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6272 if (err) {
6273 dev_err(&pdev->dev,
6274 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
6275 err); /* non-fatal, continue */
6276 }
6277
6278 return PCI_ERS_RESULT_RECOVERED;
6279 }
6280
6281 /**
6282 * bnxt_io_resume - called when traffic can start flowing again.
6283 * @pdev: Pointer to PCI device
6284 *
6285 * This callback is called when the error recovery driver tells
6286 * us that its OK to resume normal operation.
6287 */
6288 static void bnxt_io_resume(struct pci_dev *pdev)
6289 {
6290 struct net_device *netdev = pci_get_drvdata(pdev);
6291
6292 rtnl_lock();
6293
6294 netif_device_attach(netdev);
6295
6296 rtnl_unlock();
6297 }
6298
6299 static const struct pci_error_handlers bnxt_err_handler = {
6300 .error_detected = bnxt_io_error_detected,
6301 .slot_reset = bnxt_io_slot_reset,
6302 .resume = bnxt_io_resume
6303 };
6304
6305 static struct pci_driver bnxt_pci_driver = {
6306 .name = DRV_MODULE_NAME,
6307 .id_table = bnxt_pci_tbl,
6308 .probe = bnxt_init_one,
6309 .remove = bnxt_remove_one,
6310 .err_handler = &bnxt_err_handler,
6311 #if defined(CONFIG_BNXT_SRIOV)
6312 .sriov_configure = bnxt_sriov_configure,
6313 #endif
6314 };
6315
6316 module_pci_driver(bnxt_pci_driver);
Linux kernel - Deliverables
This page took 0.156251 seconds and 4 git commands to generate.