projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'regulator-v3.13' of git://git.kernel.org/pub/scm/linux/kernel/git/broonie...
[deliverable/linux.git] / drivers / net / ethernet / broadcom / bnx2x / bnx2x_cmn.c
1 /* bnx2x_cmn.c: Broadcom Everest network driver.
2 *
3 * Copyright (c) 2007-2013 Broadcom Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
8 *
9 * Maintained by: Eilon Greenstein <eilong@broadcom.com>
10 * Written by: Eliezer Tamir
11 * Based on code from Michael Chan's bnx2 driver
12 * UDP CSUM errata workaround by Arik Gendelman
13 * Slowpath and fastpath rework by Vladislav Zolotarov
14 * Statistics and Link management by Yitchak Gertner
15 *
16 */
17
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19
20 #include <linux/etherdevice.h>
21 #include <linux/if_vlan.h>
22 #include <linux/interrupt.h>
23 #include <linux/ip.h>
24 #include <net/tcp.h>
25 #include <net/ipv6.h>
26 #include <net/ip6_checksum.h>
27 #include <net/busy_poll.h>
28 #include <linux/prefetch.h>
29 #include "bnx2x_cmn.h"
30 #include "bnx2x_init.h"
31 #include "bnx2x_sp.h"
32
33 /**
34 * bnx2x_move_fp - move content of the fastpath structure.
35 *
36 * @bp: driver handle
37 * @from: source FP index
38 * @to: destination FP index
39 *
40 * Makes sure the contents of the bp->fp[to].napi is kept
41 * intact. This is done by first copying the napi struct from
42 * the target to the source, and then mem copying the entire
43 * source onto the target. Update txdata pointers and related
44 * content.
45 */
46 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
47 {
48 struct bnx2x_fastpath *from_fp = &bp->fp[from];
49 struct bnx2x_fastpath *to_fp = &bp->fp[to];
50 struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
51 struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
52 struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
53 struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
54 int old_max_eth_txqs, new_max_eth_txqs;
55 int old_txdata_index = 0, new_txdata_index = 0;
56 struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
57
58 /* Copy the NAPI object as it has been already initialized */
59 from_fp->napi = to_fp->napi;
60
61 /* Move bnx2x_fastpath contents */
62 memcpy(to_fp, from_fp, sizeof(*to_fp));
63 to_fp->index = to;
64
65 /* Retain the tpa_info of the original `to' version as we don't want
66 * 2 FPs to contain the same tpa_info pointer.
67 */
68 to_fp->tpa_info = old_tpa_info;
69
70 /* move sp_objs contents as well, as their indices match fp ones */
71 memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
72
73 /* move fp_stats contents as well, as their indices match fp ones */
74 memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
75
76 /* Update txdata pointers in fp and move txdata content accordingly:
77 * Each fp consumes 'max_cos' txdata structures, so the index should be
78 * decremented by max_cos x delta.
79 */
80
81 old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
82 new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
83 (bp)->max_cos;
84 if (from == FCOE_IDX(bp)) {
85 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
86 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
87 }
88
89 memcpy(&bp->bnx2x_txq[new_txdata_index],
90 &bp->bnx2x_txq[old_txdata_index],
91 sizeof(struct bnx2x_fp_txdata));
92 to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
93 }
94
95 /**
96 * bnx2x_fill_fw_str - Fill buffer with FW version string.
97 *
98 * @bp: driver handle
99 * @buf: character buffer to fill with the fw name
100 * @buf_len: length of the above buffer
101 *
102 */
103 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
104 {
105 if (IS_PF(bp)) {
106 u8 phy_fw_ver[PHY_FW_VER_LEN];
107
108 phy_fw_ver[0] = '\0';
109 bnx2x_get_ext_phy_fw_version(&bp->link_params,
110 phy_fw_ver, PHY_FW_VER_LEN);
111 strlcpy(buf, bp->fw_ver, buf_len);
112 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
113 "bc %d.%d.%d%s%s",
114 (bp->common.bc_ver & 0xff0000) >> 16,
115 (bp->common.bc_ver & 0xff00) >> 8,
116 (bp->common.bc_ver & 0xff),
117 ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
118 } else {
119 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
120 }
121 }
122
123 /**
124 * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
125 *
126 * @bp: driver handle
127 * @delta: number of eth queues which were not allocated
128 */
129 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
130 {
131 int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
132
133 /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
134 * backward along the array could cause memory to be overridden
135 */
136 for (cos = 1; cos < bp->max_cos; cos++) {
137 for (i = 0; i < old_eth_num - delta; i++) {
138 struct bnx2x_fastpath *fp = &bp->fp[i];
139 int new_idx = cos * (old_eth_num - delta) + i;
140
141 memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
142 sizeof(struct bnx2x_fp_txdata));
143 fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
144 }
145 }
146 }
147
148 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
149
150 /* free skb in the packet ring at pos idx
151 * return idx of last bd freed
152 */
153 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
154 u16 idx, unsigned int *pkts_compl,
155 unsigned int *bytes_compl)
156 {
157 struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
158 struct eth_tx_start_bd *tx_start_bd;
159 struct eth_tx_bd *tx_data_bd;
160 struct sk_buff *skb = tx_buf->skb;
161 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
162 int nbd;
163
164 /* prefetch skb end pointer to speedup dev_kfree_skb() */
165 prefetch(&skb->end);
166
167 DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d buff @(%p)->skb %p\n",
168 txdata->txq_index, idx, tx_buf, skb);
169
170 /* unmap first bd */
171 tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
172 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
173 BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
174
175 nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
176 #ifdef BNX2X_STOP_ON_ERROR
177 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
178 BNX2X_ERR("BAD nbd!\n");
179 bnx2x_panic();
180 }
181 #endif
182 new_cons = nbd + tx_buf->first_bd;
183
184 /* Get the next bd */
185 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
186
187 /* Skip a parse bd... */
188 --nbd;
189 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
190
191 /* ...and the TSO split header bd since they have no mapping */
192 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
193 --nbd;
194 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
195 }
196
197 /* now free frags */
198 while (nbd > 0) {
199
200 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
201 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
202 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
203 if (--nbd)
204 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
205 }
206
207 /* release skb */
208 WARN_ON(!skb);
209 if (likely(skb)) {
210 (*pkts_compl)++;
211 (*bytes_compl) += skb->len;
212 }
213
214 dev_kfree_skb_any(skb);
215 tx_buf->first_bd = 0;
216 tx_buf->skb = NULL;
217
218 return new_cons;
219 }
220
221 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
222 {
223 struct netdev_queue *txq;
224 u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
225 unsigned int pkts_compl = 0, bytes_compl = 0;
226
227 #ifdef BNX2X_STOP_ON_ERROR
228 if (unlikely(bp->panic))
229 return -1;
230 #endif
231
232 txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
233 hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
234 sw_cons = txdata->tx_pkt_cons;
235
236 while (sw_cons != hw_cons) {
237 u16 pkt_cons;
238
239 pkt_cons = TX_BD(sw_cons);
240
241 DP(NETIF_MSG_TX_DONE,
242 "queue[%d]: hw_cons %u sw_cons %u pkt_cons %u\n",
243 txdata->txq_index, hw_cons, sw_cons, pkt_cons);
244
245 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
246 &pkts_compl, &bytes_compl);
247
248 sw_cons++;
249 }
250
251 netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
252
253 txdata->tx_pkt_cons = sw_cons;
254 txdata->tx_bd_cons = bd_cons;
255
256 /* Need to make the tx_bd_cons update visible to start_xmit()
257 * before checking for netif_tx_queue_stopped(). Without the
258 * memory barrier, there is a small possibility that
259 * start_xmit() will miss it and cause the queue to be stopped
260 * forever.
261 * On the other hand we need an rmb() here to ensure the proper
262 * ordering of bit testing in the following
263 * netif_tx_queue_stopped(txq) call.
264 */
265 smp_mb();
266
267 if (unlikely(netif_tx_queue_stopped(txq))) {
268 /* Taking tx_lock() is needed to prevent re-enabling the queue
269 * while it's empty. This could have happen if rx_action() gets
270 * suspended in bnx2x_tx_int() after the condition before
271 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
272 *
273 * stops the queue->sees fresh tx_bd_cons->releases the queue->
274 * sends some packets consuming the whole queue again->
275 * stops the queue
276 */
277
278 __netif_tx_lock(txq, smp_processor_id());
279
280 if ((netif_tx_queue_stopped(txq)) &&
281 (bp->state == BNX2X_STATE_OPEN) &&
282 (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
283 netif_tx_wake_queue(txq);
284
285 __netif_tx_unlock(txq);
286 }
287 return 0;
288 }
289
290 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
291 u16 idx)
292 {
293 u16 last_max = fp->last_max_sge;
294
295 if (SUB_S16(idx, last_max) > 0)
296 fp->last_max_sge = idx;
297 }
298
299 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
300 u16 sge_len,
301 struct eth_end_agg_rx_cqe *cqe)
302 {
303 struct bnx2x *bp = fp->bp;
304 u16 last_max, last_elem, first_elem;
305 u16 delta = 0;
306 u16 i;
307
308 if (!sge_len)
309 return;
310
311 /* First mark all used pages */
312 for (i = 0; i < sge_len; i++)
313 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
314 RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
315
316 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
317 sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
318
319 /* Here we assume that the last SGE index is the biggest */
320 prefetch((void *)(fp->sge_mask));
321 bnx2x_update_last_max_sge(fp,
322 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
323
324 last_max = RX_SGE(fp->last_max_sge);
325 last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
326 first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
327
328 /* If ring is not full */
329 if (last_elem + 1 != first_elem)
330 last_elem++;
331
332 /* Now update the prod */
333 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
334 if (likely(fp->sge_mask[i]))
335 break;
336
337 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
338 delta += BIT_VEC64_ELEM_SZ;
339 }
340
341 if (delta > 0) {
342 fp->rx_sge_prod += delta;
343 /* clear page-end entries */
344 bnx2x_clear_sge_mask_next_elems(fp);
345 }
346
347 DP(NETIF_MSG_RX_STATUS,
348 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n",
349 fp->last_max_sge, fp->rx_sge_prod);
350 }
351
352 /* Get Toeplitz hash value in the skb using the value from the
353 * CQE (calculated by HW).
354 */
355 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
356 const struct eth_fast_path_rx_cqe *cqe,
357 bool *l4_rxhash)
358 {
359 /* Get Toeplitz hash from CQE */
360 if ((bp->dev->features & NETIF_F_RXHASH) &&
361 (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
362 enum eth_rss_hash_type htype;
363
364 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
365 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
366 (htype == TCP_IPV6_HASH_TYPE);
367 return le32_to_cpu(cqe->rss_hash_result);
368 }
369 *l4_rxhash = false;
370 return 0;
371 }
372
373 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
374 u16 cons, u16 prod,
375 struct eth_fast_path_rx_cqe *cqe)
376 {
377 struct bnx2x *bp = fp->bp;
378 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
379 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
380 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
381 dma_addr_t mapping;
382 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
383 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
384
385 /* print error if current state != stop */
386 if (tpa_info->tpa_state != BNX2X_TPA_STOP)
387 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
388
389 /* Try to map an empty data buffer from the aggregation info */
390 mapping = dma_map_single(&bp->pdev->dev,
391 first_buf->data + NET_SKB_PAD,
392 fp->rx_buf_size, DMA_FROM_DEVICE);
393 /*
394 * ...if it fails - move the skb from the consumer to the producer
395 * and set the current aggregation state as ERROR to drop it
396 * when TPA_STOP arrives.
397 */
398
399 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
400 /* Move the BD from the consumer to the producer */
401 bnx2x_reuse_rx_data(fp, cons, prod);
402 tpa_info->tpa_state = BNX2X_TPA_ERROR;
403 return;
404 }
405
406 /* move empty data from pool to prod */
407 prod_rx_buf->data = first_buf->data;
408 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
409 /* point prod_bd to new data */
410 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
411 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
412
413 /* move partial skb from cons to pool (don't unmap yet) */
414 *first_buf = *cons_rx_buf;
415
416 /* mark bin state as START */
417 tpa_info->parsing_flags =
418 le16_to_cpu(cqe->pars_flags.flags);
419 tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
420 tpa_info->tpa_state = BNX2X_TPA_START;
421 tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
422 tpa_info->placement_offset = cqe->placement_offset;
423 tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
424 if (fp->mode == TPA_MODE_GRO) {
425 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
426 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
427 tpa_info->gro_size = gro_size;
428 }
429
430 #ifdef BNX2X_STOP_ON_ERROR
431 fp->tpa_queue_used |= (1 << queue);
432 #ifdef _ASM_GENERIC_INT_L64_H
433 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
434 #else
435 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
436 #endif
437 fp->tpa_queue_used);
438 #endif
439 }
440
441 /* Timestamp option length allowed for TPA aggregation:
442 *
443 * nop nop kind length echo val
444 */
445 #define TPA_TSTAMP_OPT_LEN 12
446 /**
447 * bnx2x_set_gro_params - compute GRO values
448 *
449 * @skb: packet skb
450 * @parsing_flags: parsing flags from the START CQE
451 * @len_on_bd: total length of the first packet for the
452 * aggregation.
453 * @pkt_len: length of all segments
454 *
455 * Approximate value of the MSS for this aggregation calculated using
456 * the first packet of it.
457 * Compute number of aggregated segments, and gso_type.
458 */
459 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
460 u16 len_on_bd, unsigned int pkt_len,
461 u16 num_of_coalesced_segs)
462 {
463 /* TPA aggregation won't have either IP options or TCP options
464 * other than timestamp or IPv6 extension headers.
465 */
466 u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
467
468 if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
469 PRS_FLAG_OVERETH_IPV6) {
470 hdrs_len += sizeof(struct ipv6hdr);
471 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
472 } else {
473 hdrs_len += sizeof(struct iphdr);
474 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
475 }
476
477 /* Check if there was a TCP timestamp, if there is it's will
478 * always be 12 bytes length: nop nop kind length echo val.
479 *
480 * Otherwise FW would close the aggregation.
481 */
482 if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
483 hdrs_len += TPA_TSTAMP_OPT_LEN;
484
485 skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
486
487 /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
488 * to skb_shinfo(skb)->gso_segs
489 */
490 NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
491 }
492
493 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
494 u16 index, gfp_t gfp_mask)
495 {
496 struct page *page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT);
497 struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
498 struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
499 dma_addr_t mapping;
500
501 if (unlikely(page == NULL)) {
502 BNX2X_ERR("Can't alloc sge\n");
503 return -ENOMEM;
504 }
505
506 mapping = dma_map_page(&bp->pdev->dev, page, 0,
507 SGE_PAGES, DMA_FROM_DEVICE);
508 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
509 __free_pages(page, PAGES_PER_SGE_SHIFT);
510 BNX2X_ERR("Can't map sge\n");
511 return -ENOMEM;
512 }
513
514 sw_buf->page = page;
515 dma_unmap_addr_set(sw_buf, mapping, mapping);
516
517 sge->addr_hi = cpu_to_le32(U64_HI(mapping));
518 sge->addr_lo = cpu_to_le32(U64_LO(mapping));
519
520 return 0;
521 }
522
523 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
524 struct bnx2x_agg_info *tpa_info,
525 u16 pages,
526 struct sk_buff *skb,
527 struct eth_end_agg_rx_cqe *cqe,
528 u16 cqe_idx)
529 {
530 struct sw_rx_page *rx_pg, old_rx_pg;
531 u32 i, frag_len, frag_size;
532 int err, j, frag_id = 0;
533 u16 len_on_bd = tpa_info->len_on_bd;
534 u16 full_page = 0, gro_size = 0;
535
536 frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
537
538 if (fp->mode == TPA_MODE_GRO) {
539 gro_size = tpa_info->gro_size;
540 full_page = tpa_info->full_page;
541 }
542
543 /* This is needed in order to enable forwarding support */
544 if (frag_size)
545 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
546 le16_to_cpu(cqe->pkt_len),
547 le16_to_cpu(cqe->num_of_coalesced_segs));
548
549 #ifdef BNX2X_STOP_ON_ERROR
550 if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
551 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
552 pages, cqe_idx);
553 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
554 bnx2x_panic();
555 return -EINVAL;
556 }
557 #endif
558
559 /* Run through the SGL and compose the fragmented skb */
560 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
561 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
562
563 /* FW gives the indices of the SGE as if the ring is an array
564 (meaning that "next" element will consume 2 indices) */
565 if (fp->mode == TPA_MODE_GRO)
566 frag_len = min_t(u32, frag_size, (u32)full_page);
567 else /* LRO */
568 frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
569
570 rx_pg = &fp->rx_page_ring[sge_idx];
571 old_rx_pg = *rx_pg;
572
573 /* If we fail to allocate a substitute page, we simply stop
574 where we are and drop the whole packet */
575 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC);
576 if (unlikely(err)) {
577 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
578 return err;
579 }
580
581 /* Unmap the page as we're going to pass it to the stack */
582 dma_unmap_page(&bp->pdev->dev,
583 dma_unmap_addr(&old_rx_pg, mapping),
584 SGE_PAGES, DMA_FROM_DEVICE);
585 /* Add one frag and update the appropriate fields in the skb */
586 if (fp->mode == TPA_MODE_LRO)
587 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
588 else { /* GRO */
589 int rem;
590 int offset = 0;
591 for (rem = frag_len; rem > 0; rem -= gro_size) {
592 int len = rem > gro_size ? gro_size : rem;
593 skb_fill_page_desc(skb, frag_id++,
594 old_rx_pg.page, offset, len);
595 if (offset)
596 get_page(old_rx_pg.page);
597 offset += len;
598 }
599 }
600
601 skb->data_len += frag_len;
602 skb->truesize += SGE_PAGES;
603 skb->len += frag_len;
604
605 frag_size -= frag_len;
606 }
607
608 return 0;
609 }
610
611 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
612 {
613 if (fp->rx_frag_size)
614 put_page(virt_to_head_page(data));
615 else
616 kfree(data);
617 }
618
619 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
620 {
621 if (fp->rx_frag_size) {
622 /* GFP_KERNEL allocations are used only during initialization */
623 if (unlikely(gfp_mask & __GFP_WAIT))
624 return (void *)__get_free_page(gfp_mask);
625
626 return netdev_alloc_frag(fp->rx_frag_size);
627 }
628
629 return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
630 }
631
632 #ifdef CONFIG_INET
633 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
634 {
635 const struct iphdr *iph = ip_hdr(skb);
636 struct tcphdr *th;
637
638 skb_set_transport_header(skb, sizeof(struct iphdr));
639 th = tcp_hdr(skb);
640
641 th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
642 iph->saddr, iph->daddr, 0);
643 }
644
645 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
646 {
647 struct ipv6hdr *iph = ipv6_hdr(skb);
648 struct tcphdr *th;
649
650 skb_set_transport_header(skb, sizeof(struct ipv6hdr));
651 th = tcp_hdr(skb);
652
653 th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
654 &iph->saddr, &iph->daddr, 0);
655 }
656
657 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
658 void (*gro_func)(struct bnx2x*, struct sk_buff*))
659 {
660 skb_set_network_header(skb, 0);
661 gro_func(bp, skb);
662 tcp_gro_complete(skb);
663 }
664 #endif
665
666 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
667 struct sk_buff *skb)
668 {
669 #ifdef CONFIG_INET
670 if (skb_shinfo(skb)->gso_size) {
671 switch (be16_to_cpu(skb->protocol)) {
672 case ETH_P_IP:
673 bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
674 break;
675 case ETH_P_IPV6:
676 bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
677 break;
678 default:
679 BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
680 be16_to_cpu(skb->protocol));
681 }
682 }
683 #endif
684 skb_record_rx_queue(skb, fp->rx_queue);
685 napi_gro_receive(&fp->napi, skb);
686 }
687
688 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
689 struct bnx2x_agg_info *tpa_info,
690 u16 pages,
691 struct eth_end_agg_rx_cqe *cqe,
692 u16 cqe_idx)
693 {
694 struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
695 u8 pad = tpa_info->placement_offset;
696 u16 len = tpa_info->len_on_bd;
697 struct sk_buff *skb = NULL;
698 u8 *new_data, *data = rx_buf->data;
699 u8 old_tpa_state = tpa_info->tpa_state;
700
701 tpa_info->tpa_state = BNX2X_TPA_STOP;
702
703 /* If we there was an error during the handling of the TPA_START -
704 * drop this aggregation.
705 */
706 if (old_tpa_state == BNX2X_TPA_ERROR)
707 goto drop;
708
709 /* Try to allocate the new data */
710 new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC);
711 /* Unmap skb in the pool anyway, as we are going to change
712 pool entry status to BNX2X_TPA_STOP even if new skb allocation
713 fails. */
714 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
715 fp->rx_buf_size, DMA_FROM_DEVICE);
716 if (likely(new_data))
717 skb = build_skb(data, fp->rx_frag_size);
718
719 if (likely(skb)) {
720 #ifdef BNX2X_STOP_ON_ERROR
721 if (pad + len > fp->rx_buf_size) {
722 BNX2X_ERR("skb_put is about to fail... pad %d len %d rx_buf_size %d\n",
723 pad, len, fp->rx_buf_size);
724 bnx2x_panic();
725 return;
726 }
727 #endif
728
729 skb_reserve(skb, pad + NET_SKB_PAD);
730 skb_put(skb, len);
731 skb->rxhash = tpa_info->rxhash;
732 skb->l4_rxhash = tpa_info->l4_rxhash;
733
734 skb->protocol = eth_type_trans(skb, bp->dev);
735 skb->ip_summed = CHECKSUM_UNNECESSARY;
736
737 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
738 skb, cqe, cqe_idx)) {
739 if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
740 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
741 bnx2x_gro_receive(bp, fp, skb);
742 } else {
743 DP(NETIF_MSG_RX_STATUS,
744 "Failed to allocate new pages - dropping packet!\n");
745 dev_kfree_skb_any(skb);
746 }
747
748 /* put new data in bin */
749 rx_buf->data = new_data;
750
751 return;
752 }
753 bnx2x_frag_free(fp, new_data);
754 drop:
755 /* drop the packet and keep the buffer in the bin */
756 DP(NETIF_MSG_RX_STATUS,
757 "Failed to allocate or map a new skb - dropping packet!\n");
758 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
759 }
760
761 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
762 u16 index, gfp_t gfp_mask)
763 {
764 u8 *data;
765 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
766 struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
767 dma_addr_t mapping;
768
769 data = bnx2x_frag_alloc(fp, gfp_mask);
770 if (unlikely(data == NULL))
771 return -ENOMEM;
772
773 mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
774 fp->rx_buf_size,
775 DMA_FROM_DEVICE);
776 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
777 bnx2x_frag_free(fp, data);
778 BNX2X_ERR("Can't map rx data\n");
779 return -ENOMEM;
780 }
781
782 rx_buf->data = data;
783 dma_unmap_addr_set(rx_buf, mapping, mapping);
784
785 rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
786 rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
787
788 return 0;
789 }
790
791 static
792 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
793 struct bnx2x_fastpath *fp,
794 struct bnx2x_eth_q_stats *qstats)
795 {
796 /* Do nothing if no L4 csum validation was done.
797 * We do not check whether IP csum was validated. For IPv4 we assume
798 * that if the card got as far as validating the L4 csum, it also
799 * validated the IP csum. IPv6 has no IP csum.
800 */
801 if (cqe->fast_path_cqe.status_flags &
802 ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
803 return;
804
805 /* If L4 validation was done, check if an error was found. */
806
807 if (cqe->fast_path_cqe.type_error_flags &
808 (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
809 ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
810 qstats->hw_csum_err++;
811 else
812 skb->ip_summed = CHECKSUM_UNNECESSARY;
813 }
814
815 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
816 {
817 struct bnx2x *bp = fp->bp;
818 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
819 u16 sw_comp_cons, sw_comp_prod;
820 int rx_pkt = 0;
821 union eth_rx_cqe *cqe;
822 struct eth_fast_path_rx_cqe *cqe_fp;
823
824 #ifdef BNX2X_STOP_ON_ERROR
825 if (unlikely(bp->panic))
826 return 0;
827 #endif
828
829 bd_cons = fp->rx_bd_cons;
830 bd_prod = fp->rx_bd_prod;
831 bd_prod_fw = bd_prod;
832 sw_comp_cons = fp->rx_comp_cons;
833 sw_comp_prod = fp->rx_comp_prod;
834
835 comp_ring_cons = RCQ_BD(sw_comp_cons);
836 cqe = &fp->rx_comp_ring[comp_ring_cons];
837 cqe_fp = &cqe->fast_path_cqe;
838
839 DP(NETIF_MSG_RX_STATUS,
840 "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
841
842 while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
843 struct sw_rx_bd *rx_buf = NULL;
844 struct sk_buff *skb;
845 u8 cqe_fp_flags;
846 enum eth_rx_cqe_type cqe_fp_type;
847 u16 len, pad, queue;
848 u8 *data;
849 bool l4_rxhash;
850
851 #ifdef BNX2X_STOP_ON_ERROR
852 if (unlikely(bp->panic))
853 return 0;
854 #endif
855
856 bd_prod = RX_BD(bd_prod);
857 bd_cons = RX_BD(bd_cons);
858
859 cqe_fp_flags = cqe_fp->type_error_flags;
860 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
861
862 DP(NETIF_MSG_RX_STATUS,
863 "CQE type %x err %x status %x queue %x vlan %x len %u\n",
864 CQE_TYPE(cqe_fp_flags),
865 cqe_fp_flags, cqe_fp->status_flags,
866 le32_to_cpu(cqe_fp->rss_hash_result),
867 le16_to_cpu(cqe_fp->vlan_tag),
868 le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
869
870 /* is this a slowpath msg? */
871 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
872 bnx2x_sp_event(fp, cqe);
873 goto next_cqe;
874 }
875
876 rx_buf = &fp->rx_buf_ring[bd_cons];
877 data = rx_buf->data;
878
879 if (!CQE_TYPE_FAST(cqe_fp_type)) {
880 struct bnx2x_agg_info *tpa_info;
881 u16 frag_size, pages;
882 #ifdef BNX2X_STOP_ON_ERROR
883 /* sanity check */
884 if (fp->disable_tpa &&
885 (CQE_TYPE_START(cqe_fp_type) ||
886 CQE_TYPE_STOP(cqe_fp_type)))
887 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
888 CQE_TYPE(cqe_fp_type));
889 #endif
890
891 if (CQE_TYPE_START(cqe_fp_type)) {
892 u16 queue = cqe_fp->queue_index;
893 DP(NETIF_MSG_RX_STATUS,
894 "calling tpa_start on queue %d\n",
895 queue);
896
897 bnx2x_tpa_start(fp, queue,
898 bd_cons, bd_prod,
899 cqe_fp);
900
901 goto next_rx;
902 }
903 queue = cqe->end_agg_cqe.queue_index;
904 tpa_info = &fp->tpa_info[queue];
905 DP(NETIF_MSG_RX_STATUS,
906 "calling tpa_stop on queue %d\n",
907 queue);
908
909 frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
910 tpa_info->len_on_bd;
911
912 if (fp->mode == TPA_MODE_GRO)
913 pages = (frag_size + tpa_info->full_page - 1) /
914 tpa_info->full_page;
915 else
916 pages = SGE_PAGE_ALIGN(frag_size) >>
917 SGE_PAGE_SHIFT;
918
919 bnx2x_tpa_stop(bp, fp, tpa_info, pages,
920 &cqe->end_agg_cqe, comp_ring_cons);
921 #ifdef BNX2X_STOP_ON_ERROR
922 if (bp->panic)
923 return 0;
924 #endif
925
926 bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
927 goto next_cqe;
928 }
929 /* non TPA */
930 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
931 pad = cqe_fp->placement_offset;
932 dma_sync_single_for_cpu(&bp->pdev->dev,
933 dma_unmap_addr(rx_buf, mapping),
934 pad + RX_COPY_THRESH,
935 DMA_FROM_DEVICE);
936 pad += NET_SKB_PAD;
937 prefetch(data + pad); /* speedup eth_type_trans() */
938 /* is this an error packet? */
939 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
940 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
941 "ERROR flags %x rx packet %u\n",
942 cqe_fp_flags, sw_comp_cons);
943 bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
944 goto reuse_rx;
945 }
946
947 /* Since we don't have a jumbo ring
948 * copy small packets if mtu > 1500
949 */
950 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
951 (len <= RX_COPY_THRESH)) {
952 skb = netdev_alloc_skb_ip_align(bp->dev, len);
953 if (skb == NULL) {
954 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
955 "ERROR packet dropped because of alloc failure\n");
956 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
957 goto reuse_rx;
958 }
959 memcpy(skb->data, data + pad, len);
960 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
961 } else {
962 if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
963 GFP_ATOMIC) == 0)) {
964 dma_unmap_single(&bp->pdev->dev,
965 dma_unmap_addr(rx_buf, mapping),
966 fp->rx_buf_size,
967 DMA_FROM_DEVICE);
968 skb = build_skb(data, fp->rx_frag_size);
969 if (unlikely(!skb)) {
970 bnx2x_frag_free(fp, data);
971 bnx2x_fp_qstats(bp, fp)->
972 rx_skb_alloc_failed++;
973 goto next_rx;
974 }
975 skb_reserve(skb, pad);
976 } else {
977 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
978 "ERROR packet dropped because of alloc failure\n");
979 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
980 reuse_rx:
981 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
982 goto next_rx;
983 }
984 }
985
986 skb_put(skb, len);
987 skb->protocol = eth_type_trans(skb, bp->dev);
988
989 /* Set Toeplitz hash for a none-LRO skb */
990 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
991 skb->l4_rxhash = l4_rxhash;
992
993 skb_checksum_none_assert(skb);
994
995 if (bp->dev->features & NETIF_F_RXCSUM)
996 bnx2x_csum_validate(skb, cqe, fp,
997 bnx2x_fp_qstats(bp, fp));
998
999 skb_record_rx_queue(skb, fp->rx_queue);
1000
1001 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1002 PARSING_FLAGS_VLAN)
1003 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1004 le16_to_cpu(cqe_fp->vlan_tag));
1005
1006 skb_mark_napi_id(skb, &fp->napi);
1007
1008 if (bnx2x_fp_ll_polling(fp))
1009 netif_receive_skb(skb);
1010 else
1011 napi_gro_receive(&fp->napi, skb);
1012 next_rx:
1013 rx_buf->data = NULL;
1014
1015 bd_cons = NEXT_RX_IDX(bd_cons);
1016 bd_prod = NEXT_RX_IDX(bd_prod);
1017 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1018 rx_pkt++;
1019 next_cqe:
1020 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1021 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1022
1023 /* mark CQE as free */
1024 BNX2X_SEED_CQE(cqe_fp);
1025
1026 if (rx_pkt == budget)
1027 break;
1028
1029 comp_ring_cons = RCQ_BD(sw_comp_cons);
1030 cqe = &fp->rx_comp_ring[comp_ring_cons];
1031 cqe_fp = &cqe->fast_path_cqe;
1032 } /* while */
1033
1034 fp->rx_bd_cons = bd_cons;
1035 fp->rx_bd_prod = bd_prod_fw;
1036 fp->rx_comp_cons = sw_comp_cons;
1037 fp->rx_comp_prod = sw_comp_prod;
1038
1039 /* Update producers */
1040 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1041 fp->rx_sge_prod);
1042
1043 fp->rx_pkt += rx_pkt;
1044 fp->rx_calls++;
1045
1046 return rx_pkt;
1047 }
1048
1049 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1050 {
1051 struct bnx2x_fastpath *fp = fp_cookie;
1052 struct bnx2x *bp = fp->bp;
1053 u8 cos;
1054
1055 DP(NETIF_MSG_INTR,
1056 "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1057 fp->index, fp->fw_sb_id, fp->igu_sb_id);
1058
1059 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1060
1061 #ifdef BNX2X_STOP_ON_ERROR
1062 if (unlikely(bp->panic))
1063 return IRQ_HANDLED;
1064 #endif
1065
1066 /* Handle Rx and Tx according to MSI-X vector */
1067 for_each_cos_in_tx_queue(fp, cos)
1068 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1069
1070 prefetch(&fp->sb_running_index[SM_RX_ID]);
1071 napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1072
1073 return IRQ_HANDLED;
1074 }
1075
1076 /* HW Lock for shared dual port PHYs */
1077 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1078 {
1079 mutex_lock(&bp->port.phy_mutex);
1080
1081 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1082 }
1083
1084 void bnx2x_release_phy_lock(struct bnx2x *bp)
1085 {
1086 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1087
1088 mutex_unlock(&bp->port.phy_mutex);
1089 }
1090
1091 /* calculates MF speed according to current linespeed and MF configuration */
1092 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1093 {
1094 u16 line_speed = bp->link_vars.line_speed;
1095 if (IS_MF(bp)) {
1096 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1097 bp->mf_config[BP_VN(bp)]);
1098
1099 /* Calculate the current MAX line speed limit for the MF
1100 * devices
1101 */
1102 if (IS_MF_SI(bp))
1103 line_speed = (line_speed * maxCfg) / 100;
1104 else { /* SD mode */
1105 u16 vn_max_rate = maxCfg * 100;
1106
1107 if (vn_max_rate < line_speed)
1108 line_speed = vn_max_rate;
1109 }
1110 }
1111
1112 return line_speed;
1113 }
1114
1115 /**
1116 * bnx2x_fill_report_data - fill link report data to report
1117 *
1118 * @bp: driver handle
1119 * @data: link state to update
1120 *
1121 * It uses a none-atomic bit operations because is called under the mutex.
1122 */
1123 static void bnx2x_fill_report_data(struct bnx2x *bp,
1124 struct bnx2x_link_report_data *data)
1125 {
1126 u16 line_speed = bnx2x_get_mf_speed(bp);
1127
1128 memset(data, 0, sizeof(*data));
1129
1130 /* Fill the report data: effective line speed */
1131 data->line_speed = line_speed;
1132
1133 /* Link is down */
1134 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1135 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1136 &data->link_report_flags);
1137
1138 /* Full DUPLEX */
1139 if (bp->link_vars.duplex == DUPLEX_FULL)
1140 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1141
1142 /* Rx Flow Control is ON */
1143 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1144 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1145
1146 /* Tx Flow Control is ON */
1147 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1148 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1149 }
1150
1151 /**
1152 * bnx2x_link_report - report link status to OS.
1153 *
1154 * @bp: driver handle
1155 *
1156 * Calls the __bnx2x_link_report() under the same locking scheme
1157 * as a link/PHY state managing code to ensure a consistent link
1158 * reporting.
1159 */
1160
1161 void bnx2x_link_report(struct bnx2x *bp)
1162 {
1163 bnx2x_acquire_phy_lock(bp);
1164 __bnx2x_link_report(bp);
1165 bnx2x_release_phy_lock(bp);
1166 }
1167
1168 /**
1169 * __bnx2x_link_report - report link status to OS.
1170 *
1171 * @bp: driver handle
1172 *
1173 * None atomic implementation.
1174 * Should be called under the phy_lock.
1175 */
1176 void __bnx2x_link_report(struct bnx2x *bp)
1177 {
1178 struct bnx2x_link_report_data cur_data;
1179
1180 /* reread mf_cfg */
1181 if (IS_PF(bp) && !CHIP_IS_E1(bp))
1182 bnx2x_read_mf_cfg(bp);
1183
1184 /* Read the current link report info */
1185 bnx2x_fill_report_data(bp, &cur_data);
1186
1187 /* Don't report link down or exactly the same link status twice */
1188 if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1189 (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1190 &bp->last_reported_link.link_report_flags) &&
1191 test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1192 &cur_data.link_report_flags)))
1193 return;
1194
1195 bp->link_cnt++;
1196
1197 /* We are going to report a new link parameters now -
1198 * remember the current data for the next time.
1199 */
1200 memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1201
1202 if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1203 &cur_data.link_report_flags)) {
1204 netif_carrier_off(bp->dev);
1205 netdev_err(bp->dev, "NIC Link is Down\n");
1206 return;
1207 } else {
1208 const char *duplex;
1209 const char *flow;
1210
1211 netif_carrier_on(bp->dev);
1212
1213 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1214 &cur_data.link_report_flags))
1215 duplex = "full";
1216 else
1217 duplex = "half";
1218
1219 /* Handle the FC at the end so that only these flags would be
1220 * possibly set. This way we may easily check if there is no FC
1221 * enabled.
1222 */
1223 if (cur_data.link_report_flags) {
1224 if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1225 &cur_data.link_report_flags)) {
1226 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1227 &cur_data.link_report_flags))
1228 flow = "ON - receive & transmit";
1229 else
1230 flow = "ON - receive";
1231 } else {
1232 flow = "ON - transmit";
1233 }
1234 } else {
1235 flow = "none";
1236 }
1237 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1238 cur_data.line_speed, duplex, flow);
1239 }
1240 }
1241
1242 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1243 {
1244 int i;
1245
1246 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1247 struct eth_rx_sge *sge;
1248
1249 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1250 sge->addr_hi =
1251 cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1252 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1253
1254 sge->addr_lo =
1255 cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1256 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1257 }
1258 }
1259
1260 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1261 struct bnx2x_fastpath *fp, int last)
1262 {
1263 int i;
1264
1265 for (i = 0; i < last; i++) {
1266 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1267 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1268 u8 *data = first_buf->data;
1269
1270 if (data == NULL) {
1271 DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1272 continue;
1273 }
1274 if (tpa_info->tpa_state == BNX2X_TPA_START)
1275 dma_unmap_single(&bp->pdev->dev,
1276 dma_unmap_addr(first_buf, mapping),
1277 fp->rx_buf_size, DMA_FROM_DEVICE);
1278 bnx2x_frag_free(fp, data);
1279 first_buf->data = NULL;
1280 }
1281 }
1282
1283 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1284 {
1285 int j;
1286
1287 for_each_rx_queue_cnic(bp, j) {
1288 struct bnx2x_fastpath *fp = &bp->fp[j];
1289
1290 fp->rx_bd_cons = 0;
1291
1292 /* Activate BD ring */
1293 /* Warning!
1294 * this will generate an interrupt (to the TSTORM)
1295 * must only be done after chip is initialized
1296 */
1297 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1298 fp->rx_sge_prod);
1299 }
1300 }
1301
1302 void bnx2x_init_rx_rings(struct bnx2x *bp)
1303 {
1304 int func = BP_FUNC(bp);
1305 u16 ring_prod;
1306 int i, j;
1307
1308 /* Allocate TPA resources */
1309 for_each_eth_queue(bp, j) {
1310 struct bnx2x_fastpath *fp = &bp->fp[j];
1311
1312 DP(NETIF_MSG_IFUP,
1313 "mtu %d rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1314
1315 if (!fp->disable_tpa) {
1316 /* Fill the per-aggregation pool */
1317 for (i = 0; i < MAX_AGG_QS(bp); i++) {
1318 struct bnx2x_agg_info *tpa_info =
1319 &fp->tpa_info[i];
1320 struct sw_rx_bd *first_buf =
1321 &tpa_info->first_buf;
1322
1323 first_buf->data =
1324 bnx2x_frag_alloc(fp, GFP_KERNEL);
1325 if (!first_buf->data) {
1326 BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1327 j);
1328 bnx2x_free_tpa_pool(bp, fp, i);
1329 fp->disable_tpa = 1;
1330 break;
1331 }
1332 dma_unmap_addr_set(first_buf, mapping, 0);
1333 tpa_info->tpa_state = BNX2X_TPA_STOP;
1334 }
1335
1336 /* "next page" elements initialization */
1337 bnx2x_set_next_page_sgl(fp);
1338
1339 /* set SGEs bit mask */
1340 bnx2x_init_sge_ring_bit_mask(fp);
1341
1342 /* Allocate SGEs and initialize the ring elements */
1343 for (i = 0, ring_prod = 0;
1344 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1345
1346 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1347 GFP_KERNEL) < 0) {
1348 BNX2X_ERR("was only able to allocate %d rx sges\n",
1349 i);
1350 BNX2X_ERR("disabling TPA for queue[%d]\n",
1351 j);
1352 /* Cleanup already allocated elements */
1353 bnx2x_free_rx_sge_range(bp, fp,
1354 ring_prod);
1355 bnx2x_free_tpa_pool(bp, fp,
1356 MAX_AGG_QS(bp));
1357 fp->disable_tpa = 1;
1358 ring_prod = 0;
1359 break;
1360 }
1361 ring_prod = NEXT_SGE_IDX(ring_prod);
1362 }
1363
1364 fp->rx_sge_prod = ring_prod;
1365 }
1366 }
1367
1368 for_each_eth_queue(bp, j) {
1369 struct bnx2x_fastpath *fp = &bp->fp[j];
1370
1371 fp->rx_bd_cons = 0;
1372
1373 /* Activate BD ring */
1374 /* Warning!
1375 * this will generate an interrupt (to the TSTORM)
1376 * must only be done after chip is initialized
1377 */
1378 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1379 fp->rx_sge_prod);
1380
1381 if (j != 0)
1382 continue;
1383
1384 if (CHIP_IS_E1(bp)) {
1385 REG_WR(bp, BAR_USTRORM_INTMEM +
1386 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1387 U64_LO(fp->rx_comp_mapping));
1388 REG_WR(bp, BAR_USTRORM_INTMEM +
1389 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1390 U64_HI(fp->rx_comp_mapping));
1391 }
1392 }
1393 }
1394
1395 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1396 {
1397 u8 cos;
1398 struct bnx2x *bp = fp->bp;
1399
1400 for_each_cos_in_tx_queue(fp, cos) {
1401 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1402 unsigned pkts_compl = 0, bytes_compl = 0;
1403
1404 u16 sw_prod = txdata->tx_pkt_prod;
1405 u16 sw_cons = txdata->tx_pkt_cons;
1406
1407 while (sw_cons != sw_prod) {
1408 bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1409 &pkts_compl, &bytes_compl);
1410 sw_cons++;
1411 }
1412
1413 netdev_tx_reset_queue(
1414 netdev_get_tx_queue(bp->dev,
1415 txdata->txq_index));
1416 }
1417 }
1418
1419 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1420 {
1421 int i;
1422
1423 for_each_tx_queue_cnic(bp, i) {
1424 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1425 }
1426 }
1427
1428 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1429 {
1430 int i;
1431
1432 for_each_eth_queue(bp, i) {
1433 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1434 }
1435 }
1436
1437 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1438 {
1439 struct bnx2x *bp = fp->bp;
1440 int i;
1441
1442 /* ring wasn't allocated */
1443 if (fp->rx_buf_ring == NULL)
1444 return;
1445
1446 for (i = 0; i < NUM_RX_BD; i++) {
1447 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1448 u8 *data = rx_buf->data;
1449
1450 if (data == NULL)
1451 continue;
1452 dma_unmap_single(&bp->pdev->dev,
1453 dma_unmap_addr(rx_buf, mapping),
1454 fp->rx_buf_size, DMA_FROM_DEVICE);
1455
1456 rx_buf->data = NULL;
1457 bnx2x_frag_free(fp, data);
1458 }
1459 }
1460
1461 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1462 {
1463 int j;
1464
1465 for_each_rx_queue_cnic(bp, j) {
1466 bnx2x_free_rx_bds(&bp->fp[j]);
1467 }
1468 }
1469
1470 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1471 {
1472 int j;
1473
1474 for_each_eth_queue(bp, j) {
1475 struct bnx2x_fastpath *fp = &bp->fp[j];
1476
1477 bnx2x_free_rx_bds(fp);
1478
1479 if (!fp->disable_tpa)
1480 bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1481 }
1482 }
1483
1484 void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1485 {
1486 bnx2x_free_tx_skbs_cnic(bp);
1487 bnx2x_free_rx_skbs_cnic(bp);
1488 }
1489
1490 void bnx2x_free_skbs(struct bnx2x *bp)
1491 {
1492 bnx2x_free_tx_skbs(bp);
1493 bnx2x_free_rx_skbs(bp);
1494 }
1495
1496 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1497 {
1498 /* load old values */
1499 u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1500
1501 if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1502 /* leave all but MAX value */
1503 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1504
1505 /* set new MAX value */
1506 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1507 & FUNC_MF_CFG_MAX_BW_MASK;
1508
1509 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1510 }
1511 }
1512
1513 /**
1514 * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1515 *
1516 * @bp: driver handle
1517 * @nvecs: number of vectors to be released
1518 */
1519 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1520 {
1521 int i, offset = 0;
1522
1523 if (nvecs == offset)
1524 return;
1525
1526 /* VFs don't have a default SB */
1527 if (IS_PF(bp)) {
1528 free_irq(bp->msix_table[offset].vector, bp->dev);
1529 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1530 bp->msix_table[offset].vector);
1531 offset++;
1532 }
1533
1534 if (CNIC_SUPPORT(bp)) {
1535 if (nvecs == offset)
1536 return;
1537 offset++;
1538 }
1539
1540 for_each_eth_queue(bp, i) {
1541 if (nvecs == offset)
1542 return;
1543 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1544 i, bp->msix_table[offset].vector);
1545
1546 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1547 }
1548 }
1549
1550 void bnx2x_free_irq(struct bnx2x *bp)
1551 {
1552 if (bp->flags & USING_MSIX_FLAG &&
1553 !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1554 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1555
1556 /* vfs don't have a default status block */
1557 if (IS_PF(bp))
1558 nvecs++;
1559
1560 bnx2x_free_msix_irqs(bp, nvecs);
1561 } else {
1562 free_irq(bp->dev->irq, bp->dev);
1563 }
1564 }
1565
1566 int bnx2x_enable_msix(struct bnx2x *bp)
1567 {
1568 int msix_vec = 0, i, rc;
1569
1570 /* VFs don't have a default status block */
1571 if (IS_PF(bp)) {
1572 bp->msix_table[msix_vec].entry = msix_vec;
1573 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1574 bp->msix_table[0].entry);
1575 msix_vec++;
1576 }
1577
1578 /* Cnic requires an msix vector for itself */
1579 if (CNIC_SUPPORT(bp)) {
1580 bp->msix_table[msix_vec].entry = msix_vec;
1581 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1582 msix_vec, bp->msix_table[msix_vec].entry);
1583 msix_vec++;
1584 }
1585
1586 /* We need separate vectors for ETH queues only (not FCoE) */
1587 for_each_eth_queue(bp, i) {
1588 bp->msix_table[msix_vec].entry = msix_vec;
1589 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1590 msix_vec, msix_vec, i);
1591 msix_vec++;
1592 }
1593
1594 DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1595 msix_vec);
1596
1597 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], msix_vec);
1598
1599 /*
1600 * reconfigure number of tx/rx queues according to available
1601 * MSI-X vectors
1602 */
1603 if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) {
1604 /* how less vectors we will have? */
1605 int diff = msix_vec - rc;
1606
1607 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1608
1609 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1610
1611 if (rc) {
1612 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1613 goto no_msix;
1614 }
1615 /*
1616 * decrease number of queues by number of unallocated entries
1617 */
1618 bp->num_ethernet_queues -= diff;
1619 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1620
1621 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1622 bp->num_queues);
1623 } else if (rc > 0) {
1624 /* Get by with single vector */
1625 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1626 if (rc) {
1627 BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1628 rc);
1629 goto no_msix;
1630 }
1631
1632 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1633 bp->flags |= USING_SINGLE_MSIX_FLAG;
1634
1635 BNX2X_DEV_INFO("set number of queues to 1\n");
1636 bp->num_ethernet_queues = 1;
1637 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1638 } else if (rc < 0) {
1639 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1640 goto no_msix;
1641 }
1642
1643 bp->flags |= USING_MSIX_FLAG;
1644
1645 return 0;
1646
1647 no_msix:
1648 /* fall to INTx if not enough memory */
1649 if (rc == -ENOMEM)
1650 bp->flags |= DISABLE_MSI_FLAG;
1651
1652 return rc;
1653 }
1654
1655 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1656 {
1657 int i, rc, offset = 0;
1658
1659 /* no default status block for vf */
1660 if (IS_PF(bp)) {
1661 rc = request_irq(bp->msix_table[offset++].vector,
1662 bnx2x_msix_sp_int, 0,
1663 bp->dev->name, bp->dev);
1664 if (rc) {
1665 BNX2X_ERR("request sp irq failed\n");
1666 return -EBUSY;
1667 }
1668 }
1669
1670 if (CNIC_SUPPORT(bp))
1671 offset++;
1672
1673 for_each_eth_queue(bp, i) {
1674 struct bnx2x_fastpath *fp = &bp->fp[i];
1675 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1676 bp->dev->name, i);
1677
1678 rc = request_irq(bp->msix_table[offset].vector,
1679 bnx2x_msix_fp_int, 0, fp->name, fp);
1680 if (rc) {
1681 BNX2X_ERR("request fp #%d irq (%d) failed rc %d\n", i,
1682 bp->msix_table[offset].vector, rc);
1683 bnx2x_free_msix_irqs(bp, offset);
1684 return -EBUSY;
1685 }
1686
1687 offset++;
1688 }
1689
1690 i = BNX2X_NUM_ETH_QUEUES(bp);
1691 if (IS_PF(bp)) {
1692 offset = 1 + CNIC_SUPPORT(bp);
1693 netdev_info(bp->dev,
1694 "using MSI-X IRQs: sp %d fp[%d] %d ... fp[%d] %d\n",
1695 bp->msix_table[0].vector,
1696 0, bp->msix_table[offset].vector,
1697 i - 1, bp->msix_table[offset + i - 1].vector);
1698 } else {
1699 offset = CNIC_SUPPORT(bp);
1700 netdev_info(bp->dev,
1701 "using MSI-X IRQs: fp[%d] %d ... fp[%d] %d\n",
1702 0, bp->msix_table[offset].vector,
1703 i - 1, bp->msix_table[offset + i - 1].vector);
1704 }
1705 return 0;
1706 }
1707
1708 int bnx2x_enable_msi(struct bnx2x *bp)
1709 {
1710 int rc;
1711
1712 rc = pci_enable_msi(bp->pdev);
1713 if (rc) {
1714 BNX2X_DEV_INFO("MSI is not attainable\n");
1715 return -1;
1716 }
1717 bp->flags |= USING_MSI_FLAG;
1718
1719 return 0;
1720 }
1721
1722 static int bnx2x_req_irq(struct bnx2x *bp)
1723 {
1724 unsigned long flags;
1725 unsigned int irq;
1726
1727 if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1728 flags = 0;
1729 else
1730 flags = IRQF_SHARED;
1731
1732 if (bp->flags & USING_MSIX_FLAG)
1733 irq = bp->msix_table[0].vector;
1734 else
1735 irq = bp->pdev->irq;
1736
1737 return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1738 }
1739
1740 static int bnx2x_setup_irqs(struct bnx2x *bp)
1741 {
1742 int rc = 0;
1743 if (bp->flags & USING_MSIX_FLAG &&
1744 !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1745 rc = bnx2x_req_msix_irqs(bp);
1746 if (rc)
1747 return rc;
1748 } else {
1749 rc = bnx2x_req_irq(bp);
1750 if (rc) {
1751 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc);
1752 return rc;
1753 }
1754 if (bp->flags & USING_MSI_FLAG) {
1755 bp->dev->irq = bp->pdev->irq;
1756 netdev_info(bp->dev, "using MSI IRQ %d\n",
1757 bp->dev->irq);
1758 }
1759 if (bp->flags & USING_MSIX_FLAG) {
1760 bp->dev->irq = bp->msix_table[0].vector;
1761 netdev_info(bp->dev, "using MSIX IRQ %d\n",
1762 bp->dev->irq);
1763 }
1764 }
1765
1766 return 0;
1767 }
1768
1769 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1770 {
1771 int i;
1772
1773 for_each_rx_queue_cnic(bp, i) {
1774 bnx2x_fp_init_lock(&bp->fp[i]);
1775 napi_enable(&bnx2x_fp(bp, i, napi));
1776 }
1777 }
1778
1779 static void bnx2x_napi_enable(struct bnx2x *bp)
1780 {
1781 int i;
1782
1783 for_each_eth_queue(bp, i) {
1784 bnx2x_fp_init_lock(&bp->fp[i]);
1785 napi_enable(&bnx2x_fp(bp, i, napi));
1786 }
1787 }
1788
1789 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1790 {
1791 int i;
1792
1793 local_bh_disable();
1794 for_each_rx_queue_cnic(bp, i) {
1795 napi_disable(&bnx2x_fp(bp, i, napi));
1796 while (!bnx2x_fp_lock_napi(&bp->fp[i]))
1797 mdelay(1);
1798 }
1799 local_bh_enable();
1800 }
1801
1802 static void bnx2x_napi_disable(struct bnx2x *bp)
1803 {
1804 int i;
1805
1806 local_bh_disable();
1807 for_each_eth_queue(bp, i) {
1808 napi_disable(&bnx2x_fp(bp, i, napi));
1809 while (!bnx2x_fp_lock_napi(&bp->fp[i]))
1810 mdelay(1);
1811 }
1812 local_bh_enable();
1813 }
1814
1815 void bnx2x_netif_start(struct bnx2x *bp)
1816 {
1817 if (netif_running(bp->dev)) {
1818 bnx2x_napi_enable(bp);
1819 if (CNIC_LOADED(bp))
1820 bnx2x_napi_enable_cnic(bp);
1821 bnx2x_int_enable(bp);
1822 if (bp->state == BNX2X_STATE_OPEN)
1823 netif_tx_wake_all_queues(bp->dev);
1824 }
1825 }
1826
1827 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1828 {
1829 bnx2x_int_disable_sync(bp, disable_hw);
1830 bnx2x_napi_disable(bp);
1831 if (CNIC_LOADED(bp))
1832 bnx2x_napi_disable_cnic(bp);
1833 }
1834
1835 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1836 {
1837 struct bnx2x *bp = netdev_priv(dev);
1838
1839 if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1840 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1841 u16 ether_type = ntohs(hdr->h_proto);
1842
1843 /* Skip VLAN tag if present */
1844 if (ether_type == ETH_P_8021Q) {
1845 struct vlan_ethhdr *vhdr =
1846 (struct vlan_ethhdr *)skb->data;
1847
1848 ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1849 }
1850
1851 /* If ethertype is FCoE or FIP - use FCoE ring */
1852 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1853 return bnx2x_fcoe_tx(bp, txq_index);
1854 }
1855
1856 /* select a non-FCoE queue */
1857 return __netdev_pick_tx(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp);
1858 }
1859
1860 void bnx2x_set_num_queues(struct bnx2x *bp)
1861 {
1862 /* RSS queues */
1863 bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1864
1865 /* override in STORAGE SD modes */
1866 if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1867 bp->num_ethernet_queues = 1;
1868
1869 /* Add special queues */
1870 bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1871 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1872
1873 BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1874 }
1875
1876 /**
1877 * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1878 *
1879 * @bp: Driver handle
1880 *
1881 * We currently support for at most 16 Tx queues for each CoS thus we will
1882 * allocate a multiple of 16 for ETH L2 rings according to the value of the
1883 * bp->max_cos.
1884 *
1885 * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1886 * index after all ETH L2 indices.
1887 *
1888 * If the actual number of Tx queues (for each CoS) is less than 16 then there
1889 * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1890 * 16..31,...) with indices that are not coupled with any real Tx queue.
1891 *
1892 * The proper configuration of skb->queue_mapping is handled by
1893 * bnx2x_select_queue() and __skb_tx_hash().
1894 *
1895 * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1896 * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1897 */
1898 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1899 {
1900 int rc, tx, rx;
1901
1902 tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1903 rx = BNX2X_NUM_ETH_QUEUES(bp);
1904
1905 /* account for fcoe queue */
1906 if (include_cnic && !NO_FCOE(bp)) {
1907 rx++;
1908 tx++;
1909 }
1910
1911 rc = netif_set_real_num_tx_queues(bp->dev, tx);
1912 if (rc) {
1913 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1914 return rc;
1915 }
1916 rc = netif_set_real_num_rx_queues(bp->dev, rx);
1917 if (rc) {
1918 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1919 return rc;
1920 }
1921
1922 DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1923 tx, rx);
1924
1925 return rc;
1926 }
1927
1928 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1929 {
1930 int i;
1931
1932 for_each_queue(bp, i) {
1933 struct bnx2x_fastpath *fp = &bp->fp[i];
1934 u32 mtu;
1935
1936 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1937 if (IS_FCOE_IDX(i))
1938 /*
1939 * Although there are no IP frames expected to arrive to
1940 * this ring we still want to add an
1941 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1942 * overrun attack.
1943 */
1944 mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1945 else
1946 mtu = bp->dev->mtu;
1947 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1948 IP_HEADER_ALIGNMENT_PADDING +
1949 ETH_OVREHEAD +
1950 mtu +
1951 BNX2X_FW_RX_ALIGN_END;
1952 /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
1953 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
1954 fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
1955 else
1956 fp->rx_frag_size = 0;
1957 }
1958 }
1959
1960 static int bnx2x_init_rss(struct bnx2x *bp)
1961 {
1962 int i;
1963 u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1964
1965 /* Prepare the initial contents for the indirection table if RSS is
1966 * enabled
1967 */
1968 for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1969 bp->rss_conf_obj.ind_table[i] =
1970 bp->fp->cl_id +
1971 ethtool_rxfh_indir_default(i, num_eth_queues);
1972
1973 /*
1974 * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1975 * per-port, so if explicit configuration is needed , do it only
1976 * for a PMF.
1977 *
1978 * For 57712 and newer on the other hand it's a per-function
1979 * configuration.
1980 */
1981 return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
1982 }
1983
1984 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1985 bool config_hash, bool enable)
1986 {
1987 struct bnx2x_config_rss_params params = {NULL};
1988
1989 /* Although RSS is meaningless when there is a single HW queue we
1990 * still need it enabled in order to have HW Rx hash generated.
1991 *
1992 * if (!is_eth_multi(bp))
1993 * bp->multi_mode = ETH_RSS_MODE_DISABLED;
1994 */
1995
1996 params.rss_obj = rss_obj;
1997
1998 __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1999
2000 if (enable) {
2001 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
2002
2003 /* RSS configuration */
2004 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
2005 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
2006 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
2007 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
2008 if (rss_obj->udp_rss_v4)
2009 __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2010 if (rss_obj->udp_rss_v6)
2011 __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2012 } else {
2013 __set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
2014 }
2015
2016 /* Hash bits */
2017 params.rss_result_mask = MULTI_MASK;
2018
2019 memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2020
2021 if (config_hash) {
2022 /* RSS keys */
2023 prandom_bytes(params.rss_key, T_ETH_RSS_KEY * 4);
2024 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2025 }
2026
2027 if (IS_PF(bp))
2028 return bnx2x_config_rss(bp, &params);
2029 else
2030 return bnx2x_vfpf_config_rss(bp, &params);
2031 }
2032
2033 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2034 {
2035 struct bnx2x_func_state_params func_params = {NULL};
2036
2037 /* Prepare parameters for function state transitions */
2038 __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2039
2040 func_params.f_obj = &bp->func_obj;
2041 func_params.cmd = BNX2X_F_CMD_HW_INIT;
2042
2043 func_params.params.hw_init.load_phase = load_code;
2044
2045 return bnx2x_func_state_change(bp, &func_params);
2046 }
2047
2048 /*
2049 * Cleans the object that have internal lists without sending
2050 * ramrods. Should be run when interrupts are disabled.
2051 */
2052 void bnx2x_squeeze_objects(struct bnx2x *bp)
2053 {
2054 int rc;
2055 unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2056 struct bnx2x_mcast_ramrod_params rparam = {NULL};
2057 struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2058
2059 /***************** Cleanup MACs' object first *************************/
2060
2061 /* Wait for completion of requested */
2062 __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2063 /* Perform a dry cleanup */
2064 __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2065
2066 /* Clean ETH primary MAC */
2067 __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2068 rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2069 &ramrod_flags);
2070 if (rc != 0)
2071 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2072
2073 /* Cleanup UC list */
2074 vlan_mac_flags = 0;
2075 __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2076 rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2077 &ramrod_flags);
2078 if (rc != 0)
2079 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2080
2081 /***************** Now clean mcast object *****************************/
2082 rparam.mcast_obj = &bp->mcast_obj;
2083 __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2084
2085 /* Add a DEL command... - Since we're doing a driver cleanup only,
2086 * we take a lock surrounding both the initial send and the CONTs,
2087 * as we don't want a true completion to disrupt us in the middle.
2088 */
2089 netif_addr_lock_bh(bp->dev);
2090 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2091 if (rc < 0)
2092 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2093 rc);
2094
2095 /* ...and wait until all pending commands are cleared */
2096 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2097 while (rc != 0) {
2098 if (rc < 0) {
2099 BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2100 rc);
2101 netif_addr_unlock_bh(bp->dev);
2102 return;
2103 }
2104
2105 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2106 }
2107 netif_addr_unlock_bh(bp->dev);
2108 }
2109
2110 #ifndef BNX2X_STOP_ON_ERROR
2111 #define LOAD_ERROR_EXIT(bp, label) \
2112 do { \
2113 (bp)->state = BNX2X_STATE_ERROR; \
2114 goto label; \
2115 } while (0)
2116
2117 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2118 do { \
2119 bp->cnic_loaded = false; \
2120 goto label; \
2121 } while (0)
2122 #else /*BNX2X_STOP_ON_ERROR*/
2123 #define LOAD_ERROR_EXIT(bp, label) \
2124 do { \
2125 (bp)->state = BNX2X_STATE_ERROR; \
2126 (bp)->panic = 1; \
2127 return -EBUSY; \
2128 } while (0)
2129 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2130 do { \
2131 bp->cnic_loaded = false; \
2132 (bp)->panic = 1; \
2133 return -EBUSY; \
2134 } while (0)
2135 #endif /*BNX2X_STOP_ON_ERROR*/
2136
2137 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2138 {
2139 BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2140 bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2141 return;
2142 }
2143
2144 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2145 {
2146 int num_groups, vf_headroom = 0;
2147 int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2148
2149 /* number of queues for statistics is number of eth queues + FCoE */
2150 u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2151
2152 /* Total number of FW statistics requests =
2153 * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2154 * and fcoe l2 queue) stats + num of queues (which includes another 1
2155 * for fcoe l2 queue if applicable)
2156 */
2157 bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2158
2159 /* vf stats appear in the request list, but their data is allocated by
2160 * the VFs themselves. We don't include them in the bp->fw_stats_num as
2161 * it is used to determine where to place the vf stats queries in the
2162 * request struct
2163 */
2164 if (IS_SRIOV(bp))
2165 vf_headroom = bnx2x_vf_headroom(bp);
2166
2167 /* Request is built from stats_query_header and an array of
2168 * stats_query_cmd_group each of which contains
2169 * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2170 * configured in the stats_query_header.
2171 */
2172 num_groups =
2173 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2174 (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2175 1 : 0));
2176
2177 DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2178 bp->fw_stats_num, vf_headroom, num_groups);
2179 bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2180 num_groups * sizeof(struct stats_query_cmd_group);
2181
2182 /* Data for statistics requests + stats_counter
2183 * stats_counter holds per-STORM counters that are incremented
2184 * when STORM has finished with the current request.
2185 * memory for FCoE offloaded statistics are counted anyway,
2186 * even if they will not be sent.
2187 * VF stats are not accounted for here as the data of VF stats is stored
2188 * in memory allocated by the VF, not here.
2189 */
2190 bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2191 sizeof(struct per_pf_stats) +
2192 sizeof(struct fcoe_statistics_params) +
2193 sizeof(struct per_queue_stats) * num_queue_stats +
2194 sizeof(struct stats_counter);
2195
2196 BNX2X_PCI_ALLOC(bp->fw_stats, &bp->fw_stats_mapping,
2197 bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2198
2199 /* Set shortcuts */
2200 bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2201 bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2202 bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2203 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2204 bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2205 bp->fw_stats_req_sz;
2206
2207 DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2208 U64_HI(bp->fw_stats_req_mapping),
2209 U64_LO(bp->fw_stats_req_mapping));
2210 DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2211 U64_HI(bp->fw_stats_data_mapping),
2212 U64_LO(bp->fw_stats_data_mapping));
2213 return 0;
2214
2215 alloc_mem_err:
2216 bnx2x_free_fw_stats_mem(bp);
2217 BNX2X_ERR("Can't allocate FW stats memory\n");
2218 return -ENOMEM;
2219 }
2220
2221 /* send load request to mcp and analyze response */
2222 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2223 {
2224 u32 param;
2225
2226 /* init fw_seq */
2227 bp->fw_seq =
2228 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2229 DRV_MSG_SEQ_NUMBER_MASK);
2230 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2231
2232 /* Get current FW pulse sequence */
2233 bp->fw_drv_pulse_wr_seq =
2234 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2235 DRV_PULSE_SEQ_MASK);
2236 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2237
2238 param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2239
2240 if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2241 param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2242
2243 /* load request */
2244 (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2245
2246 /* if mcp fails to respond we must abort */
2247 if (!(*load_code)) {
2248 BNX2X_ERR("MCP response failure, aborting\n");
2249 return -EBUSY;
2250 }
2251
2252 /* If mcp refused (e.g. other port is in diagnostic mode) we
2253 * must abort
2254 */
2255 if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2256 BNX2X_ERR("MCP refused load request, aborting\n");
2257 return -EBUSY;
2258 }
2259 return 0;
2260 }
2261
2262 /* check whether another PF has already loaded FW to chip. In
2263 * virtualized environments a pf from another VM may have already
2264 * initialized the device including loading FW
2265 */
2266 int bnx2x_nic_load_analyze_req(struct bnx2x *bp, u32 load_code)
2267 {
2268 /* is another pf loaded on this engine? */
2269 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2270 load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2271 /* build my FW version dword */
2272 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2273 (BCM_5710_FW_MINOR_VERSION << 8) +
2274 (BCM_5710_FW_REVISION_VERSION << 16) +
2275 (BCM_5710_FW_ENGINEERING_VERSION << 24);
2276
2277 /* read loaded FW from chip */
2278 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2279
2280 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2281 loaded_fw, my_fw);
2282
2283 /* abort nic load if version mismatch */
2284 if (my_fw != loaded_fw) {
2285 BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
2286 loaded_fw, my_fw);
2287 return -EBUSY;
2288 }
2289 }
2290 return 0;
2291 }
2292
2293 /* returns the "mcp load_code" according to global load_count array */
2294 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2295 {
2296 int path = BP_PATH(bp);
2297
2298 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d] %d, %d, %d\n",
2299 path, load_count[path][0], load_count[path][1],
2300 load_count[path][2]);
2301 load_count[path][0]++;
2302 load_count[path][1 + port]++;
2303 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d] %d, %d, %d\n",
2304 path, load_count[path][0], load_count[path][1],
2305 load_count[path][2]);
2306 if (load_count[path][0] == 1)
2307 return FW_MSG_CODE_DRV_LOAD_COMMON;
2308 else if (load_count[path][1 + port] == 1)
2309 return FW_MSG_CODE_DRV_LOAD_PORT;
2310 else
2311 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2312 }
2313
2314 /* mark PMF if applicable */
2315 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2316 {
2317 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2318 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2319 (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2320 bp->port.pmf = 1;
2321 /* We need the barrier to ensure the ordering between the
2322 * writing to bp->port.pmf here and reading it from the
2323 * bnx2x_periodic_task().
2324 */
2325 smp_mb();
2326 } else {
2327 bp->port.pmf = 0;
2328 }
2329
2330 DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2331 }
2332
2333 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2334 {
2335 if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2336 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2337 (bp->common.shmem2_base)) {
2338 if (SHMEM2_HAS(bp, dcc_support))
2339 SHMEM2_WR(bp, dcc_support,
2340 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2341 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2342 if (SHMEM2_HAS(bp, afex_driver_support))
2343 SHMEM2_WR(bp, afex_driver_support,
2344 SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2345 }
2346
2347 /* Set AFEX default VLAN tag to an invalid value */
2348 bp->afex_def_vlan_tag = -1;
2349 }
2350
2351 /**
2352 * bnx2x_bz_fp - zero content of the fastpath structure.
2353 *
2354 * @bp: driver handle
2355 * @index: fastpath index to be zeroed
2356 *
2357 * Makes sure the contents of the bp->fp[index].napi is kept
2358 * intact.
2359 */
2360 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2361 {
2362 struct bnx2x_fastpath *fp = &bp->fp[index];
2363 int cos;
2364 struct napi_struct orig_napi = fp->napi;
2365 struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2366
2367 /* bzero bnx2x_fastpath contents */
2368 if (fp->tpa_info)
2369 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2370 sizeof(struct bnx2x_agg_info));
2371 memset(fp, 0, sizeof(*fp));
2372
2373 /* Restore the NAPI object as it has been already initialized */
2374 fp->napi = orig_napi;
2375 fp->tpa_info = orig_tpa_info;
2376 fp->bp = bp;
2377 fp->index = index;
2378 if (IS_ETH_FP(fp))
2379 fp->max_cos = bp->max_cos;
2380 else
2381 /* Special queues support only one CoS */
2382 fp->max_cos = 1;
2383
2384 /* Init txdata pointers */
2385 if (IS_FCOE_FP(fp))
2386 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2387 if (IS_ETH_FP(fp))
2388 for_each_cos_in_tx_queue(fp, cos)
2389 fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2390 BNX2X_NUM_ETH_QUEUES(bp) + index];
2391
2392 /* set the tpa flag for each queue. The tpa flag determines the queue
2393 * minimal size so it must be set prior to queue memory allocation
2394 */
2395 fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2396 (bp->flags & GRO_ENABLE_FLAG &&
2397 bnx2x_mtu_allows_gro(bp->dev->mtu)));
2398 if (bp->flags & TPA_ENABLE_FLAG)
2399 fp->mode = TPA_MODE_LRO;
2400 else if (bp->flags & GRO_ENABLE_FLAG)
2401 fp->mode = TPA_MODE_GRO;
2402
2403 /* We don't want TPA on an FCoE L2 ring */
2404 if (IS_FCOE_FP(fp))
2405 fp->disable_tpa = 1;
2406 }
2407
2408 int bnx2x_load_cnic(struct bnx2x *bp)
2409 {
2410 int i, rc, port = BP_PORT(bp);
2411
2412 DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2413
2414 mutex_init(&bp->cnic_mutex);
2415
2416 if (IS_PF(bp)) {
2417 rc = bnx2x_alloc_mem_cnic(bp);
2418 if (rc) {
2419 BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2420 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2421 }
2422 }
2423
2424 rc = bnx2x_alloc_fp_mem_cnic(bp);
2425 if (rc) {
2426 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2427 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2428 }
2429
2430 /* Update the number of queues with the cnic queues */
2431 rc = bnx2x_set_real_num_queues(bp, 1);
2432 if (rc) {
2433 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2434 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2435 }
2436
2437 /* Add all CNIC NAPI objects */
2438 bnx2x_add_all_napi_cnic(bp);
2439 DP(NETIF_MSG_IFUP, "cnic napi added\n");
2440 bnx2x_napi_enable_cnic(bp);
2441
2442 rc = bnx2x_init_hw_func_cnic(bp);
2443 if (rc)
2444 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2445
2446 bnx2x_nic_init_cnic(bp);
2447
2448 if (IS_PF(bp)) {
2449 /* Enable Timer scan */
2450 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2451
2452 /* setup cnic queues */
2453 for_each_cnic_queue(bp, i) {
2454 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2455 if (rc) {
2456 BNX2X_ERR("Queue setup failed\n");
2457 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2458 }
2459 }
2460 }
2461
2462 /* Initialize Rx filter. */
2463 bnx2x_set_rx_mode_inner(bp);
2464
2465 /* re-read iscsi info */
2466 bnx2x_get_iscsi_info(bp);
2467 bnx2x_setup_cnic_irq_info(bp);
2468 bnx2x_setup_cnic_info(bp);
2469 bp->cnic_loaded = true;
2470 if (bp->state == BNX2X_STATE_OPEN)
2471 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2472
2473 DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2474
2475 return 0;
2476
2477 #ifndef BNX2X_STOP_ON_ERROR
2478 load_error_cnic2:
2479 /* Disable Timer scan */
2480 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2481
2482 load_error_cnic1:
2483 bnx2x_napi_disable_cnic(bp);
2484 /* Update the number of queues without the cnic queues */
2485 if (bnx2x_set_real_num_queues(bp, 0))
2486 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2487 load_error_cnic0:
2488 BNX2X_ERR("CNIC-related load failed\n");
2489 bnx2x_free_fp_mem_cnic(bp);
2490 bnx2x_free_mem_cnic(bp);
2491 return rc;
2492 #endif /* ! BNX2X_STOP_ON_ERROR */
2493 }
2494
2495 /* must be called with rtnl_lock */
2496 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2497 {
2498 int port = BP_PORT(bp);
2499 int i, rc = 0, load_code = 0;
2500
2501 DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2502 DP(NETIF_MSG_IFUP,
2503 "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2504
2505 #ifdef BNX2X_STOP_ON_ERROR
2506 if (unlikely(bp->panic)) {
2507 BNX2X_ERR("Can't load NIC when there is panic\n");
2508 return -EPERM;
2509 }
2510 #endif
2511
2512 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2513
2514 /* zero the structure w/o any lock, before SP handler is initialized */
2515 memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2516 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2517 &bp->last_reported_link.link_report_flags);
2518
2519 if (IS_PF(bp))
2520 /* must be called before memory allocation and HW init */
2521 bnx2x_ilt_set_info(bp);
2522
2523 /*
2524 * Zero fastpath structures preserving invariants like napi, which are
2525 * allocated only once, fp index, max_cos, bp pointer.
2526 * Also set fp->disable_tpa and txdata_ptr.
2527 */
2528 DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2529 for_each_queue(bp, i)
2530 bnx2x_bz_fp(bp, i);
2531 memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2532 bp->num_cnic_queues) *
2533 sizeof(struct bnx2x_fp_txdata));
2534
2535 bp->fcoe_init = false;
2536
2537 /* Set the receive queues buffer size */
2538 bnx2x_set_rx_buf_size(bp);
2539
2540 if (IS_PF(bp)) {
2541 rc = bnx2x_alloc_mem(bp);
2542 if (rc) {
2543 BNX2X_ERR("Unable to allocate bp memory\n");
2544 return rc;
2545 }
2546 }
2547
2548 /* need to be done after alloc mem, since it's self adjusting to amount
2549 * of memory available for RSS queues
2550 */
2551 rc = bnx2x_alloc_fp_mem(bp);
2552 if (rc) {
2553 BNX2X_ERR("Unable to allocate memory for fps\n");
2554 LOAD_ERROR_EXIT(bp, load_error0);
2555 }
2556
2557 /* Allocated memory for FW statistics */
2558 if (bnx2x_alloc_fw_stats_mem(bp))
2559 LOAD_ERROR_EXIT(bp, load_error0);
2560
2561 /* request pf to initialize status blocks */
2562 if (IS_VF(bp)) {
2563 rc = bnx2x_vfpf_init(bp);
2564 if (rc)
2565 LOAD_ERROR_EXIT(bp, load_error0);
2566 }
2567
2568 /* As long as bnx2x_alloc_mem() may possibly update
2569 * bp->num_queues, bnx2x_set_real_num_queues() should always
2570 * come after it. At this stage cnic queues are not counted.
2571 */
2572 rc = bnx2x_set_real_num_queues(bp, 0);
2573 if (rc) {
2574 BNX2X_ERR("Unable to set real_num_queues\n");
2575 LOAD_ERROR_EXIT(bp, load_error0);
2576 }
2577
2578 /* configure multi cos mappings in kernel.
2579 * this configuration may be overridden by a multi class queue
2580 * discipline or by a dcbx negotiation result.
2581 */
2582 bnx2x_setup_tc(bp->dev, bp->max_cos);
2583
2584 /* Add all NAPI objects */
2585 bnx2x_add_all_napi(bp);
2586 DP(NETIF_MSG_IFUP, "napi added\n");
2587 bnx2x_napi_enable(bp);
2588
2589 if (IS_PF(bp)) {
2590 /* set pf load just before approaching the MCP */
2591 bnx2x_set_pf_load(bp);
2592
2593 /* if mcp exists send load request and analyze response */
2594 if (!BP_NOMCP(bp)) {
2595 /* attempt to load pf */
2596 rc = bnx2x_nic_load_request(bp, &load_code);
2597 if (rc)
2598 LOAD_ERROR_EXIT(bp, load_error1);
2599
2600 /* what did mcp say? */
2601 rc = bnx2x_nic_load_analyze_req(bp, load_code);
2602 if (rc) {
2603 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2604 LOAD_ERROR_EXIT(bp, load_error2);
2605 }
2606 } else {
2607 load_code = bnx2x_nic_load_no_mcp(bp, port);
2608 }
2609
2610 /* mark pmf if applicable */
2611 bnx2x_nic_load_pmf(bp, load_code);
2612
2613 /* Init Function state controlling object */
2614 bnx2x__init_func_obj(bp);
2615
2616 /* Initialize HW */
2617 rc = bnx2x_init_hw(bp, load_code);
2618 if (rc) {
2619 BNX2X_ERR("HW init failed, aborting\n");
2620 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2621 LOAD_ERROR_EXIT(bp, load_error2);
2622 }
2623 }
2624
2625 bnx2x_pre_irq_nic_init(bp);
2626
2627 /* Connect to IRQs */
2628 rc = bnx2x_setup_irqs(bp);
2629 if (rc) {
2630 BNX2X_ERR("setup irqs failed\n");
2631 if (IS_PF(bp))
2632 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2633 LOAD_ERROR_EXIT(bp, load_error2);
2634 }
2635
2636 /* Init per-function objects */
2637 if (IS_PF(bp)) {
2638 /* Setup NIC internals and enable interrupts */
2639 bnx2x_post_irq_nic_init(bp, load_code);
2640
2641 bnx2x_init_bp_objs(bp);
2642 bnx2x_iov_nic_init(bp);
2643
2644 /* Set AFEX default VLAN tag to an invalid value */
2645 bp->afex_def_vlan_tag = -1;
2646 bnx2x_nic_load_afex_dcc(bp, load_code);
2647 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2648 rc = bnx2x_func_start(bp);
2649 if (rc) {
2650 BNX2X_ERR("Function start failed!\n");
2651 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2652
2653 LOAD_ERROR_EXIT(bp, load_error3);
2654 }
2655
2656 /* Send LOAD_DONE command to MCP */
2657 if (!BP_NOMCP(bp)) {
2658 load_code = bnx2x_fw_command(bp,
2659 DRV_MSG_CODE_LOAD_DONE, 0);
2660 if (!load_code) {
2661 BNX2X_ERR("MCP response failure, aborting\n");
2662 rc = -EBUSY;
2663 LOAD_ERROR_EXIT(bp, load_error3);
2664 }
2665 }
2666
2667 /* initialize FW coalescing state machines in RAM */
2668 bnx2x_update_coalesce(bp);
2669 }
2670
2671 /* setup the leading queue */
2672 rc = bnx2x_setup_leading(bp);
2673 if (rc) {
2674 BNX2X_ERR("Setup leading failed!\n");
2675 LOAD_ERROR_EXIT(bp, load_error3);
2676 }
2677
2678 /* set up the rest of the queues */
2679 for_each_nondefault_eth_queue(bp, i) {
2680 if (IS_PF(bp))
2681 rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2682 else /* VF */
2683 rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2684 if (rc) {
2685 BNX2X_ERR("Queue %d setup failed\n", i);
2686 LOAD_ERROR_EXIT(bp, load_error3);
2687 }
2688 }
2689
2690 /* setup rss */
2691 rc = bnx2x_init_rss(bp);
2692 if (rc) {
2693 BNX2X_ERR("PF RSS init failed\n");
2694 LOAD_ERROR_EXIT(bp, load_error3);
2695 }
2696
2697 /* Now when Clients are configured we are ready to work */
2698 bp->state = BNX2X_STATE_OPEN;
2699
2700 /* Configure a ucast MAC */
2701 if (IS_PF(bp))
2702 rc = bnx2x_set_eth_mac(bp, true);
2703 else /* vf */
2704 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2705 true);
2706 if (rc) {
2707 BNX2X_ERR("Setting Ethernet MAC failed\n");
2708 LOAD_ERROR_EXIT(bp, load_error3);
2709 }
2710
2711 if (IS_PF(bp) && bp->pending_max) {
2712 bnx2x_update_max_mf_config(bp, bp->pending_max);
2713 bp->pending_max = 0;
2714 }
2715
2716 if (bp->port.pmf) {
2717 rc = bnx2x_initial_phy_init(bp, load_mode);
2718 if (rc)
2719 LOAD_ERROR_EXIT(bp, load_error3);
2720 }
2721 bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2722
2723 /* Start fast path */
2724
2725 /* Initialize Rx filter. */
2726 bnx2x_set_rx_mode_inner(bp);
2727
2728 /* Start the Tx */
2729 switch (load_mode) {
2730 case LOAD_NORMAL:
2731 /* Tx queue should be only re-enabled */
2732 netif_tx_wake_all_queues(bp->dev);
2733 break;
2734
2735 case LOAD_OPEN:
2736 netif_tx_start_all_queues(bp->dev);
2737 smp_mb__after_clear_bit();
2738 break;
2739
2740 case LOAD_DIAG:
2741 case LOAD_LOOPBACK_EXT:
2742 bp->state = BNX2X_STATE_DIAG;
2743 break;
2744
2745 default:
2746 break;
2747 }
2748
2749 if (bp->port.pmf)
2750 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2751 else
2752 bnx2x__link_status_update(bp);
2753
2754 /* start the timer */
2755 mod_timer(&bp->timer, jiffies + bp->current_interval);
2756
2757 if (CNIC_ENABLED(bp))
2758 bnx2x_load_cnic(bp);
2759
2760 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2761 /* mark driver is loaded in shmem2 */
2762 u32 val;
2763 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2764 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2765 val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2766 DRV_FLAGS_CAPABILITIES_LOADED_L2);
2767 }
2768
2769 /* Wait for all pending SP commands to complete */
2770 if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2771 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2772 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2773 return -EBUSY;
2774 }
2775
2776 /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2777 if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2778 bnx2x_dcbx_init(bp, false);
2779
2780 DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2781
2782 return 0;
2783
2784 #ifndef BNX2X_STOP_ON_ERROR
2785 load_error3:
2786 if (IS_PF(bp)) {
2787 bnx2x_int_disable_sync(bp, 1);
2788
2789 /* Clean queueable objects */
2790 bnx2x_squeeze_objects(bp);
2791 }
2792
2793 /* Free SKBs, SGEs, TPA pool and driver internals */
2794 bnx2x_free_skbs(bp);
2795 for_each_rx_queue(bp, i)
2796 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2797
2798 /* Release IRQs */
2799 bnx2x_free_irq(bp);
2800 load_error2:
2801 if (IS_PF(bp) && !BP_NOMCP(bp)) {
2802 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2803 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2804 }
2805
2806 bp->port.pmf = 0;
2807 load_error1:
2808 bnx2x_napi_disable(bp);
2809 bnx2x_del_all_napi(bp);
2810
2811 /* clear pf_load status, as it was already set */
2812 if (IS_PF(bp))
2813 bnx2x_clear_pf_load(bp);
2814 load_error0:
2815 bnx2x_free_fw_stats_mem(bp);
2816 bnx2x_free_fp_mem(bp);
2817 bnx2x_free_mem(bp);
2818
2819 return rc;
2820 #endif /* ! BNX2X_STOP_ON_ERROR */
2821 }
2822
2823 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2824 {
2825 u8 rc = 0, cos, i;
2826
2827 /* Wait until tx fastpath tasks complete */
2828 for_each_tx_queue(bp, i) {
2829 struct bnx2x_fastpath *fp = &bp->fp[i];
2830
2831 for_each_cos_in_tx_queue(fp, cos)
2832 rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2833 if (rc)
2834 return rc;
2835 }
2836 return 0;
2837 }
2838
2839 /* must be called with rtnl_lock */
2840 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2841 {
2842 int i;
2843 bool global = false;
2844
2845 DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2846
2847 /* mark driver is unloaded in shmem2 */
2848 if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2849 u32 val;
2850 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2851 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2852 val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2853 }
2854
2855 if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2856 (bp->state == BNX2X_STATE_CLOSED ||
2857 bp->state == BNX2X_STATE_ERROR)) {
2858 /* We can get here if the driver has been unloaded
2859 * during parity error recovery and is either waiting for a
2860 * leader to complete or for other functions to unload and
2861 * then ifdown has been issued. In this case we want to
2862 * unload and let other functions to complete a recovery
2863 * process.
2864 */
2865 bp->recovery_state = BNX2X_RECOVERY_DONE;
2866 bp->is_leader = 0;
2867 bnx2x_release_leader_lock(bp);
2868 smp_mb();
2869
2870 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2871 BNX2X_ERR("Can't unload in closed or error state\n");
2872 return -EINVAL;
2873 }
2874
2875 /* Nothing to do during unload if previous bnx2x_nic_load()
2876 * have not completed successfully - all resources are released.
2877 *
2878 * we can get here only after unsuccessful ndo_* callback, during which
2879 * dev->IFF_UP flag is still on.
2880 */
2881 if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2882 return 0;
2883
2884 /* It's important to set the bp->state to the value different from
2885 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2886 * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2887 */
2888 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2889 smp_mb();
2890
2891 /* indicate to VFs that the PF is going down */
2892 bnx2x_iov_channel_down(bp);
2893
2894 if (CNIC_LOADED(bp))
2895 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2896
2897 /* Stop Tx */
2898 bnx2x_tx_disable(bp);
2899 netdev_reset_tc(bp->dev);
2900
2901 bp->rx_mode = BNX2X_RX_MODE_NONE;
2902
2903 del_timer_sync(&bp->timer);
2904
2905 if (IS_PF(bp)) {
2906 /* Set ALWAYS_ALIVE bit in shmem */
2907 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2908 bnx2x_drv_pulse(bp);
2909 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2910 bnx2x_save_statistics(bp);
2911 }
2912
2913 /* wait till consumers catch up with producers in all queues */
2914 bnx2x_drain_tx_queues(bp);
2915
2916 /* if VF indicate to PF this function is going down (PF will delete sp
2917 * elements and clear initializations
2918 */
2919 if (IS_VF(bp))
2920 bnx2x_vfpf_close_vf(bp);
2921 else if (unload_mode != UNLOAD_RECOVERY)
2922 /* if this is a normal/close unload need to clean up chip*/
2923 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2924 else {
2925 /* Send the UNLOAD_REQUEST to the MCP */
2926 bnx2x_send_unload_req(bp, unload_mode);
2927
2928 /* Prevent transactions to host from the functions on the
2929 * engine that doesn't reset global blocks in case of global
2930 * attention once global blocks are reset and gates are opened
2931 * (the engine which leader will perform the recovery
2932 * last).
2933 */
2934 if (!CHIP_IS_E1x(bp))
2935 bnx2x_pf_disable(bp);
2936
2937 /* Disable HW interrupts, NAPI */
2938 bnx2x_netif_stop(bp, 1);
2939 /* Delete all NAPI objects */
2940 bnx2x_del_all_napi(bp);
2941 if (CNIC_LOADED(bp))
2942 bnx2x_del_all_napi_cnic(bp);
2943 /* Release IRQs */
2944 bnx2x_free_irq(bp);
2945
2946 /* Report UNLOAD_DONE to MCP */
2947 bnx2x_send_unload_done(bp, false);
2948 }
2949
2950 /*
2951 * At this stage no more interrupts will arrive so we may safely clean
2952 * the queueable objects here in case they failed to get cleaned so far.
2953 */
2954 if (IS_PF(bp))
2955 bnx2x_squeeze_objects(bp);
2956
2957 /* There should be no more pending SP commands at this stage */
2958 bp->sp_state = 0;
2959
2960 bp->port.pmf = 0;
2961
2962 /* Free SKBs, SGEs, TPA pool and driver internals */
2963 bnx2x_free_skbs(bp);
2964 if (CNIC_LOADED(bp))
2965 bnx2x_free_skbs_cnic(bp);
2966 for_each_rx_queue(bp, i)
2967 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2968
2969 bnx2x_free_fp_mem(bp);
2970 if (CNIC_LOADED(bp))
2971 bnx2x_free_fp_mem_cnic(bp);
2972
2973 if (IS_PF(bp)) {
2974 if (CNIC_LOADED(bp))
2975 bnx2x_free_mem_cnic(bp);
2976 }
2977 bnx2x_free_mem(bp);
2978
2979 bp->state = BNX2X_STATE_CLOSED;
2980 bp->cnic_loaded = false;
2981
2982 /* Check if there are pending parity attentions. If there are - set
2983 * RECOVERY_IN_PROGRESS.
2984 */
2985 if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
2986 bnx2x_set_reset_in_progress(bp);
2987
2988 /* Set RESET_IS_GLOBAL if needed */
2989 if (global)
2990 bnx2x_set_reset_global(bp);
2991 }
2992
2993 /* The last driver must disable a "close the gate" if there is no
2994 * parity attention or "process kill" pending.
2995 */
2996 if (IS_PF(bp) &&
2997 !bnx2x_clear_pf_load(bp) &&
2998 bnx2x_reset_is_done(bp, BP_PATH(bp)))
2999 bnx2x_disable_close_the_gate(bp);
3000
3001 DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3002
3003 return 0;
3004 }
3005
3006 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3007 {
3008 u16 pmcsr;
3009
3010 /* If there is no power capability, silently succeed */
3011 if (!bp->pdev->pm_cap) {
3012 BNX2X_DEV_INFO("No power capability. Breaking.\n");
3013 return 0;
3014 }
3015
3016 pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
3017
3018 switch (state) {
3019 case PCI_D0:
3020 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3021 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3022 PCI_PM_CTRL_PME_STATUS));
3023
3024 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3025 /* delay required during transition out of D3hot */
3026 msleep(20);
3027 break;
3028
3029 case PCI_D3hot:
3030 /* If there are other clients above don't
3031 shut down the power */
3032 if (atomic_read(&bp->pdev->enable_cnt) != 1)
3033 return 0;
3034 /* Don't shut down the power for emulation and FPGA */
3035 if (CHIP_REV_IS_SLOW(bp))
3036 return 0;
3037
3038 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3039 pmcsr |= 3;
3040
3041 if (bp->wol)
3042 pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3043
3044 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3045 pmcsr);
3046
3047 /* No more memory access after this point until
3048 * device is brought back to D0.
3049 */
3050 break;
3051
3052 default:
3053 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3054 return -EINVAL;
3055 }
3056 return 0;
3057 }
3058
3059 /*
3060 * net_device service functions
3061 */
3062 int bnx2x_poll(struct napi_struct *napi, int budget)
3063 {
3064 int work_done = 0;
3065 u8 cos;
3066 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3067 napi);
3068 struct bnx2x *bp = fp->bp;
3069
3070 while (1) {
3071 #ifdef BNX2X_STOP_ON_ERROR
3072 if (unlikely(bp->panic)) {
3073 napi_complete(napi);
3074 return 0;
3075 }
3076 #endif
3077 if (!bnx2x_fp_lock_napi(fp))
3078 return work_done;
3079
3080 for_each_cos_in_tx_queue(fp, cos)
3081 if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3082 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3083
3084 if (bnx2x_has_rx_work(fp)) {
3085 work_done += bnx2x_rx_int(fp, budget - work_done);
3086
3087 /* must not complete if we consumed full budget */
3088 if (work_done >= budget) {
3089 bnx2x_fp_unlock_napi(fp);
3090 break;
3091 }
3092 }
3093
3094 /* Fall out from the NAPI loop if needed */
3095 if (!bnx2x_fp_unlock_napi(fp) &&
3096 !(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3097
3098 /* No need to update SB for FCoE L2 ring as long as
3099 * it's connected to the default SB and the SB
3100 * has been updated when NAPI was scheduled.
3101 */
3102 if (IS_FCOE_FP(fp)) {
3103 napi_complete(napi);
3104 break;
3105 }
3106 bnx2x_update_fpsb_idx(fp);
3107 /* bnx2x_has_rx_work() reads the status block,
3108 * thus we need to ensure that status block indices
3109 * have been actually read (bnx2x_update_fpsb_idx)
3110 * prior to this check (bnx2x_has_rx_work) so that
3111 * we won't write the "newer" value of the status block
3112 * to IGU (if there was a DMA right after
3113 * bnx2x_has_rx_work and if there is no rmb, the memory
3114 * reading (bnx2x_update_fpsb_idx) may be postponed
3115 * to right before bnx2x_ack_sb). In this case there
3116 * will never be another interrupt until there is
3117 * another update of the status block, while there
3118 * is still unhandled work.
3119 */
3120 rmb();
3121
3122 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3123 napi_complete(napi);
3124 /* Re-enable interrupts */
3125 DP(NETIF_MSG_RX_STATUS,
3126 "Update index to %d\n", fp->fp_hc_idx);
3127 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3128 le16_to_cpu(fp->fp_hc_idx),
3129 IGU_INT_ENABLE, 1);
3130 break;
3131 }
3132 }
3133 }
3134
3135 return work_done;
3136 }
3137
3138 #ifdef CONFIG_NET_RX_BUSY_POLL
3139 /* must be called with local_bh_disable()d */
3140 int bnx2x_low_latency_recv(struct napi_struct *napi)
3141 {
3142 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3143 napi);
3144 struct bnx2x *bp = fp->bp;
3145 int found = 0;
3146
3147 if ((bp->state == BNX2X_STATE_CLOSED) ||
3148 (bp->state == BNX2X_STATE_ERROR) ||
3149 (bp->flags & (TPA_ENABLE_FLAG | GRO_ENABLE_FLAG)))
3150 return LL_FLUSH_FAILED;
3151
3152 if (!bnx2x_fp_lock_poll(fp))
3153 return LL_FLUSH_BUSY;
3154
3155 if (bnx2x_has_rx_work(fp))
3156 found = bnx2x_rx_int(fp, 4);
3157
3158 bnx2x_fp_unlock_poll(fp);
3159
3160 return found;
3161 }
3162 #endif
3163
3164 /* we split the first BD into headers and data BDs
3165 * to ease the pain of our fellow microcode engineers
3166 * we use one mapping for both BDs
3167 */
3168 static u16 bnx2x_tx_split(struct bnx2x *bp,
3169 struct bnx2x_fp_txdata *txdata,
3170 struct sw_tx_bd *tx_buf,
3171 struct eth_tx_start_bd **tx_bd, u16 hlen,
3172 u16 bd_prod)
3173 {
3174 struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3175 struct eth_tx_bd *d_tx_bd;
3176 dma_addr_t mapping;
3177 int old_len = le16_to_cpu(h_tx_bd->nbytes);
3178
3179 /* first fix first BD */
3180 h_tx_bd->nbytes = cpu_to_le16(hlen);
3181
3182 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3183 h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3184
3185 /* now get a new data BD
3186 * (after the pbd) and fill it */
3187 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3188 d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3189
3190 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3191 le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3192
3193 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3194 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3195 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3196
3197 /* this marks the BD as one that has no individual mapping */
3198 tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3199
3200 DP(NETIF_MSG_TX_QUEUED,
3201 "TSO split data size is %d (%x:%x)\n",
3202 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3203
3204 /* update tx_bd */
3205 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3206
3207 return bd_prod;
3208 }
3209
3210 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3211 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3212 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3213 {
3214 __sum16 tsum = (__force __sum16) csum;
3215
3216 if (fix > 0)
3217 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3218 csum_partial(t_header - fix, fix, 0)));
3219
3220 else if (fix < 0)
3221 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3222 csum_partial(t_header, -fix, 0)));
3223
3224 return bswab16(tsum);
3225 }
3226
3227 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3228 {
3229 u32 rc;
3230 __u8 prot = 0;
3231 __be16 protocol;
3232
3233 if (skb->ip_summed != CHECKSUM_PARTIAL)
3234 return XMIT_PLAIN;
3235
3236 protocol = vlan_get_protocol(skb);
3237 if (protocol == htons(ETH_P_IPV6)) {
3238 rc = XMIT_CSUM_V6;
3239 prot = ipv6_hdr(skb)->nexthdr;
3240 } else {
3241 rc = XMIT_CSUM_V4;
3242 prot = ip_hdr(skb)->protocol;
3243 }
3244
3245 if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3246 if (inner_ip_hdr(skb)->version == 6) {
3247 rc |= XMIT_CSUM_ENC_V6;
3248 if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3249 rc |= XMIT_CSUM_TCP;
3250 } else {
3251 rc |= XMIT_CSUM_ENC_V4;
3252 if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3253 rc |= XMIT_CSUM_TCP;
3254 }
3255 }
3256 if (prot == IPPROTO_TCP)
3257 rc |= XMIT_CSUM_TCP;
3258
3259 if (skb_is_gso_v6(skb)) {
3260 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3261 if (rc & XMIT_CSUM_ENC)
3262 rc |= XMIT_GSO_ENC_V6;
3263 } else if (skb_is_gso(skb)) {
3264 rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3265 if (rc & XMIT_CSUM_ENC)
3266 rc |= XMIT_GSO_ENC_V4;
3267 }
3268
3269 return rc;
3270 }
3271
3272 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3273 /* check if packet requires linearization (packet is too fragmented)
3274 no need to check fragmentation if page size > 8K (there will be no
3275 violation to FW restrictions) */
3276 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3277 u32 xmit_type)
3278 {
3279 int to_copy = 0;
3280 int hlen = 0;
3281 int first_bd_sz = 0;
3282
3283 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3284 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3285
3286 if (xmit_type & XMIT_GSO) {
3287 unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3288 /* Check if LSO packet needs to be copied:
3289 3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3290 int wnd_size = MAX_FETCH_BD - 3;
3291 /* Number of windows to check */
3292 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3293 int wnd_idx = 0;
3294 int frag_idx = 0;
3295 u32 wnd_sum = 0;
3296
3297 /* Headers length */
3298 hlen = (int)(skb_transport_header(skb) - skb->data) +
3299 tcp_hdrlen(skb);
3300
3301 /* Amount of data (w/o headers) on linear part of SKB*/
3302 first_bd_sz = skb_headlen(skb) - hlen;
3303
3304 wnd_sum = first_bd_sz;
3305
3306 /* Calculate the first sum - it's special */
3307 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3308 wnd_sum +=
3309 skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3310
3311 /* If there was data on linear skb data - check it */
3312 if (first_bd_sz > 0) {
3313 if (unlikely(wnd_sum < lso_mss)) {
3314 to_copy = 1;
3315 goto exit_lbl;
3316 }
3317
3318 wnd_sum -= first_bd_sz;
3319 }
3320
3321 /* Others are easier: run through the frag list and
3322 check all windows */
3323 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3324 wnd_sum +=
3325 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3326
3327 if (unlikely(wnd_sum < lso_mss)) {
3328 to_copy = 1;
3329 break;
3330 }
3331 wnd_sum -=
3332 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3333 }
3334 } else {
3335 /* in non-LSO too fragmented packet should always
3336 be linearized */
3337 to_copy = 1;
3338 }
3339 }
3340
3341 exit_lbl:
3342 if (unlikely(to_copy))
3343 DP(NETIF_MSG_TX_QUEUED,
3344 "Linearization IS REQUIRED for %s packet. num_frags %d hlen %d first_bd_sz %d\n",
3345 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3346 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3347
3348 return to_copy;
3349 }
3350 #endif
3351
3352 static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
3353 u32 xmit_type)
3354 {
3355 struct ipv6hdr *ipv6;
3356
3357 *parsing_data |= (skb_shinfo(skb)->gso_size <<
3358 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
3359 ETH_TX_PARSE_BD_E2_LSO_MSS;
3360
3361 if (xmit_type & XMIT_GSO_ENC_V6)
3362 ipv6 = inner_ipv6_hdr(skb);
3363 else if (xmit_type & XMIT_GSO_V6)
3364 ipv6 = ipv6_hdr(skb);
3365 else
3366 ipv6 = NULL;
3367
3368 if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6)
3369 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3370 }
3371
3372 /**
3373 * bnx2x_set_pbd_gso - update PBD in GSO case.
3374 *
3375 * @skb: packet skb
3376 * @pbd: parse BD
3377 * @xmit_type: xmit flags
3378 */
3379 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3380 struct eth_tx_parse_bd_e1x *pbd,
3381 struct eth_tx_start_bd *tx_start_bd,
3382 u32 xmit_type)
3383 {
3384 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3385 pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3386 pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3387
3388 if (xmit_type & XMIT_GSO_V4) {
3389 pbd->ip_id = bswab16(ip_hdr(skb)->id);
3390 pbd->tcp_pseudo_csum =
3391 bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3392 ip_hdr(skb)->daddr,
3393 0, IPPROTO_TCP, 0));
3394
3395 /* GSO on 57710/57711 needs FW to calculate IP checksum */
3396 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IP_CSUM;
3397 } else {
3398 pbd->tcp_pseudo_csum =
3399 bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3400 &ipv6_hdr(skb)->daddr,
3401 0, IPPROTO_TCP, 0));
3402 }
3403
3404 pbd->global_data |=
3405 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3406 }
3407
3408 /**
3409 * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3410 *
3411 * @bp: driver handle
3412 * @skb: packet skb
3413 * @parsing_data: data to be updated
3414 * @xmit_type: xmit flags
3415 *
3416 * 57712/578xx related, when skb has encapsulation
3417 */
3418 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3419 u32 *parsing_data, u32 xmit_type)
3420 {
3421 *parsing_data |=
3422 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3423 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3424 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3425
3426 if (xmit_type & XMIT_CSUM_TCP) {
3427 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3428 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3429 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3430
3431 return skb_inner_transport_header(skb) +
3432 inner_tcp_hdrlen(skb) - skb->data;
3433 }
3434
3435 /* We support checksum offload for TCP and UDP only.
3436 * No need to pass the UDP header length - it's a constant.
3437 */
3438 return skb_inner_transport_header(skb) +
3439 sizeof(struct udphdr) - skb->data;
3440 }
3441
3442 /**
3443 * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3444 *
3445 * @bp: driver handle
3446 * @skb: packet skb
3447 * @parsing_data: data to be updated
3448 * @xmit_type: xmit flags
3449 *
3450 * 57712/578xx related
3451 */
3452 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3453 u32 *parsing_data, u32 xmit_type)
3454 {
3455 *parsing_data |=
3456 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3457 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3458 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3459
3460 if (xmit_type & XMIT_CSUM_TCP) {
3461 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3462 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3463 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3464
3465 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3466 }
3467 /* We support checksum offload for TCP and UDP only.
3468 * No need to pass the UDP header length - it's a constant.
3469 */
3470 return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3471 }
3472
3473 /* set FW indication according to inner or outer protocols if tunneled */
3474 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3475 struct eth_tx_start_bd *tx_start_bd,
3476 u32 xmit_type)
3477 {
3478 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3479
3480 if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3481 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3482
3483 if (!(xmit_type & XMIT_CSUM_TCP))
3484 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3485 }
3486
3487 /**
3488 * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3489 *
3490 * @bp: driver handle
3491 * @skb: packet skb
3492 * @pbd: parse BD to be updated
3493 * @xmit_type: xmit flags
3494 */
3495 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3496 struct eth_tx_parse_bd_e1x *pbd,
3497 u32 xmit_type)
3498 {
3499 u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3500
3501 /* for now NS flag is not used in Linux */
3502 pbd->global_data =
3503 cpu_to_le16(hlen |
3504 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3505 ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3506
3507 pbd->ip_hlen_w = (skb_transport_header(skb) -
3508 skb_network_header(skb)) >> 1;
3509
3510 hlen += pbd->ip_hlen_w;
3511
3512 /* We support checksum offload for TCP and UDP only */
3513 if (xmit_type & XMIT_CSUM_TCP)
3514 hlen += tcp_hdrlen(skb) / 2;
3515 else
3516 hlen += sizeof(struct udphdr) / 2;
3517
3518 pbd->total_hlen_w = cpu_to_le16(hlen);
3519 hlen = hlen*2;
3520
3521 if (xmit_type & XMIT_CSUM_TCP) {
3522 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3523
3524 } else {
3525 s8 fix = SKB_CS_OFF(skb); /* signed! */
3526
3527 DP(NETIF_MSG_TX_QUEUED,
3528 "hlen %d fix %d csum before fix %x\n",
3529 le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3530
3531 /* HW bug: fixup the CSUM */
3532 pbd->tcp_pseudo_csum =
3533 bnx2x_csum_fix(skb_transport_header(skb),
3534 SKB_CS(skb), fix);
3535
3536 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3537 pbd->tcp_pseudo_csum);
3538 }
3539
3540 return hlen;
3541 }
3542
3543 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3544 struct eth_tx_parse_bd_e2 *pbd_e2,
3545 struct eth_tx_parse_2nd_bd *pbd2,
3546 u16 *global_data,
3547 u32 xmit_type)
3548 {
3549 u16 hlen_w = 0;
3550 u8 outerip_off, outerip_len = 0;
3551
3552 /* from outer IP to transport */
3553 hlen_w = (skb_inner_transport_header(skb) -
3554 skb_network_header(skb)) >> 1;
3555
3556 /* transport len */
3557 hlen_w += inner_tcp_hdrlen(skb) >> 1;
3558
3559 pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3560
3561 /* outer IP header info */
3562 if (xmit_type & XMIT_CSUM_V4) {
3563 struct iphdr *iph = ip_hdr(skb);
3564 u32 csum = (__force u32)(~iph->check) -
3565 (__force u32)iph->tot_len -
3566 (__force u32)iph->frag_off;
3567
3568 pbd2->fw_ip_csum_wo_len_flags_frag =
3569 bswab16(csum_fold((__force __wsum)csum));
3570 } else {
3571 pbd2->fw_ip_hdr_to_payload_w =
3572 hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3573 }
3574
3575 pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3576
3577 pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3578
3579 if (xmit_type & XMIT_GSO_V4) {
3580 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3581
3582 pbd_e2->data.tunnel_data.pseudo_csum =
3583 bswab16(~csum_tcpudp_magic(
3584 inner_ip_hdr(skb)->saddr,
3585 inner_ip_hdr(skb)->daddr,
3586 0, IPPROTO_TCP, 0));
3587
3588 outerip_len = ip_hdr(skb)->ihl << 1;
3589 } else {
3590 pbd_e2->data.tunnel_data.pseudo_csum =
3591 bswab16(~csum_ipv6_magic(
3592 &inner_ipv6_hdr(skb)->saddr,
3593 &inner_ipv6_hdr(skb)->daddr,
3594 0, IPPROTO_TCP, 0));
3595 }
3596
3597 outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3598
3599 *global_data |=
3600 outerip_off |
3601 (!!(xmit_type & XMIT_CSUM_V6) <<
3602 ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) |
3603 (outerip_len <<
3604 ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3605 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3606 ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3607
3608 if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3609 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3610 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3611 }
3612 }
3613
3614 /* called with netif_tx_lock
3615 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3616 * netif_wake_queue()
3617 */
3618 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3619 {
3620 struct bnx2x *bp = netdev_priv(dev);
3621
3622 struct netdev_queue *txq;
3623 struct bnx2x_fp_txdata *txdata;
3624 struct sw_tx_bd *tx_buf;
3625 struct eth_tx_start_bd *tx_start_bd, *first_bd;
3626 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3627 struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3628 struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3629 struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3630 u32 pbd_e2_parsing_data = 0;
3631 u16 pkt_prod, bd_prod;
3632 int nbd, txq_index;
3633 dma_addr_t mapping;
3634 u32 xmit_type = bnx2x_xmit_type(bp, skb);
3635 int i;
3636 u8 hlen = 0;
3637 __le16 pkt_size = 0;
3638 struct ethhdr *eth;
3639 u8 mac_type = UNICAST_ADDRESS;
3640
3641 #ifdef BNX2X_STOP_ON_ERROR
3642 if (unlikely(bp->panic))
3643 return NETDEV_TX_BUSY;
3644 #endif
3645
3646 txq_index = skb_get_queue_mapping(skb);
3647 txq = netdev_get_tx_queue(dev, txq_index);
3648
3649 BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3650
3651 txdata = &bp->bnx2x_txq[txq_index];
3652
3653 /* enable this debug print to view the transmission queue being used
3654 DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3655 txq_index, fp_index, txdata_index); */
3656
3657 /* enable this debug print to view the transmission details
3658 DP(NETIF_MSG_TX_QUEUED,
3659 "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3660 txdata->cid, fp_index, txdata_index, txdata, fp); */
3661
3662 if (unlikely(bnx2x_tx_avail(bp, txdata) <
3663 skb_shinfo(skb)->nr_frags +
3664 BDS_PER_TX_PKT +
3665 NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3666 /* Handle special storage cases separately */
3667 if (txdata->tx_ring_size == 0) {
3668 struct bnx2x_eth_q_stats *q_stats =
3669 bnx2x_fp_qstats(bp, txdata->parent_fp);
3670 q_stats->driver_filtered_tx_pkt++;
3671 dev_kfree_skb(skb);
3672 return NETDEV_TX_OK;
3673 }
3674 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3675 netif_tx_stop_queue(txq);
3676 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3677
3678 return NETDEV_TX_BUSY;
3679 }
3680
3681 DP(NETIF_MSG_TX_QUEUED,
3682 "queue[%d]: SKB: summed %x protocol %x protocol(%x,%x) gso type %x xmit_type %x len %d\n",
3683 txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3684 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3685 skb->len);
3686
3687 eth = (struct ethhdr *)skb->data;
3688
3689 /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3690 if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3691 if (is_broadcast_ether_addr(eth->h_dest))
3692 mac_type = BROADCAST_ADDRESS;
3693 else
3694 mac_type = MULTICAST_ADDRESS;
3695 }
3696
3697 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3698 /* First, check if we need to linearize the skb (due to FW
3699 restrictions). No need to check fragmentation if page size > 8K
3700 (there will be no violation to FW restrictions) */
3701 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3702 /* Statistics of linearization */
3703 bp->lin_cnt++;
3704 if (skb_linearize(skb) != 0) {
3705 DP(NETIF_MSG_TX_QUEUED,
3706 "SKB linearization failed - silently dropping this SKB\n");
3707 dev_kfree_skb_any(skb);
3708 return NETDEV_TX_OK;
3709 }
3710 }
3711 #endif
3712 /* Map skb linear data for DMA */
3713 mapping = dma_map_single(&bp->pdev->dev, skb->data,
3714 skb_headlen(skb), DMA_TO_DEVICE);
3715 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3716 DP(NETIF_MSG_TX_QUEUED,
3717 "SKB mapping failed - silently dropping this SKB\n");
3718 dev_kfree_skb_any(skb);
3719 return NETDEV_TX_OK;
3720 }
3721 /*
3722 Please read carefully. First we use one BD which we mark as start,
3723 then we have a parsing info BD (used for TSO or xsum),
3724 and only then we have the rest of the TSO BDs.
3725 (don't forget to mark the last one as last,
3726 and to unmap only AFTER you write to the BD ...)
3727 And above all, all pdb sizes are in words - NOT DWORDS!
3728 */
3729
3730 /* get current pkt produced now - advance it just before sending packet
3731 * since mapping of pages may fail and cause packet to be dropped
3732 */
3733 pkt_prod = txdata->tx_pkt_prod;
3734 bd_prod = TX_BD(txdata->tx_bd_prod);
3735
3736 /* get a tx_buf and first BD
3737 * tx_start_bd may be changed during SPLIT,
3738 * but first_bd will always stay first
3739 */
3740 tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3741 tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3742 first_bd = tx_start_bd;
3743
3744 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3745
3746 /* header nbd: indirectly zero other flags! */
3747 tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3748
3749 /* remember the first BD of the packet */
3750 tx_buf->first_bd = txdata->tx_bd_prod;
3751 tx_buf->skb = skb;
3752 tx_buf->flags = 0;
3753
3754 DP(NETIF_MSG_TX_QUEUED,
3755 "sending pkt %u @%p next_idx %u bd %u @%p\n",
3756 pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3757
3758 if (vlan_tx_tag_present(skb)) {
3759 tx_start_bd->vlan_or_ethertype =
3760 cpu_to_le16(vlan_tx_tag_get(skb));
3761 tx_start_bd->bd_flags.as_bitfield |=
3762 (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3763 } else {
3764 /* when transmitting in a vf, start bd must hold the ethertype
3765 * for fw to enforce it
3766 */
3767 if (IS_VF(bp))
3768 tx_start_bd->vlan_or_ethertype =
3769 cpu_to_le16(ntohs(eth->h_proto));
3770 else
3771 /* used by FW for packet accounting */
3772 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3773 }
3774
3775 nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3776
3777 /* turn on parsing and get a BD */
3778 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3779
3780 if (xmit_type & XMIT_CSUM)
3781 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3782
3783 if (!CHIP_IS_E1x(bp)) {
3784 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3785 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3786
3787 if (xmit_type & XMIT_CSUM_ENC) {
3788 u16 global_data = 0;
3789
3790 /* Set PBD in enc checksum offload case */
3791 hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3792 &pbd_e2_parsing_data,
3793 xmit_type);
3794
3795 /* turn on 2nd parsing and get a BD */
3796 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3797
3798 pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3799
3800 memset(pbd2, 0, sizeof(*pbd2));
3801
3802 pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3803 (skb_inner_network_header(skb) -
3804 skb->data) >> 1;
3805
3806 if (xmit_type & XMIT_GSO_ENC)
3807 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3808 &global_data,
3809 xmit_type);
3810
3811 pbd2->global_data = cpu_to_le16(global_data);
3812
3813 /* add addition parse BD indication to start BD */
3814 SET_FLAG(tx_start_bd->general_data,
3815 ETH_TX_START_BD_PARSE_NBDS, 1);
3816 /* set encapsulation flag in start BD */
3817 SET_FLAG(tx_start_bd->general_data,
3818 ETH_TX_START_BD_TUNNEL_EXIST, 1);
3819 nbd++;
3820 } else if (xmit_type & XMIT_CSUM) {
3821 /* Set PBD in checksum offload case w/o encapsulation */
3822 hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3823 &pbd_e2_parsing_data,
3824 xmit_type);
3825 }
3826
3827 /* Add the macs to the parsing BD this is a vf */
3828 if (IS_VF(bp)) {
3829 /* override GRE parameters in BD */
3830 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3831 &pbd_e2->data.mac_addr.src_mid,
3832 &pbd_e2->data.mac_addr.src_lo,
3833 eth->h_source);
3834
3835 bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3836 &pbd_e2->data.mac_addr.dst_mid,
3837 &pbd_e2->data.mac_addr.dst_lo,
3838 eth->h_dest);
3839 }
3840
3841 SET_FLAG(pbd_e2_parsing_data,
3842 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3843 } else {
3844 u16 global_data = 0;
3845 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3846 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3847 /* Set PBD in checksum offload case */
3848 if (xmit_type & XMIT_CSUM)
3849 hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3850
3851 SET_FLAG(global_data,
3852 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3853 pbd_e1x->global_data |= cpu_to_le16(global_data);
3854 }
3855
3856 /* Setup the data pointer of the first BD of the packet */
3857 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3858 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3859 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3860 pkt_size = tx_start_bd->nbytes;
3861
3862 DP(NETIF_MSG_TX_QUEUED,
3863 "first bd @%p addr (%x:%x) nbytes %d flags %x vlan %x\n",
3864 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3865 le16_to_cpu(tx_start_bd->nbytes),
3866 tx_start_bd->bd_flags.as_bitfield,
3867 le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3868
3869 if (xmit_type & XMIT_GSO) {
3870
3871 DP(NETIF_MSG_TX_QUEUED,
3872 "TSO packet len %d hlen %d total len %d tso size %d\n",
3873 skb->len, hlen, skb_headlen(skb),
3874 skb_shinfo(skb)->gso_size);
3875
3876 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3877
3878 if (unlikely(skb_headlen(skb) > hlen)) {
3879 nbd++;
3880 bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3881 &tx_start_bd, hlen,
3882 bd_prod);
3883 }
3884 if (!CHIP_IS_E1x(bp))
3885 bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3886 xmit_type);
3887 else
3888 bnx2x_set_pbd_gso(skb, pbd_e1x, first_bd, xmit_type);
3889 }
3890
3891 /* Set the PBD's parsing_data field if not zero
3892 * (for the chips newer than 57711).
3893 */
3894 if (pbd_e2_parsing_data)
3895 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3896
3897 tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3898
3899 /* Handle fragmented skb */
3900 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3901 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3902
3903 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3904 skb_frag_size(frag), DMA_TO_DEVICE);
3905 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3906 unsigned int pkts_compl = 0, bytes_compl = 0;
3907
3908 DP(NETIF_MSG_TX_QUEUED,
3909 "Unable to map page - dropping packet...\n");
3910
3911 /* we need unmap all buffers already mapped
3912 * for this SKB;
3913 * first_bd->nbd need to be properly updated
3914 * before call to bnx2x_free_tx_pkt
3915 */
3916 first_bd->nbd = cpu_to_le16(nbd);
3917 bnx2x_free_tx_pkt(bp, txdata,
3918 TX_BD(txdata->tx_pkt_prod),
3919 &pkts_compl, &bytes_compl);
3920 return NETDEV_TX_OK;
3921 }
3922
3923 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3924 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3925 if (total_pkt_bd == NULL)
3926 total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3927
3928 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3929 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3930 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
3931 le16_add_cpu(&pkt_size, skb_frag_size(frag));
3932 nbd++;
3933
3934 DP(NETIF_MSG_TX_QUEUED,
3935 "frag %d bd @%p addr (%x:%x) nbytes %d\n",
3936 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
3937 le16_to_cpu(tx_data_bd->nbytes));
3938 }
3939
3940 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
3941
3942 /* update with actual num BDs */
3943 first_bd->nbd = cpu_to_le16(nbd);
3944
3945 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3946
3947 /* now send a tx doorbell, counting the next BD
3948 * if the packet contains or ends with it
3949 */
3950 if (TX_BD_POFF(bd_prod) < nbd)
3951 nbd++;
3952
3953 /* total_pkt_bytes should be set on the first data BD if
3954 * it's not an LSO packet and there is more than one
3955 * data BD. In this case pkt_size is limited by an MTU value.
3956 * However we prefer to set it for an LSO packet (while we don't
3957 * have to) in order to save some CPU cycles in a none-LSO
3958 * case, when we much more care about them.
3959 */
3960 if (total_pkt_bd != NULL)
3961 total_pkt_bd->total_pkt_bytes = pkt_size;
3962
3963 if (pbd_e1x)
3964 DP(NETIF_MSG_TX_QUEUED,
3965 "PBD (E1X) @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u tcp_flags %x xsum %x seq %u hlen %u\n",
3966 pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
3967 pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
3968 pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
3969 le16_to_cpu(pbd_e1x->total_hlen_w));
3970 if (pbd_e2)
3971 DP(NETIF_MSG_TX_QUEUED,
3972 "PBD (E2) @%p dst %x %x %x src %x %x %x parsing_data %x\n",
3973 pbd_e2,
3974 pbd_e2->data.mac_addr.dst_hi,
3975 pbd_e2->data.mac_addr.dst_mid,
3976 pbd_e2->data.mac_addr.dst_lo,
3977 pbd_e2->data.mac_addr.src_hi,
3978 pbd_e2->data.mac_addr.src_mid,
3979 pbd_e2->data.mac_addr.src_lo,
3980 pbd_e2->parsing_data);
3981 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod);
3982
3983 netdev_tx_sent_queue(txq, skb->len);
3984
3985 skb_tx_timestamp(skb);
3986
3987 txdata->tx_pkt_prod++;
3988 /*
3989 * Make sure that the BD data is updated before updating the producer
3990 * since FW might read the BD right after the producer is updated.
3991 * This is only applicable for weak-ordered memory model archs such
3992 * as IA-64. The following barrier is also mandatory since FW will
3993 * assumes packets must have BDs.
3994 */
3995 wmb();
3996
3997 txdata->tx_db.data.prod += nbd;
3998 barrier();
3999
4000 DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
4001
4002 mmiowb();
4003
4004 txdata->tx_bd_prod += nbd;
4005
4006 if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
4007 netif_tx_stop_queue(txq);
4008
4009 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
4010 * ordering of set_bit() in netif_tx_stop_queue() and read of
4011 * fp->bd_tx_cons */
4012 smp_mb();
4013
4014 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
4015 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
4016 netif_tx_wake_queue(txq);
4017 }
4018 txdata->tx_pkt++;
4019
4020 return NETDEV_TX_OK;
4021 }
4022
4023 /**
4024 * bnx2x_setup_tc - routine to configure net_device for multi tc
4025 *
4026 * @netdev: net device to configure
4027 * @tc: number of traffic classes to enable
4028 *
4029 * callback connected to the ndo_setup_tc function pointer
4030 */
4031 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4032 {
4033 int cos, prio, count, offset;
4034 struct bnx2x *bp = netdev_priv(dev);
4035
4036 /* setup tc must be called under rtnl lock */
4037 ASSERT_RTNL();
4038
4039 /* no traffic classes requested. Aborting */
4040 if (!num_tc) {
4041 netdev_reset_tc(dev);
4042 return 0;
4043 }
4044
4045 /* requested to support too many traffic classes */
4046 if (num_tc > bp->max_cos) {
4047 BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
4048 num_tc, bp->max_cos);
4049 return -EINVAL;
4050 }
4051
4052 /* declare amount of supported traffic classes */
4053 if (netdev_set_num_tc(dev, num_tc)) {
4054 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4055 return -EINVAL;
4056 }
4057
4058 /* configure priority to traffic class mapping */
4059 for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4060 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
4061 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4062 "mapping priority %d to tc %d\n",
4063 prio, bp->prio_to_cos[prio]);
4064 }
4065
4066 /* Use this configuration to differentiate tc0 from other COSes
4067 This can be used for ets or pfc, and save the effort of setting
4068 up a multio class queue disc or negotiating DCBX with a switch
4069 netdev_set_prio_tc_map(dev, 0, 0);
4070 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4071 for (prio = 1; prio < 16; prio++) {
4072 netdev_set_prio_tc_map(dev, prio, 1);
4073 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4074 } */
4075
4076 /* configure traffic class to transmission queue mapping */
4077 for (cos = 0; cos < bp->max_cos; cos++) {
4078 count = BNX2X_NUM_ETH_QUEUES(bp);
4079 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4080 netdev_set_tc_queue(dev, cos, count, offset);
4081 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4082 "mapping tc %d to offset %d count %d\n",
4083 cos, offset, count);
4084 }
4085
4086 return 0;
4087 }
4088
4089 /* called with rtnl_lock */
4090 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4091 {
4092 struct sockaddr *addr = p;
4093 struct bnx2x *bp = netdev_priv(dev);
4094 int rc = 0;
4095
4096 if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
4097 BNX2X_ERR("Requested MAC address is not valid\n");
4098 return -EINVAL;
4099 }
4100
4101 if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
4102 !is_zero_ether_addr(addr->sa_data)) {
4103 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
4104 return -EINVAL;
4105 }
4106
4107 if (netif_running(dev)) {
4108 rc = bnx2x_set_eth_mac(bp, false);
4109 if (rc)
4110 return rc;
4111 }
4112
4113 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4114
4115 if (netif_running(dev))
4116 rc = bnx2x_set_eth_mac(bp, true);
4117
4118 return rc;
4119 }
4120
4121 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4122 {
4123 union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4124 struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4125 u8 cos;
4126
4127 /* Common */
4128
4129 if (IS_FCOE_IDX(fp_index)) {
4130 memset(sb, 0, sizeof(union host_hc_status_block));
4131 fp->status_blk_mapping = 0;
4132 } else {
4133 /* status blocks */
4134 if (!CHIP_IS_E1x(bp))
4135 BNX2X_PCI_FREE(sb->e2_sb,
4136 bnx2x_fp(bp, fp_index,
4137 status_blk_mapping),
4138 sizeof(struct host_hc_status_block_e2));
4139 else
4140 BNX2X_PCI_FREE(sb->e1x_sb,
4141 bnx2x_fp(bp, fp_index,
4142 status_blk_mapping),
4143 sizeof(struct host_hc_status_block_e1x));
4144 }
4145
4146 /* Rx */
4147 if (!skip_rx_queue(bp, fp_index)) {
4148 bnx2x_free_rx_bds(fp);
4149
4150 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4151 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4152 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4153 bnx2x_fp(bp, fp_index, rx_desc_mapping),
4154 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4155
4156 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4157 bnx2x_fp(bp, fp_index, rx_comp_mapping),
4158 sizeof(struct eth_fast_path_rx_cqe) *
4159 NUM_RCQ_BD);
4160
4161 /* SGE ring */
4162 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4163 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4164 bnx2x_fp(bp, fp_index, rx_sge_mapping),
4165 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4166 }
4167
4168 /* Tx */
4169 if (!skip_tx_queue(bp, fp_index)) {
4170 /* fastpath tx rings: tx_buf tx_desc */
4171 for_each_cos_in_tx_queue(fp, cos) {
4172 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4173
4174 DP(NETIF_MSG_IFDOWN,
4175 "freeing tx memory of fp %d cos %d cid %d\n",
4176 fp_index, cos, txdata->cid);
4177
4178 BNX2X_FREE(txdata->tx_buf_ring);
4179 BNX2X_PCI_FREE(txdata->tx_desc_ring,
4180 txdata->tx_desc_mapping,
4181 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4182 }
4183 }
4184 /* end of fastpath */
4185 }
4186
4187 void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4188 {
4189 int i;
4190 for_each_cnic_queue(bp, i)
4191 bnx2x_free_fp_mem_at(bp, i);
4192 }
4193
4194 void bnx2x_free_fp_mem(struct bnx2x *bp)
4195 {
4196 int i;
4197 for_each_eth_queue(bp, i)
4198 bnx2x_free_fp_mem_at(bp, i);
4199 }
4200
4201 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4202 {
4203 union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4204 if (!CHIP_IS_E1x(bp)) {
4205 bnx2x_fp(bp, index, sb_index_values) =
4206 (__le16 *)status_blk.e2_sb->sb.index_values;
4207 bnx2x_fp(bp, index, sb_running_index) =
4208 (__le16 *)status_blk.e2_sb->sb.running_index;
4209 } else {
4210 bnx2x_fp(bp, index, sb_index_values) =
4211 (__le16 *)status_blk.e1x_sb->sb.index_values;
4212 bnx2x_fp(bp, index, sb_running_index) =
4213 (__le16 *)status_blk.e1x_sb->sb.running_index;
4214 }
4215 }
4216
4217 /* Returns the number of actually allocated BDs */
4218 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4219 int rx_ring_size)
4220 {
4221 struct bnx2x *bp = fp->bp;
4222 u16 ring_prod, cqe_ring_prod;
4223 int i, failure_cnt = 0;
4224
4225 fp->rx_comp_cons = 0;
4226 cqe_ring_prod = ring_prod = 0;
4227
4228 /* This routine is called only during fo init so
4229 * fp->eth_q_stats.rx_skb_alloc_failed = 0
4230 */
4231 for (i = 0; i < rx_ring_size; i++) {
4232 if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) {
4233 failure_cnt++;
4234 continue;
4235 }
4236 ring_prod = NEXT_RX_IDX(ring_prod);
4237 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4238 WARN_ON(ring_prod <= (i - failure_cnt));
4239 }
4240
4241 if (failure_cnt)
4242 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4243 i - failure_cnt, fp->index);
4244
4245 fp->rx_bd_prod = ring_prod;
4246 /* Limit the CQE producer by the CQE ring size */
4247 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4248 cqe_ring_prod);
4249 fp->rx_pkt = fp->rx_calls = 0;
4250
4251 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4252
4253 return i - failure_cnt;
4254 }
4255
4256 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4257 {
4258 int i;
4259
4260 for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4261 struct eth_rx_cqe_next_page *nextpg;
4262
4263 nextpg = (struct eth_rx_cqe_next_page *)
4264 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4265 nextpg->addr_hi =
4266 cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4267 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4268 nextpg->addr_lo =
4269 cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4270 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4271 }
4272 }
4273
4274 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4275 {
4276 union host_hc_status_block *sb;
4277 struct bnx2x_fastpath *fp = &bp->fp[index];
4278 int ring_size = 0;
4279 u8 cos;
4280 int rx_ring_size = 0;
4281
4282 if (!bp->rx_ring_size &&
4283 (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
4284 rx_ring_size = MIN_RX_SIZE_NONTPA;
4285 bp->rx_ring_size = rx_ring_size;
4286 } else if (!bp->rx_ring_size) {
4287 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4288
4289 if (CHIP_IS_E3(bp)) {
4290 u32 cfg = SHMEM_RD(bp,
4291 dev_info.port_hw_config[BP_PORT(bp)].
4292 default_cfg);
4293
4294 /* Decrease ring size for 1G functions */
4295 if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4296 PORT_HW_CFG_NET_SERDES_IF_SGMII)
4297 rx_ring_size /= 10;
4298 }
4299
4300 /* allocate at least number of buffers required by FW */
4301 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4302 MIN_RX_SIZE_TPA, rx_ring_size);
4303
4304 bp->rx_ring_size = rx_ring_size;
4305 } else /* if rx_ring_size specified - use it */
4306 rx_ring_size = bp->rx_ring_size;
4307
4308 DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4309
4310 /* Common */
4311 sb = &bnx2x_fp(bp, index, status_blk);
4312
4313 if (!IS_FCOE_IDX(index)) {
4314 /* status blocks */
4315 if (!CHIP_IS_E1x(bp))
4316 BNX2X_PCI_ALLOC(sb->e2_sb,
4317 &bnx2x_fp(bp, index, status_blk_mapping),
4318 sizeof(struct host_hc_status_block_e2));
4319 else
4320 BNX2X_PCI_ALLOC(sb->e1x_sb,
4321 &bnx2x_fp(bp, index, status_blk_mapping),
4322 sizeof(struct host_hc_status_block_e1x));
4323 }
4324
4325 /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4326 * set shortcuts for it.
4327 */
4328 if (!IS_FCOE_IDX(index))
4329 set_sb_shortcuts(bp, index);
4330
4331 /* Tx */
4332 if (!skip_tx_queue(bp, index)) {
4333 /* fastpath tx rings: tx_buf tx_desc */
4334 for_each_cos_in_tx_queue(fp, cos) {
4335 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4336
4337 DP(NETIF_MSG_IFUP,
4338 "allocating tx memory of fp %d cos %d\n",
4339 index, cos);
4340
4341 BNX2X_ALLOC(txdata->tx_buf_ring,
4342 sizeof(struct sw_tx_bd) * NUM_TX_BD);
4343 BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
4344 &txdata->tx_desc_mapping,
4345 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4346 }
4347 }
4348
4349 /* Rx */
4350 if (!skip_rx_queue(bp, index)) {
4351 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4352 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
4353 sizeof(struct sw_rx_bd) * NUM_RX_BD);
4354 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
4355 &bnx2x_fp(bp, index, rx_desc_mapping),
4356 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4357
4358 /* Seed all CQEs by 1s */
4359 BNX2X_PCI_FALLOC(bnx2x_fp(bp, index, rx_comp_ring),
4360 &bnx2x_fp(bp, index, rx_comp_mapping),
4361 sizeof(struct eth_fast_path_rx_cqe) *
4362 NUM_RCQ_BD);
4363
4364 /* SGE ring */
4365 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
4366 sizeof(struct sw_rx_page) * NUM_RX_SGE);
4367 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
4368 &bnx2x_fp(bp, index, rx_sge_mapping),
4369 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4370 /* RX BD ring */
4371 bnx2x_set_next_page_rx_bd(fp);
4372
4373 /* CQ ring */
4374 bnx2x_set_next_page_rx_cq(fp);
4375
4376 /* BDs */
4377 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4378 if (ring_size < rx_ring_size)
4379 goto alloc_mem_err;
4380 }
4381
4382 return 0;
4383
4384 /* handles low memory cases */
4385 alloc_mem_err:
4386 BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4387 index, ring_size);
4388 /* FW will drop all packets if queue is not big enough,
4389 * In these cases we disable the queue
4390 * Min size is different for OOO, TPA and non-TPA queues
4391 */
4392 if (ring_size < (fp->disable_tpa ?
4393 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4394 /* release memory allocated for this queue */
4395 bnx2x_free_fp_mem_at(bp, index);
4396 return -ENOMEM;
4397 }
4398 return 0;
4399 }
4400
4401 int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4402 {
4403 if (!NO_FCOE(bp))
4404 /* FCoE */
4405 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4406 /* we will fail load process instead of mark
4407 * NO_FCOE_FLAG
4408 */
4409 return -ENOMEM;
4410
4411 return 0;
4412 }
4413
4414 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4415 {
4416 int i;
4417
4418 /* 1. Allocate FP for leading - fatal if error
4419 * 2. Allocate RSS - fix number of queues if error
4420 */
4421
4422 /* leading */
4423 if (bnx2x_alloc_fp_mem_at(bp, 0))
4424 return -ENOMEM;
4425
4426 /* RSS */
4427 for_each_nondefault_eth_queue(bp, i)
4428 if (bnx2x_alloc_fp_mem_at(bp, i))
4429 break;
4430
4431 /* handle memory failures */
4432 if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4433 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4434
4435 WARN_ON(delta < 0);
4436 bnx2x_shrink_eth_fp(bp, delta);
4437 if (CNIC_SUPPORT(bp))
4438 /* move non eth FPs next to last eth FP
4439 * must be done in that order
4440 * FCOE_IDX < FWD_IDX < OOO_IDX
4441 */
4442
4443 /* move FCoE fp even NO_FCOE_FLAG is on */
4444 bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4445 bp->num_ethernet_queues -= delta;
4446 bp->num_queues = bp->num_ethernet_queues +
4447 bp->num_cnic_queues;
4448 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4449 bp->num_queues + delta, bp->num_queues);
4450 }
4451
4452 return 0;
4453 }
4454
4455 void bnx2x_free_mem_bp(struct bnx2x *bp)
4456 {
4457 int i;
4458
4459 for (i = 0; i < bp->fp_array_size; i++)
4460 kfree(bp->fp[i].tpa_info);
4461 kfree(bp->fp);
4462 kfree(bp->sp_objs);
4463 kfree(bp->fp_stats);
4464 kfree(bp->bnx2x_txq);
4465 kfree(bp->msix_table);
4466 kfree(bp->ilt);
4467 }
4468
4469 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4470 {
4471 struct bnx2x_fastpath *fp;
4472 struct msix_entry *tbl;
4473 struct bnx2x_ilt *ilt;
4474 int msix_table_size = 0;
4475 int fp_array_size, txq_array_size;
4476 int i;
4477
4478 /*
4479 * The biggest MSI-X table we might need is as a maximum number of fast
4480 * path IGU SBs plus default SB (for PF only).
4481 */
4482 msix_table_size = bp->igu_sb_cnt;
4483 if (IS_PF(bp))
4484 msix_table_size++;
4485 BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4486
4487 /* fp array: RSS plus CNIC related L2 queues */
4488 fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4489 bp->fp_array_size = fp_array_size;
4490 BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4491
4492 fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4493 if (!fp)
4494 goto alloc_err;
4495 for (i = 0; i < bp->fp_array_size; i++) {
4496 fp[i].tpa_info =
4497 kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4498 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4499 if (!(fp[i].tpa_info))
4500 goto alloc_err;
4501 }
4502
4503 bp->fp = fp;
4504
4505 /* allocate sp objs */
4506 bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4507 GFP_KERNEL);
4508 if (!bp->sp_objs)
4509 goto alloc_err;
4510
4511 /* allocate fp_stats */
4512 bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4513 GFP_KERNEL);
4514 if (!bp->fp_stats)
4515 goto alloc_err;
4516
4517 /* Allocate memory for the transmission queues array */
4518 txq_array_size =
4519 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4520 BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4521
4522 bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4523 GFP_KERNEL);
4524 if (!bp->bnx2x_txq)
4525 goto alloc_err;
4526
4527 /* msix table */
4528 tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4529 if (!tbl)
4530 goto alloc_err;
4531 bp->msix_table = tbl;
4532
4533 /* ilt */
4534 ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4535 if (!ilt)
4536 goto alloc_err;
4537 bp->ilt = ilt;
4538
4539 return 0;
4540 alloc_err:
4541 bnx2x_free_mem_bp(bp);
4542 return -ENOMEM;
4543 }
4544
4545 int bnx2x_reload_if_running(struct net_device *dev)
4546 {
4547 struct bnx2x *bp = netdev_priv(dev);
4548
4549 if (unlikely(!netif_running(dev)))
4550 return 0;
4551
4552 bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4553 return bnx2x_nic_load(bp, LOAD_NORMAL);
4554 }
4555
4556 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4557 {
4558 u32 sel_phy_idx = 0;
4559 if (bp->link_params.num_phys <= 1)
4560 return INT_PHY;
4561
4562 if (bp->link_vars.link_up) {
4563 sel_phy_idx = EXT_PHY1;
4564 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4565 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4566 (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4567 sel_phy_idx = EXT_PHY2;
4568 } else {
4569
4570 switch (bnx2x_phy_selection(&bp->link_params)) {
4571 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4572 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4573 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4574 sel_phy_idx = EXT_PHY1;
4575 break;
4576 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4577 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4578 sel_phy_idx = EXT_PHY2;
4579 break;
4580 }
4581 }
4582
4583 return sel_phy_idx;
4584 }
4585 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4586 {
4587 u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4588 /*
4589 * The selected activated PHY is always after swapping (in case PHY
4590 * swapping is enabled). So when swapping is enabled, we need to reverse
4591 * the configuration
4592 */
4593
4594 if (bp->link_params.multi_phy_config &
4595 PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4596 if (sel_phy_idx == EXT_PHY1)
4597 sel_phy_idx = EXT_PHY2;
4598 else if (sel_phy_idx == EXT_PHY2)
4599 sel_phy_idx = EXT_PHY1;
4600 }
4601 return LINK_CONFIG_IDX(sel_phy_idx);
4602 }
4603
4604 #ifdef NETDEV_FCOE_WWNN
4605 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4606 {
4607 struct bnx2x *bp = netdev_priv(dev);
4608 struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4609
4610 switch (type) {
4611 case NETDEV_FCOE_WWNN:
4612 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4613 cp->fcoe_wwn_node_name_lo);
4614 break;
4615 case NETDEV_FCOE_WWPN:
4616 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4617 cp->fcoe_wwn_port_name_lo);
4618 break;
4619 default:
4620 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4621 return -EINVAL;
4622 }
4623
4624 return 0;
4625 }
4626 #endif
4627
4628 /* called with rtnl_lock */
4629 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4630 {
4631 struct bnx2x *bp = netdev_priv(dev);
4632
4633 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4634 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4635 return -EAGAIN;
4636 }
4637
4638 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4639 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4640 BNX2X_ERR("Can't support requested MTU size\n");
4641 return -EINVAL;
4642 }
4643
4644 /* This does not race with packet allocation
4645 * because the actual alloc size is
4646 * only updated as part of load
4647 */
4648 dev->mtu = new_mtu;
4649
4650 return bnx2x_reload_if_running(dev);
4651 }
4652
4653 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4654 netdev_features_t features)
4655 {
4656 struct bnx2x *bp = netdev_priv(dev);
4657
4658 /* TPA requires Rx CSUM offloading */
4659 if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
4660 features &= ~NETIF_F_LRO;
4661 features &= ~NETIF_F_GRO;
4662 }
4663
4664 return features;
4665 }
4666
4667 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4668 {
4669 struct bnx2x *bp = netdev_priv(dev);
4670 u32 flags = bp->flags;
4671 u32 changes;
4672 bool bnx2x_reload = false;
4673
4674 if (features & NETIF_F_LRO)
4675 flags |= TPA_ENABLE_FLAG;
4676 else
4677 flags &= ~TPA_ENABLE_FLAG;
4678
4679 if (features & NETIF_F_GRO)
4680 flags |= GRO_ENABLE_FLAG;
4681 else
4682 flags &= ~GRO_ENABLE_FLAG;
4683
4684 if (features & NETIF_F_LOOPBACK) {
4685 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4686 bp->link_params.loopback_mode = LOOPBACK_BMAC;
4687 bnx2x_reload = true;
4688 }
4689 } else {
4690 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4691 bp->link_params.loopback_mode = LOOPBACK_NONE;
4692 bnx2x_reload = true;
4693 }
4694 }
4695
4696 changes = flags ^ bp->flags;
4697
4698 /* if GRO is changed while LRO is enabled, don't force a reload */
4699 if ((changes & GRO_ENABLE_FLAG) && (flags & TPA_ENABLE_FLAG))
4700 changes &= ~GRO_ENABLE_FLAG;
4701
4702 if (changes)
4703 bnx2x_reload = true;
4704
4705 bp->flags = flags;
4706
4707 if (bnx2x_reload) {
4708 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4709 return bnx2x_reload_if_running(dev);
4710 /* else: bnx2x_nic_load() will be called at end of recovery */
4711 }
4712
4713 return 0;
4714 }
4715
4716 void bnx2x_tx_timeout(struct net_device *dev)
4717 {
4718 struct bnx2x *bp = netdev_priv(dev);
4719
4720 #ifdef BNX2X_STOP_ON_ERROR
4721 if (!bp->panic)
4722 bnx2x_panic();
4723 #endif
4724
4725 smp_mb__before_clear_bit();
4726 set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
4727 smp_mb__after_clear_bit();
4728
4729 /* This allows the netif to be shutdown gracefully before resetting */
4730 schedule_delayed_work(&bp->sp_rtnl_task, 0);
4731 }
4732
4733 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4734 {
4735 struct net_device *dev = pci_get_drvdata(pdev);
4736 struct bnx2x *bp;
4737
4738 if (!dev) {
4739 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4740 return -ENODEV;
4741 }
4742 bp = netdev_priv(dev);
4743
4744 rtnl_lock();
4745
4746 pci_save_state(pdev);
4747
4748 if (!netif_running(dev)) {
4749 rtnl_unlock();
4750 return 0;
4751 }
4752
4753 netif_device_detach(dev);
4754
4755 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4756
4757 bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4758
4759 rtnl_unlock();
4760
4761 return 0;
4762 }
4763
4764 int bnx2x_resume(struct pci_dev *pdev)
4765 {
4766 struct net_device *dev = pci_get_drvdata(pdev);
4767 struct bnx2x *bp;
4768 int rc;
4769
4770 if (!dev) {
4771 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4772 return -ENODEV;
4773 }
4774 bp = netdev_priv(dev);
4775
4776 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4777 BNX2X_ERR("Handling parity error recovery. Try again later\n");
4778 return -EAGAIN;
4779 }
4780
4781 rtnl_lock();
4782
4783 pci_restore_state(pdev);
4784
4785 if (!netif_running(dev)) {
4786 rtnl_unlock();
4787 return 0;
4788 }
4789
4790 bnx2x_set_power_state(bp, PCI_D0);
4791 netif_device_attach(dev);
4792
4793 rc = bnx2x_nic_load(bp, LOAD_OPEN);
4794
4795 rtnl_unlock();
4796
4797 return rc;
4798 }
4799
4800 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4801 u32 cid)
4802 {
4803 if (!cxt) {
4804 BNX2X_ERR("bad context pointer %p\n", cxt);
4805 return;
4806 }
4807
4808 /* ustorm cxt validation */
4809 cxt->ustorm_ag_context.cdu_usage =
4810 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4811 CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4812 /* xcontext validation */
4813 cxt->xstorm_ag_context.cdu_reserved =
4814 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4815 CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4816 }
4817
4818 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4819 u8 fw_sb_id, u8 sb_index,
4820 u8 ticks)
4821 {
4822 u32 addr = BAR_CSTRORM_INTMEM +
4823 CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4824 REG_WR8(bp, addr, ticks);
4825 DP(NETIF_MSG_IFUP,
4826 "port %x fw_sb_id %d sb_index %d ticks %d\n",
4827 port, fw_sb_id, sb_index, ticks);
4828 }
4829
4830 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4831 u16 fw_sb_id, u8 sb_index,
4832 u8 disable)
4833 {
4834 u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4835 u32 addr = BAR_CSTRORM_INTMEM +
4836 CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4837 u8 flags = REG_RD8(bp, addr);
4838 /* clear and set */
4839 flags &= ~HC_INDEX_DATA_HC_ENABLED;
4840 flags |= enable_flag;
4841 REG_WR8(bp, addr, flags);
4842 DP(NETIF_MSG_IFUP,
4843 "port %x fw_sb_id %d sb_index %d disable %d\n",
4844 port, fw_sb_id, sb_index, disable);
4845 }
4846
4847 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4848 u8 sb_index, u8 disable, u16 usec)
4849 {
4850 int port = BP_PORT(bp);
4851 u8 ticks = usec / BNX2X_BTR;
4852
4853 storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4854
4855 disable = disable ? 1 : (usec ? 0 : 1);
4856 storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4857 }
Linux kernel - Deliverables
This page took 0.162398 seconds and 5 git commands to generate.